@mastra/clickhouse 0.0.0-tsconfig-compile-20250703214351 → 0.0.0-unified-sidebar-20251010130811

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +524 -4
  2. package/LICENSE.md +12 -4
  3. package/dist/index.cjs +2278 -598
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.ts +3 -4
  6. package/dist/index.d.ts.map +1 -0
  7. package/dist/index.js +2278 -598
  8. package/dist/index.js.map +1 -0
  9. package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
  10. package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
  11. package/dist/storage/domains/memory/index.d.ts +87 -0
  12. package/dist/storage/domains/memory/index.d.ts.map +1 -0
  13. package/dist/storage/domains/operations/index.d.ts +42 -0
  14. package/dist/storage/domains/operations/index.d.ts.map +1 -0
  15. package/dist/storage/domains/scores/index.d.ts +54 -0
  16. package/dist/storage/domains/scores/index.d.ts.map +1 -0
  17. package/dist/storage/domains/traces/index.d.ts +21 -0
  18. package/dist/storage/domains/traces/index.d.ts.map +1 -0
  19. package/dist/storage/domains/utils.d.ts +28 -0
  20. package/dist/storage/domains/utils.d.ts.map +1 -0
  21. package/dist/storage/domains/workflows/index.d.ts +55 -0
  22. package/dist/storage/domains/workflows/index.d.ts.map +1 -0
  23. package/dist/storage/index.d.ts +245 -0
  24. package/dist/storage/index.d.ts.map +1 -0
  25. package/package.json +25 -11
  26. package/dist/_tsup-dts-rollup.d.cts +0 -191
  27. package/dist/_tsup-dts-rollup.d.ts +0 -191
  28. package/dist/index.d.cts +0 -4
  29. package/docker-compose.yaml +0 -15
  30. package/eslint.config.js +0 -6
  31. package/src/index.ts +0 -1
  32. package/src/storage/index.test.ts +0 -1154
  33. package/src/storage/index.ts +0 -1464
  34. package/tsconfig.json +0 -5
  35. package/vitest.config.ts +0 -12
package/dist/index.cjs CHANGED
@@ -1,24 +1,22 @@
1
1
  'use strict';
2
2
 
3
3
  var client = require('@clickhouse/client');
4
- var agent = require('@mastra/core/agent');
5
4
  var error = require('@mastra/core/error');
6
5
  var storage = require('@mastra/core/storage');
6
+ var agent = require('@mastra/core/agent');
7
+ var scores = require('@mastra/core/scores');
7
8
 
8
9
  // src/storage/index.ts
9
- function safelyParseJSON(jsonString) {
10
- try {
11
- return JSON.parse(jsonString);
12
- } catch {
13
- return {};
14
- }
15
- }
16
10
  var TABLE_ENGINES = {
17
11
  [storage.TABLE_MESSAGES]: `MergeTree()`,
18
12
  [storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
19
13
  [storage.TABLE_TRACES]: `MergeTree()`,
20
14
  [storage.TABLE_THREADS]: `ReplacingMergeTree()`,
21
- [storage.TABLE_EVALS]: `MergeTree()`
15
+ [storage.TABLE_EVALS]: `MergeTree()`,
16
+ [storage.TABLE_SCORERS]: `MergeTree()`,
17
+ [storage.TABLE_RESOURCES]: `ReplacingMergeTree()`,
18
+ // TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
19
+ [storage.TABLE_AI_SPANS]: `ReplacingMergeTree()`
22
20
  };
23
21
  var COLUMN_TYPES = {
24
22
  text: "String",
@@ -26,11 +24,10 @@ var COLUMN_TYPES = {
26
24
  uuid: "String",
27
25
  jsonb: "String",
28
26
  integer: "Int64",
29
- bigint: "Int64"
27
+ float: "Float64",
28
+ bigint: "Int64",
29
+ boolean: "Bool"
30
30
  };
31
- function transformRows(rows) {
32
- return rows.map((row) => transformRow(row));
33
- }
34
31
  function transformRow(row) {
35
32
  if (!row) {
36
33
  return row;
@@ -41,31 +38,56 @@ function transformRow(row) {
41
38
  if (row.updatedAt) {
42
39
  row.updatedAt = new Date(row.updatedAt);
43
40
  }
41
+ if (row.content && typeof row.content === "string") {
42
+ row.content = storage.safelyParseJSON(row.content);
43
+ }
44
44
  return row;
45
45
  }
46
- var ClickhouseStore = class extends storage.MastraStorage {
47
- db;
48
- ttl = {};
49
- constructor(config) {
50
- super({ name: "ClickhouseStore" });
51
- this.db = client.createClient({
52
- url: config.url,
53
- username: config.username,
54
- password: config.password,
55
- clickhouse_settings: {
56
- date_time_input_format: "best_effort",
57
- date_time_output_format: "iso",
58
- // This is crucial
59
- use_client_time_zone: 1,
60
- output_format_json_quote_64bit_integers: 0
61
- }
62
- });
63
- this.ttl = config.ttl;
46
+ function transformRows(rows) {
47
+ return rows.map((row) => transformRow(row));
48
+ }
49
+
50
+ // src/storage/domains/legacy-evals/index.ts
51
+ var LegacyEvalsStorageClickhouse = class extends storage.LegacyEvalsStorage {
52
+ client;
53
+ operations;
54
+ constructor({ client, operations }) {
55
+ super();
56
+ this.client = client;
57
+ this.operations = operations;
64
58
  }
65
59
  transformEvalRow(row) {
66
60
  row = transformRow(row);
67
- const resultValue = JSON.parse(row.result);
68
- const testInfoValue = row.test_info ? JSON.parse(row.test_info) : void 0;
61
+ let resultValue;
62
+ try {
63
+ if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
64
+ resultValue = JSON.parse(row.result);
65
+ } else if (typeof row.result === "object" && row.result !== null) {
66
+ resultValue = row.result;
67
+ } else if (row.result === null || row.result === void 0 || row.result === "") {
68
+ resultValue = { score: 0 };
69
+ } else {
70
+ throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
71
+ }
72
+ } catch (error$1) {
73
+ console.error("Error parsing result field:", row.result, error$1);
74
+ throw new error.MastraError({
75
+ id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
76
+ text: `Invalid result format: ${JSON.stringify(row.result)}`,
77
+ domain: error.ErrorDomain.STORAGE,
78
+ category: error.ErrorCategory.USER
79
+ });
80
+ }
81
+ let testInfoValue;
82
+ try {
83
+ if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
84
+ testInfoValue = JSON.parse(row.test_info);
85
+ } else if (typeof row.test_info === "object" && row.test_info !== null) {
86
+ testInfoValue = row.test_info;
87
+ }
88
+ } catch {
89
+ testInfoValue = void 0;
90
+ }
69
91
  if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
70
92
  throw new error.MastraError({
71
93
  id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
@@ -87,23 +109,11 @@ var ClickhouseStore = class extends storage.MastraStorage {
87
109
  createdAt: row.created_at
88
110
  };
89
111
  }
90
- escape(value) {
91
- if (typeof value === "string") {
92
- return `'${value.replace(/'/g, "''")}'`;
93
- }
94
- if (value instanceof Date) {
95
- return `'${value.toISOString()}'`;
96
- }
97
- if (value === null || value === void 0) {
98
- return "NULL";
99
- }
100
- return value.toString();
101
- }
102
112
  async getEvalsByAgentName(agentName, type) {
103
113
  try {
104
- const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
105
- const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
106
- const result = await this.db.query({
114
+ const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
115
+ const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
116
+ const result = await this.client.query({
107
117
  query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
108
118
  query_params: { var_agent_name: agentName },
109
119
  clickhouse_settings: {
@@ -133,388 +143,438 @@ var ClickhouseStore = class extends storage.MastraStorage {
133
143
  );
134
144
  }
135
145
  }
136
- async batchInsert({ tableName, records }) {
137
- try {
138
- await this.db.insert({
139
- table: tableName,
140
- values: records.map((record) => ({
141
- ...Object.fromEntries(
142
- Object.entries(record).map(([key, value]) => [
143
- key,
144
- storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
145
- ])
146
- )
147
- })),
148
- format: "JSONEachRow",
149
- clickhouse_settings: {
150
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
151
- date_time_input_format: "best_effort",
152
- use_client_time_zone: 1,
153
- output_format_json_quote_64bit_integers: 0
154
- }
155
- });
156
- } catch (error$1) {
157
- throw new error.MastraError(
158
- {
159
- id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
160
- domain: error.ErrorDomain.STORAGE,
161
- category: error.ErrorCategory.THIRD_PARTY,
162
- details: { tableName }
163
- },
164
- error$1
165
- );
166
- }
167
- }
168
- async getTraces({
169
- name,
170
- scope,
171
- page,
172
- perPage,
173
- attributes,
174
- filters,
175
- fromDate,
176
- toDate
177
- }) {
178
- const limit = perPage;
179
- const offset = page * perPage;
180
- const args = {};
146
+ async getEvals(options = {}) {
147
+ const { agentName, type, page = 0, perPage = 100, dateRange } = options;
148
+ const fromDate = dateRange?.start;
149
+ const toDate = dateRange?.end;
181
150
  const conditions = [];
182
- if (name) {
183
- conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
184
- args.var_name = name;
185
- }
186
- if (scope) {
187
- conditions.push(`scope = {var_scope:String}`);
188
- args.var_scope = scope;
151
+ if (agentName) {
152
+ conditions.push(`agent_name = {var_agent_name:String}`);
189
153
  }
190
- if (attributes) {
191
- Object.entries(attributes).forEach(([key, value]) => {
192
- conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
193
- args[`var_attr_${key}`] = value;
194
- });
195
- }
196
- if (filters) {
197
- Object.entries(filters).forEach(([key, value]) => {
198
- conditions.push(
199
- `${key} = {var_col_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
200
- );
201
- args[`var_col_${key}`] = value;
202
- });
154
+ if (type === "test") {
155
+ conditions.push(
156
+ `(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
157
+ );
158
+ } else if (type === "live") {
159
+ conditions.push(
160
+ `(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
161
+ );
203
162
  }
204
163
  if (fromDate) {
205
- conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
206
- args.var_from_date = fromDate.getTime() / 1e3;
164
+ conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
165
+ fromDate.toISOString();
207
166
  }
208
167
  if (toDate) {
209
- conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
210
- args.var_to_date = toDate.getTime() / 1e3;
168
+ conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
169
+ toDate.toISOString();
211
170
  }
212
171
  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
213
172
  try {
214
- const result = await this.db.query({
215
- query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
216
- query_params: args,
173
+ const countResult = await this.client.query({
174
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_EVALS} ${whereClause}`,
175
+ query_params: {
176
+ ...agentName ? { var_agent_name: agentName } : {},
177
+ ...fromDate ? { var_from_date: fromDate.toISOString() } : {},
178
+ ...toDate ? { var_to_date: toDate.toISOString() } : {}
179
+ },
217
180
  clickhouse_settings: {
218
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
219
181
  date_time_input_format: "best_effort",
220
182
  date_time_output_format: "iso",
221
183
  use_client_time_zone: 1,
222
184
  output_format_json_quote_64bit_integers: 0
223
185
  }
224
186
  });
225
- if (!result) {
226
- return [];
187
+ const countData = await countResult.json();
188
+ const total = Number(countData.data?.[0]?.count ?? 0);
189
+ const currentOffset = page * perPage;
190
+ const hasMore = currentOffset + perPage < total;
191
+ if (total === 0) {
192
+ return {
193
+ evals: [],
194
+ total: 0,
195
+ page,
196
+ perPage,
197
+ hasMore: false
198
+ };
227
199
  }
228
- const resp = await result.json();
229
- const rows = resp.data;
230
- return rows.map((row) => ({
231
- id: row.id,
232
- parentSpanId: row.parentSpanId,
233
- traceId: row.traceId,
234
- name: row.name,
235
- scope: row.scope,
236
- kind: row.kind,
237
- status: safelyParseJSON(row.status),
238
- events: safelyParseJSON(row.events),
239
- links: safelyParseJSON(row.links),
240
- attributes: safelyParseJSON(row.attributes),
241
- startTime: row.startTime,
242
- endTime: row.endTime,
243
- other: safelyParseJSON(row.other),
244
- createdAt: row.createdAt
245
- }));
200
+ const dataResult = await this.client.query({
201
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
202
+ query_params: {
203
+ ...agentName ? { var_agent_name: agentName } : {},
204
+ ...fromDate ? { var_from_date: fromDate.toISOString() } : {},
205
+ ...toDate ? { var_to_date: toDate.toISOString() } : {},
206
+ var_limit: perPage || 100,
207
+ var_offset: currentOffset || 0
208
+ },
209
+ clickhouse_settings: {
210
+ date_time_input_format: "best_effort",
211
+ date_time_output_format: "iso",
212
+ use_client_time_zone: 1,
213
+ output_format_json_quote_64bit_integers: 0
214
+ }
215
+ });
216
+ const rows = await dataResult.json();
217
+ return {
218
+ evals: rows.data.map((row) => this.transformEvalRow(row)),
219
+ total,
220
+ page,
221
+ perPage,
222
+ hasMore
223
+ };
246
224
  } catch (error$1) {
247
225
  if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
248
- return [];
226
+ return {
227
+ evals: [],
228
+ total: 0,
229
+ page,
230
+ perPage,
231
+ hasMore: false
232
+ };
249
233
  }
250
234
  throw new error.MastraError(
251
235
  {
252
- id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
236
+ id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
253
237
  domain: error.ErrorDomain.STORAGE,
254
238
  category: error.ErrorCategory.THIRD_PARTY,
255
- details: {
256
- name: name ?? null,
257
- scope: scope ?? null,
258
- page,
259
- perPage,
260
- attributes: attributes ? JSON.stringify(attributes) : null,
261
- filters: filters ? JSON.stringify(filters) : null,
262
- fromDate: fromDate?.toISOString() ?? null,
263
- toDate: toDate?.toISOString() ?? null
264
- }
239
+ details: { agentName: agentName ?? "all", type: type ?? "all" }
265
240
  },
266
241
  error$1
267
242
  );
268
243
  }
269
244
  }
270
- async optimizeTable({ tableName }) {
245
+ };
246
+ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
247
+ client;
248
+ operations;
249
+ constructor({ client, operations }) {
250
+ super();
251
+ this.client = client;
252
+ this.operations = operations;
253
+ }
254
+ async getMessages({
255
+ threadId,
256
+ resourceId,
257
+ selectBy,
258
+ format
259
+ }) {
271
260
  try {
272
- await this.db.command({
273
- query: `OPTIMIZE TABLE ${tableName} FINAL`
261
+ if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
262
+ const messages = [];
263
+ const limit = storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
264
+ const include = selectBy?.include || [];
265
+ if (include.length) {
266
+ const unionQueries = [];
267
+ const params = [];
268
+ let paramIdx = 1;
269
+ for (const inc of include) {
270
+ const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
271
+ const searchId = inc.threadId || threadId;
272
+ unionQueries.push(`
273
+ SELECT * FROM (
274
+ WITH numbered_messages AS (
275
+ SELECT
276
+ id, content, role, type, "createdAt", thread_id, "resourceId",
277
+ ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
278
+ FROM "${storage.TABLE_MESSAGES}"
279
+ WHERE thread_id = {var_thread_id_${paramIdx}:String}
280
+ ),
281
+ target_positions AS (
282
+ SELECT row_num as target_pos
283
+ FROM numbered_messages
284
+ WHERE id = {var_include_id_${paramIdx}:String}
285
+ )
286
+ SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
287
+ FROM numbered_messages m
288
+ CROSS JOIN target_positions t
289
+ WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
290
+ ) AS query_${paramIdx}
291
+ `);
292
+ params.push(
293
+ { [`var_thread_id_${paramIdx}`]: searchId },
294
+ { [`var_include_id_${paramIdx}`]: id },
295
+ { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
296
+ { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
297
+ );
298
+ paramIdx++;
299
+ }
300
+ const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
301
+ const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
302
+ const includeResult = await this.client.query({
303
+ query: finalQuery,
304
+ query_params: mergedParams,
305
+ clickhouse_settings: {
306
+ date_time_input_format: "best_effort",
307
+ date_time_output_format: "iso",
308
+ use_client_time_zone: 1,
309
+ output_format_json_quote_64bit_integers: 0
310
+ }
311
+ });
312
+ const rows2 = await includeResult.json();
313
+ const includedMessages = transformRows(rows2.data);
314
+ const seen = /* @__PURE__ */ new Set();
315
+ const dedupedMessages = includedMessages.filter((message) => {
316
+ if (seen.has(message.id)) return false;
317
+ seen.add(message.id);
318
+ return true;
319
+ });
320
+ messages.push(...dedupedMessages);
321
+ }
322
+ const result = await this.client.query({
323
+ query: `
324
+ SELECT
325
+ id,
326
+ content,
327
+ role,
328
+ type,
329
+ toDateTime64(createdAt, 3) as createdAt,
330
+ thread_id AS "threadId"
331
+ FROM "${storage.TABLE_MESSAGES}"
332
+ WHERE thread_id = {threadId:String}
333
+ AND id NOT IN ({exclude:Array(String)})
334
+ ORDER BY "createdAt" DESC
335
+ LIMIT {limit:Int64}
336
+ `,
337
+ query_params: {
338
+ threadId,
339
+ exclude: messages.map((m) => m.id),
340
+ limit
341
+ },
342
+ clickhouse_settings: {
343
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
344
+ date_time_input_format: "best_effort",
345
+ date_time_output_format: "iso",
346
+ use_client_time_zone: 1,
347
+ output_format_json_quote_64bit_integers: 0
348
+ }
349
+ });
350
+ const rows = await result.json();
351
+ messages.push(...transformRows(rows.data));
352
+ messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
353
+ messages.forEach((message) => {
354
+ if (typeof message.content === "string") {
355
+ try {
356
+ message.content = JSON.parse(message.content);
357
+ } catch {
358
+ }
359
+ }
274
360
  });
361
+ const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
362
+ if (format === `v2`) return list.get.all.v2();
363
+ return list.get.all.v1();
275
364
  } catch (error$1) {
276
365
  throw new error.MastraError(
277
366
  {
278
- id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
367
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
279
368
  domain: error.ErrorDomain.STORAGE,
280
369
  category: error.ErrorCategory.THIRD_PARTY,
281
- details: { tableName }
370
+ details: { threadId, resourceId: resourceId ?? "" }
282
371
  },
283
372
  error$1
284
373
  );
285
374
  }
286
375
  }
287
- async materializeTtl({ tableName }) {
376
+ async getMessagesById({
377
+ messageIds,
378
+ format
379
+ }) {
380
+ if (messageIds.length === 0) return [];
288
381
  try {
289
- await this.db.command({
290
- query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
382
+ const result = await this.client.query({
383
+ query: `
384
+ SELECT
385
+ id,
386
+ content,
387
+ role,
388
+ type,
389
+ toDateTime64(createdAt, 3) as createdAt,
390
+ thread_id AS "threadId",
391
+ "resourceId"
392
+ FROM "${storage.TABLE_MESSAGES}"
393
+ WHERE id IN {messageIds:Array(String)}
394
+ ORDER BY "createdAt" DESC
395
+ `,
396
+ query_params: {
397
+ messageIds
398
+ },
399
+ clickhouse_settings: {
400
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
401
+ date_time_input_format: "best_effort",
402
+ date_time_output_format: "iso",
403
+ use_client_time_zone: 1,
404
+ output_format_json_quote_64bit_integers: 0
405
+ }
406
+ });
407
+ const rows = await result.json();
408
+ const messages = transformRows(rows.data);
409
+ messages.forEach((message) => {
410
+ if (typeof message.content === "string") {
411
+ try {
412
+ message.content = JSON.parse(message.content);
413
+ } catch {
414
+ }
415
+ }
291
416
  });
417
+ const list = new agent.MessageList().add(messages, "memory");
418
+ if (format === `v1`) return list.get.all.v1();
419
+ return list.get.all.v2();
292
420
  } catch (error$1) {
293
421
  throw new error.MastraError(
294
422
  {
295
- id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
423
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
296
424
  domain: error.ErrorDomain.STORAGE,
297
425
  category: error.ErrorCategory.THIRD_PARTY,
298
- details: { tableName }
426
+ details: { messageIds: JSON.stringify(messageIds) }
299
427
  },
300
428
  error$1
301
429
  );
302
430
  }
303
431
  }
304
- async createTable({
305
- tableName,
306
- schema
307
- }) {
308
- try {
309
- const columns = Object.entries(schema).map(([name, def]) => {
310
- const constraints = [];
311
- if (!def.nullable) constraints.push("NOT NULL");
312
- const columnTtl = this.ttl?.[tableName]?.columns?.[name];
313
- return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
314
- }).join(",\n");
315
- const rowTtl = this.ttl?.[tableName]?.row;
316
- const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
317
- CREATE TABLE IF NOT EXISTS ${tableName} (
318
- ${["id String"].concat(columns)}
319
- )
320
- ENGINE = ${TABLE_ENGINES[tableName]}
321
- PRIMARY KEY (createdAt, run_id, workflow_name)
322
- ORDER BY (createdAt, run_id, workflow_name)
323
- ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
324
- SETTINGS index_granularity = 8192
325
- ` : `
326
- CREATE TABLE IF NOT EXISTS ${tableName} (
327
- ${columns}
328
- )
329
- ENGINE = ${TABLE_ENGINES[tableName]}
330
- PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
331
- ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
332
- ${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
333
- SETTINGS index_granularity = 8192
334
- `;
335
- await this.db.query({
336
- query: sql,
337
- clickhouse_settings: {
338
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
339
- date_time_input_format: "best_effort",
340
- date_time_output_format: "iso",
341
- use_client_time_zone: 1,
342
- output_format_json_quote_64bit_integers: 0
343
- }
344
- });
345
- } catch (error$1) {
346
- throw new error.MastraError(
347
- {
348
- id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
349
- domain: error.ErrorDomain.STORAGE,
350
- category: error.ErrorCategory.THIRD_PARTY,
351
- details: { tableName }
352
- },
353
- error$1
354
- );
355
- }
356
- }
357
- getSqlType(type) {
358
- switch (type) {
359
- case "text":
360
- return "String";
361
- case "timestamp":
362
- return "DateTime64(3)";
363
- case "integer":
364
- case "bigint":
365
- return "Int64";
366
- case "jsonb":
367
- return "String";
368
- default:
369
- return super.getSqlType(type);
370
- }
371
- }
372
- /**
373
- * Alters table schema to add columns if they don't exist
374
- * @param tableName Name of the table
375
- * @param schema Schema of the table
376
- * @param ifNotExists Array of column names to add if they don't exist
377
- */
378
- async alterTable({
379
- tableName,
380
- schema,
381
- ifNotExists
382
- }) {
383
- try {
384
- const describeSql = `DESCRIBE TABLE ${tableName}`;
385
- const result = await this.db.query({
386
- query: describeSql
387
- });
388
- const rows = await result.json();
389
- const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
390
- for (const columnName of ifNotExists) {
391
- if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
392
- const columnDef = schema[columnName];
393
- let sqlType = this.getSqlType(columnDef.type);
394
- if (columnDef.nullable !== false) {
395
- sqlType = `Nullable(${sqlType})`;
396
- }
397
- const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
398
- const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
399
- await this.db.query({
400
- query: alterSql
401
- });
402
- this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
403
- }
432
+ async saveMessages(args) {
433
+ const { messages, format = "v1" } = args;
434
+ if (messages.length === 0) return messages;
435
+ for (const message of messages) {
436
+ const resourceId = message.resourceId;
437
+ if (!resourceId) {
438
+ throw new Error("Resource ID is required");
439
+ }
440
+ if (!message.threadId) {
441
+ throw new Error("Thread ID is required");
442
+ }
443
+ const thread = await this.getThreadById({ threadId: message.threadId });
444
+ if (!thread) {
445
+ throw new Error(`Thread ${message.threadId} not found`);
404
446
  }
405
- } catch (error$1) {
406
- throw new error.MastraError(
407
- {
408
- id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
409
- domain: error.ErrorDomain.STORAGE,
410
- category: error.ErrorCategory.THIRD_PARTY,
411
- details: { tableName }
412
- },
413
- error$1
414
- );
415
447
  }
416
- }
417
- async clearTable({ tableName }) {
448
+ const threadIdSet = /* @__PURE__ */ new Map();
449
+ await Promise.all(
450
+ messages.map(async (m) => {
451
+ const resourceId = m.resourceId;
452
+ if (!resourceId) {
453
+ throw new Error("Resource ID is required");
454
+ }
455
+ if (!m.threadId) {
456
+ throw new Error("Thread ID is required");
457
+ }
458
+ const thread = await this.getThreadById({ threadId: m.threadId });
459
+ if (!thread) {
460
+ throw new Error(`Thread ${m.threadId} not found`);
461
+ }
462
+ threadIdSet.set(m.threadId, thread);
463
+ })
464
+ );
418
465
  try {
419
- await this.db.query({
420
- query: `TRUNCATE TABLE ${tableName}`,
466
+ const existingResult = await this.client.query({
467
+ query: `SELECT id, thread_id FROM ${storage.TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
468
+ query_params: {
469
+ ids: messages.map((m) => m.id)
470
+ },
421
471
  clickhouse_settings: {
422
472
  // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
423
473
  date_time_input_format: "best_effort",
424
474
  date_time_output_format: "iso",
425
475
  use_client_time_zone: 1,
426
476
  output_format_json_quote_64bit_integers: 0
427
- }
428
- });
429
- } catch (error$1) {
430
- throw new error.MastraError(
431
- {
432
- id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
433
- domain: error.ErrorDomain.STORAGE,
434
- category: error.ErrorCategory.THIRD_PARTY,
435
- details: { tableName }
436
477
  },
437
- error$1
438
- );
439
- }
440
- }
441
- async insert({ tableName, record }) {
442
- try {
443
- await this.db.insert({
444
- table: tableName,
445
- values: [
446
- {
447
- ...record,
448
- createdAt: record.createdAt.toISOString(),
449
- updatedAt: record.updatedAt.toISOString()
478
+ format: "JSONEachRow"
479
+ });
480
+ const existingRows = await existingResult.json();
481
+ const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
482
+ const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
483
+ const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
484
+ const toMove = messages.filter((m) => {
485
+ const existingRow = existingRows.find((row) => row.id === m.id);
486
+ return existingRow && existingRow.thread_id !== m.threadId;
487
+ });
488
+ const deletePromises = toMove.map((message) => {
489
+ const existingRow = existingRows.find((row) => row.id === message.id);
490
+ if (!existingRow) return Promise.resolve();
491
+ return this.client.command({
492
+ query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
493
+ query_params: {
494
+ var_id: message.id,
495
+ var_old_thread_id: existingRow.thread_id
496
+ },
497
+ clickhouse_settings: {
498
+ date_time_input_format: "best_effort",
499
+ use_client_time_zone: 1,
500
+ output_format_json_quote_64bit_integers: 0
450
501
  }
451
- ],
452
- format: "JSONEachRow",
453
- clickhouse_settings: {
454
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
455
- output_format_json_quote_64bit_integers: 0,
456
- date_time_input_format: "best_effort",
457
- use_client_time_zone: 1
458
- }
502
+ });
459
503
  });
460
- } catch (error$1) {
461
- throw new error.MastraError(
462
- {
463
- id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
464
- domain: error.ErrorDomain.STORAGE,
465
- category: error.ErrorCategory.THIRD_PARTY,
466
- details: { tableName }
467
- },
468
- error$1
504
+ const updatePromises = toUpdate.map(
505
+ (message) => this.client.command({
506
+ query: `
507
+ ALTER TABLE ${storage.TABLE_MESSAGES}
508
+ UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
509
+ WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
510
+ `,
511
+ query_params: {
512
+ var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
513
+ var_role: message.role,
514
+ var_type: message.type || "v2",
515
+ var_resourceId: message.resourceId,
516
+ var_id: message.id,
517
+ var_thread_id: message.threadId
518
+ },
519
+ clickhouse_settings: {
520
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
521
+ date_time_input_format: "best_effort",
522
+ use_client_time_zone: 1,
523
+ output_format_json_quote_64bit_integers: 0
524
+ }
525
+ })
469
526
  );
470
- }
471
- }
472
- async load({
473
- tableName,
474
- keys
475
- }) {
476
- try {
477
- const keyEntries = Object.entries(keys);
478
- const conditions = keyEntries.map(
479
- ([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
480
- ).join(" AND ");
481
- const values = keyEntries.reduce((acc, [key, value]) => {
482
- return { ...acc, [`var_${key}`]: value };
483
- }, {});
484
- const result = await this.db.query({
485
- query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName].startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
486
- query_params: values,
487
- clickhouse_settings: {
488
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
489
- date_time_input_format: "best_effort",
490
- date_time_output_format: "iso",
491
- use_client_time_zone: 1,
492
- output_format_json_quote_64bit_integers: 0
493
- }
494
- });
495
- if (!result) {
496
- return null;
497
- }
498
- const rows = await result.json();
499
- if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
500
- const snapshot = rows.data[0];
501
- if (!snapshot) {
502
- return null;
503
- }
504
- if (typeof snapshot.snapshot === "string") {
505
- snapshot.snapshot = JSON.parse(snapshot.snapshot);
506
- }
507
- return transformRow(snapshot);
508
- }
509
- const data = transformRow(rows.data[0]);
510
- return data;
527
+ await Promise.all([
528
+ // Insert new messages (including moved messages)
529
+ this.client.insert({
530
+ table: storage.TABLE_MESSAGES,
531
+ format: "JSONEachRow",
532
+ values: toInsert.map((message) => ({
533
+ id: message.id,
534
+ thread_id: message.threadId,
535
+ resourceId: message.resourceId,
536
+ content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
537
+ createdAt: message.createdAt.toISOString(),
538
+ role: message.role,
539
+ type: message.type || "v2"
540
+ })),
541
+ clickhouse_settings: {
542
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
543
+ date_time_input_format: "best_effort",
544
+ use_client_time_zone: 1,
545
+ output_format_json_quote_64bit_integers: 0
546
+ }
547
+ }),
548
+ ...updatePromises,
549
+ ...deletePromises,
550
+ // Update thread's updatedAt timestamp
551
+ this.client.insert({
552
+ table: storage.TABLE_THREADS,
553
+ format: "JSONEachRow",
554
+ values: Array.from(threadIdSet.values()).map((thread) => ({
555
+ id: thread.id,
556
+ resourceId: thread.resourceId,
557
+ title: thread.title,
558
+ metadata: thread.metadata,
559
+ createdAt: thread.createdAt,
560
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
561
+ })),
562
+ clickhouse_settings: {
563
+ date_time_input_format: "best_effort",
564
+ use_client_time_zone: 1,
565
+ output_format_json_quote_64bit_integers: 0
566
+ }
567
+ })
568
+ ]);
569
+ const list = new agent.MessageList().add(messages, "memory");
570
+ if (format === `v2`) return list.get.all.v2();
571
+ return list.get.all.v1();
511
572
  } catch (error$1) {
512
573
  throw new error.MastraError(
513
574
  {
514
- id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
575
+ id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
515
576
  domain: error.ErrorDomain.STORAGE,
516
- category: error.ErrorCategory.THIRD_PARTY,
517
- details: { tableName }
577
+ category: error.ErrorCategory.THIRD_PARTY
518
578
  },
519
579
  error$1
520
580
  );
@@ -522,7 +582,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
522
582
  }
523
583
  async getThreadById({ threadId }) {
524
584
  try {
525
- const result = await this.db.query({
585
+ const result = await this.client.query({
526
586
  query: `SELECT
527
587
  id,
528
588
  "resourceId",
@@ -567,7 +627,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
567
627
  }
568
628
  async getThreadsByResourceId({ resourceId }) {
569
629
  try {
570
- const result = await this.db.query({
630
+ const result = await this.client.query({
571
631
  query: `SELECT
572
632
  id,
573
633
  "resourceId",
@@ -608,7 +668,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
608
668
  }
609
669
  async saveThread({ thread }) {
610
670
  try {
611
- await this.db.insert({
671
+ await this.client.insert({
612
672
  table: storage.TABLE_THREADS,
613
673
  values: [
614
674
  {
@@ -658,7 +718,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
658
718
  metadata: mergedMetadata,
659
719
  updatedAt: /* @__PURE__ */ new Date()
660
720
  };
661
- await this.db.insert({
721
+ await this.client.insert({
662
722
  table: storage.TABLE_THREADS,
663
723
  format: "JSONEachRow",
664
724
  values: [
@@ -692,14 +752,14 @@ var ClickhouseStore = class extends storage.MastraStorage {
692
752
  }
693
753
  async deleteThread({ threadId }) {
694
754
  try {
695
- await this.db.command({
755
+ await this.client.command({
696
756
  query: `DELETE FROM "${storage.TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
697
757
  query_params: { var_thread_id: threadId },
698
758
  clickhouse_settings: {
699
759
  output_format_json_quote_64bit_integers: 0
700
760
  }
701
761
  });
702
- await this.db.command({
762
+ await this.client.command({
703
763
  query: `DELETE FROM "${storage.TABLE_THREADS}" WHERE id = {var_id:String};`,
704
764
  query_params: { var_id: threadId },
705
765
  clickhouse_settings: {
@@ -718,254 +778,1644 @@ var ClickhouseStore = class extends storage.MastraStorage {
718
778
  );
719
779
  }
720
780
  }
721
- async getMessages({
722
- threadId,
723
- resourceId,
724
- selectBy,
725
- format
726
- }) {
781
+ async getThreadsByResourceIdPaginated(args) {
782
+ const { resourceId, page = 0, perPage = 100 } = args;
727
783
  try {
728
- const messages = [];
729
- const limit = this.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
730
- const include = selectBy?.include || [];
731
- if (include.length) {
732
- const includeResult = await this.db.query({
733
- query: `
734
- WITH ordered_messages AS (
735
- SELECT
736
- *,
737
- toDateTime64(createdAt, 3) as createdAt,
738
- toDateTime64(updatedAt, 3) as updatedAt,
739
- ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
740
- FROM "${storage.TABLE_MESSAGES}"
741
- WHERE thread_id = {var_thread_id:String}
742
- )
743
- SELECT
744
- m.id AS id,
745
- m.content as content,
746
- m.role as role,
747
- m.type as type,
748
- m.createdAt as createdAt,
749
- m.updatedAt as updatedAt,
750
- m.thread_id AS "threadId"
751
- FROM ordered_messages m
752
- WHERE m.id = ANY({var_include:Array(String)})
753
- OR EXISTS (
754
- SELECT 1 FROM ordered_messages target
755
- WHERE target.id = ANY({var_include:Array(String)})
756
- AND (
757
- -- Get previous messages based on the max withPreviousMessages
758
- (m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
759
- OR
760
- -- Get next messages based on the max withNextMessages
761
- (m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
762
- )
763
- )
764
- ORDER BY m."createdAt" DESC
765
- `,
766
- query_params: {
767
- var_thread_id: threadId,
768
- var_include: include.map((i) => i.id),
769
- var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
770
- var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
771
- },
772
- clickhouse_settings: {
773
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
774
- date_time_input_format: "best_effort",
775
- date_time_output_format: "iso",
776
- use_client_time_zone: 1,
777
- output_format_json_quote_64bit_integers: 0
778
- }
779
- });
780
- const rows2 = await includeResult.json();
781
- messages.push(...transformRows(rows2.data));
784
+ const currentOffset = page * perPage;
785
+ const countResult = await this.client.query({
786
+ query: `SELECT count() as total FROM ${storage.TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
787
+ query_params: { resourceId },
788
+ clickhouse_settings: {
789
+ date_time_input_format: "best_effort",
790
+ date_time_output_format: "iso",
791
+ use_client_time_zone: 1,
792
+ output_format_json_quote_64bit_integers: 0
793
+ }
794
+ });
795
+ const countData = await countResult.json();
796
+ const total = countData.data[0].total;
797
+ if (total === 0) {
798
+ return {
799
+ threads: [],
800
+ total: 0,
801
+ page,
802
+ perPage,
803
+ hasMore: false
804
+ };
782
805
  }
783
- const result = await this.db.query({
806
+ const dataResult = await this.client.query({
784
807
  query: `
785
- SELECT
786
- id,
787
- content,
788
- role,
789
- type,
790
- toDateTime64(createdAt, 3) as createdAt,
791
- thread_id AS "threadId"
792
- FROM "${storage.TABLE_MESSAGES}"
793
- WHERE thread_id = {threadId:String}
794
- AND id NOT IN ({exclude:Array(String)})
795
- ORDER BY "createdAt" DESC
796
- LIMIT {limit:Int64}
797
- `,
808
+ SELECT
809
+ id,
810
+ resourceId,
811
+ title,
812
+ metadata,
813
+ toDateTime64(createdAt, 3) as createdAt,
814
+ toDateTime64(updatedAt, 3) as updatedAt
815
+ FROM ${storage.TABLE_THREADS}
816
+ WHERE resourceId = {resourceId:String}
817
+ ORDER BY createdAt DESC
818
+ LIMIT {limit:Int64} OFFSET {offset:Int64}
819
+ `,
798
820
  query_params: {
799
- threadId,
800
- exclude: messages.map((m) => m.id),
801
- limit
821
+ resourceId,
822
+ limit: perPage,
823
+ offset: currentOffset
802
824
  },
803
825
  clickhouse_settings: {
804
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
805
826
  date_time_input_format: "best_effort",
806
827
  date_time_output_format: "iso",
807
828
  use_client_time_zone: 1,
808
829
  output_format_json_quote_64bit_integers: 0
809
830
  }
810
831
  });
811
- const rows = await result.json();
812
- messages.push(...transformRows(rows.data));
813
- messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
814
- messages.forEach((message) => {
815
- if (typeof message.content === "string") {
816
- try {
817
- message.content = JSON.parse(message.content);
818
- } catch {
819
- }
820
- }
821
- });
822
- const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
823
- if (format === `v2`) return list.get.all.v2();
824
- return list.get.all.v1();
832
+ const rows = await dataResult.json();
833
+ const threads = transformRows(rows.data);
834
+ return {
835
+ threads,
836
+ total,
837
+ page,
838
+ perPage,
839
+ hasMore: currentOffset + threads.length < total
840
+ };
825
841
  } catch (error$1) {
826
842
  throw new error.MastraError(
827
843
  {
828
- id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
844
+ id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
829
845
  domain: error.ErrorDomain.STORAGE,
830
846
  category: error.ErrorCategory.THIRD_PARTY,
831
- details: { threadId, resourceId: resourceId ?? "" }
847
+ details: { resourceId, page }
832
848
  },
833
849
  error$1
834
850
  );
835
851
  }
836
852
  }
837
- async saveMessages(args) {
838
- const { messages, format = "v1" } = args;
839
- if (messages.length === 0) return messages;
853
+ async getMessagesPaginated(args) {
854
+ const { threadId, resourceId, selectBy, format = "v1" } = args;
855
+ const page = selectBy?.pagination?.page || 0;
856
+ const perPageInput = selectBy?.pagination?.perPage;
857
+ const perPage = perPageInput !== void 0 ? perPageInput : storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
840
858
  try {
841
- const threadId = messages[0]?.threadId;
842
- const resourceId = messages[0]?.resourceId;
843
- if (!threadId) {
844
- throw new Error("Thread ID is required");
859
+ if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
860
+ const offset = page * perPage;
861
+ const dateRange = selectBy?.pagination?.dateRange;
862
+ const fromDate = dateRange?.start;
863
+ const toDate = dateRange?.end;
864
+ const messages = [];
865
+ if (selectBy?.include?.length) {
866
+ const include = selectBy.include;
867
+ const unionQueries = [];
868
+ const params = [];
869
+ let paramIdx = 1;
870
+ for (const inc of include) {
871
+ const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
872
+ const searchId = inc.threadId || threadId;
873
+ unionQueries.push(`
874
+ SELECT * FROM (
875
+ WITH numbered_messages AS (
876
+ SELECT
877
+ id, content, role, type, "createdAt", thread_id, "resourceId",
878
+ ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
879
+ FROM "${storage.TABLE_MESSAGES}"
880
+ WHERE thread_id = {var_thread_id_${paramIdx}:String}
881
+ ),
882
+ target_positions AS (
883
+ SELECT row_num as target_pos
884
+ FROM numbered_messages
885
+ WHERE id = {var_include_id_${paramIdx}:String}
886
+ )
887
+ SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
888
+ FROM numbered_messages m
889
+ CROSS JOIN target_positions t
890
+ WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
891
+ ) AS query_${paramIdx}
892
+ `);
893
+ params.push(
894
+ { [`var_thread_id_${paramIdx}`]: searchId },
895
+ { [`var_include_id_${paramIdx}`]: id },
896
+ { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
897
+ { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
898
+ );
899
+ paramIdx++;
900
+ }
901
+ const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
902
+ const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
903
+ const includeResult = await this.client.query({
904
+ query: finalQuery,
905
+ query_params: mergedParams,
906
+ clickhouse_settings: {
907
+ date_time_input_format: "best_effort",
908
+ date_time_output_format: "iso",
909
+ use_client_time_zone: 1,
910
+ output_format_json_quote_64bit_integers: 0
911
+ }
912
+ });
913
+ const rows2 = await includeResult.json();
914
+ const includedMessages = transformRows(rows2.data);
915
+ const seen = /* @__PURE__ */ new Set();
916
+ const dedupedMessages = includedMessages.filter((message) => {
917
+ if (seen.has(message.id)) return false;
918
+ seen.add(message.id);
919
+ return true;
920
+ });
921
+ messages.push(...dedupedMessages);
845
922
  }
846
- const thread = await this.getThreadById({ threadId });
847
- if (!thread) {
848
- throw new Error(`Thread ${threadId} not found`);
923
+ let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
924
+ const countParams = { threadId };
925
+ if (fromDate) {
926
+ countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
927
+ countParams.fromDate = fromDate.toISOString();
849
928
  }
850
- const existingResult = await this.db.query({
851
- query: `SELECT id, thread_id FROM ${storage.TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
852
- query_params: {
853
- ids: messages.map((m) => m.id)
854
- },
929
+ if (toDate) {
930
+ countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
931
+ countParams.toDate = toDate.toISOString();
932
+ }
933
+ const countResult = await this.client.query({
934
+ query: countQuery,
935
+ query_params: countParams,
936
+ clickhouse_settings: {
937
+ date_time_input_format: "best_effort",
938
+ date_time_output_format: "iso",
939
+ use_client_time_zone: 1,
940
+ output_format_json_quote_64bit_integers: 0
941
+ }
942
+ });
943
+ const countData = await countResult.json();
944
+ const total = countData.data[0].total;
945
+ if (total === 0 && messages.length === 0) {
946
+ return {
947
+ messages: [],
948
+ total: 0,
949
+ page,
950
+ perPage,
951
+ hasMore: false
952
+ };
953
+ }
954
+ const excludeIds = messages.map((m) => m.id);
955
+ let dataQuery = `
956
+ SELECT
957
+ id,
958
+ content,
959
+ role,
960
+ type,
961
+ toDateTime64(createdAt, 3) as createdAt,
962
+ thread_id AS "threadId",
963
+ resourceId
964
+ FROM ${storage.TABLE_MESSAGES}
965
+ WHERE thread_id = {threadId:String}
966
+ `;
967
+ const dataParams = { threadId };
968
+ if (fromDate) {
969
+ dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
970
+ dataParams.fromDate = fromDate.toISOString();
971
+ }
972
+ if (toDate) {
973
+ dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
974
+ dataParams.toDate = toDate.toISOString();
975
+ }
976
+ if (excludeIds.length > 0) {
977
+ dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
978
+ dataParams.excludeIds = excludeIds;
979
+ }
980
+ if (selectBy?.last) {
981
+ dataQuery += `
982
+ ORDER BY createdAt DESC
983
+ LIMIT {limit:Int64}
984
+ `;
985
+ dataParams.limit = perPage;
986
+ } else {
987
+ dataQuery += `
988
+ ORDER BY createdAt ASC
989
+ LIMIT {limit:Int64} OFFSET {offset:Int64}
990
+ `;
991
+ dataParams.limit = perPage;
992
+ dataParams.offset = offset;
993
+ }
994
+ const result = await this.client.query({
995
+ query: dataQuery,
996
+ query_params: dataParams,
855
997
  clickhouse_settings: {
856
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
857
998
  date_time_input_format: "best_effort",
858
999
  date_time_output_format: "iso",
859
1000
  use_client_time_zone: 1,
860
1001
  output_format_json_quote_64bit_integers: 0
1002
+ }
1003
+ });
1004
+ const rows = await result.json();
1005
+ const paginatedMessages = transformRows(rows.data);
1006
+ messages.push(...paginatedMessages);
1007
+ if (selectBy?.last) {
1008
+ messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
1009
+ }
1010
+ return {
1011
+ messages: format === "v2" ? messages : messages,
1012
+ total,
1013
+ page,
1014
+ perPage,
1015
+ hasMore: offset + perPage < total
1016
+ };
1017
+ } catch (error$1) {
1018
+ const mastraError = new error.MastraError(
1019
+ {
1020
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
1021
+ domain: error.ErrorDomain.STORAGE,
1022
+ category: error.ErrorCategory.THIRD_PARTY,
1023
+ details: {
1024
+ threadId,
1025
+ resourceId: resourceId ?? ""
1026
+ }
861
1027
  },
862
- format: "JSONEachRow"
1028
+ error$1
1029
+ );
1030
+ this.logger?.trackException?.(mastraError);
1031
+ this.logger?.error?.(mastraError.toString());
1032
+ return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
1033
+ }
1034
+ }
1035
+ async updateMessages(args) {
1036
+ const { messages } = args;
1037
+ if (messages.length === 0) {
1038
+ return [];
1039
+ }
1040
+ try {
1041
+ const messageIds = messages.map((m) => m.id);
1042
+ const existingResult = await this.client.query({
1043
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
1044
+ query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
1045
+ clickhouse_settings: {
1046
+ date_time_input_format: "best_effort",
1047
+ date_time_output_format: "iso",
1048
+ use_client_time_zone: 1,
1049
+ output_format_json_quote_64bit_integers: 0
1050
+ }
863
1051
  });
864
1052
  const existingRows = await existingResult.json();
865
- const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
866
- const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${threadId}`));
867
- const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${threadId}`));
868
- const updatePromises = toUpdate.map(
869
- (message) => this.db.command({
870
- query: `
871
- ALTER TABLE ${storage.TABLE_MESSAGES}
872
- UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}
873
- WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
874
- `,
875
- query_params: {
876
- var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
877
- var_role: message.role,
878
- var_type: message.type || "v2",
879
- var_id: message.id,
880
- var_thread_id: threadId
881
- },
882
- clickhouse_settings: {
883
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
884
- date_time_input_format: "best_effort",
885
- use_client_time_zone: 1,
886
- output_format_json_quote_64bit_integers: 0
1053
+ const existingMessages = transformRows(existingRows.data);
1054
+ if (existingMessages.length === 0) {
1055
+ return [];
1056
+ }
1057
+ const parsedExistingMessages = existingMessages.map((msg) => {
1058
+ if (typeof msg.content === "string") {
1059
+ try {
1060
+ msg.content = JSON.parse(msg.content);
1061
+ } catch {
887
1062
  }
888
- })
889
- );
890
- await Promise.all([
891
- // Insert messages
892
- this.db.insert({
893
- table: storage.TABLE_MESSAGES,
894
- format: "JSONEachRow",
895
- values: toInsert.map((message) => ({
896
- id: message.id,
897
- thread_id: threadId,
898
- content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
899
- createdAt: message.createdAt.toISOString(),
900
- role: message.role,
901
- type: message.type || "v2"
902
- })),
1063
+ }
1064
+ return msg;
1065
+ });
1066
+ const threadIdsToUpdate = /* @__PURE__ */ new Set();
1067
+ const updatePromises = [];
1068
+ for (const existingMessage of parsedExistingMessages) {
1069
+ const updatePayload = messages.find((m) => m.id === existingMessage.id);
1070
+ if (!updatePayload) continue;
1071
+ const { id, ...fieldsToUpdate } = updatePayload;
1072
+ if (Object.keys(fieldsToUpdate).length === 0) continue;
1073
+ threadIdsToUpdate.add(existingMessage.threadId);
1074
+ if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
1075
+ threadIdsToUpdate.add(updatePayload.threadId);
1076
+ }
1077
+ const setClauses = [];
1078
+ const values = {};
1079
+ let paramIdx = 1;
1080
+ let newContent = null;
1081
+ const updatableFields = { ...fieldsToUpdate };
1082
+ if (updatableFields.content) {
1083
+ const existingContent = existingMessage.content || {};
1084
+ const existingMetadata = existingContent.metadata || {};
1085
+ const updateMetadata = updatableFields.content.metadata || {};
1086
+ newContent = {
1087
+ ...existingContent,
1088
+ ...updatableFields.content,
1089
+ // Deep merge metadata
1090
+ metadata: {
1091
+ ...existingMetadata,
1092
+ ...updateMetadata
1093
+ }
1094
+ };
1095
+ setClauses.push(`content = {var_content_${paramIdx}:String}`);
1096
+ values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
1097
+ paramIdx++;
1098
+ delete updatableFields.content;
1099
+ }
1100
+ for (const key in updatableFields) {
1101
+ if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
1102
+ const dbColumn = key === "threadId" ? "thread_id" : key;
1103
+ setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
1104
+ values[`var_${key}_${paramIdx}`] = updatableFields[key];
1105
+ paramIdx++;
1106
+ }
1107
+ }
1108
+ if (setClauses.length > 0) {
1109
+ values[`var_id_${paramIdx}`] = id;
1110
+ const updateQuery = `
1111
+ ALTER TABLE ${storage.TABLE_MESSAGES}
1112
+ UPDATE ${setClauses.join(", ")}
1113
+ WHERE id = {var_id_${paramIdx}:String}
1114
+ `;
1115
+ console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
1116
+ updatePromises.push(
1117
+ this.client.command({
1118
+ query: updateQuery,
1119
+ query_params: values,
1120
+ clickhouse_settings: {
1121
+ date_time_input_format: "best_effort",
1122
+ use_client_time_zone: 1,
1123
+ output_format_json_quote_64bit_integers: 0
1124
+ }
1125
+ })
1126
+ );
1127
+ }
1128
+ }
1129
+ if (updatePromises.length > 0) {
1130
+ await Promise.all(updatePromises);
1131
+ }
1132
+ await this.client.command({
1133
+ query: `OPTIMIZE TABLE ${storage.TABLE_MESSAGES} FINAL`,
1134
+ clickhouse_settings: {
1135
+ date_time_input_format: "best_effort",
1136
+ use_client_time_zone: 1,
1137
+ output_format_json_quote_64bit_integers: 0
1138
+ }
1139
+ });
1140
+ for (const existingMessage of parsedExistingMessages) {
1141
+ const updatePayload = messages.find((m) => m.id === existingMessage.id);
1142
+ if (!updatePayload) continue;
1143
+ const { id, ...fieldsToUpdate } = updatePayload;
1144
+ if (Object.keys(fieldsToUpdate).length === 0) continue;
1145
+ const verifyResult = await this.client.query({
1146
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
1147
+ query_params: { messageId: id },
903
1148
  clickhouse_settings: {
904
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
905
1149
  date_time_input_format: "best_effort",
1150
+ date_time_output_format: "iso",
906
1151
  use_client_time_zone: 1,
907
1152
  output_format_json_quote_64bit_integers: 0
908
1153
  }
909
- }),
910
- ...updatePromises,
911
- // Update thread's updatedAt timestamp
912
- this.db.insert({
913
- table: storage.TABLE_THREADS,
914
- format: "JSONEachRow",
915
- values: [
916
- {
917
- id: thread.id,
918
- resourceId: thread.resourceId,
919
- title: thread.title,
920
- metadata: thread.metadata,
921
- createdAt: thread.createdAt,
922
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1154
+ });
1155
+ const verifyRows = await verifyResult.json();
1156
+ if (verifyRows.data.length > 0) {
1157
+ const updatedMessage = transformRows(verifyRows.data)[0];
1158
+ if (updatedMessage) {
1159
+ let needsRetry = false;
1160
+ for (const [key, value] of Object.entries(fieldsToUpdate)) {
1161
+ if (key === "content") {
1162
+ const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
1163
+ const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
1164
+ if (actualContent !== expectedContent) {
1165
+ needsRetry = true;
1166
+ break;
1167
+ }
1168
+ } else if (updatedMessage[key] !== value) {
1169
+ needsRetry = true;
1170
+ break;
1171
+ }
1172
+ }
1173
+ if (needsRetry) {
1174
+ console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
1175
+ await this.client.command({
1176
+ query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
1177
+ query_params: { messageId: id },
1178
+ clickhouse_settings: {
1179
+ date_time_input_format: "best_effort",
1180
+ use_client_time_zone: 1,
1181
+ output_format_json_quote_64bit_integers: 0
1182
+ }
1183
+ });
1184
+ let updatedContent = existingMessage.content || {};
1185
+ if (fieldsToUpdate.content) {
1186
+ const existingContent = existingMessage.content || {};
1187
+ const existingMetadata = existingContent.metadata || {};
1188
+ const updateMetadata = fieldsToUpdate.content.metadata || {};
1189
+ updatedContent = {
1190
+ ...existingContent,
1191
+ ...fieldsToUpdate.content,
1192
+ metadata: {
1193
+ ...existingMetadata,
1194
+ ...updateMetadata
1195
+ }
1196
+ };
1197
+ }
1198
+ const updatedMessageData = {
1199
+ ...existingMessage,
1200
+ ...fieldsToUpdate,
1201
+ content: updatedContent
1202
+ };
1203
+ await this.client.insert({
1204
+ table: storage.TABLE_MESSAGES,
1205
+ format: "JSONEachRow",
1206
+ values: [
1207
+ {
1208
+ id: updatedMessageData.id,
1209
+ thread_id: updatedMessageData.threadId,
1210
+ resourceId: updatedMessageData.resourceId,
1211
+ content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
1212
+ createdAt: updatedMessageData.createdAt.toISOString(),
1213
+ role: updatedMessageData.role,
1214
+ type: updatedMessageData.type || "v2"
1215
+ }
1216
+ ],
1217
+ clickhouse_settings: {
1218
+ date_time_input_format: "best_effort",
1219
+ use_client_time_zone: 1,
1220
+ output_format_json_quote_64bit_integers: 0
1221
+ }
1222
+ });
1223
+ }
1224
+ }
1225
+ }
1226
+ }
1227
+ if (threadIdsToUpdate.size > 0) {
1228
+ await new Promise((resolve) => setTimeout(resolve, 10));
1229
+ const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
1230
+ const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
1231
+ const threadResult = await this.client.query({
1232
+ query: `SELECT id, resourceId, title, metadata, createdAt FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
1233
+ query_params: { threadId },
1234
+ clickhouse_settings: {
1235
+ date_time_input_format: "best_effort",
1236
+ date_time_output_format: "iso",
1237
+ use_client_time_zone: 1,
1238
+ output_format_json_quote_64bit_integers: 0
923
1239
  }
924
- ],
1240
+ });
1241
+ const threadRows = await threadResult.json();
1242
+ if (threadRows.data.length > 0) {
1243
+ const existingThread = threadRows.data[0];
1244
+ await this.client.command({
1245
+ query: `DELETE FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
1246
+ query_params: { threadId },
1247
+ clickhouse_settings: {
1248
+ date_time_input_format: "best_effort",
1249
+ use_client_time_zone: 1,
1250
+ output_format_json_quote_64bit_integers: 0
1251
+ }
1252
+ });
1253
+ await this.client.insert({
1254
+ table: storage.TABLE_THREADS,
1255
+ format: "JSONEachRow",
1256
+ values: [
1257
+ {
1258
+ id: existingThread.id,
1259
+ resourceId: existingThread.resourceId,
1260
+ title: existingThread.title,
1261
+ metadata: existingThread.metadata,
1262
+ createdAt: existingThread.createdAt,
1263
+ updatedAt: now
1264
+ }
1265
+ ],
1266
+ clickhouse_settings: {
1267
+ date_time_input_format: "best_effort",
1268
+ use_client_time_zone: 1,
1269
+ output_format_json_quote_64bit_integers: 0
1270
+ }
1271
+ });
1272
+ }
1273
+ });
1274
+ await Promise.all(threadUpdatePromises);
1275
+ }
1276
+ const updatedMessages = [];
1277
+ for (const messageId of messageIds) {
1278
+ const updatedResult = await this.client.query({
1279
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
1280
+ query_params: { messageId },
925
1281
  clickhouse_settings: {
926
1282
  date_time_input_format: "best_effort",
1283
+ date_time_output_format: "iso",
927
1284
  use_client_time_zone: 1,
928
1285
  output_format_json_quote_64bit_integers: 0
929
1286
  }
930
- })
931
- ]);
932
- const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
933
- if (format === `v2`) return list.get.all.v2();
934
- return list.get.all.v1();
1287
+ });
1288
+ const updatedRows = await updatedResult.json();
1289
+ if (updatedRows.data.length > 0) {
1290
+ const message = transformRows(updatedRows.data)[0];
1291
+ if (message) {
1292
+ updatedMessages.push(message);
1293
+ }
1294
+ }
1295
+ }
1296
+ return updatedMessages.map((message) => {
1297
+ if (typeof message.content === "string") {
1298
+ try {
1299
+ message.content = JSON.parse(message.content);
1300
+ } catch {
1301
+ }
1302
+ }
1303
+ return message;
1304
+ });
1305
+ } catch (error$1) {
1306
+ throw new error.MastraError(
1307
+ {
1308
+ id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
1309
+ domain: error.ErrorDomain.STORAGE,
1310
+ category: error.ErrorCategory.THIRD_PARTY,
1311
+ details: { messageIds: messages.map((m) => m.id).join(",") }
1312
+ },
1313
+ error$1
1314
+ );
1315
+ }
1316
+ }
1317
+ async getResourceById({ resourceId }) {
1318
+ try {
1319
+ const result = await this.client.query({
1320
+ query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${storage.TABLE_RESOURCES} WHERE id = {resourceId:String}`,
1321
+ query_params: { resourceId },
1322
+ clickhouse_settings: {
1323
+ date_time_input_format: "best_effort",
1324
+ date_time_output_format: "iso",
1325
+ use_client_time_zone: 1,
1326
+ output_format_json_quote_64bit_integers: 0
1327
+ }
1328
+ });
1329
+ const rows = await result.json();
1330
+ if (rows.data.length === 0) {
1331
+ return null;
1332
+ }
1333
+ const resource = rows.data[0];
1334
+ return {
1335
+ id: resource.id,
1336
+ workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
1337
+ metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
1338
+ createdAt: new Date(resource.createdAt),
1339
+ updatedAt: new Date(resource.updatedAt)
1340
+ };
1341
+ } catch (error$1) {
1342
+ throw new error.MastraError(
1343
+ {
1344
+ id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
1345
+ domain: error.ErrorDomain.STORAGE,
1346
+ category: error.ErrorCategory.THIRD_PARTY,
1347
+ details: { resourceId }
1348
+ },
1349
+ error$1
1350
+ );
1351
+ }
1352
+ }
1353
+ async saveResource({ resource }) {
1354
+ try {
1355
+ await this.client.insert({
1356
+ table: storage.TABLE_RESOURCES,
1357
+ format: "JSONEachRow",
1358
+ values: [
1359
+ {
1360
+ id: resource.id,
1361
+ workingMemory: resource.workingMemory,
1362
+ metadata: JSON.stringify(resource.metadata),
1363
+ createdAt: resource.createdAt.toISOString(),
1364
+ updatedAt: resource.updatedAt.toISOString()
1365
+ }
1366
+ ],
1367
+ clickhouse_settings: {
1368
+ date_time_input_format: "best_effort",
1369
+ use_client_time_zone: 1,
1370
+ output_format_json_quote_64bit_integers: 0
1371
+ }
1372
+ });
1373
+ return resource;
1374
+ } catch (error$1) {
1375
+ throw new error.MastraError(
1376
+ {
1377
+ id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
1378
+ domain: error.ErrorDomain.STORAGE,
1379
+ category: error.ErrorCategory.THIRD_PARTY,
1380
+ details: { resourceId: resource.id }
1381
+ },
1382
+ error$1
1383
+ );
1384
+ }
1385
+ }
1386
+ async updateResource({
1387
+ resourceId,
1388
+ workingMemory,
1389
+ metadata
1390
+ }) {
1391
+ try {
1392
+ const existingResource = await this.getResourceById({ resourceId });
1393
+ if (!existingResource) {
1394
+ const newResource = {
1395
+ id: resourceId,
1396
+ workingMemory,
1397
+ metadata: metadata || {},
1398
+ createdAt: /* @__PURE__ */ new Date(),
1399
+ updatedAt: /* @__PURE__ */ new Date()
1400
+ };
1401
+ return this.saveResource({ resource: newResource });
1402
+ }
1403
+ const updatedResource = {
1404
+ ...existingResource,
1405
+ workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
1406
+ metadata: {
1407
+ ...existingResource.metadata,
1408
+ ...metadata
1409
+ },
1410
+ updatedAt: /* @__PURE__ */ new Date()
1411
+ };
1412
+ const updateQuery = `
1413
+ ALTER TABLE ${storage.TABLE_RESOURCES}
1414
+ UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
1415
+ WHERE id = {resourceId:String}
1416
+ `;
1417
+ await this.client.command({
1418
+ query: updateQuery,
1419
+ query_params: {
1420
+ workingMemory: updatedResource.workingMemory,
1421
+ metadata: JSON.stringify(updatedResource.metadata),
1422
+ updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
1423
+ resourceId
1424
+ },
1425
+ clickhouse_settings: {
1426
+ date_time_input_format: "best_effort",
1427
+ use_client_time_zone: 1,
1428
+ output_format_json_quote_64bit_integers: 0
1429
+ }
1430
+ });
1431
+ await this.client.command({
1432
+ query: `OPTIMIZE TABLE ${storage.TABLE_RESOURCES} FINAL`,
1433
+ clickhouse_settings: {
1434
+ date_time_input_format: "best_effort",
1435
+ use_client_time_zone: 1,
1436
+ output_format_json_quote_64bit_integers: 0
1437
+ }
1438
+ });
1439
+ return updatedResource;
1440
+ } catch (error$1) {
1441
+ throw new error.MastraError(
1442
+ {
1443
+ id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
1444
+ domain: error.ErrorDomain.STORAGE,
1445
+ category: error.ErrorCategory.THIRD_PARTY,
1446
+ details: { resourceId }
1447
+ },
1448
+ error$1
1449
+ );
1450
+ }
1451
+ }
1452
+ };
1453
+ var StoreOperationsClickhouse = class extends storage.StoreOperations {
1454
+ ttl;
1455
+ client;
1456
+ constructor({ client, ttl }) {
1457
+ super();
1458
+ this.ttl = ttl;
1459
+ this.client = client;
1460
+ }
1461
+ async hasColumn(table, column) {
1462
+ const result = await this.client.query({
1463
+ query: `DESCRIBE TABLE ${table}`,
1464
+ format: "JSONEachRow"
1465
+ });
1466
+ const columns = await result.json();
1467
+ return columns.some((c) => c.name === column);
1468
+ }
1469
+ getSqlType(type) {
1470
+ switch (type) {
1471
+ case "text":
1472
+ return "String";
1473
+ case "timestamp":
1474
+ return "DateTime64(3)";
1475
+ case "integer":
1476
+ case "bigint":
1477
+ return "Int64";
1478
+ case "jsonb":
1479
+ return "String";
1480
+ default:
1481
+ return super.getSqlType(type);
1482
+ }
1483
+ }
1484
+ async createTable({
1485
+ tableName,
1486
+ schema
1487
+ }) {
1488
+ try {
1489
+ const columns = Object.entries(schema).map(([name, def]) => {
1490
+ const constraints = [];
1491
+ if (!def.nullable) constraints.push("NOT NULL");
1492
+ const columnTtl = this.ttl?.[tableName]?.columns?.[name];
1493
+ return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
1494
+ }).join(",\n");
1495
+ const rowTtl = this.ttl?.[tableName]?.row;
1496
+ const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
1497
+ CREATE TABLE IF NOT EXISTS ${tableName} (
1498
+ ${["id String"].concat(columns)}
1499
+ )
1500
+ ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
1501
+ PRIMARY KEY (createdAt, run_id, workflow_name)
1502
+ ORDER BY (createdAt, run_id, workflow_name)
1503
+ ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
1504
+ SETTINGS index_granularity = 8192
1505
+ ` : `
1506
+ CREATE TABLE IF NOT EXISTS ${tableName} (
1507
+ ${columns}
1508
+ )
1509
+ ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
1510
+ PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
1511
+ ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
1512
+ ${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
1513
+ SETTINGS index_granularity = 8192
1514
+ `;
1515
+ await this.client.query({
1516
+ query: sql,
1517
+ clickhouse_settings: {
1518
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1519
+ date_time_input_format: "best_effort",
1520
+ date_time_output_format: "iso",
1521
+ use_client_time_zone: 1,
1522
+ output_format_json_quote_64bit_integers: 0
1523
+ }
1524
+ });
1525
+ } catch (error$1) {
1526
+ throw new error.MastraError(
1527
+ {
1528
+ id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
1529
+ domain: error.ErrorDomain.STORAGE,
1530
+ category: error.ErrorCategory.THIRD_PARTY,
1531
+ details: { tableName }
1532
+ },
1533
+ error$1
1534
+ );
1535
+ }
1536
+ }
1537
+ async alterTable({
1538
+ tableName,
1539
+ schema,
1540
+ ifNotExists
1541
+ }) {
1542
+ try {
1543
+ const describeSql = `DESCRIBE TABLE ${tableName}`;
1544
+ const result = await this.client.query({
1545
+ query: describeSql
1546
+ });
1547
+ const rows = await result.json();
1548
+ const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
1549
+ for (const columnName of ifNotExists) {
1550
+ if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
1551
+ const columnDef = schema[columnName];
1552
+ let sqlType = this.getSqlType(columnDef.type);
1553
+ if (columnDef.nullable !== false) {
1554
+ sqlType = `Nullable(${sqlType})`;
1555
+ }
1556
+ const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
1557
+ const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
1558
+ await this.client.query({
1559
+ query: alterSql
1560
+ });
1561
+ this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
1562
+ }
1563
+ }
1564
+ } catch (error$1) {
1565
+ throw new error.MastraError(
1566
+ {
1567
+ id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
1568
+ domain: error.ErrorDomain.STORAGE,
1569
+ category: error.ErrorCategory.THIRD_PARTY,
1570
+ details: { tableName }
1571
+ },
1572
+ error$1
1573
+ );
1574
+ }
1575
+ }
1576
+ async clearTable({ tableName }) {
1577
+ try {
1578
+ await this.client.query({
1579
+ query: `TRUNCATE TABLE ${tableName}`,
1580
+ clickhouse_settings: {
1581
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1582
+ date_time_input_format: "best_effort",
1583
+ date_time_output_format: "iso",
1584
+ use_client_time_zone: 1,
1585
+ output_format_json_quote_64bit_integers: 0
1586
+ }
1587
+ });
1588
+ } catch (error$1) {
1589
+ throw new error.MastraError(
1590
+ {
1591
+ id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
1592
+ domain: error.ErrorDomain.STORAGE,
1593
+ category: error.ErrorCategory.THIRD_PARTY,
1594
+ details: { tableName }
1595
+ },
1596
+ error$1
1597
+ );
1598
+ }
1599
+ }
1600
+ async dropTable({ tableName }) {
1601
+ await this.client.query({
1602
+ query: `DROP TABLE IF EXISTS ${tableName}`
1603
+ });
1604
+ }
1605
+ async insert({ tableName, record }) {
1606
+ const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
1607
+ const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
1608
+ try {
1609
+ const result = await this.client.insert({
1610
+ table: tableName,
1611
+ values: [
1612
+ {
1613
+ ...record,
1614
+ createdAt,
1615
+ updatedAt
1616
+ }
1617
+ ],
1618
+ format: "JSONEachRow",
1619
+ clickhouse_settings: {
1620
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1621
+ output_format_json_quote_64bit_integers: 0,
1622
+ date_time_input_format: "best_effort",
1623
+ use_client_time_zone: 1
1624
+ }
1625
+ });
1626
+ console.info("INSERT RESULT", result);
1627
+ } catch (error$1) {
1628
+ throw new error.MastraError(
1629
+ {
1630
+ id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
1631
+ domain: error.ErrorDomain.STORAGE,
1632
+ category: error.ErrorCategory.THIRD_PARTY,
1633
+ details: { tableName }
1634
+ },
1635
+ error$1
1636
+ );
1637
+ }
1638
+ }
1639
+ async batchInsert({ tableName, records }) {
1640
+ const recordsToBeInserted = records.map((record) => ({
1641
+ ...Object.fromEntries(
1642
+ Object.entries(record).map(([key, value]) => [
1643
+ key,
1644
+ storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
1645
+ ])
1646
+ )
1647
+ }));
1648
+ try {
1649
+ await this.client.insert({
1650
+ table: tableName,
1651
+ values: recordsToBeInserted,
1652
+ format: "JSONEachRow",
1653
+ clickhouse_settings: {
1654
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1655
+ date_time_input_format: "best_effort",
1656
+ use_client_time_zone: 1,
1657
+ output_format_json_quote_64bit_integers: 0
1658
+ }
1659
+ });
1660
+ } catch (error$1) {
1661
+ throw new error.MastraError(
1662
+ {
1663
+ id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
1664
+ domain: error.ErrorDomain.STORAGE,
1665
+ category: error.ErrorCategory.THIRD_PARTY,
1666
+ details: { tableName }
1667
+ },
1668
+ error$1
1669
+ );
1670
+ }
1671
+ }
1672
+ async load({ tableName, keys }) {
1673
+ try {
1674
+ const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
1675
+ const keyEntries = Object.entries(keys);
1676
+ const conditions = keyEntries.map(
1677
+ ([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
1678
+ ).join(" AND ");
1679
+ const values = keyEntries.reduce((acc, [key, value]) => {
1680
+ return { ...acc, [`var_${key}`]: value };
1681
+ }, {});
1682
+ const hasUpdatedAt = storage.TABLE_SCHEMAS[tableName]?.updatedAt;
1683
+ const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
1684
+ const result = await this.client.query({
1685
+ query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
1686
+ query_params: values,
1687
+ clickhouse_settings: {
1688
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1689
+ date_time_input_format: "best_effort",
1690
+ date_time_output_format: "iso",
1691
+ use_client_time_zone: 1,
1692
+ output_format_json_quote_64bit_integers: 0
1693
+ }
1694
+ });
1695
+ if (!result) {
1696
+ return null;
1697
+ }
1698
+ const rows = await result.json();
1699
+ if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
1700
+ const snapshot = rows.data[0];
1701
+ if (!snapshot) {
1702
+ return null;
1703
+ }
1704
+ if (typeof snapshot.snapshot === "string") {
1705
+ snapshot.snapshot = JSON.parse(snapshot.snapshot);
1706
+ }
1707
+ return transformRow(snapshot);
1708
+ }
1709
+ const data = transformRow(rows.data[0]);
1710
+ return data;
1711
+ } catch (error$1) {
1712
+ throw new error.MastraError(
1713
+ {
1714
+ id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
1715
+ domain: error.ErrorDomain.STORAGE,
1716
+ category: error.ErrorCategory.THIRD_PARTY,
1717
+ details: { tableName }
1718
+ },
1719
+ error$1
1720
+ );
1721
+ }
1722
+ }
1723
+ };
1724
+ var ScoresStorageClickhouse = class extends storage.ScoresStorage {
1725
+ client;
1726
+ operations;
1727
+ constructor({ client, operations }) {
1728
+ super();
1729
+ this.client = client;
1730
+ this.operations = operations;
1731
+ }
1732
+ transformScoreRow(row) {
1733
+ const scorer = storage.safelyParseJSON(row.scorer);
1734
+ const preprocessStepResult = storage.safelyParseJSON(row.preprocessStepResult);
1735
+ const analyzeStepResult = storage.safelyParseJSON(row.analyzeStepResult);
1736
+ const metadata = storage.safelyParseJSON(row.metadata);
1737
+ const input = storage.safelyParseJSON(row.input);
1738
+ const output = storage.safelyParseJSON(row.output);
1739
+ const additionalContext = storage.safelyParseJSON(row.additionalContext);
1740
+ const runtimeContext = storage.safelyParseJSON(row.runtimeContext);
1741
+ const entity = storage.safelyParseJSON(row.entity);
1742
+ return {
1743
+ ...row,
1744
+ scorer,
1745
+ preprocessStepResult,
1746
+ analyzeStepResult,
1747
+ metadata,
1748
+ input,
1749
+ output,
1750
+ additionalContext,
1751
+ runtimeContext,
1752
+ entity,
1753
+ createdAt: new Date(row.createdAt),
1754
+ updatedAt: new Date(row.updatedAt)
1755
+ };
1756
+ }
1757
+ async getScoreById({ id }) {
1758
+ try {
1759
+ const result = await this.client.query({
1760
+ query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE id = {var_id:String}`,
1761
+ query_params: { var_id: id },
1762
+ format: "JSONEachRow",
1763
+ clickhouse_settings: {
1764
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1765
+ date_time_input_format: "best_effort",
1766
+ date_time_output_format: "iso",
1767
+ use_client_time_zone: 1,
1768
+ output_format_json_quote_64bit_integers: 0
1769
+ }
1770
+ });
1771
+ const resultJson = await result.json();
1772
+ if (!Array.isArray(resultJson) || resultJson.length === 0) {
1773
+ return null;
1774
+ }
1775
+ return this.transformScoreRow(resultJson[0]);
1776
+ } catch (error$1) {
1777
+ throw new error.MastraError(
1778
+ {
1779
+ id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
1780
+ domain: error.ErrorDomain.STORAGE,
1781
+ category: error.ErrorCategory.THIRD_PARTY,
1782
+ details: { scoreId: id }
1783
+ },
1784
+ error$1
1785
+ );
1786
+ }
1787
+ }
1788
+ async saveScore(score) {
1789
+ let parsedScore;
1790
+ try {
1791
+ parsedScore = scores.saveScorePayloadSchema.parse(score);
1792
+ } catch (error$1) {
1793
+ throw new error.MastraError(
1794
+ {
1795
+ id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED_INVALID_SCORE_PAYLOAD",
1796
+ domain: error.ErrorDomain.STORAGE,
1797
+ category: error.ErrorCategory.USER,
1798
+ details: { scoreId: score.id }
1799
+ },
1800
+ error$1
1801
+ );
1802
+ }
1803
+ try {
1804
+ const record = {
1805
+ ...parsedScore
1806
+ };
1807
+ await this.client.insert({
1808
+ table: storage.TABLE_SCORERS,
1809
+ values: [record],
1810
+ format: "JSONEachRow",
1811
+ clickhouse_settings: {
1812
+ date_time_input_format: "best_effort",
1813
+ use_client_time_zone: 1,
1814
+ output_format_json_quote_64bit_integers: 0
1815
+ }
1816
+ });
1817
+ return { score };
1818
+ } catch (error$1) {
1819
+ throw new error.MastraError(
1820
+ {
1821
+ id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
1822
+ domain: error.ErrorDomain.STORAGE,
1823
+ category: error.ErrorCategory.THIRD_PARTY,
1824
+ details: { scoreId: score.id }
1825
+ },
1826
+ error$1
1827
+ );
1828
+ }
1829
+ }
1830
+ async getScoresByRunId({
1831
+ runId,
1832
+ pagination
1833
+ }) {
1834
+ try {
1835
+ const countResult = await this.client.query({
1836
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String}`,
1837
+ query_params: { var_runId: runId },
1838
+ format: "JSONEachRow"
1839
+ });
1840
+ const countRows = await countResult.json();
1841
+ let total = 0;
1842
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1843
+ const countObj = countRows[0];
1844
+ total = Number(countObj.count);
1845
+ }
1846
+ if (!total) {
1847
+ return {
1848
+ pagination: {
1849
+ total: 0,
1850
+ page: pagination.page,
1851
+ perPage: pagination.perPage,
1852
+ hasMore: false
1853
+ },
1854
+ scores: []
1855
+ };
1856
+ }
1857
+ const offset = pagination.page * pagination.perPage;
1858
+ const result = await this.client.query({
1859
+ query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
1860
+ query_params: {
1861
+ var_runId: runId,
1862
+ var_limit: pagination.perPage,
1863
+ var_offset: offset
1864
+ },
1865
+ format: "JSONEachRow",
1866
+ clickhouse_settings: {
1867
+ date_time_input_format: "best_effort",
1868
+ date_time_output_format: "iso",
1869
+ use_client_time_zone: 1,
1870
+ output_format_json_quote_64bit_integers: 0
1871
+ }
1872
+ });
1873
+ const rows = await result.json();
1874
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
1875
+ return {
1876
+ pagination: {
1877
+ total,
1878
+ page: pagination.page,
1879
+ perPage: pagination.perPage,
1880
+ hasMore: total > (pagination.page + 1) * pagination.perPage
1881
+ },
1882
+ scores
1883
+ };
1884
+ } catch (error$1) {
1885
+ throw new error.MastraError(
1886
+ {
1887
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
1888
+ domain: error.ErrorDomain.STORAGE,
1889
+ category: error.ErrorCategory.THIRD_PARTY,
1890
+ details: { runId }
1891
+ },
1892
+ error$1
1893
+ );
1894
+ }
1895
+ }
1896
+ async getScoresByScorerId({
1897
+ scorerId,
1898
+ entityId,
1899
+ entityType,
1900
+ source,
1901
+ pagination
1902
+ }) {
1903
+ let whereClause = `scorerId = {var_scorerId:String}`;
1904
+ if (entityId) {
1905
+ whereClause += ` AND entityId = {var_entityId:String}`;
1906
+ }
1907
+ if (entityType) {
1908
+ whereClause += ` AND entityType = {var_entityType:String}`;
1909
+ }
1910
+ if (source) {
1911
+ whereClause += ` AND source = {var_source:String}`;
1912
+ }
1913
+ try {
1914
+ const countResult = await this.client.query({
1915
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE ${whereClause}`,
1916
+ query_params: {
1917
+ var_scorerId: scorerId,
1918
+ var_entityId: entityId,
1919
+ var_entityType: entityType,
1920
+ var_source: source
1921
+ },
1922
+ format: "JSONEachRow"
1923
+ });
1924
+ const countRows = await countResult.json();
1925
+ let total = 0;
1926
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1927
+ const countObj = countRows[0];
1928
+ total = Number(countObj.count);
1929
+ }
1930
+ if (!total) {
1931
+ return {
1932
+ pagination: {
1933
+ total: 0,
1934
+ page: pagination.page,
1935
+ perPage: pagination.perPage,
1936
+ hasMore: false
1937
+ },
1938
+ scores: []
1939
+ };
1940
+ }
1941
+ const offset = pagination.page * pagination.perPage;
1942
+ const result = await this.client.query({
1943
+ query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
1944
+ query_params: {
1945
+ var_scorerId: scorerId,
1946
+ var_limit: pagination.perPage,
1947
+ var_offset: offset,
1948
+ var_entityId: entityId,
1949
+ var_entityType: entityType,
1950
+ var_source: source
1951
+ },
1952
+ format: "JSONEachRow",
1953
+ clickhouse_settings: {
1954
+ date_time_input_format: "best_effort",
1955
+ date_time_output_format: "iso",
1956
+ use_client_time_zone: 1,
1957
+ output_format_json_quote_64bit_integers: 0
1958
+ }
1959
+ });
1960
+ const rows = await result.json();
1961
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
1962
+ return {
1963
+ pagination: {
1964
+ total,
1965
+ page: pagination.page,
1966
+ perPage: pagination.perPage,
1967
+ hasMore: total > (pagination.page + 1) * pagination.perPage
1968
+ },
1969
+ scores
1970
+ };
1971
+ } catch (error$1) {
1972
+ throw new error.MastraError(
1973
+ {
1974
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
1975
+ domain: error.ErrorDomain.STORAGE,
1976
+ category: error.ErrorCategory.THIRD_PARTY,
1977
+ details: { scorerId }
1978
+ },
1979
+ error$1
1980
+ );
1981
+ }
1982
+ }
1983
+ async getScoresByEntityId({
1984
+ entityId,
1985
+ entityType,
1986
+ pagination
1987
+ }) {
1988
+ try {
1989
+ const countResult = await this.client.query({
1990
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
1991
+ query_params: { var_entityId: entityId, var_entityType: entityType },
1992
+ format: "JSONEachRow"
1993
+ });
1994
+ const countRows = await countResult.json();
1995
+ let total = 0;
1996
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1997
+ const countObj = countRows[0];
1998
+ total = Number(countObj.count);
1999
+ }
2000
+ if (!total) {
2001
+ return {
2002
+ pagination: {
2003
+ total: 0,
2004
+ page: pagination.page,
2005
+ perPage: pagination.perPage,
2006
+ hasMore: false
2007
+ },
2008
+ scores: []
2009
+ };
2010
+ }
2011
+ const offset = pagination.page * pagination.perPage;
2012
+ const result = await this.client.query({
2013
+ query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
2014
+ query_params: {
2015
+ var_entityId: entityId,
2016
+ var_entityType: entityType,
2017
+ var_limit: pagination.perPage,
2018
+ var_offset: offset
2019
+ },
2020
+ format: "JSONEachRow",
2021
+ clickhouse_settings: {
2022
+ date_time_input_format: "best_effort",
2023
+ date_time_output_format: "iso",
2024
+ use_client_time_zone: 1,
2025
+ output_format_json_quote_64bit_integers: 0
2026
+ }
2027
+ });
2028
+ const rows = await result.json();
2029
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
2030
+ return {
2031
+ pagination: {
2032
+ total,
2033
+ page: pagination.page,
2034
+ perPage: pagination.perPage,
2035
+ hasMore: total > (pagination.page + 1) * pagination.perPage
2036
+ },
2037
+ scores
2038
+ };
2039
+ } catch (error$1) {
2040
+ throw new error.MastraError(
2041
+ {
2042
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
2043
+ domain: error.ErrorDomain.STORAGE,
2044
+ category: error.ErrorCategory.THIRD_PARTY,
2045
+ details: { entityId, entityType }
2046
+ },
2047
+ error$1
2048
+ );
2049
+ }
2050
+ }
2051
+ async getScoresBySpan({
2052
+ traceId,
2053
+ spanId,
2054
+ pagination
2055
+ }) {
2056
+ try {
2057
+ const countResult = await this.client.query({
2058
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String}`,
2059
+ query_params: {
2060
+ var_traceId: traceId,
2061
+ var_spanId: spanId
2062
+ },
2063
+ format: "JSONEachRow"
2064
+ });
2065
+ const countRows = await countResult.json();
2066
+ let total = 0;
2067
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
2068
+ const countObj = countRows[0];
2069
+ total = Number(countObj.count);
2070
+ }
2071
+ if (!total) {
2072
+ return {
2073
+ pagination: {
2074
+ total: 0,
2075
+ page: pagination.page,
2076
+ perPage: pagination.perPage,
2077
+ hasMore: false
2078
+ },
2079
+ scores: []
2080
+ };
2081
+ }
2082
+ const limit = pagination.perPage + 1;
2083
+ const offset = pagination.page * pagination.perPage;
2084
+ const result = await this.client.query({
2085
+ query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
2086
+ query_params: {
2087
+ var_traceId: traceId,
2088
+ var_spanId: spanId,
2089
+ var_limit: limit,
2090
+ var_offset: offset
2091
+ },
2092
+ format: "JSONEachRow",
2093
+ clickhouse_settings: {
2094
+ date_time_input_format: "best_effort",
2095
+ date_time_output_format: "iso",
2096
+ use_client_time_zone: 1,
2097
+ output_format_json_quote_64bit_integers: 0
2098
+ }
2099
+ });
2100
+ const rows = await result.json();
2101
+ const transformedRows = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
2102
+ const hasMore = transformedRows.length > pagination.perPage;
2103
+ const scores = hasMore ? transformedRows.slice(0, pagination.perPage) : transformedRows;
2104
+ return {
2105
+ pagination: {
2106
+ total,
2107
+ page: pagination.page,
2108
+ perPage: pagination.perPage,
2109
+ hasMore
2110
+ },
2111
+ scores
2112
+ };
2113
+ } catch (error$1) {
2114
+ throw new error.MastraError(
2115
+ {
2116
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
2117
+ domain: error.ErrorDomain.STORAGE,
2118
+ category: error.ErrorCategory.THIRD_PARTY,
2119
+ details: { traceId, spanId }
2120
+ },
2121
+ error$1
2122
+ );
2123
+ }
2124
+ }
2125
+ };
2126
+ var TracesStorageClickhouse = class extends storage.TracesStorage {
2127
+ client;
2128
+ operations;
2129
+ constructor({ client, operations }) {
2130
+ super();
2131
+ this.client = client;
2132
+ this.operations = operations;
2133
+ }
2134
+ async getTracesPaginated(args) {
2135
+ const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
2136
+ const fromDate = dateRange?.start;
2137
+ const toDate = dateRange?.end;
2138
+ const currentOffset = page * perPage;
2139
+ const queryArgs = {};
2140
+ const conditions = [];
2141
+ if (name) {
2142
+ conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
2143
+ queryArgs.var_name = name;
2144
+ }
2145
+ if (scope) {
2146
+ conditions.push(`scope = {var_scope:String}`);
2147
+ queryArgs.var_scope = scope;
2148
+ }
2149
+ if (attributes) {
2150
+ Object.entries(attributes).forEach(([key, value]) => {
2151
+ conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
2152
+ queryArgs[`var_attr_${key}`] = value;
2153
+ });
2154
+ }
2155
+ if (filters) {
2156
+ Object.entries(filters).forEach(([key, value]) => {
2157
+ conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
2158
+ queryArgs[`var_col_${key}`] = value;
2159
+ });
2160
+ }
2161
+ if (fromDate) {
2162
+ conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
2163
+ queryArgs.var_from_date = fromDate.toISOString();
2164
+ }
2165
+ if (toDate) {
2166
+ conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
2167
+ queryArgs.var_to_date = toDate.toISOString();
2168
+ }
2169
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
2170
+ try {
2171
+ const countResult = await this.client.query({
2172
+ query: `SELECT COUNT(*) as count FROM ${storage.TABLE_TRACES} ${whereClause}`,
2173
+ query_params: queryArgs,
2174
+ clickhouse_settings: {
2175
+ date_time_input_format: "best_effort",
2176
+ date_time_output_format: "iso",
2177
+ use_client_time_zone: 1,
2178
+ output_format_json_quote_64bit_integers: 0
2179
+ }
2180
+ });
2181
+ const countData = await countResult.json();
2182
+ const total = Number(countData.data?.[0]?.count ?? 0);
2183
+ if (total === 0) {
2184
+ return {
2185
+ traces: [],
2186
+ total: 0,
2187
+ page,
2188
+ perPage,
2189
+ hasMore: false
2190
+ };
2191
+ }
2192
+ const result = await this.client.query({
2193
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
2194
+ query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
2195
+ clickhouse_settings: {
2196
+ date_time_input_format: "best_effort",
2197
+ date_time_output_format: "iso",
2198
+ use_client_time_zone: 1,
2199
+ output_format_json_quote_64bit_integers: 0
2200
+ }
2201
+ });
2202
+ if (!result) {
2203
+ return {
2204
+ traces: [],
2205
+ total,
2206
+ page,
2207
+ perPage,
2208
+ hasMore: false
2209
+ };
2210
+ }
2211
+ const resp = await result.json();
2212
+ const rows = resp.data;
2213
+ const traces = rows.map((row) => ({
2214
+ id: row.id,
2215
+ parentSpanId: row.parentSpanId,
2216
+ traceId: row.traceId,
2217
+ name: row.name,
2218
+ scope: row.scope,
2219
+ kind: row.kind,
2220
+ status: storage.safelyParseJSON(row.status),
2221
+ events: storage.safelyParseJSON(row.events),
2222
+ links: storage.safelyParseJSON(row.links),
2223
+ attributes: storage.safelyParseJSON(row.attributes),
2224
+ startTime: row.startTime,
2225
+ endTime: row.endTime,
2226
+ other: storage.safelyParseJSON(row.other),
2227
+ createdAt: row.createdAt
2228
+ }));
2229
+ return {
2230
+ traces,
2231
+ total,
2232
+ page,
2233
+ perPage,
2234
+ hasMore: currentOffset + traces.length < total
2235
+ };
2236
+ } catch (error$1) {
2237
+ if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
2238
+ return {
2239
+ traces: [],
2240
+ total: 0,
2241
+ page,
2242
+ perPage,
2243
+ hasMore: false
2244
+ };
2245
+ }
2246
+ throw new error.MastraError(
2247
+ {
2248
+ id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
2249
+ domain: error.ErrorDomain.STORAGE,
2250
+ category: error.ErrorCategory.THIRD_PARTY,
2251
+ details: {
2252
+ name: name ?? null,
2253
+ scope: scope ?? null,
2254
+ page,
2255
+ perPage,
2256
+ attributes: attributes ? JSON.stringify(attributes) : null,
2257
+ filters: filters ? JSON.stringify(filters) : null,
2258
+ dateRange: dateRange ? JSON.stringify(dateRange) : null
2259
+ }
2260
+ },
2261
+ error$1
2262
+ );
2263
+ }
2264
+ }
2265
+ async getTraces({
2266
+ name,
2267
+ scope,
2268
+ page,
2269
+ perPage,
2270
+ attributes,
2271
+ filters,
2272
+ fromDate,
2273
+ toDate
2274
+ }) {
2275
+ const limit = perPage;
2276
+ const offset = page * perPage;
2277
+ const args = {};
2278
+ const conditions = [];
2279
+ if (name) {
2280
+ conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
2281
+ args.var_name = name;
2282
+ }
2283
+ if (scope) {
2284
+ conditions.push(`scope = {var_scope:String}`);
2285
+ args.var_scope = scope;
2286
+ }
2287
+ if (attributes) {
2288
+ Object.entries(attributes).forEach(([key, value]) => {
2289
+ conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
2290
+ args[`var_attr_${key}`] = value;
2291
+ });
2292
+ }
2293
+ if (filters) {
2294
+ Object.entries(filters).forEach(([key, value]) => {
2295
+ conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
2296
+ args[`var_col_${key}`] = value;
2297
+ });
2298
+ }
2299
+ if (fromDate) {
2300
+ conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
2301
+ args.var_from_date = fromDate.getTime() / 1e3;
2302
+ }
2303
+ if (toDate) {
2304
+ conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
2305
+ args.var_to_date = toDate.getTime() / 1e3;
2306
+ }
2307
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
2308
+ try {
2309
+ const result = await this.client.query({
2310
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
2311
+ query_params: args,
2312
+ clickhouse_settings: {
2313
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
2314
+ date_time_input_format: "best_effort",
2315
+ date_time_output_format: "iso",
2316
+ use_client_time_zone: 1,
2317
+ output_format_json_quote_64bit_integers: 0
2318
+ }
2319
+ });
2320
+ if (!result) {
2321
+ return [];
2322
+ }
2323
+ const resp = await result.json();
2324
+ const rows = resp.data;
2325
+ return rows.map((row) => ({
2326
+ id: row.id,
2327
+ parentSpanId: row.parentSpanId,
2328
+ traceId: row.traceId,
2329
+ name: row.name,
2330
+ scope: row.scope,
2331
+ kind: row.kind,
2332
+ status: storage.safelyParseJSON(row.status),
2333
+ events: storage.safelyParseJSON(row.events),
2334
+ links: storage.safelyParseJSON(row.links),
2335
+ attributes: storage.safelyParseJSON(row.attributes),
2336
+ startTime: row.startTime,
2337
+ endTime: row.endTime,
2338
+ other: storage.safelyParseJSON(row.other),
2339
+ createdAt: row.createdAt
2340
+ }));
935
2341
  } catch (error$1) {
2342
+ if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
2343
+ return [];
2344
+ }
936
2345
  throw new error.MastraError(
937
2346
  {
938
- id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
2347
+ id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
939
2348
  domain: error.ErrorDomain.STORAGE,
940
- category: error.ErrorCategory.THIRD_PARTY
2349
+ category: error.ErrorCategory.THIRD_PARTY,
2350
+ details: {
2351
+ name: name ?? null,
2352
+ scope: scope ?? null,
2353
+ page,
2354
+ perPage,
2355
+ attributes: attributes ? JSON.stringify(attributes) : null,
2356
+ filters: filters ? JSON.stringify(filters) : null,
2357
+ fromDate: fromDate?.toISOString() ?? null,
2358
+ toDate: toDate?.toISOString() ?? null
2359
+ }
941
2360
  },
942
2361
  error$1
943
2362
  );
944
2363
  }
945
2364
  }
2365
+ async batchTraceInsert(args) {
2366
+ await this.operations.batchInsert({ tableName: storage.TABLE_TRACES, records: args.records });
2367
+ }
2368
+ };
2369
+ var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
2370
+ client;
2371
+ operations;
2372
+ constructor({ client, operations }) {
2373
+ super();
2374
+ this.operations = operations;
2375
+ this.client = client;
2376
+ }
2377
+ updateWorkflowResults({
2378
+ // workflowName,
2379
+ // runId,
2380
+ // stepId,
2381
+ // result,
2382
+ // runtimeContext,
2383
+ }) {
2384
+ throw new Error("Method not implemented.");
2385
+ }
2386
+ updateWorkflowState({
2387
+ // workflowName,
2388
+ // runId,
2389
+ // opts,
2390
+ }) {
2391
+ throw new Error("Method not implemented.");
2392
+ }
946
2393
  async persistWorkflowSnapshot({
947
2394
  workflowName,
948
2395
  runId,
2396
+ resourceId,
949
2397
  snapshot
950
2398
  }) {
951
2399
  try {
952
- const currentSnapshot = await this.load({
2400
+ const currentSnapshot = await this.operations.load({
953
2401
  tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
954
2402
  keys: { workflow_name: workflowName, run_id: runId }
955
2403
  });
956
2404
  const now = /* @__PURE__ */ new Date();
957
2405
  const persisting = currentSnapshot ? {
958
2406
  ...currentSnapshot,
2407
+ resourceId,
959
2408
  snapshot: JSON.stringify(snapshot),
960
2409
  updatedAt: now.toISOString()
961
2410
  } : {
962
2411
  workflow_name: workflowName,
963
2412
  run_id: runId,
2413
+ resourceId,
964
2414
  snapshot: JSON.stringify(snapshot),
965
2415
  createdAt: now.toISOString(),
966
2416
  updatedAt: now.toISOString()
967
2417
  };
968
- await this.db.insert({
2418
+ await this.client.insert({
969
2419
  table: storage.TABLE_WORKFLOW_SNAPSHOT,
970
2420
  format: "JSONEachRow",
971
2421
  values: [persisting],
@@ -993,7 +2443,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
993
2443
  runId
994
2444
  }) {
995
2445
  try {
996
- const result = await this.load({
2446
+ const result = await this.operations.load({
997
2447
  tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
998
2448
  keys: {
999
2449
  workflow_name: workflowName,
@@ -1050,7 +2500,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
1050
2500
  values.var_workflow_name = workflowName;
1051
2501
  }
1052
2502
  if (resourceId) {
1053
- const hasResourceId = await this.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
2503
+ const hasResourceId = await this.operations.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
1054
2504
  if (hasResourceId) {
1055
2505
  conditions.push(`resourceId = {var_resourceId:String}`);
1056
2506
  values.var_resourceId = resourceId;
@@ -1071,7 +2521,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
1071
2521
  const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
1072
2522
  let total = 0;
1073
2523
  if (limit !== void 0 && offset !== void 0) {
1074
- const countResult = await this.db.query({
2524
+ const countResult = await this.client.query({
1075
2525
  query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
1076
2526
  query_params: values,
1077
2527
  format: "JSONEachRow"
@@ -1079,21 +2529,21 @@ var ClickhouseStore = class extends storage.MastraStorage {
1079
2529
  const countRows = await countResult.json();
1080
2530
  total = Number(countRows[0]?.count ?? 0);
1081
2531
  }
1082
- const result = await this.db.query({
2532
+ const result = await this.client.query({
1083
2533
  query: `
1084
- SELECT
1085
- workflow_name,
1086
- run_id,
1087
- snapshot,
1088
- toDateTime64(createdAt, 3) as createdAt,
1089
- toDateTime64(updatedAt, 3) as updatedAt,
1090
- resourceId
1091
- FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
1092
- ${whereClause}
1093
- ORDER BY createdAt DESC
1094
- ${limitClause}
1095
- ${offsetClause}
1096
- `,
2534
+ SELECT
2535
+ workflow_name,
2536
+ run_id,
2537
+ snapshot,
2538
+ toDateTime64(createdAt, 3) as createdAt,
2539
+ toDateTime64(updatedAt, 3) as updatedAt,
2540
+ resourceId
2541
+ FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
2542
+ ${whereClause}
2543
+ ORDER BY createdAt DESC
2544
+ ${limitClause}
2545
+ ${offsetClause}
2546
+ `,
1097
2547
  query_params: values,
1098
2548
  format: "JSONEachRow"
1099
2549
  });
@@ -1131,18 +2581,19 @@ var ClickhouseStore = class extends storage.MastraStorage {
1131
2581
  values.var_workflow_name = workflowName;
1132
2582
  }
1133
2583
  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
1134
- const result = await this.db.query({
2584
+ const result = await this.client.query({
1135
2585
  query: `
1136
- SELECT
1137
- workflow_name,
1138
- run_id,
1139
- snapshot,
1140
- toDateTime64(createdAt, 3) as createdAt,
1141
- toDateTime64(updatedAt, 3) as updatedAt,
1142
- resourceId
1143
- FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
1144
- ${whereClause}
1145
- `,
2586
+ SELECT
2587
+ workflow_name,
2588
+ run_id,
2589
+ snapshot,
2590
+ toDateTime64(createdAt, 3) as createdAt,
2591
+ toDateTime64(updatedAt, 3) as updatedAt,
2592
+ resourceId
2593
+ FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
2594
+ ${whereClause}
2595
+ ORDER BY createdAt DESC LIMIT 1
2596
+ `,
1146
2597
  query_params: values,
1147
2598
  format: "JSONEachRow"
1148
2599
  });
@@ -1163,47 +2614,276 @@ var ClickhouseStore = class extends storage.MastraStorage {
1163
2614
  );
1164
2615
  }
1165
2616
  }
1166
- async hasColumn(table, column) {
1167
- const result = await this.db.query({
1168
- query: `DESCRIBE TABLE ${table}`,
1169
- format: "JSONEachRow"
2617
+ };
2618
+
2619
+ // src/storage/index.ts
2620
+ var ClickhouseStore = class extends storage.MastraStorage {
2621
+ db;
2622
+ ttl = {};
2623
+ stores;
2624
+ constructor(config) {
2625
+ super({ name: "ClickhouseStore" });
2626
+ this.db = client.createClient({
2627
+ url: config.url,
2628
+ username: config.username,
2629
+ password: config.password,
2630
+ clickhouse_settings: {
2631
+ date_time_input_format: "best_effort",
2632
+ date_time_output_format: "iso",
2633
+ // This is crucial
2634
+ use_client_time_zone: 1,
2635
+ output_format_json_quote_64bit_integers: 0
2636
+ }
1170
2637
  });
1171
- const columns = await result.json();
1172
- return columns.some((c) => c.name === column);
2638
+ this.ttl = config.ttl;
2639
+ const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
2640
+ const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
2641
+ const scores = new ScoresStorageClickhouse({ client: this.db, operations });
2642
+ const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
2643
+ const traces = new TracesStorageClickhouse({ client: this.db, operations });
2644
+ const memory = new MemoryStorageClickhouse({ client: this.db, operations });
2645
+ this.stores = {
2646
+ operations,
2647
+ workflows,
2648
+ scores,
2649
+ legacyEvals,
2650
+ traces,
2651
+ memory
2652
+ };
1173
2653
  }
1174
- async getTracesPaginated(_args) {
1175
- throw new error.MastraError({
1176
- id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
1177
- domain: error.ErrorDomain.STORAGE,
1178
- category: error.ErrorCategory.USER,
1179
- text: "Method not implemented."
1180
- });
2654
+ get supports() {
2655
+ return {
2656
+ selectByIncludeResourceScope: true,
2657
+ resourceWorkingMemory: true,
2658
+ hasColumn: true,
2659
+ createTable: true,
2660
+ deleteMessages: false,
2661
+ getScoresBySpan: true
2662
+ };
1181
2663
  }
1182
- async getThreadsByResourceIdPaginated(_args) {
1183
- throw new error.MastraError({
1184
- id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
1185
- domain: error.ErrorDomain.STORAGE,
1186
- category: error.ErrorCategory.USER,
1187
- text: "Method not implemented."
1188
- });
2664
+ async getEvalsByAgentName(agentName, type) {
2665
+ return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
1189
2666
  }
1190
- async getMessagesPaginated(_args) {
1191
- throw new error.MastraError({
1192
- id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
1193
- domain: error.ErrorDomain.STORAGE,
1194
- category: error.ErrorCategory.USER,
1195
- text: "Method not implemented."
1196
- });
2667
+ async getEvals(options) {
2668
+ return this.stores.legacyEvals.getEvals(options);
2669
+ }
2670
+ async batchInsert({ tableName, records }) {
2671
+ await this.stores.operations.batchInsert({ tableName, records });
2672
+ }
2673
+ async optimizeTable({ tableName }) {
2674
+ try {
2675
+ await this.db.command({
2676
+ query: `OPTIMIZE TABLE ${tableName} FINAL`
2677
+ });
2678
+ } catch (error$1) {
2679
+ throw new error.MastraError(
2680
+ {
2681
+ id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
2682
+ domain: error.ErrorDomain.STORAGE,
2683
+ category: error.ErrorCategory.THIRD_PARTY,
2684
+ details: { tableName }
2685
+ },
2686
+ error$1
2687
+ );
2688
+ }
2689
+ }
2690
+ async materializeTtl({ tableName }) {
2691
+ try {
2692
+ await this.db.command({
2693
+ query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
2694
+ });
2695
+ } catch (error$1) {
2696
+ throw new error.MastraError(
2697
+ {
2698
+ id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
2699
+ domain: error.ErrorDomain.STORAGE,
2700
+ category: error.ErrorCategory.THIRD_PARTY,
2701
+ details: { tableName }
2702
+ },
2703
+ error$1
2704
+ );
2705
+ }
2706
+ }
2707
+ async createTable({
2708
+ tableName,
2709
+ schema
2710
+ }) {
2711
+ return this.stores.operations.createTable({ tableName, schema });
2712
+ }
2713
+ async dropTable({ tableName }) {
2714
+ return this.stores.operations.dropTable({ tableName });
2715
+ }
2716
+ async alterTable({
2717
+ tableName,
2718
+ schema,
2719
+ ifNotExists
2720
+ }) {
2721
+ return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
2722
+ }
2723
+ async clearTable({ tableName }) {
2724
+ return this.stores.operations.clearTable({ tableName });
2725
+ }
2726
+ async insert({ tableName, record }) {
2727
+ return this.stores.operations.insert({ tableName, record });
2728
+ }
2729
+ async load({ tableName, keys }) {
2730
+ return this.stores.operations.load({ tableName, keys });
2731
+ }
2732
+ async updateWorkflowResults({
2733
+ workflowName,
2734
+ runId,
2735
+ stepId,
2736
+ result,
2737
+ runtimeContext
2738
+ }) {
2739
+ return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
2740
+ }
2741
+ async updateWorkflowState({
2742
+ workflowName,
2743
+ runId,
2744
+ opts
2745
+ }) {
2746
+ return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
2747
+ }
2748
+ async persistWorkflowSnapshot({
2749
+ workflowName,
2750
+ runId,
2751
+ resourceId,
2752
+ snapshot
2753
+ }) {
2754
+ return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
2755
+ }
2756
+ async loadWorkflowSnapshot({
2757
+ workflowName,
2758
+ runId
2759
+ }) {
2760
+ return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
2761
+ }
2762
+ async getWorkflowRuns({
2763
+ workflowName,
2764
+ fromDate,
2765
+ toDate,
2766
+ limit,
2767
+ offset,
2768
+ resourceId
2769
+ } = {}) {
2770
+ return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
2771
+ }
2772
+ async getWorkflowRunById({
2773
+ runId,
2774
+ workflowName
2775
+ }) {
2776
+ return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
2777
+ }
2778
+ async getTraces(args) {
2779
+ return this.stores.traces.getTraces(args);
2780
+ }
2781
+ async getTracesPaginated(args) {
2782
+ return this.stores.traces.getTracesPaginated(args);
2783
+ }
2784
+ async batchTraceInsert(args) {
2785
+ return this.stores.traces.batchTraceInsert(args);
2786
+ }
2787
+ async getThreadById({ threadId }) {
2788
+ return this.stores.memory.getThreadById({ threadId });
2789
+ }
2790
+ async getThreadsByResourceId({ resourceId }) {
2791
+ return this.stores.memory.getThreadsByResourceId({ resourceId });
2792
+ }
2793
+ async saveThread({ thread }) {
2794
+ return this.stores.memory.saveThread({ thread });
2795
+ }
2796
+ async updateThread({
2797
+ id,
2798
+ title,
2799
+ metadata
2800
+ }) {
2801
+ return this.stores.memory.updateThread({ id, title, metadata });
2802
+ }
2803
+ async deleteThread({ threadId }) {
2804
+ return this.stores.memory.deleteThread({ threadId });
2805
+ }
2806
+ async getThreadsByResourceIdPaginated(args) {
2807
+ return this.stores.memory.getThreadsByResourceIdPaginated(args);
2808
+ }
2809
+ async getMessages({
2810
+ threadId,
2811
+ resourceId,
2812
+ selectBy,
2813
+ format
2814
+ }) {
2815
+ return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
2816
+ }
2817
+ async getMessagesById({
2818
+ messageIds,
2819
+ format
2820
+ }) {
2821
+ return this.stores.memory.getMessagesById({ messageIds, format });
2822
+ }
2823
+ async saveMessages(args) {
2824
+ return this.stores.memory.saveMessages(args);
2825
+ }
2826
+ async getMessagesPaginated(args) {
2827
+ return this.stores.memory.getMessagesPaginated(args);
2828
+ }
2829
+ async updateMessages(args) {
2830
+ return this.stores.memory.updateMessages(args);
2831
+ }
2832
+ async getResourceById({ resourceId }) {
2833
+ return this.stores.memory.getResourceById({ resourceId });
2834
+ }
2835
+ async saveResource({ resource }) {
2836
+ return this.stores.memory.saveResource({ resource });
2837
+ }
2838
+ async updateResource({
2839
+ resourceId,
2840
+ workingMemory,
2841
+ metadata
2842
+ }) {
2843
+ return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
2844
+ }
2845
+ async getScoreById({ id }) {
2846
+ return this.stores.scores.getScoreById({ id });
2847
+ }
2848
+ async saveScore(_score) {
2849
+ return this.stores.scores.saveScore(_score);
2850
+ }
2851
+ async getScoresByRunId({
2852
+ runId,
2853
+ pagination
2854
+ }) {
2855
+ return this.stores.scores.getScoresByRunId({ runId, pagination });
2856
+ }
2857
+ async getScoresByEntityId({
2858
+ entityId,
2859
+ entityType,
2860
+ pagination
2861
+ }) {
2862
+ return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
2863
+ }
2864
+ async getScoresByScorerId({
2865
+ scorerId,
2866
+ pagination,
2867
+ entityId,
2868
+ entityType,
2869
+ source
2870
+ }) {
2871
+ return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
2872
+ }
2873
+ async getScoresBySpan({
2874
+ traceId,
2875
+ spanId,
2876
+ pagination
2877
+ }) {
2878
+ return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
1197
2879
  }
1198
2880
  async close() {
1199
2881
  await this.db.close();
1200
2882
  }
1201
- async updateMessages(_args) {
1202
- this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
1203
- throw new Error("Method not implemented");
1204
- }
1205
2883
  };
1206
2884
 
1207
2885
  exports.COLUMN_TYPES = COLUMN_TYPES;
1208
2886
  exports.ClickhouseStore = ClickhouseStore;
1209
2887
  exports.TABLE_ENGINES = TABLE_ENGINES;
2888
+ //# sourceMappingURL=index.cjs.map
2889
+ //# sourceMappingURL=index.cjs.map