@mastra/clickhouse 0.0.0-trigger-playground-ui-package-20250506151043 → 0.0.0-update-stores-peerDeps-20250723031338
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +477 -2
- package/LICENSE.md +12 -4
- package/dist/_tsup-dts-rollup.d.cts +373 -33
- package/dist/_tsup-dts-rollup.d.ts +373 -33
- package/dist/index.cjs +2235 -440
- package/dist/index.d.cts +0 -2
- package/dist/index.d.ts +0 -2
- package/dist/index.js +2223 -426
- package/package.json +14 -10
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +14 -844
- package/src/storage/index.ts +241 -897
package/dist/index.js
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import {
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { MastraStorage, StoreOperations, TABLE_SCHEMAS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, ScoresStorage, safelyParseJSON, TABLE_SCORERS, LegacyEvalsStorage, TABLE_EVALS, TracesStorage, TABLE_TRACES, MemoryStorage, resolveMessageLimit, TABLE_MESSAGES, TABLE_THREADS, TABLE_RESOURCES } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
3
5
|
|
|
4
6
|
// src/storage/index.ts
|
|
5
|
-
function safelyParseJSON(jsonString) {
|
|
6
|
-
try {
|
|
7
|
-
return JSON.parse(jsonString);
|
|
8
|
-
} catch {
|
|
9
|
-
return {};
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
7
|
var TABLE_ENGINES = {
|
|
13
8
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
14
9
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
15
10
|
[TABLE_TRACES]: `MergeTree()`,
|
|
16
11
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
17
|
-
[TABLE_EVALS]: `MergeTree()
|
|
12
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
13
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
14
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`
|
|
18
15
|
};
|
|
19
16
|
var COLUMN_TYPES = {
|
|
20
17
|
text: "String",
|
|
@@ -22,11 +19,9 @@ var COLUMN_TYPES = {
|
|
|
22
19
|
uuid: "String",
|
|
23
20
|
jsonb: "String",
|
|
24
21
|
integer: "Int64",
|
|
22
|
+
float: "Float64",
|
|
25
23
|
bigint: "Int64"
|
|
26
24
|
};
|
|
27
|
-
function transformRows(rows) {
|
|
28
|
-
return rows.map((row) => transformRow(row));
|
|
29
|
-
}
|
|
30
25
|
function transformRow(row) {
|
|
31
26
|
if (!row) {
|
|
32
27
|
return row;
|
|
@@ -37,33 +32,63 @@ function transformRow(row) {
|
|
|
37
32
|
if (row.updatedAt) {
|
|
38
33
|
row.updatedAt = new Date(row.updatedAt);
|
|
39
34
|
}
|
|
35
|
+
if (row.content && typeof row.content === "string") {
|
|
36
|
+
row.content = safelyParseJSON(row.content);
|
|
37
|
+
}
|
|
40
38
|
return row;
|
|
41
39
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
// This is crucial
|
|
55
|
-
use_client_time_zone: 1,
|
|
56
|
-
output_format_json_quote_64bit_integers: 0
|
|
57
|
-
}
|
|
58
|
-
});
|
|
59
|
-
this.ttl = config.ttl;
|
|
40
|
+
function transformRows(rows) {
|
|
41
|
+
return rows.map((row) => transformRow(row));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
45
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
46
|
+
client;
|
|
47
|
+
operations;
|
|
48
|
+
constructor({ client, operations }) {
|
|
49
|
+
super();
|
|
50
|
+
this.client = client;
|
|
51
|
+
this.operations = operations;
|
|
60
52
|
}
|
|
61
53
|
transformEvalRow(row) {
|
|
62
54
|
row = transformRow(row);
|
|
63
|
-
|
|
64
|
-
|
|
55
|
+
let resultValue;
|
|
56
|
+
try {
|
|
57
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
58
|
+
resultValue = JSON.parse(row.result);
|
|
59
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
60
|
+
resultValue = row.result;
|
|
61
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
62
|
+
resultValue = { score: 0 };
|
|
63
|
+
} else {
|
|
64
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
console.error("Error parsing result field:", row.result, error);
|
|
68
|
+
throw new MastraError({
|
|
69
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
70
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
71
|
+
domain: ErrorDomain.STORAGE,
|
|
72
|
+
category: ErrorCategory.USER
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
let testInfoValue;
|
|
76
|
+
try {
|
|
77
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
78
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
79
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
80
|
+
testInfoValue = row.test_info;
|
|
81
|
+
}
|
|
82
|
+
} catch {
|
|
83
|
+
testInfoValue = void 0;
|
|
84
|
+
}
|
|
65
85
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
66
|
-
throw new
|
|
86
|
+
throw new MastraError({
|
|
87
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
88
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
89
|
+
domain: ErrorDomain.STORAGE,
|
|
90
|
+
category: ErrorCategory.USER
|
|
91
|
+
});
|
|
67
92
|
}
|
|
68
93
|
return {
|
|
69
94
|
input: row.input,
|
|
@@ -80,9 +105,9 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
80
105
|
}
|
|
81
106
|
async getEvalsByAgentName(agentName, type) {
|
|
82
107
|
try {
|
|
83
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
84
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
85
|
-
const result = await this.
|
|
108
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
109
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
110
|
+
const result = await this.client.query({
|
|
86
111
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
87
112
|
query_params: { var_agent_name: agentName },
|
|
88
113
|
clickhouse_settings: {
|
|
@@ -98,125 +123,1292 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
98
123
|
const rows = await result.json();
|
|
99
124
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
100
125
|
} catch (error) {
|
|
101
|
-
if (error
|
|
126
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
102
127
|
return [];
|
|
103
128
|
}
|
|
104
|
-
|
|
105
|
-
|
|
129
|
+
throw new MastraError(
|
|
130
|
+
{
|
|
131
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
132
|
+
domain: ErrorDomain.STORAGE,
|
|
133
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
134
|
+
details: { agentName, type: type ?? null }
|
|
135
|
+
},
|
|
136
|
+
error
|
|
137
|
+
);
|
|
106
138
|
}
|
|
107
139
|
}
|
|
108
|
-
async
|
|
140
|
+
async getEvals(options = {}) {
|
|
141
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
142
|
+
const fromDate = dateRange?.start;
|
|
143
|
+
const toDate = dateRange?.end;
|
|
144
|
+
const conditions = [];
|
|
145
|
+
if (agentName) {
|
|
146
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
147
|
+
}
|
|
148
|
+
if (type === "test") {
|
|
149
|
+
conditions.push(
|
|
150
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
151
|
+
);
|
|
152
|
+
} else if (type === "live") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
if (fromDate) {
|
|
158
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
159
|
+
fromDate.toISOString();
|
|
160
|
+
}
|
|
161
|
+
if (toDate) {
|
|
162
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
163
|
+
toDate.toISOString();
|
|
164
|
+
}
|
|
165
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
109
166
|
try {
|
|
110
|
-
await this.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
...
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
167
|
+
const countResult = await this.client.query({
|
|
168
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
169
|
+
query_params: {
|
|
170
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
171
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
172
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
173
|
+
},
|
|
174
|
+
clickhouse_settings: {
|
|
175
|
+
date_time_input_format: "best_effort",
|
|
176
|
+
date_time_output_format: "iso",
|
|
177
|
+
use_client_time_zone: 1,
|
|
178
|
+
output_format_json_quote_64bit_integers: 0
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
const countData = await countResult.json();
|
|
182
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
183
|
+
const currentOffset = page * perPage;
|
|
184
|
+
const hasMore = currentOffset + perPage < total;
|
|
185
|
+
if (total === 0) {
|
|
186
|
+
return {
|
|
187
|
+
evals: [],
|
|
188
|
+
total: 0,
|
|
189
|
+
page,
|
|
190
|
+
perPage,
|
|
191
|
+
hasMore: false
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
const dataResult = await this.client.query({
|
|
195
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
196
|
+
query_params: {
|
|
197
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
198
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
199
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
200
|
+
var_limit: perPage || 100,
|
|
201
|
+
var_offset: currentOffset || 0
|
|
202
|
+
},
|
|
121
203
|
clickhouse_settings: {
|
|
122
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
123
204
|
date_time_input_format: "best_effort",
|
|
205
|
+
date_time_output_format: "iso",
|
|
124
206
|
use_client_time_zone: 1,
|
|
125
207
|
output_format_json_quote_64bit_integers: 0
|
|
126
208
|
}
|
|
127
209
|
});
|
|
210
|
+
const rows = await dataResult.json();
|
|
211
|
+
return {
|
|
212
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
213
|
+
total,
|
|
214
|
+
page,
|
|
215
|
+
perPage,
|
|
216
|
+
hasMore
|
|
217
|
+
};
|
|
128
218
|
} catch (error) {
|
|
129
|
-
|
|
130
|
-
|
|
219
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
220
|
+
return {
|
|
221
|
+
evals: [],
|
|
222
|
+
total: 0,
|
|
223
|
+
page,
|
|
224
|
+
perPage,
|
|
225
|
+
hasMore: false
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
throw new MastraError(
|
|
229
|
+
{
|
|
230
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
231
|
+
domain: ErrorDomain.STORAGE,
|
|
232
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
233
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
234
|
+
},
|
|
235
|
+
error
|
|
236
|
+
);
|
|
131
237
|
}
|
|
132
238
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
239
|
+
};
|
|
240
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
241
|
+
client;
|
|
242
|
+
operations;
|
|
243
|
+
constructor({ client, operations }) {
|
|
244
|
+
super();
|
|
245
|
+
this.client = client;
|
|
246
|
+
this.operations = operations;
|
|
247
|
+
}
|
|
248
|
+
async getMessages({
|
|
249
|
+
threadId,
|
|
250
|
+
resourceId,
|
|
251
|
+
selectBy,
|
|
252
|
+
format
|
|
142
253
|
}) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
254
|
+
try {
|
|
255
|
+
const messages = [];
|
|
256
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
257
|
+
const include = selectBy?.include || [];
|
|
258
|
+
if (include.length) {
|
|
259
|
+
const unionQueries = [];
|
|
260
|
+
const params = [];
|
|
261
|
+
let paramIdx = 1;
|
|
262
|
+
for (const inc of include) {
|
|
263
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
264
|
+
const searchId = inc.threadId || threadId;
|
|
265
|
+
unionQueries.push(`
|
|
266
|
+
SELECT * FROM (
|
|
267
|
+
WITH numbered_messages AS (
|
|
268
|
+
SELECT
|
|
269
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
270
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
271
|
+
FROM "${TABLE_MESSAGES}"
|
|
272
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
273
|
+
),
|
|
274
|
+
target_positions AS (
|
|
275
|
+
SELECT row_num as target_pos
|
|
276
|
+
FROM numbered_messages
|
|
277
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
278
|
+
)
|
|
279
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
280
|
+
FROM numbered_messages m
|
|
281
|
+
CROSS JOIN target_positions t
|
|
282
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
283
|
+
) AS query_${paramIdx}
|
|
284
|
+
`);
|
|
285
|
+
params.push(
|
|
286
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
287
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
288
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
289
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
290
|
+
);
|
|
291
|
+
paramIdx++;
|
|
292
|
+
}
|
|
293
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
294
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
295
|
+
const includeResult = await this.client.query({
|
|
296
|
+
query: finalQuery,
|
|
297
|
+
query_params: mergedParams,
|
|
298
|
+
clickhouse_settings: {
|
|
299
|
+
date_time_input_format: "best_effort",
|
|
300
|
+
date_time_output_format: "iso",
|
|
301
|
+
use_client_time_zone: 1,
|
|
302
|
+
output_format_json_quote_64bit_integers: 0
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
const rows2 = await includeResult.json();
|
|
306
|
+
const includedMessages = transformRows(rows2.data);
|
|
307
|
+
const seen = /* @__PURE__ */ new Set();
|
|
308
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
309
|
+
if (seen.has(message.id)) return false;
|
|
310
|
+
seen.add(message.id);
|
|
311
|
+
return true;
|
|
312
|
+
});
|
|
313
|
+
messages.push(...dedupedMessages);
|
|
314
|
+
}
|
|
315
|
+
const result = await this.client.query({
|
|
316
|
+
query: `
|
|
317
|
+
SELECT
|
|
318
|
+
id,
|
|
319
|
+
content,
|
|
320
|
+
role,
|
|
321
|
+
type,
|
|
322
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
323
|
+
thread_id AS "threadId"
|
|
324
|
+
FROM "${TABLE_MESSAGES}"
|
|
325
|
+
WHERE thread_id = {threadId:String}
|
|
326
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
327
|
+
ORDER BY "createdAt" DESC
|
|
328
|
+
LIMIT {limit:Int64}
|
|
329
|
+
`,
|
|
330
|
+
query_params: {
|
|
331
|
+
threadId,
|
|
332
|
+
exclude: messages.map((m) => m.id),
|
|
333
|
+
limit
|
|
334
|
+
},
|
|
335
|
+
clickhouse_settings: {
|
|
336
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
337
|
+
date_time_input_format: "best_effort",
|
|
338
|
+
date_time_output_format: "iso",
|
|
339
|
+
use_client_time_zone: 1,
|
|
340
|
+
output_format_json_quote_64bit_integers: 0
|
|
341
|
+
}
|
|
342
|
+
});
|
|
343
|
+
const rows = await result.json();
|
|
344
|
+
messages.push(...transformRows(rows.data));
|
|
345
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
346
|
+
messages.forEach((message) => {
|
|
347
|
+
if (typeof message.content === "string") {
|
|
348
|
+
try {
|
|
349
|
+
message.content = JSON.parse(message.content);
|
|
350
|
+
} catch {
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
355
|
+
if (format === `v2`) return list.get.all.v2();
|
|
356
|
+
return list.get.all.v1();
|
|
357
|
+
} catch (error) {
|
|
358
|
+
throw new MastraError(
|
|
359
|
+
{
|
|
360
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
361
|
+
domain: ErrorDomain.STORAGE,
|
|
362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
363
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
364
|
+
},
|
|
365
|
+
error
|
|
366
|
+
);
|
|
150
367
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
368
|
+
}
|
|
369
|
+
async saveMessages(args) {
|
|
370
|
+
const { messages, format = "v1" } = args;
|
|
371
|
+
if (messages.length === 0) return messages;
|
|
372
|
+
for (const message of messages) {
|
|
373
|
+
const resourceId = message.resourceId;
|
|
374
|
+
if (!resourceId) {
|
|
375
|
+
throw new Error("Resource ID is required");
|
|
376
|
+
}
|
|
377
|
+
if (!message.threadId) {
|
|
378
|
+
throw new Error("Thread ID is required");
|
|
379
|
+
}
|
|
380
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
381
|
+
if (!thread) {
|
|
382
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
383
|
+
}
|
|
154
384
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
385
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
386
|
+
await Promise.all(
|
|
387
|
+
messages.map(async (m) => {
|
|
388
|
+
const resourceId = m.resourceId;
|
|
389
|
+
if (!resourceId) {
|
|
390
|
+
throw new Error("Resource ID is required");
|
|
391
|
+
}
|
|
392
|
+
if (!m.threadId) {
|
|
393
|
+
throw new Error("Thread ID is required");
|
|
394
|
+
}
|
|
395
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
396
|
+
if (!thread) {
|
|
397
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
398
|
+
}
|
|
399
|
+
threadIdSet.set(m.threadId, thread);
|
|
400
|
+
})
|
|
401
|
+
);
|
|
402
|
+
try {
|
|
403
|
+
const existingResult = await this.client.query({
|
|
404
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
405
|
+
query_params: {
|
|
406
|
+
ids: messages.map((m) => m.id)
|
|
407
|
+
},
|
|
408
|
+
clickhouse_settings: {
|
|
409
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
410
|
+
date_time_input_format: "best_effort",
|
|
411
|
+
date_time_output_format: "iso",
|
|
412
|
+
use_client_time_zone: 1,
|
|
413
|
+
output_format_json_quote_64bit_integers: 0
|
|
414
|
+
},
|
|
415
|
+
format: "JSONEachRow"
|
|
159
416
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
);
|
|
166
|
-
|
|
417
|
+
const existingRows = await existingResult.json();
|
|
418
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
419
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
420
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
421
|
+
const toMove = messages.filter((m) => {
|
|
422
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
423
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
424
|
+
});
|
|
425
|
+
const deletePromises = toMove.map((message) => {
|
|
426
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
427
|
+
if (!existingRow) return Promise.resolve();
|
|
428
|
+
return this.client.command({
|
|
429
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
430
|
+
query_params: {
|
|
431
|
+
var_id: message.id,
|
|
432
|
+
var_old_thread_id: existingRow.thread_id
|
|
433
|
+
},
|
|
434
|
+
clickhouse_settings: {
|
|
435
|
+
date_time_input_format: "best_effort",
|
|
436
|
+
use_client_time_zone: 1,
|
|
437
|
+
output_format_json_quote_64bit_integers: 0
|
|
438
|
+
}
|
|
439
|
+
});
|
|
167
440
|
});
|
|
441
|
+
const updatePromises = toUpdate.map(
|
|
442
|
+
(message) => this.client.command({
|
|
443
|
+
query: `
|
|
444
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
445
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
446
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
447
|
+
`,
|
|
448
|
+
query_params: {
|
|
449
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
450
|
+
var_role: message.role,
|
|
451
|
+
var_type: message.type || "v2",
|
|
452
|
+
var_resourceId: message.resourceId,
|
|
453
|
+
var_id: message.id,
|
|
454
|
+
var_thread_id: message.threadId
|
|
455
|
+
},
|
|
456
|
+
clickhouse_settings: {
|
|
457
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
458
|
+
date_time_input_format: "best_effort",
|
|
459
|
+
use_client_time_zone: 1,
|
|
460
|
+
output_format_json_quote_64bit_integers: 0
|
|
461
|
+
}
|
|
462
|
+
})
|
|
463
|
+
);
|
|
464
|
+
await Promise.all([
|
|
465
|
+
// Insert new messages (including moved messages)
|
|
466
|
+
this.client.insert({
|
|
467
|
+
table: TABLE_MESSAGES,
|
|
468
|
+
format: "JSONEachRow",
|
|
469
|
+
values: toInsert.map((message) => ({
|
|
470
|
+
id: message.id,
|
|
471
|
+
thread_id: message.threadId,
|
|
472
|
+
resourceId: message.resourceId,
|
|
473
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
474
|
+
createdAt: message.createdAt.toISOString(),
|
|
475
|
+
role: message.role,
|
|
476
|
+
type: message.type || "v2"
|
|
477
|
+
})),
|
|
478
|
+
clickhouse_settings: {
|
|
479
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
480
|
+
date_time_input_format: "best_effort",
|
|
481
|
+
use_client_time_zone: 1,
|
|
482
|
+
output_format_json_quote_64bit_integers: 0
|
|
483
|
+
}
|
|
484
|
+
}),
|
|
485
|
+
...updatePromises,
|
|
486
|
+
...deletePromises,
|
|
487
|
+
// Update thread's updatedAt timestamp
|
|
488
|
+
this.client.insert({
|
|
489
|
+
table: TABLE_THREADS,
|
|
490
|
+
format: "JSONEachRow",
|
|
491
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
492
|
+
id: thread.id,
|
|
493
|
+
resourceId: thread.resourceId,
|
|
494
|
+
title: thread.title,
|
|
495
|
+
metadata: thread.metadata,
|
|
496
|
+
createdAt: thread.createdAt,
|
|
497
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
498
|
+
})),
|
|
499
|
+
clickhouse_settings: {
|
|
500
|
+
date_time_input_format: "best_effort",
|
|
501
|
+
use_client_time_zone: 1,
|
|
502
|
+
output_format_json_quote_64bit_integers: 0
|
|
503
|
+
}
|
|
504
|
+
})
|
|
505
|
+
]);
|
|
506
|
+
const list = new MessageList().add(messages, "memory");
|
|
507
|
+
if (format === `v2`) return list.get.all.v2();
|
|
508
|
+
return list.get.all.v1();
|
|
509
|
+
} catch (error) {
|
|
510
|
+
throw new MastraError(
|
|
511
|
+
{
|
|
512
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
513
|
+
domain: ErrorDomain.STORAGE,
|
|
514
|
+
category: ErrorCategory.THIRD_PARTY
|
|
515
|
+
},
|
|
516
|
+
error
|
|
517
|
+
);
|
|
168
518
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
519
|
+
}
|
|
520
|
+
async getThreadById({ threadId }) {
|
|
521
|
+
try {
|
|
522
|
+
const result = await this.client.query({
|
|
523
|
+
query: `SELECT
|
|
524
|
+
id,
|
|
525
|
+
"resourceId",
|
|
526
|
+
title,
|
|
527
|
+
metadata,
|
|
528
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
529
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
530
|
+
FROM "${TABLE_THREADS}"
|
|
531
|
+
FINAL
|
|
532
|
+
WHERE id = {var_id:String}`,
|
|
533
|
+
query_params: { var_id: threadId },
|
|
534
|
+
clickhouse_settings: {
|
|
535
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
536
|
+
date_time_input_format: "best_effort",
|
|
537
|
+
date_time_output_format: "iso",
|
|
538
|
+
use_client_time_zone: 1,
|
|
539
|
+
output_format_json_quote_64bit_integers: 0
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
const rows = await result.json();
|
|
543
|
+
const thread = transformRow(rows.data[0]);
|
|
544
|
+
if (!thread) {
|
|
545
|
+
return null;
|
|
546
|
+
}
|
|
547
|
+
return {
|
|
548
|
+
...thread,
|
|
549
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
550
|
+
createdAt: thread.createdAt,
|
|
551
|
+
updatedAt: thread.updatedAt
|
|
552
|
+
};
|
|
553
|
+
} catch (error) {
|
|
554
|
+
throw new MastraError(
|
|
555
|
+
{
|
|
556
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
557
|
+
domain: ErrorDomain.STORAGE,
|
|
558
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
559
|
+
details: { threadId }
|
|
560
|
+
},
|
|
561
|
+
error
|
|
562
|
+
);
|
|
172
563
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
564
|
+
}
|
|
565
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
566
|
+
try {
|
|
567
|
+
const result = await this.client.query({
|
|
568
|
+
query: `SELECT
|
|
569
|
+
id,
|
|
570
|
+
"resourceId",
|
|
571
|
+
title,
|
|
572
|
+
metadata,
|
|
573
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
574
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
575
|
+
FROM "${TABLE_THREADS}"
|
|
576
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
577
|
+
query_params: { var_resourceId: resourceId },
|
|
578
|
+
clickhouse_settings: {
|
|
579
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
580
|
+
date_time_input_format: "best_effort",
|
|
581
|
+
date_time_output_format: "iso",
|
|
582
|
+
use_client_time_zone: 1,
|
|
583
|
+
output_format_json_quote_64bit_integers: 0
|
|
584
|
+
}
|
|
585
|
+
});
|
|
586
|
+
const rows = await result.json();
|
|
587
|
+
const threads = transformRows(rows.data);
|
|
588
|
+
return threads.map((thread) => ({
|
|
589
|
+
...thread,
|
|
590
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
591
|
+
createdAt: thread.createdAt,
|
|
592
|
+
updatedAt: thread.updatedAt
|
|
593
|
+
}));
|
|
594
|
+
} catch (error) {
|
|
595
|
+
throw new MastraError(
|
|
596
|
+
{
|
|
597
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
598
|
+
domain: ErrorDomain.STORAGE,
|
|
599
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
600
|
+
details: { resourceId }
|
|
601
|
+
},
|
|
602
|
+
error
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
async saveThread({ thread }) {
|
|
607
|
+
try {
|
|
608
|
+
await this.client.insert({
|
|
609
|
+
table: TABLE_THREADS,
|
|
610
|
+
values: [
|
|
611
|
+
{
|
|
612
|
+
...thread,
|
|
613
|
+
createdAt: thread.createdAt.toISOString(),
|
|
614
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
615
|
+
}
|
|
616
|
+
],
|
|
617
|
+
format: "JSONEachRow",
|
|
618
|
+
clickhouse_settings: {
|
|
619
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
620
|
+
date_time_input_format: "best_effort",
|
|
621
|
+
use_client_time_zone: 1,
|
|
622
|
+
output_format_json_quote_64bit_integers: 0
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
return thread;
|
|
626
|
+
} catch (error) {
|
|
627
|
+
throw new MastraError(
|
|
628
|
+
{
|
|
629
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
630
|
+
domain: ErrorDomain.STORAGE,
|
|
631
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
632
|
+
details: { threadId: thread.id }
|
|
633
|
+
},
|
|
634
|
+
error
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
async updateThread({
|
|
639
|
+
id,
|
|
640
|
+
title,
|
|
641
|
+
metadata
|
|
642
|
+
}) {
|
|
643
|
+
try {
|
|
644
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
645
|
+
if (!existingThread) {
|
|
646
|
+
throw new Error(`Thread ${id} not found`);
|
|
647
|
+
}
|
|
648
|
+
const mergedMetadata = {
|
|
649
|
+
...existingThread.metadata,
|
|
650
|
+
...metadata
|
|
651
|
+
};
|
|
652
|
+
const updatedThread = {
|
|
653
|
+
...existingThread,
|
|
654
|
+
title,
|
|
655
|
+
metadata: mergedMetadata,
|
|
656
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
657
|
+
};
|
|
658
|
+
await this.client.insert({
|
|
659
|
+
table: TABLE_THREADS,
|
|
660
|
+
format: "JSONEachRow",
|
|
661
|
+
values: [
|
|
662
|
+
{
|
|
663
|
+
id: updatedThread.id,
|
|
664
|
+
resourceId: updatedThread.resourceId,
|
|
665
|
+
title: updatedThread.title,
|
|
666
|
+
metadata: updatedThread.metadata,
|
|
667
|
+
createdAt: updatedThread.createdAt,
|
|
668
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
669
|
+
}
|
|
670
|
+
],
|
|
671
|
+
clickhouse_settings: {
|
|
672
|
+
date_time_input_format: "best_effort",
|
|
673
|
+
use_client_time_zone: 1,
|
|
674
|
+
output_format_json_quote_64bit_integers: 0
|
|
675
|
+
}
|
|
676
|
+
});
|
|
677
|
+
return updatedThread;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
throw new MastraError(
|
|
680
|
+
{
|
|
681
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
682
|
+
domain: ErrorDomain.STORAGE,
|
|
683
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
684
|
+
details: { threadId: id, title }
|
|
685
|
+
},
|
|
686
|
+
error
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
async deleteThread({ threadId }) {
|
|
691
|
+
try {
|
|
692
|
+
await this.client.command({
|
|
693
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
694
|
+
query_params: { var_thread_id: threadId },
|
|
695
|
+
clickhouse_settings: {
|
|
696
|
+
output_format_json_quote_64bit_integers: 0
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
await this.client.command({
|
|
700
|
+
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
701
|
+
query_params: { var_id: threadId },
|
|
702
|
+
clickhouse_settings: {
|
|
703
|
+
output_format_json_quote_64bit_integers: 0
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
} catch (error) {
|
|
707
|
+
throw new MastraError(
|
|
708
|
+
{
|
|
709
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
710
|
+
domain: ErrorDomain.STORAGE,
|
|
711
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
712
|
+
details: { threadId }
|
|
713
|
+
},
|
|
714
|
+
error
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
719
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
720
|
+
try {
|
|
721
|
+
const currentOffset = page * perPage;
|
|
722
|
+
const countResult = await this.client.query({
|
|
723
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
724
|
+
query_params: { resourceId },
|
|
725
|
+
clickhouse_settings: {
|
|
726
|
+
date_time_input_format: "best_effort",
|
|
727
|
+
date_time_output_format: "iso",
|
|
728
|
+
use_client_time_zone: 1,
|
|
729
|
+
output_format_json_quote_64bit_integers: 0
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
const countData = await countResult.json();
|
|
733
|
+
const total = countData.data[0].total;
|
|
734
|
+
if (total === 0) {
|
|
735
|
+
return {
|
|
736
|
+
threads: [],
|
|
737
|
+
total: 0,
|
|
738
|
+
page,
|
|
739
|
+
perPage,
|
|
740
|
+
hasMore: false
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
const dataResult = await this.client.query({
|
|
744
|
+
query: `
|
|
745
|
+
SELECT
|
|
746
|
+
id,
|
|
747
|
+
resourceId,
|
|
748
|
+
title,
|
|
749
|
+
metadata,
|
|
750
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
751
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
752
|
+
FROM ${TABLE_THREADS}
|
|
753
|
+
WHERE resourceId = {resourceId:String}
|
|
754
|
+
ORDER BY createdAt DESC
|
|
755
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
756
|
+
`,
|
|
757
|
+
query_params: {
|
|
758
|
+
resourceId,
|
|
759
|
+
limit: perPage,
|
|
760
|
+
offset: currentOffset
|
|
761
|
+
},
|
|
762
|
+
clickhouse_settings: {
|
|
763
|
+
date_time_input_format: "best_effort",
|
|
764
|
+
date_time_output_format: "iso",
|
|
765
|
+
use_client_time_zone: 1,
|
|
766
|
+
output_format_json_quote_64bit_integers: 0
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
const rows = await dataResult.json();
|
|
770
|
+
const threads = transformRows(rows.data);
|
|
771
|
+
return {
|
|
772
|
+
threads,
|
|
773
|
+
total,
|
|
774
|
+
page,
|
|
775
|
+
perPage,
|
|
776
|
+
hasMore: currentOffset + threads.length < total
|
|
777
|
+
};
|
|
778
|
+
} catch (error) {
|
|
779
|
+
throw new MastraError(
|
|
780
|
+
{
|
|
781
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
782
|
+
domain: ErrorDomain.STORAGE,
|
|
783
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
784
|
+
details: { resourceId, page }
|
|
785
|
+
},
|
|
786
|
+
error
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async getMessagesPaginated(args) {
|
|
791
|
+
try {
|
|
792
|
+
const { threadId, selectBy, format = "v1" } = args;
|
|
793
|
+
const page = selectBy?.pagination?.page || 0;
|
|
794
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
795
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
796
|
+
const offset = page * perPage;
|
|
797
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
798
|
+
const fromDate = dateRange?.start;
|
|
799
|
+
const toDate = dateRange?.end;
|
|
800
|
+
const messages = [];
|
|
801
|
+
if (selectBy?.include?.length) {
|
|
802
|
+
const include = selectBy.include;
|
|
803
|
+
const unionQueries = [];
|
|
804
|
+
const params = [];
|
|
805
|
+
let paramIdx = 1;
|
|
806
|
+
for (const inc of include) {
|
|
807
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
808
|
+
const searchId = inc.threadId || threadId;
|
|
809
|
+
unionQueries.push(`
|
|
810
|
+
SELECT * FROM (
|
|
811
|
+
WITH numbered_messages AS (
|
|
812
|
+
SELECT
|
|
813
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
814
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
815
|
+
FROM "${TABLE_MESSAGES}"
|
|
816
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
817
|
+
),
|
|
818
|
+
target_positions AS (
|
|
819
|
+
SELECT row_num as target_pos
|
|
820
|
+
FROM numbered_messages
|
|
821
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
822
|
+
)
|
|
823
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
824
|
+
FROM numbered_messages m
|
|
825
|
+
CROSS JOIN target_positions t
|
|
826
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
827
|
+
) AS query_${paramIdx}
|
|
828
|
+
`);
|
|
829
|
+
params.push(
|
|
830
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
831
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
832
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
833
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
834
|
+
);
|
|
835
|
+
paramIdx++;
|
|
836
|
+
}
|
|
837
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
838
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
839
|
+
const includeResult = await this.client.query({
|
|
840
|
+
query: finalQuery,
|
|
841
|
+
query_params: mergedParams,
|
|
842
|
+
clickhouse_settings: {
|
|
843
|
+
date_time_input_format: "best_effort",
|
|
844
|
+
date_time_output_format: "iso",
|
|
845
|
+
use_client_time_zone: 1,
|
|
846
|
+
output_format_json_quote_64bit_integers: 0
|
|
847
|
+
}
|
|
848
|
+
});
|
|
849
|
+
const rows2 = await includeResult.json();
|
|
850
|
+
const includedMessages = transformRows(rows2.data);
|
|
851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
852
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
853
|
+
if (seen.has(message.id)) return false;
|
|
854
|
+
seen.add(message.id);
|
|
855
|
+
return true;
|
|
856
|
+
});
|
|
857
|
+
messages.push(...dedupedMessages);
|
|
858
|
+
}
|
|
859
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
860
|
+
const countParams = { threadId };
|
|
861
|
+
if (fromDate) {
|
|
862
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
863
|
+
countParams.fromDate = fromDate.toISOString();
|
|
864
|
+
}
|
|
865
|
+
if (toDate) {
|
|
866
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
867
|
+
countParams.toDate = toDate.toISOString();
|
|
868
|
+
}
|
|
869
|
+
const countResult = await this.client.query({
|
|
870
|
+
query: countQuery,
|
|
871
|
+
query_params: countParams,
|
|
872
|
+
clickhouse_settings: {
|
|
873
|
+
date_time_input_format: "best_effort",
|
|
874
|
+
date_time_output_format: "iso",
|
|
875
|
+
use_client_time_zone: 1,
|
|
876
|
+
output_format_json_quote_64bit_integers: 0
|
|
877
|
+
}
|
|
878
|
+
});
|
|
879
|
+
const countData = await countResult.json();
|
|
880
|
+
const total = countData.data[0].total;
|
|
881
|
+
if (total === 0 && messages.length === 0) {
|
|
882
|
+
return {
|
|
883
|
+
messages: [],
|
|
884
|
+
total: 0,
|
|
885
|
+
page,
|
|
886
|
+
perPage,
|
|
887
|
+
hasMore: false
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const excludeIds = messages.map((m) => m.id);
|
|
891
|
+
let dataQuery = `
|
|
892
|
+
SELECT
|
|
893
|
+
id,
|
|
894
|
+
content,
|
|
895
|
+
role,
|
|
896
|
+
type,
|
|
897
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
898
|
+
thread_id AS "threadId",
|
|
899
|
+
resourceId
|
|
900
|
+
FROM ${TABLE_MESSAGES}
|
|
901
|
+
WHERE thread_id = {threadId:String}
|
|
902
|
+
`;
|
|
903
|
+
const dataParams = { threadId };
|
|
904
|
+
if (fromDate) {
|
|
905
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
906
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
907
|
+
}
|
|
908
|
+
if (toDate) {
|
|
909
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
910
|
+
dataParams.toDate = toDate.toISOString();
|
|
911
|
+
}
|
|
912
|
+
if (excludeIds.length > 0) {
|
|
913
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
914
|
+
dataParams.excludeIds = excludeIds;
|
|
915
|
+
}
|
|
916
|
+
if (selectBy?.last) {
|
|
917
|
+
dataQuery += `
|
|
918
|
+
ORDER BY createdAt DESC
|
|
919
|
+
LIMIT {limit:Int64}
|
|
920
|
+
`;
|
|
921
|
+
dataParams.limit = perPage;
|
|
922
|
+
} else {
|
|
923
|
+
dataQuery += `
|
|
924
|
+
ORDER BY createdAt ASC
|
|
925
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
926
|
+
`;
|
|
927
|
+
dataParams.limit = perPage;
|
|
928
|
+
dataParams.offset = offset;
|
|
929
|
+
}
|
|
930
|
+
const result = await this.client.query({
|
|
931
|
+
query: dataQuery,
|
|
932
|
+
query_params: dataParams,
|
|
933
|
+
clickhouse_settings: {
|
|
934
|
+
date_time_input_format: "best_effort",
|
|
935
|
+
date_time_output_format: "iso",
|
|
936
|
+
use_client_time_zone: 1,
|
|
937
|
+
output_format_json_quote_64bit_integers: 0
|
|
938
|
+
}
|
|
939
|
+
});
|
|
940
|
+
const rows = await result.json();
|
|
941
|
+
const paginatedMessages = transformRows(rows.data);
|
|
942
|
+
messages.push(...paginatedMessages);
|
|
943
|
+
if (selectBy?.last) {
|
|
944
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
945
|
+
}
|
|
946
|
+
return {
|
|
947
|
+
messages: format === "v2" ? messages : messages,
|
|
948
|
+
total,
|
|
949
|
+
page,
|
|
950
|
+
perPage,
|
|
951
|
+
hasMore: offset + perPage < total
|
|
952
|
+
};
|
|
953
|
+
} catch (error) {
|
|
954
|
+
throw new MastraError(
|
|
955
|
+
{
|
|
956
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
957
|
+
domain: ErrorDomain.STORAGE,
|
|
958
|
+
category: ErrorCategory.THIRD_PARTY
|
|
959
|
+
},
|
|
960
|
+
error
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
async updateMessages(args) {
|
|
965
|
+
const { messages } = args;
|
|
966
|
+
if (messages.length === 0) {
|
|
967
|
+
return [];
|
|
968
|
+
}
|
|
969
|
+
try {
|
|
970
|
+
const messageIds = messages.map((m) => m.id);
|
|
971
|
+
const existingResult = await this.client.query({
|
|
972
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
973
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
974
|
+
clickhouse_settings: {
|
|
975
|
+
date_time_input_format: "best_effort",
|
|
976
|
+
date_time_output_format: "iso",
|
|
977
|
+
use_client_time_zone: 1,
|
|
978
|
+
output_format_json_quote_64bit_integers: 0
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
const existingRows = await existingResult.json();
|
|
982
|
+
const existingMessages = transformRows(existingRows.data);
|
|
983
|
+
if (existingMessages.length === 0) {
|
|
984
|
+
return [];
|
|
985
|
+
}
|
|
986
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
987
|
+
if (typeof msg.content === "string") {
|
|
988
|
+
try {
|
|
989
|
+
msg.content = JSON.parse(msg.content);
|
|
990
|
+
} catch {
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return msg;
|
|
994
|
+
});
|
|
995
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
996
|
+
const updatePromises = [];
|
|
997
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
998
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
999
|
+
if (!updatePayload) continue;
|
|
1000
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1001
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1002
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1003
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1004
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1005
|
+
}
|
|
1006
|
+
const setClauses = [];
|
|
1007
|
+
const values = {};
|
|
1008
|
+
let paramIdx = 1;
|
|
1009
|
+
let newContent = null;
|
|
1010
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1011
|
+
if (updatableFields.content) {
|
|
1012
|
+
const existingContent = existingMessage.content || {};
|
|
1013
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1014
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1015
|
+
newContent = {
|
|
1016
|
+
...existingContent,
|
|
1017
|
+
...updatableFields.content,
|
|
1018
|
+
// Deep merge metadata
|
|
1019
|
+
metadata: {
|
|
1020
|
+
...existingMetadata,
|
|
1021
|
+
...updateMetadata
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1025
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1026
|
+
paramIdx++;
|
|
1027
|
+
delete updatableFields.content;
|
|
1028
|
+
}
|
|
1029
|
+
for (const key in updatableFields) {
|
|
1030
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1031
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1032
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1033
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1034
|
+
paramIdx++;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
if (setClauses.length > 0) {
|
|
1038
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1039
|
+
const updateQuery = `
|
|
1040
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1041
|
+
UPDATE ${setClauses.join(", ")}
|
|
1042
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1043
|
+
`;
|
|
1044
|
+
console.log("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1045
|
+
updatePromises.push(
|
|
1046
|
+
this.client.command({
|
|
1047
|
+
query: updateQuery,
|
|
1048
|
+
query_params: values,
|
|
1049
|
+
clickhouse_settings: {
|
|
1050
|
+
date_time_input_format: "best_effort",
|
|
1051
|
+
use_client_time_zone: 1,
|
|
1052
|
+
output_format_json_quote_64bit_integers: 0
|
|
1053
|
+
}
|
|
1054
|
+
})
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
if (updatePromises.length > 0) {
|
|
1059
|
+
await Promise.all(updatePromises);
|
|
1060
|
+
}
|
|
1061
|
+
await this.client.command({
|
|
1062
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1063
|
+
clickhouse_settings: {
|
|
1064
|
+
date_time_input_format: "best_effort",
|
|
1065
|
+
use_client_time_zone: 1,
|
|
1066
|
+
output_format_json_quote_64bit_integers: 0
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
1069
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1070
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1071
|
+
if (!updatePayload) continue;
|
|
1072
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1073
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1074
|
+
const verifyResult = await this.client.query({
|
|
1075
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1076
|
+
query_params: { messageId: id },
|
|
1077
|
+
clickhouse_settings: {
|
|
1078
|
+
date_time_input_format: "best_effort",
|
|
1079
|
+
date_time_output_format: "iso",
|
|
1080
|
+
use_client_time_zone: 1,
|
|
1081
|
+
output_format_json_quote_64bit_integers: 0
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
const verifyRows = await verifyResult.json();
|
|
1085
|
+
if (verifyRows.data.length > 0) {
|
|
1086
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1087
|
+
if (updatedMessage) {
|
|
1088
|
+
let needsRetry = false;
|
|
1089
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1090
|
+
if (key === "content") {
|
|
1091
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1092
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1093
|
+
if (actualContent !== expectedContent) {
|
|
1094
|
+
needsRetry = true;
|
|
1095
|
+
break;
|
|
1096
|
+
}
|
|
1097
|
+
} else if (updatedMessage[key] !== value) {
|
|
1098
|
+
needsRetry = true;
|
|
1099
|
+
break;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
if (needsRetry) {
|
|
1103
|
+
console.log("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1104
|
+
await this.client.command({
|
|
1105
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1106
|
+
query_params: { messageId: id },
|
|
1107
|
+
clickhouse_settings: {
|
|
1108
|
+
date_time_input_format: "best_effort",
|
|
1109
|
+
use_client_time_zone: 1,
|
|
1110
|
+
output_format_json_quote_64bit_integers: 0
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
let updatedContent = existingMessage.content || {};
|
|
1114
|
+
if (fieldsToUpdate.content) {
|
|
1115
|
+
const existingContent = existingMessage.content || {};
|
|
1116
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1117
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1118
|
+
updatedContent = {
|
|
1119
|
+
...existingContent,
|
|
1120
|
+
...fieldsToUpdate.content,
|
|
1121
|
+
metadata: {
|
|
1122
|
+
...existingMetadata,
|
|
1123
|
+
...updateMetadata
|
|
1124
|
+
}
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
const updatedMessageData = {
|
|
1128
|
+
...existingMessage,
|
|
1129
|
+
...fieldsToUpdate,
|
|
1130
|
+
content: updatedContent
|
|
1131
|
+
};
|
|
1132
|
+
await this.client.insert({
|
|
1133
|
+
table: TABLE_MESSAGES,
|
|
1134
|
+
format: "JSONEachRow",
|
|
1135
|
+
values: [
|
|
1136
|
+
{
|
|
1137
|
+
id: updatedMessageData.id,
|
|
1138
|
+
thread_id: updatedMessageData.threadId,
|
|
1139
|
+
resourceId: updatedMessageData.resourceId,
|
|
1140
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1141
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1142
|
+
role: updatedMessageData.role,
|
|
1143
|
+
type: updatedMessageData.type || "v2"
|
|
1144
|
+
}
|
|
1145
|
+
],
|
|
1146
|
+
clickhouse_settings: {
|
|
1147
|
+
date_time_input_format: "best_effort",
|
|
1148
|
+
use_client_time_zone: 1,
|
|
1149
|
+
output_format_json_quote_64bit_integers: 0
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1157
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1158
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1159
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1160
|
+
const threadResult = await this.client.query({
|
|
1161
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1162
|
+
query_params: { threadId },
|
|
1163
|
+
clickhouse_settings: {
|
|
1164
|
+
date_time_input_format: "best_effort",
|
|
1165
|
+
date_time_output_format: "iso",
|
|
1166
|
+
use_client_time_zone: 1,
|
|
1167
|
+
output_format_json_quote_64bit_integers: 0
|
|
1168
|
+
}
|
|
1169
|
+
});
|
|
1170
|
+
const threadRows = await threadResult.json();
|
|
1171
|
+
if (threadRows.data.length > 0) {
|
|
1172
|
+
const existingThread = threadRows.data[0];
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1175
|
+
query_params: { threadId },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
await this.client.insert({
|
|
1183
|
+
table: TABLE_THREADS,
|
|
1184
|
+
format: "JSONEachRow",
|
|
1185
|
+
values: [
|
|
1186
|
+
{
|
|
1187
|
+
id: existingThread.id,
|
|
1188
|
+
resourceId: existingThread.resourceId,
|
|
1189
|
+
title: existingThread.title,
|
|
1190
|
+
metadata: existingThread.metadata,
|
|
1191
|
+
createdAt: existingThread.createdAt,
|
|
1192
|
+
updatedAt: now
|
|
1193
|
+
}
|
|
1194
|
+
],
|
|
1195
|
+
clickhouse_settings: {
|
|
1196
|
+
date_time_input_format: "best_effort",
|
|
1197
|
+
use_client_time_zone: 1,
|
|
1198
|
+
output_format_json_quote_64bit_integers: 0
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
await Promise.all(threadUpdatePromises);
|
|
1204
|
+
}
|
|
1205
|
+
const updatedMessages = [];
|
|
1206
|
+
for (const messageId of messageIds) {
|
|
1207
|
+
const updatedResult = await this.client.query({
|
|
1208
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1209
|
+
query_params: { messageId },
|
|
1210
|
+
clickhouse_settings: {
|
|
1211
|
+
date_time_input_format: "best_effort",
|
|
1212
|
+
date_time_output_format: "iso",
|
|
1213
|
+
use_client_time_zone: 1,
|
|
1214
|
+
output_format_json_quote_64bit_integers: 0
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
const updatedRows = await updatedResult.json();
|
|
1218
|
+
if (updatedRows.data.length > 0) {
|
|
1219
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1220
|
+
if (message) {
|
|
1221
|
+
updatedMessages.push(message);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return updatedMessages.map((message) => {
|
|
1226
|
+
if (typeof message.content === "string") {
|
|
1227
|
+
try {
|
|
1228
|
+
message.content = JSON.parse(message.content);
|
|
1229
|
+
} catch {
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
return message;
|
|
1233
|
+
});
|
|
1234
|
+
} catch (error) {
|
|
1235
|
+
throw new MastraError(
|
|
1236
|
+
{
|
|
1237
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1238
|
+
domain: ErrorDomain.STORAGE,
|
|
1239
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1240
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1241
|
+
},
|
|
1242
|
+
error
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async getResourceById({ resourceId }) {
|
|
1247
|
+
try {
|
|
1248
|
+
const result = await this.client.query({
|
|
1249
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1250
|
+
query_params: { resourceId },
|
|
1251
|
+
clickhouse_settings: {
|
|
1252
|
+
date_time_input_format: "best_effort",
|
|
1253
|
+
date_time_output_format: "iso",
|
|
1254
|
+
use_client_time_zone: 1,
|
|
1255
|
+
output_format_json_quote_64bit_integers: 0
|
|
1256
|
+
}
|
|
1257
|
+
});
|
|
1258
|
+
const rows = await result.json();
|
|
1259
|
+
if (rows.data.length === 0) {
|
|
1260
|
+
return null;
|
|
1261
|
+
}
|
|
1262
|
+
const resource = rows.data[0];
|
|
1263
|
+
return {
|
|
1264
|
+
id: resource.id,
|
|
1265
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1266
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1267
|
+
createdAt: new Date(resource.createdAt),
|
|
1268
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1269
|
+
};
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
throw new MastraError(
|
|
1272
|
+
{
|
|
1273
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1274
|
+
domain: ErrorDomain.STORAGE,
|
|
1275
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1276
|
+
details: { resourceId }
|
|
1277
|
+
},
|
|
1278
|
+
error
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
async saveResource({ resource }) {
|
|
1283
|
+
try {
|
|
1284
|
+
await this.client.insert({
|
|
1285
|
+
table: TABLE_RESOURCES,
|
|
1286
|
+
format: "JSONEachRow",
|
|
1287
|
+
values: [
|
|
1288
|
+
{
|
|
1289
|
+
id: resource.id,
|
|
1290
|
+
workingMemory: resource.workingMemory,
|
|
1291
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1292
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1293
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1294
|
+
}
|
|
1295
|
+
],
|
|
1296
|
+
clickhouse_settings: {
|
|
1297
|
+
date_time_input_format: "best_effort",
|
|
1298
|
+
use_client_time_zone: 1,
|
|
1299
|
+
output_format_json_quote_64bit_integers: 0
|
|
1300
|
+
}
|
|
1301
|
+
});
|
|
1302
|
+
return resource;
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { resourceId: resource.id }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
176
1313
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
1314
|
+
}
|
|
1315
|
+
async updateResource({
|
|
1316
|
+
resourceId,
|
|
1317
|
+
workingMemory,
|
|
1318
|
+
metadata
|
|
1319
|
+
}) {
|
|
1320
|
+
try {
|
|
1321
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1322
|
+
if (!existingResource) {
|
|
1323
|
+
const newResource = {
|
|
1324
|
+
id: resourceId,
|
|
1325
|
+
workingMemory,
|
|
1326
|
+
metadata: metadata || {},
|
|
1327
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1328
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1329
|
+
};
|
|
1330
|
+
return this.saveResource({ resource: newResource });
|
|
187
1331
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
1332
|
+
const updatedResource = {
|
|
1333
|
+
...existingResource,
|
|
1334
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1335
|
+
metadata: {
|
|
1336
|
+
...existingResource.metadata,
|
|
1337
|
+
...metadata
|
|
1338
|
+
},
|
|
1339
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1340
|
+
};
|
|
1341
|
+
const updateQuery = `
|
|
1342
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1343
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1344
|
+
WHERE id = {resourceId:String}
|
|
1345
|
+
`;
|
|
1346
|
+
await this.client.command({
|
|
1347
|
+
query: updateQuery,
|
|
1348
|
+
query_params: {
|
|
1349
|
+
workingMemory: updatedResource.workingMemory,
|
|
1350
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1351
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1352
|
+
resourceId
|
|
1353
|
+
},
|
|
1354
|
+
clickhouse_settings: {
|
|
1355
|
+
date_time_input_format: "best_effort",
|
|
1356
|
+
use_client_time_zone: 1,
|
|
1357
|
+
output_format_json_quote_64bit_integers: 0
|
|
1358
|
+
}
|
|
1359
|
+
});
|
|
1360
|
+
await this.client.command({
|
|
1361
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1362
|
+
clickhouse_settings: {
|
|
1363
|
+
date_time_input_format: "best_effort",
|
|
1364
|
+
use_client_time_zone: 1,
|
|
1365
|
+
output_format_json_quote_64bit_integers: 0
|
|
1366
|
+
}
|
|
1367
|
+
});
|
|
1368
|
+
return updatedResource;
|
|
1369
|
+
} catch (error) {
|
|
1370
|
+
throw new MastraError(
|
|
1371
|
+
{
|
|
1372
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1373
|
+
domain: ErrorDomain.STORAGE,
|
|
1374
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1375
|
+
details: { resourceId }
|
|
1376
|
+
},
|
|
1377
|
+
error
|
|
1378
|
+
);
|
|
191
1379
|
}
|
|
192
|
-
const resp = await result.json();
|
|
193
|
-
const rows = resp.data;
|
|
194
|
-
return rows.map((row) => ({
|
|
195
|
-
id: row.id,
|
|
196
|
-
parentSpanId: row.parentSpanId,
|
|
197
|
-
traceId: row.traceId,
|
|
198
|
-
name: row.name,
|
|
199
|
-
scope: row.scope,
|
|
200
|
-
kind: row.kind,
|
|
201
|
-
status: safelyParseJSON(row.status),
|
|
202
|
-
events: safelyParseJSON(row.events),
|
|
203
|
-
links: safelyParseJSON(row.links),
|
|
204
|
-
attributes: safelyParseJSON(row.attributes),
|
|
205
|
-
startTime: row.startTime,
|
|
206
|
-
endTime: row.endTime,
|
|
207
|
-
other: safelyParseJSON(row.other),
|
|
208
|
-
createdAt: row.createdAt
|
|
209
|
-
}));
|
|
210
1380
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
1381
|
+
};
|
|
1382
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1383
|
+
ttl;
|
|
1384
|
+
client;
|
|
1385
|
+
constructor({ client, ttl }) {
|
|
1386
|
+
super();
|
|
1387
|
+
this.ttl = ttl;
|
|
1388
|
+
this.client = client;
|
|
215
1389
|
}
|
|
216
|
-
async
|
|
217
|
-
await this.
|
|
218
|
-
query: `
|
|
1390
|
+
async hasColumn(table, column) {
|
|
1391
|
+
const result = await this.client.query({
|
|
1392
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1393
|
+
format: "JSONEachRow"
|
|
219
1394
|
});
|
|
1395
|
+
const columns = await result.json();
|
|
1396
|
+
return columns.some((c) => c.name === column);
|
|
1397
|
+
}
|
|
1398
|
+
getSqlType(type) {
|
|
1399
|
+
switch (type) {
|
|
1400
|
+
case "text":
|
|
1401
|
+
return "String";
|
|
1402
|
+
case "timestamp":
|
|
1403
|
+
return "DateTime64(3)";
|
|
1404
|
+
case "integer":
|
|
1405
|
+
case "bigint":
|
|
1406
|
+
return "Int64";
|
|
1407
|
+
case "jsonb":
|
|
1408
|
+
return "String";
|
|
1409
|
+
default:
|
|
1410
|
+
return super.getSqlType(type);
|
|
1411
|
+
}
|
|
220
1412
|
}
|
|
221
1413
|
async createTable({
|
|
222
1414
|
tableName,
|
|
@@ -231,27 +1423,25 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
231
1423
|
}).join(",\n");
|
|
232
1424
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
233
1425
|
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
`;
|
|
254
|
-
await this.db.query({
|
|
1426
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1427
|
+
${["id String"].concat(columns)}
|
|
1428
|
+
)
|
|
1429
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1430
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1431
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1432
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1433
|
+
SETTINGS index_granularity = 8192
|
|
1434
|
+
` : `
|
|
1435
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1436
|
+
${columns}
|
|
1437
|
+
)
|
|
1438
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1439
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1440
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1441
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1442
|
+
SETTINGS index_granularity = 8192
|
|
1443
|
+
`;
|
|
1444
|
+
await this.client.query({
|
|
255
1445
|
query: sql,
|
|
256
1446
|
clickhouse_settings: {
|
|
257
1447
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -262,13 +1452,59 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
262
1452
|
}
|
|
263
1453
|
});
|
|
264
1454
|
} catch (error) {
|
|
265
|
-
|
|
266
|
-
|
|
1455
|
+
throw new MastraError(
|
|
1456
|
+
{
|
|
1457
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1458
|
+
domain: ErrorDomain.STORAGE,
|
|
1459
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1460
|
+
details: { tableName }
|
|
1461
|
+
},
|
|
1462
|
+
error
|
|
1463
|
+
);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
async alterTable({
|
|
1467
|
+
tableName,
|
|
1468
|
+
schema,
|
|
1469
|
+
ifNotExists
|
|
1470
|
+
}) {
|
|
1471
|
+
try {
|
|
1472
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1473
|
+
const result = await this.client.query({
|
|
1474
|
+
query: describeSql
|
|
1475
|
+
});
|
|
1476
|
+
const rows = await result.json();
|
|
1477
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1478
|
+
for (const columnName of ifNotExists) {
|
|
1479
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1480
|
+
const columnDef = schema[columnName];
|
|
1481
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1482
|
+
if (columnDef.nullable !== false) {
|
|
1483
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1484
|
+
}
|
|
1485
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1486
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1487
|
+
await this.client.query({
|
|
1488
|
+
query: alterSql
|
|
1489
|
+
});
|
|
1490
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
throw new MastraError(
|
|
1495
|
+
{
|
|
1496
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1497
|
+
domain: ErrorDomain.STORAGE,
|
|
1498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1499
|
+
details: { tableName }
|
|
1500
|
+
},
|
|
1501
|
+
error
|
|
1502
|
+
);
|
|
267
1503
|
}
|
|
268
1504
|
}
|
|
269
1505
|
async clearTable({ tableName }) {
|
|
270
1506
|
try {
|
|
271
|
-
await this.
|
|
1507
|
+
await this.client.query({
|
|
272
1508
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
273
1509
|
clickhouse_settings: {
|
|
274
1510
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -279,19 +1515,33 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
279
1515
|
}
|
|
280
1516
|
});
|
|
281
1517
|
} catch (error) {
|
|
282
|
-
|
|
283
|
-
|
|
1518
|
+
throw new MastraError(
|
|
1519
|
+
{
|
|
1520
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1521
|
+
domain: ErrorDomain.STORAGE,
|
|
1522
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1523
|
+
details: { tableName }
|
|
1524
|
+
},
|
|
1525
|
+
error
|
|
1526
|
+
);
|
|
284
1527
|
}
|
|
285
1528
|
}
|
|
1529
|
+
async dropTable({ tableName }) {
|
|
1530
|
+
await this.client.query({
|
|
1531
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
286
1534
|
async insert({ tableName, record }) {
|
|
1535
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1536
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
287
1537
|
try {
|
|
288
|
-
await this.
|
|
1538
|
+
const result = await this.client.insert({
|
|
289
1539
|
table: tableName,
|
|
290
1540
|
values: [
|
|
291
1541
|
{
|
|
292
1542
|
...record,
|
|
293
|
-
createdAt
|
|
294
|
-
updatedAt
|
|
1543
|
+
createdAt,
|
|
1544
|
+
updatedAt
|
|
295
1545
|
}
|
|
296
1546
|
],
|
|
297
1547
|
format: "JSONEachRow",
|
|
@@ -302,13 +1552,55 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
302
1552
|
use_client_time_zone: 1
|
|
303
1553
|
}
|
|
304
1554
|
});
|
|
1555
|
+
console.log("INSERT RESULT", result);
|
|
305
1556
|
} catch (error) {
|
|
306
|
-
|
|
307
|
-
|
|
1557
|
+
throw new MastraError(
|
|
1558
|
+
{
|
|
1559
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1560
|
+
domain: ErrorDomain.STORAGE,
|
|
1561
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1562
|
+
details: { tableName }
|
|
1563
|
+
},
|
|
1564
|
+
error
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
async batchInsert({ tableName, records }) {
|
|
1569
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1570
|
+
...Object.fromEntries(
|
|
1571
|
+
Object.entries(record).map(([key, value]) => [
|
|
1572
|
+
key,
|
|
1573
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1574
|
+
])
|
|
1575
|
+
)
|
|
1576
|
+
}));
|
|
1577
|
+
try {
|
|
1578
|
+
await this.client.insert({
|
|
1579
|
+
table: tableName,
|
|
1580
|
+
values: recordsToBeInserted,
|
|
1581
|
+
format: "JSONEachRow",
|
|
1582
|
+
clickhouse_settings: {
|
|
1583
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1584
|
+
date_time_input_format: "best_effort",
|
|
1585
|
+
use_client_time_zone: 1,
|
|
1586
|
+
output_format_json_quote_64bit_integers: 0
|
|
1587
|
+
}
|
|
1588
|
+
});
|
|
1589
|
+
} catch (error) {
|
|
1590
|
+
throw new MastraError(
|
|
1591
|
+
{
|
|
1592
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1593
|
+
domain: ErrorDomain.STORAGE,
|
|
1594
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1595
|
+
details: { tableName }
|
|
1596
|
+
},
|
|
1597
|
+
error
|
|
1598
|
+
);
|
|
308
1599
|
}
|
|
309
1600
|
}
|
|
310
1601
|
async load({ tableName, keys }) {
|
|
311
1602
|
try {
|
|
1603
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
312
1604
|
const keyEntries = Object.entries(keys);
|
|
313
1605
|
const conditions = keyEntries.map(
|
|
314
1606
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -316,8 +1608,10 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
316
1608
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
317
1609
|
return { ...acc, [`var_${key}`]: value };
|
|
318
1610
|
}, {});
|
|
319
|
-
const
|
|
320
|
-
|
|
1611
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1612
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1613
|
+
const result = await this.client.query({
|
|
1614
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
321
1615
|
query_params: values,
|
|
322
1616
|
clickhouse_settings: {
|
|
323
1617
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -344,24 +1638,57 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
344
1638
|
const data = transformRow(rows.data[0]);
|
|
345
1639
|
return data;
|
|
346
1640
|
} catch (error) {
|
|
347
|
-
|
|
348
|
-
|
|
1641
|
+
throw new MastraError(
|
|
1642
|
+
{
|
|
1643
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1644
|
+
domain: ErrorDomain.STORAGE,
|
|
1645
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1646
|
+
details: { tableName }
|
|
1647
|
+
},
|
|
1648
|
+
error
|
|
1649
|
+
);
|
|
349
1650
|
}
|
|
350
1651
|
}
|
|
351
|
-
|
|
1652
|
+
};
|
|
1653
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1654
|
+
client;
|
|
1655
|
+
operations;
|
|
1656
|
+
constructor({ client, operations }) {
|
|
1657
|
+
super();
|
|
1658
|
+
this.client = client;
|
|
1659
|
+
this.operations = operations;
|
|
1660
|
+
}
|
|
1661
|
+
transformScoreRow(row) {
|
|
1662
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1663
|
+
const extractStepResult = safelyParseJSON(row.extractStepResult);
|
|
1664
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1665
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1666
|
+
const input = safelyParseJSON(row.input);
|
|
1667
|
+
const output = safelyParseJSON(row.output);
|
|
1668
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1669
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1670
|
+
const entity = safelyParseJSON(row.entity);
|
|
1671
|
+
return {
|
|
1672
|
+
...row,
|
|
1673
|
+
scorer,
|
|
1674
|
+
extractStepResult,
|
|
1675
|
+
analyzeStepResult,
|
|
1676
|
+
metadata,
|
|
1677
|
+
input,
|
|
1678
|
+
output,
|
|
1679
|
+
additionalContext,
|
|
1680
|
+
runtimeContext,
|
|
1681
|
+
entity,
|
|
1682
|
+
createdAt: new Date(row.createdAt),
|
|
1683
|
+
updatedAt: new Date(row.updatedAt)
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
async getScoreById({ id }) {
|
|
352
1687
|
try {
|
|
353
|
-
const result = await this.
|
|
354
|
-
query: `SELECT
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
title,
|
|
358
|
-
metadata,
|
|
359
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
360
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
361
|
-
FROM "${TABLE_THREADS}"
|
|
362
|
-
FINAL
|
|
363
|
-
WHERE id = {var_id:String}`,
|
|
364
|
-
query_params: { var_id: threadId },
|
|
1688
|
+
const result = await this.client.query({
|
|
1689
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1690
|
+
query_params: { var_id: id },
|
|
1691
|
+
format: "JSONEachRow",
|
|
365
1692
|
clickhouse_settings: {
|
|
366
1693
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
367
1694
|
date_time_input_format: "best_effort",
|
|
@@ -370,288 +1697,510 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
370
1697
|
output_format_json_quote_64bit_integers: 0
|
|
371
1698
|
}
|
|
372
1699
|
});
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
if (!thread) {
|
|
1700
|
+
const resultJson = await result.json();
|
|
1701
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
376
1702
|
return null;
|
|
377
1703
|
}
|
|
378
|
-
return
|
|
379
|
-
...thread,
|
|
380
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
381
|
-
createdAt: thread.createdAt,
|
|
382
|
-
updatedAt: thread.updatedAt
|
|
383
|
-
};
|
|
1704
|
+
return this.transformScoreRow(resultJson[0]);
|
|
384
1705
|
} catch (error) {
|
|
385
|
-
|
|
386
|
-
|
|
1706
|
+
throw new MastraError(
|
|
1707
|
+
{
|
|
1708
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1709
|
+
domain: ErrorDomain.STORAGE,
|
|
1710
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1711
|
+
details: { scoreId: id }
|
|
1712
|
+
},
|
|
1713
|
+
error
|
|
1714
|
+
);
|
|
387
1715
|
}
|
|
388
1716
|
}
|
|
389
|
-
async
|
|
1717
|
+
async saveScore(score) {
|
|
390
1718
|
try {
|
|
391
|
-
const
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
399
|
-
FROM "${TABLE_THREADS}"
|
|
400
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
401
|
-
query_params: { var_resourceId: resourceId },
|
|
1719
|
+
const record = {
|
|
1720
|
+
...score
|
|
1721
|
+
};
|
|
1722
|
+
await this.client.insert({
|
|
1723
|
+
table: TABLE_SCORERS,
|
|
1724
|
+
values: [record],
|
|
1725
|
+
format: "JSONEachRow",
|
|
402
1726
|
clickhouse_settings: {
|
|
403
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
404
1727
|
date_time_input_format: "best_effort",
|
|
405
|
-
date_time_output_format: "iso",
|
|
406
1728
|
use_client_time_zone: 1,
|
|
407
1729
|
output_format_json_quote_64bit_integers: 0
|
|
408
1730
|
}
|
|
409
1731
|
});
|
|
410
|
-
|
|
411
|
-
const threads = transformRows(rows.data);
|
|
412
|
-
return threads.map((thread) => ({
|
|
413
|
-
...thread,
|
|
414
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
415
|
-
createdAt: thread.createdAt,
|
|
416
|
-
updatedAt: thread.updatedAt
|
|
417
|
-
}));
|
|
1732
|
+
return { score };
|
|
418
1733
|
} catch (error) {
|
|
419
|
-
|
|
420
|
-
|
|
1734
|
+
throw new MastraError(
|
|
1735
|
+
{
|
|
1736
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1737
|
+
domain: ErrorDomain.STORAGE,
|
|
1738
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1739
|
+
details: { scoreId: score.id }
|
|
1740
|
+
},
|
|
1741
|
+
error
|
|
1742
|
+
);
|
|
421
1743
|
}
|
|
422
1744
|
}
|
|
423
|
-
async
|
|
1745
|
+
async getScoresByRunId({
|
|
1746
|
+
runId,
|
|
1747
|
+
pagination
|
|
1748
|
+
}) {
|
|
424
1749
|
try {
|
|
425
|
-
await this.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
]
|
|
1750
|
+
const countResult = await this.client.query({
|
|
1751
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1752
|
+
query_params: { var_runId: runId },
|
|
1753
|
+
format: "JSONEachRow"
|
|
1754
|
+
});
|
|
1755
|
+
const countRows = await countResult.json();
|
|
1756
|
+
let total = 0;
|
|
1757
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1758
|
+
const countObj = countRows[0];
|
|
1759
|
+
total = Number(countObj.count);
|
|
1760
|
+
}
|
|
1761
|
+
if (!total) {
|
|
1762
|
+
return {
|
|
1763
|
+
pagination: {
|
|
1764
|
+
total: 0,
|
|
1765
|
+
page: pagination.page,
|
|
1766
|
+
perPage: pagination.perPage,
|
|
1767
|
+
hasMore: false
|
|
1768
|
+
},
|
|
1769
|
+
scores: []
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
const offset = pagination.page * pagination.perPage;
|
|
1773
|
+
const result = await this.client.query({
|
|
1774
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1775
|
+
query_params: {
|
|
1776
|
+
var_runId: runId,
|
|
1777
|
+
var_limit: pagination.perPage,
|
|
1778
|
+
var_offset: offset
|
|
1779
|
+
},
|
|
434
1780
|
format: "JSONEachRow",
|
|
435
1781
|
clickhouse_settings: {
|
|
436
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
437
1782
|
date_time_input_format: "best_effort",
|
|
1783
|
+
date_time_output_format: "iso",
|
|
438
1784
|
use_client_time_zone: 1,
|
|
439
1785
|
output_format_json_quote_64bit_integers: 0
|
|
440
1786
|
}
|
|
441
1787
|
});
|
|
442
|
-
|
|
1788
|
+
const rows = await result.json();
|
|
1789
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1790
|
+
return {
|
|
1791
|
+
pagination: {
|
|
1792
|
+
total,
|
|
1793
|
+
page: pagination.page,
|
|
1794
|
+
perPage: pagination.perPage,
|
|
1795
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1796
|
+
},
|
|
1797
|
+
scores
|
|
1798
|
+
};
|
|
443
1799
|
} catch (error) {
|
|
444
|
-
|
|
445
|
-
|
|
1800
|
+
throw new MastraError(
|
|
1801
|
+
{
|
|
1802
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1803
|
+
domain: ErrorDomain.STORAGE,
|
|
1804
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1805
|
+
details: { runId }
|
|
1806
|
+
},
|
|
1807
|
+
error
|
|
1808
|
+
);
|
|
446
1809
|
}
|
|
447
1810
|
}
|
|
448
|
-
async
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
metadata
|
|
1811
|
+
async getScoresByScorerId({
|
|
1812
|
+
scorerId,
|
|
1813
|
+
pagination
|
|
452
1814
|
}) {
|
|
453
1815
|
try {
|
|
454
|
-
const
|
|
455
|
-
|
|
456
|
-
|
|
1816
|
+
const countResult = await this.client.query({
|
|
1817
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String}`,
|
|
1818
|
+
query_params: { var_scorerId: scorerId },
|
|
1819
|
+
format: "JSONEachRow"
|
|
1820
|
+
});
|
|
1821
|
+
const countRows = await countResult.json();
|
|
1822
|
+
let total = 0;
|
|
1823
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1824
|
+
const countObj = countRows[0];
|
|
1825
|
+
total = Number(countObj.count);
|
|
457
1826
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
1827
|
+
if (!total) {
|
|
1828
|
+
return {
|
|
1829
|
+
pagination: {
|
|
1830
|
+
total: 0,
|
|
1831
|
+
page: pagination.page,
|
|
1832
|
+
perPage: pagination.perPage,
|
|
1833
|
+
hasMore: false
|
|
1834
|
+
},
|
|
1835
|
+
scores: []
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
const offset = pagination.page * pagination.perPage;
|
|
1839
|
+
const result = await this.client.query({
|
|
1840
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1841
|
+
query_params: {
|
|
1842
|
+
var_scorerId: scorerId,
|
|
1843
|
+
var_limit: pagination.perPage,
|
|
1844
|
+
var_offset: offset
|
|
1845
|
+
},
|
|
476
1846
|
format: "JSONEachRow",
|
|
477
1847
|
clickhouse_settings: {
|
|
478
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
479
1848
|
date_time_input_format: "best_effort",
|
|
1849
|
+
date_time_output_format: "iso",
|
|
480
1850
|
use_client_time_zone: 1,
|
|
481
1851
|
output_format_json_quote_64bit_integers: 0
|
|
482
1852
|
}
|
|
483
1853
|
});
|
|
484
|
-
|
|
1854
|
+
const rows = await result.json();
|
|
1855
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1856
|
+
return {
|
|
1857
|
+
pagination: {
|
|
1858
|
+
total,
|
|
1859
|
+
page: pagination.page,
|
|
1860
|
+
perPage: pagination.perPage,
|
|
1861
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1862
|
+
},
|
|
1863
|
+
scores
|
|
1864
|
+
};
|
|
485
1865
|
} catch (error) {
|
|
486
|
-
|
|
487
|
-
|
|
1866
|
+
throw new MastraError(
|
|
1867
|
+
{
|
|
1868
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1869
|
+
domain: ErrorDomain.STORAGE,
|
|
1870
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1871
|
+
details: { scorerId }
|
|
1872
|
+
},
|
|
1873
|
+
error
|
|
1874
|
+
);
|
|
488
1875
|
}
|
|
489
1876
|
}
|
|
490
|
-
async
|
|
1877
|
+
async getScoresByEntityId({
|
|
1878
|
+
entityId,
|
|
1879
|
+
entityType,
|
|
1880
|
+
pagination
|
|
1881
|
+
}) {
|
|
491
1882
|
try {
|
|
492
|
-
await this.
|
|
493
|
-
query: `
|
|
494
|
-
query_params: {
|
|
1883
|
+
const countResult = await this.client.query({
|
|
1884
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1885
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1886
|
+
format: "JSONEachRow"
|
|
1887
|
+
});
|
|
1888
|
+
const countRows = await countResult.json();
|
|
1889
|
+
let total = 0;
|
|
1890
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1891
|
+
const countObj = countRows[0];
|
|
1892
|
+
total = Number(countObj.count);
|
|
1893
|
+
}
|
|
1894
|
+
if (!total) {
|
|
1895
|
+
return {
|
|
1896
|
+
pagination: {
|
|
1897
|
+
total: 0,
|
|
1898
|
+
page: pagination.page,
|
|
1899
|
+
perPage: pagination.perPage,
|
|
1900
|
+
hasMore: false
|
|
1901
|
+
},
|
|
1902
|
+
scores: []
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
const offset = pagination.page * pagination.perPage;
|
|
1906
|
+
const result = await this.client.query({
|
|
1907
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1908
|
+
query_params: {
|
|
1909
|
+
var_entityId: entityId,
|
|
1910
|
+
var_entityType: entityType,
|
|
1911
|
+
var_limit: pagination.perPage,
|
|
1912
|
+
var_offset: offset
|
|
1913
|
+
},
|
|
1914
|
+
format: "JSONEachRow",
|
|
495
1915
|
clickhouse_settings: {
|
|
1916
|
+
date_time_input_format: "best_effort",
|
|
1917
|
+
date_time_output_format: "iso",
|
|
1918
|
+
use_client_time_zone: 1,
|
|
496
1919
|
output_format_json_quote_64bit_integers: 0
|
|
497
1920
|
}
|
|
498
1921
|
});
|
|
499
|
-
await
|
|
500
|
-
|
|
501
|
-
|
|
1922
|
+
const rows = await result.json();
|
|
1923
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1924
|
+
return {
|
|
1925
|
+
pagination: {
|
|
1926
|
+
total,
|
|
1927
|
+
page: pagination.page,
|
|
1928
|
+
perPage: pagination.perPage,
|
|
1929
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1930
|
+
},
|
|
1931
|
+
scores
|
|
1932
|
+
};
|
|
1933
|
+
} catch (error) {
|
|
1934
|
+
throw new MastraError(
|
|
1935
|
+
{
|
|
1936
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1937
|
+
domain: ErrorDomain.STORAGE,
|
|
1938
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1939
|
+
details: { entityId, entityType }
|
|
1940
|
+
},
|
|
1941
|
+
error
|
|
1942
|
+
);
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
};
|
|
1946
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
1947
|
+
client;
|
|
1948
|
+
operations;
|
|
1949
|
+
constructor({ client, operations }) {
|
|
1950
|
+
super();
|
|
1951
|
+
this.client = client;
|
|
1952
|
+
this.operations = operations;
|
|
1953
|
+
}
|
|
1954
|
+
async getTracesPaginated(args) {
|
|
1955
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
1956
|
+
const fromDate = dateRange?.start;
|
|
1957
|
+
const toDate = dateRange?.end;
|
|
1958
|
+
const currentOffset = page * perPage;
|
|
1959
|
+
const queryArgs = {};
|
|
1960
|
+
const conditions = [];
|
|
1961
|
+
if (name) {
|
|
1962
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
1963
|
+
queryArgs.var_name = name;
|
|
1964
|
+
}
|
|
1965
|
+
if (scope) {
|
|
1966
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
1967
|
+
queryArgs.var_scope = scope;
|
|
1968
|
+
}
|
|
1969
|
+
if (attributes) {
|
|
1970
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
1971
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
1972
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
if (filters) {
|
|
1976
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
1977
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
1978
|
+
queryArgs[`var_col_${key}`] = value;
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
if (fromDate) {
|
|
1982
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
1983
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
1984
|
+
}
|
|
1985
|
+
if (toDate) {
|
|
1986
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
1987
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
1988
|
+
}
|
|
1989
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1990
|
+
try {
|
|
1991
|
+
const countResult = await this.client.query({
|
|
1992
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
1993
|
+
query_params: queryArgs,
|
|
502
1994
|
clickhouse_settings: {
|
|
1995
|
+
date_time_input_format: "best_effort",
|
|
1996
|
+
date_time_output_format: "iso",
|
|
1997
|
+
use_client_time_zone: 1,
|
|
503
1998
|
output_format_json_quote_64bit_integers: 0
|
|
504
1999
|
}
|
|
505
2000
|
});
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if (include.length) {
|
|
517
|
-
const includeResult = await this.db.query({
|
|
518
|
-
query: `
|
|
519
|
-
WITH ordered_messages AS (
|
|
520
|
-
SELECT
|
|
521
|
-
*,
|
|
522
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
523
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
524
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
525
|
-
FROM "${TABLE_MESSAGES}"
|
|
526
|
-
WHERE thread_id = {var_thread_id:String}
|
|
527
|
-
)
|
|
528
|
-
SELECT
|
|
529
|
-
m.id AS id,
|
|
530
|
-
m.content as content,
|
|
531
|
-
m.role as role,
|
|
532
|
-
m.type as type,
|
|
533
|
-
m.createdAt as createdAt,
|
|
534
|
-
m.updatedAt as updatedAt,
|
|
535
|
-
m.thread_id AS "threadId"
|
|
536
|
-
FROM ordered_messages m
|
|
537
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
538
|
-
OR EXISTS (
|
|
539
|
-
SELECT 1 FROM ordered_messages target
|
|
540
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
541
|
-
AND (
|
|
542
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
543
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
544
|
-
OR
|
|
545
|
-
-- Get next messages based on the max withNextMessages
|
|
546
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
547
|
-
)
|
|
548
|
-
)
|
|
549
|
-
ORDER BY m."createdAt" DESC
|
|
550
|
-
`,
|
|
551
|
-
query_params: {
|
|
552
|
-
var_thread_id: threadId,
|
|
553
|
-
var_include: include.map((i) => i.id),
|
|
554
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
555
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
556
|
-
},
|
|
557
|
-
clickhouse_settings: {
|
|
558
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
559
|
-
date_time_input_format: "best_effort",
|
|
560
|
-
date_time_output_format: "iso",
|
|
561
|
-
use_client_time_zone: 1,
|
|
562
|
-
output_format_json_quote_64bit_integers: 0
|
|
563
|
-
}
|
|
564
|
-
});
|
|
565
|
-
const rows2 = await includeResult.json();
|
|
566
|
-
messages.push(...transformRows(rows2.data));
|
|
2001
|
+
const countData = await countResult.json();
|
|
2002
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2003
|
+
if (total === 0) {
|
|
2004
|
+
return {
|
|
2005
|
+
traces: [],
|
|
2006
|
+
total: 0,
|
|
2007
|
+
page,
|
|
2008
|
+
perPage,
|
|
2009
|
+
hasMore: false
|
|
2010
|
+
};
|
|
567
2011
|
}
|
|
568
|
-
const result = await this.
|
|
569
|
-
query: `
|
|
570
|
-
|
|
571
|
-
id,
|
|
572
|
-
content,
|
|
573
|
-
role,
|
|
574
|
-
type,
|
|
575
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
576
|
-
thread_id AS "threadId"
|
|
577
|
-
FROM "${TABLE_MESSAGES}"
|
|
578
|
-
WHERE thread_id = {threadId:String}
|
|
579
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
580
|
-
ORDER BY "createdAt" DESC
|
|
581
|
-
LIMIT {limit:Int64}
|
|
582
|
-
`,
|
|
583
|
-
query_params: {
|
|
584
|
-
threadId,
|
|
585
|
-
exclude: messages.map((m) => m.id),
|
|
586
|
-
limit
|
|
587
|
-
},
|
|
2012
|
+
const result = await this.client.query({
|
|
2013
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2014
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
588
2015
|
clickhouse_settings: {
|
|
589
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
590
2016
|
date_time_input_format: "best_effort",
|
|
591
2017
|
date_time_output_format: "iso",
|
|
592
2018
|
use_client_time_zone: 1,
|
|
593
2019
|
output_format_json_quote_64bit_integers: 0
|
|
594
2020
|
}
|
|
595
2021
|
});
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
2022
|
+
if (!result) {
|
|
2023
|
+
return {
|
|
2024
|
+
traces: [],
|
|
2025
|
+
total,
|
|
2026
|
+
page,
|
|
2027
|
+
perPage,
|
|
2028
|
+
hasMore: false
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
const resp = await result.json();
|
|
2032
|
+
const rows = resp.data;
|
|
2033
|
+
const traces = rows.map((row) => ({
|
|
2034
|
+
id: row.id,
|
|
2035
|
+
parentSpanId: row.parentSpanId,
|
|
2036
|
+
traceId: row.traceId,
|
|
2037
|
+
name: row.name,
|
|
2038
|
+
scope: row.scope,
|
|
2039
|
+
kind: row.kind,
|
|
2040
|
+
status: safelyParseJSON(row.status),
|
|
2041
|
+
events: safelyParseJSON(row.events),
|
|
2042
|
+
links: safelyParseJSON(row.links),
|
|
2043
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2044
|
+
startTime: row.startTime,
|
|
2045
|
+
endTime: row.endTime,
|
|
2046
|
+
other: safelyParseJSON(row.other),
|
|
2047
|
+
createdAt: row.createdAt
|
|
2048
|
+
}));
|
|
2049
|
+
return {
|
|
2050
|
+
traces,
|
|
2051
|
+
total,
|
|
2052
|
+
page,
|
|
2053
|
+
perPage,
|
|
2054
|
+
hasMore: currentOffset + traces.length < total
|
|
2055
|
+
};
|
|
608
2056
|
} catch (error) {
|
|
609
|
-
|
|
610
|
-
|
|
2057
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2058
|
+
return {
|
|
2059
|
+
traces: [],
|
|
2060
|
+
total: 0,
|
|
2061
|
+
page,
|
|
2062
|
+
perPage,
|
|
2063
|
+
hasMore: false
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
throw new MastraError(
|
|
2067
|
+
{
|
|
2068
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2069
|
+
domain: ErrorDomain.STORAGE,
|
|
2070
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2071
|
+
details: {
|
|
2072
|
+
name: name ?? null,
|
|
2073
|
+
scope: scope ?? null,
|
|
2074
|
+
page,
|
|
2075
|
+
perPage,
|
|
2076
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2077
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2078
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2079
|
+
}
|
|
2080
|
+
},
|
|
2081
|
+
error
|
|
2082
|
+
);
|
|
611
2083
|
}
|
|
612
2084
|
}
|
|
613
|
-
async
|
|
614
|
-
|
|
2085
|
+
async getTraces({
|
|
2086
|
+
name,
|
|
2087
|
+
scope,
|
|
2088
|
+
page,
|
|
2089
|
+
perPage,
|
|
2090
|
+
attributes,
|
|
2091
|
+
filters,
|
|
2092
|
+
fromDate,
|
|
2093
|
+
toDate
|
|
2094
|
+
}) {
|
|
2095
|
+
const limit = perPage;
|
|
2096
|
+
const offset = page * perPage;
|
|
2097
|
+
const args = {};
|
|
2098
|
+
const conditions = [];
|
|
2099
|
+
if (name) {
|
|
2100
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2101
|
+
args.var_name = name;
|
|
2102
|
+
}
|
|
2103
|
+
if (scope) {
|
|
2104
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2105
|
+
args.var_scope = scope;
|
|
2106
|
+
}
|
|
2107
|
+
if (attributes) {
|
|
2108
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2109
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2110
|
+
args[`var_attr_${key}`] = value;
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
if (filters) {
|
|
2114
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2115
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2116
|
+
args[`var_col_${key}`] = value;
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
if (fromDate) {
|
|
2120
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2121
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2122
|
+
}
|
|
2123
|
+
if (toDate) {
|
|
2124
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2125
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2126
|
+
}
|
|
2127
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
615
2128
|
try {
|
|
616
|
-
const
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
}
|
|
620
|
-
const thread = await this.getThreadById({ threadId });
|
|
621
|
-
if (!thread) {
|
|
622
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
623
|
-
}
|
|
624
|
-
await this.db.insert({
|
|
625
|
-
table: TABLE_MESSAGES,
|
|
626
|
-
format: "JSONEachRow",
|
|
627
|
-
values: messages.map((message) => ({
|
|
628
|
-
id: message.id,
|
|
629
|
-
thread_id: threadId,
|
|
630
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
631
|
-
createdAt: message.createdAt.toISOString(),
|
|
632
|
-
role: message.role,
|
|
633
|
-
type: message.type
|
|
634
|
-
})),
|
|
2129
|
+
const result = await this.client.query({
|
|
2130
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2131
|
+
query_params: args,
|
|
635
2132
|
clickhouse_settings: {
|
|
636
2133
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
637
2134
|
date_time_input_format: "best_effort",
|
|
2135
|
+
date_time_output_format: "iso",
|
|
638
2136
|
use_client_time_zone: 1,
|
|
639
2137
|
output_format_json_quote_64bit_integers: 0
|
|
640
2138
|
}
|
|
641
2139
|
});
|
|
642
|
-
|
|
2140
|
+
if (!result) {
|
|
2141
|
+
return [];
|
|
2142
|
+
}
|
|
2143
|
+
const resp = await result.json();
|
|
2144
|
+
const rows = resp.data;
|
|
2145
|
+
return rows.map((row) => ({
|
|
2146
|
+
id: row.id,
|
|
2147
|
+
parentSpanId: row.parentSpanId,
|
|
2148
|
+
traceId: row.traceId,
|
|
2149
|
+
name: row.name,
|
|
2150
|
+
scope: row.scope,
|
|
2151
|
+
kind: row.kind,
|
|
2152
|
+
status: safelyParseJSON(row.status),
|
|
2153
|
+
events: safelyParseJSON(row.events),
|
|
2154
|
+
links: safelyParseJSON(row.links),
|
|
2155
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2156
|
+
startTime: row.startTime,
|
|
2157
|
+
endTime: row.endTime,
|
|
2158
|
+
other: safelyParseJSON(row.other),
|
|
2159
|
+
createdAt: row.createdAt
|
|
2160
|
+
}));
|
|
643
2161
|
} catch (error) {
|
|
644
|
-
|
|
645
|
-
|
|
2162
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2163
|
+
return [];
|
|
2164
|
+
}
|
|
2165
|
+
throw new MastraError(
|
|
2166
|
+
{
|
|
2167
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2168
|
+
domain: ErrorDomain.STORAGE,
|
|
2169
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2170
|
+
details: {
|
|
2171
|
+
name: name ?? null,
|
|
2172
|
+
scope: scope ?? null,
|
|
2173
|
+
page,
|
|
2174
|
+
perPage,
|
|
2175
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2176
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2177
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2178
|
+
toDate: toDate?.toISOString() ?? null
|
|
2179
|
+
}
|
|
2180
|
+
},
|
|
2181
|
+
error
|
|
2182
|
+
);
|
|
646
2183
|
}
|
|
647
2184
|
}
|
|
2185
|
+
async batchTraceInsert(args) {
|
|
2186
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2187
|
+
}
|
|
2188
|
+
};
|
|
2189
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2190
|
+
client;
|
|
2191
|
+
operations;
|
|
2192
|
+
constructor({ client, operations }) {
|
|
2193
|
+
super();
|
|
2194
|
+
this.operations = operations;
|
|
2195
|
+
this.client = client;
|
|
2196
|
+
}
|
|
648
2197
|
async persistWorkflowSnapshot({
|
|
649
2198
|
workflowName,
|
|
650
2199
|
runId,
|
|
651
2200
|
snapshot
|
|
652
2201
|
}) {
|
|
653
2202
|
try {
|
|
654
|
-
const currentSnapshot = await this.load({
|
|
2203
|
+
const currentSnapshot = await this.operations.load({
|
|
655
2204
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
656
2205
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
657
2206
|
});
|
|
@@ -667,7 +2216,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
667
2216
|
createdAt: now.toISOString(),
|
|
668
2217
|
updatedAt: now.toISOString()
|
|
669
2218
|
};
|
|
670
|
-
await this.
|
|
2219
|
+
await this.client.insert({
|
|
671
2220
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
672
2221
|
format: "JSONEachRow",
|
|
673
2222
|
values: [persisting],
|
|
@@ -679,8 +2228,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
679
2228
|
}
|
|
680
2229
|
});
|
|
681
2230
|
} catch (error) {
|
|
682
|
-
|
|
683
|
-
|
|
2231
|
+
throw new MastraError(
|
|
2232
|
+
{
|
|
2233
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2234
|
+
domain: ErrorDomain.STORAGE,
|
|
2235
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2236
|
+
details: { workflowName, runId }
|
|
2237
|
+
},
|
|
2238
|
+
error
|
|
2239
|
+
);
|
|
684
2240
|
}
|
|
685
2241
|
}
|
|
686
2242
|
async loadWorkflowSnapshot({
|
|
@@ -688,7 +2244,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
688
2244
|
runId
|
|
689
2245
|
}) {
|
|
690
2246
|
try {
|
|
691
|
-
const result = await this.load({
|
|
2247
|
+
const result = await this.operations.load({
|
|
692
2248
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
693
2249
|
keys: {
|
|
694
2250
|
workflow_name: workflowName,
|
|
@@ -700,8 +2256,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
700
2256
|
}
|
|
701
2257
|
return result.snapshot;
|
|
702
2258
|
} catch (error) {
|
|
703
|
-
|
|
704
|
-
|
|
2259
|
+
throw new MastraError(
|
|
2260
|
+
{
|
|
2261
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2262
|
+
domain: ErrorDomain.STORAGE,
|
|
2263
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2264
|
+
details: { workflowName, runId }
|
|
2265
|
+
},
|
|
2266
|
+
error
|
|
2267
|
+
);
|
|
705
2268
|
}
|
|
706
2269
|
}
|
|
707
2270
|
parseWorkflowRun(row) {
|
|
@@ -738,7 +2301,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
738
2301
|
values.var_workflow_name = workflowName;
|
|
739
2302
|
}
|
|
740
2303
|
if (resourceId) {
|
|
741
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2304
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
742
2305
|
if (hasResourceId) {
|
|
743
2306
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
744
2307
|
values.var_resourceId = resourceId;
|
|
@@ -759,7 +2322,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
759
2322
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
760
2323
|
let total = 0;
|
|
761
2324
|
if (limit !== void 0 && offset !== void 0) {
|
|
762
|
-
const countResult = await this.
|
|
2325
|
+
const countResult = await this.client.query({
|
|
763
2326
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
764
2327
|
query_params: values,
|
|
765
2328
|
format: "JSONEachRow"
|
|
@@ -767,21 +2330,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
767
2330
|
const countRows = await countResult.json();
|
|
768
2331
|
total = Number(countRows[0]?.count ?? 0);
|
|
769
2332
|
}
|
|
770
|
-
const result = await this.
|
|
2333
|
+
const result = await this.client.query({
|
|
771
2334
|
query: `
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2335
|
+
SELECT
|
|
2336
|
+
workflow_name,
|
|
2337
|
+
run_id,
|
|
2338
|
+
snapshot,
|
|
2339
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2340
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2341
|
+
resourceId
|
|
2342
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2343
|
+
${whereClause}
|
|
2344
|
+
ORDER BY createdAt DESC
|
|
2345
|
+
${limitClause}
|
|
2346
|
+
${offsetClause}
|
|
2347
|
+
`,
|
|
785
2348
|
query_params: values,
|
|
786
2349
|
format: "JSONEachRow"
|
|
787
2350
|
});
|
|
@@ -792,8 +2355,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
792
2355
|
});
|
|
793
2356
|
return { runs, total: total || runs.length };
|
|
794
2357
|
} catch (error) {
|
|
795
|
-
|
|
796
|
-
|
|
2358
|
+
throw new MastraError(
|
|
2359
|
+
{
|
|
2360
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2361
|
+
domain: ErrorDomain.STORAGE,
|
|
2362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2363
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2364
|
+
},
|
|
2365
|
+
error
|
|
2366
|
+
);
|
|
797
2367
|
}
|
|
798
2368
|
}
|
|
799
2369
|
async getWorkflowRunById({
|
|
@@ -812,18 +2382,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
812
2382
|
values.var_workflow_name = workflowName;
|
|
813
2383
|
}
|
|
814
2384
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
815
|
-
const result = await this.
|
|
2385
|
+
const result = await this.client.query({
|
|
816
2386
|
query: `
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
2387
|
+
SELECT
|
|
2388
|
+
workflow_name,
|
|
2389
|
+
run_id,
|
|
2390
|
+
snapshot,
|
|
2391
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2392
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2393
|
+
resourceId
|
|
2394
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2395
|
+
${whereClause}
|
|
2396
|
+
`,
|
|
827
2397
|
query_params: values,
|
|
828
2398
|
format: "JSONEachRow"
|
|
829
2399
|
});
|
|
@@ -833,21 +2403,248 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
833
2403
|
}
|
|
834
2404
|
return this.parseWorkflowRun(resultJson[0]);
|
|
835
2405
|
} catch (error) {
|
|
836
|
-
|
|
837
|
-
|
|
2406
|
+
throw new MastraError(
|
|
2407
|
+
{
|
|
2408
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2409
|
+
domain: ErrorDomain.STORAGE,
|
|
2410
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2411
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2412
|
+
},
|
|
2413
|
+
error
|
|
2414
|
+
);
|
|
838
2415
|
}
|
|
839
2416
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
2417
|
+
};
|
|
2418
|
+
|
|
2419
|
+
// src/storage/index.ts
|
|
2420
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2421
|
+
db;
|
|
2422
|
+
ttl = {};
|
|
2423
|
+
stores;
|
|
2424
|
+
constructor(config) {
|
|
2425
|
+
super({ name: "ClickhouseStore" });
|
|
2426
|
+
this.db = createClient({
|
|
2427
|
+
url: config.url,
|
|
2428
|
+
username: config.username,
|
|
2429
|
+
password: config.password,
|
|
2430
|
+
clickhouse_settings: {
|
|
2431
|
+
date_time_input_format: "best_effort",
|
|
2432
|
+
date_time_output_format: "iso",
|
|
2433
|
+
// This is crucial
|
|
2434
|
+
use_client_time_zone: 1,
|
|
2435
|
+
output_format_json_quote_64bit_integers: 0
|
|
2436
|
+
}
|
|
844
2437
|
});
|
|
845
|
-
|
|
846
|
-
|
|
2438
|
+
this.ttl = config.ttl;
|
|
2439
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2440
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2441
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2442
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2443
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2444
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2445
|
+
this.stores = {
|
|
2446
|
+
operations,
|
|
2447
|
+
workflows,
|
|
2448
|
+
scores,
|
|
2449
|
+
legacyEvals,
|
|
2450
|
+
traces,
|
|
2451
|
+
memory
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
get supports() {
|
|
2455
|
+
return {
|
|
2456
|
+
selectByIncludeResourceScope: true,
|
|
2457
|
+
resourceWorkingMemory: true,
|
|
2458
|
+
hasColumn: true,
|
|
2459
|
+
createTable: true
|
|
2460
|
+
};
|
|
2461
|
+
}
|
|
2462
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2463
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2464
|
+
}
|
|
2465
|
+
async getEvals(options) {
|
|
2466
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2467
|
+
}
|
|
2468
|
+
async batchInsert({ tableName, records }) {
|
|
2469
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2470
|
+
}
|
|
2471
|
+
async optimizeTable({ tableName }) {
|
|
2472
|
+
try {
|
|
2473
|
+
await this.db.command({
|
|
2474
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2475
|
+
});
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
throw new MastraError(
|
|
2478
|
+
{
|
|
2479
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2480
|
+
domain: ErrorDomain.STORAGE,
|
|
2481
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2482
|
+
details: { tableName }
|
|
2483
|
+
},
|
|
2484
|
+
error
|
|
2485
|
+
);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
async materializeTtl({ tableName }) {
|
|
2489
|
+
try {
|
|
2490
|
+
await this.db.command({
|
|
2491
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2492
|
+
});
|
|
2493
|
+
} catch (error) {
|
|
2494
|
+
throw new MastraError(
|
|
2495
|
+
{
|
|
2496
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2497
|
+
domain: ErrorDomain.STORAGE,
|
|
2498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2499
|
+
details: { tableName }
|
|
2500
|
+
},
|
|
2501
|
+
error
|
|
2502
|
+
);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
async createTable({
|
|
2506
|
+
tableName,
|
|
2507
|
+
schema
|
|
2508
|
+
}) {
|
|
2509
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2510
|
+
}
|
|
2511
|
+
async dropTable({ tableName }) {
|
|
2512
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2513
|
+
}
|
|
2514
|
+
async alterTable({
|
|
2515
|
+
tableName,
|
|
2516
|
+
schema,
|
|
2517
|
+
ifNotExists
|
|
2518
|
+
}) {
|
|
2519
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2520
|
+
}
|
|
2521
|
+
async clearTable({ tableName }) {
|
|
2522
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2523
|
+
}
|
|
2524
|
+
async insert({ tableName, record }) {
|
|
2525
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2526
|
+
}
|
|
2527
|
+
async load({ tableName, keys }) {
|
|
2528
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2529
|
+
}
|
|
2530
|
+
async persistWorkflowSnapshot({
|
|
2531
|
+
workflowName,
|
|
2532
|
+
runId,
|
|
2533
|
+
snapshot
|
|
2534
|
+
}) {
|
|
2535
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2536
|
+
}
|
|
2537
|
+
async loadWorkflowSnapshot({
|
|
2538
|
+
workflowName,
|
|
2539
|
+
runId
|
|
2540
|
+
}) {
|
|
2541
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2542
|
+
}
|
|
2543
|
+
async getWorkflowRuns({
|
|
2544
|
+
workflowName,
|
|
2545
|
+
fromDate,
|
|
2546
|
+
toDate,
|
|
2547
|
+
limit,
|
|
2548
|
+
offset,
|
|
2549
|
+
resourceId
|
|
2550
|
+
} = {}) {
|
|
2551
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2552
|
+
}
|
|
2553
|
+
async getWorkflowRunById({
|
|
2554
|
+
runId,
|
|
2555
|
+
workflowName
|
|
2556
|
+
}) {
|
|
2557
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2558
|
+
}
|
|
2559
|
+
async getTraces(args) {
|
|
2560
|
+
return this.stores.traces.getTraces(args);
|
|
2561
|
+
}
|
|
2562
|
+
async getTracesPaginated(args) {
|
|
2563
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2564
|
+
}
|
|
2565
|
+
async batchTraceInsert(args) {
|
|
2566
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2567
|
+
}
|
|
2568
|
+
async getThreadById({ threadId }) {
|
|
2569
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2570
|
+
}
|
|
2571
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2572
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2573
|
+
}
|
|
2574
|
+
async saveThread({ thread }) {
|
|
2575
|
+
return this.stores.memory.saveThread({ thread });
|
|
2576
|
+
}
|
|
2577
|
+
async updateThread({
|
|
2578
|
+
id,
|
|
2579
|
+
title,
|
|
2580
|
+
metadata
|
|
2581
|
+
}) {
|
|
2582
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2583
|
+
}
|
|
2584
|
+
async deleteThread({ threadId }) {
|
|
2585
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2586
|
+
}
|
|
2587
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2588
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2589
|
+
}
|
|
2590
|
+
async getMessages({
|
|
2591
|
+
threadId,
|
|
2592
|
+
resourceId,
|
|
2593
|
+
selectBy,
|
|
2594
|
+
format
|
|
2595
|
+
}) {
|
|
2596
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2597
|
+
}
|
|
2598
|
+
async saveMessages(args) {
|
|
2599
|
+
return this.stores.memory.saveMessages(args);
|
|
2600
|
+
}
|
|
2601
|
+
async getMessagesPaginated(args) {
|
|
2602
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2603
|
+
}
|
|
2604
|
+
async updateMessages(args) {
|
|
2605
|
+
return this.stores.memory.updateMessages(args);
|
|
2606
|
+
}
|
|
2607
|
+
async getResourceById({ resourceId }) {
|
|
2608
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2609
|
+
}
|
|
2610
|
+
async saveResource({ resource }) {
|
|
2611
|
+
return this.stores.memory.saveResource({ resource });
|
|
2612
|
+
}
|
|
2613
|
+
async updateResource({
|
|
2614
|
+
resourceId,
|
|
2615
|
+
workingMemory,
|
|
2616
|
+
metadata
|
|
2617
|
+
}) {
|
|
2618
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2619
|
+
}
|
|
2620
|
+
async getScoreById({ id }) {
|
|
2621
|
+
return this.stores.scores.getScoreById({ id });
|
|
2622
|
+
}
|
|
2623
|
+
async saveScore(_score) {
|
|
2624
|
+
return this.stores.scores.saveScore(_score);
|
|
2625
|
+
}
|
|
2626
|
+
async getScoresByRunId({
|
|
2627
|
+
runId,
|
|
2628
|
+
pagination
|
|
2629
|
+
}) {
|
|
2630
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2631
|
+
}
|
|
2632
|
+
async getScoresByEntityId({
|
|
2633
|
+
entityId,
|
|
2634
|
+
entityType,
|
|
2635
|
+
pagination
|
|
2636
|
+
}) {
|
|
2637
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2638
|
+
}
|
|
2639
|
+
async getScoresByScorerId({
|
|
2640
|
+
scorerId,
|
|
2641
|
+
pagination
|
|
2642
|
+
}) {
|
|
2643
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
847
2644
|
}
|
|
848
2645
|
async close() {
|
|
849
2646
|
await this.db.close();
|
|
850
2647
|
}
|
|
851
2648
|
};
|
|
852
2649
|
|
|
853
|
-
export {
|
|
2650
|
+
export { ClickhouseStore };
|