@mastra/clickhouse 0.0.0-trigger-playground-ui-package-20250506151043 → 0.0.0-update-stores-peerDeps-20250723031338
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +477 -2
- package/LICENSE.md +12 -4
- package/dist/_tsup-dts-rollup.d.cts +373 -33
- package/dist/_tsup-dts-rollup.d.ts +373 -33
- package/dist/index.cjs +2235 -440
- package/dist/index.d.cts +0 -2
- package/dist/index.d.ts +0 -2
- package/dist/index.js +2223 -426
- package/package.json +14 -10
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +14 -844
- package/src/storage/index.ts +241 -897
package/dist/index.cjs
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var client = require('@clickhouse/client');
|
|
4
|
+
var error = require('@mastra/core/error');
|
|
4
5
|
var storage = require('@mastra/core/storage');
|
|
6
|
+
var agent = require('@mastra/core/agent');
|
|
5
7
|
|
|
6
8
|
// src/storage/index.ts
|
|
7
|
-
function safelyParseJSON(jsonString) {
|
|
8
|
-
try {
|
|
9
|
-
return JSON.parse(jsonString);
|
|
10
|
-
} catch {
|
|
11
|
-
return {};
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
9
|
var TABLE_ENGINES = {
|
|
15
10
|
[storage.TABLE_MESSAGES]: `MergeTree()`,
|
|
16
11
|
[storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
17
12
|
[storage.TABLE_TRACES]: `MergeTree()`,
|
|
18
13
|
[storage.TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
19
|
-
[storage.TABLE_EVALS]: `MergeTree()
|
|
14
|
+
[storage.TABLE_EVALS]: `MergeTree()`,
|
|
15
|
+
[storage.TABLE_SCORERS]: `MergeTree()`,
|
|
16
|
+
[storage.TABLE_RESOURCES]: `ReplacingMergeTree()`
|
|
20
17
|
};
|
|
21
18
|
var COLUMN_TYPES = {
|
|
22
19
|
text: "String",
|
|
@@ -24,11 +21,9 @@ var COLUMN_TYPES = {
|
|
|
24
21
|
uuid: "String",
|
|
25
22
|
jsonb: "String",
|
|
26
23
|
integer: "Int64",
|
|
24
|
+
float: "Float64",
|
|
27
25
|
bigint: "Int64"
|
|
28
26
|
};
|
|
29
|
-
function transformRows(rows) {
|
|
30
|
-
return rows.map((row) => transformRow(row));
|
|
31
|
-
}
|
|
32
27
|
function transformRow(row) {
|
|
33
28
|
if (!row) {
|
|
34
29
|
return row;
|
|
@@ -39,33 +34,63 @@ function transformRow(row) {
|
|
|
39
34
|
if (row.updatedAt) {
|
|
40
35
|
row.updatedAt = new Date(row.updatedAt);
|
|
41
36
|
}
|
|
37
|
+
if (row.content && typeof row.content === "string") {
|
|
38
|
+
row.content = storage.safelyParseJSON(row.content);
|
|
39
|
+
}
|
|
42
40
|
return row;
|
|
43
41
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// This is crucial
|
|
57
|
-
use_client_time_zone: 1,
|
|
58
|
-
output_format_json_quote_64bit_integers: 0
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
this.ttl = config.ttl;
|
|
42
|
+
function transformRows(rows) {
|
|
43
|
+
return rows.map((row) => transformRow(row));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
47
|
+
var LegacyEvalsStorageClickhouse = class extends storage.LegacyEvalsStorage {
|
|
48
|
+
client;
|
|
49
|
+
operations;
|
|
50
|
+
constructor({ client, operations }) {
|
|
51
|
+
super();
|
|
52
|
+
this.client = client;
|
|
53
|
+
this.operations = operations;
|
|
62
54
|
}
|
|
63
55
|
transformEvalRow(row) {
|
|
64
56
|
row = transformRow(row);
|
|
65
|
-
|
|
66
|
-
|
|
57
|
+
let resultValue;
|
|
58
|
+
try {
|
|
59
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
60
|
+
resultValue = JSON.parse(row.result);
|
|
61
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
62
|
+
resultValue = row.result;
|
|
63
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
64
|
+
resultValue = { score: 0 };
|
|
65
|
+
} else {
|
|
66
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
67
|
+
}
|
|
68
|
+
} catch (error$1) {
|
|
69
|
+
console.error("Error parsing result field:", row.result, error$1);
|
|
70
|
+
throw new error.MastraError({
|
|
71
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
72
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
73
|
+
domain: error.ErrorDomain.STORAGE,
|
|
74
|
+
category: error.ErrorCategory.USER
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
let testInfoValue;
|
|
78
|
+
try {
|
|
79
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
80
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
81
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
82
|
+
testInfoValue = row.test_info;
|
|
83
|
+
}
|
|
84
|
+
} catch {
|
|
85
|
+
testInfoValue = void 0;
|
|
86
|
+
}
|
|
67
87
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
|
-
throw new
|
|
88
|
+
throw new error.MastraError({
|
|
89
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
90
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
91
|
+
domain: error.ErrorDomain.STORAGE,
|
|
92
|
+
category: error.ErrorCategory.USER
|
|
93
|
+
});
|
|
69
94
|
}
|
|
70
95
|
return {
|
|
71
96
|
input: row.input,
|
|
@@ -82,9 +107,9 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
82
107
|
}
|
|
83
108
|
async getEvalsByAgentName(agentName, type) {
|
|
84
109
|
try {
|
|
85
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
86
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
87
|
-
const result = await this.
|
|
110
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
111
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
112
|
+
const result = await this.client.query({
|
|
88
113
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
89
114
|
query_params: { var_agent_name: agentName },
|
|
90
115
|
clickhouse_settings: {
|
|
@@ -99,26 +124,498 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
99
124
|
}
|
|
100
125
|
const rows = await result.json();
|
|
101
126
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
102
|
-
} catch (error) {
|
|
103
|
-
if (error
|
|
127
|
+
} catch (error$1) {
|
|
128
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
104
129
|
return [];
|
|
105
130
|
}
|
|
106
|
-
|
|
107
|
-
|
|
131
|
+
throw new error.MastraError(
|
|
132
|
+
{
|
|
133
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
134
|
+
domain: error.ErrorDomain.STORAGE,
|
|
135
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
136
|
+
details: { agentName, type: type ?? null }
|
|
137
|
+
},
|
|
138
|
+
error$1
|
|
139
|
+
);
|
|
108
140
|
}
|
|
109
141
|
}
|
|
110
|
-
async
|
|
142
|
+
async getEvals(options = {}) {
|
|
143
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
144
|
+
const fromDate = dateRange?.start;
|
|
145
|
+
const toDate = dateRange?.end;
|
|
146
|
+
const conditions = [];
|
|
147
|
+
if (agentName) {
|
|
148
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
149
|
+
}
|
|
150
|
+
if (type === "test") {
|
|
151
|
+
conditions.push(
|
|
152
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
153
|
+
);
|
|
154
|
+
} else if (type === "live") {
|
|
155
|
+
conditions.push(
|
|
156
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (fromDate) {
|
|
160
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
161
|
+
fromDate.toISOString();
|
|
162
|
+
}
|
|
163
|
+
if (toDate) {
|
|
164
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
165
|
+
toDate.toISOString();
|
|
166
|
+
}
|
|
167
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
111
168
|
try {
|
|
112
|
-
await this.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
...
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
169
|
+
const countResult = await this.client.query({
|
|
170
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_EVALS} ${whereClause}`,
|
|
171
|
+
query_params: {
|
|
172
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
173
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
174
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
175
|
+
},
|
|
176
|
+
clickhouse_settings: {
|
|
177
|
+
date_time_input_format: "best_effort",
|
|
178
|
+
date_time_output_format: "iso",
|
|
179
|
+
use_client_time_zone: 1,
|
|
180
|
+
output_format_json_quote_64bit_integers: 0
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
const countData = await countResult.json();
|
|
184
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
185
|
+
const currentOffset = page * perPage;
|
|
186
|
+
const hasMore = currentOffset + perPage < total;
|
|
187
|
+
if (total === 0) {
|
|
188
|
+
return {
|
|
189
|
+
evals: [],
|
|
190
|
+
total: 0,
|
|
191
|
+
page,
|
|
192
|
+
perPage,
|
|
193
|
+
hasMore: false
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
const dataResult = await this.client.query({
|
|
197
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
198
|
+
query_params: {
|
|
199
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
200
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
201
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
202
|
+
var_limit: perPage || 100,
|
|
203
|
+
var_offset: currentOffset || 0
|
|
204
|
+
},
|
|
205
|
+
clickhouse_settings: {
|
|
206
|
+
date_time_input_format: "best_effort",
|
|
207
|
+
date_time_output_format: "iso",
|
|
208
|
+
use_client_time_zone: 1,
|
|
209
|
+
output_format_json_quote_64bit_integers: 0
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
const rows = await dataResult.json();
|
|
213
|
+
return {
|
|
214
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
215
|
+
total,
|
|
216
|
+
page,
|
|
217
|
+
perPage,
|
|
218
|
+
hasMore
|
|
219
|
+
};
|
|
220
|
+
} catch (error$1) {
|
|
221
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
222
|
+
return {
|
|
223
|
+
evals: [],
|
|
224
|
+
total: 0,
|
|
225
|
+
page,
|
|
226
|
+
perPage,
|
|
227
|
+
hasMore: false
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
throw new error.MastraError(
|
|
231
|
+
{
|
|
232
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
233
|
+
domain: error.ErrorDomain.STORAGE,
|
|
234
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
235
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
236
|
+
},
|
|
237
|
+
error$1
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
var MemoryStorageClickhouse = class extends storage.MemoryStorage {
|
|
243
|
+
client;
|
|
244
|
+
operations;
|
|
245
|
+
constructor({ client, operations }) {
|
|
246
|
+
super();
|
|
247
|
+
this.client = client;
|
|
248
|
+
this.operations = operations;
|
|
249
|
+
}
|
|
250
|
+
async getMessages({
|
|
251
|
+
threadId,
|
|
252
|
+
resourceId,
|
|
253
|
+
selectBy,
|
|
254
|
+
format
|
|
255
|
+
}) {
|
|
256
|
+
try {
|
|
257
|
+
const messages = [];
|
|
258
|
+
const limit = storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
259
|
+
const include = selectBy?.include || [];
|
|
260
|
+
if (include.length) {
|
|
261
|
+
const unionQueries = [];
|
|
262
|
+
const params = [];
|
|
263
|
+
let paramIdx = 1;
|
|
264
|
+
for (const inc of include) {
|
|
265
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
266
|
+
const searchId = inc.threadId || threadId;
|
|
267
|
+
unionQueries.push(`
|
|
268
|
+
SELECT * FROM (
|
|
269
|
+
WITH numbered_messages AS (
|
|
270
|
+
SELECT
|
|
271
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
272
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
273
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
274
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
275
|
+
),
|
|
276
|
+
target_positions AS (
|
|
277
|
+
SELECT row_num as target_pos
|
|
278
|
+
FROM numbered_messages
|
|
279
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
280
|
+
)
|
|
281
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
282
|
+
FROM numbered_messages m
|
|
283
|
+
CROSS JOIN target_positions t
|
|
284
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
285
|
+
) AS query_${paramIdx}
|
|
286
|
+
`);
|
|
287
|
+
params.push(
|
|
288
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
289
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
290
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
291
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
292
|
+
);
|
|
293
|
+
paramIdx++;
|
|
294
|
+
}
|
|
295
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
296
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
297
|
+
const includeResult = await this.client.query({
|
|
298
|
+
query: finalQuery,
|
|
299
|
+
query_params: mergedParams,
|
|
300
|
+
clickhouse_settings: {
|
|
301
|
+
date_time_input_format: "best_effort",
|
|
302
|
+
date_time_output_format: "iso",
|
|
303
|
+
use_client_time_zone: 1,
|
|
304
|
+
output_format_json_quote_64bit_integers: 0
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
const rows2 = await includeResult.json();
|
|
308
|
+
const includedMessages = transformRows(rows2.data);
|
|
309
|
+
const seen = /* @__PURE__ */ new Set();
|
|
310
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
311
|
+
if (seen.has(message.id)) return false;
|
|
312
|
+
seen.add(message.id);
|
|
313
|
+
return true;
|
|
314
|
+
});
|
|
315
|
+
messages.push(...dedupedMessages);
|
|
316
|
+
}
|
|
317
|
+
const result = await this.client.query({
|
|
318
|
+
query: `
|
|
319
|
+
SELECT
|
|
320
|
+
id,
|
|
321
|
+
content,
|
|
322
|
+
role,
|
|
323
|
+
type,
|
|
324
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
325
|
+
thread_id AS "threadId"
|
|
326
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
327
|
+
WHERE thread_id = {threadId:String}
|
|
328
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
329
|
+
ORDER BY "createdAt" DESC
|
|
330
|
+
LIMIT {limit:Int64}
|
|
331
|
+
`,
|
|
332
|
+
query_params: {
|
|
333
|
+
threadId,
|
|
334
|
+
exclude: messages.map((m) => m.id),
|
|
335
|
+
limit
|
|
336
|
+
},
|
|
337
|
+
clickhouse_settings: {
|
|
338
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
339
|
+
date_time_input_format: "best_effort",
|
|
340
|
+
date_time_output_format: "iso",
|
|
341
|
+
use_client_time_zone: 1,
|
|
342
|
+
output_format_json_quote_64bit_integers: 0
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
const rows = await result.json();
|
|
346
|
+
messages.push(...transformRows(rows.data));
|
|
347
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
348
|
+
messages.forEach((message) => {
|
|
349
|
+
if (typeof message.content === "string") {
|
|
350
|
+
try {
|
|
351
|
+
message.content = JSON.parse(message.content);
|
|
352
|
+
} catch {
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
357
|
+
if (format === `v2`) return list.get.all.v2();
|
|
358
|
+
return list.get.all.v1();
|
|
359
|
+
} catch (error$1) {
|
|
360
|
+
throw new error.MastraError(
|
|
361
|
+
{
|
|
362
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
363
|
+
domain: error.ErrorDomain.STORAGE,
|
|
364
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
365
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
366
|
+
},
|
|
367
|
+
error$1
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
async saveMessages(args) {
|
|
372
|
+
const { messages, format = "v1" } = args;
|
|
373
|
+
if (messages.length === 0) return messages;
|
|
374
|
+
for (const message of messages) {
|
|
375
|
+
const resourceId = message.resourceId;
|
|
376
|
+
if (!resourceId) {
|
|
377
|
+
throw new Error("Resource ID is required");
|
|
378
|
+
}
|
|
379
|
+
if (!message.threadId) {
|
|
380
|
+
throw new Error("Thread ID is required");
|
|
381
|
+
}
|
|
382
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
383
|
+
if (!thread) {
|
|
384
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
388
|
+
await Promise.all(
|
|
389
|
+
messages.map(async (m) => {
|
|
390
|
+
const resourceId = m.resourceId;
|
|
391
|
+
if (!resourceId) {
|
|
392
|
+
throw new Error("Resource ID is required");
|
|
393
|
+
}
|
|
394
|
+
if (!m.threadId) {
|
|
395
|
+
throw new Error("Thread ID is required");
|
|
396
|
+
}
|
|
397
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
398
|
+
if (!thread) {
|
|
399
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
400
|
+
}
|
|
401
|
+
threadIdSet.set(m.threadId, thread);
|
|
402
|
+
})
|
|
403
|
+
);
|
|
404
|
+
try {
|
|
405
|
+
const existingResult = await this.client.query({
|
|
406
|
+
query: `SELECT id, thread_id FROM ${storage.TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
407
|
+
query_params: {
|
|
408
|
+
ids: messages.map((m) => m.id)
|
|
409
|
+
},
|
|
410
|
+
clickhouse_settings: {
|
|
411
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
412
|
+
date_time_input_format: "best_effort",
|
|
413
|
+
date_time_output_format: "iso",
|
|
414
|
+
use_client_time_zone: 1,
|
|
415
|
+
output_format_json_quote_64bit_integers: 0
|
|
416
|
+
},
|
|
417
|
+
format: "JSONEachRow"
|
|
418
|
+
});
|
|
419
|
+
const existingRows = await existingResult.json();
|
|
420
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
421
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
422
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
423
|
+
const toMove = messages.filter((m) => {
|
|
424
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
425
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
426
|
+
});
|
|
427
|
+
const deletePromises = toMove.map((message) => {
|
|
428
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
429
|
+
if (!existingRow) return Promise.resolve();
|
|
430
|
+
return this.client.command({
|
|
431
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
432
|
+
query_params: {
|
|
433
|
+
var_id: message.id,
|
|
434
|
+
var_old_thread_id: existingRow.thread_id
|
|
435
|
+
},
|
|
436
|
+
clickhouse_settings: {
|
|
437
|
+
date_time_input_format: "best_effort",
|
|
438
|
+
use_client_time_zone: 1,
|
|
439
|
+
output_format_json_quote_64bit_integers: 0
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
});
|
|
443
|
+
const updatePromises = toUpdate.map(
|
|
444
|
+
(message) => this.client.command({
|
|
445
|
+
query: `
|
|
446
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
447
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
448
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
449
|
+
`,
|
|
450
|
+
query_params: {
|
|
451
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
452
|
+
var_role: message.role,
|
|
453
|
+
var_type: message.type || "v2",
|
|
454
|
+
var_resourceId: message.resourceId,
|
|
455
|
+
var_id: message.id,
|
|
456
|
+
var_thread_id: message.threadId
|
|
457
|
+
},
|
|
458
|
+
clickhouse_settings: {
|
|
459
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
460
|
+
date_time_input_format: "best_effort",
|
|
461
|
+
use_client_time_zone: 1,
|
|
462
|
+
output_format_json_quote_64bit_integers: 0
|
|
463
|
+
}
|
|
464
|
+
})
|
|
465
|
+
);
|
|
466
|
+
await Promise.all([
|
|
467
|
+
// Insert new messages (including moved messages)
|
|
468
|
+
this.client.insert({
|
|
469
|
+
table: storage.TABLE_MESSAGES,
|
|
470
|
+
format: "JSONEachRow",
|
|
471
|
+
values: toInsert.map((message) => ({
|
|
472
|
+
id: message.id,
|
|
473
|
+
thread_id: message.threadId,
|
|
474
|
+
resourceId: message.resourceId,
|
|
475
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
476
|
+
createdAt: message.createdAt.toISOString(),
|
|
477
|
+
role: message.role,
|
|
478
|
+
type: message.type || "v2"
|
|
479
|
+
})),
|
|
480
|
+
clickhouse_settings: {
|
|
481
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
482
|
+
date_time_input_format: "best_effort",
|
|
483
|
+
use_client_time_zone: 1,
|
|
484
|
+
output_format_json_quote_64bit_integers: 0
|
|
485
|
+
}
|
|
486
|
+
}),
|
|
487
|
+
...updatePromises,
|
|
488
|
+
...deletePromises,
|
|
489
|
+
// Update thread's updatedAt timestamp
|
|
490
|
+
this.client.insert({
|
|
491
|
+
table: storage.TABLE_THREADS,
|
|
492
|
+
format: "JSONEachRow",
|
|
493
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
494
|
+
id: thread.id,
|
|
495
|
+
resourceId: thread.resourceId,
|
|
496
|
+
title: thread.title,
|
|
497
|
+
metadata: thread.metadata,
|
|
498
|
+
createdAt: thread.createdAt,
|
|
499
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
500
|
+
})),
|
|
501
|
+
clickhouse_settings: {
|
|
502
|
+
date_time_input_format: "best_effort",
|
|
503
|
+
use_client_time_zone: 1,
|
|
504
|
+
output_format_json_quote_64bit_integers: 0
|
|
505
|
+
}
|
|
506
|
+
})
|
|
507
|
+
]);
|
|
508
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
509
|
+
if (format === `v2`) return list.get.all.v2();
|
|
510
|
+
return list.get.all.v1();
|
|
511
|
+
} catch (error$1) {
|
|
512
|
+
throw new error.MastraError(
|
|
513
|
+
{
|
|
514
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
515
|
+
domain: error.ErrorDomain.STORAGE,
|
|
516
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
517
|
+
},
|
|
518
|
+
error$1
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
async getThreadById({ threadId }) {
|
|
523
|
+
try {
|
|
524
|
+
const result = await this.client.query({
|
|
525
|
+
query: `SELECT
|
|
526
|
+
id,
|
|
527
|
+
"resourceId",
|
|
528
|
+
title,
|
|
529
|
+
metadata,
|
|
530
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
531
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
532
|
+
FROM "${storage.TABLE_THREADS}"
|
|
533
|
+
FINAL
|
|
534
|
+
WHERE id = {var_id:String}`,
|
|
535
|
+
query_params: { var_id: threadId },
|
|
536
|
+
clickhouse_settings: {
|
|
537
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
538
|
+
date_time_input_format: "best_effort",
|
|
539
|
+
date_time_output_format: "iso",
|
|
540
|
+
use_client_time_zone: 1,
|
|
541
|
+
output_format_json_quote_64bit_integers: 0
|
|
542
|
+
}
|
|
543
|
+
});
|
|
544
|
+
const rows = await result.json();
|
|
545
|
+
const thread = transformRow(rows.data[0]);
|
|
546
|
+
if (!thread) {
|
|
547
|
+
return null;
|
|
548
|
+
}
|
|
549
|
+
return {
|
|
550
|
+
...thread,
|
|
551
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
552
|
+
createdAt: thread.createdAt,
|
|
553
|
+
updatedAt: thread.updatedAt
|
|
554
|
+
};
|
|
555
|
+
} catch (error$1) {
|
|
556
|
+
throw new error.MastraError(
|
|
557
|
+
{
|
|
558
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
559
|
+
domain: error.ErrorDomain.STORAGE,
|
|
560
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
561
|
+
details: { threadId }
|
|
562
|
+
},
|
|
563
|
+
error$1
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
568
|
+
try {
|
|
569
|
+
const result = await this.client.query({
|
|
570
|
+
query: `SELECT
|
|
571
|
+
id,
|
|
572
|
+
"resourceId",
|
|
573
|
+
title,
|
|
574
|
+
metadata,
|
|
575
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
576
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
577
|
+
FROM "${storage.TABLE_THREADS}"
|
|
578
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
579
|
+
query_params: { var_resourceId: resourceId },
|
|
580
|
+
clickhouse_settings: {
|
|
581
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
582
|
+
date_time_input_format: "best_effort",
|
|
583
|
+
date_time_output_format: "iso",
|
|
584
|
+
use_client_time_zone: 1,
|
|
585
|
+
output_format_json_quote_64bit_integers: 0
|
|
586
|
+
}
|
|
587
|
+
});
|
|
588
|
+
const rows = await result.json();
|
|
589
|
+
const threads = transformRows(rows.data);
|
|
590
|
+
return threads.map((thread) => ({
|
|
591
|
+
...thread,
|
|
592
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
593
|
+
createdAt: thread.createdAt,
|
|
594
|
+
updatedAt: thread.updatedAt
|
|
595
|
+
}));
|
|
596
|
+
} catch (error$1) {
|
|
597
|
+
throw new error.MastraError(
|
|
598
|
+
{
|
|
599
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
600
|
+
domain: error.ErrorDomain.STORAGE,
|
|
601
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
602
|
+
details: { resourceId }
|
|
603
|
+
},
|
|
604
|
+
error$1
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
async saveThread({ thread }) {
|
|
609
|
+
try {
|
|
610
|
+
await this.client.insert({
|
|
611
|
+
table: storage.TABLE_THREADS,
|
|
612
|
+
values: [
|
|
613
|
+
{
|
|
614
|
+
...thread,
|
|
615
|
+
createdAt: thread.createdAt.toISOString(),
|
|
616
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
617
|
+
}
|
|
618
|
+
],
|
|
122
619
|
format: "JSONEachRow",
|
|
123
620
|
clickhouse_settings: {
|
|
124
621
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -127,98 +624,793 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
127
624
|
output_format_json_quote_64bit_integers: 0
|
|
128
625
|
}
|
|
129
626
|
});
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
throw error
|
|
627
|
+
return thread;
|
|
628
|
+
} catch (error$1) {
|
|
629
|
+
throw new error.MastraError(
|
|
630
|
+
{
|
|
631
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
632
|
+
domain: error.ErrorDomain.STORAGE,
|
|
633
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
634
|
+
details: { threadId: thread.id }
|
|
635
|
+
},
|
|
636
|
+
error$1
|
|
637
|
+
);
|
|
133
638
|
}
|
|
134
639
|
}
|
|
135
|
-
async
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
perPage,
|
|
140
|
-
attributes,
|
|
141
|
-
filters,
|
|
142
|
-
fromDate,
|
|
143
|
-
toDate
|
|
640
|
+
async updateThread({
|
|
641
|
+
id,
|
|
642
|
+
title,
|
|
643
|
+
metadata
|
|
144
644
|
}) {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
645
|
+
try {
|
|
646
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
647
|
+
if (!existingThread) {
|
|
648
|
+
throw new Error(`Thread ${id} not found`);
|
|
649
|
+
}
|
|
650
|
+
const mergedMetadata = {
|
|
651
|
+
...existingThread.metadata,
|
|
652
|
+
...metadata
|
|
653
|
+
};
|
|
654
|
+
const updatedThread = {
|
|
655
|
+
...existingThread,
|
|
656
|
+
title,
|
|
657
|
+
metadata: mergedMetadata,
|
|
658
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
659
|
+
};
|
|
660
|
+
await this.client.insert({
|
|
661
|
+
table: storage.TABLE_THREADS,
|
|
662
|
+
format: "JSONEachRow",
|
|
663
|
+
values: [
|
|
664
|
+
{
|
|
665
|
+
id: updatedThread.id,
|
|
666
|
+
resourceId: updatedThread.resourceId,
|
|
667
|
+
title: updatedThread.title,
|
|
668
|
+
metadata: updatedThread.metadata,
|
|
669
|
+
createdAt: updatedThread.createdAt,
|
|
670
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
671
|
+
}
|
|
672
|
+
],
|
|
673
|
+
clickhouse_settings: {
|
|
674
|
+
date_time_input_format: "best_effort",
|
|
675
|
+
use_client_time_zone: 1,
|
|
676
|
+
output_format_json_quote_64bit_integers: 0
|
|
677
|
+
}
|
|
678
|
+
});
|
|
679
|
+
return updatedThread;
|
|
680
|
+
} catch (error$1) {
|
|
681
|
+
throw new error.MastraError(
|
|
682
|
+
{
|
|
683
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
684
|
+
domain: error.ErrorDomain.STORAGE,
|
|
685
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
686
|
+
details: { threadId: id, title }
|
|
687
|
+
},
|
|
688
|
+
error$1
|
|
689
|
+
);
|
|
152
690
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
691
|
+
}
|
|
692
|
+
async deleteThread({ threadId }) {
|
|
693
|
+
try {
|
|
694
|
+
await this.client.command({
|
|
695
|
+
query: `DELETE FROM "${storage.TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
696
|
+
query_params: { var_thread_id: threadId },
|
|
697
|
+
clickhouse_settings: {
|
|
698
|
+
output_format_json_quote_64bit_integers: 0
|
|
699
|
+
}
|
|
700
|
+
});
|
|
701
|
+
await this.client.command({
|
|
702
|
+
query: `DELETE FROM "${storage.TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
703
|
+
query_params: { var_id: threadId },
|
|
704
|
+
clickhouse_settings: {
|
|
705
|
+
output_format_json_quote_64bit_integers: 0
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
} catch (error$1) {
|
|
709
|
+
throw new error.MastraError(
|
|
710
|
+
{
|
|
711
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
712
|
+
domain: error.ErrorDomain.STORAGE,
|
|
713
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
714
|
+
details: { threadId }
|
|
715
|
+
},
|
|
716
|
+
error$1
|
|
717
|
+
);
|
|
156
718
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
719
|
+
}
|
|
720
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
721
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
722
|
+
try {
|
|
723
|
+
const currentOffset = page * perPage;
|
|
724
|
+
const countResult = await this.client.query({
|
|
725
|
+
query: `SELECT count() as total FROM ${storage.TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
726
|
+
query_params: { resourceId },
|
|
727
|
+
clickhouse_settings: {
|
|
728
|
+
date_time_input_format: "best_effort",
|
|
729
|
+
date_time_output_format: "iso",
|
|
730
|
+
use_client_time_zone: 1,
|
|
731
|
+
output_format_json_quote_64bit_integers: 0
|
|
732
|
+
}
|
|
733
|
+
});
|
|
734
|
+
const countData = await countResult.json();
|
|
735
|
+
const total = countData.data[0].total;
|
|
736
|
+
if (total === 0) {
|
|
737
|
+
return {
|
|
738
|
+
threads: [],
|
|
739
|
+
total: 0,
|
|
740
|
+
page,
|
|
741
|
+
perPage,
|
|
742
|
+
hasMore: false
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
const dataResult = await this.client.query({
|
|
746
|
+
query: `
|
|
747
|
+
SELECT
|
|
748
|
+
id,
|
|
749
|
+
resourceId,
|
|
750
|
+
title,
|
|
751
|
+
metadata,
|
|
752
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
753
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
754
|
+
FROM ${storage.TABLE_THREADS}
|
|
755
|
+
WHERE resourceId = {resourceId:String}
|
|
756
|
+
ORDER BY createdAt DESC
|
|
757
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
758
|
+
`,
|
|
759
|
+
query_params: {
|
|
760
|
+
resourceId,
|
|
761
|
+
limit: perPage,
|
|
762
|
+
offset: currentOffset
|
|
763
|
+
},
|
|
764
|
+
clickhouse_settings: {
|
|
765
|
+
date_time_input_format: "best_effort",
|
|
766
|
+
date_time_output_format: "iso",
|
|
767
|
+
use_client_time_zone: 1,
|
|
768
|
+
output_format_json_quote_64bit_integers: 0
|
|
769
|
+
}
|
|
161
770
|
});
|
|
771
|
+
const rows = await dataResult.json();
|
|
772
|
+
const threads = transformRows(rows.data);
|
|
773
|
+
return {
|
|
774
|
+
threads,
|
|
775
|
+
total,
|
|
776
|
+
page,
|
|
777
|
+
perPage,
|
|
778
|
+
hasMore: currentOffset + threads.length < total
|
|
779
|
+
};
|
|
780
|
+
} catch (error$1) {
|
|
781
|
+
throw new error.MastraError(
|
|
782
|
+
{
|
|
783
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
784
|
+
domain: error.ErrorDomain.STORAGE,
|
|
785
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
786
|
+
details: { resourceId, page }
|
|
787
|
+
},
|
|
788
|
+
error$1
|
|
789
|
+
);
|
|
162
790
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
791
|
+
}
|
|
792
|
+
async getMessagesPaginated(args) {
|
|
793
|
+
try {
|
|
794
|
+
const { threadId, selectBy, format = "v1" } = args;
|
|
795
|
+
const page = selectBy?.pagination?.page || 0;
|
|
796
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
797
|
+
const perPage = perPageInput !== void 0 ? perPageInput : storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
798
|
+
const offset = page * perPage;
|
|
799
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
800
|
+
const fromDate = dateRange?.start;
|
|
801
|
+
const toDate = dateRange?.end;
|
|
802
|
+
const messages = [];
|
|
803
|
+
if (selectBy?.include?.length) {
|
|
804
|
+
const include = selectBy.include;
|
|
805
|
+
const unionQueries = [];
|
|
806
|
+
const params = [];
|
|
807
|
+
let paramIdx = 1;
|
|
808
|
+
for (const inc of include) {
|
|
809
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
810
|
+
const searchId = inc.threadId || threadId;
|
|
811
|
+
unionQueries.push(`
|
|
812
|
+
SELECT * FROM (
|
|
813
|
+
WITH numbered_messages AS (
|
|
814
|
+
SELECT
|
|
815
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
816
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
817
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
818
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
819
|
+
),
|
|
820
|
+
target_positions AS (
|
|
821
|
+
SELECT row_num as target_pos
|
|
822
|
+
FROM numbered_messages
|
|
823
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
824
|
+
)
|
|
825
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
826
|
+
FROM numbered_messages m
|
|
827
|
+
CROSS JOIN target_positions t
|
|
828
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
829
|
+
) AS query_${paramIdx}
|
|
830
|
+
`);
|
|
831
|
+
params.push(
|
|
832
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
833
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
834
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
835
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
836
|
+
);
|
|
837
|
+
paramIdx++;
|
|
838
|
+
}
|
|
839
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
840
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
841
|
+
const includeResult = await this.client.query({
|
|
842
|
+
query: finalQuery,
|
|
843
|
+
query_params: mergedParams,
|
|
844
|
+
clickhouse_settings: {
|
|
845
|
+
date_time_input_format: "best_effort",
|
|
846
|
+
date_time_output_format: "iso",
|
|
847
|
+
use_client_time_zone: 1,
|
|
848
|
+
output_format_json_quote_64bit_integers: 0
|
|
849
|
+
}
|
|
850
|
+
});
|
|
851
|
+
const rows2 = await includeResult.json();
|
|
852
|
+
const includedMessages = transformRows(rows2.data);
|
|
853
|
+
const seen = /* @__PURE__ */ new Set();
|
|
854
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
855
|
+
if (seen.has(message.id)) return false;
|
|
856
|
+
seen.add(message.id);
|
|
857
|
+
return true;
|
|
858
|
+
});
|
|
859
|
+
messages.push(...dedupedMessages);
|
|
860
|
+
}
|
|
861
|
+
let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
862
|
+
const countParams = { threadId };
|
|
863
|
+
if (fromDate) {
|
|
864
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
865
|
+
countParams.fromDate = fromDate.toISOString();
|
|
866
|
+
}
|
|
867
|
+
if (toDate) {
|
|
868
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
869
|
+
countParams.toDate = toDate.toISOString();
|
|
870
|
+
}
|
|
871
|
+
const countResult = await this.client.query({
|
|
872
|
+
query: countQuery,
|
|
873
|
+
query_params: countParams,
|
|
874
|
+
clickhouse_settings: {
|
|
875
|
+
date_time_input_format: "best_effort",
|
|
876
|
+
date_time_output_format: "iso",
|
|
877
|
+
use_client_time_zone: 1,
|
|
878
|
+
output_format_json_quote_64bit_integers: 0
|
|
879
|
+
}
|
|
880
|
+
});
|
|
881
|
+
const countData = await countResult.json();
|
|
882
|
+
const total = countData.data[0].total;
|
|
883
|
+
if (total === 0 && messages.length === 0) {
|
|
884
|
+
return {
|
|
885
|
+
messages: [],
|
|
886
|
+
total: 0,
|
|
887
|
+
page,
|
|
888
|
+
perPage,
|
|
889
|
+
hasMore: false
|
|
890
|
+
};
|
|
891
|
+
}
|
|
892
|
+
const excludeIds = messages.map((m) => m.id);
|
|
893
|
+
let dataQuery = `
|
|
894
|
+
SELECT
|
|
895
|
+
id,
|
|
896
|
+
content,
|
|
897
|
+
role,
|
|
898
|
+
type,
|
|
899
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
900
|
+
thread_id AS "threadId",
|
|
901
|
+
resourceId
|
|
902
|
+
FROM ${storage.TABLE_MESSAGES}
|
|
903
|
+
WHERE thread_id = {threadId:String}
|
|
904
|
+
`;
|
|
905
|
+
const dataParams = { threadId };
|
|
906
|
+
if (fromDate) {
|
|
907
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
908
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
909
|
+
}
|
|
910
|
+
if (toDate) {
|
|
911
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
912
|
+
dataParams.toDate = toDate.toISOString();
|
|
913
|
+
}
|
|
914
|
+
if (excludeIds.length > 0) {
|
|
915
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
916
|
+
dataParams.excludeIds = excludeIds;
|
|
917
|
+
}
|
|
918
|
+
if (selectBy?.last) {
|
|
919
|
+
dataQuery += `
|
|
920
|
+
ORDER BY createdAt DESC
|
|
921
|
+
LIMIT {limit:Int64}
|
|
922
|
+
`;
|
|
923
|
+
dataParams.limit = perPage;
|
|
924
|
+
} else {
|
|
925
|
+
dataQuery += `
|
|
926
|
+
ORDER BY createdAt ASC
|
|
927
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
928
|
+
`;
|
|
929
|
+
dataParams.limit = perPage;
|
|
930
|
+
dataParams.offset = offset;
|
|
931
|
+
}
|
|
932
|
+
const result = await this.client.query({
|
|
933
|
+
query: dataQuery,
|
|
934
|
+
query_params: dataParams,
|
|
935
|
+
clickhouse_settings: {
|
|
936
|
+
date_time_input_format: "best_effort",
|
|
937
|
+
date_time_output_format: "iso",
|
|
938
|
+
use_client_time_zone: 1,
|
|
939
|
+
output_format_json_quote_64bit_integers: 0
|
|
940
|
+
}
|
|
941
|
+
});
|
|
942
|
+
const rows = await result.json();
|
|
943
|
+
const paginatedMessages = transformRows(rows.data);
|
|
944
|
+
messages.push(...paginatedMessages);
|
|
945
|
+
if (selectBy?.last) {
|
|
946
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
947
|
+
}
|
|
948
|
+
return {
|
|
949
|
+
messages: format === "v2" ? messages : messages,
|
|
950
|
+
total,
|
|
951
|
+
page,
|
|
952
|
+
perPage,
|
|
953
|
+
hasMore: offset + perPage < total
|
|
954
|
+
};
|
|
955
|
+
} catch (error$1) {
|
|
956
|
+
throw new error.MastraError(
|
|
957
|
+
{
|
|
958
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
959
|
+
domain: error.ErrorDomain.STORAGE,
|
|
960
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
961
|
+
},
|
|
962
|
+
error$1
|
|
963
|
+
);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
async updateMessages(args) {
|
|
967
|
+
const { messages } = args;
|
|
968
|
+
if (messages.length === 0) {
|
|
969
|
+
return [];
|
|
970
|
+
}
|
|
971
|
+
try {
|
|
972
|
+
const messageIds = messages.map((m) => m.id);
|
|
973
|
+
const existingResult = await this.client.query({
|
|
974
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
975
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
976
|
+
clickhouse_settings: {
|
|
977
|
+
date_time_input_format: "best_effort",
|
|
978
|
+
date_time_output_format: "iso",
|
|
979
|
+
use_client_time_zone: 1,
|
|
980
|
+
output_format_json_quote_64bit_integers: 0
|
|
981
|
+
}
|
|
982
|
+
});
|
|
983
|
+
const existingRows = await existingResult.json();
|
|
984
|
+
const existingMessages = transformRows(existingRows.data);
|
|
985
|
+
if (existingMessages.length === 0) {
|
|
986
|
+
return [];
|
|
987
|
+
}
|
|
988
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
989
|
+
if (typeof msg.content === "string") {
|
|
990
|
+
try {
|
|
991
|
+
msg.content = JSON.parse(msg.content);
|
|
992
|
+
} catch {
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
return msg;
|
|
996
|
+
});
|
|
997
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
998
|
+
const updatePromises = [];
|
|
999
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1000
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1001
|
+
if (!updatePayload) continue;
|
|
1002
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1003
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1004
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1005
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1006
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1007
|
+
}
|
|
1008
|
+
const setClauses = [];
|
|
1009
|
+
const values = {};
|
|
1010
|
+
let paramIdx = 1;
|
|
1011
|
+
let newContent = null;
|
|
1012
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1013
|
+
if (updatableFields.content) {
|
|
1014
|
+
const existingContent = existingMessage.content || {};
|
|
1015
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1016
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1017
|
+
newContent = {
|
|
1018
|
+
...existingContent,
|
|
1019
|
+
...updatableFields.content,
|
|
1020
|
+
// Deep merge metadata
|
|
1021
|
+
metadata: {
|
|
1022
|
+
...existingMetadata,
|
|
1023
|
+
...updateMetadata
|
|
1024
|
+
}
|
|
1025
|
+
};
|
|
1026
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1027
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1028
|
+
paramIdx++;
|
|
1029
|
+
delete updatableFields.content;
|
|
1030
|
+
}
|
|
1031
|
+
for (const key in updatableFields) {
|
|
1032
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1033
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1034
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1035
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1036
|
+
paramIdx++;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
if (setClauses.length > 0) {
|
|
1040
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1041
|
+
const updateQuery = `
|
|
1042
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
1043
|
+
UPDATE ${setClauses.join(", ")}
|
|
1044
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1045
|
+
`;
|
|
1046
|
+
console.log("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1047
|
+
updatePromises.push(
|
|
1048
|
+
this.client.command({
|
|
1049
|
+
query: updateQuery,
|
|
1050
|
+
query_params: values,
|
|
1051
|
+
clickhouse_settings: {
|
|
1052
|
+
date_time_input_format: "best_effort",
|
|
1053
|
+
use_client_time_zone: 1,
|
|
1054
|
+
output_format_json_quote_64bit_integers: 0
|
|
1055
|
+
}
|
|
1056
|
+
})
|
|
1057
|
+
);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
if (updatePromises.length > 0) {
|
|
1061
|
+
await Promise.all(updatePromises);
|
|
1062
|
+
}
|
|
1063
|
+
await this.client.command({
|
|
1064
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_MESSAGES} FINAL`,
|
|
1065
|
+
clickhouse_settings: {
|
|
1066
|
+
date_time_input_format: "best_effort",
|
|
1067
|
+
use_client_time_zone: 1,
|
|
1068
|
+
output_format_json_quote_64bit_integers: 0
|
|
1069
|
+
}
|
|
1070
|
+
});
|
|
1071
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1072
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1073
|
+
if (!updatePayload) continue;
|
|
1074
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1075
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1076
|
+
const verifyResult = await this.client.query({
|
|
1077
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1078
|
+
query_params: { messageId: id },
|
|
1079
|
+
clickhouse_settings: {
|
|
1080
|
+
date_time_input_format: "best_effort",
|
|
1081
|
+
date_time_output_format: "iso",
|
|
1082
|
+
use_client_time_zone: 1,
|
|
1083
|
+
output_format_json_quote_64bit_integers: 0
|
|
1084
|
+
}
|
|
1085
|
+
});
|
|
1086
|
+
const verifyRows = await verifyResult.json();
|
|
1087
|
+
if (verifyRows.data.length > 0) {
|
|
1088
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1089
|
+
if (updatedMessage) {
|
|
1090
|
+
let needsRetry = false;
|
|
1091
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1092
|
+
if (key === "content") {
|
|
1093
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1094
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1095
|
+
if (actualContent !== expectedContent) {
|
|
1096
|
+
needsRetry = true;
|
|
1097
|
+
break;
|
|
1098
|
+
}
|
|
1099
|
+
} else if (updatedMessage[key] !== value) {
|
|
1100
|
+
needsRetry = true;
|
|
1101
|
+
break;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
if (needsRetry) {
|
|
1105
|
+
console.log("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1106
|
+
await this.client.command({
|
|
1107
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1108
|
+
query_params: { messageId: id },
|
|
1109
|
+
clickhouse_settings: {
|
|
1110
|
+
date_time_input_format: "best_effort",
|
|
1111
|
+
use_client_time_zone: 1,
|
|
1112
|
+
output_format_json_quote_64bit_integers: 0
|
|
1113
|
+
}
|
|
1114
|
+
});
|
|
1115
|
+
let updatedContent = existingMessage.content || {};
|
|
1116
|
+
if (fieldsToUpdate.content) {
|
|
1117
|
+
const existingContent = existingMessage.content || {};
|
|
1118
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1119
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1120
|
+
updatedContent = {
|
|
1121
|
+
...existingContent,
|
|
1122
|
+
...fieldsToUpdate.content,
|
|
1123
|
+
metadata: {
|
|
1124
|
+
...existingMetadata,
|
|
1125
|
+
...updateMetadata
|
|
1126
|
+
}
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
const updatedMessageData = {
|
|
1130
|
+
...existingMessage,
|
|
1131
|
+
...fieldsToUpdate,
|
|
1132
|
+
content: updatedContent
|
|
1133
|
+
};
|
|
1134
|
+
await this.client.insert({
|
|
1135
|
+
table: storage.TABLE_MESSAGES,
|
|
1136
|
+
format: "JSONEachRow",
|
|
1137
|
+
values: [
|
|
1138
|
+
{
|
|
1139
|
+
id: updatedMessageData.id,
|
|
1140
|
+
thread_id: updatedMessageData.threadId,
|
|
1141
|
+
resourceId: updatedMessageData.resourceId,
|
|
1142
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1143
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1144
|
+
role: updatedMessageData.role,
|
|
1145
|
+
type: updatedMessageData.type || "v2"
|
|
1146
|
+
}
|
|
1147
|
+
],
|
|
1148
|
+
clickhouse_settings: {
|
|
1149
|
+
date_time_input_format: "best_effort",
|
|
1150
|
+
use_client_time_zone: 1,
|
|
1151
|
+
output_format_json_quote_64bit_integers: 0
|
|
1152
|
+
}
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1159
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1160
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1161
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1162
|
+
const threadResult = await this.client.query({
|
|
1163
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1164
|
+
query_params: { threadId },
|
|
1165
|
+
clickhouse_settings: {
|
|
1166
|
+
date_time_input_format: "best_effort",
|
|
1167
|
+
date_time_output_format: "iso",
|
|
1168
|
+
use_client_time_zone: 1,
|
|
1169
|
+
output_format_json_quote_64bit_integers: 0
|
|
1170
|
+
}
|
|
1171
|
+
});
|
|
1172
|
+
const threadRows = await threadResult.json();
|
|
1173
|
+
if (threadRows.data.length > 0) {
|
|
1174
|
+
const existingThread = threadRows.data[0];
|
|
1175
|
+
await this.client.command({
|
|
1176
|
+
query: `DELETE FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1177
|
+
query_params: { threadId },
|
|
1178
|
+
clickhouse_settings: {
|
|
1179
|
+
date_time_input_format: "best_effort",
|
|
1180
|
+
use_client_time_zone: 1,
|
|
1181
|
+
output_format_json_quote_64bit_integers: 0
|
|
1182
|
+
}
|
|
1183
|
+
});
|
|
1184
|
+
await this.client.insert({
|
|
1185
|
+
table: storage.TABLE_THREADS,
|
|
1186
|
+
format: "JSONEachRow",
|
|
1187
|
+
values: [
|
|
1188
|
+
{
|
|
1189
|
+
id: existingThread.id,
|
|
1190
|
+
resourceId: existingThread.resourceId,
|
|
1191
|
+
title: existingThread.title,
|
|
1192
|
+
metadata: existingThread.metadata,
|
|
1193
|
+
createdAt: existingThread.createdAt,
|
|
1194
|
+
updatedAt: now
|
|
1195
|
+
}
|
|
1196
|
+
],
|
|
1197
|
+
clickhouse_settings: {
|
|
1198
|
+
date_time_input_format: "best_effort",
|
|
1199
|
+
use_client_time_zone: 1,
|
|
1200
|
+
output_format_json_quote_64bit_integers: 0
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
});
|
|
1205
|
+
await Promise.all(threadUpdatePromises);
|
|
1206
|
+
}
|
|
1207
|
+
const updatedMessages = [];
|
|
1208
|
+
for (const messageId of messageIds) {
|
|
1209
|
+
const updatedResult = await this.client.query({
|
|
1210
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1211
|
+
query_params: { messageId },
|
|
1212
|
+
clickhouse_settings: {
|
|
1213
|
+
date_time_input_format: "best_effort",
|
|
1214
|
+
date_time_output_format: "iso",
|
|
1215
|
+
use_client_time_zone: 1,
|
|
1216
|
+
output_format_json_quote_64bit_integers: 0
|
|
1217
|
+
}
|
|
1218
|
+
});
|
|
1219
|
+
const updatedRows = await updatedResult.json();
|
|
1220
|
+
if (updatedRows.data.length > 0) {
|
|
1221
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1222
|
+
if (message) {
|
|
1223
|
+
updatedMessages.push(message);
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
return updatedMessages.map((message) => {
|
|
1228
|
+
if (typeof message.content === "string") {
|
|
1229
|
+
try {
|
|
1230
|
+
message.content = JSON.parse(message.content);
|
|
1231
|
+
} catch {
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
return message;
|
|
1235
|
+
});
|
|
1236
|
+
} catch (error$1) {
|
|
1237
|
+
throw new error.MastraError(
|
|
1238
|
+
{
|
|
1239
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1240
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1241
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1242
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1243
|
+
},
|
|
1244
|
+
error$1
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
async getResourceById({ resourceId }) {
|
|
1249
|
+
try {
|
|
1250
|
+
const result = await this.client.query({
|
|
1251
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${storage.TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1252
|
+
query_params: { resourceId },
|
|
1253
|
+
clickhouse_settings: {
|
|
1254
|
+
date_time_input_format: "best_effort",
|
|
1255
|
+
date_time_output_format: "iso",
|
|
1256
|
+
use_client_time_zone: 1,
|
|
1257
|
+
output_format_json_quote_64bit_integers: 0
|
|
1258
|
+
}
|
|
1259
|
+
});
|
|
1260
|
+
const rows = await result.json();
|
|
1261
|
+
if (rows.data.length === 0) {
|
|
1262
|
+
return null;
|
|
1263
|
+
}
|
|
1264
|
+
const resource = rows.data[0];
|
|
1265
|
+
return {
|
|
1266
|
+
id: resource.id,
|
|
1267
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1268
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1269
|
+
createdAt: new Date(resource.createdAt),
|
|
1270
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1271
|
+
};
|
|
1272
|
+
} catch (error$1) {
|
|
1273
|
+
throw new error.MastraError(
|
|
1274
|
+
{
|
|
1275
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1276
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1277
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1278
|
+
details: { resourceId }
|
|
1279
|
+
},
|
|
1280
|
+
error$1
|
|
1281
|
+
);
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
async saveResource({ resource }) {
|
|
1285
|
+
try {
|
|
1286
|
+
await this.client.insert({
|
|
1287
|
+
table: storage.TABLE_RESOURCES,
|
|
1288
|
+
format: "JSONEachRow",
|
|
1289
|
+
values: [
|
|
1290
|
+
{
|
|
1291
|
+
id: resource.id,
|
|
1292
|
+
workingMemory: resource.workingMemory,
|
|
1293
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1294
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1295
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1296
|
+
}
|
|
1297
|
+
],
|
|
1298
|
+
clickhouse_settings: {
|
|
1299
|
+
date_time_input_format: "best_effort",
|
|
1300
|
+
use_client_time_zone: 1,
|
|
1301
|
+
output_format_json_quote_64bit_integers: 0
|
|
1302
|
+
}
|
|
169
1303
|
});
|
|
1304
|
+
return resource;
|
|
1305
|
+
} catch (error$1) {
|
|
1306
|
+
throw new error.MastraError(
|
|
1307
|
+
{
|
|
1308
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1309
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1310
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1311
|
+
details: { resourceId: resource.id }
|
|
1312
|
+
},
|
|
1313
|
+
error$1
|
|
1314
|
+
);
|
|
170
1315
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
output_format_json_quote_64bit_integers: 0
|
|
1316
|
+
}
|
|
1317
|
+
async updateResource({
|
|
1318
|
+
resourceId,
|
|
1319
|
+
workingMemory,
|
|
1320
|
+
metadata
|
|
1321
|
+
}) {
|
|
1322
|
+
try {
|
|
1323
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1324
|
+
if (!existingResource) {
|
|
1325
|
+
const newResource = {
|
|
1326
|
+
id: resourceId,
|
|
1327
|
+
workingMemory,
|
|
1328
|
+
metadata: metadata || {},
|
|
1329
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1330
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1331
|
+
};
|
|
1332
|
+
return this.saveResource({ resource: newResource });
|
|
189
1333
|
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
1334
|
+
const updatedResource = {
|
|
1335
|
+
...existingResource,
|
|
1336
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1337
|
+
metadata: {
|
|
1338
|
+
...existingResource.metadata,
|
|
1339
|
+
...metadata
|
|
1340
|
+
},
|
|
1341
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1342
|
+
};
|
|
1343
|
+
const updateQuery = `
|
|
1344
|
+
ALTER TABLE ${storage.TABLE_RESOURCES}
|
|
1345
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1346
|
+
WHERE id = {resourceId:String}
|
|
1347
|
+
`;
|
|
1348
|
+
await this.client.command({
|
|
1349
|
+
query: updateQuery,
|
|
1350
|
+
query_params: {
|
|
1351
|
+
workingMemory: updatedResource.workingMemory,
|
|
1352
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1353
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1354
|
+
resourceId
|
|
1355
|
+
},
|
|
1356
|
+
clickhouse_settings: {
|
|
1357
|
+
date_time_input_format: "best_effort",
|
|
1358
|
+
use_client_time_zone: 1,
|
|
1359
|
+
output_format_json_quote_64bit_integers: 0
|
|
1360
|
+
}
|
|
1361
|
+
});
|
|
1362
|
+
await this.client.command({
|
|
1363
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_RESOURCES} FINAL`,
|
|
1364
|
+
clickhouse_settings: {
|
|
1365
|
+
date_time_input_format: "best_effort",
|
|
1366
|
+
use_client_time_zone: 1,
|
|
1367
|
+
output_format_json_quote_64bit_integers: 0
|
|
1368
|
+
}
|
|
1369
|
+
});
|
|
1370
|
+
return updatedResource;
|
|
1371
|
+
} catch (error$1) {
|
|
1372
|
+
throw new error.MastraError(
|
|
1373
|
+
{
|
|
1374
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1375
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1376
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1377
|
+
details: { resourceId }
|
|
1378
|
+
},
|
|
1379
|
+
error$1
|
|
1380
|
+
);
|
|
193
1381
|
}
|
|
194
|
-
const resp = await result.json();
|
|
195
|
-
const rows = resp.data;
|
|
196
|
-
return rows.map((row) => ({
|
|
197
|
-
id: row.id,
|
|
198
|
-
parentSpanId: row.parentSpanId,
|
|
199
|
-
traceId: row.traceId,
|
|
200
|
-
name: row.name,
|
|
201
|
-
scope: row.scope,
|
|
202
|
-
kind: row.kind,
|
|
203
|
-
status: safelyParseJSON(row.status),
|
|
204
|
-
events: safelyParseJSON(row.events),
|
|
205
|
-
links: safelyParseJSON(row.links),
|
|
206
|
-
attributes: safelyParseJSON(row.attributes),
|
|
207
|
-
startTime: row.startTime,
|
|
208
|
-
endTime: row.endTime,
|
|
209
|
-
other: safelyParseJSON(row.other),
|
|
210
|
-
createdAt: row.createdAt
|
|
211
|
-
}));
|
|
212
1382
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
1383
|
+
};
|
|
1384
|
+
var StoreOperationsClickhouse = class extends storage.StoreOperations {
|
|
1385
|
+
ttl;
|
|
1386
|
+
client;
|
|
1387
|
+
constructor({ client, ttl }) {
|
|
1388
|
+
super();
|
|
1389
|
+
this.ttl = ttl;
|
|
1390
|
+
this.client = client;
|
|
217
1391
|
}
|
|
218
|
-
async
|
|
219
|
-
await this.
|
|
220
|
-
query: `
|
|
1392
|
+
async hasColumn(table, column) {
|
|
1393
|
+
const result = await this.client.query({
|
|
1394
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1395
|
+
format: "JSONEachRow"
|
|
221
1396
|
});
|
|
1397
|
+
const columns = await result.json();
|
|
1398
|
+
return columns.some((c) => c.name === column);
|
|
1399
|
+
}
|
|
1400
|
+
getSqlType(type) {
|
|
1401
|
+
switch (type) {
|
|
1402
|
+
case "text":
|
|
1403
|
+
return "String";
|
|
1404
|
+
case "timestamp":
|
|
1405
|
+
return "DateTime64(3)";
|
|
1406
|
+
case "integer":
|
|
1407
|
+
case "bigint":
|
|
1408
|
+
return "Int64";
|
|
1409
|
+
case "jsonb":
|
|
1410
|
+
return "String";
|
|
1411
|
+
default:
|
|
1412
|
+
return super.getSqlType(type);
|
|
1413
|
+
}
|
|
222
1414
|
}
|
|
223
1415
|
async createTable({
|
|
224
1416
|
tableName,
|
|
@@ -233,27 +1425,25 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
233
1425
|
}).join(",\n");
|
|
234
1426
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
235
1427
|
const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
`;
|
|
256
|
-
await this.db.query({
|
|
1428
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1429
|
+
${["id String"].concat(columns)}
|
|
1430
|
+
)
|
|
1431
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1432
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1433
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1434
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1435
|
+
SETTINGS index_granularity = 8192
|
|
1436
|
+
` : `
|
|
1437
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1438
|
+
${columns}
|
|
1439
|
+
)
|
|
1440
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1441
|
+
PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1442
|
+
ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1443
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1444
|
+
SETTINGS index_granularity = 8192
|
|
1445
|
+
`;
|
|
1446
|
+
await this.client.query({
|
|
257
1447
|
query: sql,
|
|
258
1448
|
clickhouse_settings: {
|
|
259
1449
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -263,14 +1453,60 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
263
1453
|
output_format_json_quote_64bit_integers: 0
|
|
264
1454
|
}
|
|
265
1455
|
});
|
|
266
|
-
} catch (error) {
|
|
267
|
-
|
|
268
|
-
|
|
1456
|
+
} catch (error$1) {
|
|
1457
|
+
throw new error.MastraError(
|
|
1458
|
+
{
|
|
1459
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1460
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1461
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1462
|
+
details: { tableName }
|
|
1463
|
+
},
|
|
1464
|
+
error$1
|
|
1465
|
+
);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
async alterTable({
|
|
1469
|
+
tableName,
|
|
1470
|
+
schema,
|
|
1471
|
+
ifNotExists
|
|
1472
|
+
}) {
|
|
1473
|
+
try {
|
|
1474
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1475
|
+
const result = await this.client.query({
|
|
1476
|
+
query: describeSql
|
|
1477
|
+
});
|
|
1478
|
+
const rows = await result.json();
|
|
1479
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1480
|
+
for (const columnName of ifNotExists) {
|
|
1481
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1482
|
+
const columnDef = schema[columnName];
|
|
1483
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1484
|
+
if (columnDef.nullable !== false) {
|
|
1485
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1486
|
+
}
|
|
1487
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1488
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1489
|
+
await this.client.query({
|
|
1490
|
+
query: alterSql
|
|
1491
|
+
});
|
|
1492
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
} catch (error$1) {
|
|
1496
|
+
throw new error.MastraError(
|
|
1497
|
+
{
|
|
1498
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1499
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1500
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1501
|
+
details: { tableName }
|
|
1502
|
+
},
|
|
1503
|
+
error$1
|
|
1504
|
+
);
|
|
269
1505
|
}
|
|
270
1506
|
}
|
|
271
1507
|
async clearTable({ tableName }) {
|
|
272
1508
|
try {
|
|
273
|
-
await this.
|
|
1509
|
+
await this.client.query({
|
|
274
1510
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
275
1511
|
clickhouse_settings: {
|
|
276
1512
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -280,20 +1516,34 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
280
1516
|
output_format_json_quote_64bit_integers: 0
|
|
281
1517
|
}
|
|
282
1518
|
});
|
|
283
|
-
} catch (error) {
|
|
284
|
-
|
|
285
|
-
|
|
1519
|
+
} catch (error$1) {
|
|
1520
|
+
throw new error.MastraError(
|
|
1521
|
+
{
|
|
1522
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1523
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1524
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1525
|
+
details: { tableName }
|
|
1526
|
+
},
|
|
1527
|
+
error$1
|
|
1528
|
+
);
|
|
286
1529
|
}
|
|
287
1530
|
}
|
|
1531
|
+
async dropTable({ tableName }) {
|
|
1532
|
+
await this.client.query({
|
|
1533
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1534
|
+
});
|
|
1535
|
+
}
|
|
288
1536
|
async insert({ tableName, record }) {
|
|
1537
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1538
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
289
1539
|
try {
|
|
290
|
-
await this.
|
|
1540
|
+
const result = await this.client.insert({
|
|
291
1541
|
table: tableName,
|
|
292
1542
|
values: [
|
|
293
1543
|
{
|
|
294
1544
|
...record,
|
|
295
|
-
createdAt
|
|
296
|
-
updatedAt
|
|
1545
|
+
createdAt,
|
|
1546
|
+
updatedAt
|
|
297
1547
|
}
|
|
298
1548
|
],
|
|
299
1549
|
format: "JSONEachRow",
|
|
@@ -304,13 +1554,55 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
304
1554
|
use_client_time_zone: 1
|
|
305
1555
|
}
|
|
306
1556
|
});
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
throw error
|
|
1557
|
+
console.log("INSERT RESULT", result);
|
|
1558
|
+
} catch (error$1) {
|
|
1559
|
+
throw new error.MastraError(
|
|
1560
|
+
{
|
|
1561
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1562
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1563
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1564
|
+
details: { tableName }
|
|
1565
|
+
},
|
|
1566
|
+
error$1
|
|
1567
|
+
);
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
async batchInsert({ tableName, records }) {
|
|
1571
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1572
|
+
...Object.fromEntries(
|
|
1573
|
+
Object.entries(record).map(([key, value]) => [
|
|
1574
|
+
key,
|
|
1575
|
+
storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1576
|
+
])
|
|
1577
|
+
)
|
|
1578
|
+
}));
|
|
1579
|
+
try {
|
|
1580
|
+
await this.client.insert({
|
|
1581
|
+
table: tableName,
|
|
1582
|
+
values: recordsToBeInserted,
|
|
1583
|
+
format: "JSONEachRow",
|
|
1584
|
+
clickhouse_settings: {
|
|
1585
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1586
|
+
date_time_input_format: "best_effort",
|
|
1587
|
+
use_client_time_zone: 1,
|
|
1588
|
+
output_format_json_quote_64bit_integers: 0
|
|
1589
|
+
}
|
|
1590
|
+
});
|
|
1591
|
+
} catch (error$1) {
|
|
1592
|
+
throw new error.MastraError(
|
|
1593
|
+
{
|
|
1594
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1595
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1596
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1597
|
+
details: { tableName }
|
|
1598
|
+
},
|
|
1599
|
+
error$1
|
|
1600
|
+
);
|
|
310
1601
|
}
|
|
311
1602
|
}
|
|
312
1603
|
async load({ tableName, keys }) {
|
|
313
1604
|
try {
|
|
1605
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
314
1606
|
const keyEntries = Object.entries(keys);
|
|
315
1607
|
const conditions = keyEntries.map(
|
|
316
1608
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -318,8 +1610,10 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
318
1610
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
319
1611
|
return { ...acc, [`var_${key}`]: value };
|
|
320
1612
|
}, {});
|
|
321
|
-
const
|
|
322
|
-
|
|
1613
|
+
const hasUpdatedAt = storage.TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1614
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1615
|
+
const result = await this.client.query({
|
|
1616
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
323
1617
|
query_params: values,
|
|
324
1618
|
clickhouse_settings: {
|
|
325
1619
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -345,25 +1639,58 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
345
1639
|
}
|
|
346
1640
|
const data = transformRow(rows.data[0]);
|
|
347
1641
|
return data;
|
|
348
|
-
} catch (error) {
|
|
349
|
-
|
|
350
|
-
|
|
1642
|
+
} catch (error$1) {
|
|
1643
|
+
throw new error.MastraError(
|
|
1644
|
+
{
|
|
1645
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1646
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1647
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1648
|
+
details: { tableName }
|
|
1649
|
+
},
|
|
1650
|
+
error$1
|
|
1651
|
+
);
|
|
351
1652
|
}
|
|
352
1653
|
}
|
|
353
|
-
|
|
1654
|
+
};
|
|
1655
|
+
var ScoresStorageClickhouse = class extends storage.ScoresStorage {
|
|
1656
|
+
client;
|
|
1657
|
+
operations;
|
|
1658
|
+
constructor({ client, operations }) {
|
|
1659
|
+
super();
|
|
1660
|
+
this.client = client;
|
|
1661
|
+
this.operations = operations;
|
|
1662
|
+
}
|
|
1663
|
+
transformScoreRow(row) {
|
|
1664
|
+
const scorer = storage.safelyParseJSON(row.scorer);
|
|
1665
|
+
const extractStepResult = storage.safelyParseJSON(row.extractStepResult);
|
|
1666
|
+
const analyzeStepResult = storage.safelyParseJSON(row.analyzeStepResult);
|
|
1667
|
+
const metadata = storage.safelyParseJSON(row.metadata);
|
|
1668
|
+
const input = storage.safelyParseJSON(row.input);
|
|
1669
|
+
const output = storage.safelyParseJSON(row.output);
|
|
1670
|
+
const additionalContext = storage.safelyParseJSON(row.additionalContext);
|
|
1671
|
+
const runtimeContext = storage.safelyParseJSON(row.runtimeContext);
|
|
1672
|
+
const entity = storage.safelyParseJSON(row.entity);
|
|
1673
|
+
return {
|
|
1674
|
+
...row,
|
|
1675
|
+
scorer,
|
|
1676
|
+
extractStepResult,
|
|
1677
|
+
analyzeStepResult,
|
|
1678
|
+
metadata,
|
|
1679
|
+
input,
|
|
1680
|
+
output,
|
|
1681
|
+
additionalContext,
|
|
1682
|
+
runtimeContext,
|
|
1683
|
+
entity,
|
|
1684
|
+
createdAt: new Date(row.createdAt),
|
|
1685
|
+
updatedAt: new Date(row.updatedAt)
|
|
1686
|
+
};
|
|
1687
|
+
}
|
|
1688
|
+
async getScoreById({ id }) {
|
|
354
1689
|
try {
|
|
355
|
-
const result = await this.
|
|
356
|
-
query: `SELECT
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
title,
|
|
360
|
-
metadata,
|
|
361
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
362
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
363
|
-
FROM "${storage.TABLE_THREADS}"
|
|
364
|
-
FINAL
|
|
365
|
-
WHERE id = {var_id:String}`,
|
|
366
|
-
query_params: { var_id: threadId },
|
|
1690
|
+
const result = await this.client.query({
|
|
1691
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1692
|
+
query_params: { var_id: id },
|
|
1693
|
+
format: "JSONEachRow",
|
|
367
1694
|
clickhouse_settings: {
|
|
368
1695
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
369
1696
|
date_time_input_format: "best_effort",
|
|
@@ -372,288 +1699,510 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
372
1699
|
output_format_json_quote_64bit_integers: 0
|
|
373
1700
|
}
|
|
374
1701
|
});
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
if (!thread) {
|
|
1702
|
+
const resultJson = await result.json();
|
|
1703
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
378
1704
|
return null;
|
|
379
1705
|
}
|
|
380
|
-
return
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
1706
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1707
|
+
} catch (error$1) {
|
|
1708
|
+
throw new error.MastraError(
|
|
1709
|
+
{
|
|
1710
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1711
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1712
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1713
|
+
details: { scoreId: id }
|
|
1714
|
+
},
|
|
1715
|
+
error$1
|
|
1716
|
+
);
|
|
389
1717
|
}
|
|
390
1718
|
}
|
|
391
|
-
async
|
|
1719
|
+
async saveScore(score) {
|
|
392
1720
|
try {
|
|
393
|
-
const
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
401
|
-
FROM "${storage.TABLE_THREADS}"
|
|
402
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
403
|
-
query_params: { var_resourceId: resourceId },
|
|
1721
|
+
const record = {
|
|
1722
|
+
...score
|
|
1723
|
+
};
|
|
1724
|
+
await this.client.insert({
|
|
1725
|
+
table: storage.TABLE_SCORERS,
|
|
1726
|
+
values: [record],
|
|
1727
|
+
format: "JSONEachRow",
|
|
404
1728
|
clickhouse_settings: {
|
|
405
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
406
1729
|
date_time_input_format: "best_effort",
|
|
407
|
-
date_time_output_format: "iso",
|
|
408
1730
|
use_client_time_zone: 1,
|
|
409
1731
|
output_format_json_quote_64bit_integers: 0
|
|
410
1732
|
}
|
|
411
1733
|
});
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
1734
|
+
return { score };
|
|
1735
|
+
} catch (error$1) {
|
|
1736
|
+
throw new error.MastraError(
|
|
1737
|
+
{
|
|
1738
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1739
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1740
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1741
|
+
details: { scoreId: score.id }
|
|
1742
|
+
},
|
|
1743
|
+
error$1
|
|
1744
|
+
);
|
|
423
1745
|
}
|
|
424
1746
|
}
|
|
425
|
-
async
|
|
1747
|
+
async getScoresByRunId({
|
|
1748
|
+
runId,
|
|
1749
|
+
pagination
|
|
1750
|
+
}) {
|
|
426
1751
|
try {
|
|
427
|
-
await this.
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
]
|
|
1752
|
+
const countResult = await this.client.query({
|
|
1753
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1754
|
+
query_params: { var_runId: runId },
|
|
1755
|
+
format: "JSONEachRow"
|
|
1756
|
+
});
|
|
1757
|
+
const countRows = await countResult.json();
|
|
1758
|
+
let total = 0;
|
|
1759
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1760
|
+
const countObj = countRows[0];
|
|
1761
|
+
total = Number(countObj.count);
|
|
1762
|
+
}
|
|
1763
|
+
if (!total) {
|
|
1764
|
+
return {
|
|
1765
|
+
pagination: {
|
|
1766
|
+
total: 0,
|
|
1767
|
+
page: pagination.page,
|
|
1768
|
+
perPage: pagination.perPage,
|
|
1769
|
+
hasMore: false
|
|
1770
|
+
},
|
|
1771
|
+
scores: []
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
const offset = pagination.page * pagination.perPage;
|
|
1775
|
+
const result = await this.client.query({
|
|
1776
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1777
|
+
query_params: {
|
|
1778
|
+
var_runId: runId,
|
|
1779
|
+
var_limit: pagination.perPage,
|
|
1780
|
+
var_offset: offset
|
|
1781
|
+
},
|
|
436
1782
|
format: "JSONEachRow",
|
|
437
1783
|
clickhouse_settings: {
|
|
438
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
439
1784
|
date_time_input_format: "best_effort",
|
|
1785
|
+
date_time_output_format: "iso",
|
|
440
1786
|
use_client_time_zone: 1,
|
|
441
1787
|
output_format_json_quote_64bit_integers: 0
|
|
442
1788
|
}
|
|
443
1789
|
});
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
1790
|
+
const rows = await result.json();
|
|
1791
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1792
|
+
return {
|
|
1793
|
+
pagination: {
|
|
1794
|
+
total,
|
|
1795
|
+
page: pagination.page,
|
|
1796
|
+
perPage: pagination.perPage,
|
|
1797
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1798
|
+
},
|
|
1799
|
+
scores
|
|
1800
|
+
};
|
|
1801
|
+
} catch (error$1) {
|
|
1802
|
+
throw new error.MastraError(
|
|
1803
|
+
{
|
|
1804
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1805
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1806
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1807
|
+
details: { runId }
|
|
1808
|
+
},
|
|
1809
|
+
error$1
|
|
1810
|
+
);
|
|
448
1811
|
}
|
|
449
1812
|
}
|
|
450
|
-
async
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
metadata
|
|
1813
|
+
async getScoresByScorerId({
|
|
1814
|
+
scorerId,
|
|
1815
|
+
pagination
|
|
454
1816
|
}) {
|
|
455
1817
|
try {
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
|
|
1818
|
+
const countResult = await this.client.query({
|
|
1819
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE scorerId = {var_scorerId:String}`,
|
|
1820
|
+
query_params: { var_scorerId: scorerId },
|
|
1821
|
+
format: "JSONEachRow"
|
|
1822
|
+
});
|
|
1823
|
+
const countRows = await countResult.json();
|
|
1824
|
+
let total = 0;
|
|
1825
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1826
|
+
const countObj = countRows[0];
|
|
1827
|
+
total = Number(countObj.count);
|
|
459
1828
|
}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
1829
|
+
if (!total) {
|
|
1830
|
+
return {
|
|
1831
|
+
pagination: {
|
|
1832
|
+
total: 0,
|
|
1833
|
+
page: pagination.page,
|
|
1834
|
+
perPage: pagination.perPage,
|
|
1835
|
+
hasMore: false
|
|
1836
|
+
},
|
|
1837
|
+
scores: []
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
const offset = pagination.page * pagination.perPage;
|
|
1841
|
+
const result = await this.client.query({
|
|
1842
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE scorerId = {var_scorerId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1843
|
+
query_params: {
|
|
1844
|
+
var_scorerId: scorerId,
|
|
1845
|
+
var_limit: pagination.perPage,
|
|
1846
|
+
var_offset: offset
|
|
1847
|
+
},
|
|
478
1848
|
format: "JSONEachRow",
|
|
479
1849
|
clickhouse_settings: {
|
|
480
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
481
1850
|
date_time_input_format: "best_effort",
|
|
1851
|
+
date_time_output_format: "iso",
|
|
482
1852
|
use_client_time_zone: 1,
|
|
483
1853
|
output_format_json_quote_64bit_integers: 0
|
|
484
1854
|
}
|
|
485
1855
|
});
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
1856
|
+
const rows = await result.json();
|
|
1857
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1858
|
+
return {
|
|
1859
|
+
pagination: {
|
|
1860
|
+
total,
|
|
1861
|
+
page: pagination.page,
|
|
1862
|
+
perPage: pagination.perPage,
|
|
1863
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1864
|
+
},
|
|
1865
|
+
scores
|
|
1866
|
+
};
|
|
1867
|
+
} catch (error$1) {
|
|
1868
|
+
throw new error.MastraError(
|
|
1869
|
+
{
|
|
1870
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1871
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1872
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1873
|
+
details: { scorerId }
|
|
1874
|
+
},
|
|
1875
|
+
error$1
|
|
1876
|
+
);
|
|
490
1877
|
}
|
|
491
1878
|
}
|
|
492
|
-
async
|
|
1879
|
+
async getScoresByEntityId({
|
|
1880
|
+
entityId,
|
|
1881
|
+
entityType,
|
|
1882
|
+
pagination
|
|
1883
|
+
}) {
|
|
493
1884
|
try {
|
|
494
|
-
await this.
|
|
495
|
-
query: `
|
|
496
|
-
query_params: {
|
|
1885
|
+
const countResult = await this.client.query({
|
|
1886
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1887
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1888
|
+
format: "JSONEachRow"
|
|
1889
|
+
});
|
|
1890
|
+
const countRows = await countResult.json();
|
|
1891
|
+
let total = 0;
|
|
1892
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1893
|
+
const countObj = countRows[0];
|
|
1894
|
+
total = Number(countObj.count);
|
|
1895
|
+
}
|
|
1896
|
+
if (!total) {
|
|
1897
|
+
return {
|
|
1898
|
+
pagination: {
|
|
1899
|
+
total: 0,
|
|
1900
|
+
page: pagination.page,
|
|
1901
|
+
perPage: pagination.perPage,
|
|
1902
|
+
hasMore: false
|
|
1903
|
+
},
|
|
1904
|
+
scores: []
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
const offset = pagination.page * pagination.perPage;
|
|
1908
|
+
const result = await this.client.query({
|
|
1909
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1910
|
+
query_params: {
|
|
1911
|
+
var_entityId: entityId,
|
|
1912
|
+
var_entityType: entityType,
|
|
1913
|
+
var_limit: pagination.perPage,
|
|
1914
|
+
var_offset: offset
|
|
1915
|
+
},
|
|
1916
|
+
format: "JSONEachRow",
|
|
497
1917
|
clickhouse_settings: {
|
|
1918
|
+
date_time_input_format: "best_effort",
|
|
1919
|
+
date_time_output_format: "iso",
|
|
1920
|
+
use_client_time_zone: 1,
|
|
498
1921
|
output_format_json_quote_64bit_integers: 0
|
|
499
1922
|
}
|
|
500
1923
|
});
|
|
501
|
-
await
|
|
502
|
-
|
|
503
|
-
|
|
1924
|
+
const rows = await result.json();
|
|
1925
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1926
|
+
return {
|
|
1927
|
+
pagination: {
|
|
1928
|
+
total,
|
|
1929
|
+
page: pagination.page,
|
|
1930
|
+
perPage: pagination.perPage,
|
|
1931
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1932
|
+
},
|
|
1933
|
+
scores
|
|
1934
|
+
};
|
|
1935
|
+
} catch (error$1) {
|
|
1936
|
+
throw new error.MastraError(
|
|
1937
|
+
{
|
|
1938
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1939
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1940
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1941
|
+
details: { entityId, entityType }
|
|
1942
|
+
},
|
|
1943
|
+
error$1
|
|
1944
|
+
);
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
};
|
|
1948
|
+
var TracesStorageClickhouse = class extends storage.TracesStorage {
|
|
1949
|
+
client;
|
|
1950
|
+
operations;
|
|
1951
|
+
constructor({ client, operations }) {
|
|
1952
|
+
super();
|
|
1953
|
+
this.client = client;
|
|
1954
|
+
this.operations = operations;
|
|
1955
|
+
}
|
|
1956
|
+
async getTracesPaginated(args) {
|
|
1957
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
1958
|
+
const fromDate = dateRange?.start;
|
|
1959
|
+
const toDate = dateRange?.end;
|
|
1960
|
+
const currentOffset = page * perPage;
|
|
1961
|
+
const queryArgs = {};
|
|
1962
|
+
const conditions = [];
|
|
1963
|
+
if (name) {
|
|
1964
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
1965
|
+
queryArgs.var_name = name;
|
|
1966
|
+
}
|
|
1967
|
+
if (scope) {
|
|
1968
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
1969
|
+
queryArgs.var_scope = scope;
|
|
1970
|
+
}
|
|
1971
|
+
if (attributes) {
|
|
1972
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
1973
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
1974
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
1975
|
+
});
|
|
1976
|
+
}
|
|
1977
|
+
if (filters) {
|
|
1978
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
1979
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
1980
|
+
queryArgs[`var_col_${key}`] = value;
|
|
1981
|
+
});
|
|
1982
|
+
}
|
|
1983
|
+
if (fromDate) {
|
|
1984
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
1985
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
1986
|
+
}
|
|
1987
|
+
if (toDate) {
|
|
1988
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
1989
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
1990
|
+
}
|
|
1991
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1992
|
+
try {
|
|
1993
|
+
const countResult = await this.client.query({
|
|
1994
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_TRACES} ${whereClause}`,
|
|
1995
|
+
query_params: queryArgs,
|
|
504
1996
|
clickhouse_settings: {
|
|
1997
|
+
date_time_input_format: "best_effort",
|
|
1998
|
+
date_time_output_format: "iso",
|
|
1999
|
+
use_client_time_zone: 1,
|
|
505
2000
|
output_format_json_quote_64bit_integers: 0
|
|
506
2001
|
}
|
|
507
2002
|
});
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
if (include.length) {
|
|
519
|
-
const includeResult = await this.db.query({
|
|
520
|
-
query: `
|
|
521
|
-
WITH ordered_messages AS (
|
|
522
|
-
SELECT
|
|
523
|
-
*,
|
|
524
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
525
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
526
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
527
|
-
FROM "${storage.TABLE_MESSAGES}"
|
|
528
|
-
WHERE thread_id = {var_thread_id:String}
|
|
529
|
-
)
|
|
530
|
-
SELECT
|
|
531
|
-
m.id AS id,
|
|
532
|
-
m.content as content,
|
|
533
|
-
m.role as role,
|
|
534
|
-
m.type as type,
|
|
535
|
-
m.createdAt as createdAt,
|
|
536
|
-
m.updatedAt as updatedAt,
|
|
537
|
-
m.thread_id AS "threadId"
|
|
538
|
-
FROM ordered_messages m
|
|
539
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
540
|
-
OR EXISTS (
|
|
541
|
-
SELECT 1 FROM ordered_messages target
|
|
542
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
543
|
-
AND (
|
|
544
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
545
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
546
|
-
OR
|
|
547
|
-
-- Get next messages based on the max withNextMessages
|
|
548
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
549
|
-
)
|
|
550
|
-
)
|
|
551
|
-
ORDER BY m."createdAt" DESC
|
|
552
|
-
`,
|
|
553
|
-
query_params: {
|
|
554
|
-
var_thread_id: threadId,
|
|
555
|
-
var_include: include.map((i) => i.id),
|
|
556
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
557
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
558
|
-
},
|
|
559
|
-
clickhouse_settings: {
|
|
560
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
561
|
-
date_time_input_format: "best_effort",
|
|
562
|
-
date_time_output_format: "iso",
|
|
563
|
-
use_client_time_zone: 1,
|
|
564
|
-
output_format_json_quote_64bit_integers: 0
|
|
565
|
-
}
|
|
566
|
-
});
|
|
567
|
-
const rows2 = await includeResult.json();
|
|
568
|
-
messages.push(...transformRows(rows2.data));
|
|
2003
|
+
const countData = await countResult.json();
|
|
2004
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2005
|
+
if (total === 0) {
|
|
2006
|
+
return {
|
|
2007
|
+
traces: [],
|
|
2008
|
+
total: 0,
|
|
2009
|
+
page,
|
|
2010
|
+
perPage,
|
|
2011
|
+
hasMore: false
|
|
2012
|
+
};
|
|
569
2013
|
}
|
|
570
|
-
const result = await this.
|
|
571
|
-
query: `
|
|
572
|
-
|
|
573
|
-
id,
|
|
574
|
-
content,
|
|
575
|
-
role,
|
|
576
|
-
type,
|
|
577
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
578
|
-
thread_id AS "threadId"
|
|
579
|
-
FROM "${storage.TABLE_MESSAGES}"
|
|
580
|
-
WHERE thread_id = {threadId:String}
|
|
581
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
582
|
-
ORDER BY "createdAt" DESC
|
|
583
|
-
LIMIT {limit:Int64}
|
|
584
|
-
`,
|
|
585
|
-
query_params: {
|
|
586
|
-
threadId,
|
|
587
|
-
exclude: messages.map((m) => m.id),
|
|
588
|
-
limit
|
|
589
|
-
},
|
|
2014
|
+
const result = await this.client.query({
|
|
2015
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2016
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
590
2017
|
clickhouse_settings: {
|
|
591
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
592
2018
|
date_time_input_format: "best_effort",
|
|
593
2019
|
date_time_output_format: "iso",
|
|
594
2020
|
use_client_time_zone: 1,
|
|
595
2021
|
output_format_json_quote_64bit_integers: 0
|
|
596
2022
|
}
|
|
597
2023
|
});
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
2024
|
+
if (!result) {
|
|
2025
|
+
return {
|
|
2026
|
+
traces: [],
|
|
2027
|
+
total,
|
|
2028
|
+
page,
|
|
2029
|
+
perPage,
|
|
2030
|
+
hasMore: false
|
|
2031
|
+
};
|
|
2032
|
+
}
|
|
2033
|
+
const resp = await result.json();
|
|
2034
|
+
const rows = resp.data;
|
|
2035
|
+
const traces = rows.map((row) => ({
|
|
2036
|
+
id: row.id,
|
|
2037
|
+
parentSpanId: row.parentSpanId,
|
|
2038
|
+
traceId: row.traceId,
|
|
2039
|
+
name: row.name,
|
|
2040
|
+
scope: row.scope,
|
|
2041
|
+
kind: row.kind,
|
|
2042
|
+
status: storage.safelyParseJSON(row.status),
|
|
2043
|
+
events: storage.safelyParseJSON(row.events),
|
|
2044
|
+
links: storage.safelyParseJSON(row.links),
|
|
2045
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2046
|
+
startTime: row.startTime,
|
|
2047
|
+
endTime: row.endTime,
|
|
2048
|
+
other: storage.safelyParseJSON(row.other),
|
|
2049
|
+
createdAt: row.createdAt
|
|
2050
|
+
}));
|
|
2051
|
+
return {
|
|
2052
|
+
traces,
|
|
2053
|
+
total,
|
|
2054
|
+
page,
|
|
2055
|
+
perPage,
|
|
2056
|
+
hasMore: currentOffset + traces.length < total
|
|
2057
|
+
};
|
|
2058
|
+
} catch (error$1) {
|
|
2059
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2060
|
+
return {
|
|
2061
|
+
traces: [],
|
|
2062
|
+
total: 0,
|
|
2063
|
+
page,
|
|
2064
|
+
perPage,
|
|
2065
|
+
hasMore: false
|
|
2066
|
+
};
|
|
2067
|
+
}
|
|
2068
|
+
throw new error.MastraError(
|
|
2069
|
+
{
|
|
2070
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2071
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2072
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2073
|
+
details: {
|
|
2074
|
+
name: name ?? null,
|
|
2075
|
+
scope: scope ?? null,
|
|
2076
|
+
page,
|
|
2077
|
+
perPage,
|
|
2078
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2079
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2080
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
606
2081
|
}
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
} catch (error) {
|
|
611
|
-
console.error("Error getting messages:", error);
|
|
612
|
-
throw error;
|
|
2082
|
+
},
|
|
2083
|
+
error$1
|
|
2084
|
+
);
|
|
613
2085
|
}
|
|
614
2086
|
}
|
|
615
|
-
async
|
|
616
|
-
|
|
2087
|
+
async getTraces({
|
|
2088
|
+
name,
|
|
2089
|
+
scope,
|
|
2090
|
+
page,
|
|
2091
|
+
perPage,
|
|
2092
|
+
attributes,
|
|
2093
|
+
filters,
|
|
2094
|
+
fromDate,
|
|
2095
|
+
toDate
|
|
2096
|
+
}) {
|
|
2097
|
+
const limit = perPage;
|
|
2098
|
+
const offset = page * perPage;
|
|
2099
|
+
const args = {};
|
|
2100
|
+
const conditions = [];
|
|
2101
|
+
if (name) {
|
|
2102
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2103
|
+
args.var_name = name;
|
|
2104
|
+
}
|
|
2105
|
+
if (scope) {
|
|
2106
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2107
|
+
args.var_scope = scope;
|
|
2108
|
+
}
|
|
2109
|
+
if (attributes) {
|
|
2110
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2111
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2112
|
+
args[`var_attr_${key}`] = value;
|
|
2113
|
+
});
|
|
2114
|
+
}
|
|
2115
|
+
if (filters) {
|
|
2116
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2117
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2118
|
+
args[`var_col_${key}`] = value;
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
if (fromDate) {
|
|
2122
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2123
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2124
|
+
}
|
|
2125
|
+
if (toDate) {
|
|
2126
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2127
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2128
|
+
}
|
|
2129
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
617
2130
|
try {
|
|
618
|
-
const
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
}
|
|
622
|
-
const thread = await this.getThreadById({ threadId });
|
|
623
|
-
if (!thread) {
|
|
624
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
625
|
-
}
|
|
626
|
-
await this.db.insert({
|
|
627
|
-
table: storage.TABLE_MESSAGES,
|
|
628
|
-
format: "JSONEachRow",
|
|
629
|
-
values: messages.map((message) => ({
|
|
630
|
-
id: message.id,
|
|
631
|
-
thread_id: threadId,
|
|
632
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
633
|
-
createdAt: message.createdAt.toISOString(),
|
|
634
|
-
role: message.role,
|
|
635
|
-
type: message.type
|
|
636
|
-
})),
|
|
2131
|
+
const result = await this.client.query({
|
|
2132
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2133
|
+
query_params: args,
|
|
637
2134
|
clickhouse_settings: {
|
|
638
2135
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
639
2136
|
date_time_input_format: "best_effort",
|
|
2137
|
+
date_time_output_format: "iso",
|
|
640
2138
|
use_client_time_zone: 1,
|
|
641
2139
|
output_format_json_quote_64bit_integers: 0
|
|
642
2140
|
}
|
|
643
2141
|
});
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
2142
|
+
if (!result) {
|
|
2143
|
+
return [];
|
|
2144
|
+
}
|
|
2145
|
+
const resp = await result.json();
|
|
2146
|
+
const rows = resp.data;
|
|
2147
|
+
return rows.map((row) => ({
|
|
2148
|
+
id: row.id,
|
|
2149
|
+
parentSpanId: row.parentSpanId,
|
|
2150
|
+
traceId: row.traceId,
|
|
2151
|
+
name: row.name,
|
|
2152
|
+
scope: row.scope,
|
|
2153
|
+
kind: row.kind,
|
|
2154
|
+
status: storage.safelyParseJSON(row.status),
|
|
2155
|
+
events: storage.safelyParseJSON(row.events),
|
|
2156
|
+
links: storage.safelyParseJSON(row.links),
|
|
2157
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2158
|
+
startTime: row.startTime,
|
|
2159
|
+
endTime: row.endTime,
|
|
2160
|
+
other: storage.safelyParseJSON(row.other),
|
|
2161
|
+
createdAt: row.createdAt
|
|
2162
|
+
}));
|
|
2163
|
+
} catch (error$1) {
|
|
2164
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2165
|
+
return [];
|
|
2166
|
+
}
|
|
2167
|
+
throw new error.MastraError(
|
|
2168
|
+
{
|
|
2169
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2170
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2171
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2172
|
+
details: {
|
|
2173
|
+
name: name ?? null,
|
|
2174
|
+
scope: scope ?? null,
|
|
2175
|
+
page,
|
|
2176
|
+
perPage,
|
|
2177
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2178
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2179
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2180
|
+
toDate: toDate?.toISOString() ?? null
|
|
2181
|
+
}
|
|
2182
|
+
},
|
|
2183
|
+
error$1
|
|
2184
|
+
);
|
|
648
2185
|
}
|
|
649
2186
|
}
|
|
2187
|
+
async batchTraceInsert(args) {
|
|
2188
|
+
await this.operations.batchInsert({ tableName: storage.TABLE_TRACES, records: args.records });
|
|
2189
|
+
}
|
|
2190
|
+
};
|
|
2191
|
+
var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
|
|
2192
|
+
client;
|
|
2193
|
+
operations;
|
|
2194
|
+
constructor({ client, operations }) {
|
|
2195
|
+
super();
|
|
2196
|
+
this.operations = operations;
|
|
2197
|
+
this.client = client;
|
|
2198
|
+
}
|
|
650
2199
|
async persistWorkflowSnapshot({
|
|
651
2200
|
workflowName,
|
|
652
2201
|
runId,
|
|
653
2202
|
snapshot
|
|
654
2203
|
}) {
|
|
655
2204
|
try {
|
|
656
|
-
const currentSnapshot = await this.load({
|
|
2205
|
+
const currentSnapshot = await this.operations.load({
|
|
657
2206
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
658
2207
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
659
2208
|
});
|
|
@@ -669,7 +2218,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
669
2218
|
createdAt: now.toISOString(),
|
|
670
2219
|
updatedAt: now.toISOString()
|
|
671
2220
|
};
|
|
672
|
-
await this.
|
|
2221
|
+
await this.client.insert({
|
|
673
2222
|
table: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
674
2223
|
format: "JSONEachRow",
|
|
675
2224
|
values: [persisting],
|
|
@@ -680,9 +2229,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
680
2229
|
output_format_json_quote_64bit_integers: 0
|
|
681
2230
|
}
|
|
682
2231
|
});
|
|
683
|
-
} catch (error) {
|
|
684
|
-
|
|
685
|
-
|
|
2232
|
+
} catch (error$1) {
|
|
2233
|
+
throw new error.MastraError(
|
|
2234
|
+
{
|
|
2235
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2236
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2237
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2238
|
+
details: { workflowName, runId }
|
|
2239
|
+
},
|
|
2240
|
+
error$1
|
|
2241
|
+
);
|
|
686
2242
|
}
|
|
687
2243
|
}
|
|
688
2244
|
async loadWorkflowSnapshot({
|
|
@@ -690,7 +2246,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
690
2246
|
runId
|
|
691
2247
|
}) {
|
|
692
2248
|
try {
|
|
693
|
-
const result = await this.load({
|
|
2249
|
+
const result = await this.operations.load({
|
|
694
2250
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
695
2251
|
keys: {
|
|
696
2252
|
workflow_name: workflowName,
|
|
@@ -701,9 +2257,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
701
2257
|
return null;
|
|
702
2258
|
}
|
|
703
2259
|
return result.snapshot;
|
|
704
|
-
} catch (error) {
|
|
705
|
-
|
|
706
|
-
|
|
2260
|
+
} catch (error$1) {
|
|
2261
|
+
throw new error.MastraError(
|
|
2262
|
+
{
|
|
2263
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2264
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2265
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2266
|
+
details: { workflowName, runId }
|
|
2267
|
+
},
|
|
2268
|
+
error$1
|
|
2269
|
+
);
|
|
707
2270
|
}
|
|
708
2271
|
}
|
|
709
2272
|
parseWorkflowRun(row) {
|
|
@@ -740,7 +2303,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
740
2303
|
values.var_workflow_name = workflowName;
|
|
741
2304
|
}
|
|
742
2305
|
if (resourceId) {
|
|
743
|
-
const hasResourceId = await this.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2306
|
+
const hasResourceId = await this.operations.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
744
2307
|
if (hasResourceId) {
|
|
745
2308
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
746
2309
|
values.var_resourceId = resourceId;
|
|
@@ -761,7 +2324,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
761
2324
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
762
2325
|
let total = 0;
|
|
763
2326
|
if (limit !== void 0 && offset !== void 0) {
|
|
764
|
-
const countResult = await this.
|
|
2327
|
+
const countResult = await this.client.query({
|
|
765
2328
|
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
766
2329
|
query_params: values,
|
|
767
2330
|
format: "JSONEachRow"
|
|
@@ -769,21 +2332,21 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
769
2332
|
const countRows = await countResult.json();
|
|
770
2333
|
total = Number(countRows[0]?.count ?? 0);
|
|
771
2334
|
}
|
|
772
|
-
const result = await this.
|
|
2335
|
+
const result = await this.client.query({
|
|
773
2336
|
query: `
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
2337
|
+
SELECT
|
|
2338
|
+
workflow_name,
|
|
2339
|
+
run_id,
|
|
2340
|
+
snapshot,
|
|
2341
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2342
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2343
|
+
resourceId
|
|
2344
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2345
|
+
${whereClause}
|
|
2346
|
+
ORDER BY createdAt DESC
|
|
2347
|
+
${limitClause}
|
|
2348
|
+
${offsetClause}
|
|
2349
|
+
`,
|
|
787
2350
|
query_params: values,
|
|
788
2351
|
format: "JSONEachRow"
|
|
789
2352
|
});
|
|
@@ -793,9 +2356,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
793
2356
|
return this.parseWorkflowRun(row);
|
|
794
2357
|
});
|
|
795
2358
|
return { runs, total: total || runs.length };
|
|
796
|
-
} catch (error) {
|
|
797
|
-
|
|
798
|
-
|
|
2359
|
+
} catch (error$1) {
|
|
2360
|
+
throw new error.MastraError(
|
|
2361
|
+
{
|
|
2362
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2363
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2364
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2365
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2366
|
+
},
|
|
2367
|
+
error$1
|
|
2368
|
+
);
|
|
799
2369
|
}
|
|
800
2370
|
}
|
|
801
2371
|
async getWorkflowRunById({
|
|
@@ -814,18 +2384,18 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
814
2384
|
values.var_workflow_name = workflowName;
|
|
815
2385
|
}
|
|
816
2386
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
817
|
-
const result = await this.
|
|
2387
|
+
const result = await this.client.query({
|
|
818
2388
|
query: `
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
2389
|
+
SELECT
|
|
2390
|
+
workflow_name,
|
|
2391
|
+
run_id,
|
|
2392
|
+
snapshot,
|
|
2393
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2394
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2395
|
+
resourceId
|
|
2396
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2397
|
+
${whereClause}
|
|
2398
|
+
`,
|
|
829
2399
|
query_params: values,
|
|
830
2400
|
format: "JSONEachRow"
|
|
831
2401
|
});
|
|
@@ -834,24 +2404,249 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
834
2404
|
return null;
|
|
835
2405
|
}
|
|
836
2406
|
return this.parseWorkflowRun(resultJson[0]);
|
|
837
|
-
} catch (error) {
|
|
838
|
-
|
|
839
|
-
|
|
2407
|
+
} catch (error$1) {
|
|
2408
|
+
throw new error.MastraError(
|
|
2409
|
+
{
|
|
2410
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2411
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2412
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2413
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2414
|
+
},
|
|
2415
|
+
error$1
|
|
2416
|
+
);
|
|
840
2417
|
}
|
|
841
2418
|
}
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
2419
|
+
};
|
|
2420
|
+
|
|
2421
|
+
// src/storage/index.ts
|
|
2422
|
+
var ClickhouseStore = class extends storage.MastraStorage {
|
|
2423
|
+
db;
|
|
2424
|
+
ttl = {};
|
|
2425
|
+
stores;
|
|
2426
|
+
constructor(config) {
|
|
2427
|
+
super({ name: "ClickhouseStore" });
|
|
2428
|
+
this.db = client.createClient({
|
|
2429
|
+
url: config.url,
|
|
2430
|
+
username: config.username,
|
|
2431
|
+
password: config.password,
|
|
2432
|
+
clickhouse_settings: {
|
|
2433
|
+
date_time_input_format: "best_effort",
|
|
2434
|
+
date_time_output_format: "iso",
|
|
2435
|
+
// This is crucial
|
|
2436
|
+
use_client_time_zone: 1,
|
|
2437
|
+
output_format_json_quote_64bit_integers: 0
|
|
2438
|
+
}
|
|
846
2439
|
});
|
|
847
|
-
|
|
848
|
-
|
|
2440
|
+
this.ttl = config.ttl;
|
|
2441
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2442
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2443
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2444
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2445
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2446
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2447
|
+
this.stores = {
|
|
2448
|
+
operations,
|
|
2449
|
+
workflows,
|
|
2450
|
+
scores,
|
|
2451
|
+
legacyEvals,
|
|
2452
|
+
traces,
|
|
2453
|
+
memory
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2456
|
+
get supports() {
|
|
2457
|
+
return {
|
|
2458
|
+
selectByIncludeResourceScope: true,
|
|
2459
|
+
resourceWorkingMemory: true,
|
|
2460
|
+
hasColumn: true,
|
|
2461
|
+
createTable: true
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2465
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2466
|
+
}
|
|
2467
|
+
async getEvals(options) {
|
|
2468
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2469
|
+
}
|
|
2470
|
+
async batchInsert({ tableName, records }) {
|
|
2471
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2472
|
+
}
|
|
2473
|
+
async optimizeTable({ tableName }) {
|
|
2474
|
+
try {
|
|
2475
|
+
await this.db.command({
|
|
2476
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2477
|
+
});
|
|
2478
|
+
} catch (error$1) {
|
|
2479
|
+
throw new error.MastraError(
|
|
2480
|
+
{
|
|
2481
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2482
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2483
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2484
|
+
details: { tableName }
|
|
2485
|
+
},
|
|
2486
|
+
error$1
|
|
2487
|
+
);
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
async materializeTtl({ tableName }) {
|
|
2491
|
+
try {
|
|
2492
|
+
await this.db.command({
|
|
2493
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2494
|
+
});
|
|
2495
|
+
} catch (error$1) {
|
|
2496
|
+
throw new error.MastraError(
|
|
2497
|
+
{
|
|
2498
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2499
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2500
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2501
|
+
details: { tableName }
|
|
2502
|
+
},
|
|
2503
|
+
error$1
|
|
2504
|
+
);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
async createTable({
|
|
2508
|
+
tableName,
|
|
2509
|
+
schema
|
|
2510
|
+
}) {
|
|
2511
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2512
|
+
}
|
|
2513
|
+
async dropTable({ tableName }) {
|
|
2514
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2515
|
+
}
|
|
2516
|
+
async alterTable({
|
|
2517
|
+
tableName,
|
|
2518
|
+
schema,
|
|
2519
|
+
ifNotExists
|
|
2520
|
+
}) {
|
|
2521
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2522
|
+
}
|
|
2523
|
+
async clearTable({ tableName }) {
|
|
2524
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2525
|
+
}
|
|
2526
|
+
async insert({ tableName, record }) {
|
|
2527
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2528
|
+
}
|
|
2529
|
+
async load({ tableName, keys }) {
|
|
2530
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2531
|
+
}
|
|
2532
|
+
async persistWorkflowSnapshot({
|
|
2533
|
+
workflowName,
|
|
2534
|
+
runId,
|
|
2535
|
+
snapshot
|
|
2536
|
+
}) {
|
|
2537
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2538
|
+
}
|
|
2539
|
+
async loadWorkflowSnapshot({
|
|
2540
|
+
workflowName,
|
|
2541
|
+
runId
|
|
2542
|
+
}) {
|
|
2543
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2544
|
+
}
|
|
2545
|
+
async getWorkflowRuns({
|
|
2546
|
+
workflowName,
|
|
2547
|
+
fromDate,
|
|
2548
|
+
toDate,
|
|
2549
|
+
limit,
|
|
2550
|
+
offset,
|
|
2551
|
+
resourceId
|
|
2552
|
+
} = {}) {
|
|
2553
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2554
|
+
}
|
|
2555
|
+
async getWorkflowRunById({
|
|
2556
|
+
runId,
|
|
2557
|
+
workflowName
|
|
2558
|
+
}) {
|
|
2559
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2560
|
+
}
|
|
2561
|
+
async getTraces(args) {
|
|
2562
|
+
return this.stores.traces.getTraces(args);
|
|
2563
|
+
}
|
|
2564
|
+
async getTracesPaginated(args) {
|
|
2565
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2566
|
+
}
|
|
2567
|
+
async batchTraceInsert(args) {
|
|
2568
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2569
|
+
}
|
|
2570
|
+
async getThreadById({ threadId }) {
|
|
2571
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2572
|
+
}
|
|
2573
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2574
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2575
|
+
}
|
|
2576
|
+
async saveThread({ thread }) {
|
|
2577
|
+
return this.stores.memory.saveThread({ thread });
|
|
2578
|
+
}
|
|
2579
|
+
async updateThread({
|
|
2580
|
+
id,
|
|
2581
|
+
title,
|
|
2582
|
+
metadata
|
|
2583
|
+
}) {
|
|
2584
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2585
|
+
}
|
|
2586
|
+
async deleteThread({ threadId }) {
|
|
2587
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2588
|
+
}
|
|
2589
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2590
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2591
|
+
}
|
|
2592
|
+
async getMessages({
|
|
2593
|
+
threadId,
|
|
2594
|
+
resourceId,
|
|
2595
|
+
selectBy,
|
|
2596
|
+
format
|
|
2597
|
+
}) {
|
|
2598
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2599
|
+
}
|
|
2600
|
+
async saveMessages(args) {
|
|
2601
|
+
return this.stores.memory.saveMessages(args);
|
|
2602
|
+
}
|
|
2603
|
+
async getMessagesPaginated(args) {
|
|
2604
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2605
|
+
}
|
|
2606
|
+
async updateMessages(args) {
|
|
2607
|
+
return this.stores.memory.updateMessages(args);
|
|
2608
|
+
}
|
|
2609
|
+
async getResourceById({ resourceId }) {
|
|
2610
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2611
|
+
}
|
|
2612
|
+
async saveResource({ resource }) {
|
|
2613
|
+
return this.stores.memory.saveResource({ resource });
|
|
2614
|
+
}
|
|
2615
|
+
async updateResource({
|
|
2616
|
+
resourceId,
|
|
2617
|
+
workingMemory,
|
|
2618
|
+
metadata
|
|
2619
|
+
}) {
|
|
2620
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2621
|
+
}
|
|
2622
|
+
async getScoreById({ id }) {
|
|
2623
|
+
return this.stores.scores.getScoreById({ id });
|
|
2624
|
+
}
|
|
2625
|
+
async saveScore(_score) {
|
|
2626
|
+
return this.stores.scores.saveScore(_score);
|
|
2627
|
+
}
|
|
2628
|
+
async getScoresByRunId({
|
|
2629
|
+
runId,
|
|
2630
|
+
pagination
|
|
2631
|
+
}) {
|
|
2632
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2633
|
+
}
|
|
2634
|
+
async getScoresByEntityId({
|
|
2635
|
+
entityId,
|
|
2636
|
+
entityType,
|
|
2637
|
+
pagination
|
|
2638
|
+
}) {
|
|
2639
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2640
|
+
}
|
|
2641
|
+
async getScoresByScorerId({
|
|
2642
|
+
scorerId,
|
|
2643
|
+
pagination
|
|
2644
|
+
}) {
|
|
2645
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
849
2646
|
}
|
|
850
2647
|
async close() {
|
|
851
2648
|
await this.db.close();
|
|
852
2649
|
}
|
|
853
2650
|
};
|
|
854
2651
|
|
|
855
|
-
exports.COLUMN_TYPES = COLUMN_TYPES;
|
|
856
2652
|
exports.ClickhouseStore = ClickhouseStore;
|
|
857
|
-
exports.TABLE_ENGINES = TABLE_ENGINES;
|