@mastra/clickhouse 0.0.0-trigger-playground-ui-package-20250506151043 → 0.0.0-unified-sidebar-20251010130811
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +946 -2
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2471 -439
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2455 -423
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +87 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +54 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +55 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +245 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +32 -14
- package/dist/_tsup-dts-rollup.d.cts +0 -138
- package/dist/_tsup-dts-rollup.d.ts +0 -138
- package/dist/index.d.cts +0 -4
- package/docker-compose.yaml +0 -15
- package/eslint.config.js +0 -6
- package/src/index.ts +0 -1
- package/src/storage/index.test.ts +0 -856
- package/src/storage/index.ts +0 -1058
- package/tsconfig.json +0 -5
- package/vitest.config.ts +0 -12
package/dist/index.js
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import {
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { TABLE_AI_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, ScoresStorage, safelyParseJSON, LegacyEvalsStorage, TracesStorage, MemoryStorage, resolveMessageLimit } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { saveScorePayloadSchema } from '@mastra/core/scores';
|
|
3
6
|
|
|
4
7
|
// src/storage/index.ts
|
|
5
|
-
function safelyParseJSON(jsonString) {
|
|
6
|
-
try {
|
|
7
|
-
return JSON.parse(jsonString);
|
|
8
|
-
} catch {
|
|
9
|
-
return {};
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
8
|
var TABLE_ENGINES = {
|
|
13
9
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
14
10
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
15
11
|
[TABLE_TRACES]: `MergeTree()`,
|
|
16
12
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
17
|
-
[TABLE_EVALS]: `MergeTree()
|
|
13
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
14
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
15
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`,
|
|
16
|
+
// TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
|
|
17
|
+
[TABLE_AI_SPANS]: `ReplacingMergeTree()`
|
|
18
18
|
};
|
|
19
19
|
var COLUMN_TYPES = {
|
|
20
20
|
text: "String",
|
|
@@ -22,11 +22,10 @@ var COLUMN_TYPES = {
|
|
|
22
22
|
uuid: "String",
|
|
23
23
|
jsonb: "String",
|
|
24
24
|
integer: "Int64",
|
|
25
|
-
|
|
25
|
+
float: "Float64",
|
|
26
|
+
bigint: "Int64",
|
|
27
|
+
boolean: "Bool"
|
|
26
28
|
};
|
|
27
|
-
function transformRows(rows) {
|
|
28
|
-
return rows.map((row) => transformRow(row));
|
|
29
|
-
}
|
|
30
29
|
function transformRow(row) {
|
|
31
30
|
if (!row) {
|
|
32
31
|
return row;
|
|
@@ -37,33 +36,63 @@ function transformRow(row) {
|
|
|
37
36
|
if (row.updatedAt) {
|
|
38
37
|
row.updatedAt = new Date(row.updatedAt);
|
|
39
38
|
}
|
|
39
|
+
if (row.content && typeof row.content === "string") {
|
|
40
|
+
row.content = safelyParseJSON(row.content);
|
|
41
|
+
}
|
|
40
42
|
return row;
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
// This is crucial
|
|
55
|
-
use_client_time_zone: 1,
|
|
56
|
-
output_format_json_quote_64bit_integers: 0
|
|
57
|
-
}
|
|
58
|
-
});
|
|
59
|
-
this.ttl = config.ttl;
|
|
44
|
+
function transformRows(rows) {
|
|
45
|
+
return rows.map((row) => transformRow(row));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
49
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
50
|
+
client;
|
|
51
|
+
operations;
|
|
52
|
+
constructor({ client, operations }) {
|
|
53
|
+
super();
|
|
54
|
+
this.client = client;
|
|
55
|
+
this.operations = operations;
|
|
60
56
|
}
|
|
61
57
|
transformEvalRow(row) {
|
|
62
58
|
row = transformRow(row);
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
let resultValue;
|
|
60
|
+
try {
|
|
61
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
62
|
+
resultValue = JSON.parse(row.result);
|
|
63
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
64
|
+
resultValue = row.result;
|
|
65
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
66
|
+
resultValue = { score: 0 };
|
|
67
|
+
} else {
|
|
68
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
69
|
+
}
|
|
70
|
+
} catch (error) {
|
|
71
|
+
console.error("Error parsing result field:", row.result, error);
|
|
72
|
+
throw new MastraError({
|
|
73
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
74
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
75
|
+
domain: ErrorDomain.STORAGE,
|
|
76
|
+
category: ErrorCategory.USER
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
let testInfoValue;
|
|
80
|
+
try {
|
|
81
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
82
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
83
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
84
|
+
testInfoValue = row.test_info;
|
|
85
|
+
}
|
|
86
|
+
} catch {
|
|
87
|
+
testInfoValue = void 0;
|
|
88
|
+
}
|
|
65
89
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
66
|
-
throw new
|
|
90
|
+
throw new MastraError({
|
|
91
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
92
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
93
|
+
domain: ErrorDomain.STORAGE,
|
|
94
|
+
category: ErrorCategory.USER
|
|
95
|
+
});
|
|
67
96
|
}
|
|
68
97
|
return {
|
|
69
98
|
input: row.input,
|
|
@@ -80,9 +109,9 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
80
109
|
}
|
|
81
110
|
async getEvalsByAgentName(agentName, type) {
|
|
82
111
|
try {
|
|
83
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
84
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
85
|
-
const result = await this.
|
|
112
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
113
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
114
|
+
const result = await this.client.query({
|
|
86
115
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
87
116
|
query_params: { var_agent_name: agentName },
|
|
88
117
|
clickhouse_settings: {
|
|
@@ -98,125 +127,1357 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
98
127
|
const rows = await result.json();
|
|
99
128
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
100
129
|
} catch (error) {
|
|
101
|
-
if (error
|
|
130
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
102
131
|
return [];
|
|
103
132
|
}
|
|
104
|
-
|
|
105
|
-
|
|
133
|
+
throw new MastraError(
|
|
134
|
+
{
|
|
135
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
136
|
+
domain: ErrorDomain.STORAGE,
|
|
137
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
138
|
+
details: { agentName, type: type ?? null }
|
|
139
|
+
},
|
|
140
|
+
error
|
|
141
|
+
);
|
|
106
142
|
}
|
|
107
143
|
}
|
|
108
|
-
async
|
|
144
|
+
async getEvals(options = {}) {
|
|
145
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
146
|
+
const fromDate = dateRange?.start;
|
|
147
|
+
const toDate = dateRange?.end;
|
|
148
|
+
const conditions = [];
|
|
149
|
+
if (agentName) {
|
|
150
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
151
|
+
}
|
|
152
|
+
if (type === "test") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
155
|
+
);
|
|
156
|
+
} else if (type === "live") {
|
|
157
|
+
conditions.push(
|
|
158
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
if (fromDate) {
|
|
162
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
163
|
+
fromDate.toISOString();
|
|
164
|
+
}
|
|
165
|
+
if (toDate) {
|
|
166
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
167
|
+
toDate.toISOString();
|
|
168
|
+
}
|
|
169
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
109
170
|
try {
|
|
110
|
-
await this.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
...
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
171
|
+
const countResult = await this.client.query({
|
|
172
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
173
|
+
query_params: {
|
|
174
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
175
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
176
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
177
|
+
},
|
|
178
|
+
clickhouse_settings: {
|
|
179
|
+
date_time_input_format: "best_effort",
|
|
180
|
+
date_time_output_format: "iso",
|
|
181
|
+
use_client_time_zone: 1,
|
|
182
|
+
output_format_json_quote_64bit_integers: 0
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
const countData = await countResult.json();
|
|
186
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
187
|
+
const currentOffset = page * perPage;
|
|
188
|
+
const hasMore = currentOffset + perPage < total;
|
|
189
|
+
if (total === 0) {
|
|
190
|
+
return {
|
|
191
|
+
evals: [],
|
|
192
|
+
total: 0,
|
|
193
|
+
page,
|
|
194
|
+
perPage,
|
|
195
|
+
hasMore: false
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
const dataResult = await this.client.query({
|
|
199
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
200
|
+
query_params: {
|
|
201
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
202
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
203
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
204
|
+
var_limit: perPage || 100,
|
|
205
|
+
var_offset: currentOffset || 0
|
|
206
|
+
},
|
|
121
207
|
clickhouse_settings: {
|
|
122
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
123
208
|
date_time_input_format: "best_effort",
|
|
209
|
+
date_time_output_format: "iso",
|
|
124
210
|
use_client_time_zone: 1,
|
|
125
211
|
output_format_json_quote_64bit_integers: 0
|
|
126
212
|
}
|
|
127
213
|
});
|
|
214
|
+
const rows = await dataResult.json();
|
|
215
|
+
return {
|
|
216
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
217
|
+
total,
|
|
218
|
+
page,
|
|
219
|
+
perPage,
|
|
220
|
+
hasMore
|
|
221
|
+
};
|
|
128
222
|
} catch (error) {
|
|
129
|
-
|
|
130
|
-
|
|
223
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
224
|
+
return {
|
|
225
|
+
evals: [],
|
|
226
|
+
total: 0,
|
|
227
|
+
page,
|
|
228
|
+
perPage,
|
|
229
|
+
hasMore: false
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
throw new MastraError(
|
|
233
|
+
{
|
|
234
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
235
|
+
domain: ErrorDomain.STORAGE,
|
|
236
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
237
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
238
|
+
},
|
|
239
|
+
error
|
|
240
|
+
);
|
|
131
241
|
}
|
|
132
242
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
243
|
+
};
|
|
244
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
245
|
+
client;
|
|
246
|
+
operations;
|
|
247
|
+
constructor({ client, operations }) {
|
|
248
|
+
super();
|
|
249
|
+
this.client = client;
|
|
250
|
+
this.operations = operations;
|
|
251
|
+
}
|
|
252
|
+
async getMessages({
|
|
253
|
+
threadId,
|
|
254
|
+
resourceId,
|
|
255
|
+
selectBy,
|
|
256
|
+
format
|
|
142
257
|
}) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
258
|
+
try {
|
|
259
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
260
|
+
const messages = [];
|
|
261
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
262
|
+
const include = selectBy?.include || [];
|
|
263
|
+
if (include.length) {
|
|
264
|
+
const unionQueries = [];
|
|
265
|
+
const params = [];
|
|
266
|
+
let paramIdx = 1;
|
|
267
|
+
for (const inc of include) {
|
|
268
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
269
|
+
const searchId = inc.threadId || threadId;
|
|
270
|
+
unionQueries.push(`
|
|
271
|
+
SELECT * FROM (
|
|
272
|
+
WITH numbered_messages AS (
|
|
273
|
+
SELECT
|
|
274
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
275
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
276
|
+
FROM "${TABLE_MESSAGES}"
|
|
277
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
278
|
+
),
|
|
279
|
+
target_positions AS (
|
|
280
|
+
SELECT row_num as target_pos
|
|
281
|
+
FROM numbered_messages
|
|
282
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
283
|
+
)
|
|
284
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
285
|
+
FROM numbered_messages m
|
|
286
|
+
CROSS JOIN target_positions t
|
|
287
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
288
|
+
) AS query_${paramIdx}
|
|
289
|
+
`);
|
|
290
|
+
params.push(
|
|
291
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
292
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
293
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
294
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
295
|
+
);
|
|
296
|
+
paramIdx++;
|
|
297
|
+
}
|
|
298
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
299
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
300
|
+
const includeResult = await this.client.query({
|
|
301
|
+
query: finalQuery,
|
|
302
|
+
query_params: mergedParams,
|
|
303
|
+
clickhouse_settings: {
|
|
304
|
+
date_time_input_format: "best_effort",
|
|
305
|
+
date_time_output_format: "iso",
|
|
306
|
+
use_client_time_zone: 1,
|
|
307
|
+
output_format_json_quote_64bit_integers: 0
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
const rows2 = await includeResult.json();
|
|
311
|
+
const includedMessages = transformRows(rows2.data);
|
|
312
|
+
const seen = /* @__PURE__ */ new Set();
|
|
313
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
314
|
+
if (seen.has(message.id)) return false;
|
|
315
|
+
seen.add(message.id);
|
|
316
|
+
return true;
|
|
317
|
+
});
|
|
318
|
+
messages.push(...dedupedMessages);
|
|
319
|
+
}
|
|
320
|
+
const result = await this.client.query({
|
|
321
|
+
query: `
|
|
322
|
+
SELECT
|
|
323
|
+
id,
|
|
324
|
+
content,
|
|
325
|
+
role,
|
|
326
|
+
type,
|
|
327
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
328
|
+
thread_id AS "threadId"
|
|
329
|
+
FROM "${TABLE_MESSAGES}"
|
|
330
|
+
WHERE thread_id = {threadId:String}
|
|
331
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
332
|
+
ORDER BY "createdAt" DESC
|
|
333
|
+
LIMIT {limit:Int64}
|
|
334
|
+
`,
|
|
335
|
+
query_params: {
|
|
336
|
+
threadId,
|
|
337
|
+
exclude: messages.map((m) => m.id),
|
|
338
|
+
limit
|
|
339
|
+
},
|
|
340
|
+
clickhouse_settings: {
|
|
341
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
342
|
+
date_time_input_format: "best_effort",
|
|
343
|
+
date_time_output_format: "iso",
|
|
344
|
+
use_client_time_zone: 1,
|
|
345
|
+
output_format_json_quote_64bit_integers: 0
|
|
346
|
+
}
|
|
159
347
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
348
|
+
const rows = await result.json();
|
|
349
|
+
messages.push(...transformRows(rows.data));
|
|
350
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
351
|
+
messages.forEach((message) => {
|
|
352
|
+
if (typeof message.content === "string") {
|
|
353
|
+
try {
|
|
354
|
+
message.content = JSON.parse(message.content);
|
|
355
|
+
} catch {
|
|
356
|
+
}
|
|
357
|
+
}
|
|
167
358
|
});
|
|
359
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
360
|
+
if (format === `v2`) return list.get.all.v2();
|
|
361
|
+
return list.get.all.v1();
|
|
362
|
+
} catch (error) {
|
|
363
|
+
throw new MastraError(
|
|
364
|
+
{
|
|
365
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
366
|
+
domain: ErrorDomain.STORAGE,
|
|
367
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
368
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
369
|
+
},
|
|
370
|
+
error
|
|
371
|
+
);
|
|
168
372
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
373
|
+
}
|
|
374
|
+
async getMessagesById({
|
|
375
|
+
messageIds,
|
|
376
|
+
format
|
|
377
|
+
}) {
|
|
378
|
+
if (messageIds.length === 0) return [];
|
|
379
|
+
try {
|
|
380
|
+
const result = await this.client.query({
|
|
381
|
+
query: `
|
|
382
|
+
SELECT
|
|
383
|
+
id,
|
|
384
|
+
content,
|
|
385
|
+
role,
|
|
386
|
+
type,
|
|
387
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
388
|
+
thread_id AS "threadId",
|
|
389
|
+
"resourceId"
|
|
390
|
+
FROM "${TABLE_MESSAGES}"
|
|
391
|
+
WHERE id IN {messageIds:Array(String)}
|
|
392
|
+
ORDER BY "createdAt" DESC
|
|
393
|
+
`,
|
|
394
|
+
query_params: {
|
|
395
|
+
messageIds
|
|
396
|
+
},
|
|
397
|
+
clickhouse_settings: {
|
|
398
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
399
|
+
date_time_input_format: "best_effort",
|
|
400
|
+
date_time_output_format: "iso",
|
|
401
|
+
use_client_time_zone: 1,
|
|
402
|
+
output_format_json_quote_64bit_integers: 0
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
const rows = await result.json();
|
|
406
|
+
const messages = transformRows(rows.data);
|
|
407
|
+
messages.forEach((message) => {
|
|
408
|
+
if (typeof message.content === "string") {
|
|
409
|
+
try {
|
|
410
|
+
message.content = JSON.parse(message.content);
|
|
411
|
+
} catch {
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
});
|
|
415
|
+
const list = new MessageList().add(messages, "memory");
|
|
416
|
+
if (format === `v1`) return list.get.all.v1();
|
|
417
|
+
return list.get.all.v2();
|
|
418
|
+
} catch (error) {
|
|
419
|
+
throw new MastraError(
|
|
420
|
+
{
|
|
421
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
|
|
422
|
+
domain: ErrorDomain.STORAGE,
|
|
423
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
424
|
+
details: { messageIds: JSON.stringify(messageIds) }
|
|
425
|
+
},
|
|
426
|
+
error
|
|
427
|
+
);
|
|
176
428
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
429
|
+
}
|
|
430
|
+
async saveMessages(args) {
|
|
431
|
+
const { messages, format = "v1" } = args;
|
|
432
|
+
if (messages.length === 0) return messages;
|
|
433
|
+
for (const message of messages) {
|
|
434
|
+
const resourceId = message.resourceId;
|
|
435
|
+
if (!resourceId) {
|
|
436
|
+
throw new Error("Resource ID is required");
|
|
437
|
+
}
|
|
438
|
+
if (!message.threadId) {
|
|
439
|
+
throw new Error("Thread ID is required");
|
|
440
|
+
}
|
|
441
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
442
|
+
if (!thread) {
|
|
443
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
187
444
|
}
|
|
188
|
-
});
|
|
189
|
-
if (!result) {
|
|
190
|
-
return [];
|
|
191
445
|
}
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
446
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
447
|
+
await Promise.all(
|
|
448
|
+
messages.map(async (m) => {
|
|
449
|
+
const resourceId = m.resourceId;
|
|
450
|
+
if (!resourceId) {
|
|
451
|
+
throw new Error("Resource ID is required");
|
|
452
|
+
}
|
|
453
|
+
if (!m.threadId) {
|
|
454
|
+
throw new Error("Thread ID is required");
|
|
455
|
+
}
|
|
456
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
457
|
+
if (!thread) {
|
|
458
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
459
|
+
}
|
|
460
|
+
threadIdSet.set(m.threadId, thread);
|
|
461
|
+
})
|
|
462
|
+
);
|
|
463
|
+
try {
|
|
464
|
+
const existingResult = await this.client.query({
|
|
465
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
466
|
+
query_params: {
|
|
467
|
+
ids: messages.map((m) => m.id)
|
|
468
|
+
},
|
|
469
|
+
clickhouse_settings: {
|
|
470
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
471
|
+
date_time_input_format: "best_effort",
|
|
472
|
+
date_time_output_format: "iso",
|
|
473
|
+
use_client_time_zone: 1,
|
|
474
|
+
output_format_json_quote_64bit_integers: 0
|
|
475
|
+
},
|
|
476
|
+
format: "JSONEachRow"
|
|
477
|
+
});
|
|
478
|
+
const existingRows = await existingResult.json();
|
|
479
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
480
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
481
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
482
|
+
const toMove = messages.filter((m) => {
|
|
483
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
484
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
485
|
+
});
|
|
486
|
+
const deletePromises = toMove.map((message) => {
|
|
487
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
488
|
+
if (!existingRow) return Promise.resolve();
|
|
489
|
+
return this.client.command({
|
|
490
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
491
|
+
query_params: {
|
|
492
|
+
var_id: message.id,
|
|
493
|
+
var_old_thread_id: existingRow.thread_id
|
|
494
|
+
},
|
|
495
|
+
clickhouse_settings: {
|
|
496
|
+
date_time_input_format: "best_effort",
|
|
497
|
+
use_client_time_zone: 1,
|
|
498
|
+
output_format_json_quote_64bit_integers: 0
|
|
499
|
+
}
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
const updatePromises = toUpdate.map(
|
|
503
|
+
(message) => this.client.command({
|
|
504
|
+
query: `
|
|
505
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
506
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
507
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
508
|
+
`,
|
|
509
|
+
query_params: {
|
|
510
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
511
|
+
var_role: message.role,
|
|
512
|
+
var_type: message.type || "v2",
|
|
513
|
+
var_resourceId: message.resourceId,
|
|
514
|
+
var_id: message.id,
|
|
515
|
+
var_thread_id: message.threadId
|
|
516
|
+
},
|
|
517
|
+
clickhouse_settings: {
|
|
518
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
519
|
+
date_time_input_format: "best_effort",
|
|
520
|
+
use_client_time_zone: 1,
|
|
521
|
+
output_format_json_quote_64bit_integers: 0
|
|
522
|
+
}
|
|
523
|
+
})
|
|
524
|
+
);
|
|
525
|
+
await Promise.all([
|
|
526
|
+
// Insert new messages (including moved messages)
|
|
527
|
+
this.client.insert({
|
|
528
|
+
table: TABLE_MESSAGES,
|
|
529
|
+
format: "JSONEachRow",
|
|
530
|
+
values: toInsert.map((message) => ({
|
|
531
|
+
id: message.id,
|
|
532
|
+
thread_id: message.threadId,
|
|
533
|
+
resourceId: message.resourceId,
|
|
534
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
535
|
+
createdAt: message.createdAt.toISOString(),
|
|
536
|
+
role: message.role,
|
|
537
|
+
type: message.type || "v2"
|
|
538
|
+
})),
|
|
539
|
+
clickhouse_settings: {
|
|
540
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
541
|
+
date_time_input_format: "best_effort",
|
|
542
|
+
use_client_time_zone: 1,
|
|
543
|
+
output_format_json_quote_64bit_integers: 0
|
|
544
|
+
}
|
|
545
|
+
}),
|
|
546
|
+
...updatePromises,
|
|
547
|
+
...deletePromises,
|
|
548
|
+
// Update thread's updatedAt timestamp
|
|
549
|
+
this.client.insert({
|
|
550
|
+
table: TABLE_THREADS,
|
|
551
|
+
format: "JSONEachRow",
|
|
552
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
553
|
+
id: thread.id,
|
|
554
|
+
resourceId: thread.resourceId,
|
|
555
|
+
title: thread.title,
|
|
556
|
+
metadata: thread.metadata,
|
|
557
|
+
createdAt: thread.createdAt,
|
|
558
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
559
|
+
})),
|
|
560
|
+
clickhouse_settings: {
|
|
561
|
+
date_time_input_format: "best_effort",
|
|
562
|
+
use_client_time_zone: 1,
|
|
563
|
+
output_format_json_quote_64bit_integers: 0
|
|
564
|
+
}
|
|
565
|
+
})
|
|
566
|
+
]);
|
|
567
|
+
const list = new MessageList().add(messages, "memory");
|
|
568
|
+
if (format === `v2`) return list.get.all.v2();
|
|
569
|
+
return list.get.all.v1();
|
|
570
|
+
} catch (error) {
|
|
571
|
+
throw new MastraError(
|
|
572
|
+
{
|
|
573
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
574
|
+
domain: ErrorDomain.STORAGE,
|
|
575
|
+
category: ErrorCategory.THIRD_PARTY
|
|
576
|
+
},
|
|
577
|
+
error
|
|
578
|
+
);
|
|
579
|
+
}
|
|
210
580
|
}
|
|
211
|
-
async
|
|
212
|
-
|
|
213
|
-
|
|
581
|
+
async getThreadById({ threadId }) {
|
|
582
|
+
try {
|
|
583
|
+
const result = await this.client.query({
|
|
584
|
+
query: `SELECT
|
|
585
|
+
id,
|
|
586
|
+
"resourceId",
|
|
587
|
+
title,
|
|
588
|
+
metadata,
|
|
589
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
590
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
591
|
+
FROM "${TABLE_THREADS}"
|
|
592
|
+
FINAL
|
|
593
|
+
WHERE id = {var_id:String}`,
|
|
594
|
+
query_params: { var_id: threadId },
|
|
595
|
+
clickhouse_settings: {
|
|
596
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
597
|
+
date_time_input_format: "best_effort",
|
|
598
|
+
date_time_output_format: "iso",
|
|
599
|
+
use_client_time_zone: 1,
|
|
600
|
+
output_format_json_quote_64bit_integers: 0
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
const rows = await result.json();
|
|
604
|
+
const thread = transformRow(rows.data[0]);
|
|
605
|
+
if (!thread) {
|
|
606
|
+
return null;
|
|
607
|
+
}
|
|
608
|
+
return {
|
|
609
|
+
...thread,
|
|
610
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
611
|
+
createdAt: thread.createdAt,
|
|
612
|
+
updatedAt: thread.updatedAt
|
|
613
|
+
};
|
|
614
|
+
} catch (error) {
|
|
615
|
+
throw new MastraError(
|
|
616
|
+
{
|
|
617
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
618
|
+
domain: ErrorDomain.STORAGE,
|
|
619
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
620
|
+
details: { threadId }
|
|
621
|
+
},
|
|
622
|
+
error
|
|
623
|
+
);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
627
|
+
try {
|
|
628
|
+
const result = await this.client.query({
|
|
629
|
+
query: `SELECT
|
|
630
|
+
id,
|
|
631
|
+
"resourceId",
|
|
632
|
+
title,
|
|
633
|
+
metadata,
|
|
634
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
635
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
636
|
+
FROM "${TABLE_THREADS}"
|
|
637
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
638
|
+
query_params: { var_resourceId: resourceId },
|
|
639
|
+
clickhouse_settings: {
|
|
640
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
641
|
+
date_time_input_format: "best_effort",
|
|
642
|
+
date_time_output_format: "iso",
|
|
643
|
+
use_client_time_zone: 1,
|
|
644
|
+
output_format_json_quote_64bit_integers: 0
|
|
645
|
+
}
|
|
646
|
+
});
|
|
647
|
+
const rows = await result.json();
|
|
648
|
+
const threads = transformRows(rows.data);
|
|
649
|
+
return threads.map((thread) => ({
|
|
650
|
+
...thread,
|
|
651
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
652
|
+
createdAt: thread.createdAt,
|
|
653
|
+
updatedAt: thread.updatedAt
|
|
654
|
+
}));
|
|
655
|
+
} catch (error) {
|
|
656
|
+
throw new MastraError(
|
|
657
|
+
{
|
|
658
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
659
|
+
domain: ErrorDomain.STORAGE,
|
|
660
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
661
|
+
details: { resourceId }
|
|
662
|
+
},
|
|
663
|
+
error
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
async saveThread({ thread }) {
|
|
668
|
+
try {
|
|
669
|
+
await this.client.insert({
|
|
670
|
+
table: TABLE_THREADS,
|
|
671
|
+
values: [
|
|
672
|
+
{
|
|
673
|
+
...thread,
|
|
674
|
+
createdAt: thread.createdAt.toISOString(),
|
|
675
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
676
|
+
}
|
|
677
|
+
],
|
|
678
|
+
format: "JSONEachRow",
|
|
679
|
+
clickhouse_settings: {
|
|
680
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
681
|
+
date_time_input_format: "best_effort",
|
|
682
|
+
use_client_time_zone: 1,
|
|
683
|
+
output_format_json_quote_64bit_integers: 0
|
|
684
|
+
}
|
|
685
|
+
});
|
|
686
|
+
return thread;
|
|
687
|
+
} catch (error) {
|
|
688
|
+
throw new MastraError(
|
|
689
|
+
{
|
|
690
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
691
|
+
domain: ErrorDomain.STORAGE,
|
|
692
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
693
|
+
details: { threadId: thread.id }
|
|
694
|
+
},
|
|
695
|
+
error
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
async updateThread({
|
|
700
|
+
id,
|
|
701
|
+
title,
|
|
702
|
+
metadata
|
|
703
|
+
}) {
|
|
704
|
+
try {
|
|
705
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
706
|
+
if (!existingThread) {
|
|
707
|
+
throw new Error(`Thread ${id} not found`);
|
|
708
|
+
}
|
|
709
|
+
const mergedMetadata = {
|
|
710
|
+
...existingThread.metadata,
|
|
711
|
+
...metadata
|
|
712
|
+
};
|
|
713
|
+
const updatedThread = {
|
|
714
|
+
...existingThread,
|
|
715
|
+
title,
|
|
716
|
+
metadata: mergedMetadata,
|
|
717
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
718
|
+
};
|
|
719
|
+
await this.client.insert({
|
|
720
|
+
table: TABLE_THREADS,
|
|
721
|
+
format: "JSONEachRow",
|
|
722
|
+
values: [
|
|
723
|
+
{
|
|
724
|
+
id: updatedThread.id,
|
|
725
|
+
resourceId: updatedThread.resourceId,
|
|
726
|
+
title: updatedThread.title,
|
|
727
|
+
metadata: updatedThread.metadata,
|
|
728
|
+
createdAt: updatedThread.createdAt,
|
|
729
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
730
|
+
}
|
|
731
|
+
],
|
|
732
|
+
clickhouse_settings: {
|
|
733
|
+
date_time_input_format: "best_effort",
|
|
734
|
+
use_client_time_zone: 1,
|
|
735
|
+
output_format_json_quote_64bit_integers: 0
|
|
736
|
+
}
|
|
737
|
+
});
|
|
738
|
+
return updatedThread;
|
|
739
|
+
} catch (error) {
|
|
740
|
+
throw new MastraError(
|
|
741
|
+
{
|
|
742
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
743
|
+
domain: ErrorDomain.STORAGE,
|
|
744
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
745
|
+
details: { threadId: id, title }
|
|
746
|
+
},
|
|
747
|
+
error
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
async deleteThread({ threadId }) {
|
|
752
|
+
try {
|
|
753
|
+
await this.client.command({
|
|
754
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
755
|
+
query_params: { var_thread_id: threadId },
|
|
756
|
+
clickhouse_settings: {
|
|
757
|
+
output_format_json_quote_64bit_integers: 0
|
|
758
|
+
}
|
|
759
|
+
});
|
|
760
|
+
await this.client.command({
|
|
761
|
+
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
762
|
+
query_params: { var_id: threadId },
|
|
763
|
+
clickhouse_settings: {
|
|
764
|
+
output_format_json_quote_64bit_integers: 0
|
|
765
|
+
}
|
|
766
|
+
});
|
|
767
|
+
} catch (error) {
|
|
768
|
+
throw new MastraError(
|
|
769
|
+
{
|
|
770
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
771
|
+
domain: ErrorDomain.STORAGE,
|
|
772
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
773
|
+
details: { threadId }
|
|
774
|
+
},
|
|
775
|
+
error
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
780
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
781
|
+
try {
|
|
782
|
+
const currentOffset = page * perPage;
|
|
783
|
+
const countResult = await this.client.query({
|
|
784
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
785
|
+
query_params: { resourceId },
|
|
786
|
+
clickhouse_settings: {
|
|
787
|
+
date_time_input_format: "best_effort",
|
|
788
|
+
date_time_output_format: "iso",
|
|
789
|
+
use_client_time_zone: 1,
|
|
790
|
+
output_format_json_quote_64bit_integers: 0
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
const countData = await countResult.json();
|
|
794
|
+
const total = countData.data[0].total;
|
|
795
|
+
if (total === 0) {
|
|
796
|
+
return {
|
|
797
|
+
threads: [],
|
|
798
|
+
total: 0,
|
|
799
|
+
page,
|
|
800
|
+
perPage,
|
|
801
|
+
hasMore: false
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
const dataResult = await this.client.query({
|
|
805
|
+
query: `
|
|
806
|
+
SELECT
|
|
807
|
+
id,
|
|
808
|
+
resourceId,
|
|
809
|
+
title,
|
|
810
|
+
metadata,
|
|
811
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
812
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
813
|
+
FROM ${TABLE_THREADS}
|
|
814
|
+
WHERE resourceId = {resourceId:String}
|
|
815
|
+
ORDER BY createdAt DESC
|
|
816
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
817
|
+
`,
|
|
818
|
+
query_params: {
|
|
819
|
+
resourceId,
|
|
820
|
+
limit: perPage,
|
|
821
|
+
offset: currentOffset
|
|
822
|
+
},
|
|
823
|
+
clickhouse_settings: {
|
|
824
|
+
date_time_input_format: "best_effort",
|
|
825
|
+
date_time_output_format: "iso",
|
|
826
|
+
use_client_time_zone: 1,
|
|
827
|
+
output_format_json_quote_64bit_integers: 0
|
|
828
|
+
}
|
|
829
|
+
});
|
|
830
|
+
const rows = await dataResult.json();
|
|
831
|
+
const threads = transformRows(rows.data);
|
|
832
|
+
return {
|
|
833
|
+
threads,
|
|
834
|
+
total,
|
|
835
|
+
page,
|
|
836
|
+
perPage,
|
|
837
|
+
hasMore: currentOffset + threads.length < total
|
|
838
|
+
};
|
|
839
|
+
} catch (error) {
|
|
840
|
+
throw new MastraError(
|
|
841
|
+
{
|
|
842
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
843
|
+
domain: ErrorDomain.STORAGE,
|
|
844
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
845
|
+
details: { resourceId, page }
|
|
846
|
+
},
|
|
847
|
+
error
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
async getMessagesPaginated(args) {
|
|
852
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
853
|
+
const page = selectBy?.pagination?.page || 0;
|
|
854
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
855
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
856
|
+
try {
|
|
857
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
858
|
+
const offset = page * perPage;
|
|
859
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
860
|
+
const fromDate = dateRange?.start;
|
|
861
|
+
const toDate = dateRange?.end;
|
|
862
|
+
const messages = [];
|
|
863
|
+
if (selectBy?.include?.length) {
|
|
864
|
+
const include = selectBy.include;
|
|
865
|
+
const unionQueries = [];
|
|
866
|
+
const params = [];
|
|
867
|
+
let paramIdx = 1;
|
|
868
|
+
for (const inc of include) {
|
|
869
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
870
|
+
const searchId = inc.threadId || threadId;
|
|
871
|
+
unionQueries.push(`
|
|
872
|
+
SELECT * FROM (
|
|
873
|
+
WITH numbered_messages AS (
|
|
874
|
+
SELECT
|
|
875
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
876
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
877
|
+
FROM "${TABLE_MESSAGES}"
|
|
878
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
879
|
+
),
|
|
880
|
+
target_positions AS (
|
|
881
|
+
SELECT row_num as target_pos
|
|
882
|
+
FROM numbered_messages
|
|
883
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
884
|
+
)
|
|
885
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
886
|
+
FROM numbered_messages m
|
|
887
|
+
CROSS JOIN target_positions t
|
|
888
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
889
|
+
) AS query_${paramIdx}
|
|
890
|
+
`);
|
|
891
|
+
params.push(
|
|
892
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
893
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
894
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
895
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
896
|
+
);
|
|
897
|
+
paramIdx++;
|
|
898
|
+
}
|
|
899
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
900
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
901
|
+
const includeResult = await this.client.query({
|
|
902
|
+
query: finalQuery,
|
|
903
|
+
query_params: mergedParams,
|
|
904
|
+
clickhouse_settings: {
|
|
905
|
+
date_time_input_format: "best_effort",
|
|
906
|
+
date_time_output_format: "iso",
|
|
907
|
+
use_client_time_zone: 1,
|
|
908
|
+
output_format_json_quote_64bit_integers: 0
|
|
909
|
+
}
|
|
910
|
+
});
|
|
911
|
+
const rows2 = await includeResult.json();
|
|
912
|
+
const includedMessages = transformRows(rows2.data);
|
|
913
|
+
const seen = /* @__PURE__ */ new Set();
|
|
914
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
915
|
+
if (seen.has(message.id)) return false;
|
|
916
|
+
seen.add(message.id);
|
|
917
|
+
return true;
|
|
918
|
+
});
|
|
919
|
+
messages.push(...dedupedMessages);
|
|
920
|
+
}
|
|
921
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
922
|
+
const countParams = { threadId };
|
|
923
|
+
if (fromDate) {
|
|
924
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
925
|
+
countParams.fromDate = fromDate.toISOString();
|
|
926
|
+
}
|
|
927
|
+
if (toDate) {
|
|
928
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
929
|
+
countParams.toDate = toDate.toISOString();
|
|
930
|
+
}
|
|
931
|
+
const countResult = await this.client.query({
|
|
932
|
+
query: countQuery,
|
|
933
|
+
query_params: countParams,
|
|
934
|
+
clickhouse_settings: {
|
|
935
|
+
date_time_input_format: "best_effort",
|
|
936
|
+
date_time_output_format: "iso",
|
|
937
|
+
use_client_time_zone: 1,
|
|
938
|
+
output_format_json_quote_64bit_integers: 0
|
|
939
|
+
}
|
|
940
|
+
});
|
|
941
|
+
const countData = await countResult.json();
|
|
942
|
+
const total = countData.data[0].total;
|
|
943
|
+
if (total === 0 && messages.length === 0) {
|
|
944
|
+
return {
|
|
945
|
+
messages: [],
|
|
946
|
+
total: 0,
|
|
947
|
+
page,
|
|
948
|
+
perPage,
|
|
949
|
+
hasMore: false
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
const excludeIds = messages.map((m) => m.id);
|
|
953
|
+
let dataQuery = `
|
|
954
|
+
SELECT
|
|
955
|
+
id,
|
|
956
|
+
content,
|
|
957
|
+
role,
|
|
958
|
+
type,
|
|
959
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
960
|
+
thread_id AS "threadId",
|
|
961
|
+
resourceId
|
|
962
|
+
FROM ${TABLE_MESSAGES}
|
|
963
|
+
WHERE thread_id = {threadId:String}
|
|
964
|
+
`;
|
|
965
|
+
const dataParams = { threadId };
|
|
966
|
+
if (fromDate) {
|
|
967
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
968
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
969
|
+
}
|
|
970
|
+
if (toDate) {
|
|
971
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
972
|
+
dataParams.toDate = toDate.toISOString();
|
|
973
|
+
}
|
|
974
|
+
if (excludeIds.length > 0) {
|
|
975
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
976
|
+
dataParams.excludeIds = excludeIds;
|
|
977
|
+
}
|
|
978
|
+
if (selectBy?.last) {
|
|
979
|
+
dataQuery += `
|
|
980
|
+
ORDER BY createdAt DESC
|
|
981
|
+
LIMIT {limit:Int64}
|
|
982
|
+
`;
|
|
983
|
+
dataParams.limit = perPage;
|
|
984
|
+
} else {
|
|
985
|
+
dataQuery += `
|
|
986
|
+
ORDER BY createdAt ASC
|
|
987
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
988
|
+
`;
|
|
989
|
+
dataParams.limit = perPage;
|
|
990
|
+
dataParams.offset = offset;
|
|
991
|
+
}
|
|
992
|
+
const result = await this.client.query({
|
|
993
|
+
query: dataQuery,
|
|
994
|
+
query_params: dataParams,
|
|
995
|
+
clickhouse_settings: {
|
|
996
|
+
date_time_input_format: "best_effort",
|
|
997
|
+
date_time_output_format: "iso",
|
|
998
|
+
use_client_time_zone: 1,
|
|
999
|
+
output_format_json_quote_64bit_integers: 0
|
|
1000
|
+
}
|
|
1001
|
+
});
|
|
1002
|
+
const rows = await result.json();
|
|
1003
|
+
const paginatedMessages = transformRows(rows.data);
|
|
1004
|
+
messages.push(...paginatedMessages);
|
|
1005
|
+
if (selectBy?.last) {
|
|
1006
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1007
|
+
}
|
|
1008
|
+
return {
|
|
1009
|
+
messages: format === "v2" ? messages : messages,
|
|
1010
|
+
total,
|
|
1011
|
+
page,
|
|
1012
|
+
perPage,
|
|
1013
|
+
hasMore: offset + perPage < total
|
|
1014
|
+
};
|
|
1015
|
+
} catch (error) {
|
|
1016
|
+
const mastraError = new MastraError(
|
|
1017
|
+
{
|
|
1018
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
1019
|
+
domain: ErrorDomain.STORAGE,
|
|
1020
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1021
|
+
details: {
|
|
1022
|
+
threadId,
|
|
1023
|
+
resourceId: resourceId ?? ""
|
|
1024
|
+
}
|
|
1025
|
+
},
|
|
1026
|
+
error
|
|
1027
|
+
);
|
|
1028
|
+
this.logger?.trackException?.(mastraError);
|
|
1029
|
+
this.logger?.error?.(mastraError.toString());
|
|
1030
|
+
return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
async updateMessages(args) {
|
|
1034
|
+
const { messages } = args;
|
|
1035
|
+
if (messages.length === 0) {
|
|
1036
|
+
return [];
|
|
1037
|
+
}
|
|
1038
|
+
try {
|
|
1039
|
+
const messageIds = messages.map((m) => m.id);
|
|
1040
|
+
const existingResult = await this.client.query({
|
|
1041
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
1042
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
1043
|
+
clickhouse_settings: {
|
|
1044
|
+
date_time_input_format: "best_effort",
|
|
1045
|
+
date_time_output_format: "iso",
|
|
1046
|
+
use_client_time_zone: 1,
|
|
1047
|
+
output_format_json_quote_64bit_integers: 0
|
|
1048
|
+
}
|
|
1049
|
+
});
|
|
1050
|
+
const existingRows = await existingResult.json();
|
|
1051
|
+
const existingMessages = transformRows(existingRows.data);
|
|
1052
|
+
if (existingMessages.length === 0) {
|
|
1053
|
+
return [];
|
|
1054
|
+
}
|
|
1055
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
1056
|
+
if (typeof msg.content === "string") {
|
|
1057
|
+
try {
|
|
1058
|
+
msg.content = JSON.parse(msg.content);
|
|
1059
|
+
} catch {
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
return msg;
|
|
1063
|
+
});
|
|
1064
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
1065
|
+
const updatePromises = [];
|
|
1066
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1067
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1068
|
+
if (!updatePayload) continue;
|
|
1069
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1070
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1071
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1072
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1073
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1074
|
+
}
|
|
1075
|
+
const setClauses = [];
|
|
1076
|
+
const values = {};
|
|
1077
|
+
let paramIdx = 1;
|
|
1078
|
+
let newContent = null;
|
|
1079
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1080
|
+
if (updatableFields.content) {
|
|
1081
|
+
const existingContent = existingMessage.content || {};
|
|
1082
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1083
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1084
|
+
newContent = {
|
|
1085
|
+
...existingContent,
|
|
1086
|
+
...updatableFields.content,
|
|
1087
|
+
// Deep merge metadata
|
|
1088
|
+
metadata: {
|
|
1089
|
+
...existingMetadata,
|
|
1090
|
+
...updateMetadata
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1094
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1095
|
+
paramIdx++;
|
|
1096
|
+
delete updatableFields.content;
|
|
1097
|
+
}
|
|
1098
|
+
for (const key in updatableFields) {
|
|
1099
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1100
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1101
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1102
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1103
|
+
paramIdx++;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
if (setClauses.length > 0) {
|
|
1107
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1108
|
+
const updateQuery = `
|
|
1109
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1110
|
+
UPDATE ${setClauses.join(", ")}
|
|
1111
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1112
|
+
`;
|
|
1113
|
+
console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1114
|
+
updatePromises.push(
|
|
1115
|
+
this.client.command({
|
|
1116
|
+
query: updateQuery,
|
|
1117
|
+
query_params: values,
|
|
1118
|
+
clickhouse_settings: {
|
|
1119
|
+
date_time_input_format: "best_effort",
|
|
1120
|
+
use_client_time_zone: 1,
|
|
1121
|
+
output_format_json_quote_64bit_integers: 0
|
|
1122
|
+
}
|
|
1123
|
+
})
|
|
1124
|
+
);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
if (updatePromises.length > 0) {
|
|
1128
|
+
await Promise.all(updatePromises);
|
|
1129
|
+
}
|
|
1130
|
+
await this.client.command({
|
|
1131
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1132
|
+
clickhouse_settings: {
|
|
1133
|
+
date_time_input_format: "best_effort",
|
|
1134
|
+
use_client_time_zone: 1,
|
|
1135
|
+
output_format_json_quote_64bit_integers: 0
|
|
1136
|
+
}
|
|
1137
|
+
});
|
|
1138
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1139
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1140
|
+
if (!updatePayload) continue;
|
|
1141
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1142
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1143
|
+
const verifyResult = await this.client.query({
|
|
1144
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1145
|
+
query_params: { messageId: id },
|
|
1146
|
+
clickhouse_settings: {
|
|
1147
|
+
date_time_input_format: "best_effort",
|
|
1148
|
+
date_time_output_format: "iso",
|
|
1149
|
+
use_client_time_zone: 1,
|
|
1150
|
+
output_format_json_quote_64bit_integers: 0
|
|
1151
|
+
}
|
|
1152
|
+
});
|
|
1153
|
+
const verifyRows = await verifyResult.json();
|
|
1154
|
+
if (verifyRows.data.length > 0) {
|
|
1155
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1156
|
+
if (updatedMessage) {
|
|
1157
|
+
let needsRetry = false;
|
|
1158
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1159
|
+
if (key === "content") {
|
|
1160
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1161
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1162
|
+
if (actualContent !== expectedContent) {
|
|
1163
|
+
needsRetry = true;
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
} else if (updatedMessage[key] !== value) {
|
|
1167
|
+
needsRetry = true;
|
|
1168
|
+
break;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
if (needsRetry) {
|
|
1172
|
+
console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1175
|
+
query_params: { messageId: id },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
let updatedContent = existingMessage.content || {};
|
|
1183
|
+
if (fieldsToUpdate.content) {
|
|
1184
|
+
const existingContent = existingMessage.content || {};
|
|
1185
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1186
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1187
|
+
updatedContent = {
|
|
1188
|
+
...existingContent,
|
|
1189
|
+
...fieldsToUpdate.content,
|
|
1190
|
+
metadata: {
|
|
1191
|
+
...existingMetadata,
|
|
1192
|
+
...updateMetadata
|
|
1193
|
+
}
|
|
1194
|
+
};
|
|
1195
|
+
}
|
|
1196
|
+
const updatedMessageData = {
|
|
1197
|
+
...existingMessage,
|
|
1198
|
+
...fieldsToUpdate,
|
|
1199
|
+
content: updatedContent
|
|
1200
|
+
};
|
|
1201
|
+
await this.client.insert({
|
|
1202
|
+
table: TABLE_MESSAGES,
|
|
1203
|
+
format: "JSONEachRow",
|
|
1204
|
+
values: [
|
|
1205
|
+
{
|
|
1206
|
+
id: updatedMessageData.id,
|
|
1207
|
+
thread_id: updatedMessageData.threadId,
|
|
1208
|
+
resourceId: updatedMessageData.resourceId,
|
|
1209
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1210
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1211
|
+
role: updatedMessageData.role,
|
|
1212
|
+
type: updatedMessageData.type || "v2"
|
|
1213
|
+
}
|
|
1214
|
+
],
|
|
1215
|
+
clickhouse_settings: {
|
|
1216
|
+
date_time_input_format: "best_effort",
|
|
1217
|
+
use_client_time_zone: 1,
|
|
1218
|
+
output_format_json_quote_64bit_integers: 0
|
|
1219
|
+
}
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1226
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1227
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1228
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1229
|
+
const threadResult = await this.client.query({
|
|
1230
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1231
|
+
query_params: { threadId },
|
|
1232
|
+
clickhouse_settings: {
|
|
1233
|
+
date_time_input_format: "best_effort",
|
|
1234
|
+
date_time_output_format: "iso",
|
|
1235
|
+
use_client_time_zone: 1,
|
|
1236
|
+
output_format_json_quote_64bit_integers: 0
|
|
1237
|
+
}
|
|
1238
|
+
});
|
|
1239
|
+
const threadRows = await threadResult.json();
|
|
1240
|
+
if (threadRows.data.length > 0) {
|
|
1241
|
+
const existingThread = threadRows.data[0];
|
|
1242
|
+
await this.client.command({
|
|
1243
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1244
|
+
query_params: { threadId },
|
|
1245
|
+
clickhouse_settings: {
|
|
1246
|
+
date_time_input_format: "best_effort",
|
|
1247
|
+
use_client_time_zone: 1,
|
|
1248
|
+
output_format_json_quote_64bit_integers: 0
|
|
1249
|
+
}
|
|
1250
|
+
});
|
|
1251
|
+
await this.client.insert({
|
|
1252
|
+
table: TABLE_THREADS,
|
|
1253
|
+
format: "JSONEachRow",
|
|
1254
|
+
values: [
|
|
1255
|
+
{
|
|
1256
|
+
id: existingThread.id,
|
|
1257
|
+
resourceId: existingThread.resourceId,
|
|
1258
|
+
title: existingThread.title,
|
|
1259
|
+
metadata: existingThread.metadata,
|
|
1260
|
+
createdAt: existingThread.createdAt,
|
|
1261
|
+
updatedAt: now
|
|
1262
|
+
}
|
|
1263
|
+
],
|
|
1264
|
+
clickhouse_settings: {
|
|
1265
|
+
date_time_input_format: "best_effort",
|
|
1266
|
+
use_client_time_zone: 1,
|
|
1267
|
+
output_format_json_quote_64bit_integers: 0
|
|
1268
|
+
}
|
|
1269
|
+
});
|
|
1270
|
+
}
|
|
1271
|
+
});
|
|
1272
|
+
await Promise.all(threadUpdatePromises);
|
|
1273
|
+
}
|
|
1274
|
+
const updatedMessages = [];
|
|
1275
|
+
for (const messageId of messageIds) {
|
|
1276
|
+
const updatedResult = await this.client.query({
|
|
1277
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1278
|
+
query_params: { messageId },
|
|
1279
|
+
clickhouse_settings: {
|
|
1280
|
+
date_time_input_format: "best_effort",
|
|
1281
|
+
date_time_output_format: "iso",
|
|
1282
|
+
use_client_time_zone: 1,
|
|
1283
|
+
output_format_json_quote_64bit_integers: 0
|
|
1284
|
+
}
|
|
1285
|
+
});
|
|
1286
|
+
const updatedRows = await updatedResult.json();
|
|
1287
|
+
if (updatedRows.data.length > 0) {
|
|
1288
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1289
|
+
if (message) {
|
|
1290
|
+
updatedMessages.push(message);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
return updatedMessages.map((message) => {
|
|
1295
|
+
if (typeof message.content === "string") {
|
|
1296
|
+
try {
|
|
1297
|
+
message.content = JSON.parse(message.content);
|
|
1298
|
+
} catch {
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
return message;
|
|
1302
|
+
});
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
async getResourceById({ resourceId }) {
|
|
1316
|
+
try {
|
|
1317
|
+
const result = await this.client.query({
|
|
1318
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1319
|
+
query_params: { resourceId },
|
|
1320
|
+
clickhouse_settings: {
|
|
1321
|
+
date_time_input_format: "best_effort",
|
|
1322
|
+
date_time_output_format: "iso",
|
|
1323
|
+
use_client_time_zone: 1,
|
|
1324
|
+
output_format_json_quote_64bit_integers: 0
|
|
1325
|
+
}
|
|
1326
|
+
});
|
|
1327
|
+
const rows = await result.json();
|
|
1328
|
+
if (rows.data.length === 0) {
|
|
1329
|
+
return null;
|
|
1330
|
+
}
|
|
1331
|
+
const resource = rows.data[0];
|
|
1332
|
+
return {
|
|
1333
|
+
id: resource.id,
|
|
1334
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1335
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1336
|
+
createdAt: new Date(resource.createdAt),
|
|
1337
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1338
|
+
};
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
throw new MastraError(
|
|
1341
|
+
{
|
|
1342
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1343
|
+
domain: ErrorDomain.STORAGE,
|
|
1344
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1345
|
+
details: { resourceId }
|
|
1346
|
+
},
|
|
1347
|
+
error
|
|
1348
|
+
);
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
async saveResource({ resource }) {
|
|
1352
|
+
try {
|
|
1353
|
+
await this.client.insert({
|
|
1354
|
+
table: TABLE_RESOURCES,
|
|
1355
|
+
format: "JSONEachRow",
|
|
1356
|
+
values: [
|
|
1357
|
+
{
|
|
1358
|
+
id: resource.id,
|
|
1359
|
+
workingMemory: resource.workingMemory,
|
|
1360
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1361
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1362
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1363
|
+
}
|
|
1364
|
+
],
|
|
1365
|
+
clickhouse_settings: {
|
|
1366
|
+
date_time_input_format: "best_effort",
|
|
1367
|
+
use_client_time_zone: 1,
|
|
1368
|
+
output_format_json_quote_64bit_integers: 0
|
|
1369
|
+
}
|
|
1370
|
+
});
|
|
1371
|
+
return resource;
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
throw new MastraError(
|
|
1374
|
+
{
|
|
1375
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1376
|
+
domain: ErrorDomain.STORAGE,
|
|
1377
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1378
|
+
details: { resourceId: resource.id }
|
|
1379
|
+
},
|
|
1380
|
+
error
|
|
1381
|
+
);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
async updateResource({
|
|
1385
|
+
resourceId,
|
|
1386
|
+
workingMemory,
|
|
1387
|
+
metadata
|
|
1388
|
+
}) {
|
|
1389
|
+
try {
|
|
1390
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1391
|
+
if (!existingResource) {
|
|
1392
|
+
const newResource = {
|
|
1393
|
+
id: resourceId,
|
|
1394
|
+
workingMemory,
|
|
1395
|
+
metadata: metadata || {},
|
|
1396
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1397
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1398
|
+
};
|
|
1399
|
+
return this.saveResource({ resource: newResource });
|
|
1400
|
+
}
|
|
1401
|
+
const updatedResource = {
|
|
1402
|
+
...existingResource,
|
|
1403
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1404
|
+
metadata: {
|
|
1405
|
+
...existingResource.metadata,
|
|
1406
|
+
...metadata
|
|
1407
|
+
},
|
|
1408
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1409
|
+
};
|
|
1410
|
+
const updateQuery = `
|
|
1411
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1412
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1413
|
+
WHERE id = {resourceId:String}
|
|
1414
|
+
`;
|
|
1415
|
+
await this.client.command({
|
|
1416
|
+
query: updateQuery,
|
|
1417
|
+
query_params: {
|
|
1418
|
+
workingMemory: updatedResource.workingMemory,
|
|
1419
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1420
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1421
|
+
resourceId
|
|
1422
|
+
},
|
|
1423
|
+
clickhouse_settings: {
|
|
1424
|
+
date_time_input_format: "best_effort",
|
|
1425
|
+
use_client_time_zone: 1,
|
|
1426
|
+
output_format_json_quote_64bit_integers: 0
|
|
1427
|
+
}
|
|
1428
|
+
});
|
|
1429
|
+
await this.client.command({
|
|
1430
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1431
|
+
clickhouse_settings: {
|
|
1432
|
+
date_time_input_format: "best_effort",
|
|
1433
|
+
use_client_time_zone: 1,
|
|
1434
|
+
output_format_json_quote_64bit_integers: 0
|
|
1435
|
+
}
|
|
1436
|
+
});
|
|
1437
|
+
return updatedResource;
|
|
1438
|
+
} catch (error) {
|
|
1439
|
+
throw new MastraError(
|
|
1440
|
+
{
|
|
1441
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1442
|
+
domain: ErrorDomain.STORAGE,
|
|
1443
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1444
|
+
details: { resourceId }
|
|
1445
|
+
},
|
|
1446
|
+
error
|
|
1447
|
+
);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
};
|
|
1451
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1452
|
+
ttl;
|
|
1453
|
+
client;
|
|
1454
|
+
constructor({ client, ttl }) {
|
|
1455
|
+
super();
|
|
1456
|
+
this.ttl = ttl;
|
|
1457
|
+
this.client = client;
|
|
1458
|
+
}
|
|
1459
|
+
async hasColumn(table, column) {
|
|
1460
|
+
const result = await this.client.query({
|
|
1461
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1462
|
+
format: "JSONEachRow"
|
|
214
1463
|
});
|
|
1464
|
+
const columns = await result.json();
|
|
1465
|
+
return columns.some((c) => c.name === column);
|
|
215
1466
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
1467
|
+
getSqlType(type) {
|
|
1468
|
+
switch (type) {
|
|
1469
|
+
case "text":
|
|
1470
|
+
return "String";
|
|
1471
|
+
case "timestamp":
|
|
1472
|
+
return "DateTime64(3)";
|
|
1473
|
+
case "integer":
|
|
1474
|
+
case "bigint":
|
|
1475
|
+
return "Int64";
|
|
1476
|
+
case "jsonb":
|
|
1477
|
+
return "String";
|
|
1478
|
+
default:
|
|
1479
|
+
return super.getSqlType(type);
|
|
1480
|
+
}
|
|
220
1481
|
}
|
|
221
1482
|
async createTable({
|
|
222
1483
|
tableName,
|
|
@@ -231,27 +1492,25 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
231
1492
|
}).join(",\n");
|
|
232
1493
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
233
1494
|
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
`;
|
|
254
|
-
await this.db.query({
|
|
1495
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1496
|
+
${["id String"].concat(columns)}
|
|
1497
|
+
)
|
|
1498
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1499
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1500
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1501
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1502
|
+
SETTINGS index_granularity = 8192
|
|
1503
|
+
` : `
|
|
1504
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1505
|
+
${columns}
|
|
1506
|
+
)
|
|
1507
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1508
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1509
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1510
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1511
|
+
SETTINGS index_granularity = 8192
|
|
1512
|
+
`;
|
|
1513
|
+
await this.client.query({
|
|
255
1514
|
query: sql,
|
|
256
1515
|
clickhouse_settings: {
|
|
257
1516
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -262,13 +1521,59 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
262
1521
|
}
|
|
263
1522
|
});
|
|
264
1523
|
} catch (error) {
|
|
265
|
-
|
|
266
|
-
|
|
1524
|
+
throw new MastraError(
|
|
1525
|
+
{
|
|
1526
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1527
|
+
domain: ErrorDomain.STORAGE,
|
|
1528
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1529
|
+
details: { tableName }
|
|
1530
|
+
},
|
|
1531
|
+
error
|
|
1532
|
+
);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
async alterTable({
|
|
1536
|
+
tableName,
|
|
1537
|
+
schema,
|
|
1538
|
+
ifNotExists
|
|
1539
|
+
}) {
|
|
1540
|
+
try {
|
|
1541
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1542
|
+
const result = await this.client.query({
|
|
1543
|
+
query: describeSql
|
|
1544
|
+
});
|
|
1545
|
+
const rows = await result.json();
|
|
1546
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1547
|
+
for (const columnName of ifNotExists) {
|
|
1548
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1549
|
+
const columnDef = schema[columnName];
|
|
1550
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1551
|
+
if (columnDef.nullable !== false) {
|
|
1552
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1553
|
+
}
|
|
1554
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1555
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1556
|
+
await this.client.query({
|
|
1557
|
+
query: alterSql
|
|
1558
|
+
});
|
|
1559
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
} catch (error) {
|
|
1563
|
+
throw new MastraError(
|
|
1564
|
+
{
|
|
1565
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1566
|
+
domain: ErrorDomain.STORAGE,
|
|
1567
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1568
|
+
details: { tableName }
|
|
1569
|
+
},
|
|
1570
|
+
error
|
|
1571
|
+
);
|
|
267
1572
|
}
|
|
268
1573
|
}
|
|
269
1574
|
async clearTable({ tableName }) {
|
|
270
1575
|
try {
|
|
271
|
-
await this.
|
|
1576
|
+
await this.client.query({
|
|
272
1577
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
273
1578
|
clickhouse_settings: {
|
|
274
1579
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -279,19 +1584,33 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
279
1584
|
}
|
|
280
1585
|
});
|
|
281
1586
|
} catch (error) {
|
|
282
|
-
|
|
283
|
-
|
|
1587
|
+
throw new MastraError(
|
|
1588
|
+
{
|
|
1589
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1590
|
+
domain: ErrorDomain.STORAGE,
|
|
1591
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1592
|
+
details: { tableName }
|
|
1593
|
+
},
|
|
1594
|
+
error
|
|
1595
|
+
);
|
|
284
1596
|
}
|
|
285
1597
|
}
|
|
1598
|
+
async dropTable({ tableName }) {
|
|
1599
|
+
await this.client.query({
|
|
1600
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1601
|
+
});
|
|
1602
|
+
}
|
|
286
1603
|
async insert({ tableName, record }) {
|
|
1604
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1605
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
287
1606
|
try {
|
|
288
|
-
await this.
|
|
1607
|
+
const result = await this.client.insert({
|
|
289
1608
|
table: tableName,
|
|
290
1609
|
values: [
|
|
291
1610
|
{
|
|
292
1611
|
...record,
|
|
293
|
-
createdAt
|
|
294
|
-
updatedAt
|
|
1612
|
+
createdAt,
|
|
1613
|
+
updatedAt
|
|
295
1614
|
}
|
|
296
1615
|
],
|
|
297
1616
|
format: "JSONEachRow",
|
|
@@ -302,13 +1621,55 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
302
1621
|
use_client_time_zone: 1
|
|
303
1622
|
}
|
|
304
1623
|
});
|
|
1624
|
+
console.info("INSERT RESULT", result);
|
|
1625
|
+
} catch (error) {
|
|
1626
|
+
throw new MastraError(
|
|
1627
|
+
{
|
|
1628
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1629
|
+
domain: ErrorDomain.STORAGE,
|
|
1630
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1631
|
+
details: { tableName }
|
|
1632
|
+
},
|
|
1633
|
+
error
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
async batchInsert({ tableName, records }) {
|
|
1638
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1639
|
+
...Object.fromEntries(
|
|
1640
|
+
Object.entries(record).map(([key, value]) => [
|
|
1641
|
+
key,
|
|
1642
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1643
|
+
])
|
|
1644
|
+
)
|
|
1645
|
+
}));
|
|
1646
|
+
try {
|
|
1647
|
+
await this.client.insert({
|
|
1648
|
+
table: tableName,
|
|
1649
|
+
values: recordsToBeInserted,
|
|
1650
|
+
format: "JSONEachRow",
|
|
1651
|
+
clickhouse_settings: {
|
|
1652
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1653
|
+
date_time_input_format: "best_effort",
|
|
1654
|
+
use_client_time_zone: 1,
|
|
1655
|
+
output_format_json_quote_64bit_integers: 0
|
|
1656
|
+
}
|
|
1657
|
+
});
|
|
305
1658
|
} catch (error) {
|
|
306
|
-
|
|
307
|
-
|
|
1659
|
+
throw new MastraError(
|
|
1660
|
+
{
|
|
1661
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1662
|
+
domain: ErrorDomain.STORAGE,
|
|
1663
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1664
|
+
details: { tableName }
|
|
1665
|
+
},
|
|
1666
|
+
error
|
|
1667
|
+
);
|
|
308
1668
|
}
|
|
309
1669
|
}
|
|
310
1670
|
async load({ tableName, keys }) {
|
|
311
1671
|
try {
|
|
1672
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
312
1673
|
const keyEntries = Object.entries(keys);
|
|
313
1674
|
const conditions = keyEntries.map(
|
|
314
1675
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -316,8 +1677,10 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
316
1677
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
317
1678
|
return { ...acc, [`var_${key}`]: value };
|
|
318
1679
|
}, {});
|
|
319
|
-
const
|
|
320
|
-
|
|
1680
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1681
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1682
|
+
const result = await this.client.query({
|
|
1683
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
321
1684
|
query_params: values,
|
|
322
1685
|
clickhouse_settings: {
|
|
323
1686
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -344,24 +1707,57 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
344
1707
|
const data = transformRow(rows.data[0]);
|
|
345
1708
|
return data;
|
|
346
1709
|
} catch (error) {
|
|
347
|
-
|
|
348
|
-
|
|
1710
|
+
throw new MastraError(
|
|
1711
|
+
{
|
|
1712
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1713
|
+
domain: ErrorDomain.STORAGE,
|
|
1714
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1715
|
+
details: { tableName }
|
|
1716
|
+
},
|
|
1717
|
+
error
|
|
1718
|
+
);
|
|
349
1719
|
}
|
|
350
1720
|
}
|
|
351
|
-
|
|
1721
|
+
};
|
|
1722
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1723
|
+
client;
|
|
1724
|
+
operations;
|
|
1725
|
+
constructor({ client, operations }) {
|
|
1726
|
+
super();
|
|
1727
|
+
this.client = client;
|
|
1728
|
+
this.operations = operations;
|
|
1729
|
+
}
|
|
1730
|
+
transformScoreRow(row) {
|
|
1731
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1732
|
+
const preprocessStepResult = safelyParseJSON(row.preprocessStepResult);
|
|
1733
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1734
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1735
|
+
const input = safelyParseJSON(row.input);
|
|
1736
|
+
const output = safelyParseJSON(row.output);
|
|
1737
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1738
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1739
|
+
const entity = safelyParseJSON(row.entity);
|
|
1740
|
+
return {
|
|
1741
|
+
...row,
|
|
1742
|
+
scorer,
|
|
1743
|
+
preprocessStepResult,
|
|
1744
|
+
analyzeStepResult,
|
|
1745
|
+
metadata,
|
|
1746
|
+
input,
|
|
1747
|
+
output,
|
|
1748
|
+
additionalContext,
|
|
1749
|
+
runtimeContext,
|
|
1750
|
+
entity,
|
|
1751
|
+
createdAt: new Date(row.createdAt),
|
|
1752
|
+
updatedAt: new Date(row.updatedAt)
|
|
1753
|
+
};
|
|
1754
|
+
}
|
|
1755
|
+
async getScoreById({ id }) {
|
|
352
1756
|
try {
|
|
353
|
-
const result = await this.
|
|
354
|
-
query: `SELECT
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
title,
|
|
358
|
-
metadata,
|
|
359
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
360
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
361
|
-
FROM "${TABLE_THREADS}"
|
|
362
|
-
FINAL
|
|
363
|
-
WHERE id = {var_id:String}`,
|
|
364
|
-
query_params: { var_id: threadId },
|
|
1757
|
+
const result = await this.client.query({
|
|
1758
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1759
|
+
query_params: { var_id: id },
|
|
1760
|
+
format: "JSONEachRow",
|
|
365
1761
|
clickhouse_settings: {
|
|
366
1762
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
367
1763
|
date_time_input_format: "best_effort",
|
|
@@ -370,304 +1766,654 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
370
1766
|
output_format_json_quote_64bit_integers: 0
|
|
371
1767
|
}
|
|
372
1768
|
});
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
if (!thread) {
|
|
1769
|
+
const resultJson = await result.json();
|
|
1770
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
376
1771
|
return null;
|
|
377
1772
|
}
|
|
378
|
-
return
|
|
379
|
-
...thread,
|
|
380
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
381
|
-
createdAt: thread.createdAt,
|
|
382
|
-
updatedAt: thread.updatedAt
|
|
383
|
-
};
|
|
1773
|
+
return this.transformScoreRow(resultJson[0]);
|
|
384
1774
|
} catch (error) {
|
|
385
|
-
|
|
386
|
-
|
|
1775
|
+
throw new MastraError(
|
|
1776
|
+
{
|
|
1777
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1778
|
+
domain: ErrorDomain.STORAGE,
|
|
1779
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1780
|
+
details: { scoreId: id }
|
|
1781
|
+
},
|
|
1782
|
+
error
|
|
1783
|
+
);
|
|
387
1784
|
}
|
|
388
1785
|
}
|
|
389
|
-
async
|
|
1786
|
+
async saveScore(score) {
|
|
1787
|
+
let parsedScore;
|
|
390
1788
|
try {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
1789
|
+
parsedScore = saveScorePayloadSchema.parse(score);
|
|
1790
|
+
} catch (error) {
|
|
1791
|
+
throw new MastraError(
|
|
1792
|
+
{
|
|
1793
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED_INVALID_SCORE_PAYLOAD",
|
|
1794
|
+
domain: ErrorDomain.STORAGE,
|
|
1795
|
+
category: ErrorCategory.USER,
|
|
1796
|
+
details: { scoreId: score.id }
|
|
1797
|
+
},
|
|
1798
|
+
error
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
try {
|
|
1802
|
+
const record = {
|
|
1803
|
+
...parsedScore
|
|
1804
|
+
};
|
|
1805
|
+
await this.client.insert({
|
|
1806
|
+
table: TABLE_SCORERS,
|
|
1807
|
+
values: [record],
|
|
1808
|
+
format: "JSONEachRow",
|
|
402
1809
|
clickhouse_settings: {
|
|
403
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
404
1810
|
date_time_input_format: "best_effort",
|
|
405
|
-
date_time_output_format: "iso",
|
|
406
1811
|
use_client_time_zone: 1,
|
|
407
1812
|
output_format_json_quote_64bit_integers: 0
|
|
408
1813
|
}
|
|
409
1814
|
});
|
|
410
|
-
|
|
411
|
-
const threads = transformRows(rows.data);
|
|
412
|
-
return threads.map((thread) => ({
|
|
413
|
-
...thread,
|
|
414
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
415
|
-
createdAt: thread.createdAt,
|
|
416
|
-
updatedAt: thread.updatedAt
|
|
417
|
-
}));
|
|
1815
|
+
return { score };
|
|
418
1816
|
} catch (error) {
|
|
419
|
-
|
|
420
|
-
|
|
1817
|
+
throw new MastraError(
|
|
1818
|
+
{
|
|
1819
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1820
|
+
domain: ErrorDomain.STORAGE,
|
|
1821
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1822
|
+
details: { scoreId: score.id }
|
|
1823
|
+
},
|
|
1824
|
+
error
|
|
1825
|
+
);
|
|
421
1826
|
}
|
|
422
1827
|
}
|
|
423
|
-
async
|
|
1828
|
+
async getScoresByRunId({
|
|
1829
|
+
runId,
|
|
1830
|
+
pagination
|
|
1831
|
+
}) {
|
|
424
1832
|
try {
|
|
425
|
-
await this.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
]
|
|
1833
|
+
const countResult = await this.client.query({
|
|
1834
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1835
|
+
query_params: { var_runId: runId },
|
|
1836
|
+
format: "JSONEachRow"
|
|
1837
|
+
});
|
|
1838
|
+
const countRows = await countResult.json();
|
|
1839
|
+
let total = 0;
|
|
1840
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1841
|
+
const countObj = countRows[0];
|
|
1842
|
+
total = Number(countObj.count);
|
|
1843
|
+
}
|
|
1844
|
+
if (!total) {
|
|
1845
|
+
return {
|
|
1846
|
+
pagination: {
|
|
1847
|
+
total: 0,
|
|
1848
|
+
page: pagination.page,
|
|
1849
|
+
perPage: pagination.perPage,
|
|
1850
|
+
hasMore: false
|
|
1851
|
+
},
|
|
1852
|
+
scores: []
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
const offset = pagination.page * pagination.perPage;
|
|
1856
|
+
const result = await this.client.query({
|
|
1857
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1858
|
+
query_params: {
|
|
1859
|
+
var_runId: runId,
|
|
1860
|
+
var_limit: pagination.perPage,
|
|
1861
|
+
var_offset: offset
|
|
1862
|
+
},
|
|
434
1863
|
format: "JSONEachRow",
|
|
435
1864
|
clickhouse_settings: {
|
|
436
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
437
1865
|
date_time_input_format: "best_effort",
|
|
1866
|
+
date_time_output_format: "iso",
|
|
438
1867
|
use_client_time_zone: 1,
|
|
439
1868
|
output_format_json_quote_64bit_integers: 0
|
|
440
1869
|
}
|
|
441
1870
|
});
|
|
442
|
-
|
|
1871
|
+
const rows = await result.json();
|
|
1872
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1873
|
+
return {
|
|
1874
|
+
pagination: {
|
|
1875
|
+
total,
|
|
1876
|
+
page: pagination.page,
|
|
1877
|
+
perPage: pagination.perPage,
|
|
1878
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1879
|
+
},
|
|
1880
|
+
scores
|
|
1881
|
+
};
|
|
443
1882
|
} catch (error) {
|
|
444
|
-
|
|
445
|
-
|
|
1883
|
+
throw new MastraError(
|
|
1884
|
+
{
|
|
1885
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1886
|
+
domain: ErrorDomain.STORAGE,
|
|
1887
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1888
|
+
details: { runId }
|
|
1889
|
+
},
|
|
1890
|
+
error
|
|
1891
|
+
);
|
|
446
1892
|
}
|
|
447
1893
|
}
|
|
448
|
-
async
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
1894
|
+
async getScoresByScorerId({
|
|
1895
|
+
scorerId,
|
|
1896
|
+
entityId,
|
|
1897
|
+
entityType,
|
|
1898
|
+
source,
|
|
1899
|
+
pagination
|
|
452
1900
|
}) {
|
|
1901
|
+
let whereClause = `scorerId = {var_scorerId:String}`;
|
|
1902
|
+
if (entityId) {
|
|
1903
|
+
whereClause += ` AND entityId = {var_entityId:String}`;
|
|
1904
|
+
}
|
|
1905
|
+
if (entityType) {
|
|
1906
|
+
whereClause += ` AND entityType = {var_entityType:String}`;
|
|
1907
|
+
}
|
|
1908
|
+
if (source) {
|
|
1909
|
+
whereClause += ` AND source = {var_source:String}`;
|
|
1910
|
+
}
|
|
453
1911
|
try {
|
|
454
|
-
const
|
|
455
|
-
|
|
456
|
-
|
|
1912
|
+
const countResult = await this.client.query({
|
|
1913
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE ${whereClause}`,
|
|
1914
|
+
query_params: {
|
|
1915
|
+
var_scorerId: scorerId,
|
|
1916
|
+
var_entityId: entityId,
|
|
1917
|
+
var_entityType: entityType,
|
|
1918
|
+
var_source: source
|
|
1919
|
+
},
|
|
1920
|
+
format: "JSONEachRow"
|
|
1921
|
+
});
|
|
1922
|
+
const countRows = await countResult.json();
|
|
1923
|
+
let total = 0;
|
|
1924
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1925
|
+
const countObj = countRows[0];
|
|
1926
|
+
total = Number(countObj.count);
|
|
457
1927
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
1928
|
+
if (!total) {
|
|
1929
|
+
return {
|
|
1930
|
+
pagination: {
|
|
1931
|
+
total: 0,
|
|
1932
|
+
page: pagination.page,
|
|
1933
|
+
perPage: pagination.perPage,
|
|
1934
|
+
hasMore: false
|
|
1935
|
+
},
|
|
1936
|
+
scores: []
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
const offset = pagination.page * pagination.perPage;
|
|
1940
|
+
const result = await this.client.query({
|
|
1941
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1942
|
+
query_params: {
|
|
1943
|
+
var_scorerId: scorerId,
|
|
1944
|
+
var_limit: pagination.perPage,
|
|
1945
|
+
var_offset: offset,
|
|
1946
|
+
var_entityId: entityId,
|
|
1947
|
+
var_entityType: entityType,
|
|
1948
|
+
var_source: source
|
|
1949
|
+
},
|
|
476
1950
|
format: "JSONEachRow",
|
|
477
1951
|
clickhouse_settings: {
|
|
478
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
479
1952
|
date_time_input_format: "best_effort",
|
|
1953
|
+
date_time_output_format: "iso",
|
|
480
1954
|
use_client_time_zone: 1,
|
|
481
1955
|
output_format_json_quote_64bit_integers: 0
|
|
482
1956
|
}
|
|
483
1957
|
});
|
|
484
|
-
|
|
1958
|
+
const rows = await result.json();
|
|
1959
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1960
|
+
return {
|
|
1961
|
+
pagination: {
|
|
1962
|
+
total,
|
|
1963
|
+
page: pagination.page,
|
|
1964
|
+
perPage: pagination.perPage,
|
|
1965
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1966
|
+
},
|
|
1967
|
+
scores
|
|
1968
|
+
};
|
|
485
1969
|
} catch (error) {
|
|
486
|
-
|
|
487
|
-
|
|
1970
|
+
throw new MastraError(
|
|
1971
|
+
{
|
|
1972
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1973
|
+
domain: ErrorDomain.STORAGE,
|
|
1974
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1975
|
+
details: { scorerId }
|
|
1976
|
+
},
|
|
1977
|
+
error
|
|
1978
|
+
);
|
|
488
1979
|
}
|
|
489
1980
|
}
|
|
490
|
-
async
|
|
1981
|
+
async getScoresByEntityId({
|
|
1982
|
+
entityId,
|
|
1983
|
+
entityType,
|
|
1984
|
+
pagination
|
|
1985
|
+
}) {
|
|
491
1986
|
try {
|
|
492
|
-
await this.
|
|
493
|
-
query: `
|
|
494
|
-
query_params: {
|
|
495
|
-
|
|
496
|
-
output_format_json_quote_64bit_integers: 0
|
|
497
|
-
}
|
|
1987
|
+
const countResult = await this.client.query({
|
|
1988
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1989
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1990
|
+
format: "JSONEachRow"
|
|
498
1991
|
});
|
|
499
|
-
await
|
|
500
|
-
|
|
501
|
-
|
|
1992
|
+
const countRows = await countResult.json();
|
|
1993
|
+
let total = 0;
|
|
1994
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1995
|
+
const countObj = countRows[0];
|
|
1996
|
+
total = Number(countObj.count);
|
|
1997
|
+
}
|
|
1998
|
+
if (!total) {
|
|
1999
|
+
return {
|
|
2000
|
+
pagination: {
|
|
2001
|
+
total: 0,
|
|
2002
|
+
page: pagination.page,
|
|
2003
|
+
perPage: pagination.perPage,
|
|
2004
|
+
hasMore: false
|
|
2005
|
+
},
|
|
2006
|
+
scores: []
|
|
2007
|
+
};
|
|
2008
|
+
}
|
|
2009
|
+
const offset = pagination.page * pagination.perPage;
|
|
2010
|
+
const result = await this.client.query({
|
|
2011
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
2012
|
+
query_params: {
|
|
2013
|
+
var_entityId: entityId,
|
|
2014
|
+
var_entityType: entityType,
|
|
2015
|
+
var_limit: pagination.perPage,
|
|
2016
|
+
var_offset: offset
|
|
2017
|
+
},
|
|
2018
|
+
format: "JSONEachRow",
|
|
502
2019
|
clickhouse_settings: {
|
|
2020
|
+
date_time_input_format: "best_effort",
|
|
2021
|
+
date_time_output_format: "iso",
|
|
2022
|
+
use_client_time_zone: 1,
|
|
503
2023
|
output_format_json_quote_64bit_integers: 0
|
|
504
2024
|
}
|
|
505
2025
|
});
|
|
2026
|
+
const rows = await result.json();
|
|
2027
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2028
|
+
return {
|
|
2029
|
+
pagination: {
|
|
2030
|
+
total,
|
|
2031
|
+
page: pagination.page,
|
|
2032
|
+
perPage: pagination.perPage,
|
|
2033
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
2034
|
+
},
|
|
2035
|
+
scores
|
|
2036
|
+
};
|
|
506
2037
|
} catch (error) {
|
|
507
|
-
|
|
508
|
-
|
|
2038
|
+
throw new MastraError(
|
|
2039
|
+
{
|
|
2040
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
2041
|
+
domain: ErrorDomain.STORAGE,
|
|
2042
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2043
|
+
details: { entityId, entityType }
|
|
2044
|
+
},
|
|
2045
|
+
error
|
|
2046
|
+
);
|
|
509
2047
|
}
|
|
510
2048
|
}
|
|
511
|
-
async
|
|
2049
|
+
async getScoresBySpan({
|
|
2050
|
+
traceId,
|
|
2051
|
+
spanId,
|
|
2052
|
+
pagination
|
|
2053
|
+
}) {
|
|
512
2054
|
try {
|
|
513
|
-
const
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
m.updatedAt as updatedAt,
|
|
535
|
-
m.thread_id AS "threadId"
|
|
536
|
-
FROM ordered_messages m
|
|
537
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
538
|
-
OR EXISTS (
|
|
539
|
-
SELECT 1 FROM ordered_messages target
|
|
540
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
541
|
-
AND (
|
|
542
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
543
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
544
|
-
OR
|
|
545
|
-
-- Get next messages based on the max withNextMessages
|
|
546
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
547
|
-
)
|
|
548
|
-
)
|
|
549
|
-
ORDER BY m."createdAt" DESC
|
|
550
|
-
`,
|
|
551
|
-
query_params: {
|
|
552
|
-
var_thread_id: threadId,
|
|
553
|
-
var_include: include.map((i) => i.id),
|
|
554
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
555
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
2055
|
+
const countResult = await this.client.query({
|
|
2056
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String}`,
|
|
2057
|
+
query_params: {
|
|
2058
|
+
var_traceId: traceId,
|
|
2059
|
+
var_spanId: spanId
|
|
2060
|
+
},
|
|
2061
|
+
format: "JSONEachRow"
|
|
2062
|
+
});
|
|
2063
|
+
const countRows = await countResult.json();
|
|
2064
|
+
let total = 0;
|
|
2065
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
2066
|
+
const countObj = countRows[0];
|
|
2067
|
+
total = Number(countObj.count);
|
|
2068
|
+
}
|
|
2069
|
+
if (!total) {
|
|
2070
|
+
return {
|
|
2071
|
+
pagination: {
|
|
2072
|
+
total: 0,
|
|
2073
|
+
page: pagination.page,
|
|
2074
|
+
perPage: pagination.perPage,
|
|
2075
|
+
hasMore: false
|
|
556
2076
|
},
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
date_time_input_format: "best_effort",
|
|
560
|
-
date_time_output_format: "iso",
|
|
561
|
-
use_client_time_zone: 1,
|
|
562
|
-
output_format_json_quote_64bit_integers: 0
|
|
563
|
-
}
|
|
564
|
-
});
|
|
565
|
-
const rows2 = await includeResult.json();
|
|
566
|
-
messages.push(...transformRows(rows2.data));
|
|
2077
|
+
scores: []
|
|
2078
|
+
};
|
|
567
2079
|
}
|
|
568
|
-
const
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
content,
|
|
573
|
-
role,
|
|
574
|
-
type,
|
|
575
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
576
|
-
thread_id AS "threadId"
|
|
577
|
-
FROM "${TABLE_MESSAGES}"
|
|
578
|
-
WHERE thread_id = {threadId:String}
|
|
579
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
580
|
-
ORDER BY "createdAt" DESC
|
|
581
|
-
LIMIT {limit:Int64}
|
|
582
|
-
`,
|
|
2080
|
+
const limit = pagination.perPage + 1;
|
|
2081
|
+
const offset = pagination.page * pagination.perPage;
|
|
2082
|
+
const result = await this.client.query({
|
|
2083
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
583
2084
|
query_params: {
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
limit
|
|
2085
|
+
var_traceId: traceId,
|
|
2086
|
+
var_spanId: spanId,
|
|
2087
|
+
var_limit: limit,
|
|
2088
|
+
var_offset: offset
|
|
2089
|
+
},
|
|
2090
|
+
format: "JSONEachRow",
|
|
2091
|
+
clickhouse_settings: {
|
|
2092
|
+
date_time_input_format: "best_effort",
|
|
2093
|
+
date_time_output_format: "iso",
|
|
2094
|
+
use_client_time_zone: 1,
|
|
2095
|
+
output_format_json_quote_64bit_integers: 0
|
|
2096
|
+
}
|
|
2097
|
+
});
|
|
2098
|
+
const rows = await result.json();
|
|
2099
|
+
const transformedRows = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2100
|
+
const hasMore = transformedRows.length > pagination.perPage;
|
|
2101
|
+
const scores = hasMore ? transformedRows.slice(0, pagination.perPage) : transformedRows;
|
|
2102
|
+
return {
|
|
2103
|
+
pagination: {
|
|
2104
|
+
total,
|
|
2105
|
+
page: pagination.page,
|
|
2106
|
+
perPage: pagination.perPage,
|
|
2107
|
+
hasMore
|
|
587
2108
|
},
|
|
2109
|
+
scores
|
|
2110
|
+
};
|
|
2111
|
+
} catch (error) {
|
|
2112
|
+
throw new MastraError(
|
|
2113
|
+
{
|
|
2114
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
|
|
2115
|
+
domain: ErrorDomain.STORAGE,
|
|
2116
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2117
|
+
details: { traceId, spanId }
|
|
2118
|
+
},
|
|
2119
|
+
error
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
};
|
|
2124
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
2125
|
+
client;
|
|
2126
|
+
operations;
|
|
2127
|
+
constructor({ client, operations }) {
|
|
2128
|
+
super();
|
|
2129
|
+
this.client = client;
|
|
2130
|
+
this.operations = operations;
|
|
2131
|
+
}
|
|
2132
|
+
async getTracesPaginated(args) {
|
|
2133
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
2134
|
+
const fromDate = dateRange?.start;
|
|
2135
|
+
const toDate = dateRange?.end;
|
|
2136
|
+
const currentOffset = page * perPage;
|
|
2137
|
+
const queryArgs = {};
|
|
2138
|
+
const conditions = [];
|
|
2139
|
+
if (name) {
|
|
2140
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2141
|
+
queryArgs.var_name = name;
|
|
2142
|
+
}
|
|
2143
|
+
if (scope) {
|
|
2144
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2145
|
+
queryArgs.var_scope = scope;
|
|
2146
|
+
}
|
|
2147
|
+
if (attributes) {
|
|
2148
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2149
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2150
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
2151
|
+
});
|
|
2152
|
+
}
|
|
2153
|
+
if (filters) {
|
|
2154
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2155
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2156
|
+
queryArgs[`var_col_${key}`] = value;
|
|
2157
|
+
});
|
|
2158
|
+
}
|
|
2159
|
+
if (fromDate) {
|
|
2160
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
2161
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
2162
|
+
}
|
|
2163
|
+
if (toDate) {
|
|
2164
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
2165
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
2166
|
+
}
|
|
2167
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2168
|
+
try {
|
|
2169
|
+
const countResult = await this.client.query({
|
|
2170
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
2171
|
+
query_params: queryArgs,
|
|
588
2172
|
clickhouse_settings: {
|
|
589
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
590
2173
|
date_time_input_format: "best_effort",
|
|
591
2174
|
date_time_output_format: "iso",
|
|
592
2175
|
use_client_time_zone: 1,
|
|
593
2176
|
output_format_json_quote_64bit_integers: 0
|
|
594
2177
|
}
|
|
595
2178
|
});
|
|
596
|
-
const
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
2179
|
+
const countData = await countResult.json();
|
|
2180
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2181
|
+
if (total === 0) {
|
|
2182
|
+
return {
|
|
2183
|
+
traces: [],
|
|
2184
|
+
total: 0,
|
|
2185
|
+
page,
|
|
2186
|
+
perPage,
|
|
2187
|
+
hasMore: false
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
const result = await this.client.query({
|
|
2191
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2192
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2193
|
+
clickhouse_settings: {
|
|
2194
|
+
date_time_input_format: "best_effort",
|
|
2195
|
+
date_time_output_format: "iso",
|
|
2196
|
+
use_client_time_zone: 1,
|
|
2197
|
+
output_format_json_quote_64bit_integers: 0
|
|
605
2198
|
}
|
|
606
2199
|
});
|
|
607
|
-
|
|
2200
|
+
if (!result) {
|
|
2201
|
+
return {
|
|
2202
|
+
traces: [],
|
|
2203
|
+
total,
|
|
2204
|
+
page,
|
|
2205
|
+
perPage,
|
|
2206
|
+
hasMore: false
|
|
2207
|
+
};
|
|
2208
|
+
}
|
|
2209
|
+
const resp = await result.json();
|
|
2210
|
+
const rows = resp.data;
|
|
2211
|
+
const traces = rows.map((row) => ({
|
|
2212
|
+
id: row.id,
|
|
2213
|
+
parentSpanId: row.parentSpanId,
|
|
2214
|
+
traceId: row.traceId,
|
|
2215
|
+
name: row.name,
|
|
2216
|
+
scope: row.scope,
|
|
2217
|
+
kind: row.kind,
|
|
2218
|
+
status: safelyParseJSON(row.status),
|
|
2219
|
+
events: safelyParseJSON(row.events),
|
|
2220
|
+
links: safelyParseJSON(row.links),
|
|
2221
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2222
|
+
startTime: row.startTime,
|
|
2223
|
+
endTime: row.endTime,
|
|
2224
|
+
other: safelyParseJSON(row.other),
|
|
2225
|
+
createdAt: row.createdAt
|
|
2226
|
+
}));
|
|
2227
|
+
return {
|
|
2228
|
+
traces,
|
|
2229
|
+
total,
|
|
2230
|
+
page,
|
|
2231
|
+
perPage,
|
|
2232
|
+
hasMore: currentOffset + traces.length < total
|
|
2233
|
+
};
|
|
608
2234
|
} catch (error) {
|
|
609
|
-
|
|
610
|
-
|
|
2235
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2236
|
+
return {
|
|
2237
|
+
traces: [],
|
|
2238
|
+
total: 0,
|
|
2239
|
+
page,
|
|
2240
|
+
perPage,
|
|
2241
|
+
hasMore: false
|
|
2242
|
+
};
|
|
2243
|
+
}
|
|
2244
|
+
throw new MastraError(
|
|
2245
|
+
{
|
|
2246
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2247
|
+
domain: ErrorDomain.STORAGE,
|
|
2248
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2249
|
+
details: {
|
|
2250
|
+
name: name ?? null,
|
|
2251
|
+
scope: scope ?? null,
|
|
2252
|
+
page,
|
|
2253
|
+
perPage,
|
|
2254
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2255
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2256
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2257
|
+
}
|
|
2258
|
+
},
|
|
2259
|
+
error
|
|
2260
|
+
);
|
|
611
2261
|
}
|
|
612
2262
|
}
|
|
613
|
-
async
|
|
614
|
-
|
|
2263
|
+
async getTraces({
|
|
2264
|
+
name,
|
|
2265
|
+
scope,
|
|
2266
|
+
page,
|
|
2267
|
+
perPage,
|
|
2268
|
+
attributes,
|
|
2269
|
+
filters,
|
|
2270
|
+
fromDate,
|
|
2271
|
+
toDate
|
|
2272
|
+
}) {
|
|
2273
|
+
const limit = perPage;
|
|
2274
|
+
const offset = page * perPage;
|
|
2275
|
+
const args = {};
|
|
2276
|
+
const conditions = [];
|
|
2277
|
+
if (name) {
|
|
2278
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2279
|
+
args.var_name = name;
|
|
2280
|
+
}
|
|
2281
|
+
if (scope) {
|
|
2282
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2283
|
+
args.var_scope = scope;
|
|
2284
|
+
}
|
|
2285
|
+
if (attributes) {
|
|
2286
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2287
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2288
|
+
args[`var_attr_${key}`] = value;
|
|
2289
|
+
});
|
|
2290
|
+
}
|
|
2291
|
+
if (filters) {
|
|
2292
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2293
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2294
|
+
args[`var_col_${key}`] = value;
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
if (fromDate) {
|
|
2298
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2299
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2300
|
+
}
|
|
2301
|
+
if (toDate) {
|
|
2302
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2303
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2304
|
+
}
|
|
2305
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
615
2306
|
try {
|
|
616
|
-
const
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
}
|
|
620
|
-
const thread = await this.getThreadById({ threadId });
|
|
621
|
-
if (!thread) {
|
|
622
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
623
|
-
}
|
|
624
|
-
await this.db.insert({
|
|
625
|
-
table: TABLE_MESSAGES,
|
|
626
|
-
format: "JSONEachRow",
|
|
627
|
-
values: messages.map((message) => ({
|
|
628
|
-
id: message.id,
|
|
629
|
-
thread_id: threadId,
|
|
630
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
631
|
-
createdAt: message.createdAt.toISOString(),
|
|
632
|
-
role: message.role,
|
|
633
|
-
type: message.type
|
|
634
|
-
})),
|
|
2307
|
+
const result = await this.client.query({
|
|
2308
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2309
|
+
query_params: args,
|
|
635
2310
|
clickhouse_settings: {
|
|
636
2311
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
637
2312
|
date_time_input_format: "best_effort",
|
|
2313
|
+
date_time_output_format: "iso",
|
|
638
2314
|
use_client_time_zone: 1,
|
|
639
2315
|
output_format_json_quote_64bit_integers: 0
|
|
640
2316
|
}
|
|
641
2317
|
});
|
|
642
|
-
|
|
2318
|
+
if (!result) {
|
|
2319
|
+
return [];
|
|
2320
|
+
}
|
|
2321
|
+
const resp = await result.json();
|
|
2322
|
+
const rows = resp.data;
|
|
2323
|
+
return rows.map((row) => ({
|
|
2324
|
+
id: row.id,
|
|
2325
|
+
parentSpanId: row.parentSpanId,
|
|
2326
|
+
traceId: row.traceId,
|
|
2327
|
+
name: row.name,
|
|
2328
|
+
scope: row.scope,
|
|
2329
|
+
kind: row.kind,
|
|
2330
|
+
status: safelyParseJSON(row.status),
|
|
2331
|
+
events: safelyParseJSON(row.events),
|
|
2332
|
+
links: safelyParseJSON(row.links),
|
|
2333
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2334
|
+
startTime: row.startTime,
|
|
2335
|
+
endTime: row.endTime,
|
|
2336
|
+
other: safelyParseJSON(row.other),
|
|
2337
|
+
createdAt: row.createdAt
|
|
2338
|
+
}));
|
|
643
2339
|
} catch (error) {
|
|
644
|
-
|
|
645
|
-
|
|
2340
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2341
|
+
return [];
|
|
2342
|
+
}
|
|
2343
|
+
throw new MastraError(
|
|
2344
|
+
{
|
|
2345
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2346
|
+
domain: ErrorDomain.STORAGE,
|
|
2347
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2348
|
+
details: {
|
|
2349
|
+
name: name ?? null,
|
|
2350
|
+
scope: scope ?? null,
|
|
2351
|
+
page,
|
|
2352
|
+
perPage,
|
|
2353
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2354
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2355
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2356
|
+
toDate: toDate?.toISOString() ?? null
|
|
2357
|
+
}
|
|
2358
|
+
},
|
|
2359
|
+
error
|
|
2360
|
+
);
|
|
646
2361
|
}
|
|
647
2362
|
}
|
|
2363
|
+
async batchTraceInsert(args) {
|
|
2364
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2365
|
+
}
|
|
2366
|
+
};
|
|
2367
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2368
|
+
client;
|
|
2369
|
+
operations;
|
|
2370
|
+
constructor({ client, operations }) {
|
|
2371
|
+
super();
|
|
2372
|
+
this.operations = operations;
|
|
2373
|
+
this.client = client;
|
|
2374
|
+
}
|
|
2375
|
+
updateWorkflowResults({
|
|
2376
|
+
// workflowName,
|
|
2377
|
+
// runId,
|
|
2378
|
+
// stepId,
|
|
2379
|
+
// result,
|
|
2380
|
+
// runtimeContext,
|
|
2381
|
+
}) {
|
|
2382
|
+
throw new Error("Method not implemented.");
|
|
2383
|
+
}
|
|
2384
|
+
updateWorkflowState({
|
|
2385
|
+
// workflowName,
|
|
2386
|
+
// runId,
|
|
2387
|
+
// opts,
|
|
2388
|
+
}) {
|
|
2389
|
+
throw new Error("Method not implemented.");
|
|
2390
|
+
}
|
|
648
2391
|
async persistWorkflowSnapshot({
|
|
649
2392
|
workflowName,
|
|
650
2393
|
runId,
|
|
2394
|
+
resourceId,
|
|
651
2395
|
snapshot
|
|
652
2396
|
}) {
|
|
653
2397
|
try {
|
|
654
|
-
const currentSnapshot = await this.load({
|
|
2398
|
+
const currentSnapshot = await this.operations.load({
|
|
655
2399
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
656
2400
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
657
2401
|
});
|
|
658
2402
|
const now = /* @__PURE__ */ new Date();
|
|
659
2403
|
const persisting = currentSnapshot ? {
|
|
660
2404
|
...currentSnapshot,
|
|
2405
|
+
resourceId,
|
|
661
2406
|
snapshot: JSON.stringify(snapshot),
|
|
662
2407
|
updatedAt: now.toISOString()
|
|
663
2408
|
} : {
|
|
664
2409
|
workflow_name: workflowName,
|
|
665
2410
|
run_id: runId,
|
|
2411
|
+
resourceId,
|
|
666
2412
|
snapshot: JSON.stringify(snapshot),
|
|
667
2413
|
createdAt: now.toISOString(),
|
|
668
2414
|
updatedAt: now.toISOString()
|
|
669
2415
|
};
|
|
670
|
-
await this.
|
|
2416
|
+
await this.client.insert({
|
|
671
2417
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
672
2418
|
format: "JSONEachRow",
|
|
673
2419
|
values: [persisting],
|
|
@@ -679,8 +2425,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
679
2425
|
}
|
|
680
2426
|
});
|
|
681
2427
|
} catch (error) {
|
|
682
|
-
|
|
683
|
-
|
|
2428
|
+
throw new MastraError(
|
|
2429
|
+
{
|
|
2430
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2431
|
+
domain: ErrorDomain.STORAGE,
|
|
2432
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2433
|
+
details: { workflowName, runId }
|
|
2434
|
+
},
|
|
2435
|
+
error
|
|
2436
|
+
);
|
|
684
2437
|
}
|
|
685
2438
|
}
|
|
686
2439
|
async loadWorkflowSnapshot({
|
|
@@ -688,7 +2441,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
688
2441
|
runId
|
|
689
2442
|
}) {
|
|
690
2443
|
try {
|
|
691
|
-
const result = await this.load({
|
|
2444
|
+
const result = await this.operations.load({
|
|
692
2445
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
693
2446
|
keys: {
|
|
694
2447
|
workflow_name: workflowName,
|
|
@@ -700,8 +2453,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
700
2453
|
}
|
|
701
2454
|
return result.snapshot;
|
|
702
2455
|
} catch (error) {
|
|
703
|
-
|
|
704
|
-
|
|
2456
|
+
throw new MastraError(
|
|
2457
|
+
{
|
|
2458
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2459
|
+
domain: ErrorDomain.STORAGE,
|
|
2460
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2461
|
+
details: { workflowName, runId }
|
|
2462
|
+
},
|
|
2463
|
+
error
|
|
2464
|
+
);
|
|
705
2465
|
}
|
|
706
2466
|
}
|
|
707
2467
|
parseWorkflowRun(row) {
|
|
@@ -738,7 +2498,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
738
2498
|
values.var_workflow_name = workflowName;
|
|
739
2499
|
}
|
|
740
2500
|
if (resourceId) {
|
|
741
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2501
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
742
2502
|
if (hasResourceId) {
|
|
743
2503
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
744
2504
|
values.var_resourceId = resourceId;
|
|
@@ -759,7 +2519,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
759
2519
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
760
2520
|
let total = 0;
|
|
761
2521
|
if (limit !== void 0 && offset !== void 0) {
|
|
762
|
-
const countResult = await this.
|
|
2522
|
+
const countResult = await this.client.query({
|
|
763
2523
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
764
2524
|
query_params: values,
|
|
765
2525
|
format: "JSONEachRow"
|
|
@@ -767,21 +2527,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
767
2527
|
const countRows = await countResult.json();
|
|
768
2528
|
total = Number(countRows[0]?.count ?? 0);
|
|
769
2529
|
}
|
|
770
|
-
const result = await this.
|
|
2530
|
+
const result = await this.client.query({
|
|
771
2531
|
query: `
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2532
|
+
SELECT
|
|
2533
|
+
workflow_name,
|
|
2534
|
+
run_id,
|
|
2535
|
+
snapshot,
|
|
2536
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2537
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2538
|
+
resourceId
|
|
2539
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2540
|
+
${whereClause}
|
|
2541
|
+
ORDER BY createdAt DESC
|
|
2542
|
+
${limitClause}
|
|
2543
|
+
${offsetClause}
|
|
2544
|
+
`,
|
|
785
2545
|
query_params: values,
|
|
786
2546
|
format: "JSONEachRow"
|
|
787
2547
|
});
|
|
@@ -792,8 +2552,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
792
2552
|
});
|
|
793
2553
|
return { runs, total: total || runs.length };
|
|
794
2554
|
} catch (error) {
|
|
795
|
-
|
|
796
|
-
|
|
2555
|
+
throw new MastraError(
|
|
2556
|
+
{
|
|
2557
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2558
|
+
domain: ErrorDomain.STORAGE,
|
|
2559
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2560
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2561
|
+
},
|
|
2562
|
+
error
|
|
2563
|
+
);
|
|
797
2564
|
}
|
|
798
2565
|
}
|
|
799
2566
|
async getWorkflowRunById({
|
|
@@ -812,18 +2579,19 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
812
2579
|
values.var_workflow_name = workflowName;
|
|
813
2580
|
}
|
|
814
2581
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
815
|
-
const result = await this.
|
|
2582
|
+
const result = await this.client.query({
|
|
816
2583
|
query: `
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
2584
|
+
SELECT
|
|
2585
|
+
workflow_name,
|
|
2586
|
+
run_id,
|
|
2587
|
+
snapshot,
|
|
2588
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2589
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2590
|
+
resourceId
|
|
2591
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2592
|
+
${whereClause}
|
|
2593
|
+
ORDER BY createdAt DESC LIMIT 1
|
|
2594
|
+
`,
|
|
827
2595
|
query_params: values,
|
|
828
2596
|
format: "JSONEachRow"
|
|
829
2597
|
});
|
|
@@ -833,17 +2601,279 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
833
2601
|
}
|
|
834
2602
|
return this.parseWorkflowRun(resultJson[0]);
|
|
835
2603
|
} catch (error) {
|
|
836
|
-
|
|
837
|
-
|
|
2604
|
+
throw new MastraError(
|
|
2605
|
+
{
|
|
2606
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2607
|
+
domain: ErrorDomain.STORAGE,
|
|
2608
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2609
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2610
|
+
},
|
|
2611
|
+
error
|
|
2612
|
+
);
|
|
838
2613
|
}
|
|
839
2614
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
2615
|
+
};
|
|
2616
|
+
|
|
2617
|
+
// src/storage/index.ts
|
|
2618
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2619
|
+
db;
|
|
2620
|
+
ttl = {};
|
|
2621
|
+
stores;
|
|
2622
|
+
constructor(config) {
|
|
2623
|
+
super({ name: "ClickhouseStore" });
|
|
2624
|
+
this.db = createClient({
|
|
2625
|
+
url: config.url,
|
|
2626
|
+
username: config.username,
|
|
2627
|
+
password: config.password,
|
|
2628
|
+
clickhouse_settings: {
|
|
2629
|
+
date_time_input_format: "best_effort",
|
|
2630
|
+
date_time_output_format: "iso",
|
|
2631
|
+
// This is crucial
|
|
2632
|
+
use_client_time_zone: 1,
|
|
2633
|
+
output_format_json_quote_64bit_integers: 0
|
|
2634
|
+
}
|
|
844
2635
|
});
|
|
845
|
-
|
|
846
|
-
|
|
2636
|
+
this.ttl = config.ttl;
|
|
2637
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2638
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2639
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2640
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2641
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2642
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2643
|
+
this.stores = {
|
|
2644
|
+
operations,
|
|
2645
|
+
workflows,
|
|
2646
|
+
scores,
|
|
2647
|
+
legacyEvals,
|
|
2648
|
+
traces,
|
|
2649
|
+
memory
|
|
2650
|
+
};
|
|
2651
|
+
}
|
|
2652
|
+
get supports() {
|
|
2653
|
+
return {
|
|
2654
|
+
selectByIncludeResourceScope: true,
|
|
2655
|
+
resourceWorkingMemory: true,
|
|
2656
|
+
hasColumn: true,
|
|
2657
|
+
createTable: true,
|
|
2658
|
+
deleteMessages: false,
|
|
2659
|
+
getScoresBySpan: true
|
|
2660
|
+
};
|
|
2661
|
+
}
|
|
2662
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2663
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2664
|
+
}
|
|
2665
|
+
async getEvals(options) {
|
|
2666
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2667
|
+
}
|
|
2668
|
+
async batchInsert({ tableName, records }) {
|
|
2669
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2670
|
+
}
|
|
2671
|
+
async optimizeTable({ tableName }) {
|
|
2672
|
+
try {
|
|
2673
|
+
await this.db.command({
|
|
2674
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2675
|
+
});
|
|
2676
|
+
} catch (error) {
|
|
2677
|
+
throw new MastraError(
|
|
2678
|
+
{
|
|
2679
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2680
|
+
domain: ErrorDomain.STORAGE,
|
|
2681
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2682
|
+
details: { tableName }
|
|
2683
|
+
},
|
|
2684
|
+
error
|
|
2685
|
+
);
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
async materializeTtl({ tableName }) {
|
|
2689
|
+
try {
|
|
2690
|
+
await this.db.command({
|
|
2691
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2692
|
+
});
|
|
2693
|
+
} catch (error) {
|
|
2694
|
+
throw new MastraError(
|
|
2695
|
+
{
|
|
2696
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2697
|
+
domain: ErrorDomain.STORAGE,
|
|
2698
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2699
|
+
details: { tableName }
|
|
2700
|
+
},
|
|
2701
|
+
error
|
|
2702
|
+
);
|
|
2703
|
+
}
|
|
2704
|
+
}
|
|
2705
|
+
async createTable({
|
|
2706
|
+
tableName,
|
|
2707
|
+
schema
|
|
2708
|
+
}) {
|
|
2709
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2710
|
+
}
|
|
2711
|
+
async dropTable({ tableName }) {
|
|
2712
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2713
|
+
}
|
|
2714
|
+
async alterTable({
|
|
2715
|
+
tableName,
|
|
2716
|
+
schema,
|
|
2717
|
+
ifNotExists
|
|
2718
|
+
}) {
|
|
2719
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2720
|
+
}
|
|
2721
|
+
async clearTable({ tableName }) {
|
|
2722
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2723
|
+
}
|
|
2724
|
+
async insert({ tableName, record }) {
|
|
2725
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2726
|
+
}
|
|
2727
|
+
async load({ tableName, keys }) {
|
|
2728
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2729
|
+
}
|
|
2730
|
+
async updateWorkflowResults({
|
|
2731
|
+
workflowName,
|
|
2732
|
+
runId,
|
|
2733
|
+
stepId,
|
|
2734
|
+
result,
|
|
2735
|
+
runtimeContext
|
|
2736
|
+
}) {
|
|
2737
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2738
|
+
}
|
|
2739
|
+
async updateWorkflowState({
|
|
2740
|
+
workflowName,
|
|
2741
|
+
runId,
|
|
2742
|
+
opts
|
|
2743
|
+
}) {
|
|
2744
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2745
|
+
}
|
|
2746
|
+
async persistWorkflowSnapshot({
|
|
2747
|
+
workflowName,
|
|
2748
|
+
runId,
|
|
2749
|
+
resourceId,
|
|
2750
|
+
snapshot
|
|
2751
|
+
}) {
|
|
2752
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2753
|
+
}
|
|
2754
|
+
async loadWorkflowSnapshot({
|
|
2755
|
+
workflowName,
|
|
2756
|
+
runId
|
|
2757
|
+
}) {
|
|
2758
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2759
|
+
}
|
|
2760
|
+
async getWorkflowRuns({
|
|
2761
|
+
workflowName,
|
|
2762
|
+
fromDate,
|
|
2763
|
+
toDate,
|
|
2764
|
+
limit,
|
|
2765
|
+
offset,
|
|
2766
|
+
resourceId
|
|
2767
|
+
} = {}) {
|
|
2768
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2769
|
+
}
|
|
2770
|
+
async getWorkflowRunById({
|
|
2771
|
+
runId,
|
|
2772
|
+
workflowName
|
|
2773
|
+
}) {
|
|
2774
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2775
|
+
}
|
|
2776
|
+
async getTraces(args) {
|
|
2777
|
+
return this.stores.traces.getTraces(args);
|
|
2778
|
+
}
|
|
2779
|
+
async getTracesPaginated(args) {
|
|
2780
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2781
|
+
}
|
|
2782
|
+
async batchTraceInsert(args) {
|
|
2783
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2784
|
+
}
|
|
2785
|
+
async getThreadById({ threadId }) {
|
|
2786
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2787
|
+
}
|
|
2788
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2789
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2790
|
+
}
|
|
2791
|
+
async saveThread({ thread }) {
|
|
2792
|
+
return this.stores.memory.saveThread({ thread });
|
|
2793
|
+
}
|
|
2794
|
+
async updateThread({
|
|
2795
|
+
id,
|
|
2796
|
+
title,
|
|
2797
|
+
metadata
|
|
2798
|
+
}) {
|
|
2799
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2800
|
+
}
|
|
2801
|
+
async deleteThread({ threadId }) {
|
|
2802
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2803
|
+
}
|
|
2804
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2805
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2806
|
+
}
|
|
2807
|
+
async getMessages({
|
|
2808
|
+
threadId,
|
|
2809
|
+
resourceId,
|
|
2810
|
+
selectBy,
|
|
2811
|
+
format
|
|
2812
|
+
}) {
|
|
2813
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2814
|
+
}
|
|
2815
|
+
async getMessagesById({
|
|
2816
|
+
messageIds,
|
|
2817
|
+
format
|
|
2818
|
+
}) {
|
|
2819
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2820
|
+
}
|
|
2821
|
+
async saveMessages(args) {
|
|
2822
|
+
return this.stores.memory.saveMessages(args);
|
|
2823
|
+
}
|
|
2824
|
+
async getMessagesPaginated(args) {
|
|
2825
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2826
|
+
}
|
|
2827
|
+
async updateMessages(args) {
|
|
2828
|
+
return this.stores.memory.updateMessages(args);
|
|
2829
|
+
}
|
|
2830
|
+
async getResourceById({ resourceId }) {
|
|
2831
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2832
|
+
}
|
|
2833
|
+
async saveResource({ resource }) {
|
|
2834
|
+
return this.stores.memory.saveResource({ resource });
|
|
2835
|
+
}
|
|
2836
|
+
async updateResource({
|
|
2837
|
+
resourceId,
|
|
2838
|
+
workingMemory,
|
|
2839
|
+
metadata
|
|
2840
|
+
}) {
|
|
2841
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2842
|
+
}
|
|
2843
|
+
async getScoreById({ id }) {
|
|
2844
|
+
return this.stores.scores.getScoreById({ id });
|
|
2845
|
+
}
|
|
2846
|
+
async saveScore(_score) {
|
|
2847
|
+
return this.stores.scores.saveScore(_score);
|
|
2848
|
+
}
|
|
2849
|
+
async getScoresByRunId({
|
|
2850
|
+
runId,
|
|
2851
|
+
pagination
|
|
2852
|
+
}) {
|
|
2853
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2854
|
+
}
|
|
2855
|
+
async getScoresByEntityId({
|
|
2856
|
+
entityId,
|
|
2857
|
+
entityType,
|
|
2858
|
+
pagination
|
|
2859
|
+
}) {
|
|
2860
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2861
|
+
}
|
|
2862
|
+
async getScoresByScorerId({
|
|
2863
|
+
scorerId,
|
|
2864
|
+
pagination,
|
|
2865
|
+
entityId,
|
|
2866
|
+
entityType,
|
|
2867
|
+
source
|
|
2868
|
+
}) {
|
|
2869
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
|
|
2870
|
+
}
|
|
2871
|
+
async getScoresBySpan({
|
|
2872
|
+
traceId,
|
|
2873
|
+
spanId,
|
|
2874
|
+
pagination
|
|
2875
|
+
}) {
|
|
2876
|
+
return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
|
|
847
2877
|
}
|
|
848
2878
|
async close() {
|
|
849
2879
|
await this.db.close();
|
|
@@ -851,3 +2881,5 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
851
2881
|
};
|
|
852
2882
|
|
|
853
2883
|
export { COLUMN_TYPES, ClickhouseStore, TABLE_ENGINES };
|
|
2884
|
+
//# sourceMappingURL=index.js.map
|
|
2885
|
+
//# sourceMappingURL=index.js.map
|