@mastra/clickhouse 0.0.0-taofeeqInngest-20250603090617 → 0.0.0-transpile-packages-20250724123433
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +219 -2
- package/LICENSE.md +12 -4
- package/dist/_tsup-dts-rollup.d.cts +358 -29
- package/dist/_tsup-dts-rollup.d.ts +358 -29
- package/dist/index.cjs +2239 -454
- package/dist/index.d.cts +0 -2
- package/dist/index.d.ts +0 -2
- package/dist/index.js +2222 -435
- package/package.json +12 -11
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +14 -836
- package/src/storage/index.ts +228 -901
- package/.turbo/turbo-build.log +0 -23
package/dist/index.js
CHANGED
|
@@ -1,21 +1,17 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { MastraStorage, StoreOperations, TABLE_SCHEMAS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, ScoresStorage, safelyParseJSON, TABLE_SCORERS, LegacyEvalsStorage, TABLE_EVALS, TracesStorage, TABLE_TRACES, MemoryStorage, resolveMessageLimit, TABLE_MESSAGES, TABLE_THREADS, TABLE_RESOURCES } from '@mastra/core/storage';
|
|
2
4
|
import { MessageList } from '@mastra/core/agent';
|
|
3
|
-
import { TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, TABLE_SCHEMAS } from '@mastra/core/storage';
|
|
4
5
|
|
|
5
6
|
// src/storage/index.ts
|
|
6
|
-
function safelyParseJSON(jsonString) {
|
|
7
|
-
try {
|
|
8
|
-
return JSON.parse(jsonString);
|
|
9
|
-
} catch {
|
|
10
|
-
return {};
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
7
|
var TABLE_ENGINES = {
|
|
14
8
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
15
9
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
16
10
|
[TABLE_TRACES]: `MergeTree()`,
|
|
17
11
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
18
|
-
[TABLE_EVALS]: `MergeTree()
|
|
12
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
13
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
14
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`
|
|
19
15
|
};
|
|
20
16
|
var COLUMN_TYPES = {
|
|
21
17
|
text: "String",
|
|
@@ -23,11 +19,9 @@ var COLUMN_TYPES = {
|
|
|
23
19
|
uuid: "String",
|
|
24
20
|
jsonb: "String",
|
|
25
21
|
integer: "Int64",
|
|
22
|
+
float: "Float64",
|
|
26
23
|
bigint: "Int64"
|
|
27
24
|
};
|
|
28
|
-
function transformRows(rows) {
|
|
29
|
-
return rows.map((row) => transformRow(row));
|
|
30
|
-
}
|
|
31
25
|
function transformRow(row) {
|
|
32
26
|
if (!row) {
|
|
33
27
|
return row;
|
|
@@ -38,33 +32,63 @@ function transformRow(row) {
|
|
|
38
32
|
if (row.updatedAt) {
|
|
39
33
|
row.updatedAt = new Date(row.updatedAt);
|
|
40
34
|
}
|
|
35
|
+
if (row.content && typeof row.content === "string") {
|
|
36
|
+
row.content = safelyParseJSON(row.content);
|
|
37
|
+
}
|
|
41
38
|
return row;
|
|
42
39
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
// This is crucial
|
|
56
|
-
use_client_time_zone: 1,
|
|
57
|
-
output_format_json_quote_64bit_integers: 0
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
this.ttl = config.ttl;
|
|
40
|
+
function transformRows(rows) {
|
|
41
|
+
return rows.map((row) => transformRow(row));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
45
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
46
|
+
client;
|
|
47
|
+
operations;
|
|
48
|
+
constructor({ client, operations }) {
|
|
49
|
+
super();
|
|
50
|
+
this.client = client;
|
|
51
|
+
this.operations = operations;
|
|
61
52
|
}
|
|
62
53
|
transformEvalRow(row) {
|
|
63
54
|
row = transformRow(row);
|
|
64
|
-
|
|
65
|
-
|
|
55
|
+
let resultValue;
|
|
56
|
+
try {
|
|
57
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
58
|
+
resultValue = JSON.parse(row.result);
|
|
59
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
60
|
+
resultValue = row.result;
|
|
61
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
62
|
+
resultValue = { score: 0 };
|
|
63
|
+
} else {
|
|
64
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
console.error("Error parsing result field:", row.result, error);
|
|
68
|
+
throw new MastraError({
|
|
69
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
70
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
71
|
+
domain: ErrorDomain.STORAGE,
|
|
72
|
+
category: ErrorCategory.USER
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
let testInfoValue;
|
|
76
|
+
try {
|
|
77
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
78
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
79
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
80
|
+
testInfoValue = row.test_info;
|
|
81
|
+
}
|
|
82
|
+
} catch {
|
|
83
|
+
testInfoValue = void 0;
|
|
84
|
+
}
|
|
66
85
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
67
|
-
throw new
|
|
86
|
+
throw new MastraError({
|
|
87
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
88
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
89
|
+
domain: ErrorDomain.STORAGE,
|
|
90
|
+
category: ErrorCategory.USER
|
|
91
|
+
});
|
|
68
92
|
}
|
|
69
93
|
return {
|
|
70
94
|
input: row.input,
|
|
@@ -81,9 +105,9 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
81
105
|
}
|
|
82
106
|
async getEvalsByAgentName(agentName, type) {
|
|
83
107
|
try {
|
|
84
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
85
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
86
|
-
const result = await this.
|
|
108
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
109
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
110
|
+
const result = await this.client.query({
|
|
87
111
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
88
112
|
query_params: { var_agent_name: agentName },
|
|
89
113
|
clickhouse_settings: {
|
|
@@ -99,125 +123,1292 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
99
123
|
const rows = await result.json();
|
|
100
124
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
101
125
|
} catch (error) {
|
|
102
|
-
if (error
|
|
126
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
103
127
|
return [];
|
|
104
128
|
}
|
|
105
|
-
|
|
106
|
-
|
|
129
|
+
throw new MastraError(
|
|
130
|
+
{
|
|
131
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
132
|
+
domain: ErrorDomain.STORAGE,
|
|
133
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
134
|
+
details: { agentName, type: type ?? null }
|
|
135
|
+
},
|
|
136
|
+
error
|
|
137
|
+
);
|
|
107
138
|
}
|
|
108
139
|
}
|
|
109
|
-
async
|
|
140
|
+
async getEvals(options = {}) {
|
|
141
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
142
|
+
const fromDate = dateRange?.start;
|
|
143
|
+
const toDate = dateRange?.end;
|
|
144
|
+
const conditions = [];
|
|
145
|
+
if (agentName) {
|
|
146
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
147
|
+
}
|
|
148
|
+
if (type === "test") {
|
|
149
|
+
conditions.push(
|
|
150
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
151
|
+
);
|
|
152
|
+
} else if (type === "live") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
if (fromDate) {
|
|
158
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
159
|
+
fromDate.toISOString();
|
|
160
|
+
}
|
|
161
|
+
if (toDate) {
|
|
162
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
163
|
+
toDate.toISOString();
|
|
164
|
+
}
|
|
165
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
110
166
|
try {
|
|
111
|
-
await this.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
...
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
167
|
+
const countResult = await this.client.query({
|
|
168
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
169
|
+
query_params: {
|
|
170
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
171
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
172
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
173
|
+
},
|
|
174
|
+
clickhouse_settings: {
|
|
175
|
+
date_time_input_format: "best_effort",
|
|
176
|
+
date_time_output_format: "iso",
|
|
177
|
+
use_client_time_zone: 1,
|
|
178
|
+
output_format_json_quote_64bit_integers: 0
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
const countData = await countResult.json();
|
|
182
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
183
|
+
const currentOffset = page * perPage;
|
|
184
|
+
const hasMore = currentOffset + perPage < total;
|
|
185
|
+
if (total === 0) {
|
|
186
|
+
return {
|
|
187
|
+
evals: [],
|
|
188
|
+
total: 0,
|
|
189
|
+
page,
|
|
190
|
+
perPage,
|
|
191
|
+
hasMore: false
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
const dataResult = await this.client.query({
|
|
195
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
196
|
+
query_params: {
|
|
197
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
198
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
199
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
200
|
+
var_limit: perPage || 100,
|
|
201
|
+
var_offset: currentOffset || 0
|
|
202
|
+
},
|
|
122
203
|
clickhouse_settings: {
|
|
123
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
124
204
|
date_time_input_format: "best_effort",
|
|
205
|
+
date_time_output_format: "iso",
|
|
125
206
|
use_client_time_zone: 1,
|
|
126
207
|
output_format_json_quote_64bit_integers: 0
|
|
127
208
|
}
|
|
128
209
|
});
|
|
210
|
+
const rows = await dataResult.json();
|
|
211
|
+
return {
|
|
212
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
213
|
+
total,
|
|
214
|
+
page,
|
|
215
|
+
perPage,
|
|
216
|
+
hasMore
|
|
217
|
+
};
|
|
129
218
|
} catch (error) {
|
|
130
|
-
|
|
131
|
-
|
|
219
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
220
|
+
return {
|
|
221
|
+
evals: [],
|
|
222
|
+
total: 0,
|
|
223
|
+
page,
|
|
224
|
+
perPage,
|
|
225
|
+
hasMore: false
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
throw new MastraError(
|
|
229
|
+
{
|
|
230
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
231
|
+
domain: ErrorDomain.STORAGE,
|
|
232
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
233
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
234
|
+
},
|
|
235
|
+
error
|
|
236
|
+
);
|
|
132
237
|
}
|
|
133
238
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
239
|
+
};
|
|
240
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
241
|
+
client;
|
|
242
|
+
operations;
|
|
243
|
+
constructor({ client, operations }) {
|
|
244
|
+
super();
|
|
245
|
+
this.client = client;
|
|
246
|
+
this.operations = operations;
|
|
247
|
+
}
|
|
248
|
+
async getMessages({
|
|
249
|
+
threadId,
|
|
250
|
+
resourceId,
|
|
251
|
+
selectBy,
|
|
252
|
+
format
|
|
143
253
|
}) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
254
|
+
try {
|
|
255
|
+
const messages = [];
|
|
256
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
257
|
+
const include = selectBy?.include || [];
|
|
258
|
+
if (include.length) {
|
|
259
|
+
const unionQueries = [];
|
|
260
|
+
const params = [];
|
|
261
|
+
let paramIdx = 1;
|
|
262
|
+
for (const inc of include) {
|
|
263
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
264
|
+
const searchId = inc.threadId || threadId;
|
|
265
|
+
unionQueries.push(`
|
|
266
|
+
SELECT * FROM (
|
|
267
|
+
WITH numbered_messages AS (
|
|
268
|
+
SELECT
|
|
269
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
270
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
271
|
+
FROM "${TABLE_MESSAGES}"
|
|
272
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
273
|
+
),
|
|
274
|
+
target_positions AS (
|
|
275
|
+
SELECT row_num as target_pos
|
|
276
|
+
FROM numbered_messages
|
|
277
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
278
|
+
)
|
|
279
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
280
|
+
FROM numbered_messages m
|
|
281
|
+
CROSS JOIN target_positions t
|
|
282
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
283
|
+
) AS query_${paramIdx}
|
|
284
|
+
`);
|
|
285
|
+
params.push(
|
|
286
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
287
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
288
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
289
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
290
|
+
);
|
|
291
|
+
paramIdx++;
|
|
292
|
+
}
|
|
293
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
294
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
295
|
+
const includeResult = await this.client.query({
|
|
296
|
+
query: finalQuery,
|
|
297
|
+
query_params: mergedParams,
|
|
298
|
+
clickhouse_settings: {
|
|
299
|
+
date_time_input_format: "best_effort",
|
|
300
|
+
date_time_output_format: "iso",
|
|
301
|
+
use_client_time_zone: 1,
|
|
302
|
+
output_format_json_quote_64bit_integers: 0
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
const rows2 = await includeResult.json();
|
|
306
|
+
const includedMessages = transformRows(rows2.data);
|
|
307
|
+
const seen = /* @__PURE__ */ new Set();
|
|
308
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
309
|
+
if (seen.has(message.id)) return false;
|
|
310
|
+
seen.add(message.id);
|
|
311
|
+
return true;
|
|
312
|
+
});
|
|
313
|
+
messages.push(...dedupedMessages);
|
|
314
|
+
}
|
|
315
|
+
const result = await this.client.query({
|
|
316
|
+
query: `
|
|
317
|
+
SELECT
|
|
318
|
+
id,
|
|
319
|
+
content,
|
|
320
|
+
role,
|
|
321
|
+
type,
|
|
322
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
323
|
+
thread_id AS "threadId"
|
|
324
|
+
FROM "${TABLE_MESSAGES}"
|
|
325
|
+
WHERE thread_id = {threadId:String}
|
|
326
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
327
|
+
ORDER BY "createdAt" DESC
|
|
328
|
+
LIMIT {limit:Int64}
|
|
329
|
+
`,
|
|
330
|
+
query_params: {
|
|
331
|
+
threadId,
|
|
332
|
+
exclude: messages.map((m) => m.id),
|
|
333
|
+
limit
|
|
334
|
+
},
|
|
335
|
+
clickhouse_settings: {
|
|
336
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
337
|
+
date_time_input_format: "best_effort",
|
|
338
|
+
date_time_output_format: "iso",
|
|
339
|
+
use_client_time_zone: 1,
|
|
340
|
+
output_format_json_quote_64bit_integers: 0
|
|
341
|
+
}
|
|
160
342
|
});
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
343
|
+
const rows = await result.json();
|
|
344
|
+
messages.push(...transformRows(rows.data));
|
|
345
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
346
|
+
messages.forEach((message) => {
|
|
347
|
+
if (typeof message.content === "string") {
|
|
348
|
+
try {
|
|
349
|
+
message.content = JSON.parse(message.content);
|
|
350
|
+
} catch {
|
|
351
|
+
}
|
|
352
|
+
}
|
|
168
353
|
});
|
|
354
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
355
|
+
if (format === `v2`) return list.get.all.v2();
|
|
356
|
+
return list.get.all.v1();
|
|
357
|
+
} catch (error) {
|
|
358
|
+
throw new MastraError(
|
|
359
|
+
{
|
|
360
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
361
|
+
domain: ErrorDomain.STORAGE,
|
|
362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
363
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
364
|
+
},
|
|
365
|
+
error
|
|
366
|
+
);
|
|
169
367
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
368
|
+
}
|
|
369
|
+
async saveMessages(args) {
|
|
370
|
+
const { messages, format = "v1" } = args;
|
|
371
|
+
if (messages.length === 0) return messages;
|
|
372
|
+
for (const message of messages) {
|
|
373
|
+
const resourceId = message.resourceId;
|
|
374
|
+
if (!resourceId) {
|
|
375
|
+
throw new Error("Resource ID is required");
|
|
376
|
+
}
|
|
377
|
+
if (!message.threadId) {
|
|
378
|
+
throw new Error("Thread ID is required");
|
|
379
|
+
}
|
|
380
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
381
|
+
if (!thread) {
|
|
382
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
383
|
+
}
|
|
173
384
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
385
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
386
|
+
await Promise.all(
|
|
387
|
+
messages.map(async (m) => {
|
|
388
|
+
const resourceId = m.resourceId;
|
|
389
|
+
if (!resourceId) {
|
|
390
|
+
throw new Error("Resource ID is required");
|
|
391
|
+
}
|
|
392
|
+
if (!m.threadId) {
|
|
393
|
+
throw new Error("Thread ID is required");
|
|
394
|
+
}
|
|
395
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
396
|
+
if (!thread) {
|
|
397
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
398
|
+
}
|
|
399
|
+
threadIdSet.set(m.threadId, thread);
|
|
400
|
+
})
|
|
401
|
+
);
|
|
402
|
+
try {
|
|
403
|
+
const existingResult = await this.client.query({
|
|
404
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
405
|
+
query_params: {
|
|
406
|
+
ids: messages.map((m) => m.id)
|
|
407
|
+
},
|
|
408
|
+
clickhouse_settings: {
|
|
409
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
410
|
+
date_time_input_format: "best_effort",
|
|
411
|
+
date_time_output_format: "iso",
|
|
412
|
+
use_client_time_zone: 1,
|
|
413
|
+
output_format_json_quote_64bit_integers: 0
|
|
414
|
+
},
|
|
415
|
+
format: "JSONEachRow"
|
|
416
|
+
});
|
|
417
|
+
const existingRows = await existingResult.json();
|
|
418
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
419
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
420
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
421
|
+
const toMove = messages.filter((m) => {
|
|
422
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
423
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
424
|
+
});
|
|
425
|
+
const deletePromises = toMove.map((message) => {
|
|
426
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
427
|
+
if (!existingRow) return Promise.resolve();
|
|
428
|
+
return this.client.command({
|
|
429
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
430
|
+
query_params: {
|
|
431
|
+
var_id: message.id,
|
|
432
|
+
var_old_thread_id: existingRow.thread_id
|
|
433
|
+
},
|
|
434
|
+
clickhouse_settings: {
|
|
435
|
+
date_time_input_format: "best_effort",
|
|
436
|
+
use_client_time_zone: 1,
|
|
437
|
+
output_format_json_quote_64bit_integers: 0
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
});
|
|
441
|
+
const updatePromises = toUpdate.map(
|
|
442
|
+
(message) => this.client.command({
|
|
443
|
+
query: `
|
|
444
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
445
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
446
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
447
|
+
`,
|
|
448
|
+
query_params: {
|
|
449
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
450
|
+
var_role: message.role,
|
|
451
|
+
var_type: message.type || "v2",
|
|
452
|
+
var_resourceId: message.resourceId,
|
|
453
|
+
var_id: message.id,
|
|
454
|
+
var_thread_id: message.threadId
|
|
455
|
+
},
|
|
456
|
+
clickhouse_settings: {
|
|
457
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
458
|
+
date_time_input_format: "best_effort",
|
|
459
|
+
use_client_time_zone: 1,
|
|
460
|
+
output_format_json_quote_64bit_integers: 0
|
|
461
|
+
}
|
|
462
|
+
})
|
|
463
|
+
);
|
|
464
|
+
await Promise.all([
|
|
465
|
+
// Insert new messages (including moved messages)
|
|
466
|
+
this.client.insert({
|
|
467
|
+
table: TABLE_MESSAGES,
|
|
468
|
+
format: "JSONEachRow",
|
|
469
|
+
values: toInsert.map((message) => ({
|
|
470
|
+
id: message.id,
|
|
471
|
+
thread_id: message.threadId,
|
|
472
|
+
resourceId: message.resourceId,
|
|
473
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
474
|
+
createdAt: message.createdAt.toISOString(),
|
|
475
|
+
role: message.role,
|
|
476
|
+
type: message.type || "v2"
|
|
477
|
+
})),
|
|
478
|
+
clickhouse_settings: {
|
|
479
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
480
|
+
date_time_input_format: "best_effort",
|
|
481
|
+
use_client_time_zone: 1,
|
|
482
|
+
output_format_json_quote_64bit_integers: 0
|
|
483
|
+
}
|
|
484
|
+
}),
|
|
485
|
+
...updatePromises,
|
|
486
|
+
...deletePromises,
|
|
487
|
+
// Update thread's updatedAt timestamp
|
|
488
|
+
this.client.insert({
|
|
489
|
+
table: TABLE_THREADS,
|
|
490
|
+
format: "JSONEachRow",
|
|
491
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
492
|
+
id: thread.id,
|
|
493
|
+
resourceId: thread.resourceId,
|
|
494
|
+
title: thread.title,
|
|
495
|
+
metadata: thread.metadata,
|
|
496
|
+
createdAt: thread.createdAt,
|
|
497
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
498
|
+
})),
|
|
499
|
+
clickhouse_settings: {
|
|
500
|
+
date_time_input_format: "best_effort",
|
|
501
|
+
use_client_time_zone: 1,
|
|
502
|
+
output_format_json_quote_64bit_integers: 0
|
|
503
|
+
}
|
|
504
|
+
})
|
|
505
|
+
]);
|
|
506
|
+
const list = new MessageList().add(messages, "memory");
|
|
507
|
+
if (format === `v2`) return list.get.all.v2();
|
|
508
|
+
return list.get.all.v1();
|
|
509
|
+
} catch (error) {
|
|
510
|
+
throw new MastraError(
|
|
511
|
+
{
|
|
512
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
513
|
+
domain: ErrorDomain.STORAGE,
|
|
514
|
+
category: ErrorCategory.THIRD_PARTY
|
|
515
|
+
},
|
|
516
|
+
error
|
|
517
|
+
);
|
|
177
518
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
519
|
+
}
|
|
520
|
+
async getThreadById({ threadId }) {
|
|
521
|
+
try {
|
|
522
|
+
const result = await this.client.query({
|
|
523
|
+
query: `SELECT
|
|
524
|
+
id,
|
|
525
|
+
"resourceId",
|
|
526
|
+
title,
|
|
527
|
+
metadata,
|
|
528
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
529
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
530
|
+
FROM "${TABLE_THREADS}"
|
|
531
|
+
FINAL
|
|
532
|
+
WHERE id = {var_id:String}`,
|
|
533
|
+
query_params: { var_id: threadId },
|
|
534
|
+
clickhouse_settings: {
|
|
535
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
536
|
+
date_time_input_format: "best_effort",
|
|
537
|
+
date_time_output_format: "iso",
|
|
538
|
+
use_client_time_zone: 1,
|
|
539
|
+
output_format_json_quote_64bit_integers: 0
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
const rows = await result.json();
|
|
543
|
+
const thread = transformRow(rows.data[0]);
|
|
544
|
+
if (!thread) {
|
|
545
|
+
return null;
|
|
188
546
|
}
|
|
189
|
-
|
|
190
|
-
|
|
547
|
+
return {
|
|
548
|
+
...thread,
|
|
549
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
550
|
+
createdAt: thread.createdAt,
|
|
551
|
+
updatedAt: thread.updatedAt
|
|
552
|
+
};
|
|
553
|
+
} catch (error) {
|
|
554
|
+
throw new MastraError(
|
|
555
|
+
{
|
|
556
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
557
|
+
domain: ErrorDomain.STORAGE,
|
|
558
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
559
|
+
details: { threadId }
|
|
560
|
+
},
|
|
561
|
+
error
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
566
|
+
try {
|
|
567
|
+
const result = await this.client.query({
|
|
568
|
+
query: `SELECT
|
|
569
|
+
id,
|
|
570
|
+
"resourceId",
|
|
571
|
+
title,
|
|
572
|
+
metadata,
|
|
573
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
574
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
575
|
+
FROM "${TABLE_THREADS}"
|
|
576
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
577
|
+
query_params: { var_resourceId: resourceId },
|
|
578
|
+
clickhouse_settings: {
|
|
579
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
580
|
+
date_time_input_format: "best_effort",
|
|
581
|
+
date_time_output_format: "iso",
|
|
582
|
+
use_client_time_zone: 1,
|
|
583
|
+
output_format_json_quote_64bit_integers: 0
|
|
584
|
+
}
|
|
585
|
+
});
|
|
586
|
+
const rows = await result.json();
|
|
587
|
+
const threads = transformRows(rows.data);
|
|
588
|
+
return threads.map((thread) => ({
|
|
589
|
+
...thread,
|
|
590
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
591
|
+
createdAt: thread.createdAt,
|
|
592
|
+
updatedAt: thread.updatedAt
|
|
593
|
+
}));
|
|
594
|
+
} catch (error) {
|
|
595
|
+
throw new MastraError(
|
|
596
|
+
{
|
|
597
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
598
|
+
domain: ErrorDomain.STORAGE,
|
|
599
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
600
|
+
details: { resourceId }
|
|
601
|
+
},
|
|
602
|
+
error
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
async saveThread({ thread }) {
|
|
607
|
+
try {
|
|
608
|
+
await this.client.insert({
|
|
609
|
+
table: TABLE_THREADS,
|
|
610
|
+
values: [
|
|
611
|
+
{
|
|
612
|
+
...thread,
|
|
613
|
+
createdAt: thread.createdAt.toISOString(),
|
|
614
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
615
|
+
}
|
|
616
|
+
],
|
|
617
|
+
format: "JSONEachRow",
|
|
618
|
+
clickhouse_settings: {
|
|
619
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
620
|
+
date_time_input_format: "best_effort",
|
|
621
|
+
use_client_time_zone: 1,
|
|
622
|
+
output_format_json_quote_64bit_integers: 0
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
return thread;
|
|
626
|
+
} catch (error) {
|
|
627
|
+
throw new MastraError(
|
|
628
|
+
{
|
|
629
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
630
|
+
domain: ErrorDomain.STORAGE,
|
|
631
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
632
|
+
details: { threadId: thread.id }
|
|
633
|
+
},
|
|
634
|
+
error
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
async updateThread({
|
|
639
|
+
id,
|
|
640
|
+
title,
|
|
641
|
+
metadata
|
|
642
|
+
}) {
|
|
643
|
+
try {
|
|
644
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
645
|
+
if (!existingThread) {
|
|
646
|
+
throw new Error(`Thread ${id} not found`);
|
|
647
|
+
}
|
|
648
|
+
const mergedMetadata = {
|
|
649
|
+
...existingThread.metadata,
|
|
650
|
+
...metadata
|
|
651
|
+
};
|
|
652
|
+
const updatedThread = {
|
|
653
|
+
...existingThread,
|
|
654
|
+
title,
|
|
655
|
+
metadata: mergedMetadata,
|
|
656
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
657
|
+
};
|
|
658
|
+
await this.client.insert({
|
|
659
|
+
table: TABLE_THREADS,
|
|
660
|
+
format: "JSONEachRow",
|
|
661
|
+
values: [
|
|
662
|
+
{
|
|
663
|
+
id: updatedThread.id,
|
|
664
|
+
resourceId: updatedThread.resourceId,
|
|
665
|
+
title: updatedThread.title,
|
|
666
|
+
metadata: updatedThread.metadata,
|
|
667
|
+
createdAt: updatedThread.createdAt,
|
|
668
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
669
|
+
}
|
|
670
|
+
],
|
|
671
|
+
clickhouse_settings: {
|
|
672
|
+
date_time_input_format: "best_effort",
|
|
673
|
+
use_client_time_zone: 1,
|
|
674
|
+
output_format_json_quote_64bit_integers: 0
|
|
675
|
+
}
|
|
676
|
+
});
|
|
677
|
+
return updatedThread;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
throw new MastraError(
|
|
680
|
+
{
|
|
681
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
682
|
+
domain: ErrorDomain.STORAGE,
|
|
683
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
684
|
+
details: { threadId: id, title }
|
|
685
|
+
},
|
|
686
|
+
error
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
async deleteThread({ threadId }) {
|
|
691
|
+
try {
|
|
692
|
+
await this.client.command({
|
|
693
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
694
|
+
query_params: { var_thread_id: threadId },
|
|
695
|
+
clickhouse_settings: {
|
|
696
|
+
output_format_json_quote_64bit_integers: 0
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
await this.client.command({
|
|
700
|
+
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
701
|
+
query_params: { var_id: threadId },
|
|
702
|
+
clickhouse_settings: {
|
|
703
|
+
output_format_json_quote_64bit_integers: 0
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
} catch (error) {
|
|
707
|
+
throw new MastraError(
|
|
708
|
+
{
|
|
709
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
710
|
+
domain: ErrorDomain.STORAGE,
|
|
711
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
712
|
+
details: { threadId }
|
|
713
|
+
},
|
|
714
|
+
error
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
719
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
720
|
+
try {
|
|
721
|
+
const currentOffset = page * perPage;
|
|
722
|
+
const countResult = await this.client.query({
|
|
723
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
724
|
+
query_params: { resourceId },
|
|
725
|
+
clickhouse_settings: {
|
|
726
|
+
date_time_input_format: "best_effort",
|
|
727
|
+
date_time_output_format: "iso",
|
|
728
|
+
use_client_time_zone: 1,
|
|
729
|
+
output_format_json_quote_64bit_integers: 0
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
const countData = await countResult.json();
|
|
733
|
+
const total = countData.data[0].total;
|
|
734
|
+
if (total === 0) {
|
|
735
|
+
return {
|
|
736
|
+
threads: [],
|
|
737
|
+
total: 0,
|
|
738
|
+
page,
|
|
739
|
+
perPage,
|
|
740
|
+
hasMore: false
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
const dataResult = await this.client.query({
|
|
744
|
+
query: `
|
|
745
|
+
SELECT
|
|
746
|
+
id,
|
|
747
|
+
resourceId,
|
|
748
|
+
title,
|
|
749
|
+
metadata,
|
|
750
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
751
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
752
|
+
FROM ${TABLE_THREADS}
|
|
753
|
+
WHERE resourceId = {resourceId:String}
|
|
754
|
+
ORDER BY createdAt DESC
|
|
755
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
756
|
+
`,
|
|
757
|
+
query_params: {
|
|
758
|
+
resourceId,
|
|
759
|
+
limit: perPage,
|
|
760
|
+
offset: currentOffset
|
|
761
|
+
},
|
|
762
|
+
clickhouse_settings: {
|
|
763
|
+
date_time_input_format: "best_effort",
|
|
764
|
+
date_time_output_format: "iso",
|
|
765
|
+
use_client_time_zone: 1,
|
|
766
|
+
output_format_json_quote_64bit_integers: 0
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
const rows = await dataResult.json();
|
|
770
|
+
const threads = transformRows(rows.data);
|
|
771
|
+
return {
|
|
772
|
+
threads,
|
|
773
|
+
total,
|
|
774
|
+
page,
|
|
775
|
+
perPage,
|
|
776
|
+
hasMore: currentOffset + threads.length < total
|
|
777
|
+
};
|
|
778
|
+
} catch (error) {
|
|
779
|
+
throw new MastraError(
|
|
780
|
+
{
|
|
781
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
782
|
+
domain: ErrorDomain.STORAGE,
|
|
783
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
784
|
+
details: { resourceId, page }
|
|
785
|
+
},
|
|
786
|
+
error
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async getMessagesPaginated(args) {
|
|
791
|
+
try {
|
|
792
|
+
const { threadId, selectBy, format = "v1" } = args;
|
|
793
|
+
const page = selectBy?.pagination?.page || 0;
|
|
794
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
795
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
796
|
+
const offset = page * perPage;
|
|
797
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
798
|
+
const fromDate = dateRange?.start;
|
|
799
|
+
const toDate = dateRange?.end;
|
|
800
|
+
const messages = [];
|
|
801
|
+
if (selectBy?.include?.length) {
|
|
802
|
+
const include = selectBy.include;
|
|
803
|
+
const unionQueries = [];
|
|
804
|
+
const params = [];
|
|
805
|
+
let paramIdx = 1;
|
|
806
|
+
for (const inc of include) {
|
|
807
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
808
|
+
const searchId = inc.threadId || threadId;
|
|
809
|
+
unionQueries.push(`
|
|
810
|
+
SELECT * FROM (
|
|
811
|
+
WITH numbered_messages AS (
|
|
812
|
+
SELECT
|
|
813
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
814
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
815
|
+
FROM "${TABLE_MESSAGES}"
|
|
816
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
817
|
+
),
|
|
818
|
+
target_positions AS (
|
|
819
|
+
SELECT row_num as target_pos
|
|
820
|
+
FROM numbered_messages
|
|
821
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
822
|
+
)
|
|
823
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
824
|
+
FROM numbered_messages m
|
|
825
|
+
CROSS JOIN target_positions t
|
|
826
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
827
|
+
) AS query_${paramIdx}
|
|
828
|
+
`);
|
|
829
|
+
params.push(
|
|
830
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
831
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
832
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
833
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
834
|
+
);
|
|
835
|
+
paramIdx++;
|
|
836
|
+
}
|
|
837
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
838
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
839
|
+
const includeResult = await this.client.query({
|
|
840
|
+
query: finalQuery,
|
|
841
|
+
query_params: mergedParams,
|
|
842
|
+
clickhouse_settings: {
|
|
843
|
+
date_time_input_format: "best_effort",
|
|
844
|
+
date_time_output_format: "iso",
|
|
845
|
+
use_client_time_zone: 1,
|
|
846
|
+
output_format_json_quote_64bit_integers: 0
|
|
847
|
+
}
|
|
848
|
+
});
|
|
849
|
+
const rows2 = await includeResult.json();
|
|
850
|
+
const includedMessages = transformRows(rows2.data);
|
|
851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
852
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
853
|
+
if (seen.has(message.id)) return false;
|
|
854
|
+
seen.add(message.id);
|
|
855
|
+
return true;
|
|
856
|
+
});
|
|
857
|
+
messages.push(...dedupedMessages);
|
|
858
|
+
}
|
|
859
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
860
|
+
const countParams = { threadId };
|
|
861
|
+
if (fromDate) {
|
|
862
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
863
|
+
countParams.fromDate = fromDate.toISOString();
|
|
864
|
+
}
|
|
865
|
+
if (toDate) {
|
|
866
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
867
|
+
countParams.toDate = toDate.toISOString();
|
|
868
|
+
}
|
|
869
|
+
const countResult = await this.client.query({
|
|
870
|
+
query: countQuery,
|
|
871
|
+
query_params: countParams,
|
|
872
|
+
clickhouse_settings: {
|
|
873
|
+
date_time_input_format: "best_effort",
|
|
874
|
+
date_time_output_format: "iso",
|
|
875
|
+
use_client_time_zone: 1,
|
|
876
|
+
output_format_json_quote_64bit_integers: 0
|
|
877
|
+
}
|
|
878
|
+
});
|
|
879
|
+
const countData = await countResult.json();
|
|
880
|
+
const total = countData.data[0].total;
|
|
881
|
+
if (total === 0 && messages.length === 0) {
|
|
882
|
+
return {
|
|
883
|
+
messages: [],
|
|
884
|
+
total: 0,
|
|
885
|
+
page,
|
|
886
|
+
perPage,
|
|
887
|
+
hasMore: false
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const excludeIds = messages.map((m) => m.id);
|
|
891
|
+
let dataQuery = `
|
|
892
|
+
SELECT
|
|
893
|
+
id,
|
|
894
|
+
content,
|
|
895
|
+
role,
|
|
896
|
+
type,
|
|
897
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
898
|
+
thread_id AS "threadId",
|
|
899
|
+
resourceId
|
|
900
|
+
FROM ${TABLE_MESSAGES}
|
|
901
|
+
WHERE thread_id = {threadId:String}
|
|
902
|
+
`;
|
|
903
|
+
const dataParams = { threadId };
|
|
904
|
+
if (fromDate) {
|
|
905
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
906
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
907
|
+
}
|
|
908
|
+
if (toDate) {
|
|
909
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
910
|
+
dataParams.toDate = toDate.toISOString();
|
|
911
|
+
}
|
|
912
|
+
if (excludeIds.length > 0) {
|
|
913
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
914
|
+
dataParams.excludeIds = excludeIds;
|
|
915
|
+
}
|
|
916
|
+
if (selectBy?.last) {
|
|
917
|
+
dataQuery += `
|
|
918
|
+
ORDER BY createdAt DESC
|
|
919
|
+
LIMIT {limit:Int64}
|
|
920
|
+
`;
|
|
921
|
+
dataParams.limit = perPage;
|
|
922
|
+
} else {
|
|
923
|
+
dataQuery += `
|
|
924
|
+
ORDER BY createdAt ASC
|
|
925
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
926
|
+
`;
|
|
927
|
+
dataParams.limit = perPage;
|
|
928
|
+
dataParams.offset = offset;
|
|
929
|
+
}
|
|
930
|
+
const result = await this.client.query({
|
|
931
|
+
query: dataQuery,
|
|
932
|
+
query_params: dataParams,
|
|
933
|
+
clickhouse_settings: {
|
|
934
|
+
date_time_input_format: "best_effort",
|
|
935
|
+
date_time_output_format: "iso",
|
|
936
|
+
use_client_time_zone: 1,
|
|
937
|
+
output_format_json_quote_64bit_integers: 0
|
|
938
|
+
}
|
|
939
|
+
});
|
|
940
|
+
const rows = await result.json();
|
|
941
|
+
const paginatedMessages = transformRows(rows.data);
|
|
942
|
+
messages.push(...paginatedMessages);
|
|
943
|
+
if (selectBy?.last) {
|
|
944
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
945
|
+
}
|
|
946
|
+
return {
|
|
947
|
+
messages: format === "v2" ? messages : messages,
|
|
948
|
+
total,
|
|
949
|
+
page,
|
|
950
|
+
perPage,
|
|
951
|
+
hasMore: offset + perPage < total
|
|
952
|
+
};
|
|
953
|
+
} catch (error) {
|
|
954
|
+
throw new MastraError(
|
|
955
|
+
{
|
|
956
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
957
|
+
domain: ErrorDomain.STORAGE,
|
|
958
|
+
category: ErrorCategory.THIRD_PARTY
|
|
959
|
+
},
|
|
960
|
+
error
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
async updateMessages(args) {
|
|
965
|
+
const { messages } = args;
|
|
966
|
+
if (messages.length === 0) {
|
|
191
967
|
return [];
|
|
192
968
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
969
|
+
try {
|
|
970
|
+
const messageIds = messages.map((m) => m.id);
|
|
971
|
+
const existingResult = await this.client.query({
|
|
972
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
973
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
974
|
+
clickhouse_settings: {
|
|
975
|
+
date_time_input_format: "best_effort",
|
|
976
|
+
date_time_output_format: "iso",
|
|
977
|
+
use_client_time_zone: 1,
|
|
978
|
+
output_format_json_quote_64bit_integers: 0
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
const existingRows = await existingResult.json();
|
|
982
|
+
const existingMessages = transformRows(existingRows.data);
|
|
983
|
+
if (existingMessages.length === 0) {
|
|
984
|
+
return [];
|
|
985
|
+
}
|
|
986
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
987
|
+
if (typeof msg.content === "string") {
|
|
988
|
+
try {
|
|
989
|
+
msg.content = JSON.parse(msg.content);
|
|
990
|
+
} catch {
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return msg;
|
|
994
|
+
});
|
|
995
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
996
|
+
const updatePromises = [];
|
|
997
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
998
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
999
|
+
if (!updatePayload) continue;
|
|
1000
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1001
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1002
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1003
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1004
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1005
|
+
}
|
|
1006
|
+
const setClauses = [];
|
|
1007
|
+
const values = {};
|
|
1008
|
+
let paramIdx = 1;
|
|
1009
|
+
let newContent = null;
|
|
1010
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1011
|
+
if (updatableFields.content) {
|
|
1012
|
+
const existingContent = existingMessage.content || {};
|
|
1013
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1014
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1015
|
+
newContent = {
|
|
1016
|
+
...existingContent,
|
|
1017
|
+
...updatableFields.content,
|
|
1018
|
+
// Deep merge metadata
|
|
1019
|
+
metadata: {
|
|
1020
|
+
...existingMetadata,
|
|
1021
|
+
...updateMetadata
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1025
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1026
|
+
paramIdx++;
|
|
1027
|
+
delete updatableFields.content;
|
|
1028
|
+
}
|
|
1029
|
+
for (const key in updatableFields) {
|
|
1030
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1031
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1032
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1033
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1034
|
+
paramIdx++;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
if (setClauses.length > 0) {
|
|
1038
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1039
|
+
const updateQuery = `
|
|
1040
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1041
|
+
UPDATE ${setClauses.join(", ")}
|
|
1042
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1043
|
+
`;
|
|
1044
|
+
console.log("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1045
|
+
updatePromises.push(
|
|
1046
|
+
this.client.command({
|
|
1047
|
+
query: updateQuery,
|
|
1048
|
+
query_params: values,
|
|
1049
|
+
clickhouse_settings: {
|
|
1050
|
+
date_time_input_format: "best_effort",
|
|
1051
|
+
use_client_time_zone: 1,
|
|
1052
|
+
output_format_json_quote_64bit_integers: 0
|
|
1053
|
+
}
|
|
1054
|
+
})
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
if (updatePromises.length > 0) {
|
|
1059
|
+
await Promise.all(updatePromises);
|
|
1060
|
+
}
|
|
1061
|
+
await this.client.command({
|
|
1062
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1063
|
+
clickhouse_settings: {
|
|
1064
|
+
date_time_input_format: "best_effort",
|
|
1065
|
+
use_client_time_zone: 1,
|
|
1066
|
+
output_format_json_quote_64bit_integers: 0
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
1069
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1070
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1071
|
+
if (!updatePayload) continue;
|
|
1072
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1073
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1074
|
+
const verifyResult = await this.client.query({
|
|
1075
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1076
|
+
query_params: { messageId: id },
|
|
1077
|
+
clickhouse_settings: {
|
|
1078
|
+
date_time_input_format: "best_effort",
|
|
1079
|
+
date_time_output_format: "iso",
|
|
1080
|
+
use_client_time_zone: 1,
|
|
1081
|
+
output_format_json_quote_64bit_integers: 0
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
const verifyRows = await verifyResult.json();
|
|
1085
|
+
if (verifyRows.data.length > 0) {
|
|
1086
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1087
|
+
if (updatedMessage) {
|
|
1088
|
+
let needsRetry = false;
|
|
1089
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1090
|
+
if (key === "content") {
|
|
1091
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1092
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1093
|
+
if (actualContent !== expectedContent) {
|
|
1094
|
+
needsRetry = true;
|
|
1095
|
+
break;
|
|
1096
|
+
}
|
|
1097
|
+
} else if (updatedMessage[key] !== value) {
|
|
1098
|
+
needsRetry = true;
|
|
1099
|
+
break;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
if (needsRetry) {
|
|
1103
|
+
console.log("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1104
|
+
await this.client.command({
|
|
1105
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1106
|
+
query_params: { messageId: id },
|
|
1107
|
+
clickhouse_settings: {
|
|
1108
|
+
date_time_input_format: "best_effort",
|
|
1109
|
+
use_client_time_zone: 1,
|
|
1110
|
+
output_format_json_quote_64bit_integers: 0
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
let updatedContent = existingMessage.content || {};
|
|
1114
|
+
if (fieldsToUpdate.content) {
|
|
1115
|
+
const existingContent = existingMessage.content || {};
|
|
1116
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1117
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1118
|
+
updatedContent = {
|
|
1119
|
+
...existingContent,
|
|
1120
|
+
...fieldsToUpdate.content,
|
|
1121
|
+
metadata: {
|
|
1122
|
+
...existingMetadata,
|
|
1123
|
+
...updateMetadata
|
|
1124
|
+
}
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
const updatedMessageData = {
|
|
1128
|
+
...existingMessage,
|
|
1129
|
+
...fieldsToUpdate,
|
|
1130
|
+
content: updatedContent
|
|
1131
|
+
};
|
|
1132
|
+
await this.client.insert({
|
|
1133
|
+
table: TABLE_MESSAGES,
|
|
1134
|
+
format: "JSONEachRow",
|
|
1135
|
+
values: [
|
|
1136
|
+
{
|
|
1137
|
+
id: updatedMessageData.id,
|
|
1138
|
+
thread_id: updatedMessageData.threadId,
|
|
1139
|
+
resourceId: updatedMessageData.resourceId,
|
|
1140
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1141
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1142
|
+
role: updatedMessageData.role,
|
|
1143
|
+
type: updatedMessageData.type || "v2"
|
|
1144
|
+
}
|
|
1145
|
+
],
|
|
1146
|
+
clickhouse_settings: {
|
|
1147
|
+
date_time_input_format: "best_effort",
|
|
1148
|
+
use_client_time_zone: 1,
|
|
1149
|
+
output_format_json_quote_64bit_integers: 0
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1157
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1158
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1159
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1160
|
+
const threadResult = await this.client.query({
|
|
1161
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1162
|
+
query_params: { threadId },
|
|
1163
|
+
clickhouse_settings: {
|
|
1164
|
+
date_time_input_format: "best_effort",
|
|
1165
|
+
date_time_output_format: "iso",
|
|
1166
|
+
use_client_time_zone: 1,
|
|
1167
|
+
output_format_json_quote_64bit_integers: 0
|
|
1168
|
+
}
|
|
1169
|
+
});
|
|
1170
|
+
const threadRows = await threadResult.json();
|
|
1171
|
+
if (threadRows.data.length > 0) {
|
|
1172
|
+
const existingThread = threadRows.data[0];
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1175
|
+
query_params: { threadId },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
await this.client.insert({
|
|
1183
|
+
table: TABLE_THREADS,
|
|
1184
|
+
format: "JSONEachRow",
|
|
1185
|
+
values: [
|
|
1186
|
+
{
|
|
1187
|
+
id: existingThread.id,
|
|
1188
|
+
resourceId: existingThread.resourceId,
|
|
1189
|
+
title: existingThread.title,
|
|
1190
|
+
metadata: existingThread.metadata,
|
|
1191
|
+
createdAt: existingThread.createdAt,
|
|
1192
|
+
updatedAt: now
|
|
1193
|
+
}
|
|
1194
|
+
],
|
|
1195
|
+
clickhouse_settings: {
|
|
1196
|
+
date_time_input_format: "best_effort",
|
|
1197
|
+
use_client_time_zone: 1,
|
|
1198
|
+
output_format_json_quote_64bit_integers: 0
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
await Promise.all(threadUpdatePromises);
|
|
1204
|
+
}
|
|
1205
|
+
const updatedMessages = [];
|
|
1206
|
+
for (const messageId of messageIds) {
|
|
1207
|
+
const updatedResult = await this.client.query({
|
|
1208
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1209
|
+
query_params: { messageId },
|
|
1210
|
+
clickhouse_settings: {
|
|
1211
|
+
date_time_input_format: "best_effort",
|
|
1212
|
+
date_time_output_format: "iso",
|
|
1213
|
+
use_client_time_zone: 1,
|
|
1214
|
+
output_format_json_quote_64bit_integers: 0
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
const updatedRows = await updatedResult.json();
|
|
1218
|
+
if (updatedRows.data.length > 0) {
|
|
1219
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1220
|
+
if (message) {
|
|
1221
|
+
updatedMessages.push(message);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return updatedMessages.map((message) => {
|
|
1226
|
+
if (typeof message.content === "string") {
|
|
1227
|
+
try {
|
|
1228
|
+
message.content = JSON.parse(message.content);
|
|
1229
|
+
} catch {
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
return message;
|
|
1233
|
+
});
|
|
1234
|
+
} catch (error) {
|
|
1235
|
+
throw new MastraError(
|
|
1236
|
+
{
|
|
1237
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1238
|
+
domain: ErrorDomain.STORAGE,
|
|
1239
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1240
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1241
|
+
},
|
|
1242
|
+
error
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async getResourceById({ resourceId }) {
|
|
1247
|
+
try {
|
|
1248
|
+
const result = await this.client.query({
|
|
1249
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1250
|
+
query_params: { resourceId },
|
|
1251
|
+
clickhouse_settings: {
|
|
1252
|
+
date_time_input_format: "best_effort",
|
|
1253
|
+
date_time_output_format: "iso",
|
|
1254
|
+
use_client_time_zone: 1,
|
|
1255
|
+
output_format_json_quote_64bit_integers: 0
|
|
1256
|
+
}
|
|
1257
|
+
});
|
|
1258
|
+
const rows = await result.json();
|
|
1259
|
+
if (rows.data.length === 0) {
|
|
1260
|
+
return null;
|
|
1261
|
+
}
|
|
1262
|
+
const resource = rows.data[0];
|
|
1263
|
+
return {
|
|
1264
|
+
id: resource.id,
|
|
1265
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1266
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1267
|
+
createdAt: new Date(resource.createdAt),
|
|
1268
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1269
|
+
};
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
throw new MastraError(
|
|
1272
|
+
{
|
|
1273
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1274
|
+
domain: ErrorDomain.STORAGE,
|
|
1275
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1276
|
+
details: { resourceId }
|
|
1277
|
+
},
|
|
1278
|
+
error
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
async saveResource({ resource }) {
|
|
1283
|
+
try {
|
|
1284
|
+
await this.client.insert({
|
|
1285
|
+
table: TABLE_RESOURCES,
|
|
1286
|
+
format: "JSONEachRow",
|
|
1287
|
+
values: [
|
|
1288
|
+
{
|
|
1289
|
+
id: resource.id,
|
|
1290
|
+
workingMemory: resource.workingMemory,
|
|
1291
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1292
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1293
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1294
|
+
}
|
|
1295
|
+
],
|
|
1296
|
+
clickhouse_settings: {
|
|
1297
|
+
date_time_input_format: "best_effort",
|
|
1298
|
+
use_client_time_zone: 1,
|
|
1299
|
+
output_format_json_quote_64bit_integers: 0
|
|
1300
|
+
}
|
|
1301
|
+
});
|
|
1302
|
+
return resource;
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { resourceId: resource.id }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
async updateResource({
|
|
1316
|
+
resourceId,
|
|
1317
|
+
workingMemory,
|
|
1318
|
+
metadata
|
|
1319
|
+
}) {
|
|
1320
|
+
try {
|
|
1321
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1322
|
+
if (!existingResource) {
|
|
1323
|
+
const newResource = {
|
|
1324
|
+
id: resourceId,
|
|
1325
|
+
workingMemory,
|
|
1326
|
+
metadata: metadata || {},
|
|
1327
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1328
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1329
|
+
};
|
|
1330
|
+
return this.saveResource({ resource: newResource });
|
|
1331
|
+
}
|
|
1332
|
+
const updatedResource = {
|
|
1333
|
+
...existingResource,
|
|
1334
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1335
|
+
metadata: {
|
|
1336
|
+
...existingResource.metadata,
|
|
1337
|
+
...metadata
|
|
1338
|
+
},
|
|
1339
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1340
|
+
};
|
|
1341
|
+
const updateQuery = `
|
|
1342
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1343
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1344
|
+
WHERE id = {resourceId:String}
|
|
1345
|
+
`;
|
|
1346
|
+
await this.client.command({
|
|
1347
|
+
query: updateQuery,
|
|
1348
|
+
query_params: {
|
|
1349
|
+
workingMemory: updatedResource.workingMemory,
|
|
1350
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1351
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1352
|
+
resourceId
|
|
1353
|
+
},
|
|
1354
|
+
clickhouse_settings: {
|
|
1355
|
+
date_time_input_format: "best_effort",
|
|
1356
|
+
use_client_time_zone: 1,
|
|
1357
|
+
output_format_json_quote_64bit_integers: 0
|
|
1358
|
+
}
|
|
1359
|
+
});
|
|
1360
|
+
await this.client.command({
|
|
1361
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1362
|
+
clickhouse_settings: {
|
|
1363
|
+
date_time_input_format: "best_effort",
|
|
1364
|
+
use_client_time_zone: 1,
|
|
1365
|
+
output_format_json_quote_64bit_integers: 0
|
|
1366
|
+
}
|
|
1367
|
+
});
|
|
1368
|
+
return updatedResource;
|
|
1369
|
+
} catch (error) {
|
|
1370
|
+
throw new MastraError(
|
|
1371
|
+
{
|
|
1372
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1373
|
+
domain: ErrorDomain.STORAGE,
|
|
1374
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1375
|
+
details: { resourceId }
|
|
1376
|
+
},
|
|
1377
|
+
error
|
|
1378
|
+
);
|
|
1379
|
+
}
|
|
211
1380
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
1381
|
+
};
|
|
1382
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1383
|
+
ttl;
|
|
1384
|
+
client;
|
|
1385
|
+
constructor({ client, ttl }) {
|
|
1386
|
+
super();
|
|
1387
|
+
this.ttl = ttl;
|
|
1388
|
+
this.client = client;
|
|
216
1389
|
}
|
|
217
|
-
async
|
|
218
|
-
await this.
|
|
219
|
-
query: `
|
|
1390
|
+
async hasColumn(table, column) {
|
|
1391
|
+
const result = await this.client.query({
|
|
1392
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1393
|
+
format: "JSONEachRow"
|
|
220
1394
|
});
|
|
1395
|
+
const columns = await result.json();
|
|
1396
|
+
return columns.some((c) => c.name === column);
|
|
1397
|
+
}
|
|
1398
|
+
getSqlType(type) {
|
|
1399
|
+
switch (type) {
|
|
1400
|
+
case "text":
|
|
1401
|
+
return "String";
|
|
1402
|
+
case "timestamp":
|
|
1403
|
+
return "DateTime64(3)";
|
|
1404
|
+
case "integer":
|
|
1405
|
+
case "bigint":
|
|
1406
|
+
return "Int64";
|
|
1407
|
+
case "jsonb":
|
|
1408
|
+
return "String";
|
|
1409
|
+
default:
|
|
1410
|
+
return super.getSqlType(type);
|
|
1411
|
+
}
|
|
221
1412
|
}
|
|
222
1413
|
async createTable({
|
|
223
1414
|
tableName,
|
|
@@ -232,25 +1423,25 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
232
1423
|
}).join(",\n");
|
|
233
1424
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
234
1425
|
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
await this.
|
|
1426
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1427
|
+
${["id String"].concat(columns)}
|
|
1428
|
+
)
|
|
1429
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1430
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1431
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1432
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1433
|
+
SETTINGS index_granularity = 8192
|
|
1434
|
+
` : `
|
|
1435
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1436
|
+
${columns}
|
|
1437
|
+
)
|
|
1438
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1439
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1440
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1441
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1442
|
+
SETTINGS index_granularity = 8192
|
|
1443
|
+
`;
|
|
1444
|
+
await this.client.query({
|
|
254
1445
|
query: sql,
|
|
255
1446
|
clickhouse_settings: {
|
|
256
1447
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -261,13 +1452,59 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
261
1452
|
}
|
|
262
1453
|
});
|
|
263
1454
|
} catch (error) {
|
|
264
|
-
|
|
265
|
-
|
|
1455
|
+
throw new MastraError(
|
|
1456
|
+
{
|
|
1457
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1458
|
+
domain: ErrorDomain.STORAGE,
|
|
1459
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1460
|
+
details: { tableName }
|
|
1461
|
+
},
|
|
1462
|
+
error
|
|
1463
|
+
);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
async alterTable({
|
|
1467
|
+
tableName,
|
|
1468
|
+
schema,
|
|
1469
|
+
ifNotExists
|
|
1470
|
+
}) {
|
|
1471
|
+
try {
|
|
1472
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1473
|
+
const result = await this.client.query({
|
|
1474
|
+
query: describeSql
|
|
1475
|
+
});
|
|
1476
|
+
const rows = await result.json();
|
|
1477
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1478
|
+
for (const columnName of ifNotExists) {
|
|
1479
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1480
|
+
const columnDef = schema[columnName];
|
|
1481
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1482
|
+
if (columnDef.nullable !== false) {
|
|
1483
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1484
|
+
}
|
|
1485
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1486
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1487
|
+
await this.client.query({
|
|
1488
|
+
query: alterSql
|
|
1489
|
+
});
|
|
1490
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
throw new MastraError(
|
|
1495
|
+
{
|
|
1496
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1497
|
+
domain: ErrorDomain.STORAGE,
|
|
1498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1499
|
+
details: { tableName }
|
|
1500
|
+
},
|
|
1501
|
+
error
|
|
1502
|
+
);
|
|
266
1503
|
}
|
|
267
1504
|
}
|
|
268
1505
|
async clearTable({ tableName }) {
|
|
269
1506
|
try {
|
|
270
|
-
await this.
|
|
1507
|
+
await this.client.query({
|
|
271
1508
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
272
1509
|
clickhouse_settings: {
|
|
273
1510
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -278,19 +1515,33 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
278
1515
|
}
|
|
279
1516
|
});
|
|
280
1517
|
} catch (error) {
|
|
281
|
-
|
|
282
|
-
|
|
1518
|
+
throw new MastraError(
|
|
1519
|
+
{
|
|
1520
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1521
|
+
domain: ErrorDomain.STORAGE,
|
|
1522
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1523
|
+
details: { tableName }
|
|
1524
|
+
},
|
|
1525
|
+
error
|
|
1526
|
+
);
|
|
283
1527
|
}
|
|
284
1528
|
}
|
|
1529
|
+
async dropTable({ tableName }) {
|
|
1530
|
+
await this.client.query({
|
|
1531
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
285
1534
|
async insert({ tableName, record }) {
|
|
1535
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1536
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
286
1537
|
try {
|
|
287
|
-
await this.
|
|
1538
|
+
const result = await this.client.insert({
|
|
288
1539
|
table: tableName,
|
|
289
1540
|
values: [
|
|
290
1541
|
{
|
|
291
1542
|
...record,
|
|
292
|
-
createdAt
|
|
293
|
-
updatedAt
|
|
1543
|
+
createdAt,
|
|
1544
|
+
updatedAt
|
|
294
1545
|
}
|
|
295
1546
|
],
|
|
296
1547
|
format: "JSONEachRow",
|
|
@@ -301,13 +1552,55 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
301
1552
|
use_client_time_zone: 1
|
|
302
1553
|
}
|
|
303
1554
|
});
|
|
1555
|
+
console.log("INSERT RESULT", result);
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
throw new MastraError(
|
|
1558
|
+
{
|
|
1559
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1560
|
+
domain: ErrorDomain.STORAGE,
|
|
1561
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1562
|
+
details: { tableName }
|
|
1563
|
+
},
|
|
1564
|
+
error
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
async batchInsert({ tableName, records }) {
|
|
1569
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1570
|
+
...Object.fromEntries(
|
|
1571
|
+
Object.entries(record).map(([key, value]) => [
|
|
1572
|
+
key,
|
|
1573
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1574
|
+
])
|
|
1575
|
+
)
|
|
1576
|
+
}));
|
|
1577
|
+
try {
|
|
1578
|
+
await this.client.insert({
|
|
1579
|
+
table: tableName,
|
|
1580
|
+
values: recordsToBeInserted,
|
|
1581
|
+
format: "JSONEachRow",
|
|
1582
|
+
clickhouse_settings: {
|
|
1583
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1584
|
+
date_time_input_format: "best_effort",
|
|
1585
|
+
use_client_time_zone: 1,
|
|
1586
|
+
output_format_json_quote_64bit_integers: 0
|
|
1587
|
+
}
|
|
1588
|
+
});
|
|
304
1589
|
} catch (error) {
|
|
305
|
-
|
|
306
|
-
|
|
1590
|
+
throw new MastraError(
|
|
1591
|
+
{
|
|
1592
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1593
|
+
domain: ErrorDomain.STORAGE,
|
|
1594
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1595
|
+
details: { tableName }
|
|
1596
|
+
},
|
|
1597
|
+
error
|
|
1598
|
+
);
|
|
307
1599
|
}
|
|
308
1600
|
}
|
|
309
1601
|
async load({ tableName, keys }) {
|
|
310
1602
|
try {
|
|
1603
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
311
1604
|
const keyEntries = Object.entries(keys);
|
|
312
1605
|
const conditions = keyEntries.map(
|
|
313
1606
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -315,8 +1608,10 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
315
1608
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
316
1609
|
return { ...acc, [`var_${key}`]: value };
|
|
317
1610
|
}, {});
|
|
318
|
-
const
|
|
319
|
-
|
|
1611
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1612
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1613
|
+
const result = await this.client.query({
|
|
1614
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
320
1615
|
query_params: values,
|
|
321
1616
|
clickhouse_settings: {
|
|
322
1617
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -343,24 +1638,57 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
343
1638
|
const data = transformRow(rows.data[0]);
|
|
344
1639
|
return data;
|
|
345
1640
|
} catch (error) {
|
|
346
|
-
|
|
347
|
-
|
|
1641
|
+
throw new MastraError(
|
|
1642
|
+
{
|
|
1643
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1644
|
+
domain: ErrorDomain.STORAGE,
|
|
1645
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1646
|
+
details: { tableName }
|
|
1647
|
+
},
|
|
1648
|
+
error
|
|
1649
|
+
);
|
|
348
1650
|
}
|
|
349
1651
|
}
|
|
350
|
-
|
|
1652
|
+
};
|
|
1653
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1654
|
+
client;
|
|
1655
|
+
operations;
|
|
1656
|
+
constructor({ client, operations }) {
|
|
1657
|
+
super();
|
|
1658
|
+
this.client = client;
|
|
1659
|
+
this.operations = operations;
|
|
1660
|
+
}
|
|
1661
|
+
transformScoreRow(row) {
|
|
1662
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1663
|
+
const extractStepResult = safelyParseJSON(row.extractStepResult);
|
|
1664
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1665
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1666
|
+
const input = safelyParseJSON(row.input);
|
|
1667
|
+
const output = safelyParseJSON(row.output);
|
|
1668
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1669
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1670
|
+
const entity = safelyParseJSON(row.entity);
|
|
1671
|
+
return {
|
|
1672
|
+
...row,
|
|
1673
|
+
scorer,
|
|
1674
|
+
extractStepResult,
|
|
1675
|
+
analyzeStepResult,
|
|
1676
|
+
metadata,
|
|
1677
|
+
input,
|
|
1678
|
+
output,
|
|
1679
|
+
additionalContext,
|
|
1680
|
+
runtimeContext,
|
|
1681
|
+
entity,
|
|
1682
|
+
createdAt: new Date(row.createdAt),
|
|
1683
|
+
updatedAt: new Date(row.updatedAt)
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
async getScoreById({ id }) {
|
|
351
1687
|
try {
|
|
352
|
-
const result = await this.
|
|
353
|
-
query: `SELECT
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
title,
|
|
357
|
-
metadata,
|
|
358
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
359
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
360
|
-
FROM "${TABLE_THREADS}"
|
|
361
|
-
FINAL
|
|
362
|
-
WHERE id = {var_id:String}`,
|
|
363
|
-
query_params: { var_id: threadId },
|
|
1688
|
+
const result = await this.client.query({
|
|
1689
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1690
|
+
query_params: { var_id: id },
|
|
1691
|
+
format: "JSONEachRow",
|
|
364
1692
|
clickhouse_settings: {
|
|
365
1693
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
366
1694
|
date_time_input_format: "best_effort",
|
|
@@ -369,299 +1697,510 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
369
1697
|
output_format_json_quote_64bit_integers: 0
|
|
370
1698
|
}
|
|
371
1699
|
});
|
|
372
|
-
const
|
|
373
|
-
|
|
374
|
-
if (!thread) {
|
|
1700
|
+
const resultJson = await result.json();
|
|
1701
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
375
1702
|
return null;
|
|
376
1703
|
}
|
|
377
|
-
return
|
|
378
|
-
...thread,
|
|
379
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
380
|
-
createdAt: thread.createdAt,
|
|
381
|
-
updatedAt: thread.updatedAt
|
|
382
|
-
};
|
|
1704
|
+
return this.transformScoreRow(resultJson[0]);
|
|
383
1705
|
} catch (error) {
|
|
384
|
-
|
|
385
|
-
|
|
1706
|
+
throw new MastraError(
|
|
1707
|
+
{
|
|
1708
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1709
|
+
domain: ErrorDomain.STORAGE,
|
|
1710
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1711
|
+
details: { scoreId: id }
|
|
1712
|
+
},
|
|
1713
|
+
error
|
|
1714
|
+
);
|
|
386
1715
|
}
|
|
387
1716
|
}
|
|
388
|
-
async
|
|
1717
|
+
async saveScore(score) {
|
|
389
1718
|
try {
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
398
|
-
FROM "${TABLE_THREADS}"
|
|
399
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
400
|
-
query_params: { var_resourceId: resourceId },
|
|
1719
|
+
const record = {
|
|
1720
|
+
...score
|
|
1721
|
+
};
|
|
1722
|
+
await this.client.insert({
|
|
1723
|
+
table: TABLE_SCORERS,
|
|
1724
|
+
values: [record],
|
|
1725
|
+
format: "JSONEachRow",
|
|
401
1726
|
clickhouse_settings: {
|
|
402
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
403
1727
|
date_time_input_format: "best_effort",
|
|
404
|
-
date_time_output_format: "iso",
|
|
405
1728
|
use_client_time_zone: 1,
|
|
406
1729
|
output_format_json_quote_64bit_integers: 0
|
|
407
1730
|
}
|
|
408
1731
|
});
|
|
409
|
-
|
|
410
|
-
const threads = transformRows(rows.data);
|
|
411
|
-
return threads.map((thread) => ({
|
|
412
|
-
...thread,
|
|
413
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
414
|
-
createdAt: thread.createdAt,
|
|
415
|
-
updatedAt: thread.updatedAt
|
|
416
|
-
}));
|
|
1732
|
+
return { score };
|
|
417
1733
|
} catch (error) {
|
|
418
|
-
|
|
419
|
-
|
|
1734
|
+
throw new MastraError(
|
|
1735
|
+
{
|
|
1736
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1737
|
+
domain: ErrorDomain.STORAGE,
|
|
1738
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1739
|
+
details: { scoreId: score.id }
|
|
1740
|
+
},
|
|
1741
|
+
error
|
|
1742
|
+
);
|
|
420
1743
|
}
|
|
421
1744
|
}
|
|
422
|
-
async
|
|
1745
|
+
async getScoresByRunId({
|
|
1746
|
+
runId,
|
|
1747
|
+
pagination
|
|
1748
|
+
}) {
|
|
423
1749
|
try {
|
|
424
|
-
await this.
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
]
|
|
1750
|
+
const countResult = await this.client.query({
|
|
1751
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1752
|
+
query_params: { var_runId: runId },
|
|
1753
|
+
format: "JSONEachRow"
|
|
1754
|
+
});
|
|
1755
|
+
const countRows = await countResult.json();
|
|
1756
|
+
let total = 0;
|
|
1757
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1758
|
+
const countObj = countRows[0];
|
|
1759
|
+
total = Number(countObj.count);
|
|
1760
|
+
}
|
|
1761
|
+
if (!total) {
|
|
1762
|
+
return {
|
|
1763
|
+
pagination: {
|
|
1764
|
+
total: 0,
|
|
1765
|
+
page: pagination.page,
|
|
1766
|
+
perPage: pagination.perPage,
|
|
1767
|
+
hasMore: false
|
|
1768
|
+
},
|
|
1769
|
+
scores: []
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
const offset = pagination.page * pagination.perPage;
|
|
1773
|
+
const result = await this.client.query({
|
|
1774
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1775
|
+
query_params: {
|
|
1776
|
+
var_runId: runId,
|
|
1777
|
+
var_limit: pagination.perPage,
|
|
1778
|
+
var_offset: offset
|
|
1779
|
+
},
|
|
433
1780
|
format: "JSONEachRow",
|
|
434
1781
|
clickhouse_settings: {
|
|
435
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
436
1782
|
date_time_input_format: "best_effort",
|
|
1783
|
+
date_time_output_format: "iso",
|
|
437
1784
|
use_client_time_zone: 1,
|
|
438
1785
|
output_format_json_quote_64bit_integers: 0
|
|
439
1786
|
}
|
|
440
1787
|
});
|
|
441
|
-
|
|
1788
|
+
const rows = await result.json();
|
|
1789
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1790
|
+
return {
|
|
1791
|
+
pagination: {
|
|
1792
|
+
total,
|
|
1793
|
+
page: pagination.page,
|
|
1794
|
+
perPage: pagination.perPage,
|
|
1795
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1796
|
+
},
|
|
1797
|
+
scores
|
|
1798
|
+
};
|
|
442
1799
|
} catch (error) {
|
|
443
|
-
|
|
444
|
-
|
|
1800
|
+
throw new MastraError(
|
|
1801
|
+
{
|
|
1802
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1803
|
+
domain: ErrorDomain.STORAGE,
|
|
1804
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1805
|
+
details: { runId }
|
|
1806
|
+
},
|
|
1807
|
+
error
|
|
1808
|
+
);
|
|
445
1809
|
}
|
|
446
1810
|
}
|
|
447
|
-
async
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
metadata
|
|
1811
|
+
async getScoresByScorerId({
|
|
1812
|
+
scorerId,
|
|
1813
|
+
pagination
|
|
451
1814
|
}) {
|
|
452
1815
|
try {
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
1816
|
+
const countResult = await this.client.query({
|
|
1817
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String}`,
|
|
1818
|
+
query_params: { var_scorerId: scorerId },
|
|
1819
|
+
format: "JSONEachRow"
|
|
1820
|
+
});
|
|
1821
|
+
const countRows = await countResult.json();
|
|
1822
|
+
let total = 0;
|
|
1823
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1824
|
+
const countObj = countRows[0];
|
|
1825
|
+
total = Number(countObj.count);
|
|
456
1826
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
1827
|
+
if (!total) {
|
|
1828
|
+
return {
|
|
1829
|
+
pagination: {
|
|
1830
|
+
total: 0,
|
|
1831
|
+
page: pagination.page,
|
|
1832
|
+
perPage: pagination.perPage,
|
|
1833
|
+
hasMore: false
|
|
1834
|
+
},
|
|
1835
|
+
scores: []
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
const offset = pagination.page * pagination.perPage;
|
|
1839
|
+
const result = await this.client.query({
|
|
1840
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1841
|
+
query_params: {
|
|
1842
|
+
var_scorerId: scorerId,
|
|
1843
|
+
var_limit: pagination.perPage,
|
|
1844
|
+
var_offset: offset
|
|
1845
|
+
},
|
|
475
1846
|
format: "JSONEachRow",
|
|
476
1847
|
clickhouse_settings: {
|
|
477
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
478
1848
|
date_time_input_format: "best_effort",
|
|
1849
|
+
date_time_output_format: "iso",
|
|
479
1850
|
use_client_time_zone: 1,
|
|
480
1851
|
output_format_json_quote_64bit_integers: 0
|
|
481
1852
|
}
|
|
482
1853
|
});
|
|
483
|
-
|
|
1854
|
+
const rows = await result.json();
|
|
1855
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1856
|
+
return {
|
|
1857
|
+
pagination: {
|
|
1858
|
+
total,
|
|
1859
|
+
page: pagination.page,
|
|
1860
|
+
perPage: pagination.perPage,
|
|
1861
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1862
|
+
},
|
|
1863
|
+
scores
|
|
1864
|
+
};
|
|
484
1865
|
} catch (error) {
|
|
485
|
-
|
|
486
|
-
|
|
1866
|
+
throw new MastraError(
|
|
1867
|
+
{
|
|
1868
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1869
|
+
domain: ErrorDomain.STORAGE,
|
|
1870
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1871
|
+
details: { scorerId }
|
|
1872
|
+
},
|
|
1873
|
+
error
|
|
1874
|
+
);
|
|
487
1875
|
}
|
|
488
1876
|
}
|
|
489
|
-
async
|
|
1877
|
+
async getScoresByEntityId({
|
|
1878
|
+
entityId,
|
|
1879
|
+
entityType,
|
|
1880
|
+
pagination
|
|
1881
|
+
}) {
|
|
490
1882
|
try {
|
|
491
|
-
await this.
|
|
492
|
-
query: `
|
|
493
|
-
query_params: {
|
|
494
|
-
|
|
495
|
-
output_format_json_quote_64bit_integers: 0
|
|
496
|
-
}
|
|
1883
|
+
const countResult = await this.client.query({
|
|
1884
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1885
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1886
|
+
format: "JSONEachRow"
|
|
497
1887
|
});
|
|
498
|
-
await
|
|
499
|
-
|
|
500
|
-
|
|
1888
|
+
const countRows = await countResult.json();
|
|
1889
|
+
let total = 0;
|
|
1890
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1891
|
+
const countObj = countRows[0];
|
|
1892
|
+
total = Number(countObj.count);
|
|
1893
|
+
}
|
|
1894
|
+
if (!total) {
|
|
1895
|
+
return {
|
|
1896
|
+
pagination: {
|
|
1897
|
+
total: 0,
|
|
1898
|
+
page: pagination.page,
|
|
1899
|
+
perPage: pagination.perPage,
|
|
1900
|
+
hasMore: false
|
|
1901
|
+
},
|
|
1902
|
+
scores: []
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
const offset = pagination.page * pagination.perPage;
|
|
1906
|
+
const result = await this.client.query({
|
|
1907
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1908
|
+
query_params: {
|
|
1909
|
+
var_entityId: entityId,
|
|
1910
|
+
var_entityType: entityType,
|
|
1911
|
+
var_limit: pagination.perPage,
|
|
1912
|
+
var_offset: offset
|
|
1913
|
+
},
|
|
1914
|
+
format: "JSONEachRow",
|
|
501
1915
|
clickhouse_settings: {
|
|
1916
|
+
date_time_input_format: "best_effort",
|
|
1917
|
+
date_time_output_format: "iso",
|
|
1918
|
+
use_client_time_zone: 1,
|
|
502
1919
|
output_format_json_quote_64bit_integers: 0
|
|
503
1920
|
}
|
|
504
1921
|
});
|
|
1922
|
+
const rows = await result.json();
|
|
1923
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1924
|
+
return {
|
|
1925
|
+
pagination: {
|
|
1926
|
+
total,
|
|
1927
|
+
page: pagination.page,
|
|
1928
|
+
perPage: pagination.perPage,
|
|
1929
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1930
|
+
},
|
|
1931
|
+
scores
|
|
1932
|
+
};
|
|
505
1933
|
} catch (error) {
|
|
506
|
-
|
|
507
|
-
|
|
1934
|
+
throw new MastraError(
|
|
1935
|
+
{
|
|
1936
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1937
|
+
domain: ErrorDomain.STORAGE,
|
|
1938
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1939
|
+
details: { entityId, entityType }
|
|
1940
|
+
},
|
|
1941
|
+
error
|
|
1942
|
+
);
|
|
508
1943
|
}
|
|
509
1944
|
}
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
560
|
-
},
|
|
561
|
-
clickhouse_settings: {
|
|
562
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
563
|
-
date_time_input_format: "best_effort",
|
|
564
|
-
date_time_output_format: "iso",
|
|
565
|
-
use_client_time_zone: 1,
|
|
566
|
-
output_format_json_quote_64bit_integers: 0
|
|
567
|
-
}
|
|
568
|
-
});
|
|
569
|
-
const rows2 = await includeResult.json();
|
|
570
|
-
messages.push(...transformRows(rows2.data));
|
|
571
|
-
}
|
|
572
|
-
const result = await this.db.query({
|
|
573
|
-
query: `
|
|
574
|
-
SELECT
|
|
575
|
-
id,
|
|
576
|
-
content,
|
|
577
|
-
role,
|
|
578
|
-
type,
|
|
579
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
580
|
-
thread_id AS "threadId"
|
|
581
|
-
FROM "${TABLE_MESSAGES}"
|
|
582
|
-
WHERE thread_id = {threadId:String}
|
|
583
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
584
|
-
ORDER BY "createdAt" DESC
|
|
585
|
-
LIMIT {limit:Int64}
|
|
586
|
-
`,
|
|
587
|
-
query_params: {
|
|
588
|
-
threadId,
|
|
589
|
-
exclude: messages.map((m) => m.id),
|
|
590
|
-
limit
|
|
591
|
-
},
|
|
1945
|
+
};
|
|
1946
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
1947
|
+
client;
|
|
1948
|
+
operations;
|
|
1949
|
+
constructor({ client, operations }) {
|
|
1950
|
+
super();
|
|
1951
|
+
this.client = client;
|
|
1952
|
+
this.operations = operations;
|
|
1953
|
+
}
|
|
1954
|
+
async getTracesPaginated(args) {
|
|
1955
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
1956
|
+
const fromDate = dateRange?.start;
|
|
1957
|
+
const toDate = dateRange?.end;
|
|
1958
|
+
const currentOffset = page * perPage;
|
|
1959
|
+
const queryArgs = {};
|
|
1960
|
+
const conditions = [];
|
|
1961
|
+
if (name) {
|
|
1962
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
1963
|
+
queryArgs.var_name = name;
|
|
1964
|
+
}
|
|
1965
|
+
if (scope) {
|
|
1966
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
1967
|
+
queryArgs.var_scope = scope;
|
|
1968
|
+
}
|
|
1969
|
+
if (attributes) {
|
|
1970
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
1971
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
1972
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
if (filters) {
|
|
1976
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
1977
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
1978
|
+
queryArgs[`var_col_${key}`] = value;
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
if (fromDate) {
|
|
1982
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
1983
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
1984
|
+
}
|
|
1985
|
+
if (toDate) {
|
|
1986
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
1987
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
1988
|
+
}
|
|
1989
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1990
|
+
try {
|
|
1991
|
+
const countResult = await this.client.query({
|
|
1992
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
1993
|
+
query_params: queryArgs,
|
|
592
1994
|
clickhouse_settings: {
|
|
593
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
594
1995
|
date_time_input_format: "best_effort",
|
|
595
1996
|
date_time_output_format: "iso",
|
|
596
1997
|
use_client_time_zone: 1,
|
|
597
1998
|
output_format_json_quote_64bit_integers: 0
|
|
598
1999
|
}
|
|
599
2000
|
});
|
|
600
|
-
const
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
2001
|
+
const countData = await countResult.json();
|
|
2002
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2003
|
+
if (total === 0) {
|
|
2004
|
+
return {
|
|
2005
|
+
traces: [],
|
|
2006
|
+
total: 0,
|
|
2007
|
+
page,
|
|
2008
|
+
perPage,
|
|
2009
|
+
hasMore: false
|
|
2010
|
+
};
|
|
2011
|
+
}
|
|
2012
|
+
const result = await this.client.query({
|
|
2013
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2014
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2015
|
+
clickhouse_settings: {
|
|
2016
|
+
date_time_input_format: "best_effort",
|
|
2017
|
+
date_time_output_format: "iso",
|
|
2018
|
+
use_client_time_zone: 1,
|
|
2019
|
+
output_format_json_quote_64bit_integers: 0
|
|
609
2020
|
}
|
|
610
2021
|
});
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
2022
|
+
if (!result) {
|
|
2023
|
+
return {
|
|
2024
|
+
traces: [],
|
|
2025
|
+
total,
|
|
2026
|
+
page,
|
|
2027
|
+
perPage,
|
|
2028
|
+
hasMore: false
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
const resp = await result.json();
|
|
2032
|
+
const rows = resp.data;
|
|
2033
|
+
const traces = rows.map((row) => ({
|
|
2034
|
+
id: row.id,
|
|
2035
|
+
parentSpanId: row.parentSpanId,
|
|
2036
|
+
traceId: row.traceId,
|
|
2037
|
+
name: row.name,
|
|
2038
|
+
scope: row.scope,
|
|
2039
|
+
kind: row.kind,
|
|
2040
|
+
status: safelyParseJSON(row.status),
|
|
2041
|
+
events: safelyParseJSON(row.events),
|
|
2042
|
+
links: safelyParseJSON(row.links),
|
|
2043
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2044
|
+
startTime: row.startTime,
|
|
2045
|
+
endTime: row.endTime,
|
|
2046
|
+
other: safelyParseJSON(row.other),
|
|
2047
|
+
createdAt: row.createdAt
|
|
2048
|
+
}));
|
|
2049
|
+
return {
|
|
2050
|
+
traces,
|
|
2051
|
+
total,
|
|
2052
|
+
page,
|
|
2053
|
+
perPage,
|
|
2054
|
+
hasMore: currentOffset + traces.length < total
|
|
2055
|
+
};
|
|
614
2056
|
} catch (error) {
|
|
615
|
-
|
|
616
|
-
|
|
2057
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2058
|
+
return {
|
|
2059
|
+
traces: [],
|
|
2060
|
+
total: 0,
|
|
2061
|
+
page,
|
|
2062
|
+
perPage,
|
|
2063
|
+
hasMore: false
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
throw new MastraError(
|
|
2067
|
+
{
|
|
2068
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2069
|
+
domain: ErrorDomain.STORAGE,
|
|
2070
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2071
|
+
details: {
|
|
2072
|
+
name: name ?? null,
|
|
2073
|
+
scope: scope ?? null,
|
|
2074
|
+
page,
|
|
2075
|
+
perPage,
|
|
2076
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2077
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2078
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2079
|
+
}
|
|
2080
|
+
},
|
|
2081
|
+
error
|
|
2082
|
+
);
|
|
617
2083
|
}
|
|
618
2084
|
}
|
|
619
|
-
async
|
|
620
|
-
|
|
621
|
-
|
|
2085
|
+
async getTraces({
|
|
2086
|
+
name,
|
|
2087
|
+
scope,
|
|
2088
|
+
page,
|
|
2089
|
+
perPage,
|
|
2090
|
+
attributes,
|
|
2091
|
+
filters,
|
|
2092
|
+
fromDate,
|
|
2093
|
+
toDate
|
|
2094
|
+
}) {
|
|
2095
|
+
const limit = perPage;
|
|
2096
|
+
const offset = page * perPage;
|
|
2097
|
+
const args = {};
|
|
2098
|
+
const conditions = [];
|
|
2099
|
+
if (name) {
|
|
2100
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2101
|
+
args.var_name = name;
|
|
2102
|
+
}
|
|
2103
|
+
if (scope) {
|
|
2104
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2105
|
+
args.var_scope = scope;
|
|
2106
|
+
}
|
|
2107
|
+
if (attributes) {
|
|
2108
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2109
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2110
|
+
args[`var_attr_${key}`] = value;
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
if (filters) {
|
|
2114
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2115
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2116
|
+
args[`var_col_${key}`] = value;
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
if (fromDate) {
|
|
2120
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2121
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2122
|
+
}
|
|
2123
|
+
if (toDate) {
|
|
2124
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2125
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2126
|
+
}
|
|
2127
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
622
2128
|
try {
|
|
623
|
-
const
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
throw new Error("Thread ID is required");
|
|
627
|
-
}
|
|
628
|
-
const thread = await this.getThreadById({ threadId });
|
|
629
|
-
if (!thread) {
|
|
630
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
631
|
-
}
|
|
632
|
-
await this.db.insert({
|
|
633
|
-
table: TABLE_MESSAGES,
|
|
634
|
-
format: "JSONEachRow",
|
|
635
|
-
values: messages.map((message) => ({
|
|
636
|
-
id: message.id,
|
|
637
|
-
thread_id: threadId,
|
|
638
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
639
|
-
createdAt: message.createdAt.toISOString(),
|
|
640
|
-
role: message.role,
|
|
641
|
-
type: message.type || "v2"
|
|
642
|
-
})),
|
|
2129
|
+
const result = await this.client.query({
|
|
2130
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2131
|
+
query_params: args,
|
|
643
2132
|
clickhouse_settings: {
|
|
644
2133
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
645
2134
|
date_time_input_format: "best_effort",
|
|
2135
|
+
date_time_output_format: "iso",
|
|
646
2136
|
use_client_time_zone: 1,
|
|
647
2137
|
output_format_json_quote_64bit_integers: 0
|
|
648
2138
|
}
|
|
649
2139
|
});
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
2140
|
+
if (!result) {
|
|
2141
|
+
return [];
|
|
2142
|
+
}
|
|
2143
|
+
const resp = await result.json();
|
|
2144
|
+
const rows = resp.data;
|
|
2145
|
+
return rows.map((row) => ({
|
|
2146
|
+
id: row.id,
|
|
2147
|
+
parentSpanId: row.parentSpanId,
|
|
2148
|
+
traceId: row.traceId,
|
|
2149
|
+
name: row.name,
|
|
2150
|
+
scope: row.scope,
|
|
2151
|
+
kind: row.kind,
|
|
2152
|
+
status: safelyParseJSON(row.status),
|
|
2153
|
+
events: safelyParseJSON(row.events),
|
|
2154
|
+
links: safelyParseJSON(row.links),
|
|
2155
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2156
|
+
startTime: row.startTime,
|
|
2157
|
+
endTime: row.endTime,
|
|
2158
|
+
other: safelyParseJSON(row.other),
|
|
2159
|
+
createdAt: row.createdAt
|
|
2160
|
+
}));
|
|
653
2161
|
} catch (error) {
|
|
654
|
-
|
|
655
|
-
|
|
2162
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2163
|
+
return [];
|
|
2164
|
+
}
|
|
2165
|
+
throw new MastraError(
|
|
2166
|
+
{
|
|
2167
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2168
|
+
domain: ErrorDomain.STORAGE,
|
|
2169
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2170
|
+
details: {
|
|
2171
|
+
name: name ?? null,
|
|
2172
|
+
scope: scope ?? null,
|
|
2173
|
+
page,
|
|
2174
|
+
perPage,
|
|
2175
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2176
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2177
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2178
|
+
toDate: toDate?.toISOString() ?? null
|
|
2179
|
+
}
|
|
2180
|
+
},
|
|
2181
|
+
error
|
|
2182
|
+
);
|
|
656
2183
|
}
|
|
657
2184
|
}
|
|
2185
|
+
async batchTraceInsert(args) {
|
|
2186
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2187
|
+
}
|
|
2188
|
+
};
|
|
2189
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2190
|
+
client;
|
|
2191
|
+
operations;
|
|
2192
|
+
constructor({ client, operations }) {
|
|
2193
|
+
super();
|
|
2194
|
+
this.operations = operations;
|
|
2195
|
+
this.client = client;
|
|
2196
|
+
}
|
|
658
2197
|
async persistWorkflowSnapshot({
|
|
659
2198
|
workflowName,
|
|
660
2199
|
runId,
|
|
661
2200
|
snapshot
|
|
662
2201
|
}) {
|
|
663
2202
|
try {
|
|
664
|
-
const currentSnapshot = await this.load({
|
|
2203
|
+
const currentSnapshot = await this.operations.load({
|
|
665
2204
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
666
2205
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
667
2206
|
});
|
|
@@ -677,7 +2216,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
677
2216
|
createdAt: now.toISOString(),
|
|
678
2217
|
updatedAt: now.toISOString()
|
|
679
2218
|
};
|
|
680
|
-
await this.
|
|
2219
|
+
await this.client.insert({
|
|
681
2220
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
682
2221
|
format: "JSONEachRow",
|
|
683
2222
|
values: [persisting],
|
|
@@ -689,8 +2228,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
689
2228
|
}
|
|
690
2229
|
});
|
|
691
2230
|
} catch (error) {
|
|
692
|
-
|
|
693
|
-
|
|
2231
|
+
throw new MastraError(
|
|
2232
|
+
{
|
|
2233
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2234
|
+
domain: ErrorDomain.STORAGE,
|
|
2235
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2236
|
+
details: { workflowName, runId }
|
|
2237
|
+
},
|
|
2238
|
+
error
|
|
2239
|
+
);
|
|
694
2240
|
}
|
|
695
2241
|
}
|
|
696
2242
|
async loadWorkflowSnapshot({
|
|
@@ -698,7 +2244,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
698
2244
|
runId
|
|
699
2245
|
}) {
|
|
700
2246
|
try {
|
|
701
|
-
const result = await this.load({
|
|
2247
|
+
const result = await this.operations.load({
|
|
702
2248
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
703
2249
|
keys: {
|
|
704
2250
|
workflow_name: workflowName,
|
|
@@ -710,8 +2256,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
710
2256
|
}
|
|
711
2257
|
return result.snapshot;
|
|
712
2258
|
} catch (error) {
|
|
713
|
-
|
|
714
|
-
|
|
2259
|
+
throw new MastraError(
|
|
2260
|
+
{
|
|
2261
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2262
|
+
domain: ErrorDomain.STORAGE,
|
|
2263
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2264
|
+
details: { workflowName, runId }
|
|
2265
|
+
},
|
|
2266
|
+
error
|
|
2267
|
+
);
|
|
715
2268
|
}
|
|
716
2269
|
}
|
|
717
2270
|
parseWorkflowRun(row) {
|
|
@@ -748,7 +2301,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
748
2301
|
values.var_workflow_name = workflowName;
|
|
749
2302
|
}
|
|
750
2303
|
if (resourceId) {
|
|
751
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2304
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
752
2305
|
if (hasResourceId) {
|
|
753
2306
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
754
2307
|
values.var_resourceId = resourceId;
|
|
@@ -769,7 +2322,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
769
2322
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
770
2323
|
let total = 0;
|
|
771
2324
|
if (limit !== void 0 && offset !== void 0) {
|
|
772
|
-
const countResult = await this.
|
|
2325
|
+
const countResult = await this.client.query({
|
|
773
2326
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
774
2327
|
query_params: values,
|
|
775
2328
|
format: "JSONEachRow"
|
|
@@ -777,21 +2330,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
777
2330
|
const countRows = await countResult.json();
|
|
778
2331
|
total = Number(countRows[0]?.count ?? 0);
|
|
779
2332
|
}
|
|
780
|
-
const result = await this.
|
|
2333
|
+
const result = await this.client.query({
|
|
781
2334
|
query: `
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
2335
|
+
SELECT
|
|
2336
|
+
workflow_name,
|
|
2337
|
+
run_id,
|
|
2338
|
+
snapshot,
|
|
2339
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2340
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2341
|
+
resourceId
|
|
2342
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2343
|
+
${whereClause}
|
|
2344
|
+
ORDER BY createdAt DESC
|
|
2345
|
+
${limitClause}
|
|
2346
|
+
${offsetClause}
|
|
2347
|
+
`,
|
|
795
2348
|
query_params: values,
|
|
796
2349
|
format: "JSONEachRow"
|
|
797
2350
|
});
|
|
@@ -802,8 +2355,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
802
2355
|
});
|
|
803
2356
|
return { runs, total: total || runs.length };
|
|
804
2357
|
} catch (error) {
|
|
805
|
-
|
|
806
|
-
|
|
2358
|
+
throw new MastraError(
|
|
2359
|
+
{
|
|
2360
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2361
|
+
domain: ErrorDomain.STORAGE,
|
|
2362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2363
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2364
|
+
},
|
|
2365
|
+
error
|
|
2366
|
+
);
|
|
807
2367
|
}
|
|
808
2368
|
}
|
|
809
2369
|
async getWorkflowRunById({
|
|
@@ -822,18 +2382,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
822
2382
|
values.var_workflow_name = workflowName;
|
|
823
2383
|
}
|
|
824
2384
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
825
|
-
const result = await this.
|
|
2385
|
+
const result = await this.client.query({
|
|
826
2386
|
query: `
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
2387
|
+
SELECT
|
|
2388
|
+
workflow_name,
|
|
2389
|
+
run_id,
|
|
2390
|
+
snapshot,
|
|
2391
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2392
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2393
|
+
resourceId
|
|
2394
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2395
|
+
${whereClause}
|
|
2396
|
+
`,
|
|
837
2397
|
query_params: values,
|
|
838
2398
|
format: "JSONEachRow"
|
|
839
2399
|
});
|
|
@@ -843,21 +2403,248 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
843
2403
|
}
|
|
844
2404
|
return this.parseWorkflowRun(resultJson[0]);
|
|
845
2405
|
} catch (error) {
|
|
846
|
-
|
|
847
|
-
|
|
2406
|
+
throw new MastraError(
|
|
2407
|
+
{
|
|
2408
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2409
|
+
domain: ErrorDomain.STORAGE,
|
|
2410
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2411
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2412
|
+
},
|
|
2413
|
+
error
|
|
2414
|
+
);
|
|
848
2415
|
}
|
|
849
2416
|
}
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
2417
|
+
};
|
|
2418
|
+
|
|
2419
|
+
// src/storage/index.ts
|
|
2420
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2421
|
+
db;
|
|
2422
|
+
ttl = {};
|
|
2423
|
+
stores;
|
|
2424
|
+
constructor(config) {
|
|
2425
|
+
super({ name: "ClickhouseStore" });
|
|
2426
|
+
this.db = createClient({
|
|
2427
|
+
url: config.url,
|
|
2428
|
+
username: config.username,
|
|
2429
|
+
password: config.password,
|
|
2430
|
+
clickhouse_settings: {
|
|
2431
|
+
date_time_input_format: "best_effort",
|
|
2432
|
+
date_time_output_format: "iso",
|
|
2433
|
+
// This is crucial
|
|
2434
|
+
use_client_time_zone: 1,
|
|
2435
|
+
output_format_json_quote_64bit_integers: 0
|
|
2436
|
+
}
|
|
854
2437
|
});
|
|
855
|
-
|
|
856
|
-
|
|
2438
|
+
this.ttl = config.ttl;
|
|
2439
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2440
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2441
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2442
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2443
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2444
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2445
|
+
this.stores = {
|
|
2446
|
+
operations,
|
|
2447
|
+
workflows,
|
|
2448
|
+
scores,
|
|
2449
|
+
legacyEvals,
|
|
2450
|
+
traces,
|
|
2451
|
+
memory
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
get supports() {
|
|
2455
|
+
return {
|
|
2456
|
+
selectByIncludeResourceScope: true,
|
|
2457
|
+
resourceWorkingMemory: true,
|
|
2458
|
+
hasColumn: true,
|
|
2459
|
+
createTable: true
|
|
2460
|
+
};
|
|
2461
|
+
}
|
|
2462
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2463
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2464
|
+
}
|
|
2465
|
+
async getEvals(options) {
|
|
2466
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2467
|
+
}
|
|
2468
|
+
async batchInsert({ tableName, records }) {
|
|
2469
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2470
|
+
}
|
|
2471
|
+
async optimizeTable({ tableName }) {
|
|
2472
|
+
try {
|
|
2473
|
+
await this.db.command({
|
|
2474
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2475
|
+
});
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
throw new MastraError(
|
|
2478
|
+
{
|
|
2479
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2480
|
+
domain: ErrorDomain.STORAGE,
|
|
2481
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2482
|
+
details: { tableName }
|
|
2483
|
+
},
|
|
2484
|
+
error
|
|
2485
|
+
);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
async materializeTtl({ tableName }) {
|
|
2489
|
+
try {
|
|
2490
|
+
await this.db.command({
|
|
2491
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2492
|
+
});
|
|
2493
|
+
} catch (error) {
|
|
2494
|
+
throw new MastraError(
|
|
2495
|
+
{
|
|
2496
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2497
|
+
domain: ErrorDomain.STORAGE,
|
|
2498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2499
|
+
details: { tableName }
|
|
2500
|
+
},
|
|
2501
|
+
error
|
|
2502
|
+
);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
async createTable({
|
|
2506
|
+
tableName,
|
|
2507
|
+
schema
|
|
2508
|
+
}) {
|
|
2509
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2510
|
+
}
|
|
2511
|
+
async dropTable({ tableName }) {
|
|
2512
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2513
|
+
}
|
|
2514
|
+
async alterTable({
|
|
2515
|
+
tableName,
|
|
2516
|
+
schema,
|
|
2517
|
+
ifNotExists
|
|
2518
|
+
}) {
|
|
2519
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2520
|
+
}
|
|
2521
|
+
async clearTable({ tableName }) {
|
|
2522
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2523
|
+
}
|
|
2524
|
+
async insert({ tableName, record }) {
|
|
2525
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2526
|
+
}
|
|
2527
|
+
async load({ tableName, keys }) {
|
|
2528
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2529
|
+
}
|
|
2530
|
+
async persistWorkflowSnapshot({
|
|
2531
|
+
workflowName,
|
|
2532
|
+
runId,
|
|
2533
|
+
snapshot
|
|
2534
|
+
}) {
|
|
2535
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2536
|
+
}
|
|
2537
|
+
async loadWorkflowSnapshot({
|
|
2538
|
+
workflowName,
|
|
2539
|
+
runId
|
|
2540
|
+
}) {
|
|
2541
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2542
|
+
}
|
|
2543
|
+
async getWorkflowRuns({
|
|
2544
|
+
workflowName,
|
|
2545
|
+
fromDate,
|
|
2546
|
+
toDate,
|
|
2547
|
+
limit,
|
|
2548
|
+
offset,
|
|
2549
|
+
resourceId
|
|
2550
|
+
} = {}) {
|
|
2551
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2552
|
+
}
|
|
2553
|
+
async getWorkflowRunById({
|
|
2554
|
+
runId,
|
|
2555
|
+
workflowName
|
|
2556
|
+
}) {
|
|
2557
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2558
|
+
}
|
|
2559
|
+
async getTraces(args) {
|
|
2560
|
+
return this.stores.traces.getTraces(args);
|
|
2561
|
+
}
|
|
2562
|
+
async getTracesPaginated(args) {
|
|
2563
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2564
|
+
}
|
|
2565
|
+
async batchTraceInsert(args) {
|
|
2566
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2567
|
+
}
|
|
2568
|
+
async getThreadById({ threadId }) {
|
|
2569
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2570
|
+
}
|
|
2571
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2572
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2573
|
+
}
|
|
2574
|
+
async saveThread({ thread }) {
|
|
2575
|
+
return this.stores.memory.saveThread({ thread });
|
|
2576
|
+
}
|
|
2577
|
+
async updateThread({
|
|
2578
|
+
id,
|
|
2579
|
+
title,
|
|
2580
|
+
metadata
|
|
2581
|
+
}) {
|
|
2582
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2583
|
+
}
|
|
2584
|
+
async deleteThread({ threadId }) {
|
|
2585
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2586
|
+
}
|
|
2587
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2588
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2589
|
+
}
|
|
2590
|
+
async getMessages({
|
|
2591
|
+
threadId,
|
|
2592
|
+
resourceId,
|
|
2593
|
+
selectBy,
|
|
2594
|
+
format
|
|
2595
|
+
}) {
|
|
2596
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2597
|
+
}
|
|
2598
|
+
async saveMessages(args) {
|
|
2599
|
+
return this.stores.memory.saveMessages(args);
|
|
2600
|
+
}
|
|
2601
|
+
async getMessagesPaginated(args) {
|
|
2602
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2603
|
+
}
|
|
2604
|
+
async updateMessages(args) {
|
|
2605
|
+
return this.stores.memory.updateMessages(args);
|
|
2606
|
+
}
|
|
2607
|
+
async getResourceById({ resourceId }) {
|
|
2608
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2609
|
+
}
|
|
2610
|
+
async saveResource({ resource }) {
|
|
2611
|
+
return this.stores.memory.saveResource({ resource });
|
|
2612
|
+
}
|
|
2613
|
+
async updateResource({
|
|
2614
|
+
resourceId,
|
|
2615
|
+
workingMemory,
|
|
2616
|
+
metadata
|
|
2617
|
+
}) {
|
|
2618
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2619
|
+
}
|
|
2620
|
+
async getScoreById({ id }) {
|
|
2621
|
+
return this.stores.scores.getScoreById({ id });
|
|
2622
|
+
}
|
|
2623
|
+
async saveScore(_score) {
|
|
2624
|
+
return this.stores.scores.saveScore(_score);
|
|
2625
|
+
}
|
|
2626
|
+
async getScoresByRunId({
|
|
2627
|
+
runId,
|
|
2628
|
+
pagination
|
|
2629
|
+
}) {
|
|
2630
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2631
|
+
}
|
|
2632
|
+
async getScoresByEntityId({
|
|
2633
|
+
entityId,
|
|
2634
|
+
entityType,
|
|
2635
|
+
pagination
|
|
2636
|
+
}) {
|
|
2637
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2638
|
+
}
|
|
2639
|
+
async getScoresByScorerId({
|
|
2640
|
+
scorerId,
|
|
2641
|
+
pagination
|
|
2642
|
+
}) {
|
|
2643
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
857
2644
|
}
|
|
858
2645
|
async close() {
|
|
859
2646
|
await this.db.close();
|
|
860
2647
|
}
|
|
861
2648
|
};
|
|
862
2649
|
|
|
863
|
-
export {
|
|
2650
|
+
export { ClickhouseStore };
|