@mastra/clickhouse 0.0.0-vector-sources-20250516175436 → 0.0.0-vector-extension-schema-20250922130418
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +759 -2
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2379 -442
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2362 -425
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +87 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +46 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +55 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +236 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +32 -14
- package/dist/_tsup-dts-rollup.d.cts +0 -138
- package/dist/_tsup-dts-rollup.d.ts +0 -138
- package/dist/index.d.cts +0 -4
- package/docker-compose.yaml +0 -15
- package/eslint.config.js +0 -6
- package/src/index.ts +0 -1
- package/src/storage/index.test.ts +0 -856
- package/src/storage/index.ts +0 -1056
- package/tsconfig.json +0 -5
- package/vitest.config.ts +0 -12
package/dist/index.js
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import {
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { TABLE_AI_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, ScoresStorage, safelyParseJSON, LegacyEvalsStorage, TracesStorage, MemoryStorage, resolveMessageLimit } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
3
5
|
|
|
4
6
|
// src/storage/index.ts
|
|
5
|
-
function safelyParseJSON(jsonString) {
|
|
6
|
-
try {
|
|
7
|
-
return JSON.parse(jsonString);
|
|
8
|
-
} catch {
|
|
9
|
-
return {};
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
7
|
var TABLE_ENGINES = {
|
|
13
8
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
14
9
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
15
10
|
[TABLE_TRACES]: `MergeTree()`,
|
|
16
11
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
17
|
-
[TABLE_EVALS]: `MergeTree()
|
|
12
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
13
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
14
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`,
|
|
15
|
+
// TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
|
|
16
|
+
[TABLE_AI_SPANS]: `ReplacingMergeTree()`
|
|
18
17
|
};
|
|
19
18
|
var COLUMN_TYPES = {
|
|
20
19
|
text: "String",
|
|
@@ -22,11 +21,10 @@ var COLUMN_TYPES = {
|
|
|
22
21
|
uuid: "String",
|
|
23
22
|
jsonb: "String",
|
|
24
23
|
integer: "Int64",
|
|
25
|
-
|
|
24
|
+
float: "Float64",
|
|
25
|
+
bigint: "Int64",
|
|
26
|
+
boolean: "Bool"
|
|
26
27
|
};
|
|
27
|
-
function transformRows(rows) {
|
|
28
|
-
return rows.map((row) => transformRow(row));
|
|
29
|
-
}
|
|
30
28
|
function transformRow(row) {
|
|
31
29
|
if (!row) {
|
|
32
30
|
return row;
|
|
@@ -37,33 +35,63 @@ function transformRow(row) {
|
|
|
37
35
|
if (row.updatedAt) {
|
|
38
36
|
row.updatedAt = new Date(row.updatedAt);
|
|
39
37
|
}
|
|
38
|
+
if (row.content && typeof row.content === "string") {
|
|
39
|
+
row.content = safelyParseJSON(row.content);
|
|
40
|
+
}
|
|
40
41
|
return row;
|
|
41
42
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
// This is crucial
|
|
55
|
-
use_client_time_zone: 1,
|
|
56
|
-
output_format_json_quote_64bit_integers: 0
|
|
57
|
-
}
|
|
58
|
-
});
|
|
59
|
-
this.ttl = config.ttl;
|
|
43
|
+
function transformRows(rows) {
|
|
44
|
+
return rows.map((row) => transformRow(row));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
48
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
49
|
+
client;
|
|
50
|
+
operations;
|
|
51
|
+
constructor({ client, operations }) {
|
|
52
|
+
super();
|
|
53
|
+
this.client = client;
|
|
54
|
+
this.operations = operations;
|
|
60
55
|
}
|
|
61
56
|
transformEvalRow(row) {
|
|
62
57
|
row = transformRow(row);
|
|
63
|
-
|
|
64
|
-
|
|
58
|
+
let resultValue;
|
|
59
|
+
try {
|
|
60
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
61
|
+
resultValue = JSON.parse(row.result);
|
|
62
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
63
|
+
resultValue = row.result;
|
|
64
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
65
|
+
resultValue = { score: 0 };
|
|
66
|
+
} else {
|
|
67
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
68
|
+
}
|
|
69
|
+
} catch (error) {
|
|
70
|
+
console.error("Error parsing result field:", row.result, error);
|
|
71
|
+
throw new MastraError({
|
|
72
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
73
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
74
|
+
domain: ErrorDomain.STORAGE,
|
|
75
|
+
category: ErrorCategory.USER
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
let testInfoValue;
|
|
79
|
+
try {
|
|
80
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
81
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
82
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
83
|
+
testInfoValue = row.test_info;
|
|
84
|
+
}
|
|
85
|
+
} catch {
|
|
86
|
+
testInfoValue = void 0;
|
|
87
|
+
}
|
|
65
88
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
66
|
-
throw new
|
|
89
|
+
throw new MastraError({
|
|
90
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
91
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
92
|
+
domain: ErrorDomain.STORAGE,
|
|
93
|
+
category: ErrorCategory.USER
|
|
94
|
+
});
|
|
67
95
|
}
|
|
68
96
|
return {
|
|
69
97
|
input: row.input,
|
|
@@ -80,9 +108,9 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
80
108
|
}
|
|
81
109
|
async getEvalsByAgentName(agentName, type) {
|
|
82
110
|
try {
|
|
83
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
84
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
85
|
-
const result = await this.
|
|
111
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
112
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
113
|
+
const result = await this.client.query({
|
|
86
114
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
87
115
|
query_params: { var_agent_name: agentName },
|
|
88
116
|
clickhouse_settings: {
|
|
@@ -98,125 +126,1357 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
98
126
|
const rows = await result.json();
|
|
99
127
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
100
128
|
} catch (error) {
|
|
101
|
-
if (error
|
|
129
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
102
130
|
return [];
|
|
103
131
|
}
|
|
104
|
-
|
|
105
|
-
|
|
132
|
+
throw new MastraError(
|
|
133
|
+
{
|
|
134
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
135
|
+
domain: ErrorDomain.STORAGE,
|
|
136
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
137
|
+
details: { agentName, type: type ?? null }
|
|
138
|
+
},
|
|
139
|
+
error
|
|
140
|
+
);
|
|
106
141
|
}
|
|
107
142
|
}
|
|
108
|
-
async
|
|
143
|
+
async getEvals(options = {}) {
|
|
144
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
145
|
+
const fromDate = dateRange?.start;
|
|
146
|
+
const toDate = dateRange?.end;
|
|
147
|
+
const conditions = [];
|
|
148
|
+
if (agentName) {
|
|
149
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
150
|
+
}
|
|
151
|
+
if (type === "test") {
|
|
152
|
+
conditions.push(
|
|
153
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
154
|
+
);
|
|
155
|
+
} else if (type === "live") {
|
|
156
|
+
conditions.push(
|
|
157
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
if (fromDate) {
|
|
161
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
162
|
+
fromDate.toISOString();
|
|
163
|
+
}
|
|
164
|
+
if (toDate) {
|
|
165
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
166
|
+
toDate.toISOString();
|
|
167
|
+
}
|
|
168
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
109
169
|
try {
|
|
110
|
-
await this.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
...
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
170
|
+
const countResult = await this.client.query({
|
|
171
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
172
|
+
query_params: {
|
|
173
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
174
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
175
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
176
|
+
},
|
|
177
|
+
clickhouse_settings: {
|
|
178
|
+
date_time_input_format: "best_effort",
|
|
179
|
+
date_time_output_format: "iso",
|
|
180
|
+
use_client_time_zone: 1,
|
|
181
|
+
output_format_json_quote_64bit_integers: 0
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
const countData = await countResult.json();
|
|
185
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
186
|
+
const currentOffset = page * perPage;
|
|
187
|
+
const hasMore = currentOffset + perPage < total;
|
|
188
|
+
if (total === 0) {
|
|
189
|
+
return {
|
|
190
|
+
evals: [],
|
|
191
|
+
total: 0,
|
|
192
|
+
page,
|
|
193
|
+
perPage,
|
|
194
|
+
hasMore: false
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
const dataResult = await this.client.query({
|
|
198
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
199
|
+
query_params: {
|
|
200
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
201
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
202
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
203
|
+
var_limit: perPage || 100,
|
|
204
|
+
var_offset: currentOffset || 0
|
|
205
|
+
},
|
|
121
206
|
clickhouse_settings: {
|
|
122
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
123
207
|
date_time_input_format: "best_effort",
|
|
208
|
+
date_time_output_format: "iso",
|
|
124
209
|
use_client_time_zone: 1,
|
|
125
210
|
output_format_json_quote_64bit_integers: 0
|
|
126
211
|
}
|
|
127
212
|
});
|
|
213
|
+
const rows = await dataResult.json();
|
|
214
|
+
return {
|
|
215
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
216
|
+
total,
|
|
217
|
+
page,
|
|
218
|
+
perPage,
|
|
219
|
+
hasMore
|
|
220
|
+
};
|
|
128
221
|
} catch (error) {
|
|
129
|
-
|
|
130
|
-
|
|
222
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
223
|
+
return {
|
|
224
|
+
evals: [],
|
|
225
|
+
total: 0,
|
|
226
|
+
page,
|
|
227
|
+
perPage,
|
|
228
|
+
hasMore: false
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
throw new MastraError(
|
|
232
|
+
{
|
|
233
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
234
|
+
domain: ErrorDomain.STORAGE,
|
|
235
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
236
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
237
|
+
},
|
|
238
|
+
error
|
|
239
|
+
);
|
|
131
240
|
}
|
|
132
241
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
242
|
+
};
|
|
243
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
244
|
+
client;
|
|
245
|
+
operations;
|
|
246
|
+
constructor({ client, operations }) {
|
|
247
|
+
super();
|
|
248
|
+
this.client = client;
|
|
249
|
+
this.operations = operations;
|
|
250
|
+
}
|
|
251
|
+
async getMessages({
|
|
252
|
+
threadId,
|
|
253
|
+
resourceId,
|
|
254
|
+
selectBy,
|
|
255
|
+
format
|
|
142
256
|
}) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
257
|
+
try {
|
|
258
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
259
|
+
const messages = [];
|
|
260
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
261
|
+
const include = selectBy?.include || [];
|
|
262
|
+
if (include.length) {
|
|
263
|
+
const unionQueries = [];
|
|
264
|
+
const params = [];
|
|
265
|
+
let paramIdx = 1;
|
|
266
|
+
for (const inc of include) {
|
|
267
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
268
|
+
const searchId = inc.threadId || threadId;
|
|
269
|
+
unionQueries.push(`
|
|
270
|
+
SELECT * FROM (
|
|
271
|
+
WITH numbered_messages AS (
|
|
272
|
+
SELECT
|
|
273
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
274
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
275
|
+
FROM "${TABLE_MESSAGES}"
|
|
276
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
277
|
+
),
|
|
278
|
+
target_positions AS (
|
|
279
|
+
SELECT row_num as target_pos
|
|
280
|
+
FROM numbered_messages
|
|
281
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
282
|
+
)
|
|
283
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
284
|
+
FROM numbered_messages m
|
|
285
|
+
CROSS JOIN target_positions t
|
|
286
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
287
|
+
) AS query_${paramIdx}
|
|
288
|
+
`);
|
|
289
|
+
params.push(
|
|
290
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
291
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
292
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
293
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
294
|
+
);
|
|
295
|
+
paramIdx++;
|
|
296
|
+
}
|
|
297
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
298
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
299
|
+
const includeResult = await this.client.query({
|
|
300
|
+
query: finalQuery,
|
|
301
|
+
query_params: mergedParams,
|
|
302
|
+
clickhouse_settings: {
|
|
303
|
+
date_time_input_format: "best_effort",
|
|
304
|
+
date_time_output_format: "iso",
|
|
305
|
+
use_client_time_zone: 1,
|
|
306
|
+
output_format_json_quote_64bit_integers: 0
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
const rows2 = await includeResult.json();
|
|
310
|
+
const includedMessages = transformRows(rows2.data);
|
|
311
|
+
const seen = /* @__PURE__ */ new Set();
|
|
312
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
313
|
+
if (seen.has(message.id)) return false;
|
|
314
|
+
seen.add(message.id);
|
|
315
|
+
return true;
|
|
316
|
+
});
|
|
317
|
+
messages.push(...dedupedMessages);
|
|
318
|
+
}
|
|
319
|
+
const result = await this.client.query({
|
|
320
|
+
query: `
|
|
321
|
+
SELECT
|
|
322
|
+
id,
|
|
323
|
+
content,
|
|
324
|
+
role,
|
|
325
|
+
type,
|
|
326
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
327
|
+
thread_id AS "threadId"
|
|
328
|
+
FROM "${TABLE_MESSAGES}"
|
|
329
|
+
WHERE thread_id = {threadId:String}
|
|
330
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
331
|
+
ORDER BY "createdAt" DESC
|
|
332
|
+
LIMIT {limit:Int64}
|
|
333
|
+
`,
|
|
334
|
+
query_params: {
|
|
335
|
+
threadId,
|
|
336
|
+
exclude: messages.map((m) => m.id),
|
|
337
|
+
limit
|
|
338
|
+
},
|
|
339
|
+
clickhouse_settings: {
|
|
340
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
341
|
+
date_time_input_format: "best_effort",
|
|
342
|
+
date_time_output_format: "iso",
|
|
343
|
+
use_client_time_zone: 1,
|
|
344
|
+
output_format_json_quote_64bit_integers: 0
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
const rows = await result.json();
|
|
348
|
+
messages.push(...transformRows(rows.data));
|
|
349
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
350
|
+
messages.forEach((message) => {
|
|
351
|
+
if (typeof message.content === "string") {
|
|
352
|
+
try {
|
|
353
|
+
message.content = JSON.parse(message.content);
|
|
354
|
+
} catch {
|
|
355
|
+
}
|
|
356
|
+
}
|
|
159
357
|
});
|
|
358
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
359
|
+
if (format === `v2`) return list.get.all.v2();
|
|
360
|
+
return list.get.all.v1();
|
|
361
|
+
} catch (error) {
|
|
362
|
+
throw new MastraError(
|
|
363
|
+
{
|
|
364
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
365
|
+
domain: ErrorDomain.STORAGE,
|
|
366
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
367
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
368
|
+
},
|
|
369
|
+
error
|
|
370
|
+
);
|
|
160
371
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
372
|
+
}
|
|
373
|
+
async getMessagesById({
|
|
374
|
+
messageIds,
|
|
375
|
+
format
|
|
376
|
+
}) {
|
|
377
|
+
if (messageIds.length === 0) return [];
|
|
378
|
+
try {
|
|
379
|
+
const result = await this.client.query({
|
|
380
|
+
query: `
|
|
381
|
+
SELECT
|
|
382
|
+
id,
|
|
383
|
+
content,
|
|
384
|
+
role,
|
|
385
|
+
type,
|
|
386
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
387
|
+
thread_id AS "threadId",
|
|
388
|
+
"resourceId"
|
|
389
|
+
FROM "${TABLE_MESSAGES}"
|
|
390
|
+
WHERE id IN {messageIds:Array(String)}
|
|
391
|
+
ORDER BY "createdAt" DESC
|
|
392
|
+
`,
|
|
393
|
+
query_params: {
|
|
394
|
+
messageIds
|
|
395
|
+
},
|
|
396
|
+
clickhouse_settings: {
|
|
397
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
398
|
+
date_time_input_format: "best_effort",
|
|
399
|
+
date_time_output_format: "iso",
|
|
400
|
+
use_client_time_zone: 1,
|
|
401
|
+
output_format_json_quote_64bit_integers: 0
|
|
402
|
+
}
|
|
403
|
+
});
|
|
404
|
+
const rows = await result.json();
|
|
405
|
+
const messages = transformRows(rows.data);
|
|
406
|
+
messages.forEach((message) => {
|
|
407
|
+
if (typeof message.content === "string") {
|
|
408
|
+
try {
|
|
409
|
+
message.content = JSON.parse(message.content);
|
|
410
|
+
} catch {
|
|
411
|
+
}
|
|
412
|
+
}
|
|
167
413
|
});
|
|
414
|
+
const list = new MessageList().add(messages, "memory");
|
|
415
|
+
if (format === `v1`) return list.get.all.v1();
|
|
416
|
+
return list.get.all.v2();
|
|
417
|
+
} catch (error) {
|
|
418
|
+
throw new MastraError(
|
|
419
|
+
{
|
|
420
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
|
|
421
|
+
domain: ErrorDomain.STORAGE,
|
|
422
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
423
|
+
details: { messageIds: JSON.stringify(messageIds) }
|
|
424
|
+
},
|
|
425
|
+
error
|
|
426
|
+
);
|
|
168
427
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
428
|
+
}
|
|
429
|
+
async saveMessages(args) {
|
|
430
|
+
const { messages, format = "v1" } = args;
|
|
431
|
+
if (messages.length === 0) return messages;
|
|
432
|
+
for (const message of messages) {
|
|
433
|
+
const resourceId = message.resourceId;
|
|
434
|
+
if (!resourceId) {
|
|
435
|
+
throw new Error("Resource ID is required");
|
|
436
|
+
}
|
|
437
|
+
if (!message.threadId) {
|
|
438
|
+
throw new Error("Thread ID is required");
|
|
439
|
+
}
|
|
440
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
441
|
+
if (!thread) {
|
|
442
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
443
|
+
}
|
|
172
444
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
445
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
446
|
+
await Promise.all(
|
|
447
|
+
messages.map(async (m) => {
|
|
448
|
+
const resourceId = m.resourceId;
|
|
449
|
+
if (!resourceId) {
|
|
450
|
+
throw new Error("Resource ID is required");
|
|
451
|
+
}
|
|
452
|
+
if (!m.threadId) {
|
|
453
|
+
throw new Error("Thread ID is required");
|
|
454
|
+
}
|
|
455
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
456
|
+
if (!thread) {
|
|
457
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
458
|
+
}
|
|
459
|
+
threadIdSet.set(m.threadId, thread);
|
|
460
|
+
})
|
|
461
|
+
);
|
|
462
|
+
try {
|
|
463
|
+
const existingResult = await this.client.query({
|
|
464
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
465
|
+
query_params: {
|
|
466
|
+
ids: messages.map((m) => m.id)
|
|
467
|
+
},
|
|
468
|
+
clickhouse_settings: {
|
|
469
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
470
|
+
date_time_input_format: "best_effort",
|
|
471
|
+
date_time_output_format: "iso",
|
|
472
|
+
use_client_time_zone: 1,
|
|
473
|
+
output_format_json_quote_64bit_integers: 0
|
|
474
|
+
},
|
|
475
|
+
format: "JSONEachRow"
|
|
476
|
+
});
|
|
477
|
+
const existingRows = await existingResult.json();
|
|
478
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
479
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
480
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
481
|
+
const toMove = messages.filter((m) => {
|
|
482
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
483
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
484
|
+
});
|
|
485
|
+
const deletePromises = toMove.map((message) => {
|
|
486
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
487
|
+
if (!existingRow) return Promise.resolve();
|
|
488
|
+
return this.client.command({
|
|
489
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
490
|
+
query_params: {
|
|
491
|
+
var_id: message.id,
|
|
492
|
+
var_old_thread_id: existingRow.thread_id
|
|
493
|
+
},
|
|
494
|
+
clickhouse_settings: {
|
|
495
|
+
date_time_input_format: "best_effort",
|
|
496
|
+
use_client_time_zone: 1,
|
|
497
|
+
output_format_json_quote_64bit_integers: 0
|
|
498
|
+
}
|
|
499
|
+
});
|
|
500
|
+
});
|
|
501
|
+
const updatePromises = toUpdate.map(
|
|
502
|
+
(message) => this.client.command({
|
|
503
|
+
query: `
|
|
504
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
505
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
506
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
507
|
+
`,
|
|
508
|
+
query_params: {
|
|
509
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
510
|
+
var_role: message.role,
|
|
511
|
+
var_type: message.type || "v2",
|
|
512
|
+
var_resourceId: message.resourceId,
|
|
513
|
+
var_id: message.id,
|
|
514
|
+
var_thread_id: message.threadId
|
|
515
|
+
},
|
|
516
|
+
clickhouse_settings: {
|
|
517
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
518
|
+
date_time_input_format: "best_effort",
|
|
519
|
+
use_client_time_zone: 1,
|
|
520
|
+
output_format_json_quote_64bit_integers: 0
|
|
521
|
+
}
|
|
522
|
+
})
|
|
523
|
+
);
|
|
524
|
+
await Promise.all([
|
|
525
|
+
// Insert new messages (including moved messages)
|
|
526
|
+
this.client.insert({
|
|
527
|
+
table: TABLE_MESSAGES,
|
|
528
|
+
format: "JSONEachRow",
|
|
529
|
+
values: toInsert.map((message) => ({
|
|
530
|
+
id: message.id,
|
|
531
|
+
thread_id: message.threadId,
|
|
532
|
+
resourceId: message.resourceId,
|
|
533
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
534
|
+
createdAt: message.createdAt.toISOString(),
|
|
535
|
+
role: message.role,
|
|
536
|
+
type: message.type || "v2"
|
|
537
|
+
})),
|
|
538
|
+
clickhouse_settings: {
|
|
539
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
540
|
+
date_time_input_format: "best_effort",
|
|
541
|
+
use_client_time_zone: 1,
|
|
542
|
+
output_format_json_quote_64bit_integers: 0
|
|
543
|
+
}
|
|
544
|
+
}),
|
|
545
|
+
...updatePromises,
|
|
546
|
+
...deletePromises,
|
|
547
|
+
// Update thread's updatedAt timestamp
|
|
548
|
+
this.client.insert({
|
|
549
|
+
table: TABLE_THREADS,
|
|
550
|
+
format: "JSONEachRow",
|
|
551
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
552
|
+
id: thread.id,
|
|
553
|
+
resourceId: thread.resourceId,
|
|
554
|
+
title: thread.title,
|
|
555
|
+
metadata: thread.metadata,
|
|
556
|
+
createdAt: thread.createdAt,
|
|
557
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
558
|
+
})),
|
|
559
|
+
clickhouse_settings: {
|
|
560
|
+
date_time_input_format: "best_effort",
|
|
561
|
+
use_client_time_zone: 1,
|
|
562
|
+
output_format_json_quote_64bit_integers: 0
|
|
563
|
+
}
|
|
564
|
+
})
|
|
565
|
+
]);
|
|
566
|
+
const list = new MessageList().add(messages, "memory");
|
|
567
|
+
if (format === `v2`) return list.get.all.v2();
|
|
568
|
+
return list.get.all.v1();
|
|
569
|
+
} catch (error) {
|
|
570
|
+
throw new MastraError(
|
|
571
|
+
{
|
|
572
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
573
|
+
domain: ErrorDomain.STORAGE,
|
|
574
|
+
category: ErrorCategory.THIRD_PARTY
|
|
575
|
+
},
|
|
576
|
+
error
|
|
577
|
+
);
|
|
176
578
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
579
|
+
}
|
|
580
|
+
async getThreadById({ threadId }) {
|
|
581
|
+
try {
|
|
582
|
+
const result = await this.client.query({
|
|
583
|
+
query: `SELECT
|
|
584
|
+
id,
|
|
585
|
+
"resourceId",
|
|
586
|
+
title,
|
|
587
|
+
metadata,
|
|
588
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
589
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
590
|
+
FROM "${TABLE_THREADS}"
|
|
591
|
+
FINAL
|
|
592
|
+
WHERE id = {var_id:String}`,
|
|
593
|
+
query_params: { var_id: threadId },
|
|
594
|
+
clickhouse_settings: {
|
|
595
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
596
|
+
date_time_input_format: "best_effort",
|
|
597
|
+
date_time_output_format: "iso",
|
|
598
|
+
use_client_time_zone: 1,
|
|
599
|
+
output_format_json_quote_64bit_integers: 0
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
const rows = await result.json();
|
|
603
|
+
const thread = transformRow(rows.data[0]);
|
|
604
|
+
if (!thread) {
|
|
605
|
+
return null;
|
|
606
|
+
}
|
|
607
|
+
return {
|
|
608
|
+
...thread,
|
|
609
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
610
|
+
createdAt: thread.createdAt,
|
|
611
|
+
updatedAt: thread.updatedAt
|
|
612
|
+
};
|
|
613
|
+
} catch (error) {
|
|
614
|
+
throw new MastraError(
|
|
615
|
+
{
|
|
616
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
617
|
+
domain: ErrorDomain.STORAGE,
|
|
618
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
619
|
+
details: { threadId }
|
|
620
|
+
},
|
|
621
|
+
error
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
626
|
+
try {
|
|
627
|
+
const result = await this.client.query({
|
|
628
|
+
query: `SELECT
|
|
629
|
+
id,
|
|
630
|
+
"resourceId",
|
|
631
|
+
title,
|
|
632
|
+
metadata,
|
|
633
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
634
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
635
|
+
FROM "${TABLE_THREADS}"
|
|
636
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
637
|
+
query_params: { var_resourceId: resourceId },
|
|
638
|
+
clickhouse_settings: {
|
|
639
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
640
|
+
date_time_input_format: "best_effort",
|
|
641
|
+
date_time_output_format: "iso",
|
|
642
|
+
use_client_time_zone: 1,
|
|
643
|
+
output_format_json_quote_64bit_integers: 0
|
|
644
|
+
}
|
|
645
|
+
});
|
|
646
|
+
const rows = await result.json();
|
|
647
|
+
const threads = transformRows(rows.data);
|
|
648
|
+
return threads.map((thread) => ({
|
|
649
|
+
...thread,
|
|
650
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
651
|
+
createdAt: thread.createdAt,
|
|
652
|
+
updatedAt: thread.updatedAt
|
|
653
|
+
}));
|
|
654
|
+
} catch (error) {
|
|
655
|
+
throw new MastraError(
|
|
656
|
+
{
|
|
657
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
658
|
+
domain: ErrorDomain.STORAGE,
|
|
659
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
660
|
+
details: { resourceId }
|
|
661
|
+
},
|
|
662
|
+
error
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
async saveThread({ thread }) {
|
|
667
|
+
try {
|
|
668
|
+
await this.client.insert({
|
|
669
|
+
table: TABLE_THREADS,
|
|
670
|
+
values: [
|
|
671
|
+
{
|
|
672
|
+
...thread,
|
|
673
|
+
createdAt: thread.createdAt.toISOString(),
|
|
674
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
675
|
+
}
|
|
676
|
+
],
|
|
677
|
+
format: "JSONEachRow",
|
|
678
|
+
clickhouse_settings: {
|
|
679
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
680
|
+
date_time_input_format: "best_effort",
|
|
681
|
+
use_client_time_zone: 1,
|
|
682
|
+
output_format_json_quote_64bit_integers: 0
|
|
683
|
+
}
|
|
684
|
+
});
|
|
685
|
+
return thread;
|
|
686
|
+
} catch (error) {
|
|
687
|
+
throw new MastraError(
|
|
688
|
+
{
|
|
689
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
690
|
+
domain: ErrorDomain.STORAGE,
|
|
691
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
692
|
+
details: { threadId: thread.id }
|
|
693
|
+
},
|
|
694
|
+
error
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
async updateThread({
|
|
699
|
+
id,
|
|
700
|
+
title,
|
|
701
|
+
metadata
|
|
702
|
+
}) {
|
|
703
|
+
try {
|
|
704
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
705
|
+
if (!existingThread) {
|
|
706
|
+
throw new Error(`Thread ${id} not found`);
|
|
707
|
+
}
|
|
708
|
+
const mergedMetadata = {
|
|
709
|
+
...existingThread.metadata,
|
|
710
|
+
...metadata
|
|
711
|
+
};
|
|
712
|
+
const updatedThread = {
|
|
713
|
+
...existingThread,
|
|
714
|
+
title,
|
|
715
|
+
metadata: mergedMetadata,
|
|
716
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
717
|
+
};
|
|
718
|
+
await this.client.insert({
|
|
719
|
+
table: TABLE_THREADS,
|
|
720
|
+
format: "JSONEachRow",
|
|
721
|
+
values: [
|
|
722
|
+
{
|
|
723
|
+
id: updatedThread.id,
|
|
724
|
+
resourceId: updatedThread.resourceId,
|
|
725
|
+
title: updatedThread.title,
|
|
726
|
+
metadata: updatedThread.metadata,
|
|
727
|
+
createdAt: updatedThread.createdAt,
|
|
728
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
729
|
+
}
|
|
730
|
+
],
|
|
731
|
+
clickhouse_settings: {
|
|
732
|
+
date_time_input_format: "best_effort",
|
|
733
|
+
use_client_time_zone: 1,
|
|
734
|
+
output_format_json_quote_64bit_integers: 0
|
|
735
|
+
}
|
|
736
|
+
});
|
|
737
|
+
return updatedThread;
|
|
738
|
+
} catch (error) {
|
|
739
|
+
throw new MastraError(
|
|
740
|
+
{
|
|
741
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
742
|
+
domain: ErrorDomain.STORAGE,
|
|
743
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
744
|
+
details: { threadId: id, title }
|
|
745
|
+
},
|
|
746
|
+
error
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
async deleteThread({ threadId }) {
|
|
751
|
+
try {
|
|
752
|
+
await this.client.command({
|
|
753
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
754
|
+
query_params: { var_thread_id: threadId },
|
|
755
|
+
clickhouse_settings: {
|
|
756
|
+
output_format_json_quote_64bit_integers: 0
|
|
757
|
+
}
|
|
758
|
+
});
|
|
759
|
+
await this.client.command({
|
|
760
|
+
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
761
|
+
query_params: { var_id: threadId },
|
|
762
|
+
clickhouse_settings: {
|
|
763
|
+
output_format_json_quote_64bit_integers: 0
|
|
764
|
+
}
|
|
765
|
+
});
|
|
766
|
+
} catch (error) {
|
|
767
|
+
throw new MastraError(
|
|
768
|
+
{
|
|
769
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
770
|
+
domain: ErrorDomain.STORAGE,
|
|
771
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
772
|
+
details: { threadId }
|
|
773
|
+
},
|
|
774
|
+
error
|
|
775
|
+
);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
779
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
780
|
+
try {
|
|
781
|
+
const currentOffset = page * perPage;
|
|
782
|
+
const countResult = await this.client.query({
|
|
783
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
784
|
+
query_params: { resourceId },
|
|
785
|
+
clickhouse_settings: {
|
|
786
|
+
date_time_input_format: "best_effort",
|
|
787
|
+
date_time_output_format: "iso",
|
|
788
|
+
use_client_time_zone: 1,
|
|
789
|
+
output_format_json_quote_64bit_integers: 0
|
|
790
|
+
}
|
|
791
|
+
});
|
|
792
|
+
const countData = await countResult.json();
|
|
793
|
+
const total = countData.data[0].total;
|
|
794
|
+
if (total === 0) {
|
|
795
|
+
return {
|
|
796
|
+
threads: [],
|
|
797
|
+
total: 0,
|
|
798
|
+
page,
|
|
799
|
+
perPage,
|
|
800
|
+
hasMore: false
|
|
801
|
+
};
|
|
802
|
+
}
|
|
803
|
+
const dataResult = await this.client.query({
|
|
804
|
+
query: `
|
|
805
|
+
SELECT
|
|
806
|
+
id,
|
|
807
|
+
resourceId,
|
|
808
|
+
title,
|
|
809
|
+
metadata,
|
|
810
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
811
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
812
|
+
FROM ${TABLE_THREADS}
|
|
813
|
+
WHERE resourceId = {resourceId:String}
|
|
814
|
+
ORDER BY createdAt DESC
|
|
815
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
816
|
+
`,
|
|
817
|
+
query_params: {
|
|
818
|
+
resourceId,
|
|
819
|
+
limit: perPage,
|
|
820
|
+
offset: currentOffset
|
|
821
|
+
},
|
|
822
|
+
clickhouse_settings: {
|
|
823
|
+
date_time_input_format: "best_effort",
|
|
824
|
+
date_time_output_format: "iso",
|
|
825
|
+
use_client_time_zone: 1,
|
|
826
|
+
output_format_json_quote_64bit_integers: 0
|
|
827
|
+
}
|
|
828
|
+
});
|
|
829
|
+
const rows = await dataResult.json();
|
|
830
|
+
const threads = transformRows(rows.data);
|
|
831
|
+
return {
|
|
832
|
+
threads,
|
|
833
|
+
total,
|
|
834
|
+
page,
|
|
835
|
+
perPage,
|
|
836
|
+
hasMore: currentOffset + threads.length < total
|
|
837
|
+
};
|
|
838
|
+
} catch (error) {
|
|
839
|
+
throw new MastraError(
|
|
840
|
+
{
|
|
841
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
842
|
+
domain: ErrorDomain.STORAGE,
|
|
843
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
844
|
+
details: { resourceId, page }
|
|
845
|
+
},
|
|
846
|
+
error
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
async getMessagesPaginated(args) {
|
|
851
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
852
|
+
const page = selectBy?.pagination?.page || 0;
|
|
853
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
854
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
855
|
+
try {
|
|
856
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
857
|
+
const offset = page * perPage;
|
|
858
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
859
|
+
const fromDate = dateRange?.start;
|
|
860
|
+
const toDate = dateRange?.end;
|
|
861
|
+
const messages = [];
|
|
862
|
+
if (selectBy?.include?.length) {
|
|
863
|
+
const include = selectBy.include;
|
|
864
|
+
const unionQueries = [];
|
|
865
|
+
const params = [];
|
|
866
|
+
let paramIdx = 1;
|
|
867
|
+
for (const inc of include) {
|
|
868
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
869
|
+
const searchId = inc.threadId || threadId;
|
|
870
|
+
unionQueries.push(`
|
|
871
|
+
SELECT * FROM (
|
|
872
|
+
WITH numbered_messages AS (
|
|
873
|
+
SELECT
|
|
874
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
875
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
876
|
+
FROM "${TABLE_MESSAGES}"
|
|
877
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
878
|
+
),
|
|
879
|
+
target_positions AS (
|
|
880
|
+
SELECT row_num as target_pos
|
|
881
|
+
FROM numbered_messages
|
|
882
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
883
|
+
)
|
|
884
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
885
|
+
FROM numbered_messages m
|
|
886
|
+
CROSS JOIN target_positions t
|
|
887
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
888
|
+
) AS query_${paramIdx}
|
|
889
|
+
`);
|
|
890
|
+
params.push(
|
|
891
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
892
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
893
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
894
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
895
|
+
);
|
|
896
|
+
paramIdx++;
|
|
897
|
+
}
|
|
898
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
899
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
900
|
+
const includeResult = await this.client.query({
|
|
901
|
+
query: finalQuery,
|
|
902
|
+
query_params: mergedParams,
|
|
903
|
+
clickhouse_settings: {
|
|
904
|
+
date_time_input_format: "best_effort",
|
|
905
|
+
date_time_output_format: "iso",
|
|
906
|
+
use_client_time_zone: 1,
|
|
907
|
+
output_format_json_quote_64bit_integers: 0
|
|
908
|
+
}
|
|
909
|
+
});
|
|
910
|
+
const rows2 = await includeResult.json();
|
|
911
|
+
const includedMessages = transformRows(rows2.data);
|
|
912
|
+
const seen = /* @__PURE__ */ new Set();
|
|
913
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
914
|
+
if (seen.has(message.id)) return false;
|
|
915
|
+
seen.add(message.id);
|
|
916
|
+
return true;
|
|
917
|
+
});
|
|
918
|
+
messages.push(...dedupedMessages);
|
|
919
|
+
}
|
|
920
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
921
|
+
const countParams = { threadId };
|
|
922
|
+
if (fromDate) {
|
|
923
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
924
|
+
countParams.fromDate = fromDate.toISOString();
|
|
925
|
+
}
|
|
926
|
+
if (toDate) {
|
|
927
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
928
|
+
countParams.toDate = toDate.toISOString();
|
|
929
|
+
}
|
|
930
|
+
const countResult = await this.client.query({
|
|
931
|
+
query: countQuery,
|
|
932
|
+
query_params: countParams,
|
|
933
|
+
clickhouse_settings: {
|
|
934
|
+
date_time_input_format: "best_effort",
|
|
935
|
+
date_time_output_format: "iso",
|
|
936
|
+
use_client_time_zone: 1,
|
|
937
|
+
output_format_json_quote_64bit_integers: 0
|
|
938
|
+
}
|
|
939
|
+
});
|
|
940
|
+
const countData = await countResult.json();
|
|
941
|
+
const total = countData.data[0].total;
|
|
942
|
+
if (total === 0 && messages.length === 0) {
|
|
943
|
+
return {
|
|
944
|
+
messages: [],
|
|
945
|
+
total: 0,
|
|
946
|
+
page,
|
|
947
|
+
perPage,
|
|
948
|
+
hasMore: false
|
|
949
|
+
};
|
|
950
|
+
}
|
|
951
|
+
const excludeIds = messages.map((m) => m.id);
|
|
952
|
+
let dataQuery = `
|
|
953
|
+
SELECT
|
|
954
|
+
id,
|
|
955
|
+
content,
|
|
956
|
+
role,
|
|
957
|
+
type,
|
|
958
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
959
|
+
thread_id AS "threadId",
|
|
960
|
+
resourceId
|
|
961
|
+
FROM ${TABLE_MESSAGES}
|
|
962
|
+
WHERE thread_id = {threadId:String}
|
|
963
|
+
`;
|
|
964
|
+
const dataParams = { threadId };
|
|
965
|
+
if (fromDate) {
|
|
966
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
967
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
968
|
+
}
|
|
969
|
+
if (toDate) {
|
|
970
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
971
|
+
dataParams.toDate = toDate.toISOString();
|
|
972
|
+
}
|
|
973
|
+
if (excludeIds.length > 0) {
|
|
974
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
975
|
+
dataParams.excludeIds = excludeIds;
|
|
976
|
+
}
|
|
977
|
+
if (selectBy?.last) {
|
|
978
|
+
dataQuery += `
|
|
979
|
+
ORDER BY createdAt DESC
|
|
980
|
+
LIMIT {limit:Int64}
|
|
981
|
+
`;
|
|
982
|
+
dataParams.limit = perPage;
|
|
983
|
+
} else {
|
|
984
|
+
dataQuery += `
|
|
985
|
+
ORDER BY createdAt ASC
|
|
986
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
987
|
+
`;
|
|
988
|
+
dataParams.limit = perPage;
|
|
989
|
+
dataParams.offset = offset;
|
|
990
|
+
}
|
|
991
|
+
const result = await this.client.query({
|
|
992
|
+
query: dataQuery,
|
|
993
|
+
query_params: dataParams,
|
|
994
|
+
clickhouse_settings: {
|
|
995
|
+
date_time_input_format: "best_effort",
|
|
996
|
+
date_time_output_format: "iso",
|
|
997
|
+
use_client_time_zone: 1,
|
|
998
|
+
output_format_json_quote_64bit_integers: 0
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
1001
|
+
const rows = await result.json();
|
|
1002
|
+
const paginatedMessages = transformRows(rows.data);
|
|
1003
|
+
messages.push(...paginatedMessages);
|
|
1004
|
+
if (selectBy?.last) {
|
|
1005
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1006
|
+
}
|
|
1007
|
+
return {
|
|
1008
|
+
messages: format === "v2" ? messages : messages,
|
|
1009
|
+
total,
|
|
1010
|
+
page,
|
|
1011
|
+
perPage,
|
|
1012
|
+
hasMore: offset + perPage < total
|
|
1013
|
+
};
|
|
1014
|
+
} catch (error) {
|
|
1015
|
+
const mastraError = new MastraError(
|
|
1016
|
+
{
|
|
1017
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
1018
|
+
domain: ErrorDomain.STORAGE,
|
|
1019
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1020
|
+
details: {
|
|
1021
|
+
threadId,
|
|
1022
|
+
resourceId: resourceId ?? ""
|
|
1023
|
+
}
|
|
1024
|
+
},
|
|
1025
|
+
error
|
|
1026
|
+
);
|
|
1027
|
+
this.logger?.trackException?.(mastraError);
|
|
1028
|
+
this.logger?.error?.(mastraError.toString());
|
|
1029
|
+
return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
async updateMessages(args) {
|
|
1033
|
+
const { messages } = args;
|
|
1034
|
+
if (messages.length === 0) {
|
|
1035
|
+
return [];
|
|
1036
|
+
}
|
|
1037
|
+
try {
|
|
1038
|
+
const messageIds = messages.map((m) => m.id);
|
|
1039
|
+
const existingResult = await this.client.query({
|
|
1040
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
1041
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
1042
|
+
clickhouse_settings: {
|
|
1043
|
+
date_time_input_format: "best_effort",
|
|
1044
|
+
date_time_output_format: "iso",
|
|
1045
|
+
use_client_time_zone: 1,
|
|
1046
|
+
output_format_json_quote_64bit_integers: 0
|
|
1047
|
+
}
|
|
1048
|
+
});
|
|
1049
|
+
const existingRows = await existingResult.json();
|
|
1050
|
+
const existingMessages = transformRows(existingRows.data);
|
|
1051
|
+
if (existingMessages.length === 0) {
|
|
1052
|
+
return [];
|
|
1053
|
+
}
|
|
1054
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
1055
|
+
if (typeof msg.content === "string") {
|
|
1056
|
+
try {
|
|
1057
|
+
msg.content = JSON.parse(msg.content);
|
|
1058
|
+
} catch {
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
return msg;
|
|
1062
|
+
});
|
|
1063
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
1064
|
+
const updatePromises = [];
|
|
1065
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1066
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1067
|
+
if (!updatePayload) continue;
|
|
1068
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1069
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1070
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1071
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1072
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1073
|
+
}
|
|
1074
|
+
const setClauses = [];
|
|
1075
|
+
const values = {};
|
|
1076
|
+
let paramIdx = 1;
|
|
1077
|
+
let newContent = null;
|
|
1078
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1079
|
+
if (updatableFields.content) {
|
|
1080
|
+
const existingContent = existingMessage.content || {};
|
|
1081
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1082
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1083
|
+
newContent = {
|
|
1084
|
+
...existingContent,
|
|
1085
|
+
...updatableFields.content,
|
|
1086
|
+
// Deep merge metadata
|
|
1087
|
+
metadata: {
|
|
1088
|
+
...existingMetadata,
|
|
1089
|
+
...updateMetadata
|
|
1090
|
+
}
|
|
1091
|
+
};
|
|
1092
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1093
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1094
|
+
paramIdx++;
|
|
1095
|
+
delete updatableFields.content;
|
|
1096
|
+
}
|
|
1097
|
+
for (const key in updatableFields) {
|
|
1098
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1099
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1100
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1101
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1102
|
+
paramIdx++;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
if (setClauses.length > 0) {
|
|
1106
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1107
|
+
const updateQuery = `
|
|
1108
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1109
|
+
UPDATE ${setClauses.join(", ")}
|
|
1110
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1111
|
+
`;
|
|
1112
|
+
console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1113
|
+
updatePromises.push(
|
|
1114
|
+
this.client.command({
|
|
1115
|
+
query: updateQuery,
|
|
1116
|
+
query_params: values,
|
|
1117
|
+
clickhouse_settings: {
|
|
1118
|
+
date_time_input_format: "best_effort",
|
|
1119
|
+
use_client_time_zone: 1,
|
|
1120
|
+
output_format_json_quote_64bit_integers: 0
|
|
1121
|
+
}
|
|
1122
|
+
})
|
|
1123
|
+
);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
if (updatePromises.length > 0) {
|
|
1127
|
+
await Promise.all(updatePromises);
|
|
1128
|
+
}
|
|
1129
|
+
await this.client.command({
|
|
1130
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1131
|
+
clickhouse_settings: {
|
|
1132
|
+
date_time_input_format: "best_effort",
|
|
1133
|
+
use_client_time_zone: 1,
|
|
1134
|
+
output_format_json_quote_64bit_integers: 0
|
|
1135
|
+
}
|
|
1136
|
+
});
|
|
1137
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1138
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1139
|
+
if (!updatePayload) continue;
|
|
1140
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1141
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1142
|
+
const verifyResult = await this.client.query({
|
|
1143
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1144
|
+
query_params: { messageId: id },
|
|
1145
|
+
clickhouse_settings: {
|
|
1146
|
+
date_time_input_format: "best_effort",
|
|
1147
|
+
date_time_output_format: "iso",
|
|
1148
|
+
use_client_time_zone: 1,
|
|
1149
|
+
output_format_json_quote_64bit_integers: 0
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
const verifyRows = await verifyResult.json();
|
|
1153
|
+
if (verifyRows.data.length > 0) {
|
|
1154
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1155
|
+
if (updatedMessage) {
|
|
1156
|
+
let needsRetry = false;
|
|
1157
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1158
|
+
if (key === "content") {
|
|
1159
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1160
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1161
|
+
if (actualContent !== expectedContent) {
|
|
1162
|
+
needsRetry = true;
|
|
1163
|
+
break;
|
|
1164
|
+
}
|
|
1165
|
+
} else if (updatedMessage[key] !== value) {
|
|
1166
|
+
needsRetry = true;
|
|
1167
|
+
break;
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
if (needsRetry) {
|
|
1171
|
+
console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1172
|
+
await this.client.command({
|
|
1173
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1174
|
+
query_params: { messageId: id },
|
|
1175
|
+
clickhouse_settings: {
|
|
1176
|
+
date_time_input_format: "best_effort",
|
|
1177
|
+
use_client_time_zone: 1,
|
|
1178
|
+
output_format_json_quote_64bit_integers: 0
|
|
1179
|
+
}
|
|
1180
|
+
});
|
|
1181
|
+
let updatedContent = existingMessage.content || {};
|
|
1182
|
+
if (fieldsToUpdate.content) {
|
|
1183
|
+
const existingContent = existingMessage.content || {};
|
|
1184
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1185
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1186
|
+
updatedContent = {
|
|
1187
|
+
...existingContent,
|
|
1188
|
+
...fieldsToUpdate.content,
|
|
1189
|
+
metadata: {
|
|
1190
|
+
...existingMetadata,
|
|
1191
|
+
...updateMetadata
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
const updatedMessageData = {
|
|
1196
|
+
...existingMessage,
|
|
1197
|
+
...fieldsToUpdate,
|
|
1198
|
+
content: updatedContent
|
|
1199
|
+
};
|
|
1200
|
+
await this.client.insert({
|
|
1201
|
+
table: TABLE_MESSAGES,
|
|
1202
|
+
format: "JSONEachRow",
|
|
1203
|
+
values: [
|
|
1204
|
+
{
|
|
1205
|
+
id: updatedMessageData.id,
|
|
1206
|
+
thread_id: updatedMessageData.threadId,
|
|
1207
|
+
resourceId: updatedMessageData.resourceId,
|
|
1208
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1209
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1210
|
+
role: updatedMessageData.role,
|
|
1211
|
+
type: updatedMessageData.type || "v2"
|
|
1212
|
+
}
|
|
1213
|
+
],
|
|
1214
|
+
clickhouse_settings: {
|
|
1215
|
+
date_time_input_format: "best_effort",
|
|
1216
|
+
use_client_time_zone: 1,
|
|
1217
|
+
output_format_json_quote_64bit_integers: 0
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1225
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1226
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1227
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1228
|
+
const threadResult = await this.client.query({
|
|
1229
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1230
|
+
query_params: { threadId },
|
|
1231
|
+
clickhouse_settings: {
|
|
1232
|
+
date_time_input_format: "best_effort",
|
|
1233
|
+
date_time_output_format: "iso",
|
|
1234
|
+
use_client_time_zone: 1,
|
|
1235
|
+
output_format_json_quote_64bit_integers: 0
|
|
1236
|
+
}
|
|
1237
|
+
});
|
|
1238
|
+
const threadRows = await threadResult.json();
|
|
1239
|
+
if (threadRows.data.length > 0) {
|
|
1240
|
+
const existingThread = threadRows.data[0];
|
|
1241
|
+
await this.client.command({
|
|
1242
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1243
|
+
query_params: { threadId },
|
|
1244
|
+
clickhouse_settings: {
|
|
1245
|
+
date_time_input_format: "best_effort",
|
|
1246
|
+
use_client_time_zone: 1,
|
|
1247
|
+
output_format_json_quote_64bit_integers: 0
|
|
1248
|
+
}
|
|
1249
|
+
});
|
|
1250
|
+
await this.client.insert({
|
|
1251
|
+
table: TABLE_THREADS,
|
|
1252
|
+
format: "JSONEachRow",
|
|
1253
|
+
values: [
|
|
1254
|
+
{
|
|
1255
|
+
id: existingThread.id,
|
|
1256
|
+
resourceId: existingThread.resourceId,
|
|
1257
|
+
title: existingThread.title,
|
|
1258
|
+
metadata: existingThread.metadata,
|
|
1259
|
+
createdAt: existingThread.createdAt,
|
|
1260
|
+
updatedAt: now
|
|
1261
|
+
}
|
|
1262
|
+
],
|
|
1263
|
+
clickhouse_settings: {
|
|
1264
|
+
date_time_input_format: "best_effort",
|
|
1265
|
+
use_client_time_zone: 1,
|
|
1266
|
+
output_format_json_quote_64bit_integers: 0
|
|
1267
|
+
}
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
});
|
|
1271
|
+
await Promise.all(threadUpdatePromises);
|
|
1272
|
+
}
|
|
1273
|
+
const updatedMessages = [];
|
|
1274
|
+
for (const messageId of messageIds) {
|
|
1275
|
+
const updatedResult = await this.client.query({
|
|
1276
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1277
|
+
query_params: { messageId },
|
|
1278
|
+
clickhouse_settings: {
|
|
1279
|
+
date_time_input_format: "best_effort",
|
|
1280
|
+
date_time_output_format: "iso",
|
|
1281
|
+
use_client_time_zone: 1,
|
|
1282
|
+
output_format_json_quote_64bit_integers: 0
|
|
1283
|
+
}
|
|
1284
|
+
});
|
|
1285
|
+
const updatedRows = await updatedResult.json();
|
|
1286
|
+
if (updatedRows.data.length > 0) {
|
|
1287
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1288
|
+
if (message) {
|
|
1289
|
+
updatedMessages.push(message);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
return updatedMessages.map((message) => {
|
|
1294
|
+
if (typeof message.content === "string") {
|
|
1295
|
+
try {
|
|
1296
|
+
message.content = JSON.parse(message.content);
|
|
1297
|
+
} catch {
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
return message;
|
|
1301
|
+
});
|
|
1302
|
+
} catch (error) {
|
|
1303
|
+
throw new MastraError(
|
|
1304
|
+
{
|
|
1305
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1306
|
+
domain: ErrorDomain.STORAGE,
|
|
1307
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1308
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1309
|
+
},
|
|
1310
|
+
error
|
|
1311
|
+
);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
async getResourceById({ resourceId }) {
|
|
1315
|
+
try {
|
|
1316
|
+
const result = await this.client.query({
|
|
1317
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1318
|
+
query_params: { resourceId },
|
|
1319
|
+
clickhouse_settings: {
|
|
1320
|
+
date_time_input_format: "best_effort",
|
|
1321
|
+
date_time_output_format: "iso",
|
|
1322
|
+
use_client_time_zone: 1,
|
|
1323
|
+
output_format_json_quote_64bit_integers: 0
|
|
1324
|
+
}
|
|
1325
|
+
});
|
|
1326
|
+
const rows = await result.json();
|
|
1327
|
+
if (rows.data.length === 0) {
|
|
1328
|
+
return null;
|
|
1329
|
+
}
|
|
1330
|
+
const resource = rows.data[0];
|
|
1331
|
+
return {
|
|
1332
|
+
id: resource.id,
|
|
1333
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1334
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1335
|
+
createdAt: new Date(resource.createdAt),
|
|
1336
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1337
|
+
};
|
|
1338
|
+
} catch (error) {
|
|
1339
|
+
throw new MastraError(
|
|
1340
|
+
{
|
|
1341
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1342
|
+
domain: ErrorDomain.STORAGE,
|
|
1343
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1344
|
+
details: { resourceId }
|
|
1345
|
+
},
|
|
1346
|
+
error
|
|
1347
|
+
);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
async saveResource({ resource }) {
|
|
1351
|
+
try {
|
|
1352
|
+
await this.client.insert({
|
|
1353
|
+
table: TABLE_RESOURCES,
|
|
1354
|
+
format: "JSONEachRow",
|
|
1355
|
+
values: [
|
|
1356
|
+
{
|
|
1357
|
+
id: resource.id,
|
|
1358
|
+
workingMemory: resource.workingMemory,
|
|
1359
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1360
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1361
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1362
|
+
}
|
|
1363
|
+
],
|
|
1364
|
+
clickhouse_settings: {
|
|
1365
|
+
date_time_input_format: "best_effort",
|
|
1366
|
+
use_client_time_zone: 1,
|
|
1367
|
+
output_format_json_quote_64bit_integers: 0
|
|
1368
|
+
}
|
|
1369
|
+
});
|
|
1370
|
+
return resource;
|
|
1371
|
+
} catch (error) {
|
|
1372
|
+
throw new MastraError(
|
|
1373
|
+
{
|
|
1374
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1375
|
+
domain: ErrorDomain.STORAGE,
|
|
1376
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1377
|
+
details: { resourceId: resource.id }
|
|
1378
|
+
},
|
|
1379
|
+
error
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
async updateResource({
|
|
1384
|
+
resourceId,
|
|
1385
|
+
workingMemory,
|
|
1386
|
+
metadata
|
|
1387
|
+
}) {
|
|
1388
|
+
try {
|
|
1389
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1390
|
+
if (!existingResource) {
|
|
1391
|
+
const newResource = {
|
|
1392
|
+
id: resourceId,
|
|
1393
|
+
workingMemory,
|
|
1394
|
+
metadata: metadata || {},
|
|
1395
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1396
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1397
|
+
};
|
|
1398
|
+
return this.saveResource({ resource: newResource });
|
|
1399
|
+
}
|
|
1400
|
+
const updatedResource = {
|
|
1401
|
+
...existingResource,
|
|
1402
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1403
|
+
metadata: {
|
|
1404
|
+
...existingResource.metadata,
|
|
1405
|
+
...metadata
|
|
1406
|
+
},
|
|
1407
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1408
|
+
};
|
|
1409
|
+
const updateQuery = `
|
|
1410
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1411
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1412
|
+
WHERE id = {resourceId:String}
|
|
1413
|
+
`;
|
|
1414
|
+
await this.client.command({
|
|
1415
|
+
query: updateQuery,
|
|
1416
|
+
query_params: {
|
|
1417
|
+
workingMemory: updatedResource.workingMemory,
|
|
1418
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1419
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1420
|
+
resourceId
|
|
1421
|
+
},
|
|
1422
|
+
clickhouse_settings: {
|
|
1423
|
+
date_time_input_format: "best_effort",
|
|
1424
|
+
use_client_time_zone: 1,
|
|
1425
|
+
output_format_json_quote_64bit_integers: 0
|
|
1426
|
+
}
|
|
1427
|
+
});
|
|
1428
|
+
await this.client.command({
|
|
1429
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1430
|
+
clickhouse_settings: {
|
|
1431
|
+
date_time_input_format: "best_effort",
|
|
1432
|
+
use_client_time_zone: 1,
|
|
1433
|
+
output_format_json_quote_64bit_integers: 0
|
|
1434
|
+
}
|
|
1435
|
+
});
|
|
1436
|
+
return updatedResource;
|
|
1437
|
+
} catch (error) {
|
|
1438
|
+
throw new MastraError(
|
|
1439
|
+
{
|
|
1440
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1441
|
+
domain: ErrorDomain.STORAGE,
|
|
1442
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1443
|
+
details: { resourceId }
|
|
1444
|
+
},
|
|
1445
|
+
error
|
|
1446
|
+
);
|
|
191
1447
|
}
|
|
192
|
-
const resp = await result.json();
|
|
193
|
-
const rows = resp.data;
|
|
194
|
-
return rows.map((row) => ({
|
|
195
|
-
id: row.id,
|
|
196
|
-
parentSpanId: row.parentSpanId,
|
|
197
|
-
traceId: row.traceId,
|
|
198
|
-
name: row.name,
|
|
199
|
-
scope: row.scope,
|
|
200
|
-
kind: row.kind,
|
|
201
|
-
status: safelyParseJSON(row.status),
|
|
202
|
-
events: safelyParseJSON(row.events),
|
|
203
|
-
links: safelyParseJSON(row.links),
|
|
204
|
-
attributes: safelyParseJSON(row.attributes),
|
|
205
|
-
startTime: row.startTime,
|
|
206
|
-
endTime: row.endTime,
|
|
207
|
-
other: safelyParseJSON(row.other),
|
|
208
|
-
createdAt: row.createdAt
|
|
209
|
-
}));
|
|
210
1448
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
1449
|
+
};
|
|
1450
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1451
|
+
ttl;
|
|
1452
|
+
client;
|
|
1453
|
+
constructor({ client, ttl }) {
|
|
1454
|
+
super();
|
|
1455
|
+
this.ttl = ttl;
|
|
1456
|
+
this.client = client;
|
|
215
1457
|
}
|
|
216
|
-
async
|
|
217
|
-
await this.
|
|
218
|
-
query: `
|
|
1458
|
+
async hasColumn(table, column) {
|
|
1459
|
+
const result = await this.client.query({
|
|
1460
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1461
|
+
format: "JSONEachRow"
|
|
219
1462
|
});
|
|
1463
|
+
const columns = await result.json();
|
|
1464
|
+
return columns.some((c) => c.name === column);
|
|
1465
|
+
}
|
|
1466
|
+
getSqlType(type) {
|
|
1467
|
+
switch (type) {
|
|
1468
|
+
case "text":
|
|
1469
|
+
return "String";
|
|
1470
|
+
case "timestamp":
|
|
1471
|
+
return "DateTime64(3)";
|
|
1472
|
+
case "integer":
|
|
1473
|
+
case "bigint":
|
|
1474
|
+
return "Int64";
|
|
1475
|
+
case "jsonb":
|
|
1476
|
+
return "String";
|
|
1477
|
+
default:
|
|
1478
|
+
return super.getSqlType(type);
|
|
1479
|
+
}
|
|
220
1480
|
}
|
|
221
1481
|
async createTable({
|
|
222
1482
|
tableName,
|
|
@@ -231,25 +1491,25 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
231
1491
|
}).join(",\n");
|
|
232
1492
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
233
1493
|
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
await this.
|
|
1494
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1495
|
+
${["id String"].concat(columns)}
|
|
1496
|
+
)
|
|
1497
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1498
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1499
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1500
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1501
|
+
SETTINGS index_granularity = 8192
|
|
1502
|
+
` : `
|
|
1503
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1504
|
+
${columns}
|
|
1505
|
+
)
|
|
1506
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1507
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1508
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1509
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1510
|
+
SETTINGS index_granularity = 8192
|
|
1511
|
+
`;
|
|
1512
|
+
await this.client.query({
|
|
253
1513
|
query: sql,
|
|
254
1514
|
clickhouse_settings: {
|
|
255
1515
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -260,13 +1520,59 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
260
1520
|
}
|
|
261
1521
|
});
|
|
262
1522
|
} catch (error) {
|
|
263
|
-
|
|
264
|
-
|
|
1523
|
+
throw new MastraError(
|
|
1524
|
+
{
|
|
1525
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1526
|
+
domain: ErrorDomain.STORAGE,
|
|
1527
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1528
|
+
details: { tableName }
|
|
1529
|
+
},
|
|
1530
|
+
error
|
|
1531
|
+
);
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
async alterTable({
|
|
1535
|
+
tableName,
|
|
1536
|
+
schema,
|
|
1537
|
+
ifNotExists
|
|
1538
|
+
}) {
|
|
1539
|
+
try {
|
|
1540
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1541
|
+
const result = await this.client.query({
|
|
1542
|
+
query: describeSql
|
|
1543
|
+
});
|
|
1544
|
+
const rows = await result.json();
|
|
1545
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1546
|
+
for (const columnName of ifNotExists) {
|
|
1547
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1548
|
+
const columnDef = schema[columnName];
|
|
1549
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1550
|
+
if (columnDef.nullable !== false) {
|
|
1551
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1552
|
+
}
|
|
1553
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1554
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1555
|
+
await this.client.query({
|
|
1556
|
+
query: alterSql
|
|
1557
|
+
});
|
|
1558
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
} catch (error) {
|
|
1562
|
+
throw new MastraError(
|
|
1563
|
+
{
|
|
1564
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1565
|
+
domain: ErrorDomain.STORAGE,
|
|
1566
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1567
|
+
details: { tableName }
|
|
1568
|
+
},
|
|
1569
|
+
error
|
|
1570
|
+
);
|
|
265
1571
|
}
|
|
266
1572
|
}
|
|
267
1573
|
async clearTable({ tableName }) {
|
|
268
1574
|
try {
|
|
269
|
-
await this.
|
|
1575
|
+
await this.client.query({
|
|
270
1576
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
271
1577
|
clickhouse_settings: {
|
|
272
1578
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -277,19 +1583,33 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
277
1583
|
}
|
|
278
1584
|
});
|
|
279
1585
|
} catch (error) {
|
|
280
|
-
|
|
281
|
-
|
|
1586
|
+
throw new MastraError(
|
|
1587
|
+
{
|
|
1588
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1589
|
+
domain: ErrorDomain.STORAGE,
|
|
1590
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1591
|
+
details: { tableName }
|
|
1592
|
+
},
|
|
1593
|
+
error
|
|
1594
|
+
);
|
|
282
1595
|
}
|
|
283
1596
|
}
|
|
1597
|
+
async dropTable({ tableName }) {
|
|
1598
|
+
await this.client.query({
|
|
1599
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1600
|
+
});
|
|
1601
|
+
}
|
|
284
1602
|
async insert({ tableName, record }) {
|
|
1603
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1604
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
285
1605
|
try {
|
|
286
|
-
await this.
|
|
1606
|
+
const result = await this.client.insert({
|
|
287
1607
|
table: tableName,
|
|
288
1608
|
values: [
|
|
289
1609
|
{
|
|
290
1610
|
...record,
|
|
291
|
-
createdAt
|
|
292
|
-
updatedAt
|
|
1611
|
+
createdAt,
|
|
1612
|
+
updatedAt
|
|
293
1613
|
}
|
|
294
1614
|
],
|
|
295
1615
|
format: "JSONEachRow",
|
|
@@ -300,13 +1620,55 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
300
1620
|
use_client_time_zone: 1
|
|
301
1621
|
}
|
|
302
1622
|
});
|
|
1623
|
+
console.info("INSERT RESULT", result);
|
|
1624
|
+
} catch (error) {
|
|
1625
|
+
throw new MastraError(
|
|
1626
|
+
{
|
|
1627
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1628
|
+
domain: ErrorDomain.STORAGE,
|
|
1629
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1630
|
+
details: { tableName }
|
|
1631
|
+
},
|
|
1632
|
+
error
|
|
1633
|
+
);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
async batchInsert({ tableName, records }) {
|
|
1637
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1638
|
+
...Object.fromEntries(
|
|
1639
|
+
Object.entries(record).map(([key, value]) => [
|
|
1640
|
+
key,
|
|
1641
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1642
|
+
])
|
|
1643
|
+
)
|
|
1644
|
+
}));
|
|
1645
|
+
try {
|
|
1646
|
+
await this.client.insert({
|
|
1647
|
+
table: tableName,
|
|
1648
|
+
values: recordsToBeInserted,
|
|
1649
|
+
format: "JSONEachRow",
|
|
1650
|
+
clickhouse_settings: {
|
|
1651
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1652
|
+
date_time_input_format: "best_effort",
|
|
1653
|
+
use_client_time_zone: 1,
|
|
1654
|
+
output_format_json_quote_64bit_integers: 0
|
|
1655
|
+
}
|
|
1656
|
+
});
|
|
303
1657
|
} catch (error) {
|
|
304
|
-
|
|
305
|
-
|
|
1658
|
+
throw new MastraError(
|
|
1659
|
+
{
|
|
1660
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1661
|
+
domain: ErrorDomain.STORAGE,
|
|
1662
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1663
|
+
details: { tableName }
|
|
1664
|
+
},
|
|
1665
|
+
error
|
|
1666
|
+
);
|
|
306
1667
|
}
|
|
307
1668
|
}
|
|
308
1669
|
async load({ tableName, keys }) {
|
|
309
1670
|
try {
|
|
1671
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
310
1672
|
const keyEntries = Object.entries(keys);
|
|
311
1673
|
const conditions = keyEntries.map(
|
|
312
1674
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -314,8 +1676,10 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
314
1676
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
315
1677
|
return { ...acc, [`var_${key}`]: value };
|
|
316
1678
|
}, {});
|
|
317
|
-
const
|
|
318
|
-
|
|
1679
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1680
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1681
|
+
const result = await this.client.query({
|
|
1682
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
319
1683
|
query_params: values,
|
|
320
1684
|
clickhouse_settings: {
|
|
321
1685
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -342,24 +1706,57 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
342
1706
|
const data = transformRow(rows.data[0]);
|
|
343
1707
|
return data;
|
|
344
1708
|
} catch (error) {
|
|
345
|
-
|
|
346
|
-
|
|
1709
|
+
throw new MastraError(
|
|
1710
|
+
{
|
|
1711
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1712
|
+
domain: ErrorDomain.STORAGE,
|
|
1713
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1714
|
+
details: { tableName }
|
|
1715
|
+
},
|
|
1716
|
+
error
|
|
1717
|
+
);
|
|
347
1718
|
}
|
|
348
1719
|
}
|
|
349
|
-
|
|
1720
|
+
};
|
|
1721
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1722
|
+
client;
|
|
1723
|
+
operations;
|
|
1724
|
+
constructor({ client, operations }) {
|
|
1725
|
+
super();
|
|
1726
|
+
this.client = client;
|
|
1727
|
+
this.operations = operations;
|
|
1728
|
+
}
|
|
1729
|
+
transformScoreRow(row) {
|
|
1730
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1731
|
+
const preprocessStepResult = safelyParseJSON(row.preprocessStepResult);
|
|
1732
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1733
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1734
|
+
const input = safelyParseJSON(row.input);
|
|
1735
|
+
const output = safelyParseJSON(row.output);
|
|
1736
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1737
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1738
|
+
const entity = safelyParseJSON(row.entity);
|
|
1739
|
+
return {
|
|
1740
|
+
...row,
|
|
1741
|
+
scorer,
|
|
1742
|
+
preprocessStepResult,
|
|
1743
|
+
analyzeStepResult,
|
|
1744
|
+
metadata,
|
|
1745
|
+
input,
|
|
1746
|
+
output,
|
|
1747
|
+
additionalContext,
|
|
1748
|
+
runtimeContext,
|
|
1749
|
+
entity,
|
|
1750
|
+
createdAt: new Date(row.createdAt),
|
|
1751
|
+
updatedAt: new Date(row.updatedAt)
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
async getScoreById({ id }) {
|
|
350
1755
|
try {
|
|
351
|
-
const result = await this.
|
|
352
|
-
query: `SELECT
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
title,
|
|
356
|
-
metadata,
|
|
357
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
358
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
359
|
-
FROM "${TABLE_THREADS}"
|
|
360
|
-
FINAL
|
|
361
|
-
WHERE id = {var_id:String}`,
|
|
362
|
-
query_params: { var_id: threadId },
|
|
1756
|
+
const result = await this.client.query({
|
|
1757
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1758
|
+
query_params: { var_id: id },
|
|
1759
|
+
format: "JSONEachRow",
|
|
363
1760
|
clickhouse_settings: {
|
|
364
1761
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
365
1762
|
date_time_input_format: "best_effort",
|
|
@@ -368,304 +1765,566 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
368
1765
|
output_format_json_quote_64bit_integers: 0
|
|
369
1766
|
}
|
|
370
1767
|
});
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
if (!thread) {
|
|
1768
|
+
const resultJson = await result.json();
|
|
1769
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
374
1770
|
return null;
|
|
375
1771
|
}
|
|
376
|
-
return
|
|
377
|
-
...thread,
|
|
378
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
379
|
-
createdAt: thread.createdAt,
|
|
380
|
-
updatedAt: thread.updatedAt
|
|
381
|
-
};
|
|
1772
|
+
return this.transformScoreRow(resultJson[0]);
|
|
382
1773
|
} catch (error) {
|
|
383
|
-
|
|
384
|
-
|
|
1774
|
+
throw new MastraError(
|
|
1775
|
+
{
|
|
1776
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1777
|
+
domain: ErrorDomain.STORAGE,
|
|
1778
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1779
|
+
details: { scoreId: id }
|
|
1780
|
+
},
|
|
1781
|
+
error
|
|
1782
|
+
);
|
|
385
1783
|
}
|
|
386
1784
|
}
|
|
387
|
-
async
|
|
1785
|
+
async saveScore(score) {
|
|
388
1786
|
try {
|
|
389
|
-
const
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
397
|
-
FROM "${TABLE_THREADS}"
|
|
398
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
399
|
-
query_params: { var_resourceId: resourceId },
|
|
1787
|
+
const record = {
|
|
1788
|
+
...score
|
|
1789
|
+
};
|
|
1790
|
+
await this.client.insert({
|
|
1791
|
+
table: TABLE_SCORERS,
|
|
1792
|
+
values: [record],
|
|
1793
|
+
format: "JSONEachRow",
|
|
400
1794
|
clickhouse_settings: {
|
|
401
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
402
1795
|
date_time_input_format: "best_effort",
|
|
403
|
-
date_time_output_format: "iso",
|
|
404
1796
|
use_client_time_zone: 1,
|
|
405
1797
|
output_format_json_quote_64bit_integers: 0
|
|
406
1798
|
}
|
|
407
1799
|
});
|
|
408
|
-
|
|
409
|
-
const threads = transformRows(rows.data);
|
|
410
|
-
return threads.map((thread) => ({
|
|
411
|
-
...thread,
|
|
412
|
-
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
413
|
-
createdAt: thread.createdAt,
|
|
414
|
-
updatedAt: thread.updatedAt
|
|
415
|
-
}));
|
|
1800
|
+
return { score };
|
|
416
1801
|
} catch (error) {
|
|
417
|
-
|
|
418
|
-
|
|
1802
|
+
throw new MastraError(
|
|
1803
|
+
{
|
|
1804
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1805
|
+
domain: ErrorDomain.STORAGE,
|
|
1806
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1807
|
+
details: { scoreId: score.id }
|
|
1808
|
+
},
|
|
1809
|
+
error
|
|
1810
|
+
);
|
|
419
1811
|
}
|
|
420
1812
|
}
|
|
421
|
-
async
|
|
1813
|
+
async getScoresByRunId({
|
|
1814
|
+
runId,
|
|
1815
|
+
pagination
|
|
1816
|
+
}) {
|
|
422
1817
|
try {
|
|
423
|
-
await this.
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
]
|
|
1818
|
+
const countResult = await this.client.query({
|
|
1819
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1820
|
+
query_params: { var_runId: runId },
|
|
1821
|
+
format: "JSONEachRow"
|
|
1822
|
+
});
|
|
1823
|
+
const countRows = await countResult.json();
|
|
1824
|
+
let total = 0;
|
|
1825
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1826
|
+
const countObj = countRows[0];
|
|
1827
|
+
total = Number(countObj.count);
|
|
1828
|
+
}
|
|
1829
|
+
if (!total) {
|
|
1830
|
+
return {
|
|
1831
|
+
pagination: {
|
|
1832
|
+
total: 0,
|
|
1833
|
+
page: pagination.page,
|
|
1834
|
+
perPage: pagination.perPage,
|
|
1835
|
+
hasMore: false
|
|
1836
|
+
},
|
|
1837
|
+
scores: []
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
const offset = pagination.page * pagination.perPage;
|
|
1841
|
+
const result = await this.client.query({
|
|
1842
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1843
|
+
query_params: {
|
|
1844
|
+
var_runId: runId,
|
|
1845
|
+
var_limit: pagination.perPage,
|
|
1846
|
+
var_offset: offset
|
|
1847
|
+
},
|
|
432
1848
|
format: "JSONEachRow",
|
|
433
1849
|
clickhouse_settings: {
|
|
434
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
435
1850
|
date_time_input_format: "best_effort",
|
|
1851
|
+
date_time_output_format: "iso",
|
|
436
1852
|
use_client_time_zone: 1,
|
|
437
1853
|
output_format_json_quote_64bit_integers: 0
|
|
438
1854
|
}
|
|
439
1855
|
});
|
|
440
|
-
|
|
1856
|
+
const rows = await result.json();
|
|
1857
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1858
|
+
return {
|
|
1859
|
+
pagination: {
|
|
1860
|
+
total,
|
|
1861
|
+
page: pagination.page,
|
|
1862
|
+
perPage: pagination.perPage,
|
|
1863
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1864
|
+
},
|
|
1865
|
+
scores
|
|
1866
|
+
};
|
|
441
1867
|
} catch (error) {
|
|
442
|
-
|
|
443
|
-
|
|
1868
|
+
throw new MastraError(
|
|
1869
|
+
{
|
|
1870
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1871
|
+
domain: ErrorDomain.STORAGE,
|
|
1872
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1873
|
+
details: { runId }
|
|
1874
|
+
},
|
|
1875
|
+
error
|
|
1876
|
+
);
|
|
444
1877
|
}
|
|
445
1878
|
}
|
|
446
|
-
async
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
1879
|
+
async getScoresByScorerId({
|
|
1880
|
+
scorerId,
|
|
1881
|
+
entityId,
|
|
1882
|
+
entityType,
|
|
1883
|
+
source,
|
|
1884
|
+
pagination
|
|
450
1885
|
}) {
|
|
1886
|
+
let whereClause = `scorerId = {var_scorerId:String}`;
|
|
1887
|
+
if (entityId) {
|
|
1888
|
+
whereClause += ` AND entityId = {var_entityId:String}`;
|
|
1889
|
+
}
|
|
1890
|
+
if (entityType) {
|
|
1891
|
+
whereClause += ` AND entityType = {var_entityType:String}`;
|
|
1892
|
+
}
|
|
1893
|
+
if (source) {
|
|
1894
|
+
whereClause += ` AND source = {var_source:String}`;
|
|
1895
|
+
}
|
|
451
1896
|
try {
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
1897
|
+
const countResult = await this.client.query({
|
|
1898
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE ${whereClause}`,
|
|
1899
|
+
query_params: {
|
|
1900
|
+
var_scorerId: scorerId,
|
|
1901
|
+
var_entityId: entityId,
|
|
1902
|
+
var_entityType: entityType,
|
|
1903
|
+
var_source: source
|
|
1904
|
+
},
|
|
1905
|
+
format: "JSONEachRow"
|
|
1906
|
+
});
|
|
1907
|
+
const countRows = await countResult.json();
|
|
1908
|
+
let total = 0;
|
|
1909
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1910
|
+
const countObj = countRows[0];
|
|
1911
|
+
total = Number(countObj.count);
|
|
455
1912
|
}
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
1913
|
+
if (!total) {
|
|
1914
|
+
return {
|
|
1915
|
+
pagination: {
|
|
1916
|
+
total: 0,
|
|
1917
|
+
page: pagination.page,
|
|
1918
|
+
perPage: pagination.perPage,
|
|
1919
|
+
hasMore: false
|
|
1920
|
+
},
|
|
1921
|
+
scores: []
|
|
1922
|
+
};
|
|
1923
|
+
}
|
|
1924
|
+
const offset = pagination.page * pagination.perPage;
|
|
1925
|
+
const result = await this.client.query({
|
|
1926
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1927
|
+
query_params: {
|
|
1928
|
+
var_scorerId: scorerId,
|
|
1929
|
+
var_limit: pagination.perPage,
|
|
1930
|
+
var_offset: offset,
|
|
1931
|
+
var_entityId: entityId,
|
|
1932
|
+
var_entityType: entityType,
|
|
1933
|
+
var_source: source
|
|
1934
|
+
},
|
|
474
1935
|
format: "JSONEachRow",
|
|
475
1936
|
clickhouse_settings: {
|
|
476
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
477
1937
|
date_time_input_format: "best_effort",
|
|
1938
|
+
date_time_output_format: "iso",
|
|
478
1939
|
use_client_time_zone: 1,
|
|
479
1940
|
output_format_json_quote_64bit_integers: 0
|
|
480
1941
|
}
|
|
481
1942
|
});
|
|
482
|
-
|
|
1943
|
+
const rows = await result.json();
|
|
1944
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1945
|
+
return {
|
|
1946
|
+
pagination: {
|
|
1947
|
+
total,
|
|
1948
|
+
page: pagination.page,
|
|
1949
|
+
perPage: pagination.perPage,
|
|
1950
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1951
|
+
},
|
|
1952
|
+
scores
|
|
1953
|
+
};
|
|
483
1954
|
} catch (error) {
|
|
484
|
-
|
|
485
|
-
|
|
1955
|
+
throw new MastraError(
|
|
1956
|
+
{
|
|
1957
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1958
|
+
domain: ErrorDomain.STORAGE,
|
|
1959
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1960
|
+
details: { scorerId }
|
|
1961
|
+
},
|
|
1962
|
+
error
|
|
1963
|
+
);
|
|
486
1964
|
}
|
|
487
1965
|
}
|
|
488
|
-
async
|
|
1966
|
+
async getScoresByEntityId({
|
|
1967
|
+
entityId,
|
|
1968
|
+
entityType,
|
|
1969
|
+
pagination
|
|
1970
|
+
}) {
|
|
489
1971
|
try {
|
|
490
|
-
await this.
|
|
491
|
-
query: `
|
|
492
|
-
query_params: {
|
|
493
|
-
|
|
494
|
-
output_format_json_quote_64bit_integers: 0
|
|
495
|
-
}
|
|
1972
|
+
const countResult = await this.client.query({
|
|
1973
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1974
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1975
|
+
format: "JSONEachRow"
|
|
496
1976
|
});
|
|
497
|
-
await
|
|
498
|
-
|
|
499
|
-
|
|
1977
|
+
const countRows = await countResult.json();
|
|
1978
|
+
let total = 0;
|
|
1979
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1980
|
+
const countObj = countRows[0];
|
|
1981
|
+
total = Number(countObj.count);
|
|
1982
|
+
}
|
|
1983
|
+
if (!total) {
|
|
1984
|
+
return {
|
|
1985
|
+
pagination: {
|
|
1986
|
+
total: 0,
|
|
1987
|
+
page: pagination.page,
|
|
1988
|
+
perPage: pagination.perPage,
|
|
1989
|
+
hasMore: false
|
|
1990
|
+
},
|
|
1991
|
+
scores: []
|
|
1992
|
+
};
|
|
1993
|
+
}
|
|
1994
|
+
const offset = pagination.page * pagination.perPage;
|
|
1995
|
+
const result = await this.client.query({
|
|
1996
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1997
|
+
query_params: {
|
|
1998
|
+
var_entityId: entityId,
|
|
1999
|
+
var_entityType: entityType,
|
|
2000
|
+
var_limit: pagination.perPage,
|
|
2001
|
+
var_offset: offset
|
|
2002
|
+
},
|
|
2003
|
+
format: "JSONEachRow",
|
|
500
2004
|
clickhouse_settings: {
|
|
2005
|
+
date_time_input_format: "best_effort",
|
|
2006
|
+
date_time_output_format: "iso",
|
|
2007
|
+
use_client_time_zone: 1,
|
|
501
2008
|
output_format_json_quote_64bit_integers: 0
|
|
502
2009
|
}
|
|
503
2010
|
});
|
|
2011
|
+
const rows = await result.json();
|
|
2012
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2013
|
+
return {
|
|
2014
|
+
pagination: {
|
|
2015
|
+
total,
|
|
2016
|
+
page: pagination.page,
|
|
2017
|
+
perPage: pagination.perPage,
|
|
2018
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
2019
|
+
},
|
|
2020
|
+
scores
|
|
2021
|
+
};
|
|
504
2022
|
} catch (error) {
|
|
505
|
-
|
|
506
|
-
|
|
2023
|
+
throw new MastraError(
|
|
2024
|
+
{
|
|
2025
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
2026
|
+
domain: ErrorDomain.STORAGE,
|
|
2027
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2028
|
+
details: { entityId, entityType }
|
|
2029
|
+
},
|
|
2030
|
+
error
|
|
2031
|
+
);
|
|
507
2032
|
}
|
|
508
2033
|
}
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
date_time_output_format: "iso",
|
|
559
|
-
use_client_time_zone: 1,
|
|
560
|
-
output_format_json_quote_64bit_integers: 0
|
|
561
|
-
}
|
|
562
|
-
});
|
|
563
|
-
const rows2 = await includeResult.json();
|
|
564
|
-
messages.push(...transformRows(rows2.data));
|
|
565
|
-
}
|
|
566
|
-
const result = await this.db.query({
|
|
567
|
-
query: `
|
|
568
|
-
SELECT
|
|
569
|
-
id,
|
|
570
|
-
content,
|
|
571
|
-
role,
|
|
572
|
-
type,
|
|
573
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
574
|
-
thread_id AS "threadId"
|
|
575
|
-
FROM "${TABLE_MESSAGES}"
|
|
576
|
-
WHERE thread_id = {threadId:String}
|
|
577
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
578
|
-
ORDER BY "createdAt" DESC
|
|
579
|
-
LIMIT {limit:Int64}
|
|
580
|
-
`,
|
|
581
|
-
query_params: {
|
|
582
|
-
threadId,
|
|
583
|
-
exclude: messages.map((m) => m.id),
|
|
584
|
-
limit
|
|
585
|
-
},
|
|
2034
|
+
};
|
|
2035
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
2036
|
+
client;
|
|
2037
|
+
operations;
|
|
2038
|
+
constructor({ client, operations }) {
|
|
2039
|
+
super();
|
|
2040
|
+
this.client = client;
|
|
2041
|
+
this.operations = operations;
|
|
2042
|
+
}
|
|
2043
|
+
async getTracesPaginated(args) {
|
|
2044
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
2045
|
+
const fromDate = dateRange?.start;
|
|
2046
|
+
const toDate = dateRange?.end;
|
|
2047
|
+
const currentOffset = page * perPage;
|
|
2048
|
+
const queryArgs = {};
|
|
2049
|
+
const conditions = [];
|
|
2050
|
+
if (name) {
|
|
2051
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2052
|
+
queryArgs.var_name = name;
|
|
2053
|
+
}
|
|
2054
|
+
if (scope) {
|
|
2055
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2056
|
+
queryArgs.var_scope = scope;
|
|
2057
|
+
}
|
|
2058
|
+
if (attributes) {
|
|
2059
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2060
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2061
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
2062
|
+
});
|
|
2063
|
+
}
|
|
2064
|
+
if (filters) {
|
|
2065
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2066
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2067
|
+
queryArgs[`var_col_${key}`] = value;
|
|
2068
|
+
});
|
|
2069
|
+
}
|
|
2070
|
+
if (fromDate) {
|
|
2071
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
2072
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
2073
|
+
}
|
|
2074
|
+
if (toDate) {
|
|
2075
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
2076
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
2077
|
+
}
|
|
2078
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2079
|
+
try {
|
|
2080
|
+
const countResult = await this.client.query({
|
|
2081
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
2082
|
+
query_params: queryArgs,
|
|
586
2083
|
clickhouse_settings: {
|
|
587
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
588
2084
|
date_time_input_format: "best_effort",
|
|
589
2085
|
date_time_output_format: "iso",
|
|
590
2086
|
use_client_time_zone: 1,
|
|
591
2087
|
output_format_json_quote_64bit_integers: 0
|
|
592
2088
|
}
|
|
593
2089
|
});
|
|
594
|
-
const
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
2090
|
+
const countData = await countResult.json();
|
|
2091
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2092
|
+
if (total === 0) {
|
|
2093
|
+
return {
|
|
2094
|
+
traces: [],
|
|
2095
|
+
total: 0,
|
|
2096
|
+
page,
|
|
2097
|
+
perPage,
|
|
2098
|
+
hasMore: false
|
|
2099
|
+
};
|
|
2100
|
+
}
|
|
2101
|
+
const result = await this.client.query({
|
|
2102
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2103
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2104
|
+
clickhouse_settings: {
|
|
2105
|
+
date_time_input_format: "best_effort",
|
|
2106
|
+
date_time_output_format: "iso",
|
|
2107
|
+
use_client_time_zone: 1,
|
|
2108
|
+
output_format_json_quote_64bit_integers: 0
|
|
603
2109
|
}
|
|
604
2110
|
});
|
|
605
|
-
|
|
2111
|
+
if (!result) {
|
|
2112
|
+
return {
|
|
2113
|
+
traces: [],
|
|
2114
|
+
total,
|
|
2115
|
+
page,
|
|
2116
|
+
perPage,
|
|
2117
|
+
hasMore: false
|
|
2118
|
+
};
|
|
2119
|
+
}
|
|
2120
|
+
const resp = await result.json();
|
|
2121
|
+
const rows = resp.data;
|
|
2122
|
+
const traces = rows.map((row) => ({
|
|
2123
|
+
id: row.id,
|
|
2124
|
+
parentSpanId: row.parentSpanId,
|
|
2125
|
+
traceId: row.traceId,
|
|
2126
|
+
name: row.name,
|
|
2127
|
+
scope: row.scope,
|
|
2128
|
+
kind: row.kind,
|
|
2129
|
+
status: safelyParseJSON(row.status),
|
|
2130
|
+
events: safelyParseJSON(row.events),
|
|
2131
|
+
links: safelyParseJSON(row.links),
|
|
2132
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2133
|
+
startTime: row.startTime,
|
|
2134
|
+
endTime: row.endTime,
|
|
2135
|
+
other: safelyParseJSON(row.other),
|
|
2136
|
+
createdAt: row.createdAt
|
|
2137
|
+
}));
|
|
2138
|
+
return {
|
|
2139
|
+
traces,
|
|
2140
|
+
total,
|
|
2141
|
+
page,
|
|
2142
|
+
perPage,
|
|
2143
|
+
hasMore: currentOffset + traces.length < total
|
|
2144
|
+
};
|
|
606
2145
|
} catch (error) {
|
|
607
|
-
|
|
608
|
-
|
|
2146
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2147
|
+
return {
|
|
2148
|
+
traces: [],
|
|
2149
|
+
total: 0,
|
|
2150
|
+
page,
|
|
2151
|
+
perPage,
|
|
2152
|
+
hasMore: false
|
|
2153
|
+
};
|
|
2154
|
+
}
|
|
2155
|
+
throw new MastraError(
|
|
2156
|
+
{
|
|
2157
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2158
|
+
domain: ErrorDomain.STORAGE,
|
|
2159
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2160
|
+
details: {
|
|
2161
|
+
name: name ?? null,
|
|
2162
|
+
scope: scope ?? null,
|
|
2163
|
+
page,
|
|
2164
|
+
perPage,
|
|
2165
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2166
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2167
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2168
|
+
}
|
|
2169
|
+
},
|
|
2170
|
+
error
|
|
2171
|
+
);
|
|
609
2172
|
}
|
|
610
2173
|
}
|
|
611
|
-
async
|
|
612
|
-
|
|
2174
|
+
async getTraces({
|
|
2175
|
+
name,
|
|
2176
|
+
scope,
|
|
2177
|
+
page,
|
|
2178
|
+
perPage,
|
|
2179
|
+
attributes,
|
|
2180
|
+
filters,
|
|
2181
|
+
fromDate,
|
|
2182
|
+
toDate
|
|
2183
|
+
}) {
|
|
2184
|
+
const limit = perPage;
|
|
2185
|
+
const offset = page * perPage;
|
|
2186
|
+
const args = {};
|
|
2187
|
+
const conditions = [];
|
|
2188
|
+
if (name) {
|
|
2189
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2190
|
+
args.var_name = name;
|
|
2191
|
+
}
|
|
2192
|
+
if (scope) {
|
|
2193
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2194
|
+
args.var_scope = scope;
|
|
2195
|
+
}
|
|
2196
|
+
if (attributes) {
|
|
2197
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2198
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2199
|
+
args[`var_attr_${key}`] = value;
|
|
2200
|
+
});
|
|
2201
|
+
}
|
|
2202
|
+
if (filters) {
|
|
2203
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2204
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2205
|
+
args[`var_col_${key}`] = value;
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
if (fromDate) {
|
|
2209
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2210
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2211
|
+
}
|
|
2212
|
+
if (toDate) {
|
|
2213
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2214
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2215
|
+
}
|
|
2216
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
613
2217
|
try {
|
|
614
|
-
const
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
}
|
|
618
|
-
const thread = await this.getThreadById({ threadId });
|
|
619
|
-
if (!thread) {
|
|
620
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
621
|
-
}
|
|
622
|
-
await this.db.insert({
|
|
623
|
-
table: TABLE_MESSAGES,
|
|
624
|
-
format: "JSONEachRow",
|
|
625
|
-
values: messages.map((message) => ({
|
|
626
|
-
id: message.id,
|
|
627
|
-
thread_id: threadId,
|
|
628
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
629
|
-
createdAt: message.createdAt.toISOString(),
|
|
630
|
-
role: message.role,
|
|
631
|
-
type: message.type
|
|
632
|
-
})),
|
|
2218
|
+
const result = await this.client.query({
|
|
2219
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2220
|
+
query_params: args,
|
|
633
2221
|
clickhouse_settings: {
|
|
634
2222
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
635
2223
|
date_time_input_format: "best_effort",
|
|
2224
|
+
date_time_output_format: "iso",
|
|
636
2225
|
use_client_time_zone: 1,
|
|
637
2226
|
output_format_json_quote_64bit_integers: 0
|
|
638
2227
|
}
|
|
639
2228
|
});
|
|
640
|
-
|
|
2229
|
+
if (!result) {
|
|
2230
|
+
return [];
|
|
2231
|
+
}
|
|
2232
|
+
const resp = await result.json();
|
|
2233
|
+
const rows = resp.data;
|
|
2234
|
+
return rows.map((row) => ({
|
|
2235
|
+
id: row.id,
|
|
2236
|
+
parentSpanId: row.parentSpanId,
|
|
2237
|
+
traceId: row.traceId,
|
|
2238
|
+
name: row.name,
|
|
2239
|
+
scope: row.scope,
|
|
2240
|
+
kind: row.kind,
|
|
2241
|
+
status: safelyParseJSON(row.status),
|
|
2242
|
+
events: safelyParseJSON(row.events),
|
|
2243
|
+
links: safelyParseJSON(row.links),
|
|
2244
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2245
|
+
startTime: row.startTime,
|
|
2246
|
+
endTime: row.endTime,
|
|
2247
|
+
other: safelyParseJSON(row.other),
|
|
2248
|
+
createdAt: row.createdAt
|
|
2249
|
+
}));
|
|
641
2250
|
} catch (error) {
|
|
642
|
-
|
|
643
|
-
|
|
2251
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2252
|
+
return [];
|
|
2253
|
+
}
|
|
2254
|
+
throw new MastraError(
|
|
2255
|
+
{
|
|
2256
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2257
|
+
domain: ErrorDomain.STORAGE,
|
|
2258
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2259
|
+
details: {
|
|
2260
|
+
name: name ?? null,
|
|
2261
|
+
scope: scope ?? null,
|
|
2262
|
+
page,
|
|
2263
|
+
perPage,
|
|
2264
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2265
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2266
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2267
|
+
toDate: toDate?.toISOString() ?? null
|
|
2268
|
+
}
|
|
2269
|
+
},
|
|
2270
|
+
error
|
|
2271
|
+
);
|
|
644
2272
|
}
|
|
645
2273
|
}
|
|
2274
|
+
async batchTraceInsert(args) {
|
|
2275
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2276
|
+
}
|
|
2277
|
+
};
|
|
2278
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2279
|
+
client;
|
|
2280
|
+
operations;
|
|
2281
|
+
constructor({ client, operations }) {
|
|
2282
|
+
super();
|
|
2283
|
+
this.operations = operations;
|
|
2284
|
+
this.client = client;
|
|
2285
|
+
}
|
|
2286
|
+
updateWorkflowResults({
|
|
2287
|
+
// workflowName,
|
|
2288
|
+
// runId,
|
|
2289
|
+
// stepId,
|
|
2290
|
+
// result,
|
|
2291
|
+
// runtimeContext,
|
|
2292
|
+
}) {
|
|
2293
|
+
throw new Error("Method not implemented.");
|
|
2294
|
+
}
|
|
2295
|
+
updateWorkflowState({
|
|
2296
|
+
// workflowName,
|
|
2297
|
+
// runId,
|
|
2298
|
+
// opts,
|
|
2299
|
+
}) {
|
|
2300
|
+
throw new Error("Method not implemented.");
|
|
2301
|
+
}
|
|
646
2302
|
async persistWorkflowSnapshot({
|
|
647
2303
|
workflowName,
|
|
648
2304
|
runId,
|
|
2305
|
+
resourceId,
|
|
649
2306
|
snapshot
|
|
650
2307
|
}) {
|
|
651
2308
|
try {
|
|
652
|
-
const currentSnapshot = await this.load({
|
|
2309
|
+
const currentSnapshot = await this.operations.load({
|
|
653
2310
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
654
2311
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
655
2312
|
});
|
|
656
2313
|
const now = /* @__PURE__ */ new Date();
|
|
657
2314
|
const persisting = currentSnapshot ? {
|
|
658
2315
|
...currentSnapshot,
|
|
2316
|
+
resourceId,
|
|
659
2317
|
snapshot: JSON.stringify(snapshot),
|
|
660
2318
|
updatedAt: now.toISOString()
|
|
661
2319
|
} : {
|
|
662
2320
|
workflow_name: workflowName,
|
|
663
2321
|
run_id: runId,
|
|
2322
|
+
resourceId,
|
|
664
2323
|
snapshot: JSON.stringify(snapshot),
|
|
665
2324
|
createdAt: now.toISOString(),
|
|
666
2325
|
updatedAt: now.toISOString()
|
|
667
2326
|
};
|
|
668
|
-
await this.
|
|
2327
|
+
await this.client.insert({
|
|
669
2328
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
670
2329
|
format: "JSONEachRow",
|
|
671
2330
|
values: [persisting],
|
|
@@ -677,8 +2336,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
677
2336
|
}
|
|
678
2337
|
});
|
|
679
2338
|
} catch (error) {
|
|
680
|
-
|
|
681
|
-
|
|
2339
|
+
throw new MastraError(
|
|
2340
|
+
{
|
|
2341
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2342
|
+
domain: ErrorDomain.STORAGE,
|
|
2343
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2344
|
+
details: { workflowName, runId }
|
|
2345
|
+
},
|
|
2346
|
+
error
|
|
2347
|
+
);
|
|
682
2348
|
}
|
|
683
2349
|
}
|
|
684
2350
|
async loadWorkflowSnapshot({
|
|
@@ -686,7 +2352,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
686
2352
|
runId
|
|
687
2353
|
}) {
|
|
688
2354
|
try {
|
|
689
|
-
const result = await this.load({
|
|
2355
|
+
const result = await this.operations.load({
|
|
690
2356
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
691
2357
|
keys: {
|
|
692
2358
|
workflow_name: workflowName,
|
|
@@ -698,8 +2364,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
698
2364
|
}
|
|
699
2365
|
return result.snapshot;
|
|
700
2366
|
} catch (error) {
|
|
701
|
-
|
|
702
|
-
|
|
2367
|
+
throw new MastraError(
|
|
2368
|
+
{
|
|
2369
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2370
|
+
domain: ErrorDomain.STORAGE,
|
|
2371
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2372
|
+
details: { workflowName, runId }
|
|
2373
|
+
},
|
|
2374
|
+
error
|
|
2375
|
+
);
|
|
703
2376
|
}
|
|
704
2377
|
}
|
|
705
2378
|
parseWorkflowRun(row) {
|
|
@@ -736,7 +2409,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
736
2409
|
values.var_workflow_name = workflowName;
|
|
737
2410
|
}
|
|
738
2411
|
if (resourceId) {
|
|
739
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2412
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
740
2413
|
if (hasResourceId) {
|
|
741
2414
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
742
2415
|
values.var_resourceId = resourceId;
|
|
@@ -757,7 +2430,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
757
2430
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
758
2431
|
let total = 0;
|
|
759
2432
|
if (limit !== void 0 && offset !== void 0) {
|
|
760
|
-
const countResult = await this.
|
|
2433
|
+
const countResult = await this.client.query({
|
|
761
2434
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
762
2435
|
query_params: values,
|
|
763
2436
|
format: "JSONEachRow"
|
|
@@ -765,21 +2438,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
765
2438
|
const countRows = await countResult.json();
|
|
766
2439
|
total = Number(countRows[0]?.count ?? 0);
|
|
767
2440
|
}
|
|
768
|
-
const result = await this.
|
|
2441
|
+
const result = await this.client.query({
|
|
769
2442
|
query: `
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
2443
|
+
SELECT
|
|
2444
|
+
workflow_name,
|
|
2445
|
+
run_id,
|
|
2446
|
+
snapshot,
|
|
2447
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2448
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2449
|
+
resourceId
|
|
2450
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2451
|
+
${whereClause}
|
|
2452
|
+
ORDER BY createdAt DESC
|
|
2453
|
+
${limitClause}
|
|
2454
|
+
${offsetClause}
|
|
2455
|
+
`,
|
|
783
2456
|
query_params: values,
|
|
784
2457
|
format: "JSONEachRow"
|
|
785
2458
|
});
|
|
@@ -790,8 +2463,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
790
2463
|
});
|
|
791
2464
|
return { runs, total: total || runs.length };
|
|
792
2465
|
} catch (error) {
|
|
793
|
-
|
|
794
|
-
|
|
2466
|
+
throw new MastraError(
|
|
2467
|
+
{
|
|
2468
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2469
|
+
domain: ErrorDomain.STORAGE,
|
|
2470
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2471
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2472
|
+
},
|
|
2473
|
+
error
|
|
2474
|
+
);
|
|
795
2475
|
}
|
|
796
2476
|
}
|
|
797
2477
|
async getWorkflowRunById({
|
|
@@ -810,18 +2490,19 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
810
2490
|
values.var_workflow_name = workflowName;
|
|
811
2491
|
}
|
|
812
2492
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
813
|
-
const result = await this.
|
|
2493
|
+
const result = await this.client.query({
|
|
814
2494
|
query: `
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
2495
|
+
SELECT
|
|
2496
|
+
workflow_name,
|
|
2497
|
+
run_id,
|
|
2498
|
+
snapshot,
|
|
2499
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2500
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2501
|
+
resourceId
|
|
2502
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2503
|
+
${whereClause}
|
|
2504
|
+
ORDER BY createdAt DESC LIMIT 1
|
|
2505
|
+
`,
|
|
825
2506
|
query_params: values,
|
|
826
2507
|
format: "JSONEachRow"
|
|
827
2508
|
});
|
|
@@ -831,17 +2512,271 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
831
2512
|
}
|
|
832
2513
|
return this.parseWorkflowRun(resultJson[0]);
|
|
833
2514
|
} catch (error) {
|
|
834
|
-
|
|
835
|
-
|
|
2515
|
+
throw new MastraError(
|
|
2516
|
+
{
|
|
2517
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2518
|
+
domain: ErrorDomain.STORAGE,
|
|
2519
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2520
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2521
|
+
},
|
|
2522
|
+
error
|
|
2523
|
+
);
|
|
836
2524
|
}
|
|
837
2525
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
2526
|
+
};
|
|
2527
|
+
|
|
2528
|
+
// src/storage/index.ts
|
|
2529
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2530
|
+
db;
|
|
2531
|
+
ttl = {};
|
|
2532
|
+
stores;
|
|
2533
|
+
constructor(config) {
|
|
2534
|
+
super({ name: "ClickhouseStore" });
|
|
2535
|
+
this.db = createClient({
|
|
2536
|
+
url: config.url,
|
|
2537
|
+
username: config.username,
|
|
2538
|
+
password: config.password,
|
|
2539
|
+
clickhouse_settings: {
|
|
2540
|
+
date_time_input_format: "best_effort",
|
|
2541
|
+
date_time_output_format: "iso",
|
|
2542
|
+
// This is crucial
|
|
2543
|
+
use_client_time_zone: 1,
|
|
2544
|
+
output_format_json_quote_64bit_integers: 0
|
|
2545
|
+
}
|
|
842
2546
|
});
|
|
843
|
-
|
|
844
|
-
|
|
2547
|
+
this.ttl = config.ttl;
|
|
2548
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2549
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2550
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2551
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2552
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2553
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2554
|
+
this.stores = {
|
|
2555
|
+
operations,
|
|
2556
|
+
workflows,
|
|
2557
|
+
scores,
|
|
2558
|
+
legacyEvals,
|
|
2559
|
+
traces,
|
|
2560
|
+
memory
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
get supports() {
|
|
2564
|
+
return {
|
|
2565
|
+
selectByIncludeResourceScope: true,
|
|
2566
|
+
resourceWorkingMemory: true,
|
|
2567
|
+
hasColumn: true,
|
|
2568
|
+
createTable: true,
|
|
2569
|
+
deleteMessages: false
|
|
2570
|
+
};
|
|
2571
|
+
}
|
|
2572
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2573
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2574
|
+
}
|
|
2575
|
+
async getEvals(options) {
|
|
2576
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2577
|
+
}
|
|
2578
|
+
async batchInsert({ tableName, records }) {
|
|
2579
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2580
|
+
}
|
|
2581
|
+
async optimizeTable({ tableName }) {
|
|
2582
|
+
try {
|
|
2583
|
+
await this.db.command({
|
|
2584
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2585
|
+
});
|
|
2586
|
+
} catch (error) {
|
|
2587
|
+
throw new MastraError(
|
|
2588
|
+
{
|
|
2589
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2590
|
+
domain: ErrorDomain.STORAGE,
|
|
2591
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2592
|
+
details: { tableName }
|
|
2593
|
+
},
|
|
2594
|
+
error
|
|
2595
|
+
);
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
async materializeTtl({ tableName }) {
|
|
2599
|
+
try {
|
|
2600
|
+
await this.db.command({
|
|
2601
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2602
|
+
});
|
|
2603
|
+
} catch (error) {
|
|
2604
|
+
throw new MastraError(
|
|
2605
|
+
{
|
|
2606
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2607
|
+
domain: ErrorDomain.STORAGE,
|
|
2608
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2609
|
+
details: { tableName }
|
|
2610
|
+
},
|
|
2611
|
+
error
|
|
2612
|
+
);
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
async createTable({
|
|
2616
|
+
tableName,
|
|
2617
|
+
schema
|
|
2618
|
+
}) {
|
|
2619
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2620
|
+
}
|
|
2621
|
+
async dropTable({ tableName }) {
|
|
2622
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2623
|
+
}
|
|
2624
|
+
async alterTable({
|
|
2625
|
+
tableName,
|
|
2626
|
+
schema,
|
|
2627
|
+
ifNotExists
|
|
2628
|
+
}) {
|
|
2629
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2630
|
+
}
|
|
2631
|
+
async clearTable({ tableName }) {
|
|
2632
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2633
|
+
}
|
|
2634
|
+
async insert({ tableName, record }) {
|
|
2635
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2636
|
+
}
|
|
2637
|
+
async load({ tableName, keys }) {
|
|
2638
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2639
|
+
}
|
|
2640
|
+
async updateWorkflowResults({
|
|
2641
|
+
workflowName,
|
|
2642
|
+
runId,
|
|
2643
|
+
stepId,
|
|
2644
|
+
result,
|
|
2645
|
+
runtimeContext
|
|
2646
|
+
}) {
|
|
2647
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2648
|
+
}
|
|
2649
|
+
async updateWorkflowState({
|
|
2650
|
+
workflowName,
|
|
2651
|
+
runId,
|
|
2652
|
+
opts
|
|
2653
|
+
}) {
|
|
2654
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2655
|
+
}
|
|
2656
|
+
async persistWorkflowSnapshot({
|
|
2657
|
+
workflowName,
|
|
2658
|
+
runId,
|
|
2659
|
+
resourceId,
|
|
2660
|
+
snapshot
|
|
2661
|
+
}) {
|
|
2662
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2663
|
+
}
|
|
2664
|
+
async loadWorkflowSnapshot({
|
|
2665
|
+
workflowName,
|
|
2666
|
+
runId
|
|
2667
|
+
}) {
|
|
2668
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2669
|
+
}
|
|
2670
|
+
async getWorkflowRuns({
|
|
2671
|
+
workflowName,
|
|
2672
|
+
fromDate,
|
|
2673
|
+
toDate,
|
|
2674
|
+
limit,
|
|
2675
|
+
offset,
|
|
2676
|
+
resourceId
|
|
2677
|
+
} = {}) {
|
|
2678
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2679
|
+
}
|
|
2680
|
+
async getWorkflowRunById({
|
|
2681
|
+
runId,
|
|
2682
|
+
workflowName
|
|
2683
|
+
}) {
|
|
2684
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2685
|
+
}
|
|
2686
|
+
async getTraces(args) {
|
|
2687
|
+
return this.stores.traces.getTraces(args);
|
|
2688
|
+
}
|
|
2689
|
+
async getTracesPaginated(args) {
|
|
2690
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2691
|
+
}
|
|
2692
|
+
async batchTraceInsert(args) {
|
|
2693
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2694
|
+
}
|
|
2695
|
+
async getThreadById({ threadId }) {
|
|
2696
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2697
|
+
}
|
|
2698
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2699
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2700
|
+
}
|
|
2701
|
+
async saveThread({ thread }) {
|
|
2702
|
+
return this.stores.memory.saveThread({ thread });
|
|
2703
|
+
}
|
|
2704
|
+
async updateThread({
|
|
2705
|
+
id,
|
|
2706
|
+
title,
|
|
2707
|
+
metadata
|
|
2708
|
+
}) {
|
|
2709
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2710
|
+
}
|
|
2711
|
+
async deleteThread({ threadId }) {
|
|
2712
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2713
|
+
}
|
|
2714
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2715
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2716
|
+
}
|
|
2717
|
+
async getMessages({
|
|
2718
|
+
threadId,
|
|
2719
|
+
resourceId,
|
|
2720
|
+
selectBy,
|
|
2721
|
+
format
|
|
2722
|
+
}) {
|
|
2723
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2724
|
+
}
|
|
2725
|
+
async getMessagesById({
|
|
2726
|
+
messageIds,
|
|
2727
|
+
format
|
|
2728
|
+
}) {
|
|
2729
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2730
|
+
}
|
|
2731
|
+
async saveMessages(args) {
|
|
2732
|
+
return this.stores.memory.saveMessages(args);
|
|
2733
|
+
}
|
|
2734
|
+
async getMessagesPaginated(args) {
|
|
2735
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2736
|
+
}
|
|
2737
|
+
async updateMessages(args) {
|
|
2738
|
+
return this.stores.memory.updateMessages(args);
|
|
2739
|
+
}
|
|
2740
|
+
async getResourceById({ resourceId }) {
|
|
2741
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2742
|
+
}
|
|
2743
|
+
async saveResource({ resource }) {
|
|
2744
|
+
return this.stores.memory.saveResource({ resource });
|
|
2745
|
+
}
|
|
2746
|
+
async updateResource({
|
|
2747
|
+
resourceId,
|
|
2748
|
+
workingMemory,
|
|
2749
|
+
metadata
|
|
2750
|
+
}) {
|
|
2751
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2752
|
+
}
|
|
2753
|
+
async getScoreById({ id }) {
|
|
2754
|
+
return this.stores.scores.getScoreById({ id });
|
|
2755
|
+
}
|
|
2756
|
+
async saveScore(_score) {
|
|
2757
|
+
return this.stores.scores.saveScore(_score);
|
|
2758
|
+
}
|
|
2759
|
+
async getScoresByRunId({
|
|
2760
|
+
runId,
|
|
2761
|
+
pagination
|
|
2762
|
+
}) {
|
|
2763
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2764
|
+
}
|
|
2765
|
+
async getScoresByEntityId({
|
|
2766
|
+
entityId,
|
|
2767
|
+
entityType,
|
|
2768
|
+
pagination
|
|
2769
|
+
}) {
|
|
2770
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2771
|
+
}
|
|
2772
|
+
async getScoresByScorerId({
|
|
2773
|
+
scorerId,
|
|
2774
|
+
pagination,
|
|
2775
|
+
entityId,
|
|
2776
|
+
entityType,
|
|
2777
|
+
source
|
|
2778
|
+
}) {
|
|
2779
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
|
|
845
2780
|
}
|
|
846
2781
|
async close() {
|
|
847
2782
|
await this.db.close();
|
|
@@ -849,3 +2784,5 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
849
2784
|
};
|
|
850
2785
|
|
|
851
2786
|
export { COLUMN_TYPES, ClickhouseStore, TABLE_ENGINES };
|
|
2787
|
+
//# sourceMappingURL=index.js.map
|
|
2788
|
+
//# sourceMappingURL=index.js.map
|