@mastra/clickhouse 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +53 -0
- package/LICENSE.md +12 -4
- package/dist/_tsup-dts-rollup.d.cts +351 -64
- package/dist/_tsup-dts-rollup.d.ts +351 -64
- package/dist/index.cjs +2052 -609
- package/dist/index.d.cts +0 -2
- package/dist/index.d.ts +0 -2
- package/dist/index.js +2051 -606
- package/package.json +6 -6
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +15 -1091
- package/src/storage/index.ts +184 -1246
package/dist/index.js
CHANGED
|
@@ -1,22 +1,17 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import { MessageList } from '@mastra/core/agent';
|
|
3
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
4
|
-
import { TABLE_EVALS,
|
|
3
|
+
import { MastraStorage, StoreOperations, TABLE_SCHEMAS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, ScoresStorage, safelyParseJSON, TABLE_SCORERS, LegacyEvalsStorage, TABLE_EVALS, TracesStorage, TABLE_TRACES, MemoryStorage, resolveMessageLimit, TABLE_MESSAGES, TABLE_THREADS, TABLE_RESOURCES } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
5
5
|
|
|
6
6
|
// src/storage/index.ts
|
|
7
|
-
function safelyParseJSON(jsonString) {
|
|
8
|
-
try {
|
|
9
|
-
return JSON.parse(jsonString);
|
|
10
|
-
} catch {
|
|
11
|
-
return {};
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
7
|
var TABLE_ENGINES = {
|
|
15
8
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
16
9
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
17
10
|
[TABLE_TRACES]: `MergeTree()`,
|
|
18
11
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
19
|
-
[TABLE_EVALS]: `MergeTree()
|
|
12
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
13
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
14
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`
|
|
20
15
|
};
|
|
21
16
|
var COLUMN_TYPES = {
|
|
22
17
|
text: "String",
|
|
@@ -24,11 +19,9 @@ var COLUMN_TYPES = {
|
|
|
24
19
|
uuid: "String",
|
|
25
20
|
jsonb: "String",
|
|
26
21
|
integer: "Int64",
|
|
22
|
+
float: "Float64",
|
|
27
23
|
bigint: "Int64"
|
|
28
24
|
};
|
|
29
|
-
function transformRows(rows) {
|
|
30
|
-
return rows.map((row) => transformRow(row));
|
|
31
|
-
}
|
|
32
25
|
function transformRow(row) {
|
|
33
26
|
if (!row) {
|
|
34
27
|
return row;
|
|
@@ -39,31 +32,56 @@ function transformRow(row) {
|
|
|
39
32
|
if (row.updatedAt) {
|
|
40
33
|
row.updatedAt = new Date(row.updatedAt);
|
|
41
34
|
}
|
|
35
|
+
if (row.content && typeof row.content === "string") {
|
|
36
|
+
row.content = safelyParseJSON(row.content);
|
|
37
|
+
}
|
|
42
38
|
return row;
|
|
43
39
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// This is crucial
|
|
57
|
-
use_client_time_zone: 1,
|
|
58
|
-
output_format_json_quote_64bit_integers: 0
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
this.ttl = config.ttl;
|
|
40
|
+
function transformRows(rows) {
|
|
41
|
+
return rows.map((row) => transformRow(row));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
45
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
46
|
+
client;
|
|
47
|
+
operations;
|
|
48
|
+
constructor({ client, operations }) {
|
|
49
|
+
super();
|
|
50
|
+
this.client = client;
|
|
51
|
+
this.operations = operations;
|
|
62
52
|
}
|
|
63
53
|
transformEvalRow(row) {
|
|
64
54
|
row = transformRow(row);
|
|
65
|
-
|
|
66
|
-
|
|
55
|
+
let resultValue;
|
|
56
|
+
try {
|
|
57
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
58
|
+
resultValue = JSON.parse(row.result);
|
|
59
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
60
|
+
resultValue = row.result;
|
|
61
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
62
|
+
resultValue = { score: 0 };
|
|
63
|
+
} else {
|
|
64
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
console.error("Error parsing result field:", row.result, error);
|
|
68
|
+
throw new MastraError({
|
|
69
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
70
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
71
|
+
domain: ErrorDomain.STORAGE,
|
|
72
|
+
category: ErrorCategory.USER
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
let testInfoValue;
|
|
76
|
+
try {
|
|
77
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
78
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
79
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
80
|
+
testInfoValue = row.test_info;
|
|
81
|
+
}
|
|
82
|
+
} catch {
|
|
83
|
+
testInfoValue = void 0;
|
|
84
|
+
}
|
|
67
85
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
86
|
throw new MastraError({
|
|
69
87
|
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
@@ -85,23 +103,11 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
85
103
|
createdAt: row.created_at
|
|
86
104
|
};
|
|
87
105
|
}
|
|
88
|
-
escape(value) {
|
|
89
|
-
if (typeof value === "string") {
|
|
90
|
-
return `'${value.replace(/'/g, "''")}'`;
|
|
91
|
-
}
|
|
92
|
-
if (value instanceof Date) {
|
|
93
|
-
return `'${value.toISOString()}'`;
|
|
94
|
-
}
|
|
95
|
-
if (value === null || value === void 0) {
|
|
96
|
-
return "NULL";
|
|
97
|
-
}
|
|
98
|
-
return value.toString();
|
|
99
|
-
}
|
|
100
106
|
async getEvalsByAgentName(agentName, type) {
|
|
101
107
|
try {
|
|
102
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
103
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
104
|
-
const result = await this.
|
|
108
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
109
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
110
|
+
const result = await this.client.query({
|
|
105
111
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
106
112
|
query_params: { var_agent_name: agentName },
|
|
107
113
|
clickhouse_settings: {
|
|
@@ -131,207 +137,201 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
131
137
|
);
|
|
132
138
|
}
|
|
133
139
|
}
|
|
134
|
-
async
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
values: records.map((record) => ({
|
|
139
|
-
...Object.fromEntries(
|
|
140
|
-
Object.entries(record).map(([key, value]) => [
|
|
141
|
-
key,
|
|
142
|
-
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
143
|
-
])
|
|
144
|
-
)
|
|
145
|
-
})),
|
|
146
|
-
format: "JSONEachRow",
|
|
147
|
-
clickhouse_settings: {
|
|
148
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
149
|
-
date_time_input_format: "best_effort",
|
|
150
|
-
use_client_time_zone: 1,
|
|
151
|
-
output_format_json_quote_64bit_integers: 0
|
|
152
|
-
}
|
|
153
|
-
});
|
|
154
|
-
} catch (error) {
|
|
155
|
-
throw new MastraError(
|
|
156
|
-
{
|
|
157
|
-
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
158
|
-
domain: ErrorDomain.STORAGE,
|
|
159
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
160
|
-
details: { tableName }
|
|
161
|
-
},
|
|
162
|
-
error
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
async getTraces({
|
|
167
|
-
name,
|
|
168
|
-
scope,
|
|
169
|
-
page,
|
|
170
|
-
perPage,
|
|
171
|
-
attributes,
|
|
172
|
-
filters,
|
|
173
|
-
fromDate,
|
|
174
|
-
toDate
|
|
175
|
-
}) {
|
|
176
|
-
const limit = perPage;
|
|
177
|
-
const offset = page * perPage;
|
|
178
|
-
const args = {};
|
|
140
|
+
async getEvals(options = {}) {
|
|
141
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
142
|
+
const fromDate = dateRange?.start;
|
|
143
|
+
const toDate = dateRange?.end;
|
|
179
144
|
const conditions = [];
|
|
180
|
-
if (
|
|
181
|
-
conditions.push(`
|
|
182
|
-
args.var_name = name;
|
|
183
|
-
}
|
|
184
|
-
if (scope) {
|
|
185
|
-
conditions.push(`scope = {var_scope:String}`);
|
|
186
|
-
args.var_scope = scope;
|
|
145
|
+
if (agentName) {
|
|
146
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
187
147
|
}
|
|
188
|
-
if (
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
conditions.push(
|
|
197
|
-
`${key} = {var_col_${key}:${COLUMN_TYPES[TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
|
|
198
|
-
);
|
|
199
|
-
args[`var_col_${key}`] = value;
|
|
200
|
-
});
|
|
148
|
+
if (type === "test") {
|
|
149
|
+
conditions.push(
|
|
150
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
151
|
+
);
|
|
152
|
+
} else if (type === "live") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
155
|
+
);
|
|
201
156
|
}
|
|
202
157
|
if (fromDate) {
|
|
203
|
-
conditions.push(`
|
|
204
|
-
|
|
158
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
159
|
+
fromDate.toISOString();
|
|
205
160
|
}
|
|
206
161
|
if (toDate) {
|
|
207
|
-
conditions.push(`
|
|
208
|
-
|
|
162
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
163
|
+
toDate.toISOString();
|
|
209
164
|
}
|
|
210
165
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
211
166
|
try {
|
|
212
|
-
const
|
|
213
|
-
query: `SELECT
|
|
214
|
-
query_params:
|
|
167
|
+
const countResult = await this.client.query({
|
|
168
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
169
|
+
query_params: {
|
|
170
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
171
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
172
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
173
|
+
},
|
|
215
174
|
clickhouse_settings: {
|
|
216
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
217
175
|
date_time_input_format: "best_effort",
|
|
218
176
|
date_time_output_format: "iso",
|
|
219
177
|
use_client_time_zone: 1,
|
|
220
178
|
output_format_json_quote_64bit_integers: 0
|
|
221
179
|
}
|
|
222
180
|
});
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
status: safelyParseJSON(row.status),
|
|
236
|
-
events: safelyParseJSON(row.events),
|
|
237
|
-
links: safelyParseJSON(row.links),
|
|
238
|
-
attributes: safelyParseJSON(row.attributes),
|
|
239
|
-
startTime: row.startTime,
|
|
240
|
-
endTime: row.endTime,
|
|
241
|
-
other: safelyParseJSON(row.other),
|
|
242
|
-
createdAt: row.createdAt
|
|
243
|
-
}));
|
|
244
|
-
} catch (error) {
|
|
245
|
-
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
246
|
-
return [];
|
|
181
|
+
const countData = await countResult.json();
|
|
182
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
183
|
+
const currentOffset = page * perPage;
|
|
184
|
+
const hasMore = currentOffset + perPage < total;
|
|
185
|
+
if (total === 0) {
|
|
186
|
+
return {
|
|
187
|
+
evals: [],
|
|
188
|
+
total: 0,
|
|
189
|
+
page,
|
|
190
|
+
perPage,
|
|
191
|
+
hasMore: false
|
|
192
|
+
};
|
|
247
193
|
}
|
|
248
|
-
|
|
249
|
-
{
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
page,
|
|
257
|
-
perPage,
|
|
258
|
-
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
259
|
-
filters: filters ? JSON.stringify(filters) : null,
|
|
260
|
-
fromDate: fromDate?.toISOString() ?? null,
|
|
261
|
-
toDate: toDate?.toISOString() ?? null
|
|
262
|
-
}
|
|
194
|
+
const dataResult = await this.client.query({
|
|
195
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
196
|
+
query_params: {
|
|
197
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
198
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
199
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
200
|
+
var_limit: perPage || 100,
|
|
201
|
+
var_offset: currentOffset || 0
|
|
263
202
|
},
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
await this.db.command({
|
|
271
|
-
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
203
|
+
clickhouse_settings: {
|
|
204
|
+
date_time_input_format: "best_effort",
|
|
205
|
+
date_time_output_format: "iso",
|
|
206
|
+
use_client_time_zone: 1,
|
|
207
|
+
output_format_json_quote_64bit_integers: 0
|
|
208
|
+
}
|
|
272
209
|
});
|
|
210
|
+
const rows = await dataResult.json();
|
|
211
|
+
return {
|
|
212
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
213
|
+
total,
|
|
214
|
+
page,
|
|
215
|
+
perPage,
|
|
216
|
+
hasMore
|
|
217
|
+
};
|
|
273
218
|
} catch (error) {
|
|
219
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
220
|
+
return {
|
|
221
|
+
evals: [],
|
|
222
|
+
total: 0,
|
|
223
|
+
page,
|
|
224
|
+
perPage,
|
|
225
|
+
hasMore: false
|
|
226
|
+
};
|
|
227
|
+
}
|
|
274
228
|
throw new MastraError(
|
|
275
229
|
{
|
|
276
|
-
id: "
|
|
230
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
277
231
|
domain: ErrorDomain.STORAGE,
|
|
278
232
|
category: ErrorCategory.THIRD_PARTY,
|
|
279
|
-
details: {
|
|
233
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
280
234
|
},
|
|
281
235
|
error
|
|
282
236
|
);
|
|
283
237
|
}
|
|
284
238
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
294
|
-
domain: ErrorDomain.STORAGE,
|
|
295
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
296
|
-
details: { tableName }
|
|
297
|
-
},
|
|
298
|
-
error
|
|
299
|
-
);
|
|
300
|
-
}
|
|
239
|
+
};
|
|
240
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
241
|
+
client;
|
|
242
|
+
operations;
|
|
243
|
+
constructor({ client, operations }) {
|
|
244
|
+
super();
|
|
245
|
+
this.client = client;
|
|
246
|
+
this.operations = operations;
|
|
301
247
|
}
|
|
302
|
-
async
|
|
303
|
-
|
|
304
|
-
|
|
248
|
+
async getMessages({
|
|
249
|
+
threadId,
|
|
250
|
+
resourceId,
|
|
251
|
+
selectBy,
|
|
252
|
+
format
|
|
305
253
|
}) {
|
|
306
254
|
try {
|
|
307
|
-
const
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
255
|
+
const messages = [];
|
|
256
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
257
|
+
const include = selectBy?.include || [];
|
|
258
|
+
if (include.length) {
|
|
259
|
+
const unionQueries = [];
|
|
260
|
+
const params = [];
|
|
261
|
+
let paramIdx = 1;
|
|
262
|
+
for (const inc of include) {
|
|
263
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
264
|
+
const searchId = inc.threadId || threadId;
|
|
265
|
+
unionQueries.push(`
|
|
266
|
+
SELECT * FROM (
|
|
267
|
+
WITH numbered_messages AS (
|
|
268
|
+
SELECT
|
|
269
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
270
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
271
|
+
FROM "${TABLE_MESSAGES}"
|
|
272
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
273
|
+
),
|
|
274
|
+
target_positions AS (
|
|
275
|
+
SELECT row_num as target_pos
|
|
276
|
+
FROM numbered_messages
|
|
277
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
278
|
+
)
|
|
279
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
280
|
+
FROM numbered_messages m
|
|
281
|
+
CROSS JOIN target_positions t
|
|
282
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
283
|
+
) AS query_${paramIdx}
|
|
284
|
+
`);
|
|
285
|
+
params.push(
|
|
286
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
287
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
288
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
289
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
290
|
+
);
|
|
291
|
+
paramIdx++;
|
|
292
|
+
}
|
|
293
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
294
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
295
|
+
const includeResult = await this.client.query({
|
|
296
|
+
query: finalQuery,
|
|
297
|
+
query_params: mergedParams,
|
|
298
|
+
clickhouse_settings: {
|
|
299
|
+
date_time_input_format: "best_effort",
|
|
300
|
+
date_time_output_format: "iso",
|
|
301
|
+
use_client_time_zone: 1,
|
|
302
|
+
output_format_json_quote_64bit_integers: 0
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
const rows2 = await includeResult.json();
|
|
306
|
+
const includedMessages = transformRows(rows2.data);
|
|
307
|
+
const seen = /* @__PURE__ */ new Set();
|
|
308
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
309
|
+
if (seen.has(message.id)) return false;
|
|
310
|
+
seen.add(message.id);
|
|
311
|
+
return true;
|
|
312
|
+
});
|
|
313
|
+
messages.push(...dedupedMessages);
|
|
314
|
+
}
|
|
315
|
+
const result = await this.client.query({
|
|
316
|
+
query: `
|
|
317
|
+
SELECT
|
|
318
|
+
id,
|
|
319
|
+
content,
|
|
320
|
+
role,
|
|
321
|
+
type,
|
|
322
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
323
|
+
thread_id AS "threadId"
|
|
324
|
+
FROM "${TABLE_MESSAGES}"
|
|
325
|
+
WHERE thread_id = {threadId:String}
|
|
326
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
327
|
+
ORDER BY "createdAt" DESC
|
|
328
|
+
LIMIT {limit:Int64}
|
|
329
|
+
`,
|
|
330
|
+
query_params: {
|
|
331
|
+
threadId,
|
|
332
|
+
exclude: messages.map((m) => m.id),
|
|
333
|
+
limit
|
|
334
|
+
},
|
|
335
335
|
clickhouse_settings: {
|
|
336
336
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
337
337
|
date_time_input_format: "best_effort",
|
|
@@ -340,179 +340,178 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
340
340
|
output_format_json_quote_64bit_integers: 0
|
|
341
341
|
}
|
|
342
342
|
});
|
|
343
|
+
const rows = await result.json();
|
|
344
|
+
messages.push(...transformRows(rows.data));
|
|
345
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
346
|
+
messages.forEach((message) => {
|
|
347
|
+
if (typeof message.content === "string") {
|
|
348
|
+
try {
|
|
349
|
+
message.content = JSON.parse(message.content);
|
|
350
|
+
} catch {
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
355
|
+
if (format === `v2`) return list.get.all.v2();
|
|
356
|
+
return list.get.all.v1();
|
|
343
357
|
} catch (error) {
|
|
344
358
|
throw new MastraError(
|
|
345
359
|
{
|
|
346
|
-
id: "
|
|
360
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
347
361
|
domain: ErrorDomain.STORAGE,
|
|
348
362
|
category: ErrorCategory.THIRD_PARTY,
|
|
349
|
-
details: {
|
|
363
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
350
364
|
},
|
|
351
365
|
error
|
|
352
366
|
);
|
|
353
367
|
}
|
|
354
368
|
}
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
}
|
|
370
|
-
/**
|
|
371
|
-
* Alters table schema to add columns if they don't exist
|
|
372
|
-
* @param tableName Name of the table
|
|
373
|
-
* @param schema Schema of the table
|
|
374
|
-
* @param ifNotExists Array of column names to add if they don't exist
|
|
375
|
-
*/
|
|
376
|
-
async alterTable({
|
|
377
|
-
tableName,
|
|
378
|
-
schema,
|
|
379
|
-
ifNotExists
|
|
380
|
-
}) {
|
|
381
|
-
try {
|
|
382
|
-
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
383
|
-
const result = await this.db.query({
|
|
384
|
-
query: describeSql
|
|
385
|
-
});
|
|
386
|
-
const rows = await result.json();
|
|
387
|
-
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
388
|
-
for (const columnName of ifNotExists) {
|
|
389
|
-
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
390
|
-
const columnDef = schema[columnName];
|
|
391
|
-
let sqlType = this.getSqlType(columnDef.type);
|
|
392
|
-
if (columnDef.nullable !== false) {
|
|
393
|
-
sqlType = `Nullable(${sqlType})`;
|
|
394
|
-
}
|
|
395
|
-
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
396
|
-
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
397
|
-
await this.db.query({
|
|
398
|
-
query: alterSql
|
|
399
|
-
});
|
|
400
|
-
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
401
|
-
}
|
|
369
|
+
async saveMessages(args) {
|
|
370
|
+
const { messages, format = "v1" } = args;
|
|
371
|
+
if (messages.length === 0) return messages;
|
|
372
|
+
for (const message of messages) {
|
|
373
|
+
const resourceId = message.resourceId;
|
|
374
|
+
if (!resourceId) {
|
|
375
|
+
throw new Error("Resource ID is required");
|
|
376
|
+
}
|
|
377
|
+
if (!message.threadId) {
|
|
378
|
+
throw new Error("Thread ID is required");
|
|
379
|
+
}
|
|
380
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
381
|
+
if (!thread) {
|
|
382
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
402
383
|
}
|
|
403
|
-
} catch (error) {
|
|
404
|
-
throw new MastraError(
|
|
405
|
-
{
|
|
406
|
-
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
407
|
-
domain: ErrorDomain.STORAGE,
|
|
408
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
409
|
-
details: { tableName }
|
|
410
|
-
},
|
|
411
|
-
error
|
|
412
|
-
);
|
|
413
384
|
}
|
|
414
|
-
|
|
415
|
-
|
|
385
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
386
|
+
await Promise.all(
|
|
387
|
+
messages.map(async (m) => {
|
|
388
|
+
const resourceId = m.resourceId;
|
|
389
|
+
if (!resourceId) {
|
|
390
|
+
throw new Error("Resource ID is required");
|
|
391
|
+
}
|
|
392
|
+
if (!m.threadId) {
|
|
393
|
+
throw new Error("Thread ID is required");
|
|
394
|
+
}
|
|
395
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
396
|
+
if (!thread) {
|
|
397
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
398
|
+
}
|
|
399
|
+
threadIdSet.set(m.threadId, thread);
|
|
400
|
+
})
|
|
401
|
+
);
|
|
416
402
|
try {
|
|
417
|
-
await this.
|
|
418
|
-
query: `
|
|
403
|
+
const existingResult = await this.client.query({
|
|
404
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
405
|
+
query_params: {
|
|
406
|
+
ids: messages.map((m) => m.id)
|
|
407
|
+
},
|
|
419
408
|
clickhouse_settings: {
|
|
420
409
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
421
410
|
date_time_input_format: "best_effort",
|
|
422
411
|
date_time_output_format: "iso",
|
|
423
412
|
use_client_time_zone: 1,
|
|
424
413
|
output_format_json_quote_64bit_integers: 0
|
|
425
|
-
}
|
|
426
|
-
});
|
|
427
|
-
} catch (error) {
|
|
428
|
-
throw new MastraError(
|
|
429
|
-
{
|
|
430
|
-
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
431
|
-
domain: ErrorDomain.STORAGE,
|
|
432
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
433
|
-
details: { tableName }
|
|
434
414
|
},
|
|
435
|
-
|
|
436
|
-
);
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
415
|
+
format: "JSONEachRow"
|
|
416
|
+
});
|
|
417
|
+
const existingRows = await existingResult.json();
|
|
418
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
419
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
420
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
421
|
+
const toMove = messages.filter((m) => {
|
|
422
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
423
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
424
|
+
});
|
|
425
|
+
const deletePromises = toMove.map((message) => {
|
|
426
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
427
|
+
if (!existingRow) return Promise.resolve();
|
|
428
|
+
return this.client.command({
|
|
429
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
430
|
+
query_params: {
|
|
431
|
+
var_id: message.id,
|
|
432
|
+
var_old_thread_id: existingRow.thread_id
|
|
433
|
+
},
|
|
434
|
+
clickhouse_settings: {
|
|
435
|
+
date_time_input_format: "best_effort",
|
|
436
|
+
use_client_time_zone: 1,
|
|
437
|
+
output_format_json_quote_64bit_integers: 0
|
|
448
438
|
}
|
|
449
|
-
|
|
450
|
-
format: "JSONEachRow",
|
|
451
|
-
clickhouse_settings: {
|
|
452
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
453
|
-
output_format_json_quote_64bit_integers: 0,
|
|
454
|
-
date_time_input_format: "best_effort",
|
|
455
|
-
use_client_time_zone: 1
|
|
456
|
-
}
|
|
439
|
+
});
|
|
457
440
|
});
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
441
|
+
const updatePromises = toUpdate.map(
|
|
442
|
+
(message) => this.client.command({
|
|
443
|
+
query: `
|
|
444
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
445
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
446
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
447
|
+
`,
|
|
448
|
+
query_params: {
|
|
449
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
450
|
+
var_role: message.role,
|
|
451
|
+
var_type: message.type || "v2",
|
|
452
|
+
var_resourceId: message.resourceId,
|
|
453
|
+
var_id: message.id,
|
|
454
|
+
var_thread_id: message.threadId
|
|
455
|
+
},
|
|
456
|
+
clickhouse_settings: {
|
|
457
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
458
|
+
date_time_input_format: "best_effort",
|
|
459
|
+
use_client_time_zone: 1,
|
|
460
|
+
output_format_json_quote_64bit_integers: 0
|
|
461
|
+
}
|
|
462
|
+
})
|
|
467
463
|
);
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
464
|
+
await Promise.all([
|
|
465
|
+
// Insert new messages (including moved messages)
|
|
466
|
+
this.client.insert({
|
|
467
|
+
table: TABLE_MESSAGES,
|
|
468
|
+
format: "JSONEachRow",
|
|
469
|
+
values: toInsert.map((message) => ({
|
|
470
|
+
id: message.id,
|
|
471
|
+
thread_id: message.threadId,
|
|
472
|
+
resourceId: message.resourceId,
|
|
473
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
474
|
+
createdAt: message.createdAt.toISOString(),
|
|
475
|
+
role: message.role,
|
|
476
|
+
type: message.type || "v2"
|
|
477
|
+
})),
|
|
478
|
+
clickhouse_settings: {
|
|
479
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
480
|
+
date_time_input_format: "best_effort",
|
|
481
|
+
use_client_time_zone: 1,
|
|
482
|
+
output_format_json_quote_64bit_integers: 0
|
|
483
|
+
}
|
|
484
|
+
}),
|
|
485
|
+
...updatePromises,
|
|
486
|
+
...deletePromises,
|
|
487
|
+
// Update thread's updatedAt timestamp
|
|
488
|
+
this.client.insert({
|
|
489
|
+
table: TABLE_THREADS,
|
|
490
|
+
format: "JSONEachRow",
|
|
491
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
492
|
+
id: thread.id,
|
|
493
|
+
resourceId: thread.resourceId,
|
|
494
|
+
title: thread.title,
|
|
495
|
+
metadata: thread.metadata,
|
|
496
|
+
createdAt: thread.createdAt,
|
|
497
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
498
|
+
})),
|
|
499
|
+
clickhouse_settings: {
|
|
500
|
+
date_time_input_format: "best_effort",
|
|
501
|
+
use_client_time_zone: 1,
|
|
502
|
+
output_format_json_quote_64bit_integers: 0
|
|
503
|
+
}
|
|
504
|
+
})
|
|
505
|
+
]);
|
|
506
|
+
const list = new MessageList().add(messages, "memory");
|
|
507
|
+
if (format === `v2`) return list.get.all.v2();
|
|
508
|
+
return list.get.all.v1();
|
|
509
509
|
} catch (error) {
|
|
510
510
|
throw new MastraError(
|
|
511
511
|
{
|
|
512
|
-
id: "
|
|
512
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
513
513
|
domain: ErrorDomain.STORAGE,
|
|
514
|
-
category: ErrorCategory.THIRD_PARTY
|
|
515
|
-
details: { tableName }
|
|
514
|
+
category: ErrorCategory.THIRD_PARTY
|
|
516
515
|
},
|
|
517
516
|
error
|
|
518
517
|
);
|
|
@@ -520,7 +519,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
520
519
|
}
|
|
521
520
|
async getThreadById({ threadId }) {
|
|
522
521
|
try {
|
|
523
|
-
const result = await this.
|
|
522
|
+
const result = await this.client.query({
|
|
524
523
|
query: `SELECT
|
|
525
524
|
id,
|
|
526
525
|
"resourceId",
|
|
@@ -565,7 +564,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
565
564
|
}
|
|
566
565
|
async getThreadsByResourceId({ resourceId }) {
|
|
567
566
|
try {
|
|
568
|
-
const result = await this.
|
|
567
|
+
const result = await this.client.query({
|
|
569
568
|
query: `SELECT
|
|
570
569
|
id,
|
|
571
570
|
"resourceId",
|
|
@@ -606,7 +605,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
606
605
|
}
|
|
607
606
|
async saveThread({ thread }) {
|
|
608
607
|
try {
|
|
609
|
-
await this.
|
|
608
|
+
await this.client.insert({
|
|
610
609
|
table: TABLE_THREADS,
|
|
611
610
|
values: [
|
|
612
611
|
{
|
|
@@ -656,7 +655,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
656
655
|
metadata: mergedMetadata,
|
|
657
656
|
updatedAt: /* @__PURE__ */ new Date()
|
|
658
657
|
};
|
|
659
|
-
await this.
|
|
658
|
+
await this.client.insert({
|
|
660
659
|
table: TABLE_THREADS,
|
|
661
660
|
format: "JSONEachRow",
|
|
662
661
|
values: [
|
|
@@ -690,14 +689,14 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
690
689
|
}
|
|
691
690
|
async deleteThread({ threadId }) {
|
|
692
691
|
try {
|
|
693
|
-
await this.
|
|
692
|
+
await this.client.command({
|
|
694
693
|
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
695
694
|
query_params: { var_thread_id: threadId },
|
|
696
695
|
clickhouse_settings: {
|
|
697
696
|
output_format_json_quote_64bit_integers: 0
|
|
698
697
|
}
|
|
699
698
|
});
|
|
700
|
-
await this.
|
|
699
|
+
await this.client.command({
|
|
701
700
|
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
702
701
|
query_params: { var_id: threadId },
|
|
703
702
|
clickhouse_settings: {
|
|
@@ -716,238 +715,1492 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
716
715
|
);
|
|
717
716
|
}
|
|
718
717
|
}
|
|
719
|
-
async
|
|
720
|
-
|
|
721
|
-
resourceId,
|
|
722
|
-
selectBy,
|
|
723
|
-
format
|
|
724
|
-
}) {
|
|
718
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
719
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
725
720
|
try {
|
|
726
|
-
const
|
|
727
|
-
const
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
m.updatedAt as updatedAt,
|
|
748
|
-
m.thread_id AS "threadId"
|
|
749
|
-
FROM ordered_messages m
|
|
750
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
751
|
-
OR EXISTS (
|
|
752
|
-
SELECT 1 FROM ordered_messages target
|
|
753
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
754
|
-
AND (
|
|
755
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
756
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
757
|
-
OR
|
|
758
|
-
-- Get next messages based on the max withNextMessages
|
|
759
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
760
|
-
)
|
|
761
|
-
)
|
|
762
|
-
ORDER BY m."createdAt" DESC
|
|
763
|
-
`,
|
|
764
|
-
query_params: {
|
|
765
|
-
var_thread_id: threadId,
|
|
766
|
-
var_include: include.map((i) => i.id),
|
|
767
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
768
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
769
|
-
},
|
|
770
|
-
clickhouse_settings: {
|
|
771
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
772
|
-
date_time_input_format: "best_effort",
|
|
773
|
-
date_time_output_format: "iso",
|
|
774
|
-
use_client_time_zone: 1,
|
|
775
|
-
output_format_json_quote_64bit_integers: 0
|
|
776
|
-
}
|
|
777
|
-
});
|
|
778
|
-
const rows2 = await includeResult.json();
|
|
779
|
-
messages.push(...transformRows(rows2.data));
|
|
721
|
+
const currentOffset = page * perPage;
|
|
722
|
+
const countResult = await this.client.query({
|
|
723
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
724
|
+
query_params: { resourceId },
|
|
725
|
+
clickhouse_settings: {
|
|
726
|
+
date_time_input_format: "best_effort",
|
|
727
|
+
date_time_output_format: "iso",
|
|
728
|
+
use_client_time_zone: 1,
|
|
729
|
+
output_format_json_quote_64bit_integers: 0
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
const countData = await countResult.json();
|
|
733
|
+
const total = countData.data[0].total;
|
|
734
|
+
if (total === 0) {
|
|
735
|
+
return {
|
|
736
|
+
threads: [],
|
|
737
|
+
total: 0,
|
|
738
|
+
page,
|
|
739
|
+
perPage,
|
|
740
|
+
hasMore: false
|
|
741
|
+
};
|
|
780
742
|
}
|
|
781
|
-
const
|
|
743
|
+
const dataResult = await this.client.query({
|
|
782
744
|
query: `
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
`,
|
|
745
|
+
SELECT
|
|
746
|
+
id,
|
|
747
|
+
resourceId,
|
|
748
|
+
title,
|
|
749
|
+
metadata,
|
|
750
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
751
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
752
|
+
FROM ${TABLE_THREADS}
|
|
753
|
+
WHERE resourceId = {resourceId:String}
|
|
754
|
+
ORDER BY createdAt DESC
|
|
755
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
756
|
+
`,
|
|
796
757
|
query_params: {
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
758
|
+
resourceId,
|
|
759
|
+
limit: perPage,
|
|
760
|
+
offset: currentOffset
|
|
800
761
|
},
|
|
801
762
|
clickhouse_settings: {
|
|
802
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
803
763
|
date_time_input_format: "best_effort",
|
|
804
764
|
date_time_output_format: "iso",
|
|
805
765
|
use_client_time_zone: 1,
|
|
806
766
|
output_format_json_quote_64bit_integers: 0
|
|
807
767
|
}
|
|
808
768
|
});
|
|
809
|
-
const rows = await
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
769
|
+
const rows = await dataResult.json();
|
|
770
|
+
const threads = transformRows(rows.data);
|
|
771
|
+
return {
|
|
772
|
+
threads,
|
|
773
|
+
total,
|
|
774
|
+
page,
|
|
775
|
+
perPage,
|
|
776
|
+
hasMore: currentOffset + threads.length < total
|
|
777
|
+
};
|
|
778
|
+
} catch (error) {
|
|
779
|
+
throw new MastraError(
|
|
780
|
+
{
|
|
781
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
782
|
+
domain: ErrorDomain.STORAGE,
|
|
783
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
784
|
+
details: { resourceId, page }
|
|
785
|
+
},
|
|
786
|
+
error
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async getMessagesPaginated(args) {
|
|
791
|
+
try {
|
|
792
|
+
const { threadId, selectBy, format = "v1" } = args;
|
|
793
|
+
const page = selectBy?.pagination?.page || 0;
|
|
794
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
795
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
796
|
+
const offset = page * perPage;
|
|
797
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
798
|
+
const fromDate = dateRange?.start;
|
|
799
|
+
const toDate = dateRange?.end;
|
|
800
|
+
const messages = [];
|
|
801
|
+
if (selectBy?.include?.length) {
|
|
802
|
+
const include = selectBy.include;
|
|
803
|
+
const unionQueries = [];
|
|
804
|
+
const params = [];
|
|
805
|
+
let paramIdx = 1;
|
|
806
|
+
for (const inc of include) {
|
|
807
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
808
|
+
const searchId = inc.threadId || threadId;
|
|
809
|
+
unionQueries.push(`
|
|
810
|
+
SELECT * FROM (
|
|
811
|
+
WITH numbered_messages AS (
|
|
812
|
+
SELECT
|
|
813
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
814
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
815
|
+
FROM "${TABLE_MESSAGES}"
|
|
816
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
817
|
+
),
|
|
818
|
+
target_positions AS (
|
|
819
|
+
SELECT row_num as target_pos
|
|
820
|
+
FROM numbered_messages
|
|
821
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
822
|
+
)
|
|
823
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
824
|
+
FROM numbered_messages m
|
|
825
|
+
CROSS JOIN target_positions t
|
|
826
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
827
|
+
) AS query_${paramIdx}
|
|
828
|
+
`);
|
|
829
|
+
params.push(
|
|
830
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
831
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
832
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
833
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
834
|
+
);
|
|
835
|
+
paramIdx++;
|
|
836
|
+
}
|
|
837
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
838
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
839
|
+
const includeResult = await this.client.query({
|
|
840
|
+
query: finalQuery,
|
|
841
|
+
query_params: mergedParams,
|
|
842
|
+
clickhouse_settings: {
|
|
843
|
+
date_time_input_format: "best_effort",
|
|
844
|
+
date_time_output_format: "iso",
|
|
845
|
+
use_client_time_zone: 1,
|
|
846
|
+
output_format_json_quote_64bit_integers: 0
|
|
847
|
+
}
|
|
848
|
+
});
|
|
849
|
+
const rows2 = await includeResult.json();
|
|
850
|
+
const includedMessages = transformRows(rows2.data);
|
|
851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
852
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
853
|
+
if (seen.has(message.id)) return false;
|
|
854
|
+
seen.add(message.id);
|
|
855
|
+
return true;
|
|
856
|
+
});
|
|
857
|
+
messages.push(...dedupedMessages);
|
|
858
|
+
}
|
|
859
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
860
|
+
const countParams = { threadId };
|
|
861
|
+
if (fromDate) {
|
|
862
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
863
|
+
countParams.fromDate = fromDate.toISOString();
|
|
864
|
+
}
|
|
865
|
+
if (toDate) {
|
|
866
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
867
|
+
countParams.toDate = toDate.toISOString();
|
|
868
|
+
}
|
|
869
|
+
const countResult = await this.client.query({
|
|
870
|
+
query: countQuery,
|
|
871
|
+
query_params: countParams,
|
|
872
|
+
clickhouse_settings: {
|
|
873
|
+
date_time_input_format: "best_effort",
|
|
874
|
+
date_time_output_format: "iso",
|
|
875
|
+
use_client_time_zone: 1,
|
|
876
|
+
output_format_json_quote_64bit_integers: 0
|
|
877
|
+
}
|
|
878
|
+
});
|
|
879
|
+
const countData = await countResult.json();
|
|
880
|
+
const total = countData.data[0].total;
|
|
881
|
+
if (total === 0 && messages.length === 0) {
|
|
882
|
+
return {
|
|
883
|
+
messages: [],
|
|
884
|
+
total: 0,
|
|
885
|
+
page,
|
|
886
|
+
perPage,
|
|
887
|
+
hasMore: false
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const excludeIds = messages.map((m) => m.id);
|
|
891
|
+
let dataQuery = `
|
|
892
|
+
SELECT
|
|
893
|
+
id,
|
|
894
|
+
content,
|
|
895
|
+
role,
|
|
896
|
+
type,
|
|
897
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
898
|
+
thread_id AS "threadId",
|
|
899
|
+
resourceId
|
|
900
|
+
FROM ${TABLE_MESSAGES}
|
|
901
|
+
WHERE thread_id = {threadId:String}
|
|
902
|
+
`;
|
|
903
|
+
const dataParams = { threadId };
|
|
904
|
+
if (fromDate) {
|
|
905
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
906
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
907
|
+
}
|
|
908
|
+
if (toDate) {
|
|
909
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
910
|
+
dataParams.toDate = toDate.toISOString();
|
|
911
|
+
}
|
|
912
|
+
if (excludeIds.length > 0) {
|
|
913
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
914
|
+
dataParams.excludeIds = excludeIds;
|
|
915
|
+
}
|
|
916
|
+
if (selectBy?.last) {
|
|
917
|
+
dataQuery += `
|
|
918
|
+
ORDER BY createdAt DESC
|
|
919
|
+
LIMIT {limit:Int64}
|
|
920
|
+
`;
|
|
921
|
+
dataParams.limit = perPage;
|
|
922
|
+
} else {
|
|
923
|
+
dataQuery += `
|
|
924
|
+
ORDER BY createdAt ASC
|
|
925
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
926
|
+
`;
|
|
927
|
+
dataParams.limit = perPage;
|
|
928
|
+
dataParams.offset = offset;
|
|
929
|
+
}
|
|
930
|
+
const result = await this.client.query({
|
|
931
|
+
query: dataQuery,
|
|
932
|
+
query_params: dataParams,
|
|
933
|
+
clickhouse_settings: {
|
|
934
|
+
date_time_input_format: "best_effort",
|
|
935
|
+
date_time_output_format: "iso",
|
|
936
|
+
use_client_time_zone: 1,
|
|
937
|
+
output_format_json_quote_64bit_integers: 0
|
|
938
|
+
}
|
|
939
|
+
});
|
|
940
|
+
const rows = await result.json();
|
|
941
|
+
const paginatedMessages = transformRows(rows.data);
|
|
942
|
+
messages.push(...paginatedMessages);
|
|
943
|
+
if (selectBy?.last) {
|
|
944
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
945
|
+
}
|
|
946
|
+
return {
|
|
947
|
+
messages: format === "v2" ? messages : messages,
|
|
948
|
+
total,
|
|
949
|
+
page,
|
|
950
|
+
perPage,
|
|
951
|
+
hasMore: offset + perPage < total
|
|
952
|
+
};
|
|
953
|
+
} catch (error) {
|
|
954
|
+
throw new MastraError(
|
|
955
|
+
{
|
|
956
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
957
|
+
domain: ErrorDomain.STORAGE,
|
|
958
|
+
category: ErrorCategory.THIRD_PARTY
|
|
959
|
+
},
|
|
960
|
+
error
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
async updateMessages(args) {
|
|
965
|
+
const { messages } = args;
|
|
966
|
+
if (messages.length === 0) {
|
|
967
|
+
return [];
|
|
968
|
+
}
|
|
969
|
+
try {
|
|
970
|
+
const messageIds = messages.map((m) => m.id);
|
|
971
|
+
const existingResult = await this.client.query({
|
|
972
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
973
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
974
|
+
clickhouse_settings: {
|
|
975
|
+
date_time_input_format: "best_effort",
|
|
976
|
+
date_time_output_format: "iso",
|
|
977
|
+
use_client_time_zone: 1,
|
|
978
|
+
output_format_json_quote_64bit_integers: 0
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
const existingRows = await existingResult.json();
|
|
982
|
+
const existingMessages = transformRows(existingRows.data);
|
|
983
|
+
if (existingMessages.length === 0) {
|
|
984
|
+
return [];
|
|
985
|
+
}
|
|
986
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
987
|
+
if (typeof msg.content === "string") {
|
|
988
|
+
try {
|
|
989
|
+
msg.content = JSON.parse(msg.content);
|
|
990
|
+
} catch {
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return msg;
|
|
994
|
+
});
|
|
995
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
996
|
+
const updatePromises = [];
|
|
997
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
998
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
999
|
+
if (!updatePayload) continue;
|
|
1000
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1001
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1002
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1003
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1004
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1005
|
+
}
|
|
1006
|
+
const setClauses = [];
|
|
1007
|
+
const values = {};
|
|
1008
|
+
let paramIdx = 1;
|
|
1009
|
+
let newContent = null;
|
|
1010
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1011
|
+
if (updatableFields.content) {
|
|
1012
|
+
const existingContent = existingMessage.content || {};
|
|
1013
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1014
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1015
|
+
newContent = {
|
|
1016
|
+
...existingContent,
|
|
1017
|
+
...updatableFields.content,
|
|
1018
|
+
// Deep merge metadata
|
|
1019
|
+
metadata: {
|
|
1020
|
+
...existingMetadata,
|
|
1021
|
+
...updateMetadata
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1025
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1026
|
+
paramIdx++;
|
|
1027
|
+
delete updatableFields.content;
|
|
1028
|
+
}
|
|
1029
|
+
for (const key in updatableFields) {
|
|
1030
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1031
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1032
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1033
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1034
|
+
paramIdx++;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
if (setClauses.length > 0) {
|
|
1038
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1039
|
+
const updateQuery = `
|
|
1040
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1041
|
+
UPDATE ${setClauses.join(", ")}
|
|
1042
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1043
|
+
`;
|
|
1044
|
+
console.log("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1045
|
+
updatePromises.push(
|
|
1046
|
+
this.client.command({
|
|
1047
|
+
query: updateQuery,
|
|
1048
|
+
query_params: values,
|
|
1049
|
+
clickhouse_settings: {
|
|
1050
|
+
date_time_input_format: "best_effort",
|
|
1051
|
+
use_client_time_zone: 1,
|
|
1052
|
+
output_format_json_quote_64bit_integers: 0
|
|
1053
|
+
}
|
|
1054
|
+
})
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
if (updatePromises.length > 0) {
|
|
1059
|
+
await Promise.all(updatePromises);
|
|
1060
|
+
}
|
|
1061
|
+
await this.client.command({
|
|
1062
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1063
|
+
clickhouse_settings: {
|
|
1064
|
+
date_time_input_format: "best_effort",
|
|
1065
|
+
use_client_time_zone: 1,
|
|
1066
|
+
output_format_json_quote_64bit_integers: 0
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
1069
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1070
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1071
|
+
if (!updatePayload) continue;
|
|
1072
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1073
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1074
|
+
const verifyResult = await this.client.query({
|
|
1075
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1076
|
+
query_params: { messageId: id },
|
|
1077
|
+
clickhouse_settings: {
|
|
1078
|
+
date_time_input_format: "best_effort",
|
|
1079
|
+
date_time_output_format: "iso",
|
|
1080
|
+
use_client_time_zone: 1,
|
|
1081
|
+
output_format_json_quote_64bit_integers: 0
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
const verifyRows = await verifyResult.json();
|
|
1085
|
+
if (verifyRows.data.length > 0) {
|
|
1086
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1087
|
+
if (updatedMessage) {
|
|
1088
|
+
let needsRetry = false;
|
|
1089
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1090
|
+
if (key === "content") {
|
|
1091
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1092
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1093
|
+
if (actualContent !== expectedContent) {
|
|
1094
|
+
needsRetry = true;
|
|
1095
|
+
break;
|
|
1096
|
+
}
|
|
1097
|
+
} else if (updatedMessage[key] !== value) {
|
|
1098
|
+
needsRetry = true;
|
|
1099
|
+
break;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
if (needsRetry) {
|
|
1103
|
+
console.log("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1104
|
+
await this.client.command({
|
|
1105
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1106
|
+
query_params: { messageId: id },
|
|
1107
|
+
clickhouse_settings: {
|
|
1108
|
+
date_time_input_format: "best_effort",
|
|
1109
|
+
use_client_time_zone: 1,
|
|
1110
|
+
output_format_json_quote_64bit_integers: 0
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
let updatedContent = existingMessage.content || {};
|
|
1114
|
+
if (fieldsToUpdate.content) {
|
|
1115
|
+
const existingContent = existingMessage.content || {};
|
|
1116
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1117
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1118
|
+
updatedContent = {
|
|
1119
|
+
...existingContent,
|
|
1120
|
+
...fieldsToUpdate.content,
|
|
1121
|
+
metadata: {
|
|
1122
|
+
...existingMetadata,
|
|
1123
|
+
...updateMetadata
|
|
1124
|
+
}
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
const updatedMessageData = {
|
|
1128
|
+
...existingMessage,
|
|
1129
|
+
...fieldsToUpdate,
|
|
1130
|
+
content: updatedContent
|
|
1131
|
+
};
|
|
1132
|
+
await this.client.insert({
|
|
1133
|
+
table: TABLE_MESSAGES,
|
|
1134
|
+
format: "JSONEachRow",
|
|
1135
|
+
values: [
|
|
1136
|
+
{
|
|
1137
|
+
id: updatedMessageData.id,
|
|
1138
|
+
thread_id: updatedMessageData.threadId,
|
|
1139
|
+
resourceId: updatedMessageData.resourceId,
|
|
1140
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1141
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1142
|
+
role: updatedMessageData.role,
|
|
1143
|
+
type: updatedMessageData.type || "v2"
|
|
1144
|
+
}
|
|
1145
|
+
],
|
|
1146
|
+
clickhouse_settings: {
|
|
1147
|
+
date_time_input_format: "best_effort",
|
|
1148
|
+
use_client_time_zone: 1,
|
|
1149
|
+
output_format_json_quote_64bit_integers: 0
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1157
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1158
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1159
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1160
|
+
const threadResult = await this.client.query({
|
|
1161
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1162
|
+
query_params: { threadId },
|
|
1163
|
+
clickhouse_settings: {
|
|
1164
|
+
date_time_input_format: "best_effort",
|
|
1165
|
+
date_time_output_format: "iso",
|
|
1166
|
+
use_client_time_zone: 1,
|
|
1167
|
+
output_format_json_quote_64bit_integers: 0
|
|
1168
|
+
}
|
|
1169
|
+
});
|
|
1170
|
+
const threadRows = await threadResult.json();
|
|
1171
|
+
if (threadRows.data.length > 0) {
|
|
1172
|
+
const existingThread = threadRows.data[0];
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1175
|
+
query_params: { threadId },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
await this.client.insert({
|
|
1183
|
+
table: TABLE_THREADS,
|
|
1184
|
+
format: "JSONEachRow",
|
|
1185
|
+
values: [
|
|
1186
|
+
{
|
|
1187
|
+
id: existingThread.id,
|
|
1188
|
+
resourceId: existingThread.resourceId,
|
|
1189
|
+
title: existingThread.title,
|
|
1190
|
+
metadata: existingThread.metadata,
|
|
1191
|
+
createdAt: existingThread.createdAt,
|
|
1192
|
+
updatedAt: now
|
|
1193
|
+
}
|
|
1194
|
+
],
|
|
1195
|
+
clickhouse_settings: {
|
|
1196
|
+
date_time_input_format: "best_effort",
|
|
1197
|
+
use_client_time_zone: 1,
|
|
1198
|
+
output_format_json_quote_64bit_integers: 0
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
await Promise.all(threadUpdatePromises);
|
|
1204
|
+
}
|
|
1205
|
+
const updatedMessages = [];
|
|
1206
|
+
for (const messageId of messageIds) {
|
|
1207
|
+
const updatedResult = await this.client.query({
|
|
1208
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1209
|
+
query_params: { messageId },
|
|
1210
|
+
clickhouse_settings: {
|
|
1211
|
+
date_time_input_format: "best_effort",
|
|
1212
|
+
date_time_output_format: "iso",
|
|
1213
|
+
use_client_time_zone: 1,
|
|
1214
|
+
output_format_json_quote_64bit_integers: 0
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
const updatedRows = await updatedResult.json();
|
|
1218
|
+
if (updatedRows.data.length > 0) {
|
|
1219
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1220
|
+
if (message) {
|
|
1221
|
+
updatedMessages.push(message);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return updatedMessages.map((message) => {
|
|
813
1226
|
if (typeof message.content === "string") {
|
|
814
1227
|
try {
|
|
815
1228
|
message.content = JSON.parse(message.content);
|
|
816
1229
|
} catch {
|
|
817
1230
|
}
|
|
818
1231
|
}
|
|
1232
|
+
return message;
|
|
819
1233
|
});
|
|
820
|
-
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
821
|
-
if (format === `v2`) return list.get.all.v2();
|
|
822
|
-
return list.get.all.v1();
|
|
823
1234
|
} catch (error) {
|
|
824
1235
|
throw new MastraError(
|
|
825
1236
|
{
|
|
826
|
-
id: "
|
|
1237
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
827
1238
|
domain: ErrorDomain.STORAGE,
|
|
828
1239
|
category: ErrorCategory.THIRD_PARTY,
|
|
829
|
-
details: {
|
|
1240
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
830
1241
|
},
|
|
831
1242
|
error
|
|
832
1243
|
);
|
|
833
1244
|
}
|
|
834
1245
|
}
|
|
835
|
-
async
|
|
836
|
-
const { messages, format = "v1" } = args;
|
|
837
|
-
if (messages.length === 0) return messages;
|
|
1246
|
+
async getResourceById({ resourceId }) {
|
|
838
1247
|
try {
|
|
839
|
-
const
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1248
|
+
const result = await this.client.query({
|
|
1249
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1250
|
+
query_params: { resourceId },
|
|
1251
|
+
clickhouse_settings: {
|
|
1252
|
+
date_time_input_format: "best_effort",
|
|
1253
|
+
date_time_output_format: "iso",
|
|
1254
|
+
use_client_time_zone: 1,
|
|
1255
|
+
output_format_json_quote_64bit_integers: 0
|
|
1256
|
+
}
|
|
1257
|
+
});
|
|
1258
|
+
const rows = await result.json();
|
|
1259
|
+
if (rows.data.length === 0) {
|
|
1260
|
+
return null;
|
|
843
1261
|
}
|
|
844
|
-
const
|
|
845
|
-
|
|
846
|
-
|
|
1262
|
+
const resource = rows.data[0];
|
|
1263
|
+
return {
|
|
1264
|
+
id: resource.id,
|
|
1265
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1266
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1267
|
+
createdAt: new Date(resource.createdAt),
|
|
1268
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1269
|
+
};
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
throw new MastraError(
|
|
1272
|
+
{
|
|
1273
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1274
|
+
domain: ErrorDomain.STORAGE,
|
|
1275
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1276
|
+
details: { resourceId }
|
|
1277
|
+
},
|
|
1278
|
+
error
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
async saveResource({ resource }) {
|
|
1283
|
+
try {
|
|
1284
|
+
await this.client.insert({
|
|
1285
|
+
table: TABLE_RESOURCES,
|
|
1286
|
+
format: "JSONEachRow",
|
|
1287
|
+
values: [
|
|
1288
|
+
{
|
|
1289
|
+
id: resource.id,
|
|
1290
|
+
workingMemory: resource.workingMemory,
|
|
1291
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1292
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1293
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1294
|
+
}
|
|
1295
|
+
],
|
|
1296
|
+
clickhouse_settings: {
|
|
1297
|
+
date_time_input_format: "best_effort",
|
|
1298
|
+
use_client_time_zone: 1,
|
|
1299
|
+
output_format_json_quote_64bit_integers: 0
|
|
1300
|
+
}
|
|
1301
|
+
});
|
|
1302
|
+
return resource;
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { resourceId: resource.id }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
async updateResource({
|
|
1316
|
+
resourceId,
|
|
1317
|
+
workingMemory,
|
|
1318
|
+
metadata
|
|
1319
|
+
}) {
|
|
1320
|
+
try {
|
|
1321
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1322
|
+
if (!existingResource) {
|
|
1323
|
+
const newResource = {
|
|
1324
|
+
id: resourceId,
|
|
1325
|
+
workingMemory,
|
|
1326
|
+
metadata: metadata || {},
|
|
1327
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1328
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1329
|
+
};
|
|
1330
|
+
return this.saveResource({ resource: newResource });
|
|
847
1331
|
}
|
|
848
|
-
const
|
|
849
|
-
|
|
1332
|
+
const updatedResource = {
|
|
1333
|
+
...existingResource,
|
|
1334
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1335
|
+
metadata: {
|
|
1336
|
+
...existingResource.metadata,
|
|
1337
|
+
...metadata
|
|
1338
|
+
},
|
|
1339
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1340
|
+
};
|
|
1341
|
+
const updateQuery = `
|
|
1342
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1343
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1344
|
+
WHERE id = {resourceId:String}
|
|
1345
|
+
`;
|
|
1346
|
+
await this.client.command({
|
|
1347
|
+
query: updateQuery,
|
|
850
1348
|
query_params: {
|
|
851
|
-
|
|
1349
|
+
workingMemory: updatedResource.workingMemory,
|
|
1350
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1351
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1352
|
+
resourceId
|
|
1353
|
+
},
|
|
1354
|
+
clickhouse_settings: {
|
|
1355
|
+
date_time_input_format: "best_effort",
|
|
1356
|
+
use_client_time_zone: 1,
|
|
1357
|
+
output_format_json_quote_64bit_integers: 0
|
|
1358
|
+
}
|
|
1359
|
+
});
|
|
1360
|
+
await this.client.command({
|
|
1361
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1362
|
+
clickhouse_settings: {
|
|
1363
|
+
date_time_input_format: "best_effort",
|
|
1364
|
+
use_client_time_zone: 1,
|
|
1365
|
+
output_format_json_quote_64bit_integers: 0
|
|
1366
|
+
}
|
|
1367
|
+
});
|
|
1368
|
+
return updatedResource;
|
|
1369
|
+
} catch (error) {
|
|
1370
|
+
throw new MastraError(
|
|
1371
|
+
{
|
|
1372
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1373
|
+
domain: ErrorDomain.STORAGE,
|
|
1374
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1375
|
+
details: { resourceId }
|
|
852
1376
|
},
|
|
1377
|
+
error
|
|
1378
|
+
);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
};
|
|
1382
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1383
|
+
ttl;
|
|
1384
|
+
client;
|
|
1385
|
+
constructor({ client, ttl }) {
|
|
1386
|
+
super();
|
|
1387
|
+
this.ttl = ttl;
|
|
1388
|
+
this.client = client;
|
|
1389
|
+
}
|
|
1390
|
+
async hasColumn(table, column) {
|
|
1391
|
+
const result = await this.client.query({
|
|
1392
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1393
|
+
format: "JSONEachRow"
|
|
1394
|
+
});
|
|
1395
|
+
const columns = await result.json();
|
|
1396
|
+
return columns.some((c) => c.name === column);
|
|
1397
|
+
}
|
|
1398
|
+
getSqlType(type) {
|
|
1399
|
+
switch (type) {
|
|
1400
|
+
case "text":
|
|
1401
|
+
return "String";
|
|
1402
|
+
case "timestamp":
|
|
1403
|
+
return "DateTime64(3)";
|
|
1404
|
+
case "integer":
|
|
1405
|
+
case "bigint":
|
|
1406
|
+
return "Int64";
|
|
1407
|
+
case "jsonb":
|
|
1408
|
+
return "String";
|
|
1409
|
+
default:
|
|
1410
|
+
return super.getSqlType(type);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
async createTable({
|
|
1414
|
+
tableName,
|
|
1415
|
+
schema
|
|
1416
|
+
}) {
|
|
1417
|
+
try {
|
|
1418
|
+
const columns = Object.entries(schema).map(([name, def]) => {
|
|
1419
|
+
const constraints = [];
|
|
1420
|
+
if (!def.nullable) constraints.push("NOT NULL");
|
|
1421
|
+
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
1422
|
+
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
|
|
1423
|
+
}).join(",\n");
|
|
1424
|
+
const rowTtl = this.ttl?.[tableName]?.row;
|
|
1425
|
+
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
1426
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1427
|
+
${["id String"].concat(columns)}
|
|
1428
|
+
)
|
|
1429
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1430
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1431
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1432
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1433
|
+
SETTINGS index_granularity = 8192
|
|
1434
|
+
` : `
|
|
1435
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1436
|
+
${columns}
|
|
1437
|
+
)
|
|
1438
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1439
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1440
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1441
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1442
|
+
SETTINGS index_granularity = 8192
|
|
1443
|
+
`;
|
|
1444
|
+
await this.client.query({
|
|
1445
|
+
query: sql,
|
|
1446
|
+
clickhouse_settings: {
|
|
1447
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1448
|
+
date_time_input_format: "best_effort",
|
|
1449
|
+
date_time_output_format: "iso",
|
|
1450
|
+
use_client_time_zone: 1,
|
|
1451
|
+
output_format_json_quote_64bit_integers: 0
|
|
1452
|
+
}
|
|
1453
|
+
});
|
|
1454
|
+
} catch (error) {
|
|
1455
|
+
throw new MastraError(
|
|
1456
|
+
{
|
|
1457
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1458
|
+
domain: ErrorDomain.STORAGE,
|
|
1459
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1460
|
+
details: { tableName }
|
|
1461
|
+
},
|
|
1462
|
+
error
|
|
1463
|
+
);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
async alterTable({
|
|
1467
|
+
tableName,
|
|
1468
|
+
schema,
|
|
1469
|
+
ifNotExists
|
|
1470
|
+
}) {
|
|
1471
|
+
try {
|
|
1472
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1473
|
+
const result = await this.client.query({
|
|
1474
|
+
query: describeSql
|
|
1475
|
+
});
|
|
1476
|
+
const rows = await result.json();
|
|
1477
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1478
|
+
for (const columnName of ifNotExists) {
|
|
1479
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1480
|
+
const columnDef = schema[columnName];
|
|
1481
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1482
|
+
if (columnDef.nullable !== false) {
|
|
1483
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1484
|
+
}
|
|
1485
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1486
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1487
|
+
await this.client.query({
|
|
1488
|
+
query: alterSql
|
|
1489
|
+
});
|
|
1490
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
throw new MastraError(
|
|
1495
|
+
{
|
|
1496
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1497
|
+
domain: ErrorDomain.STORAGE,
|
|
1498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1499
|
+
details: { tableName }
|
|
1500
|
+
},
|
|
1501
|
+
error
|
|
1502
|
+
);
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
async clearTable({ tableName }) {
|
|
1506
|
+
try {
|
|
1507
|
+
await this.client.query({
|
|
1508
|
+
query: `TRUNCATE TABLE ${tableName}`,
|
|
1509
|
+
clickhouse_settings: {
|
|
1510
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1511
|
+
date_time_input_format: "best_effort",
|
|
1512
|
+
date_time_output_format: "iso",
|
|
1513
|
+
use_client_time_zone: 1,
|
|
1514
|
+
output_format_json_quote_64bit_integers: 0
|
|
1515
|
+
}
|
|
1516
|
+
});
|
|
1517
|
+
} catch (error) {
|
|
1518
|
+
throw new MastraError(
|
|
1519
|
+
{
|
|
1520
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1521
|
+
domain: ErrorDomain.STORAGE,
|
|
1522
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1523
|
+
details: { tableName }
|
|
1524
|
+
},
|
|
1525
|
+
error
|
|
1526
|
+
);
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
async dropTable({ tableName }) {
|
|
1530
|
+
await this.client.query({
|
|
1531
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
1534
|
+
async insert({ tableName, record }) {
|
|
1535
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1536
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
1537
|
+
try {
|
|
1538
|
+
const result = await this.client.insert({
|
|
1539
|
+
table: tableName,
|
|
1540
|
+
values: [
|
|
1541
|
+
{
|
|
1542
|
+
...record,
|
|
1543
|
+
createdAt,
|
|
1544
|
+
updatedAt
|
|
1545
|
+
}
|
|
1546
|
+
],
|
|
1547
|
+
format: "JSONEachRow",
|
|
1548
|
+
clickhouse_settings: {
|
|
1549
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1550
|
+
output_format_json_quote_64bit_integers: 0,
|
|
1551
|
+
date_time_input_format: "best_effort",
|
|
1552
|
+
use_client_time_zone: 1
|
|
1553
|
+
}
|
|
1554
|
+
});
|
|
1555
|
+
console.log("INSERT RESULT", result);
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
throw new MastraError(
|
|
1558
|
+
{
|
|
1559
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1560
|
+
domain: ErrorDomain.STORAGE,
|
|
1561
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1562
|
+
details: { tableName }
|
|
1563
|
+
},
|
|
1564
|
+
error
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
async batchInsert({ tableName, records }) {
|
|
1569
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1570
|
+
...Object.fromEntries(
|
|
1571
|
+
Object.entries(record).map(([key, value]) => [
|
|
1572
|
+
key,
|
|
1573
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1574
|
+
])
|
|
1575
|
+
)
|
|
1576
|
+
}));
|
|
1577
|
+
try {
|
|
1578
|
+
await this.client.insert({
|
|
1579
|
+
table: tableName,
|
|
1580
|
+
values: recordsToBeInserted,
|
|
1581
|
+
format: "JSONEachRow",
|
|
1582
|
+
clickhouse_settings: {
|
|
1583
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1584
|
+
date_time_input_format: "best_effort",
|
|
1585
|
+
use_client_time_zone: 1,
|
|
1586
|
+
output_format_json_quote_64bit_integers: 0
|
|
1587
|
+
}
|
|
1588
|
+
});
|
|
1589
|
+
} catch (error) {
|
|
1590
|
+
throw new MastraError(
|
|
1591
|
+
{
|
|
1592
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1593
|
+
domain: ErrorDomain.STORAGE,
|
|
1594
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1595
|
+
details: { tableName }
|
|
1596
|
+
},
|
|
1597
|
+
error
|
|
1598
|
+
);
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
async load({ tableName, keys }) {
|
|
1602
|
+
try {
|
|
1603
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
1604
|
+
const keyEntries = Object.entries(keys);
|
|
1605
|
+
const conditions = keyEntries.map(
|
|
1606
|
+
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
1607
|
+
).join(" AND ");
|
|
1608
|
+
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
1609
|
+
return { ...acc, [`var_${key}`]: value };
|
|
1610
|
+
}, {});
|
|
1611
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1612
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1613
|
+
const result = await this.client.query({
|
|
1614
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
1615
|
+
query_params: values,
|
|
1616
|
+
clickhouse_settings: {
|
|
1617
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1618
|
+
date_time_input_format: "best_effort",
|
|
1619
|
+
date_time_output_format: "iso",
|
|
1620
|
+
use_client_time_zone: 1,
|
|
1621
|
+
output_format_json_quote_64bit_integers: 0
|
|
1622
|
+
}
|
|
1623
|
+
});
|
|
1624
|
+
if (!result) {
|
|
1625
|
+
return null;
|
|
1626
|
+
}
|
|
1627
|
+
const rows = await result.json();
|
|
1628
|
+
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
1629
|
+
const snapshot = rows.data[0];
|
|
1630
|
+
if (!snapshot) {
|
|
1631
|
+
return null;
|
|
1632
|
+
}
|
|
1633
|
+
if (typeof snapshot.snapshot === "string") {
|
|
1634
|
+
snapshot.snapshot = JSON.parse(snapshot.snapshot);
|
|
1635
|
+
}
|
|
1636
|
+
return transformRow(snapshot);
|
|
1637
|
+
}
|
|
1638
|
+
const data = transformRow(rows.data[0]);
|
|
1639
|
+
return data;
|
|
1640
|
+
} catch (error) {
|
|
1641
|
+
throw new MastraError(
|
|
1642
|
+
{
|
|
1643
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1644
|
+
domain: ErrorDomain.STORAGE,
|
|
1645
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1646
|
+
details: { tableName }
|
|
1647
|
+
},
|
|
1648
|
+
error
|
|
1649
|
+
);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
};
|
|
1653
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1654
|
+
client;
|
|
1655
|
+
operations;
|
|
1656
|
+
constructor({ client, operations }) {
|
|
1657
|
+
super();
|
|
1658
|
+
this.client = client;
|
|
1659
|
+
this.operations = operations;
|
|
1660
|
+
}
|
|
1661
|
+
transformScoreRow(row) {
|
|
1662
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1663
|
+
const extractStepResult = safelyParseJSON(row.extractStepResult);
|
|
1664
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1665
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1666
|
+
const input = safelyParseJSON(row.input);
|
|
1667
|
+
const output = safelyParseJSON(row.output);
|
|
1668
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1669
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1670
|
+
const entity = safelyParseJSON(row.entity);
|
|
1671
|
+
return {
|
|
1672
|
+
...row,
|
|
1673
|
+
scorer,
|
|
1674
|
+
extractStepResult,
|
|
1675
|
+
analyzeStepResult,
|
|
1676
|
+
metadata,
|
|
1677
|
+
input,
|
|
1678
|
+
output,
|
|
1679
|
+
additionalContext,
|
|
1680
|
+
runtimeContext,
|
|
1681
|
+
entity,
|
|
1682
|
+
createdAt: new Date(row.createdAt),
|
|
1683
|
+
updatedAt: new Date(row.updatedAt)
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
async getScoreById({ id }) {
|
|
1687
|
+
try {
|
|
1688
|
+
const result = await this.client.query({
|
|
1689
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1690
|
+
query_params: { var_id: id },
|
|
1691
|
+
format: "JSONEachRow",
|
|
1692
|
+
clickhouse_settings: {
|
|
1693
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1694
|
+
date_time_input_format: "best_effort",
|
|
1695
|
+
date_time_output_format: "iso",
|
|
1696
|
+
use_client_time_zone: 1,
|
|
1697
|
+
output_format_json_quote_64bit_integers: 0
|
|
1698
|
+
}
|
|
1699
|
+
});
|
|
1700
|
+
const resultJson = await result.json();
|
|
1701
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
1702
|
+
return null;
|
|
1703
|
+
}
|
|
1704
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1705
|
+
} catch (error) {
|
|
1706
|
+
throw new MastraError(
|
|
1707
|
+
{
|
|
1708
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1709
|
+
domain: ErrorDomain.STORAGE,
|
|
1710
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1711
|
+
details: { scoreId: id }
|
|
1712
|
+
},
|
|
1713
|
+
error
|
|
1714
|
+
);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
async saveScore(score) {
|
|
1718
|
+
try {
|
|
1719
|
+
const record = {
|
|
1720
|
+
...score
|
|
1721
|
+
};
|
|
1722
|
+
await this.client.insert({
|
|
1723
|
+
table: TABLE_SCORERS,
|
|
1724
|
+
values: [record],
|
|
1725
|
+
format: "JSONEachRow",
|
|
1726
|
+
clickhouse_settings: {
|
|
1727
|
+
date_time_input_format: "best_effort",
|
|
1728
|
+
use_client_time_zone: 1,
|
|
1729
|
+
output_format_json_quote_64bit_integers: 0
|
|
1730
|
+
}
|
|
1731
|
+
});
|
|
1732
|
+
return { score };
|
|
1733
|
+
} catch (error) {
|
|
1734
|
+
throw new MastraError(
|
|
1735
|
+
{
|
|
1736
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1737
|
+
domain: ErrorDomain.STORAGE,
|
|
1738
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1739
|
+
details: { scoreId: score.id }
|
|
1740
|
+
},
|
|
1741
|
+
error
|
|
1742
|
+
);
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
async getScoresByRunId({
|
|
1746
|
+
runId,
|
|
1747
|
+
pagination
|
|
1748
|
+
}) {
|
|
1749
|
+
try {
|
|
1750
|
+
const countResult = await this.client.query({
|
|
1751
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1752
|
+
query_params: { var_runId: runId },
|
|
1753
|
+
format: "JSONEachRow"
|
|
1754
|
+
});
|
|
1755
|
+
const countRows = await countResult.json();
|
|
1756
|
+
let total = 0;
|
|
1757
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1758
|
+
const countObj = countRows[0];
|
|
1759
|
+
total = Number(countObj.count);
|
|
1760
|
+
}
|
|
1761
|
+
if (!total) {
|
|
1762
|
+
return {
|
|
1763
|
+
pagination: {
|
|
1764
|
+
total: 0,
|
|
1765
|
+
page: pagination.page,
|
|
1766
|
+
perPage: pagination.perPage,
|
|
1767
|
+
hasMore: false
|
|
1768
|
+
},
|
|
1769
|
+
scores: []
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
const offset = pagination.page * pagination.perPage;
|
|
1773
|
+
const result = await this.client.query({
|
|
1774
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1775
|
+
query_params: {
|
|
1776
|
+
var_runId: runId,
|
|
1777
|
+
var_limit: pagination.perPage,
|
|
1778
|
+
var_offset: offset
|
|
1779
|
+
},
|
|
1780
|
+
format: "JSONEachRow",
|
|
1781
|
+
clickhouse_settings: {
|
|
1782
|
+
date_time_input_format: "best_effort",
|
|
1783
|
+
date_time_output_format: "iso",
|
|
1784
|
+
use_client_time_zone: 1,
|
|
1785
|
+
output_format_json_quote_64bit_integers: 0
|
|
1786
|
+
}
|
|
1787
|
+
});
|
|
1788
|
+
const rows = await result.json();
|
|
1789
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1790
|
+
return {
|
|
1791
|
+
pagination: {
|
|
1792
|
+
total,
|
|
1793
|
+
page: pagination.page,
|
|
1794
|
+
perPage: pagination.perPage,
|
|
1795
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1796
|
+
},
|
|
1797
|
+
scores
|
|
1798
|
+
};
|
|
1799
|
+
} catch (error) {
|
|
1800
|
+
throw new MastraError(
|
|
1801
|
+
{
|
|
1802
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1803
|
+
domain: ErrorDomain.STORAGE,
|
|
1804
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1805
|
+
details: { runId }
|
|
1806
|
+
},
|
|
1807
|
+
error
|
|
1808
|
+
);
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
async getScoresByScorerId({
|
|
1812
|
+
scorerId,
|
|
1813
|
+
pagination
|
|
1814
|
+
}) {
|
|
1815
|
+
try {
|
|
1816
|
+
const countResult = await this.client.query({
|
|
1817
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String}`,
|
|
1818
|
+
query_params: { var_scorerId: scorerId },
|
|
1819
|
+
format: "JSONEachRow"
|
|
1820
|
+
});
|
|
1821
|
+
const countRows = await countResult.json();
|
|
1822
|
+
let total = 0;
|
|
1823
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1824
|
+
const countObj = countRows[0];
|
|
1825
|
+
total = Number(countObj.count);
|
|
1826
|
+
}
|
|
1827
|
+
if (!total) {
|
|
1828
|
+
return {
|
|
1829
|
+
pagination: {
|
|
1830
|
+
total: 0,
|
|
1831
|
+
page: pagination.page,
|
|
1832
|
+
perPage: pagination.perPage,
|
|
1833
|
+
hasMore: false
|
|
1834
|
+
},
|
|
1835
|
+
scores: []
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
const offset = pagination.page * pagination.perPage;
|
|
1839
|
+
const result = await this.client.query({
|
|
1840
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1841
|
+
query_params: {
|
|
1842
|
+
var_scorerId: scorerId,
|
|
1843
|
+
var_limit: pagination.perPage,
|
|
1844
|
+
var_offset: offset
|
|
1845
|
+
},
|
|
1846
|
+
format: "JSONEachRow",
|
|
1847
|
+
clickhouse_settings: {
|
|
1848
|
+
date_time_input_format: "best_effort",
|
|
1849
|
+
date_time_output_format: "iso",
|
|
1850
|
+
use_client_time_zone: 1,
|
|
1851
|
+
output_format_json_quote_64bit_integers: 0
|
|
1852
|
+
}
|
|
1853
|
+
});
|
|
1854
|
+
const rows = await result.json();
|
|
1855
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1856
|
+
return {
|
|
1857
|
+
pagination: {
|
|
1858
|
+
total,
|
|
1859
|
+
page: pagination.page,
|
|
1860
|
+
perPage: pagination.perPage,
|
|
1861
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1862
|
+
},
|
|
1863
|
+
scores
|
|
1864
|
+
};
|
|
1865
|
+
} catch (error) {
|
|
1866
|
+
throw new MastraError(
|
|
1867
|
+
{
|
|
1868
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1869
|
+
domain: ErrorDomain.STORAGE,
|
|
1870
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1871
|
+
details: { scorerId }
|
|
1872
|
+
},
|
|
1873
|
+
error
|
|
1874
|
+
);
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
async getScoresByEntityId({
|
|
1878
|
+
entityId,
|
|
1879
|
+
entityType,
|
|
1880
|
+
pagination
|
|
1881
|
+
}) {
|
|
1882
|
+
try {
|
|
1883
|
+
const countResult = await this.client.query({
|
|
1884
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1885
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1886
|
+
format: "JSONEachRow"
|
|
1887
|
+
});
|
|
1888
|
+
const countRows = await countResult.json();
|
|
1889
|
+
let total = 0;
|
|
1890
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1891
|
+
const countObj = countRows[0];
|
|
1892
|
+
total = Number(countObj.count);
|
|
1893
|
+
}
|
|
1894
|
+
if (!total) {
|
|
1895
|
+
return {
|
|
1896
|
+
pagination: {
|
|
1897
|
+
total: 0,
|
|
1898
|
+
page: pagination.page,
|
|
1899
|
+
perPage: pagination.perPage,
|
|
1900
|
+
hasMore: false
|
|
1901
|
+
},
|
|
1902
|
+
scores: []
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
const offset = pagination.page * pagination.perPage;
|
|
1906
|
+
const result = await this.client.query({
|
|
1907
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1908
|
+
query_params: {
|
|
1909
|
+
var_entityId: entityId,
|
|
1910
|
+
var_entityType: entityType,
|
|
1911
|
+
var_limit: pagination.perPage,
|
|
1912
|
+
var_offset: offset
|
|
1913
|
+
},
|
|
1914
|
+
format: "JSONEachRow",
|
|
1915
|
+
clickhouse_settings: {
|
|
1916
|
+
date_time_input_format: "best_effort",
|
|
1917
|
+
date_time_output_format: "iso",
|
|
1918
|
+
use_client_time_zone: 1,
|
|
1919
|
+
output_format_json_quote_64bit_integers: 0
|
|
1920
|
+
}
|
|
1921
|
+
});
|
|
1922
|
+
const rows = await result.json();
|
|
1923
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1924
|
+
return {
|
|
1925
|
+
pagination: {
|
|
1926
|
+
total,
|
|
1927
|
+
page: pagination.page,
|
|
1928
|
+
perPage: pagination.perPage,
|
|
1929
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1930
|
+
},
|
|
1931
|
+
scores
|
|
1932
|
+
};
|
|
1933
|
+
} catch (error) {
|
|
1934
|
+
throw new MastraError(
|
|
1935
|
+
{
|
|
1936
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1937
|
+
domain: ErrorDomain.STORAGE,
|
|
1938
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1939
|
+
details: { entityId, entityType }
|
|
1940
|
+
},
|
|
1941
|
+
error
|
|
1942
|
+
);
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
};
|
|
1946
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
1947
|
+
client;
|
|
1948
|
+
operations;
|
|
1949
|
+
constructor({ client, operations }) {
|
|
1950
|
+
super();
|
|
1951
|
+
this.client = client;
|
|
1952
|
+
this.operations = operations;
|
|
1953
|
+
}
|
|
1954
|
+
async getTracesPaginated(args) {
|
|
1955
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
1956
|
+
const fromDate = dateRange?.start;
|
|
1957
|
+
const toDate = dateRange?.end;
|
|
1958
|
+
const currentOffset = page * perPage;
|
|
1959
|
+
const queryArgs = {};
|
|
1960
|
+
const conditions = [];
|
|
1961
|
+
if (name) {
|
|
1962
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
1963
|
+
queryArgs.var_name = name;
|
|
1964
|
+
}
|
|
1965
|
+
if (scope) {
|
|
1966
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
1967
|
+
queryArgs.var_scope = scope;
|
|
1968
|
+
}
|
|
1969
|
+
if (attributes) {
|
|
1970
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
1971
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
1972
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
if (filters) {
|
|
1976
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
1977
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
1978
|
+
queryArgs[`var_col_${key}`] = value;
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
if (fromDate) {
|
|
1982
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
1983
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
1984
|
+
}
|
|
1985
|
+
if (toDate) {
|
|
1986
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
1987
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
1988
|
+
}
|
|
1989
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1990
|
+
try {
|
|
1991
|
+
const countResult = await this.client.query({
|
|
1992
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
1993
|
+
query_params: queryArgs,
|
|
1994
|
+
clickhouse_settings: {
|
|
1995
|
+
date_time_input_format: "best_effort",
|
|
1996
|
+
date_time_output_format: "iso",
|
|
1997
|
+
use_client_time_zone: 1,
|
|
1998
|
+
output_format_json_quote_64bit_integers: 0
|
|
1999
|
+
}
|
|
2000
|
+
});
|
|
2001
|
+
const countData = await countResult.json();
|
|
2002
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2003
|
+
if (total === 0) {
|
|
2004
|
+
return {
|
|
2005
|
+
traces: [],
|
|
2006
|
+
total: 0,
|
|
2007
|
+
page,
|
|
2008
|
+
perPage,
|
|
2009
|
+
hasMore: false
|
|
2010
|
+
};
|
|
2011
|
+
}
|
|
2012
|
+
const result = await this.client.query({
|
|
2013
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2014
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2015
|
+
clickhouse_settings: {
|
|
2016
|
+
date_time_input_format: "best_effort",
|
|
2017
|
+
date_time_output_format: "iso",
|
|
2018
|
+
use_client_time_zone: 1,
|
|
2019
|
+
output_format_json_quote_64bit_integers: 0
|
|
2020
|
+
}
|
|
2021
|
+
});
|
|
2022
|
+
if (!result) {
|
|
2023
|
+
return {
|
|
2024
|
+
traces: [],
|
|
2025
|
+
total,
|
|
2026
|
+
page,
|
|
2027
|
+
perPage,
|
|
2028
|
+
hasMore: false
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
const resp = await result.json();
|
|
2032
|
+
const rows = resp.data;
|
|
2033
|
+
const traces = rows.map((row) => ({
|
|
2034
|
+
id: row.id,
|
|
2035
|
+
parentSpanId: row.parentSpanId,
|
|
2036
|
+
traceId: row.traceId,
|
|
2037
|
+
name: row.name,
|
|
2038
|
+
scope: row.scope,
|
|
2039
|
+
kind: row.kind,
|
|
2040
|
+
status: safelyParseJSON(row.status),
|
|
2041
|
+
events: safelyParseJSON(row.events),
|
|
2042
|
+
links: safelyParseJSON(row.links),
|
|
2043
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2044
|
+
startTime: row.startTime,
|
|
2045
|
+
endTime: row.endTime,
|
|
2046
|
+
other: safelyParseJSON(row.other),
|
|
2047
|
+
createdAt: row.createdAt
|
|
2048
|
+
}));
|
|
2049
|
+
return {
|
|
2050
|
+
traces,
|
|
2051
|
+
total,
|
|
2052
|
+
page,
|
|
2053
|
+
perPage,
|
|
2054
|
+
hasMore: currentOffset + traces.length < total
|
|
2055
|
+
};
|
|
2056
|
+
} catch (error) {
|
|
2057
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2058
|
+
return {
|
|
2059
|
+
traces: [],
|
|
2060
|
+
total: 0,
|
|
2061
|
+
page,
|
|
2062
|
+
perPage,
|
|
2063
|
+
hasMore: false
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
throw new MastraError(
|
|
2067
|
+
{
|
|
2068
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2069
|
+
domain: ErrorDomain.STORAGE,
|
|
2070
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2071
|
+
details: {
|
|
2072
|
+
name: name ?? null,
|
|
2073
|
+
scope: scope ?? null,
|
|
2074
|
+
page,
|
|
2075
|
+
perPage,
|
|
2076
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2077
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2078
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2079
|
+
}
|
|
2080
|
+
},
|
|
2081
|
+
error
|
|
2082
|
+
);
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
async getTraces({
|
|
2086
|
+
name,
|
|
2087
|
+
scope,
|
|
2088
|
+
page,
|
|
2089
|
+
perPage,
|
|
2090
|
+
attributes,
|
|
2091
|
+
filters,
|
|
2092
|
+
fromDate,
|
|
2093
|
+
toDate
|
|
2094
|
+
}) {
|
|
2095
|
+
const limit = perPage;
|
|
2096
|
+
const offset = page * perPage;
|
|
2097
|
+
const args = {};
|
|
2098
|
+
const conditions = [];
|
|
2099
|
+
if (name) {
|
|
2100
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2101
|
+
args.var_name = name;
|
|
2102
|
+
}
|
|
2103
|
+
if (scope) {
|
|
2104
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2105
|
+
args.var_scope = scope;
|
|
2106
|
+
}
|
|
2107
|
+
if (attributes) {
|
|
2108
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2109
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2110
|
+
args[`var_attr_${key}`] = value;
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
if (filters) {
|
|
2114
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2115
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2116
|
+
args[`var_col_${key}`] = value;
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
if (fromDate) {
|
|
2120
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2121
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2122
|
+
}
|
|
2123
|
+
if (toDate) {
|
|
2124
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2125
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2126
|
+
}
|
|
2127
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2128
|
+
try {
|
|
2129
|
+
const result = await this.client.query({
|
|
2130
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2131
|
+
query_params: args,
|
|
853
2132
|
clickhouse_settings: {
|
|
854
2133
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
855
2134
|
date_time_input_format: "best_effort",
|
|
856
2135
|
date_time_output_format: "iso",
|
|
857
2136
|
use_client_time_zone: 1,
|
|
858
2137
|
output_format_json_quote_64bit_integers: 0
|
|
859
|
-
}
|
|
860
|
-
format: "JSONEachRow"
|
|
2138
|
+
}
|
|
861
2139
|
});
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
const
|
|
866
|
-
const
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
use_client_time_zone: 1,
|
|
884
|
-
output_format_json_quote_64bit_integers: 0
|
|
885
|
-
}
|
|
886
|
-
})
|
|
887
|
-
);
|
|
888
|
-
await Promise.all([
|
|
889
|
-
// Insert messages
|
|
890
|
-
this.db.insert({
|
|
891
|
-
table: TABLE_MESSAGES,
|
|
892
|
-
format: "JSONEachRow",
|
|
893
|
-
values: toInsert.map((message) => ({
|
|
894
|
-
id: message.id,
|
|
895
|
-
thread_id: threadId,
|
|
896
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
897
|
-
createdAt: message.createdAt.toISOString(),
|
|
898
|
-
role: message.role,
|
|
899
|
-
type: message.type || "v2"
|
|
900
|
-
})),
|
|
901
|
-
clickhouse_settings: {
|
|
902
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
903
|
-
date_time_input_format: "best_effort",
|
|
904
|
-
use_client_time_zone: 1,
|
|
905
|
-
output_format_json_quote_64bit_integers: 0
|
|
906
|
-
}
|
|
907
|
-
}),
|
|
908
|
-
...updatePromises,
|
|
909
|
-
// Update thread's updatedAt timestamp
|
|
910
|
-
this.db.insert({
|
|
911
|
-
table: TABLE_THREADS,
|
|
912
|
-
format: "JSONEachRow",
|
|
913
|
-
values: [
|
|
914
|
-
{
|
|
915
|
-
id: thread.id,
|
|
916
|
-
resourceId: thread.resourceId,
|
|
917
|
-
title: thread.title,
|
|
918
|
-
metadata: thread.metadata,
|
|
919
|
-
createdAt: thread.createdAt,
|
|
920
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
921
|
-
}
|
|
922
|
-
],
|
|
923
|
-
clickhouse_settings: {
|
|
924
|
-
date_time_input_format: "best_effort",
|
|
925
|
-
use_client_time_zone: 1,
|
|
926
|
-
output_format_json_quote_64bit_integers: 0
|
|
927
|
-
}
|
|
928
|
-
})
|
|
929
|
-
]);
|
|
930
|
-
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
931
|
-
if (format === `v2`) return list.get.all.v2();
|
|
932
|
-
return list.get.all.v1();
|
|
2140
|
+
if (!result) {
|
|
2141
|
+
return [];
|
|
2142
|
+
}
|
|
2143
|
+
const resp = await result.json();
|
|
2144
|
+
const rows = resp.data;
|
|
2145
|
+
return rows.map((row) => ({
|
|
2146
|
+
id: row.id,
|
|
2147
|
+
parentSpanId: row.parentSpanId,
|
|
2148
|
+
traceId: row.traceId,
|
|
2149
|
+
name: row.name,
|
|
2150
|
+
scope: row.scope,
|
|
2151
|
+
kind: row.kind,
|
|
2152
|
+
status: safelyParseJSON(row.status),
|
|
2153
|
+
events: safelyParseJSON(row.events),
|
|
2154
|
+
links: safelyParseJSON(row.links),
|
|
2155
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2156
|
+
startTime: row.startTime,
|
|
2157
|
+
endTime: row.endTime,
|
|
2158
|
+
other: safelyParseJSON(row.other),
|
|
2159
|
+
createdAt: row.createdAt
|
|
2160
|
+
}));
|
|
933
2161
|
} catch (error) {
|
|
2162
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2163
|
+
return [];
|
|
2164
|
+
}
|
|
934
2165
|
throw new MastraError(
|
|
935
2166
|
{
|
|
936
|
-
id: "
|
|
2167
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
937
2168
|
domain: ErrorDomain.STORAGE,
|
|
938
|
-
category: ErrorCategory.THIRD_PARTY
|
|
2169
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2170
|
+
details: {
|
|
2171
|
+
name: name ?? null,
|
|
2172
|
+
scope: scope ?? null,
|
|
2173
|
+
page,
|
|
2174
|
+
perPage,
|
|
2175
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2176
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2177
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2178
|
+
toDate: toDate?.toISOString() ?? null
|
|
2179
|
+
}
|
|
939
2180
|
},
|
|
940
2181
|
error
|
|
941
2182
|
);
|
|
942
2183
|
}
|
|
943
2184
|
}
|
|
2185
|
+
async batchTraceInsert(args) {
|
|
2186
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2187
|
+
}
|
|
2188
|
+
};
|
|
2189
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2190
|
+
client;
|
|
2191
|
+
operations;
|
|
2192
|
+
constructor({ client, operations }) {
|
|
2193
|
+
super();
|
|
2194
|
+
this.operations = operations;
|
|
2195
|
+
this.client = client;
|
|
2196
|
+
}
|
|
944
2197
|
async persistWorkflowSnapshot({
|
|
945
2198
|
workflowName,
|
|
946
2199
|
runId,
|
|
947
2200
|
snapshot
|
|
948
2201
|
}) {
|
|
949
2202
|
try {
|
|
950
|
-
const currentSnapshot = await this.load({
|
|
2203
|
+
const currentSnapshot = await this.operations.load({
|
|
951
2204
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
952
2205
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
953
2206
|
});
|
|
@@ -963,7 +2216,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
963
2216
|
createdAt: now.toISOString(),
|
|
964
2217
|
updatedAt: now.toISOString()
|
|
965
2218
|
};
|
|
966
|
-
await this.
|
|
2219
|
+
await this.client.insert({
|
|
967
2220
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
968
2221
|
format: "JSONEachRow",
|
|
969
2222
|
values: [persisting],
|
|
@@ -991,7 +2244,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
991
2244
|
runId
|
|
992
2245
|
}) {
|
|
993
2246
|
try {
|
|
994
|
-
const result = await this.load({
|
|
2247
|
+
const result = await this.operations.load({
|
|
995
2248
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
996
2249
|
keys: {
|
|
997
2250
|
workflow_name: workflowName,
|
|
@@ -1048,7 +2301,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1048
2301
|
values.var_workflow_name = workflowName;
|
|
1049
2302
|
}
|
|
1050
2303
|
if (resourceId) {
|
|
1051
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2304
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
1052
2305
|
if (hasResourceId) {
|
|
1053
2306
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
1054
2307
|
values.var_resourceId = resourceId;
|
|
@@ -1069,7 +2322,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1069
2322
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
1070
2323
|
let total = 0;
|
|
1071
2324
|
if (limit !== void 0 && offset !== void 0) {
|
|
1072
|
-
const countResult = await this.
|
|
2325
|
+
const countResult = await this.client.query({
|
|
1073
2326
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
1074
2327
|
query_params: values,
|
|
1075
2328
|
format: "JSONEachRow"
|
|
@@ -1077,21 +2330,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1077
2330
|
const countRows = await countResult.json();
|
|
1078
2331
|
total = Number(countRows[0]?.count ?? 0);
|
|
1079
2332
|
}
|
|
1080
|
-
const result = await this.
|
|
2333
|
+
const result = await this.client.query({
|
|
1081
2334
|
query: `
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
2335
|
+
SELECT
|
|
2336
|
+
workflow_name,
|
|
2337
|
+
run_id,
|
|
2338
|
+
snapshot,
|
|
2339
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2340
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2341
|
+
resourceId
|
|
2342
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2343
|
+
${whereClause}
|
|
2344
|
+
ORDER BY createdAt DESC
|
|
2345
|
+
${limitClause}
|
|
2346
|
+
${offsetClause}
|
|
2347
|
+
`,
|
|
1095
2348
|
query_params: values,
|
|
1096
2349
|
format: "JSONEachRow"
|
|
1097
2350
|
});
|
|
@@ -1129,18 +2382,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1129
2382
|
values.var_workflow_name = workflowName;
|
|
1130
2383
|
}
|
|
1131
2384
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1132
|
-
const result = await this.
|
|
2385
|
+
const result = await this.client.query({
|
|
1133
2386
|
query: `
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
2387
|
+
SELECT
|
|
2388
|
+
workflow_name,
|
|
2389
|
+
run_id,
|
|
2390
|
+
snapshot,
|
|
2391
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2392
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2393
|
+
resourceId
|
|
2394
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2395
|
+
${whereClause}
|
|
2396
|
+
`,
|
|
1144
2397
|
query_params: values,
|
|
1145
2398
|
format: "JSONEachRow"
|
|
1146
2399
|
});
|
|
@@ -1161,45 +2414,237 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1161
2414
|
);
|
|
1162
2415
|
}
|
|
1163
2416
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
2417
|
+
};
|
|
2418
|
+
|
|
2419
|
+
// src/storage/index.ts
|
|
2420
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2421
|
+
db;
|
|
2422
|
+
ttl = {};
|
|
2423
|
+
stores;
|
|
2424
|
+
constructor(config) {
|
|
2425
|
+
super({ name: "ClickhouseStore" });
|
|
2426
|
+
this.db = createClient({
|
|
2427
|
+
url: config.url,
|
|
2428
|
+
username: config.username,
|
|
2429
|
+
password: config.password,
|
|
2430
|
+
clickhouse_settings: {
|
|
2431
|
+
date_time_input_format: "best_effort",
|
|
2432
|
+
date_time_output_format: "iso",
|
|
2433
|
+
// This is crucial
|
|
2434
|
+
use_client_time_zone: 1,
|
|
2435
|
+
output_format_json_quote_64bit_integers: 0
|
|
2436
|
+
}
|
|
1168
2437
|
});
|
|
1169
|
-
|
|
1170
|
-
|
|
2438
|
+
this.ttl = config.ttl;
|
|
2439
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2440
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2441
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2442
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2443
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2444
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2445
|
+
this.stores = {
|
|
2446
|
+
operations,
|
|
2447
|
+
workflows,
|
|
2448
|
+
scores,
|
|
2449
|
+
legacyEvals,
|
|
2450
|
+
traces,
|
|
2451
|
+
memory
|
|
2452
|
+
};
|
|
1171
2453
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
}
|
|
2454
|
+
get supports() {
|
|
2455
|
+
return {
|
|
2456
|
+
selectByIncludeResourceScope: true,
|
|
2457
|
+
resourceWorkingMemory: true,
|
|
2458
|
+
hasColumn: true,
|
|
2459
|
+
createTable: true
|
|
2460
|
+
};
|
|
1179
2461
|
}
|
|
1180
|
-
async
|
|
1181
|
-
|
|
1182
|
-
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
1183
|
-
domain: ErrorDomain.STORAGE,
|
|
1184
|
-
category: ErrorCategory.USER,
|
|
1185
|
-
text: "Method not implemented."
|
|
1186
|
-
});
|
|
2462
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2463
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
1187
2464
|
}
|
|
1188
|
-
async
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
2465
|
+
async getEvals(options) {
|
|
2466
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2467
|
+
}
|
|
2468
|
+
async batchInsert({ tableName, records }) {
|
|
2469
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2470
|
+
}
|
|
2471
|
+
async optimizeTable({ tableName }) {
|
|
2472
|
+
try {
|
|
2473
|
+
await this.db.command({
|
|
2474
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2475
|
+
});
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
throw new MastraError(
|
|
2478
|
+
{
|
|
2479
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2480
|
+
domain: ErrorDomain.STORAGE,
|
|
2481
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2482
|
+
details: { tableName }
|
|
2483
|
+
},
|
|
2484
|
+
error
|
|
2485
|
+
);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
async materializeTtl({ tableName }) {
|
|
2489
|
+
try {
|
|
2490
|
+
await this.db.command({
|
|
2491
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2492
|
+
});
|
|
2493
|
+
} catch (error) {
|
|
2494
|
+
throw new MastraError(
|
|
2495
|
+
{
|
|
2496
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2497
|
+
domain: ErrorDomain.STORAGE,
|
|
2498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2499
|
+
details: { tableName }
|
|
2500
|
+
},
|
|
2501
|
+
error
|
|
2502
|
+
);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
async createTable({
|
|
2506
|
+
tableName,
|
|
2507
|
+
schema
|
|
2508
|
+
}) {
|
|
2509
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2510
|
+
}
|
|
2511
|
+
async dropTable({ tableName }) {
|
|
2512
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2513
|
+
}
|
|
2514
|
+
async alterTable({
|
|
2515
|
+
tableName,
|
|
2516
|
+
schema,
|
|
2517
|
+
ifNotExists
|
|
2518
|
+
}) {
|
|
2519
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2520
|
+
}
|
|
2521
|
+
async clearTable({ tableName }) {
|
|
2522
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2523
|
+
}
|
|
2524
|
+
async insert({ tableName, record }) {
|
|
2525
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2526
|
+
}
|
|
2527
|
+
async load({ tableName, keys }) {
|
|
2528
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2529
|
+
}
|
|
2530
|
+
async persistWorkflowSnapshot({
|
|
2531
|
+
workflowName,
|
|
2532
|
+
runId,
|
|
2533
|
+
snapshot
|
|
2534
|
+
}) {
|
|
2535
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2536
|
+
}
|
|
2537
|
+
async loadWorkflowSnapshot({
|
|
2538
|
+
workflowName,
|
|
2539
|
+
runId
|
|
2540
|
+
}) {
|
|
2541
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2542
|
+
}
|
|
2543
|
+
async getWorkflowRuns({
|
|
2544
|
+
workflowName,
|
|
2545
|
+
fromDate,
|
|
2546
|
+
toDate,
|
|
2547
|
+
limit,
|
|
2548
|
+
offset,
|
|
2549
|
+
resourceId
|
|
2550
|
+
} = {}) {
|
|
2551
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2552
|
+
}
|
|
2553
|
+
async getWorkflowRunById({
|
|
2554
|
+
runId,
|
|
2555
|
+
workflowName
|
|
2556
|
+
}) {
|
|
2557
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2558
|
+
}
|
|
2559
|
+
async getTraces(args) {
|
|
2560
|
+
return this.stores.traces.getTraces(args);
|
|
2561
|
+
}
|
|
2562
|
+
async getTracesPaginated(args) {
|
|
2563
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2564
|
+
}
|
|
2565
|
+
async batchTraceInsert(args) {
|
|
2566
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2567
|
+
}
|
|
2568
|
+
async getThreadById({ threadId }) {
|
|
2569
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2570
|
+
}
|
|
2571
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2572
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2573
|
+
}
|
|
2574
|
+
async saveThread({ thread }) {
|
|
2575
|
+
return this.stores.memory.saveThread({ thread });
|
|
2576
|
+
}
|
|
2577
|
+
async updateThread({
|
|
2578
|
+
id,
|
|
2579
|
+
title,
|
|
2580
|
+
metadata
|
|
2581
|
+
}) {
|
|
2582
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2583
|
+
}
|
|
2584
|
+
async deleteThread({ threadId }) {
|
|
2585
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2586
|
+
}
|
|
2587
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2588
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2589
|
+
}
|
|
2590
|
+
async getMessages({
|
|
2591
|
+
threadId,
|
|
2592
|
+
resourceId,
|
|
2593
|
+
selectBy,
|
|
2594
|
+
format
|
|
2595
|
+
}) {
|
|
2596
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2597
|
+
}
|
|
2598
|
+
async saveMessages(args) {
|
|
2599
|
+
return this.stores.memory.saveMessages(args);
|
|
2600
|
+
}
|
|
2601
|
+
async getMessagesPaginated(args) {
|
|
2602
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2603
|
+
}
|
|
2604
|
+
async updateMessages(args) {
|
|
2605
|
+
return this.stores.memory.updateMessages(args);
|
|
2606
|
+
}
|
|
2607
|
+
async getResourceById({ resourceId }) {
|
|
2608
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2609
|
+
}
|
|
2610
|
+
async saveResource({ resource }) {
|
|
2611
|
+
return this.stores.memory.saveResource({ resource });
|
|
2612
|
+
}
|
|
2613
|
+
async updateResource({
|
|
2614
|
+
resourceId,
|
|
2615
|
+
workingMemory,
|
|
2616
|
+
metadata
|
|
2617
|
+
}) {
|
|
2618
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2619
|
+
}
|
|
2620
|
+
async getScoreById({ id }) {
|
|
2621
|
+
return this.stores.scores.getScoreById({ id });
|
|
2622
|
+
}
|
|
2623
|
+
async saveScore(_score) {
|
|
2624
|
+
return this.stores.scores.saveScore(_score);
|
|
2625
|
+
}
|
|
2626
|
+
async getScoresByRunId({
|
|
2627
|
+
runId,
|
|
2628
|
+
pagination
|
|
2629
|
+
}) {
|
|
2630
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2631
|
+
}
|
|
2632
|
+
async getScoresByEntityId({
|
|
2633
|
+
entityId,
|
|
2634
|
+
entityType,
|
|
2635
|
+
pagination
|
|
2636
|
+
}) {
|
|
2637
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2638
|
+
}
|
|
2639
|
+
async getScoresByScorerId({
|
|
2640
|
+
scorerId,
|
|
2641
|
+
pagination
|
|
2642
|
+
}) {
|
|
2643
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
1195
2644
|
}
|
|
1196
2645
|
async close() {
|
|
1197
2646
|
await this.db.close();
|
|
1198
2647
|
}
|
|
1199
|
-
async updateMessages(_args) {
|
|
1200
|
-
this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
|
|
1201
|
-
throw new Error("Method not implemented");
|
|
1202
|
-
}
|
|
1203
2648
|
};
|
|
1204
2649
|
|
|
1205
|
-
export {
|
|
2650
|
+
export { ClickhouseStore };
|