@mastra/clickhouse 0.0.0-vector-sources-20250516175436 → 0.0.0-vector-extension-schema-20250922130418
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +759 -2
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2379 -442
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2362 -425
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +87 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +46 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +55 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +236 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +32 -14
- package/dist/_tsup-dts-rollup.d.cts +0 -138
- package/dist/_tsup-dts-rollup.d.ts +0 -138
- package/dist/index.d.cts +0 -4
- package/docker-compose.yaml +0 -15
- package/eslint.config.js +0 -6
- package/src/index.ts +0 -1
- package/src/storage/index.test.ts +0 -856
- package/src/storage/index.ts +0 -1056
- package/tsconfig.json +0 -5
- package/vitest.config.ts +0 -12
package/dist/index.cjs
CHANGED
|
@@ -1,22 +1,21 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var client = require('@clickhouse/client');
|
|
4
|
+
var error = require('@mastra/core/error');
|
|
4
5
|
var storage = require('@mastra/core/storage');
|
|
6
|
+
var agent = require('@mastra/core/agent');
|
|
5
7
|
|
|
6
8
|
// src/storage/index.ts
|
|
7
|
-
function safelyParseJSON(jsonString) {
|
|
8
|
-
try {
|
|
9
|
-
return JSON.parse(jsonString);
|
|
10
|
-
} catch {
|
|
11
|
-
return {};
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
9
|
var TABLE_ENGINES = {
|
|
15
10
|
[storage.TABLE_MESSAGES]: `MergeTree()`,
|
|
16
11
|
[storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
17
12
|
[storage.TABLE_TRACES]: `MergeTree()`,
|
|
18
13
|
[storage.TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
19
|
-
[storage.TABLE_EVALS]: `MergeTree()
|
|
14
|
+
[storage.TABLE_EVALS]: `MergeTree()`,
|
|
15
|
+
[storage.TABLE_SCORERS]: `MergeTree()`,
|
|
16
|
+
[storage.TABLE_RESOURCES]: `ReplacingMergeTree()`,
|
|
17
|
+
// TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
|
|
18
|
+
[storage.TABLE_AI_SPANS]: `ReplacingMergeTree()`
|
|
20
19
|
};
|
|
21
20
|
var COLUMN_TYPES = {
|
|
22
21
|
text: "String",
|
|
@@ -24,11 +23,10 @@ var COLUMN_TYPES = {
|
|
|
24
23
|
uuid: "String",
|
|
25
24
|
jsonb: "String",
|
|
26
25
|
integer: "Int64",
|
|
27
|
-
|
|
26
|
+
float: "Float64",
|
|
27
|
+
bigint: "Int64",
|
|
28
|
+
boolean: "Bool"
|
|
28
29
|
};
|
|
29
|
-
function transformRows(rows) {
|
|
30
|
-
return rows.map((row) => transformRow(row));
|
|
31
|
-
}
|
|
32
30
|
function transformRow(row) {
|
|
33
31
|
if (!row) {
|
|
34
32
|
return row;
|
|
@@ -39,33 +37,63 @@ function transformRow(row) {
|
|
|
39
37
|
if (row.updatedAt) {
|
|
40
38
|
row.updatedAt = new Date(row.updatedAt);
|
|
41
39
|
}
|
|
40
|
+
if (row.content && typeof row.content === "string") {
|
|
41
|
+
row.content = storage.safelyParseJSON(row.content);
|
|
42
|
+
}
|
|
42
43
|
return row;
|
|
43
44
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// This is crucial
|
|
57
|
-
use_client_time_zone: 1,
|
|
58
|
-
output_format_json_quote_64bit_integers: 0
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
this.ttl = config.ttl;
|
|
45
|
+
function transformRows(rows) {
|
|
46
|
+
return rows.map((row) => transformRow(row));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
50
|
+
var LegacyEvalsStorageClickhouse = class extends storage.LegacyEvalsStorage {
|
|
51
|
+
client;
|
|
52
|
+
operations;
|
|
53
|
+
constructor({ client, operations }) {
|
|
54
|
+
super();
|
|
55
|
+
this.client = client;
|
|
56
|
+
this.operations = operations;
|
|
62
57
|
}
|
|
63
58
|
transformEvalRow(row) {
|
|
64
59
|
row = transformRow(row);
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
let resultValue;
|
|
61
|
+
try {
|
|
62
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
63
|
+
resultValue = JSON.parse(row.result);
|
|
64
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
65
|
+
resultValue = row.result;
|
|
66
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
67
|
+
resultValue = { score: 0 };
|
|
68
|
+
} else {
|
|
69
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
70
|
+
}
|
|
71
|
+
} catch (error$1) {
|
|
72
|
+
console.error("Error parsing result field:", row.result, error$1);
|
|
73
|
+
throw new error.MastraError({
|
|
74
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
75
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
76
|
+
domain: error.ErrorDomain.STORAGE,
|
|
77
|
+
category: error.ErrorCategory.USER
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
let testInfoValue;
|
|
81
|
+
try {
|
|
82
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
83
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
84
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
85
|
+
testInfoValue = row.test_info;
|
|
86
|
+
}
|
|
87
|
+
} catch {
|
|
88
|
+
testInfoValue = void 0;
|
|
89
|
+
}
|
|
67
90
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
|
-
throw new
|
|
91
|
+
throw new error.MastraError({
|
|
92
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
93
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
94
|
+
domain: error.ErrorDomain.STORAGE,
|
|
95
|
+
category: error.ErrorCategory.USER
|
|
96
|
+
});
|
|
69
97
|
}
|
|
70
98
|
return {
|
|
71
99
|
input: row.input,
|
|
@@ -82,9 +110,9 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
82
110
|
}
|
|
83
111
|
async getEvalsByAgentName(agentName, type) {
|
|
84
112
|
try {
|
|
85
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
86
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
87
|
-
const result = await this.
|
|
113
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
114
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
115
|
+
const result = await this.client.query({
|
|
88
116
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
89
117
|
query_params: { var_agent_name: agentName },
|
|
90
118
|
clickhouse_settings: {
|
|
@@ -99,126 +127,1358 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
99
127
|
}
|
|
100
128
|
const rows = await result.json();
|
|
101
129
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
102
|
-
} catch (error) {
|
|
103
|
-
if (error
|
|
130
|
+
} catch (error$1) {
|
|
131
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
104
132
|
return [];
|
|
105
133
|
}
|
|
106
|
-
|
|
107
|
-
|
|
134
|
+
throw new error.MastraError(
|
|
135
|
+
{
|
|
136
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
137
|
+
domain: error.ErrorDomain.STORAGE,
|
|
138
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
139
|
+
details: { agentName, type: type ?? null }
|
|
140
|
+
},
|
|
141
|
+
error$1
|
|
142
|
+
);
|
|
108
143
|
}
|
|
109
144
|
}
|
|
110
|
-
async
|
|
145
|
+
async getEvals(options = {}) {
|
|
146
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
147
|
+
const fromDate = dateRange?.start;
|
|
148
|
+
const toDate = dateRange?.end;
|
|
149
|
+
const conditions = [];
|
|
150
|
+
if (agentName) {
|
|
151
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
152
|
+
}
|
|
153
|
+
if (type === "test") {
|
|
154
|
+
conditions.push(
|
|
155
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
156
|
+
);
|
|
157
|
+
} else if (type === "live") {
|
|
158
|
+
conditions.push(
|
|
159
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
if (fromDate) {
|
|
163
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
164
|
+
fromDate.toISOString();
|
|
165
|
+
}
|
|
166
|
+
if (toDate) {
|
|
167
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
168
|
+
toDate.toISOString();
|
|
169
|
+
}
|
|
170
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
111
171
|
try {
|
|
112
|
-
await this.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
...
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
172
|
+
const countResult = await this.client.query({
|
|
173
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_EVALS} ${whereClause}`,
|
|
174
|
+
query_params: {
|
|
175
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
176
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
177
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
178
|
+
},
|
|
179
|
+
clickhouse_settings: {
|
|
180
|
+
date_time_input_format: "best_effort",
|
|
181
|
+
date_time_output_format: "iso",
|
|
182
|
+
use_client_time_zone: 1,
|
|
183
|
+
output_format_json_quote_64bit_integers: 0
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
const countData = await countResult.json();
|
|
187
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
188
|
+
const currentOffset = page * perPage;
|
|
189
|
+
const hasMore = currentOffset + perPage < total;
|
|
190
|
+
if (total === 0) {
|
|
191
|
+
return {
|
|
192
|
+
evals: [],
|
|
193
|
+
total: 0,
|
|
194
|
+
page,
|
|
195
|
+
perPage,
|
|
196
|
+
hasMore: false
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
const dataResult = await this.client.query({
|
|
200
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
201
|
+
query_params: {
|
|
202
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
203
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
204
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
205
|
+
var_limit: perPage || 100,
|
|
206
|
+
var_offset: currentOffset || 0
|
|
207
|
+
},
|
|
123
208
|
clickhouse_settings: {
|
|
124
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
125
209
|
date_time_input_format: "best_effort",
|
|
210
|
+
date_time_output_format: "iso",
|
|
126
211
|
use_client_time_zone: 1,
|
|
127
212
|
output_format_json_quote_64bit_integers: 0
|
|
128
213
|
}
|
|
129
214
|
});
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
215
|
+
const rows = await dataResult.json();
|
|
216
|
+
return {
|
|
217
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
218
|
+
total,
|
|
219
|
+
page,
|
|
220
|
+
perPage,
|
|
221
|
+
hasMore
|
|
222
|
+
};
|
|
223
|
+
} catch (error$1) {
|
|
224
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
225
|
+
return {
|
|
226
|
+
evals: [],
|
|
227
|
+
total: 0,
|
|
228
|
+
page,
|
|
229
|
+
perPage,
|
|
230
|
+
hasMore: false
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
throw new error.MastraError(
|
|
234
|
+
{
|
|
235
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
236
|
+
domain: error.ErrorDomain.STORAGE,
|
|
237
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
238
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
239
|
+
},
|
|
240
|
+
error$1
|
|
241
|
+
);
|
|
133
242
|
}
|
|
134
243
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
244
|
+
};
|
|
245
|
+
var MemoryStorageClickhouse = class extends storage.MemoryStorage {
|
|
246
|
+
client;
|
|
247
|
+
operations;
|
|
248
|
+
constructor({ client, operations }) {
|
|
249
|
+
super();
|
|
250
|
+
this.client = client;
|
|
251
|
+
this.operations = operations;
|
|
252
|
+
}
|
|
253
|
+
async getMessages({
|
|
254
|
+
threadId,
|
|
255
|
+
resourceId,
|
|
256
|
+
selectBy,
|
|
257
|
+
format
|
|
144
258
|
}) {
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
259
|
+
try {
|
|
260
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
261
|
+
const messages = [];
|
|
262
|
+
const limit = storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
263
|
+
const include = selectBy?.include || [];
|
|
264
|
+
if (include.length) {
|
|
265
|
+
const unionQueries = [];
|
|
266
|
+
const params = [];
|
|
267
|
+
let paramIdx = 1;
|
|
268
|
+
for (const inc of include) {
|
|
269
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
270
|
+
const searchId = inc.threadId || threadId;
|
|
271
|
+
unionQueries.push(`
|
|
272
|
+
SELECT * FROM (
|
|
273
|
+
WITH numbered_messages AS (
|
|
274
|
+
SELECT
|
|
275
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
276
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
277
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
278
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
279
|
+
),
|
|
280
|
+
target_positions AS (
|
|
281
|
+
SELECT row_num as target_pos
|
|
282
|
+
FROM numbered_messages
|
|
283
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
284
|
+
)
|
|
285
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
286
|
+
FROM numbered_messages m
|
|
287
|
+
CROSS JOIN target_positions t
|
|
288
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
289
|
+
) AS query_${paramIdx}
|
|
290
|
+
`);
|
|
291
|
+
params.push(
|
|
292
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
293
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
294
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
295
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
296
|
+
);
|
|
297
|
+
paramIdx++;
|
|
298
|
+
}
|
|
299
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
300
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
301
|
+
const includeResult = await this.client.query({
|
|
302
|
+
query: finalQuery,
|
|
303
|
+
query_params: mergedParams,
|
|
304
|
+
clickhouse_settings: {
|
|
305
|
+
date_time_input_format: "best_effort",
|
|
306
|
+
date_time_output_format: "iso",
|
|
307
|
+
use_client_time_zone: 1,
|
|
308
|
+
output_format_json_quote_64bit_integers: 0
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
const rows2 = await includeResult.json();
|
|
312
|
+
const includedMessages = transformRows(rows2.data);
|
|
313
|
+
const seen = /* @__PURE__ */ new Set();
|
|
314
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
315
|
+
if (seen.has(message.id)) return false;
|
|
316
|
+
seen.add(message.id);
|
|
317
|
+
return true;
|
|
318
|
+
});
|
|
319
|
+
messages.push(...dedupedMessages);
|
|
320
|
+
}
|
|
321
|
+
const result = await this.client.query({
|
|
322
|
+
query: `
|
|
323
|
+
SELECT
|
|
324
|
+
id,
|
|
325
|
+
content,
|
|
326
|
+
role,
|
|
327
|
+
type,
|
|
328
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
329
|
+
thread_id AS "threadId"
|
|
330
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
331
|
+
WHERE thread_id = {threadId:String}
|
|
332
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
333
|
+
ORDER BY "createdAt" DESC
|
|
334
|
+
LIMIT {limit:Int64}
|
|
335
|
+
`,
|
|
336
|
+
query_params: {
|
|
337
|
+
threadId,
|
|
338
|
+
exclude: messages.map((m) => m.id),
|
|
339
|
+
limit
|
|
340
|
+
},
|
|
341
|
+
clickhouse_settings: {
|
|
342
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
343
|
+
date_time_input_format: "best_effort",
|
|
344
|
+
date_time_output_format: "iso",
|
|
345
|
+
use_client_time_zone: 1,
|
|
346
|
+
output_format_json_quote_64bit_integers: 0
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
const rows = await result.json();
|
|
350
|
+
messages.push(...transformRows(rows.data));
|
|
351
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
352
|
+
messages.forEach((message) => {
|
|
353
|
+
if (typeof message.content === "string") {
|
|
354
|
+
try {
|
|
355
|
+
message.content = JSON.parse(message.content);
|
|
356
|
+
} catch {
|
|
357
|
+
}
|
|
358
|
+
}
|
|
161
359
|
});
|
|
360
|
+
const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
361
|
+
if (format === `v2`) return list.get.all.v2();
|
|
362
|
+
return list.get.all.v1();
|
|
363
|
+
} catch (error$1) {
|
|
364
|
+
throw new error.MastraError(
|
|
365
|
+
{
|
|
366
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
367
|
+
domain: error.ErrorDomain.STORAGE,
|
|
368
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
369
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
370
|
+
},
|
|
371
|
+
error$1
|
|
372
|
+
);
|
|
162
373
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
374
|
+
}
|
|
375
|
+
async getMessagesById({
|
|
376
|
+
messageIds,
|
|
377
|
+
format
|
|
378
|
+
}) {
|
|
379
|
+
if (messageIds.length === 0) return [];
|
|
380
|
+
try {
|
|
381
|
+
const result = await this.client.query({
|
|
382
|
+
query: `
|
|
383
|
+
SELECT
|
|
384
|
+
id,
|
|
385
|
+
content,
|
|
386
|
+
role,
|
|
387
|
+
type,
|
|
388
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
389
|
+
thread_id AS "threadId",
|
|
390
|
+
"resourceId"
|
|
391
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
392
|
+
WHERE id IN {messageIds:Array(String)}
|
|
393
|
+
ORDER BY "createdAt" DESC
|
|
394
|
+
`,
|
|
395
|
+
query_params: {
|
|
396
|
+
messageIds
|
|
397
|
+
},
|
|
398
|
+
clickhouse_settings: {
|
|
399
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
400
|
+
date_time_input_format: "best_effort",
|
|
401
|
+
date_time_output_format: "iso",
|
|
402
|
+
use_client_time_zone: 1,
|
|
403
|
+
output_format_json_quote_64bit_integers: 0
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
const rows = await result.json();
|
|
407
|
+
const messages = transformRows(rows.data);
|
|
408
|
+
messages.forEach((message) => {
|
|
409
|
+
if (typeof message.content === "string") {
|
|
410
|
+
try {
|
|
411
|
+
message.content = JSON.parse(message.content);
|
|
412
|
+
} catch {
|
|
413
|
+
}
|
|
414
|
+
}
|
|
169
415
|
});
|
|
416
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
417
|
+
if (format === `v1`) return list.get.all.v1();
|
|
418
|
+
return list.get.all.v2();
|
|
419
|
+
} catch (error$1) {
|
|
420
|
+
throw new error.MastraError(
|
|
421
|
+
{
|
|
422
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
|
|
423
|
+
domain: error.ErrorDomain.STORAGE,
|
|
424
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
425
|
+
details: { messageIds: JSON.stringify(messageIds) }
|
|
426
|
+
},
|
|
427
|
+
error$1
|
|
428
|
+
);
|
|
170
429
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
430
|
+
}
|
|
431
|
+
async saveMessages(args) {
|
|
432
|
+
const { messages, format = "v1" } = args;
|
|
433
|
+
if (messages.length === 0) return messages;
|
|
434
|
+
for (const message of messages) {
|
|
435
|
+
const resourceId = message.resourceId;
|
|
436
|
+
if (!resourceId) {
|
|
437
|
+
throw new Error("Resource ID is required");
|
|
438
|
+
}
|
|
439
|
+
if (!message.threadId) {
|
|
440
|
+
throw new Error("Thread ID is required");
|
|
441
|
+
}
|
|
442
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
443
|
+
if (!thread) {
|
|
444
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
445
|
+
}
|
|
174
446
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
447
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
448
|
+
await Promise.all(
|
|
449
|
+
messages.map(async (m) => {
|
|
450
|
+
const resourceId = m.resourceId;
|
|
451
|
+
if (!resourceId) {
|
|
452
|
+
throw new Error("Resource ID is required");
|
|
453
|
+
}
|
|
454
|
+
if (!m.threadId) {
|
|
455
|
+
throw new Error("Thread ID is required");
|
|
456
|
+
}
|
|
457
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
458
|
+
if (!thread) {
|
|
459
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
460
|
+
}
|
|
461
|
+
threadIdSet.set(m.threadId, thread);
|
|
462
|
+
})
|
|
463
|
+
);
|
|
464
|
+
try {
|
|
465
|
+
const existingResult = await this.client.query({
|
|
466
|
+
query: `SELECT id, thread_id FROM ${storage.TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
467
|
+
query_params: {
|
|
468
|
+
ids: messages.map((m) => m.id)
|
|
469
|
+
},
|
|
470
|
+
clickhouse_settings: {
|
|
471
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
472
|
+
date_time_input_format: "best_effort",
|
|
473
|
+
date_time_output_format: "iso",
|
|
474
|
+
use_client_time_zone: 1,
|
|
475
|
+
output_format_json_quote_64bit_integers: 0
|
|
476
|
+
},
|
|
477
|
+
format: "JSONEachRow"
|
|
478
|
+
});
|
|
479
|
+
const existingRows = await existingResult.json();
|
|
480
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
481
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
482
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
483
|
+
const toMove = messages.filter((m) => {
|
|
484
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
485
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
486
|
+
});
|
|
487
|
+
const deletePromises = toMove.map((message) => {
|
|
488
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
489
|
+
if (!existingRow) return Promise.resolve();
|
|
490
|
+
return this.client.command({
|
|
491
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
492
|
+
query_params: {
|
|
493
|
+
var_id: message.id,
|
|
494
|
+
var_old_thread_id: existingRow.thread_id
|
|
495
|
+
},
|
|
496
|
+
clickhouse_settings: {
|
|
497
|
+
date_time_input_format: "best_effort",
|
|
498
|
+
use_client_time_zone: 1,
|
|
499
|
+
output_format_json_quote_64bit_integers: 0
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
});
|
|
503
|
+
const updatePromises = toUpdate.map(
|
|
504
|
+
(message) => this.client.command({
|
|
505
|
+
query: `
|
|
506
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
507
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
508
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
509
|
+
`,
|
|
510
|
+
query_params: {
|
|
511
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
512
|
+
var_role: message.role,
|
|
513
|
+
var_type: message.type || "v2",
|
|
514
|
+
var_resourceId: message.resourceId,
|
|
515
|
+
var_id: message.id,
|
|
516
|
+
var_thread_id: message.threadId
|
|
517
|
+
},
|
|
518
|
+
clickhouse_settings: {
|
|
519
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
520
|
+
date_time_input_format: "best_effort",
|
|
521
|
+
use_client_time_zone: 1,
|
|
522
|
+
output_format_json_quote_64bit_integers: 0
|
|
523
|
+
}
|
|
524
|
+
})
|
|
525
|
+
);
|
|
526
|
+
await Promise.all([
|
|
527
|
+
// Insert new messages (including moved messages)
|
|
528
|
+
this.client.insert({
|
|
529
|
+
table: storage.TABLE_MESSAGES,
|
|
530
|
+
format: "JSONEachRow",
|
|
531
|
+
values: toInsert.map((message) => ({
|
|
532
|
+
id: message.id,
|
|
533
|
+
thread_id: message.threadId,
|
|
534
|
+
resourceId: message.resourceId,
|
|
535
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
536
|
+
createdAt: message.createdAt.toISOString(),
|
|
537
|
+
role: message.role,
|
|
538
|
+
type: message.type || "v2"
|
|
539
|
+
})),
|
|
540
|
+
clickhouse_settings: {
|
|
541
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
542
|
+
date_time_input_format: "best_effort",
|
|
543
|
+
use_client_time_zone: 1,
|
|
544
|
+
output_format_json_quote_64bit_integers: 0
|
|
545
|
+
}
|
|
546
|
+
}),
|
|
547
|
+
...updatePromises,
|
|
548
|
+
...deletePromises,
|
|
549
|
+
// Update thread's updatedAt timestamp
|
|
550
|
+
this.client.insert({
|
|
551
|
+
table: storage.TABLE_THREADS,
|
|
552
|
+
format: "JSONEachRow",
|
|
553
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
554
|
+
id: thread.id,
|
|
555
|
+
resourceId: thread.resourceId,
|
|
556
|
+
title: thread.title,
|
|
557
|
+
metadata: thread.metadata,
|
|
558
|
+
createdAt: thread.createdAt,
|
|
559
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
560
|
+
})),
|
|
561
|
+
clickhouse_settings: {
|
|
562
|
+
date_time_input_format: "best_effort",
|
|
563
|
+
use_client_time_zone: 1,
|
|
564
|
+
output_format_json_quote_64bit_integers: 0
|
|
565
|
+
}
|
|
566
|
+
})
|
|
567
|
+
]);
|
|
568
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
569
|
+
if (format === `v2`) return list.get.all.v2();
|
|
570
|
+
return list.get.all.v1();
|
|
571
|
+
} catch (error$1) {
|
|
572
|
+
throw new error.MastraError(
|
|
573
|
+
{
|
|
574
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
575
|
+
domain: error.ErrorDomain.STORAGE,
|
|
576
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
577
|
+
},
|
|
578
|
+
error$1
|
|
579
|
+
);
|
|
178
580
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
581
|
+
}
|
|
582
|
+
async getThreadById({ threadId }) {
|
|
583
|
+
try {
|
|
584
|
+
const result = await this.client.query({
|
|
585
|
+
query: `SELECT
|
|
586
|
+
id,
|
|
587
|
+
"resourceId",
|
|
588
|
+
title,
|
|
589
|
+
metadata,
|
|
590
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
591
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
592
|
+
FROM "${storage.TABLE_THREADS}"
|
|
593
|
+
FINAL
|
|
594
|
+
WHERE id = {var_id:String}`,
|
|
595
|
+
query_params: { var_id: threadId },
|
|
596
|
+
clickhouse_settings: {
|
|
597
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
598
|
+
date_time_input_format: "best_effort",
|
|
599
|
+
date_time_output_format: "iso",
|
|
600
|
+
use_client_time_zone: 1,
|
|
601
|
+
output_format_json_quote_64bit_integers: 0
|
|
602
|
+
}
|
|
603
|
+
});
|
|
604
|
+
const rows = await result.json();
|
|
605
|
+
const thread = transformRow(rows.data[0]);
|
|
606
|
+
if (!thread) {
|
|
607
|
+
return null;
|
|
608
|
+
}
|
|
609
|
+
return {
|
|
610
|
+
...thread,
|
|
611
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
612
|
+
createdAt: thread.createdAt,
|
|
613
|
+
updatedAt: thread.updatedAt
|
|
614
|
+
};
|
|
615
|
+
} catch (error$1) {
|
|
616
|
+
throw new error.MastraError(
|
|
617
|
+
{
|
|
618
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
619
|
+
domain: error.ErrorDomain.STORAGE,
|
|
620
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
621
|
+
details: { threadId }
|
|
622
|
+
},
|
|
623
|
+
error$1
|
|
624
|
+
);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
628
|
+
try {
|
|
629
|
+
const result = await this.client.query({
|
|
630
|
+
query: `SELECT
|
|
631
|
+
id,
|
|
632
|
+
"resourceId",
|
|
633
|
+
title,
|
|
634
|
+
metadata,
|
|
635
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
636
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
637
|
+
FROM "${storage.TABLE_THREADS}"
|
|
638
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
639
|
+
query_params: { var_resourceId: resourceId },
|
|
640
|
+
clickhouse_settings: {
|
|
641
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
642
|
+
date_time_input_format: "best_effort",
|
|
643
|
+
date_time_output_format: "iso",
|
|
644
|
+
use_client_time_zone: 1,
|
|
645
|
+
output_format_json_quote_64bit_integers: 0
|
|
646
|
+
}
|
|
647
|
+
});
|
|
648
|
+
const rows = await result.json();
|
|
649
|
+
const threads = transformRows(rows.data);
|
|
650
|
+
return threads.map((thread) => ({
|
|
651
|
+
...thread,
|
|
652
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
653
|
+
createdAt: thread.createdAt,
|
|
654
|
+
updatedAt: thread.updatedAt
|
|
655
|
+
}));
|
|
656
|
+
} catch (error$1) {
|
|
657
|
+
throw new error.MastraError(
|
|
658
|
+
{
|
|
659
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
660
|
+
domain: error.ErrorDomain.STORAGE,
|
|
661
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
662
|
+
details: { resourceId }
|
|
663
|
+
},
|
|
664
|
+
error$1
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
async saveThread({ thread }) {
|
|
669
|
+
try {
|
|
670
|
+
await this.client.insert({
|
|
671
|
+
table: storage.TABLE_THREADS,
|
|
672
|
+
values: [
|
|
673
|
+
{
|
|
674
|
+
...thread,
|
|
675
|
+
createdAt: thread.createdAt.toISOString(),
|
|
676
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
677
|
+
}
|
|
678
|
+
],
|
|
679
|
+
format: "JSONEachRow",
|
|
680
|
+
clickhouse_settings: {
|
|
681
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
682
|
+
date_time_input_format: "best_effort",
|
|
683
|
+
use_client_time_zone: 1,
|
|
684
|
+
output_format_json_quote_64bit_integers: 0
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
return thread;
|
|
688
|
+
} catch (error$1) {
|
|
689
|
+
throw new error.MastraError(
|
|
690
|
+
{
|
|
691
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
692
|
+
domain: error.ErrorDomain.STORAGE,
|
|
693
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
694
|
+
details: { threadId: thread.id }
|
|
695
|
+
},
|
|
696
|
+
error$1
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
async updateThread({
|
|
701
|
+
id,
|
|
702
|
+
title,
|
|
703
|
+
metadata
|
|
704
|
+
}) {
|
|
705
|
+
try {
|
|
706
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
707
|
+
if (!existingThread) {
|
|
708
|
+
throw new Error(`Thread ${id} not found`);
|
|
709
|
+
}
|
|
710
|
+
const mergedMetadata = {
|
|
711
|
+
...existingThread.metadata,
|
|
712
|
+
...metadata
|
|
713
|
+
};
|
|
714
|
+
const updatedThread = {
|
|
715
|
+
...existingThread,
|
|
716
|
+
title,
|
|
717
|
+
metadata: mergedMetadata,
|
|
718
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
719
|
+
};
|
|
720
|
+
await this.client.insert({
|
|
721
|
+
table: storage.TABLE_THREADS,
|
|
722
|
+
format: "JSONEachRow",
|
|
723
|
+
values: [
|
|
724
|
+
{
|
|
725
|
+
id: updatedThread.id,
|
|
726
|
+
resourceId: updatedThread.resourceId,
|
|
727
|
+
title: updatedThread.title,
|
|
728
|
+
metadata: updatedThread.metadata,
|
|
729
|
+
createdAt: updatedThread.createdAt,
|
|
730
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
731
|
+
}
|
|
732
|
+
],
|
|
733
|
+
clickhouse_settings: {
|
|
734
|
+
date_time_input_format: "best_effort",
|
|
735
|
+
use_client_time_zone: 1,
|
|
736
|
+
output_format_json_quote_64bit_integers: 0
|
|
737
|
+
}
|
|
738
|
+
});
|
|
739
|
+
return updatedThread;
|
|
740
|
+
} catch (error$1) {
|
|
741
|
+
throw new error.MastraError(
|
|
742
|
+
{
|
|
743
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
744
|
+
domain: error.ErrorDomain.STORAGE,
|
|
745
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
746
|
+
details: { threadId: id, title }
|
|
747
|
+
},
|
|
748
|
+
error$1
|
|
749
|
+
);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
async deleteThread({ threadId }) {
|
|
753
|
+
try {
|
|
754
|
+
await this.client.command({
|
|
755
|
+
query: `DELETE FROM "${storage.TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
756
|
+
query_params: { var_thread_id: threadId },
|
|
757
|
+
clickhouse_settings: {
|
|
758
|
+
output_format_json_quote_64bit_integers: 0
|
|
759
|
+
}
|
|
760
|
+
});
|
|
761
|
+
await this.client.command({
|
|
762
|
+
query: `DELETE FROM "${storage.TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
763
|
+
query_params: { var_id: threadId },
|
|
764
|
+
clickhouse_settings: {
|
|
765
|
+
output_format_json_quote_64bit_integers: 0
|
|
766
|
+
}
|
|
767
|
+
});
|
|
768
|
+
} catch (error$1) {
|
|
769
|
+
throw new error.MastraError(
|
|
770
|
+
{
|
|
771
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
772
|
+
domain: error.ErrorDomain.STORAGE,
|
|
773
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
774
|
+
details: { threadId }
|
|
775
|
+
},
|
|
776
|
+
error$1
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
781
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
782
|
+
try {
|
|
783
|
+
const currentOffset = page * perPage;
|
|
784
|
+
const countResult = await this.client.query({
|
|
785
|
+
query: `SELECT count() as total FROM ${storage.TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
786
|
+
query_params: { resourceId },
|
|
787
|
+
clickhouse_settings: {
|
|
788
|
+
date_time_input_format: "best_effort",
|
|
789
|
+
date_time_output_format: "iso",
|
|
790
|
+
use_client_time_zone: 1,
|
|
791
|
+
output_format_json_quote_64bit_integers: 0
|
|
792
|
+
}
|
|
793
|
+
});
|
|
794
|
+
const countData = await countResult.json();
|
|
795
|
+
const total = countData.data[0].total;
|
|
796
|
+
if (total === 0) {
|
|
797
|
+
return {
|
|
798
|
+
threads: [],
|
|
799
|
+
total: 0,
|
|
800
|
+
page,
|
|
801
|
+
perPage,
|
|
802
|
+
hasMore: false
|
|
803
|
+
};
|
|
804
|
+
}
|
|
805
|
+
const dataResult = await this.client.query({
|
|
806
|
+
query: `
|
|
807
|
+
SELECT
|
|
808
|
+
id,
|
|
809
|
+
resourceId,
|
|
810
|
+
title,
|
|
811
|
+
metadata,
|
|
812
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
813
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
814
|
+
FROM ${storage.TABLE_THREADS}
|
|
815
|
+
WHERE resourceId = {resourceId:String}
|
|
816
|
+
ORDER BY createdAt DESC
|
|
817
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
818
|
+
`,
|
|
819
|
+
query_params: {
|
|
820
|
+
resourceId,
|
|
821
|
+
limit: perPage,
|
|
822
|
+
offset: currentOffset
|
|
823
|
+
},
|
|
824
|
+
clickhouse_settings: {
|
|
825
|
+
date_time_input_format: "best_effort",
|
|
826
|
+
date_time_output_format: "iso",
|
|
827
|
+
use_client_time_zone: 1,
|
|
828
|
+
output_format_json_quote_64bit_integers: 0
|
|
829
|
+
}
|
|
830
|
+
});
|
|
831
|
+
const rows = await dataResult.json();
|
|
832
|
+
const threads = transformRows(rows.data);
|
|
833
|
+
return {
|
|
834
|
+
threads,
|
|
835
|
+
total,
|
|
836
|
+
page,
|
|
837
|
+
perPage,
|
|
838
|
+
hasMore: currentOffset + threads.length < total
|
|
839
|
+
};
|
|
840
|
+
} catch (error$1) {
|
|
841
|
+
throw new error.MastraError(
|
|
842
|
+
{
|
|
843
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
844
|
+
domain: error.ErrorDomain.STORAGE,
|
|
845
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
846
|
+
details: { resourceId, page }
|
|
847
|
+
},
|
|
848
|
+
error$1
|
|
849
|
+
);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
async getMessagesPaginated(args) {
|
|
853
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
854
|
+
const page = selectBy?.pagination?.page || 0;
|
|
855
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
856
|
+
const perPage = perPageInput !== void 0 ? perPageInput : storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
857
|
+
try {
|
|
858
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
859
|
+
const offset = page * perPage;
|
|
860
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
861
|
+
const fromDate = dateRange?.start;
|
|
862
|
+
const toDate = dateRange?.end;
|
|
863
|
+
const messages = [];
|
|
864
|
+
if (selectBy?.include?.length) {
|
|
865
|
+
const include = selectBy.include;
|
|
866
|
+
const unionQueries = [];
|
|
867
|
+
const params = [];
|
|
868
|
+
let paramIdx = 1;
|
|
869
|
+
for (const inc of include) {
|
|
870
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
871
|
+
const searchId = inc.threadId || threadId;
|
|
872
|
+
unionQueries.push(`
|
|
873
|
+
SELECT * FROM (
|
|
874
|
+
WITH numbered_messages AS (
|
|
875
|
+
SELECT
|
|
876
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
877
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
878
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
879
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
880
|
+
),
|
|
881
|
+
target_positions AS (
|
|
882
|
+
SELECT row_num as target_pos
|
|
883
|
+
FROM numbered_messages
|
|
884
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
885
|
+
)
|
|
886
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
887
|
+
FROM numbered_messages m
|
|
888
|
+
CROSS JOIN target_positions t
|
|
889
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
890
|
+
) AS query_${paramIdx}
|
|
891
|
+
`);
|
|
892
|
+
params.push(
|
|
893
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
894
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
895
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
896
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
897
|
+
);
|
|
898
|
+
paramIdx++;
|
|
899
|
+
}
|
|
900
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
901
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
902
|
+
const includeResult = await this.client.query({
|
|
903
|
+
query: finalQuery,
|
|
904
|
+
query_params: mergedParams,
|
|
905
|
+
clickhouse_settings: {
|
|
906
|
+
date_time_input_format: "best_effort",
|
|
907
|
+
date_time_output_format: "iso",
|
|
908
|
+
use_client_time_zone: 1,
|
|
909
|
+
output_format_json_quote_64bit_integers: 0
|
|
910
|
+
}
|
|
911
|
+
});
|
|
912
|
+
const rows2 = await includeResult.json();
|
|
913
|
+
const includedMessages = transformRows(rows2.data);
|
|
914
|
+
const seen = /* @__PURE__ */ new Set();
|
|
915
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
916
|
+
if (seen.has(message.id)) return false;
|
|
917
|
+
seen.add(message.id);
|
|
918
|
+
return true;
|
|
919
|
+
});
|
|
920
|
+
messages.push(...dedupedMessages);
|
|
921
|
+
}
|
|
922
|
+
let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
923
|
+
const countParams = { threadId };
|
|
924
|
+
if (fromDate) {
|
|
925
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
926
|
+
countParams.fromDate = fromDate.toISOString();
|
|
927
|
+
}
|
|
928
|
+
if (toDate) {
|
|
929
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
930
|
+
countParams.toDate = toDate.toISOString();
|
|
931
|
+
}
|
|
932
|
+
const countResult = await this.client.query({
|
|
933
|
+
query: countQuery,
|
|
934
|
+
query_params: countParams,
|
|
935
|
+
clickhouse_settings: {
|
|
936
|
+
date_time_input_format: "best_effort",
|
|
937
|
+
date_time_output_format: "iso",
|
|
938
|
+
use_client_time_zone: 1,
|
|
939
|
+
output_format_json_quote_64bit_integers: 0
|
|
940
|
+
}
|
|
941
|
+
});
|
|
942
|
+
const countData = await countResult.json();
|
|
943
|
+
const total = countData.data[0].total;
|
|
944
|
+
if (total === 0 && messages.length === 0) {
|
|
945
|
+
return {
|
|
946
|
+
messages: [],
|
|
947
|
+
total: 0,
|
|
948
|
+
page,
|
|
949
|
+
perPage,
|
|
950
|
+
hasMore: false
|
|
951
|
+
};
|
|
952
|
+
}
|
|
953
|
+
const excludeIds = messages.map((m) => m.id);
|
|
954
|
+
let dataQuery = `
|
|
955
|
+
SELECT
|
|
956
|
+
id,
|
|
957
|
+
content,
|
|
958
|
+
role,
|
|
959
|
+
type,
|
|
960
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
961
|
+
thread_id AS "threadId",
|
|
962
|
+
resourceId
|
|
963
|
+
FROM ${storage.TABLE_MESSAGES}
|
|
964
|
+
WHERE thread_id = {threadId:String}
|
|
965
|
+
`;
|
|
966
|
+
const dataParams = { threadId };
|
|
967
|
+
if (fromDate) {
|
|
968
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
969
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
970
|
+
}
|
|
971
|
+
if (toDate) {
|
|
972
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
973
|
+
dataParams.toDate = toDate.toISOString();
|
|
974
|
+
}
|
|
975
|
+
if (excludeIds.length > 0) {
|
|
976
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
977
|
+
dataParams.excludeIds = excludeIds;
|
|
978
|
+
}
|
|
979
|
+
if (selectBy?.last) {
|
|
980
|
+
dataQuery += `
|
|
981
|
+
ORDER BY createdAt DESC
|
|
982
|
+
LIMIT {limit:Int64}
|
|
983
|
+
`;
|
|
984
|
+
dataParams.limit = perPage;
|
|
985
|
+
} else {
|
|
986
|
+
dataQuery += `
|
|
987
|
+
ORDER BY createdAt ASC
|
|
988
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
989
|
+
`;
|
|
990
|
+
dataParams.limit = perPage;
|
|
991
|
+
dataParams.offset = offset;
|
|
992
|
+
}
|
|
993
|
+
const result = await this.client.query({
|
|
994
|
+
query: dataQuery,
|
|
995
|
+
query_params: dataParams,
|
|
996
|
+
clickhouse_settings: {
|
|
997
|
+
date_time_input_format: "best_effort",
|
|
998
|
+
date_time_output_format: "iso",
|
|
999
|
+
use_client_time_zone: 1,
|
|
1000
|
+
output_format_json_quote_64bit_integers: 0
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
const rows = await result.json();
|
|
1004
|
+
const paginatedMessages = transformRows(rows.data);
|
|
1005
|
+
messages.push(...paginatedMessages);
|
|
1006
|
+
if (selectBy?.last) {
|
|
1007
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1008
|
+
}
|
|
1009
|
+
return {
|
|
1010
|
+
messages: format === "v2" ? messages : messages,
|
|
1011
|
+
total,
|
|
1012
|
+
page,
|
|
1013
|
+
perPage,
|
|
1014
|
+
hasMore: offset + perPage < total
|
|
1015
|
+
};
|
|
1016
|
+
} catch (error$1) {
|
|
1017
|
+
const mastraError = new error.MastraError(
|
|
1018
|
+
{
|
|
1019
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
1020
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1021
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1022
|
+
details: {
|
|
1023
|
+
threadId,
|
|
1024
|
+
resourceId: resourceId ?? ""
|
|
1025
|
+
}
|
|
1026
|
+
},
|
|
1027
|
+
error$1
|
|
1028
|
+
);
|
|
1029
|
+
this.logger?.trackException?.(mastraError);
|
|
1030
|
+
this.logger?.error?.(mastraError.toString());
|
|
1031
|
+
return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
async updateMessages(args) {
|
|
1035
|
+
const { messages } = args;
|
|
1036
|
+
if (messages.length === 0) {
|
|
1037
|
+
return [];
|
|
1038
|
+
}
|
|
1039
|
+
try {
|
|
1040
|
+
const messageIds = messages.map((m) => m.id);
|
|
1041
|
+
const existingResult = await this.client.query({
|
|
1042
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
1043
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
1044
|
+
clickhouse_settings: {
|
|
1045
|
+
date_time_input_format: "best_effort",
|
|
1046
|
+
date_time_output_format: "iso",
|
|
1047
|
+
use_client_time_zone: 1,
|
|
1048
|
+
output_format_json_quote_64bit_integers: 0
|
|
1049
|
+
}
|
|
1050
|
+
});
|
|
1051
|
+
const existingRows = await existingResult.json();
|
|
1052
|
+
const existingMessages = transformRows(existingRows.data);
|
|
1053
|
+
if (existingMessages.length === 0) {
|
|
1054
|
+
return [];
|
|
1055
|
+
}
|
|
1056
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
1057
|
+
if (typeof msg.content === "string") {
|
|
1058
|
+
try {
|
|
1059
|
+
msg.content = JSON.parse(msg.content);
|
|
1060
|
+
} catch {
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
return msg;
|
|
1064
|
+
});
|
|
1065
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
1066
|
+
const updatePromises = [];
|
|
1067
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1068
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1069
|
+
if (!updatePayload) continue;
|
|
1070
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1071
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1072
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1073
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1074
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1075
|
+
}
|
|
1076
|
+
const setClauses = [];
|
|
1077
|
+
const values = {};
|
|
1078
|
+
let paramIdx = 1;
|
|
1079
|
+
let newContent = null;
|
|
1080
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1081
|
+
if (updatableFields.content) {
|
|
1082
|
+
const existingContent = existingMessage.content || {};
|
|
1083
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1084
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1085
|
+
newContent = {
|
|
1086
|
+
...existingContent,
|
|
1087
|
+
...updatableFields.content,
|
|
1088
|
+
// Deep merge metadata
|
|
1089
|
+
metadata: {
|
|
1090
|
+
...existingMetadata,
|
|
1091
|
+
...updateMetadata
|
|
1092
|
+
}
|
|
1093
|
+
};
|
|
1094
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1095
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1096
|
+
paramIdx++;
|
|
1097
|
+
delete updatableFields.content;
|
|
1098
|
+
}
|
|
1099
|
+
for (const key in updatableFields) {
|
|
1100
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1101
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1102
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1103
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1104
|
+
paramIdx++;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
if (setClauses.length > 0) {
|
|
1108
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1109
|
+
const updateQuery = `
|
|
1110
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
1111
|
+
UPDATE ${setClauses.join(", ")}
|
|
1112
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1113
|
+
`;
|
|
1114
|
+
console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1115
|
+
updatePromises.push(
|
|
1116
|
+
this.client.command({
|
|
1117
|
+
query: updateQuery,
|
|
1118
|
+
query_params: values,
|
|
1119
|
+
clickhouse_settings: {
|
|
1120
|
+
date_time_input_format: "best_effort",
|
|
1121
|
+
use_client_time_zone: 1,
|
|
1122
|
+
output_format_json_quote_64bit_integers: 0
|
|
1123
|
+
}
|
|
1124
|
+
})
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
if (updatePromises.length > 0) {
|
|
1129
|
+
await Promise.all(updatePromises);
|
|
1130
|
+
}
|
|
1131
|
+
await this.client.command({
|
|
1132
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_MESSAGES} FINAL`,
|
|
1133
|
+
clickhouse_settings: {
|
|
1134
|
+
date_time_input_format: "best_effort",
|
|
1135
|
+
use_client_time_zone: 1,
|
|
1136
|
+
output_format_json_quote_64bit_integers: 0
|
|
1137
|
+
}
|
|
1138
|
+
});
|
|
1139
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1140
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1141
|
+
if (!updatePayload) continue;
|
|
1142
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1143
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1144
|
+
const verifyResult = await this.client.query({
|
|
1145
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1146
|
+
query_params: { messageId: id },
|
|
1147
|
+
clickhouse_settings: {
|
|
1148
|
+
date_time_input_format: "best_effort",
|
|
1149
|
+
date_time_output_format: "iso",
|
|
1150
|
+
use_client_time_zone: 1,
|
|
1151
|
+
output_format_json_quote_64bit_integers: 0
|
|
1152
|
+
}
|
|
1153
|
+
});
|
|
1154
|
+
const verifyRows = await verifyResult.json();
|
|
1155
|
+
if (verifyRows.data.length > 0) {
|
|
1156
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1157
|
+
if (updatedMessage) {
|
|
1158
|
+
let needsRetry = false;
|
|
1159
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1160
|
+
if (key === "content") {
|
|
1161
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1162
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1163
|
+
if (actualContent !== expectedContent) {
|
|
1164
|
+
needsRetry = true;
|
|
1165
|
+
break;
|
|
1166
|
+
}
|
|
1167
|
+
} else if (updatedMessage[key] !== value) {
|
|
1168
|
+
needsRetry = true;
|
|
1169
|
+
break;
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
if (needsRetry) {
|
|
1173
|
+
console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1174
|
+
await this.client.command({
|
|
1175
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1176
|
+
query_params: { messageId: id },
|
|
1177
|
+
clickhouse_settings: {
|
|
1178
|
+
date_time_input_format: "best_effort",
|
|
1179
|
+
use_client_time_zone: 1,
|
|
1180
|
+
output_format_json_quote_64bit_integers: 0
|
|
1181
|
+
}
|
|
1182
|
+
});
|
|
1183
|
+
let updatedContent = existingMessage.content || {};
|
|
1184
|
+
if (fieldsToUpdate.content) {
|
|
1185
|
+
const existingContent = existingMessage.content || {};
|
|
1186
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1187
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1188
|
+
updatedContent = {
|
|
1189
|
+
...existingContent,
|
|
1190
|
+
...fieldsToUpdate.content,
|
|
1191
|
+
metadata: {
|
|
1192
|
+
...existingMetadata,
|
|
1193
|
+
...updateMetadata
|
|
1194
|
+
}
|
|
1195
|
+
};
|
|
1196
|
+
}
|
|
1197
|
+
const updatedMessageData = {
|
|
1198
|
+
...existingMessage,
|
|
1199
|
+
...fieldsToUpdate,
|
|
1200
|
+
content: updatedContent
|
|
1201
|
+
};
|
|
1202
|
+
await this.client.insert({
|
|
1203
|
+
table: storage.TABLE_MESSAGES,
|
|
1204
|
+
format: "JSONEachRow",
|
|
1205
|
+
values: [
|
|
1206
|
+
{
|
|
1207
|
+
id: updatedMessageData.id,
|
|
1208
|
+
thread_id: updatedMessageData.threadId,
|
|
1209
|
+
resourceId: updatedMessageData.resourceId,
|
|
1210
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1211
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1212
|
+
role: updatedMessageData.role,
|
|
1213
|
+
type: updatedMessageData.type || "v2"
|
|
1214
|
+
}
|
|
1215
|
+
],
|
|
1216
|
+
clickhouse_settings: {
|
|
1217
|
+
date_time_input_format: "best_effort",
|
|
1218
|
+
use_client_time_zone: 1,
|
|
1219
|
+
output_format_json_quote_64bit_integers: 0
|
|
1220
|
+
}
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1227
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1228
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1229
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1230
|
+
const threadResult = await this.client.query({
|
|
1231
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1232
|
+
query_params: { threadId },
|
|
1233
|
+
clickhouse_settings: {
|
|
1234
|
+
date_time_input_format: "best_effort",
|
|
1235
|
+
date_time_output_format: "iso",
|
|
1236
|
+
use_client_time_zone: 1,
|
|
1237
|
+
output_format_json_quote_64bit_integers: 0
|
|
1238
|
+
}
|
|
1239
|
+
});
|
|
1240
|
+
const threadRows = await threadResult.json();
|
|
1241
|
+
if (threadRows.data.length > 0) {
|
|
1242
|
+
const existingThread = threadRows.data[0];
|
|
1243
|
+
await this.client.command({
|
|
1244
|
+
query: `DELETE FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1245
|
+
query_params: { threadId },
|
|
1246
|
+
clickhouse_settings: {
|
|
1247
|
+
date_time_input_format: "best_effort",
|
|
1248
|
+
use_client_time_zone: 1,
|
|
1249
|
+
output_format_json_quote_64bit_integers: 0
|
|
1250
|
+
}
|
|
1251
|
+
});
|
|
1252
|
+
await this.client.insert({
|
|
1253
|
+
table: storage.TABLE_THREADS,
|
|
1254
|
+
format: "JSONEachRow",
|
|
1255
|
+
values: [
|
|
1256
|
+
{
|
|
1257
|
+
id: existingThread.id,
|
|
1258
|
+
resourceId: existingThread.resourceId,
|
|
1259
|
+
title: existingThread.title,
|
|
1260
|
+
metadata: existingThread.metadata,
|
|
1261
|
+
createdAt: existingThread.createdAt,
|
|
1262
|
+
updatedAt: now
|
|
1263
|
+
}
|
|
1264
|
+
],
|
|
1265
|
+
clickhouse_settings: {
|
|
1266
|
+
date_time_input_format: "best_effort",
|
|
1267
|
+
use_client_time_zone: 1,
|
|
1268
|
+
output_format_json_quote_64bit_integers: 0
|
|
1269
|
+
}
|
|
1270
|
+
});
|
|
1271
|
+
}
|
|
1272
|
+
});
|
|
1273
|
+
await Promise.all(threadUpdatePromises);
|
|
1274
|
+
}
|
|
1275
|
+
const updatedMessages = [];
|
|
1276
|
+
for (const messageId of messageIds) {
|
|
1277
|
+
const updatedResult = await this.client.query({
|
|
1278
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1279
|
+
query_params: { messageId },
|
|
1280
|
+
clickhouse_settings: {
|
|
1281
|
+
date_time_input_format: "best_effort",
|
|
1282
|
+
date_time_output_format: "iso",
|
|
1283
|
+
use_client_time_zone: 1,
|
|
1284
|
+
output_format_json_quote_64bit_integers: 0
|
|
1285
|
+
}
|
|
1286
|
+
});
|
|
1287
|
+
const updatedRows = await updatedResult.json();
|
|
1288
|
+
if (updatedRows.data.length > 0) {
|
|
1289
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1290
|
+
if (message) {
|
|
1291
|
+
updatedMessages.push(message);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
return updatedMessages.map((message) => {
|
|
1296
|
+
if (typeof message.content === "string") {
|
|
1297
|
+
try {
|
|
1298
|
+
message.content = JSON.parse(message.content);
|
|
1299
|
+
} catch {
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
return message;
|
|
1303
|
+
});
|
|
1304
|
+
} catch (error$1) {
|
|
1305
|
+
throw new error.MastraError(
|
|
1306
|
+
{
|
|
1307
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1308
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1309
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1310
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1311
|
+
},
|
|
1312
|
+
error$1
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
async getResourceById({ resourceId }) {
|
|
1317
|
+
try {
|
|
1318
|
+
const result = await this.client.query({
|
|
1319
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${storage.TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1320
|
+
query_params: { resourceId },
|
|
1321
|
+
clickhouse_settings: {
|
|
1322
|
+
date_time_input_format: "best_effort",
|
|
1323
|
+
date_time_output_format: "iso",
|
|
1324
|
+
use_client_time_zone: 1,
|
|
1325
|
+
output_format_json_quote_64bit_integers: 0
|
|
1326
|
+
}
|
|
1327
|
+
});
|
|
1328
|
+
const rows = await result.json();
|
|
1329
|
+
if (rows.data.length === 0) {
|
|
1330
|
+
return null;
|
|
1331
|
+
}
|
|
1332
|
+
const resource = rows.data[0];
|
|
1333
|
+
return {
|
|
1334
|
+
id: resource.id,
|
|
1335
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1336
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1337
|
+
createdAt: new Date(resource.createdAt),
|
|
1338
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1339
|
+
};
|
|
1340
|
+
} catch (error$1) {
|
|
1341
|
+
throw new error.MastraError(
|
|
1342
|
+
{
|
|
1343
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1344
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1345
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1346
|
+
details: { resourceId }
|
|
1347
|
+
},
|
|
1348
|
+
error$1
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
async saveResource({ resource }) {
|
|
1353
|
+
try {
|
|
1354
|
+
await this.client.insert({
|
|
1355
|
+
table: storage.TABLE_RESOURCES,
|
|
1356
|
+
format: "JSONEachRow",
|
|
1357
|
+
values: [
|
|
1358
|
+
{
|
|
1359
|
+
id: resource.id,
|
|
1360
|
+
workingMemory: resource.workingMemory,
|
|
1361
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1362
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1363
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1364
|
+
}
|
|
1365
|
+
],
|
|
1366
|
+
clickhouse_settings: {
|
|
1367
|
+
date_time_input_format: "best_effort",
|
|
1368
|
+
use_client_time_zone: 1,
|
|
1369
|
+
output_format_json_quote_64bit_integers: 0
|
|
1370
|
+
}
|
|
1371
|
+
});
|
|
1372
|
+
return resource;
|
|
1373
|
+
} catch (error$1) {
|
|
1374
|
+
throw new error.MastraError(
|
|
1375
|
+
{
|
|
1376
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1377
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1378
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1379
|
+
details: { resourceId: resource.id }
|
|
1380
|
+
},
|
|
1381
|
+
error$1
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
async updateResource({
|
|
1386
|
+
resourceId,
|
|
1387
|
+
workingMemory,
|
|
1388
|
+
metadata
|
|
1389
|
+
}) {
|
|
1390
|
+
try {
|
|
1391
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1392
|
+
if (!existingResource) {
|
|
1393
|
+
const newResource = {
|
|
1394
|
+
id: resourceId,
|
|
1395
|
+
workingMemory,
|
|
1396
|
+
metadata: metadata || {},
|
|
1397
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1398
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1399
|
+
};
|
|
1400
|
+
return this.saveResource({ resource: newResource });
|
|
1401
|
+
}
|
|
1402
|
+
const updatedResource = {
|
|
1403
|
+
...existingResource,
|
|
1404
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1405
|
+
metadata: {
|
|
1406
|
+
...existingResource.metadata,
|
|
1407
|
+
...metadata
|
|
1408
|
+
},
|
|
1409
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1410
|
+
};
|
|
1411
|
+
const updateQuery = `
|
|
1412
|
+
ALTER TABLE ${storage.TABLE_RESOURCES}
|
|
1413
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1414
|
+
WHERE id = {resourceId:String}
|
|
1415
|
+
`;
|
|
1416
|
+
await this.client.command({
|
|
1417
|
+
query: updateQuery,
|
|
1418
|
+
query_params: {
|
|
1419
|
+
workingMemory: updatedResource.workingMemory,
|
|
1420
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1421
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1422
|
+
resourceId
|
|
1423
|
+
},
|
|
1424
|
+
clickhouse_settings: {
|
|
1425
|
+
date_time_input_format: "best_effort",
|
|
1426
|
+
use_client_time_zone: 1,
|
|
1427
|
+
output_format_json_quote_64bit_integers: 0
|
|
1428
|
+
}
|
|
1429
|
+
});
|
|
1430
|
+
await this.client.command({
|
|
1431
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_RESOURCES} FINAL`,
|
|
1432
|
+
clickhouse_settings: {
|
|
1433
|
+
date_time_input_format: "best_effort",
|
|
1434
|
+
use_client_time_zone: 1,
|
|
1435
|
+
output_format_json_quote_64bit_integers: 0
|
|
1436
|
+
}
|
|
1437
|
+
});
|
|
1438
|
+
return updatedResource;
|
|
1439
|
+
} catch (error$1) {
|
|
1440
|
+
throw new error.MastraError(
|
|
1441
|
+
{
|
|
1442
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1443
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1444
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1445
|
+
details: { resourceId }
|
|
1446
|
+
},
|
|
1447
|
+
error$1
|
|
1448
|
+
);
|
|
193
1449
|
}
|
|
194
|
-
const resp = await result.json();
|
|
195
|
-
const rows = resp.data;
|
|
196
|
-
return rows.map((row) => ({
|
|
197
|
-
id: row.id,
|
|
198
|
-
parentSpanId: row.parentSpanId,
|
|
199
|
-
traceId: row.traceId,
|
|
200
|
-
name: row.name,
|
|
201
|
-
scope: row.scope,
|
|
202
|
-
kind: row.kind,
|
|
203
|
-
status: safelyParseJSON(row.status),
|
|
204
|
-
events: safelyParseJSON(row.events),
|
|
205
|
-
links: safelyParseJSON(row.links),
|
|
206
|
-
attributes: safelyParseJSON(row.attributes),
|
|
207
|
-
startTime: row.startTime,
|
|
208
|
-
endTime: row.endTime,
|
|
209
|
-
other: safelyParseJSON(row.other),
|
|
210
|
-
createdAt: row.createdAt
|
|
211
|
-
}));
|
|
212
1450
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
1451
|
+
};
|
|
1452
|
+
var StoreOperationsClickhouse = class extends storage.StoreOperations {
|
|
1453
|
+
ttl;
|
|
1454
|
+
client;
|
|
1455
|
+
constructor({ client, ttl }) {
|
|
1456
|
+
super();
|
|
1457
|
+
this.ttl = ttl;
|
|
1458
|
+
this.client = client;
|
|
217
1459
|
}
|
|
218
|
-
async
|
|
219
|
-
await this.
|
|
220
|
-
query: `
|
|
1460
|
+
async hasColumn(table, column) {
|
|
1461
|
+
const result = await this.client.query({
|
|
1462
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1463
|
+
format: "JSONEachRow"
|
|
221
1464
|
});
|
|
1465
|
+
const columns = await result.json();
|
|
1466
|
+
return columns.some((c) => c.name === column);
|
|
1467
|
+
}
|
|
1468
|
+
getSqlType(type) {
|
|
1469
|
+
switch (type) {
|
|
1470
|
+
case "text":
|
|
1471
|
+
return "String";
|
|
1472
|
+
case "timestamp":
|
|
1473
|
+
return "DateTime64(3)";
|
|
1474
|
+
case "integer":
|
|
1475
|
+
case "bigint":
|
|
1476
|
+
return "Int64";
|
|
1477
|
+
case "jsonb":
|
|
1478
|
+
return "String";
|
|
1479
|
+
default:
|
|
1480
|
+
return super.getSqlType(type);
|
|
1481
|
+
}
|
|
222
1482
|
}
|
|
223
1483
|
async createTable({
|
|
224
1484
|
tableName,
|
|
@@ -233,25 +1493,25 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
233
1493
|
}).join(",\n");
|
|
234
1494
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
235
1495
|
const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
await this.
|
|
1496
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1497
|
+
${["id String"].concat(columns)}
|
|
1498
|
+
)
|
|
1499
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1500
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1501
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1502
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1503
|
+
SETTINGS index_granularity = 8192
|
|
1504
|
+
` : `
|
|
1505
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1506
|
+
${columns}
|
|
1507
|
+
)
|
|
1508
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1509
|
+
PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1510
|
+
ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1511
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1512
|
+
SETTINGS index_granularity = 8192
|
|
1513
|
+
`;
|
|
1514
|
+
await this.client.query({
|
|
255
1515
|
query: sql,
|
|
256
1516
|
clickhouse_settings: {
|
|
257
1517
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -261,14 +1521,60 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
261
1521
|
output_format_json_quote_64bit_integers: 0
|
|
262
1522
|
}
|
|
263
1523
|
});
|
|
264
|
-
} catch (error) {
|
|
265
|
-
|
|
266
|
-
|
|
1524
|
+
} catch (error$1) {
|
|
1525
|
+
throw new error.MastraError(
|
|
1526
|
+
{
|
|
1527
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1528
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1529
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1530
|
+
details: { tableName }
|
|
1531
|
+
},
|
|
1532
|
+
error$1
|
|
1533
|
+
);
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
async alterTable({
|
|
1537
|
+
tableName,
|
|
1538
|
+
schema,
|
|
1539
|
+
ifNotExists
|
|
1540
|
+
}) {
|
|
1541
|
+
try {
|
|
1542
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1543
|
+
const result = await this.client.query({
|
|
1544
|
+
query: describeSql
|
|
1545
|
+
});
|
|
1546
|
+
const rows = await result.json();
|
|
1547
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1548
|
+
for (const columnName of ifNotExists) {
|
|
1549
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1550
|
+
const columnDef = schema[columnName];
|
|
1551
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1552
|
+
if (columnDef.nullable !== false) {
|
|
1553
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1554
|
+
}
|
|
1555
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1556
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1557
|
+
await this.client.query({
|
|
1558
|
+
query: alterSql
|
|
1559
|
+
});
|
|
1560
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
} catch (error$1) {
|
|
1564
|
+
throw new error.MastraError(
|
|
1565
|
+
{
|
|
1566
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1567
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1568
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1569
|
+
details: { tableName }
|
|
1570
|
+
},
|
|
1571
|
+
error$1
|
|
1572
|
+
);
|
|
267
1573
|
}
|
|
268
1574
|
}
|
|
269
1575
|
async clearTable({ tableName }) {
|
|
270
1576
|
try {
|
|
271
|
-
await this.
|
|
1577
|
+
await this.client.query({
|
|
272
1578
|
query: `TRUNCATE TABLE ${tableName}`,
|
|
273
1579
|
clickhouse_settings: {
|
|
274
1580
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -278,20 +1584,34 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
278
1584
|
output_format_json_quote_64bit_integers: 0
|
|
279
1585
|
}
|
|
280
1586
|
});
|
|
281
|
-
} catch (error) {
|
|
282
|
-
|
|
283
|
-
|
|
1587
|
+
} catch (error$1) {
|
|
1588
|
+
throw new error.MastraError(
|
|
1589
|
+
{
|
|
1590
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1591
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1592
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1593
|
+
details: { tableName }
|
|
1594
|
+
},
|
|
1595
|
+
error$1
|
|
1596
|
+
);
|
|
284
1597
|
}
|
|
285
1598
|
}
|
|
1599
|
+
async dropTable({ tableName }) {
|
|
1600
|
+
await this.client.query({
|
|
1601
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1602
|
+
});
|
|
1603
|
+
}
|
|
286
1604
|
async insert({ tableName, record }) {
|
|
1605
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1606
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
287
1607
|
try {
|
|
288
|
-
await this.
|
|
1608
|
+
const result = await this.client.insert({
|
|
289
1609
|
table: tableName,
|
|
290
1610
|
values: [
|
|
291
1611
|
{
|
|
292
1612
|
...record,
|
|
293
|
-
createdAt
|
|
294
|
-
updatedAt
|
|
1613
|
+
createdAt,
|
|
1614
|
+
updatedAt
|
|
295
1615
|
}
|
|
296
1616
|
],
|
|
297
1617
|
format: "JSONEachRow",
|
|
@@ -302,13 +1622,55 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
302
1622
|
use_client_time_zone: 1
|
|
303
1623
|
}
|
|
304
1624
|
});
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
throw error
|
|
1625
|
+
console.info("INSERT RESULT", result);
|
|
1626
|
+
} catch (error$1) {
|
|
1627
|
+
throw new error.MastraError(
|
|
1628
|
+
{
|
|
1629
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1630
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1631
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1632
|
+
details: { tableName }
|
|
1633
|
+
},
|
|
1634
|
+
error$1
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
async batchInsert({ tableName, records }) {
|
|
1639
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1640
|
+
...Object.fromEntries(
|
|
1641
|
+
Object.entries(record).map(([key, value]) => [
|
|
1642
|
+
key,
|
|
1643
|
+
storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1644
|
+
])
|
|
1645
|
+
)
|
|
1646
|
+
}));
|
|
1647
|
+
try {
|
|
1648
|
+
await this.client.insert({
|
|
1649
|
+
table: tableName,
|
|
1650
|
+
values: recordsToBeInserted,
|
|
1651
|
+
format: "JSONEachRow",
|
|
1652
|
+
clickhouse_settings: {
|
|
1653
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1654
|
+
date_time_input_format: "best_effort",
|
|
1655
|
+
use_client_time_zone: 1,
|
|
1656
|
+
output_format_json_quote_64bit_integers: 0
|
|
1657
|
+
}
|
|
1658
|
+
});
|
|
1659
|
+
} catch (error$1) {
|
|
1660
|
+
throw new error.MastraError(
|
|
1661
|
+
{
|
|
1662
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1663
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1664
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1665
|
+
details: { tableName }
|
|
1666
|
+
},
|
|
1667
|
+
error$1
|
|
1668
|
+
);
|
|
308
1669
|
}
|
|
309
1670
|
}
|
|
310
1671
|
async load({ tableName, keys }) {
|
|
311
1672
|
try {
|
|
1673
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
312
1674
|
const keyEntries = Object.entries(keys);
|
|
313
1675
|
const conditions = keyEntries.map(
|
|
314
1676
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -316,8 +1678,10 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
316
1678
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
317
1679
|
return { ...acc, [`var_${key}`]: value };
|
|
318
1680
|
}, {});
|
|
319
|
-
const
|
|
320
|
-
|
|
1681
|
+
const hasUpdatedAt = storage.TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1682
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1683
|
+
const result = await this.client.query({
|
|
1684
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
321
1685
|
query_params: values,
|
|
322
1686
|
clickhouse_settings: {
|
|
323
1687
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -343,25 +1707,58 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
343
1707
|
}
|
|
344
1708
|
const data = transformRow(rows.data[0]);
|
|
345
1709
|
return data;
|
|
346
|
-
} catch (error) {
|
|
347
|
-
|
|
348
|
-
|
|
1710
|
+
} catch (error$1) {
|
|
1711
|
+
throw new error.MastraError(
|
|
1712
|
+
{
|
|
1713
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1714
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1715
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1716
|
+
details: { tableName }
|
|
1717
|
+
},
|
|
1718
|
+
error$1
|
|
1719
|
+
);
|
|
349
1720
|
}
|
|
350
1721
|
}
|
|
351
|
-
|
|
1722
|
+
};
|
|
1723
|
+
var ScoresStorageClickhouse = class extends storage.ScoresStorage {
|
|
1724
|
+
client;
|
|
1725
|
+
operations;
|
|
1726
|
+
constructor({ client, operations }) {
|
|
1727
|
+
super();
|
|
1728
|
+
this.client = client;
|
|
1729
|
+
this.operations = operations;
|
|
1730
|
+
}
|
|
1731
|
+
transformScoreRow(row) {
|
|
1732
|
+
const scorer = storage.safelyParseJSON(row.scorer);
|
|
1733
|
+
const preprocessStepResult = storage.safelyParseJSON(row.preprocessStepResult);
|
|
1734
|
+
const analyzeStepResult = storage.safelyParseJSON(row.analyzeStepResult);
|
|
1735
|
+
const metadata = storage.safelyParseJSON(row.metadata);
|
|
1736
|
+
const input = storage.safelyParseJSON(row.input);
|
|
1737
|
+
const output = storage.safelyParseJSON(row.output);
|
|
1738
|
+
const additionalContext = storage.safelyParseJSON(row.additionalContext);
|
|
1739
|
+
const runtimeContext = storage.safelyParseJSON(row.runtimeContext);
|
|
1740
|
+
const entity = storage.safelyParseJSON(row.entity);
|
|
1741
|
+
return {
|
|
1742
|
+
...row,
|
|
1743
|
+
scorer,
|
|
1744
|
+
preprocessStepResult,
|
|
1745
|
+
analyzeStepResult,
|
|
1746
|
+
metadata,
|
|
1747
|
+
input,
|
|
1748
|
+
output,
|
|
1749
|
+
additionalContext,
|
|
1750
|
+
runtimeContext,
|
|
1751
|
+
entity,
|
|
1752
|
+
createdAt: new Date(row.createdAt),
|
|
1753
|
+
updatedAt: new Date(row.updatedAt)
|
|
1754
|
+
};
|
|
1755
|
+
}
|
|
1756
|
+
async getScoreById({ id }) {
|
|
352
1757
|
try {
|
|
353
|
-
const result = await this.
|
|
354
|
-
query: `SELECT
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
title,
|
|
358
|
-
metadata,
|
|
359
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
360
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
361
|
-
FROM "${storage.TABLE_THREADS}"
|
|
362
|
-
FINAL
|
|
363
|
-
WHERE id = {var_id:String}`,
|
|
364
|
-
query_params: { var_id: threadId },
|
|
1758
|
+
const result = await this.client.query({
|
|
1759
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1760
|
+
query_params: { var_id: id },
|
|
1761
|
+
format: "JSONEachRow",
|
|
365
1762
|
clickhouse_settings: {
|
|
366
1763
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
367
1764
|
date_time_input_format: "best_effort",
|
|
@@ -370,304 +1767,566 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
370
1767
|
output_format_json_quote_64bit_integers: 0
|
|
371
1768
|
}
|
|
372
1769
|
});
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
if (!thread) {
|
|
1770
|
+
const resultJson = await result.json();
|
|
1771
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
376
1772
|
return null;
|
|
377
1773
|
}
|
|
378
|
-
return
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
1774
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1775
|
+
} catch (error$1) {
|
|
1776
|
+
throw new error.MastraError(
|
|
1777
|
+
{
|
|
1778
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1779
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1780
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1781
|
+
details: { scoreId: id }
|
|
1782
|
+
},
|
|
1783
|
+
error$1
|
|
1784
|
+
);
|
|
387
1785
|
}
|
|
388
1786
|
}
|
|
389
|
-
async
|
|
1787
|
+
async saveScore(score) {
|
|
390
1788
|
try {
|
|
391
|
-
const
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
399
|
-
FROM "${storage.TABLE_THREADS}"
|
|
400
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
401
|
-
query_params: { var_resourceId: resourceId },
|
|
1789
|
+
const record = {
|
|
1790
|
+
...score
|
|
1791
|
+
};
|
|
1792
|
+
await this.client.insert({
|
|
1793
|
+
table: storage.TABLE_SCORERS,
|
|
1794
|
+
values: [record],
|
|
1795
|
+
format: "JSONEachRow",
|
|
402
1796
|
clickhouse_settings: {
|
|
403
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
404
1797
|
date_time_input_format: "best_effort",
|
|
405
|
-
date_time_output_format: "iso",
|
|
406
1798
|
use_client_time_zone: 1,
|
|
407
1799
|
output_format_json_quote_64bit_integers: 0
|
|
408
1800
|
}
|
|
409
1801
|
});
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
1802
|
+
return { score };
|
|
1803
|
+
} catch (error$1) {
|
|
1804
|
+
throw new error.MastraError(
|
|
1805
|
+
{
|
|
1806
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1807
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1808
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1809
|
+
details: { scoreId: score.id }
|
|
1810
|
+
},
|
|
1811
|
+
error$1
|
|
1812
|
+
);
|
|
421
1813
|
}
|
|
422
1814
|
}
|
|
423
|
-
async
|
|
1815
|
+
async getScoresByRunId({
|
|
1816
|
+
runId,
|
|
1817
|
+
pagination
|
|
1818
|
+
}) {
|
|
424
1819
|
try {
|
|
425
|
-
await this.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
]
|
|
1820
|
+
const countResult = await this.client.query({
|
|
1821
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1822
|
+
query_params: { var_runId: runId },
|
|
1823
|
+
format: "JSONEachRow"
|
|
1824
|
+
});
|
|
1825
|
+
const countRows = await countResult.json();
|
|
1826
|
+
let total = 0;
|
|
1827
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1828
|
+
const countObj = countRows[0];
|
|
1829
|
+
total = Number(countObj.count);
|
|
1830
|
+
}
|
|
1831
|
+
if (!total) {
|
|
1832
|
+
return {
|
|
1833
|
+
pagination: {
|
|
1834
|
+
total: 0,
|
|
1835
|
+
page: pagination.page,
|
|
1836
|
+
perPage: pagination.perPage,
|
|
1837
|
+
hasMore: false
|
|
1838
|
+
},
|
|
1839
|
+
scores: []
|
|
1840
|
+
};
|
|
1841
|
+
}
|
|
1842
|
+
const offset = pagination.page * pagination.perPage;
|
|
1843
|
+
const result = await this.client.query({
|
|
1844
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1845
|
+
query_params: {
|
|
1846
|
+
var_runId: runId,
|
|
1847
|
+
var_limit: pagination.perPage,
|
|
1848
|
+
var_offset: offset
|
|
1849
|
+
},
|
|
434
1850
|
format: "JSONEachRow",
|
|
435
1851
|
clickhouse_settings: {
|
|
436
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
437
1852
|
date_time_input_format: "best_effort",
|
|
1853
|
+
date_time_output_format: "iso",
|
|
438
1854
|
use_client_time_zone: 1,
|
|
439
1855
|
output_format_json_quote_64bit_integers: 0
|
|
440
1856
|
}
|
|
441
1857
|
});
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
1858
|
+
const rows = await result.json();
|
|
1859
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1860
|
+
return {
|
|
1861
|
+
pagination: {
|
|
1862
|
+
total,
|
|
1863
|
+
page: pagination.page,
|
|
1864
|
+
perPage: pagination.perPage,
|
|
1865
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1866
|
+
},
|
|
1867
|
+
scores
|
|
1868
|
+
};
|
|
1869
|
+
} catch (error$1) {
|
|
1870
|
+
throw new error.MastraError(
|
|
1871
|
+
{
|
|
1872
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1873
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1874
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1875
|
+
details: { runId }
|
|
1876
|
+
},
|
|
1877
|
+
error$1
|
|
1878
|
+
);
|
|
446
1879
|
}
|
|
447
1880
|
}
|
|
448
|
-
async
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
1881
|
+
async getScoresByScorerId({
|
|
1882
|
+
scorerId,
|
|
1883
|
+
entityId,
|
|
1884
|
+
entityType,
|
|
1885
|
+
source,
|
|
1886
|
+
pagination
|
|
452
1887
|
}) {
|
|
1888
|
+
let whereClause = `scorerId = {var_scorerId:String}`;
|
|
1889
|
+
if (entityId) {
|
|
1890
|
+
whereClause += ` AND entityId = {var_entityId:String}`;
|
|
1891
|
+
}
|
|
1892
|
+
if (entityType) {
|
|
1893
|
+
whereClause += ` AND entityType = {var_entityType:String}`;
|
|
1894
|
+
}
|
|
1895
|
+
if (source) {
|
|
1896
|
+
whereClause += ` AND source = {var_source:String}`;
|
|
1897
|
+
}
|
|
453
1898
|
try {
|
|
454
|
-
const
|
|
455
|
-
|
|
456
|
-
|
|
1899
|
+
const countResult = await this.client.query({
|
|
1900
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE ${whereClause}`,
|
|
1901
|
+
query_params: {
|
|
1902
|
+
var_scorerId: scorerId,
|
|
1903
|
+
var_entityId: entityId,
|
|
1904
|
+
var_entityType: entityType,
|
|
1905
|
+
var_source: source
|
|
1906
|
+
},
|
|
1907
|
+
format: "JSONEachRow"
|
|
1908
|
+
});
|
|
1909
|
+
const countRows = await countResult.json();
|
|
1910
|
+
let total = 0;
|
|
1911
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1912
|
+
const countObj = countRows[0];
|
|
1913
|
+
total = Number(countObj.count);
|
|
457
1914
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
1915
|
+
if (!total) {
|
|
1916
|
+
return {
|
|
1917
|
+
pagination: {
|
|
1918
|
+
total: 0,
|
|
1919
|
+
page: pagination.page,
|
|
1920
|
+
perPage: pagination.perPage,
|
|
1921
|
+
hasMore: false
|
|
1922
|
+
},
|
|
1923
|
+
scores: []
|
|
1924
|
+
};
|
|
1925
|
+
}
|
|
1926
|
+
const offset = pagination.page * pagination.perPage;
|
|
1927
|
+
const result = await this.client.query({
|
|
1928
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1929
|
+
query_params: {
|
|
1930
|
+
var_scorerId: scorerId,
|
|
1931
|
+
var_limit: pagination.perPage,
|
|
1932
|
+
var_offset: offset,
|
|
1933
|
+
var_entityId: entityId,
|
|
1934
|
+
var_entityType: entityType,
|
|
1935
|
+
var_source: source
|
|
1936
|
+
},
|
|
476
1937
|
format: "JSONEachRow",
|
|
477
1938
|
clickhouse_settings: {
|
|
478
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
479
1939
|
date_time_input_format: "best_effort",
|
|
1940
|
+
date_time_output_format: "iso",
|
|
480
1941
|
use_client_time_zone: 1,
|
|
481
1942
|
output_format_json_quote_64bit_integers: 0
|
|
482
1943
|
}
|
|
483
1944
|
});
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
1945
|
+
const rows = await result.json();
|
|
1946
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1947
|
+
return {
|
|
1948
|
+
pagination: {
|
|
1949
|
+
total,
|
|
1950
|
+
page: pagination.page,
|
|
1951
|
+
perPage: pagination.perPage,
|
|
1952
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1953
|
+
},
|
|
1954
|
+
scores
|
|
1955
|
+
};
|
|
1956
|
+
} catch (error$1) {
|
|
1957
|
+
throw new error.MastraError(
|
|
1958
|
+
{
|
|
1959
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1960
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1961
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1962
|
+
details: { scorerId }
|
|
1963
|
+
},
|
|
1964
|
+
error$1
|
|
1965
|
+
);
|
|
488
1966
|
}
|
|
489
1967
|
}
|
|
490
|
-
async
|
|
1968
|
+
async getScoresByEntityId({
|
|
1969
|
+
entityId,
|
|
1970
|
+
entityType,
|
|
1971
|
+
pagination
|
|
1972
|
+
}) {
|
|
491
1973
|
try {
|
|
492
|
-
await this.
|
|
493
|
-
query: `
|
|
494
|
-
query_params: {
|
|
495
|
-
|
|
496
|
-
output_format_json_quote_64bit_integers: 0
|
|
497
|
-
}
|
|
1974
|
+
const countResult = await this.client.query({
|
|
1975
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1976
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1977
|
+
format: "JSONEachRow"
|
|
498
1978
|
});
|
|
499
|
-
await
|
|
500
|
-
|
|
501
|
-
|
|
1979
|
+
const countRows = await countResult.json();
|
|
1980
|
+
let total = 0;
|
|
1981
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1982
|
+
const countObj = countRows[0];
|
|
1983
|
+
total = Number(countObj.count);
|
|
1984
|
+
}
|
|
1985
|
+
if (!total) {
|
|
1986
|
+
return {
|
|
1987
|
+
pagination: {
|
|
1988
|
+
total: 0,
|
|
1989
|
+
page: pagination.page,
|
|
1990
|
+
perPage: pagination.perPage,
|
|
1991
|
+
hasMore: false
|
|
1992
|
+
},
|
|
1993
|
+
scores: []
|
|
1994
|
+
};
|
|
1995
|
+
}
|
|
1996
|
+
const offset = pagination.page * pagination.perPage;
|
|
1997
|
+
const result = await this.client.query({
|
|
1998
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1999
|
+
query_params: {
|
|
2000
|
+
var_entityId: entityId,
|
|
2001
|
+
var_entityType: entityType,
|
|
2002
|
+
var_limit: pagination.perPage,
|
|
2003
|
+
var_offset: offset
|
|
2004
|
+
},
|
|
2005
|
+
format: "JSONEachRow",
|
|
502
2006
|
clickhouse_settings: {
|
|
2007
|
+
date_time_input_format: "best_effort",
|
|
2008
|
+
date_time_output_format: "iso",
|
|
2009
|
+
use_client_time_zone: 1,
|
|
503
2010
|
output_format_json_quote_64bit_integers: 0
|
|
504
2011
|
}
|
|
505
2012
|
});
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
525
|
-
FROM "${storage.TABLE_MESSAGES}"
|
|
526
|
-
WHERE thread_id = {var_thread_id:String}
|
|
527
|
-
)
|
|
528
|
-
SELECT
|
|
529
|
-
m.id AS id,
|
|
530
|
-
m.content as content,
|
|
531
|
-
m.role as role,
|
|
532
|
-
m.type as type,
|
|
533
|
-
m.createdAt as createdAt,
|
|
534
|
-
m.updatedAt as updatedAt,
|
|
535
|
-
m.thread_id AS "threadId"
|
|
536
|
-
FROM ordered_messages m
|
|
537
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
538
|
-
OR EXISTS (
|
|
539
|
-
SELECT 1 FROM ordered_messages target
|
|
540
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
541
|
-
AND (
|
|
542
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
543
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
544
|
-
OR
|
|
545
|
-
-- Get next messages based on the max withNextMessages
|
|
546
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
547
|
-
)
|
|
548
|
-
)
|
|
549
|
-
ORDER BY m."createdAt" DESC
|
|
550
|
-
`,
|
|
551
|
-
query_params: {
|
|
552
|
-
var_thread_id: threadId,
|
|
553
|
-
var_include: include.map((i) => i.id),
|
|
554
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
555
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
556
|
-
},
|
|
557
|
-
clickhouse_settings: {
|
|
558
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
559
|
-
date_time_input_format: "best_effort",
|
|
560
|
-
date_time_output_format: "iso",
|
|
561
|
-
use_client_time_zone: 1,
|
|
562
|
-
output_format_json_quote_64bit_integers: 0
|
|
563
|
-
}
|
|
564
|
-
});
|
|
565
|
-
const rows2 = await includeResult.json();
|
|
566
|
-
messages.push(...transformRows(rows2.data));
|
|
567
|
-
}
|
|
568
|
-
const result = await this.db.query({
|
|
569
|
-
query: `
|
|
570
|
-
SELECT
|
|
571
|
-
id,
|
|
572
|
-
content,
|
|
573
|
-
role,
|
|
574
|
-
type,
|
|
575
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
576
|
-
thread_id AS "threadId"
|
|
577
|
-
FROM "${storage.TABLE_MESSAGES}"
|
|
578
|
-
WHERE thread_id = {threadId:String}
|
|
579
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
580
|
-
ORDER BY "createdAt" DESC
|
|
581
|
-
LIMIT {limit:Int64}
|
|
582
|
-
`,
|
|
583
|
-
query_params: {
|
|
584
|
-
threadId,
|
|
585
|
-
exclude: messages.map((m) => m.id),
|
|
586
|
-
limit
|
|
2013
|
+
const rows = await result.json();
|
|
2014
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2015
|
+
return {
|
|
2016
|
+
pagination: {
|
|
2017
|
+
total,
|
|
2018
|
+
page: pagination.page,
|
|
2019
|
+
perPage: pagination.perPage,
|
|
2020
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
2021
|
+
},
|
|
2022
|
+
scores
|
|
2023
|
+
};
|
|
2024
|
+
} catch (error$1) {
|
|
2025
|
+
throw new error.MastraError(
|
|
2026
|
+
{
|
|
2027
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
2028
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2029
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2030
|
+
details: { entityId, entityType }
|
|
587
2031
|
},
|
|
2032
|
+
error$1
|
|
2033
|
+
);
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
};
|
|
2037
|
+
var TracesStorageClickhouse = class extends storage.TracesStorage {
|
|
2038
|
+
client;
|
|
2039
|
+
operations;
|
|
2040
|
+
constructor({ client, operations }) {
|
|
2041
|
+
super();
|
|
2042
|
+
this.client = client;
|
|
2043
|
+
this.operations = operations;
|
|
2044
|
+
}
|
|
2045
|
+
async getTracesPaginated(args) {
|
|
2046
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
2047
|
+
const fromDate = dateRange?.start;
|
|
2048
|
+
const toDate = dateRange?.end;
|
|
2049
|
+
const currentOffset = page * perPage;
|
|
2050
|
+
const queryArgs = {};
|
|
2051
|
+
const conditions = [];
|
|
2052
|
+
if (name) {
|
|
2053
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2054
|
+
queryArgs.var_name = name;
|
|
2055
|
+
}
|
|
2056
|
+
if (scope) {
|
|
2057
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2058
|
+
queryArgs.var_scope = scope;
|
|
2059
|
+
}
|
|
2060
|
+
if (attributes) {
|
|
2061
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2062
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2063
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
if (filters) {
|
|
2067
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2068
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2069
|
+
queryArgs[`var_col_${key}`] = value;
|
|
2070
|
+
});
|
|
2071
|
+
}
|
|
2072
|
+
if (fromDate) {
|
|
2073
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
2074
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
2075
|
+
}
|
|
2076
|
+
if (toDate) {
|
|
2077
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
2078
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
2079
|
+
}
|
|
2080
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2081
|
+
try {
|
|
2082
|
+
const countResult = await this.client.query({
|
|
2083
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_TRACES} ${whereClause}`,
|
|
2084
|
+
query_params: queryArgs,
|
|
588
2085
|
clickhouse_settings: {
|
|
589
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
590
2086
|
date_time_input_format: "best_effort",
|
|
591
2087
|
date_time_output_format: "iso",
|
|
592
2088
|
use_client_time_zone: 1,
|
|
593
2089
|
output_format_json_quote_64bit_integers: 0
|
|
594
2090
|
}
|
|
595
2091
|
});
|
|
596
|
-
const
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
2092
|
+
const countData = await countResult.json();
|
|
2093
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2094
|
+
if (total === 0) {
|
|
2095
|
+
return {
|
|
2096
|
+
traces: [],
|
|
2097
|
+
total: 0,
|
|
2098
|
+
page,
|
|
2099
|
+
perPage,
|
|
2100
|
+
hasMore: false
|
|
2101
|
+
};
|
|
2102
|
+
}
|
|
2103
|
+
const result = await this.client.query({
|
|
2104
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2105
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2106
|
+
clickhouse_settings: {
|
|
2107
|
+
date_time_input_format: "best_effort",
|
|
2108
|
+
date_time_output_format: "iso",
|
|
2109
|
+
use_client_time_zone: 1,
|
|
2110
|
+
output_format_json_quote_64bit_integers: 0
|
|
605
2111
|
}
|
|
606
2112
|
});
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
2113
|
+
if (!result) {
|
|
2114
|
+
return {
|
|
2115
|
+
traces: [],
|
|
2116
|
+
total,
|
|
2117
|
+
page,
|
|
2118
|
+
perPage,
|
|
2119
|
+
hasMore: false
|
|
2120
|
+
};
|
|
2121
|
+
}
|
|
2122
|
+
const resp = await result.json();
|
|
2123
|
+
const rows = resp.data;
|
|
2124
|
+
const traces = rows.map((row) => ({
|
|
2125
|
+
id: row.id,
|
|
2126
|
+
parentSpanId: row.parentSpanId,
|
|
2127
|
+
traceId: row.traceId,
|
|
2128
|
+
name: row.name,
|
|
2129
|
+
scope: row.scope,
|
|
2130
|
+
kind: row.kind,
|
|
2131
|
+
status: storage.safelyParseJSON(row.status),
|
|
2132
|
+
events: storage.safelyParseJSON(row.events),
|
|
2133
|
+
links: storage.safelyParseJSON(row.links),
|
|
2134
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2135
|
+
startTime: row.startTime,
|
|
2136
|
+
endTime: row.endTime,
|
|
2137
|
+
other: storage.safelyParseJSON(row.other),
|
|
2138
|
+
createdAt: row.createdAt
|
|
2139
|
+
}));
|
|
2140
|
+
return {
|
|
2141
|
+
traces,
|
|
2142
|
+
total,
|
|
2143
|
+
page,
|
|
2144
|
+
perPage,
|
|
2145
|
+
hasMore: currentOffset + traces.length < total
|
|
2146
|
+
};
|
|
2147
|
+
} catch (error$1) {
|
|
2148
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2149
|
+
return {
|
|
2150
|
+
traces: [],
|
|
2151
|
+
total: 0,
|
|
2152
|
+
page,
|
|
2153
|
+
perPage,
|
|
2154
|
+
hasMore: false
|
|
2155
|
+
};
|
|
2156
|
+
}
|
|
2157
|
+
throw new error.MastraError(
|
|
2158
|
+
{
|
|
2159
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2160
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2161
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2162
|
+
details: {
|
|
2163
|
+
name: name ?? null,
|
|
2164
|
+
scope: scope ?? null,
|
|
2165
|
+
page,
|
|
2166
|
+
perPage,
|
|
2167
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2168
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2169
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2170
|
+
}
|
|
2171
|
+
},
|
|
2172
|
+
error$1
|
|
2173
|
+
);
|
|
611
2174
|
}
|
|
612
2175
|
}
|
|
613
|
-
async
|
|
614
|
-
|
|
2176
|
+
async getTraces({
|
|
2177
|
+
name,
|
|
2178
|
+
scope,
|
|
2179
|
+
page,
|
|
2180
|
+
perPage,
|
|
2181
|
+
attributes,
|
|
2182
|
+
filters,
|
|
2183
|
+
fromDate,
|
|
2184
|
+
toDate
|
|
2185
|
+
}) {
|
|
2186
|
+
const limit = perPage;
|
|
2187
|
+
const offset = page * perPage;
|
|
2188
|
+
const args = {};
|
|
2189
|
+
const conditions = [];
|
|
2190
|
+
if (name) {
|
|
2191
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2192
|
+
args.var_name = name;
|
|
2193
|
+
}
|
|
2194
|
+
if (scope) {
|
|
2195
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2196
|
+
args.var_scope = scope;
|
|
2197
|
+
}
|
|
2198
|
+
if (attributes) {
|
|
2199
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2200
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2201
|
+
args[`var_attr_${key}`] = value;
|
|
2202
|
+
});
|
|
2203
|
+
}
|
|
2204
|
+
if (filters) {
|
|
2205
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2206
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2207
|
+
args[`var_col_${key}`] = value;
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
if (fromDate) {
|
|
2211
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2212
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2213
|
+
}
|
|
2214
|
+
if (toDate) {
|
|
2215
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2216
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2217
|
+
}
|
|
2218
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
615
2219
|
try {
|
|
616
|
-
const
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
}
|
|
620
|
-
const thread = await this.getThreadById({ threadId });
|
|
621
|
-
if (!thread) {
|
|
622
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
623
|
-
}
|
|
624
|
-
await this.db.insert({
|
|
625
|
-
table: storage.TABLE_MESSAGES,
|
|
626
|
-
format: "JSONEachRow",
|
|
627
|
-
values: messages.map((message) => ({
|
|
628
|
-
id: message.id,
|
|
629
|
-
thread_id: threadId,
|
|
630
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
631
|
-
createdAt: message.createdAt.toISOString(),
|
|
632
|
-
role: message.role,
|
|
633
|
-
type: message.type
|
|
634
|
-
})),
|
|
2220
|
+
const result = await this.client.query({
|
|
2221
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2222
|
+
query_params: args,
|
|
635
2223
|
clickhouse_settings: {
|
|
636
2224
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
637
2225
|
date_time_input_format: "best_effort",
|
|
2226
|
+
date_time_output_format: "iso",
|
|
638
2227
|
use_client_time_zone: 1,
|
|
639
2228
|
output_format_json_quote_64bit_integers: 0
|
|
640
2229
|
}
|
|
641
2230
|
});
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
2231
|
+
if (!result) {
|
|
2232
|
+
return [];
|
|
2233
|
+
}
|
|
2234
|
+
const resp = await result.json();
|
|
2235
|
+
const rows = resp.data;
|
|
2236
|
+
return rows.map((row) => ({
|
|
2237
|
+
id: row.id,
|
|
2238
|
+
parentSpanId: row.parentSpanId,
|
|
2239
|
+
traceId: row.traceId,
|
|
2240
|
+
name: row.name,
|
|
2241
|
+
scope: row.scope,
|
|
2242
|
+
kind: row.kind,
|
|
2243
|
+
status: storage.safelyParseJSON(row.status),
|
|
2244
|
+
events: storage.safelyParseJSON(row.events),
|
|
2245
|
+
links: storage.safelyParseJSON(row.links),
|
|
2246
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2247
|
+
startTime: row.startTime,
|
|
2248
|
+
endTime: row.endTime,
|
|
2249
|
+
other: storage.safelyParseJSON(row.other),
|
|
2250
|
+
createdAt: row.createdAt
|
|
2251
|
+
}));
|
|
2252
|
+
} catch (error$1) {
|
|
2253
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2254
|
+
return [];
|
|
2255
|
+
}
|
|
2256
|
+
throw new error.MastraError(
|
|
2257
|
+
{
|
|
2258
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2259
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2260
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2261
|
+
details: {
|
|
2262
|
+
name: name ?? null,
|
|
2263
|
+
scope: scope ?? null,
|
|
2264
|
+
page,
|
|
2265
|
+
perPage,
|
|
2266
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2267
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2268
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2269
|
+
toDate: toDate?.toISOString() ?? null
|
|
2270
|
+
}
|
|
2271
|
+
},
|
|
2272
|
+
error$1
|
|
2273
|
+
);
|
|
646
2274
|
}
|
|
647
2275
|
}
|
|
2276
|
+
async batchTraceInsert(args) {
|
|
2277
|
+
await this.operations.batchInsert({ tableName: storage.TABLE_TRACES, records: args.records });
|
|
2278
|
+
}
|
|
2279
|
+
};
|
|
2280
|
+
var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
|
|
2281
|
+
client;
|
|
2282
|
+
operations;
|
|
2283
|
+
constructor({ client, operations }) {
|
|
2284
|
+
super();
|
|
2285
|
+
this.operations = operations;
|
|
2286
|
+
this.client = client;
|
|
2287
|
+
}
|
|
2288
|
+
updateWorkflowResults({
|
|
2289
|
+
// workflowName,
|
|
2290
|
+
// runId,
|
|
2291
|
+
// stepId,
|
|
2292
|
+
// result,
|
|
2293
|
+
// runtimeContext,
|
|
2294
|
+
}) {
|
|
2295
|
+
throw new Error("Method not implemented.");
|
|
2296
|
+
}
|
|
2297
|
+
updateWorkflowState({
|
|
2298
|
+
// workflowName,
|
|
2299
|
+
// runId,
|
|
2300
|
+
// opts,
|
|
2301
|
+
}) {
|
|
2302
|
+
throw new Error("Method not implemented.");
|
|
2303
|
+
}
|
|
648
2304
|
async persistWorkflowSnapshot({
|
|
649
2305
|
workflowName,
|
|
650
2306
|
runId,
|
|
2307
|
+
resourceId,
|
|
651
2308
|
snapshot
|
|
652
2309
|
}) {
|
|
653
2310
|
try {
|
|
654
|
-
const currentSnapshot = await this.load({
|
|
2311
|
+
const currentSnapshot = await this.operations.load({
|
|
655
2312
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
656
2313
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
657
2314
|
});
|
|
658
2315
|
const now = /* @__PURE__ */ new Date();
|
|
659
2316
|
const persisting = currentSnapshot ? {
|
|
660
2317
|
...currentSnapshot,
|
|
2318
|
+
resourceId,
|
|
661
2319
|
snapshot: JSON.stringify(snapshot),
|
|
662
2320
|
updatedAt: now.toISOString()
|
|
663
2321
|
} : {
|
|
664
2322
|
workflow_name: workflowName,
|
|
665
2323
|
run_id: runId,
|
|
2324
|
+
resourceId,
|
|
666
2325
|
snapshot: JSON.stringify(snapshot),
|
|
667
2326
|
createdAt: now.toISOString(),
|
|
668
2327
|
updatedAt: now.toISOString()
|
|
669
2328
|
};
|
|
670
|
-
await this.
|
|
2329
|
+
await this.client.insert({
|
|
671
2330
|
table: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
672
2331
|
format: "JSONEachRow",
|
|
673
2332
|
values: [persisting],
|
|
@@ -678,9 +2337,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
678
2337
|
output_format_json_quote_64bit_integers: 0
|
|
679
2338
|
}
|
|
680
2339
|
});
|
|
681
|
-
} catch (error) {
|
|
682
|
-
|
|
683
|
-
|
|
2340
|
+
} catch (error$1) {
|
|
2341
|
+
throw new error.MastraError(
|
|
2342
|
+
{
|
|
2343
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2344
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2345
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2346
|
+
details: { workflowName, runId }
|
|
2347
|
+
},
|
|
2348
|
+
error$1
|
|
2349
|
+
);
|
|
684
2350
|
}
|
|
685
2351
|
}
|
|
686
2352
|
async loadWorkflowSnapshot({
|
|
@@ -688,7 +2354,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
688
2354
|
runId
|
|
689
2355
|
}) {
|
|
690
2356
|
try {
|
|
691
|
-
const result = await this.load({
|
|
2357
|
+
const result = await this.operations.load({
|
|
692
2358
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
693
2359
|
keys: {
|
|
694
2360
|
workflow_name: workflowName,
|
|
@@ -699,9 +2365,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
699
2365
|
return null;
|
|
700
2366
|
}
|
|
701
2367
|
return result.snapshot;
|
|
702
|
-
} catch (error) {
|
|
703
|
-
|
|
704
|
-
|
|
2368
|
+
} catch (error$1) {
|
|
2369
|
+
throw new error.MastraError(
|
|
2370
|
+
{
|
|
2371
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2372
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2373
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2374
|
+
details: { workflowName, runId }
|
|
2375
|
+
},
|
|
2376
|
+
error$1
|
|
2377
|
+
);
|
|
705
2378
|
}
|
|
706
2379
|
}
|
|
707
2380
|
parseWorkflowRun(row) {
|
|
@@ -738,7 +2411,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
738
2411
|
values.var_workflow_name = workflowName;
|
|
739
2412
|
}
|
|
740
2413
|
if (resourceId) {
|
|
741
|
-
const hasResourceId = await this.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2414
|
+
const hasResourceId = await this.operations.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
742
2415
|
if (hasResourceId) {
|
|
743
2416
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
744
2417
|
values.var_resourceId = resourceId;
|
|
@@ -759,7 +2432,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
759
2432
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
760
2433
|
let total = 0;
|
|
761
2434
|
if (limit !== void 0 && offset !== void 0) {
|
|
762
|
-
const countResult = await this.
|
|
2435
|
+
const countResult = await this.client.query({
|
|
763
2436
|
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
764
2437
|
query_params: values,
|
|
765
2438
|
format: "JSONEachRow"
|
|
@@ -767,21 +2440,21 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
767
2440
|
const countRows = await countResult.json();
|
|
768
2441
|
total = Number(countRows[0]?.count ?? 0);
|
|
769
2442
|
}
|
|
770
|
-
const result = await this.
|
|
2443
|
+
const result = await this.client.query({
|
|
771
2444
|
query: `
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2445
|
+
SELECT
|
|
2446
|
+
workflow_name,
|
|
2447
|
+
run_id,
|
|
2448
|
+
snapshot,
|
|
2449
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2450
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2451
|
+
resourceId
|
|
2452
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2453
|
+
${whereClause}
|
|
2454
|
+
ORDER BY createdAt DESC
|
|
2455
|
+
${limitClause}
|
|
2456
|
+
${offsetClause}
|
|
2457
|
+
`,
|
|
785
2458
|
query_params: values,
|
|
786
2459
|
format: "JSONEachRow"
|
|
787
2460
|
});
|
|
@@ -791,9 +2464,16 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
791
2464
|
return this.parseWorkflowRun(row);
|
|
792
2465
|
});
|
|
793
2466
|
return { runs, total: total || runs.length };
|
|
794
|
-
} catch (error) {
|
|
795
|
-
|
|
796
|
-
|
|
2467
|
+
} catch (error$1) {
|
|
2468
|
+
throw new error.MastraError(
|
|
2469
|
+
{
|
|
2470
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2471
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2472
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2473
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2474
|
+
},
|
|
2475
|
+
error$1
|
|
2476
|
+
);
|
|
797
2477
|
}
|
|
798
2478
|
}
|
|
799
2479
|
async getWorkflowRunById({
|
|
@@ -812,18 +2492,19 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
812
2492
|
values.var_workflow_name = workflowName;
|
|
813
2493
|
}
|
|
814
2494
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
815
|
-
const result = await this.
|
|
2495
|
+
const result = await this.client.query({
|
|
816
2496
|
query: `
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
2497
|
+
SELECT
|
|
2498
|
+
workflow_name,
|
|
2499
|
+
run_id,
|
|
2500
|
+
snapshot,
|
|
2501
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2502
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2503
|
+
resourceId
|
|
2504
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2505
|
+
${whereClause}
|
|
2506
|
+
ORDER BY createdAt DESC LIMIT 1
|
|
2507
|
+
`,
|
|
827
2508
|
query_params: values,
|
|
828
2509
|
format: "JSONEachRow"
|
|
829
2510
|
});
|
|
@@ -832,18 +2513,272 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
832
2513
|
return null;
|
|
833
2514
|
}
|
|
834
2515
|
return this.parseWorkflowRun(resultJson[0]);
|
|
835
|
-
} catch (error) {
|
|
836
|
-
|
|
837
|
-
|
|
2516
|
+
} catch (error$1) {
|
|
2517
|
+
throw new error.MastraError(
|
|
2518
|
+
{
|
|
2519
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2520
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2521
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2522
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2523
|
+
},
|
|
2524
|
+
error$1
|
|
2525
|
+
);
|
|
838
2526
|
}
|
|
839
2527
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
2528
|
+
};
|
|
2529
|
+
|
|
2530
|
+
// src/storage/index.ts
|
|
2531
|
+
var ClickhouseStore = class extends storage.MastraStorage {
|
|
2532
|
+
db;
|
|
2533
|
+
ttl = {};
|
|
2534
|
+
stores;
|
|
2535
|
+
constructor(config) {
|
|
2536
|
+
super({ name: "ClickhouseStore" });
|
|
2537
|
+
this.db = client.createClient({
|
|
2538
|
+
url: config.url,
|
|
2539
|
+
username: config.username,
|
|
2540
|
+
password: config.password,
|
|
2541
|
+
clickhouse_settings: {
|
|
2542
|
+
date_time_input_format: "best_effort",
|
|
2543
|
+
date_time_output_format: "iso",
|
|
2544
|
+
// This is crucial
|
|
2545
|
+
use_client_time_zone: 1,
|
|
2546
|
+
output_format_json_quote_64bit_integers: 0
|
|
2547
|
+
}
|
|
844
2548
|
});
|
|
845
|
-
|
|
846
|
-
|
|
2549
|
+
this.ttl = config.ttl;
|
|
2550
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2551
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2552
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2553
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2554
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2555
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2556
|
+
this.stores = {
|
|
2557
|
+
operations,
|
|
2558
|
+
workflows,
|
|
2559
|
+
scores,
|
|
2560
|
+
legacyEvals,
|
|
2561
|
+
traces,
|
|
2562
|
+
memory
|
|
2563
|
+
};
|
|
2564
|
+
}
|
|
2565
|
+
get supports() {
|
|
2566
|
+
return {
|
|
2567
|
+
selectByIncludeResourceScope: true,
|
|
2568
|
+
resourceWorkingMemory: true,
|
|
2569
|
+
hasColumn: true,
|
|
2570
|
+
createTable: true,
|
|
2571
|
+
deleteMessages: false
|
|
2572
|
+
};
|
|
2573
|
+
}
|
|
2574
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2575
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2576
|
+
}
|
|
2577
|
+
async getEvals(options) {
|
|
2578
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2579
|
+
}
|
|
2580
|
+
async batchInsert({ tableName, records }) {
|
|
2581
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2582
|
+
}
|
|
2583
|
+
async optimizeTable({ tableName }) {
|
|
2584
|
+
try {
|
|
2585
|
+
await this.db.command({
|
|
2586
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2587
|
+
});
|
|
2588
|
+
} catch (error$1) {
|
|
2589
|
+
throw new error.MastraError(
|
|
2590
|
+
{
|
|
2591
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2592
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2593
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2594
|
+
details: { tableName }
|
|
2595
|
+
},
|
|
2596
|
+
error$1
|
|
2597
|
+
);
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
async materializeTtl({ tableName }) {
|
|
2601
|
+
try {
|
|
2602
|
+
await this.db.command({
|
|
2603
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2604
|
+
});
|
|
2605
|
+
} catch (error$1) {
|
|
2606
|
+
throw new error.MastraError(
|
|
2607
|
+
{
|
|
2608
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2609
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2610
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2611
|
+
details: { tableName }
|
|
2612
|
+
},
|
|
2613
|
+
error$1
|
|
2614
|
+
);
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
async createTable({
|
|
2618
|
+
tableName,
|
|
2619
|
+
schema
|
|
2620
|
+
}) {
|
|
2621
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2622
|
+
}
|
|
2623
|
+
async dropTable({ tableName }) {
|
|
2624
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2625
|
+
}
|
|
2626
|
+
async alterTable({
|
|
2627
|
+
tableName,
|
|
2628
|
+
schema,
|
|
2629
|
+
ifNotExists
|
|
2630
|
+
}) {
|
|
2631
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2632
|
+
}
|
|
2633
|
+
async clearTable({ tableName }) {
|
|
2634
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2635
|
+
}
|
|
2636
|
+
async insert({ tableName, record }) {
|
|
2637
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2638
|
+
}
|
|
2639
|
+
async load({ tableName, keys }) {
|
|
2640
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2641
|
+
}
|
|
2642
|
+
async updateWorkflowResults({
|
|
2643
|
+
workflowName,
|
|
2644
|
+
runId,
|
|
2645
|
+
stepId,
|
|
2646
|
+
result,
|
|
2647
|
+
runtimeContext
|
|
2648
|
+
}) {
|
|
2649
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2650
|
+
}
|
|
2651
|
+
async updateWorkflowState({
|
|
2652
|
+
workflowName,
|
|
2653
|
+
runId,
|
|
2654
|
+
opts
|
|
2655
|
+
}) {
|
|
2656
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2657
|
+
}
|
|
2658
|
+
async persistWorkflowSnapshot({
|
|
2659
|
+
workflowName,
|
|
2660
|
+
runId,
|
|
2661
|
+
resourceId,
|
|
2662
|
+
snapshot
|
|
2663
|
+
}) {
|
|
2664
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2665
|
+
}
|
|
2666
|
+
async loadWorkflowSnapshot({
|
|
2667
|
+
workflowName,
|
|
2668
|
+
runId
|
|
2669
|
+
}) {
|
|
2670
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2671
|
+
}
|
|
2672
|
+
async getWorkflowRuns({
|
|
2673
|
+
workflowName,
|
|
2674
|
+
fromDate,
|
|
2675
|
+
toDate,
|
|
2676
|
+
limit,
|
|
2677
|
+
offset,
|
|
2678
|
+
resourceId
|
|
2679
|
+
} = {}) {
|
|
2680
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2681
|
+
}
|
|
2682
|
+
async getWorkflowRunById({
|
|
2683
|
+
runId,
|
|
2684
|
+
workflowName
|
|
2685
|
+
}) {
|
|
2686
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2687
|
+
}
|
|
2688
|
+
async getTraces(args) {
|
|
2689
|
+
return this.stores.traces.getTraces(args);
|
|
2690
|
+
}
|
|
2691
|
+
async getTracesPaginated(args) {
|
|
2692
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2693
|
+
}
|
|
2694
|
+
async batchTraceInsert(args) {
|
|
2695
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2696
|
+
}
|
|
2697
|
+
async getThreadById({ threadId }) {
|
|
2698
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2699
|
+
}
|
|
2700
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2701
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2702
|
+
}
|
|
2703
|
+
async saveThread({ thread }) {
|
|
2704
|
+
return this.stores.memory.saveThread({ thread });
|
|
2705
|
+
}
|
|
2706
|
+
async updateThread({
|
|
2707
|
+
id,
|
|
2708
|
+
title,
|
|
2709
|
+
metadata
|
|
2710
|
+
}) {
|
|
2711
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2712
|
+
}
|
|
2713
|
+
async deleteThread({ threadId }) {
|
|
2714
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2715
|
+
}
|
|
2716
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2717
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2718
|
+
}
|
|
2719
|
+
async getMessages({
|
|
2720
|
+
threadId,
|
|
2721
|
+
resourceId,
|
|
2722
|
+
selectBy,
|
|
2723
|
+
format
|
|
2724
|
+
}) {
|
|
2725
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2726
|
+
}
|
|
2727
|
+
async getMessagesById({
|
|
2728
|
+
messageIds,
|
|
2729
|
+
format
|
|
2730
|
+
}) {
|
|
2731
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2732
|
+
}
|
|
2733
|
+
async saveMessages(args) {
|
|
2734
|
+
return this.stores.memory.saveMessages(args);
|
|
2735
|
+
}
|
|
2736
|
+
async getMessagesPaginated(args) {
|
|
2737
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2738
|
+
}
|
|
2739
|
+
async updateMessages(args) {
|
|
2740
|
+
return this.stores.memory.updateMessages(args);
|
|
2741
|
+
}
|
|
2742
|
+
async getResourceById({ resourceId }) {
|
|
2743
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2744
|
+
}
|
|
2745
|
+
async saveResource({ resource }) {
|
|
2746
|
+
return this.stores.memory.saveResource({ resource });
|
|
2747
|
+
}
|
|
2748
|
+
async updateResource({
|
|
2749
|
+
resourceId,
|
|
2750
|
+
workingMemory,
|
|
2751
|
+
metadata
|
|
2752
|
+
}) {
|
|
2753
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2754
|
+
}
|
|
2755
|
+
async getScoreById({ id }) {
|
|
2756
|
+
return this.stores.scores.getScoreById({ id });
|
|
2757
|
+
}
|
|
2758
|
+
async saveScore(_score) {
|
|
2759
|
+
return this.stores.scores.saveScore(_score);
|
|
2760
|
+
}
|
|
2761
|
+
async getScoresByRunId({
|
|
2762
|
+
runId,
|
|
2763
|
+
pagination
|
|
2764
|
+
}) {
|
|
2765
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2766
|
+
}
|
|
2767
|
+
async getScoresByEntityId({
|
|
2768
|
+
entityId,
|
|
2769
|
+
entityType,
|
|
2770
|
+
pagination
|
|
2771
|
+
}) {
|
|
2772
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2773
|
+
}
|
|
2774
|
+
async getScoresByScorerId({
|
|
2775
|
+
scorerId,
|
|
2776
|
+
pagination,
|
|
2777
|
+
entityId,
|
|
2778
|
+
entityType,
|
|
2779
|
+
source
|
|
2780
|
+
}) {
|
|
2781
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
|
|
847
2782
|
}
|
|
848
2783
|
async close() {
|
|
849
2784
|
await this.db.close();
|
|
@@ -853,3 +2788,5 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
853
2788
|
exports.COLUMN_TYPES = COLUMN_TYPES;
|
|
854
2789
|
exports.ClickhouseStore = ClickhouseStore;
|
|
855
2790
|
exports.TABLE_ENGINES = TABLE_ENGINES;
|
|
2791
|
+
//# sourceMappingURL=index.cjs.map
|
|
2792
|
+
//# sourceMappingURL=index.cjs.map
|