@mastra/clickhouse 0.0.0-tsconfig-compile-20250703214351 → 0.0.0-unified-sidebar-20251010130811
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +524 -4
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2278 -598
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2278 -598
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +87 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +54 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +55 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +245 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +25 -11
- package/dist/_tsup-dts-rollup.d.cts +0 -191
- package/dist/_tsup-dts-rollup.d.ts +0 -191
- package/dist/index.d.cts +0 -4
- package/docker-compose.yaml +0 -15
- package/eslint.config.js +0 -6
- package/src/index.ts +0 -1
- package/src/storage/index.test.ts +0 -1154
- package/src/storage/index.ts +0 -1464
- package/tsconfig.json +0 -5
- package/vitest.config.ts +0 -12
package/dist/index.cjs
CHANGED
|
@@ -1,24 +1,22 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var client = require('@clickhouse/client');
|
|
4
|
-
var agent = require('@mastra/core/agent');
|
|
5
4
|
var error = require('@mastra/core/error');
|
|
6
5
|
var storage = require('@mastra/core/storage');
|
|
6
|
+
var agent = require('@mastra/core/agent');
|
|
7
|
+
var scores = require('@mastra/core/scores');
|
|
7
8
|
|
|
8
9
|
// src/storage/index.ts
|
|
9
|
-
function safelyParseJSON(jsonString) {
|
|
10
|
-
try {
|
|
11
|
-
return JSON.parse(jsonString);
|
|
12
|
-
} catch {
|
|
13
|
-
return {};
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
10
|
var TABLE_ENGINES = {
|
|
17
11
|
[storage.TABLE_MESSAGES]: `MergeTree()`,
|
|
18
12
|
[storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
19
13
|
[storage.TABLE_TRACES]: `MergeTree()`,
|
|
20
14
|
[storage.TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
21
|
-
[storage.TABLE_EVALS]: `MergeTree()
|
|
15
|
+
[storage.TABLE_EVALS]: `MergeTree()`,
|
|
16
|
+
[storage.TABLE_SCORERS]: `MergeTree()`,
|
|
17
|
+
[storage.TABLE_RESOURCES]: `ReplacingMergeTree()`,
|
|
18
|
+
// TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
|
|
19
|
+
[storage.TABLE_AI_SPANS]: `ReplacingMergeTree()`
|
|
22
20
|
};
|
|
23
21
|
var COLUMN_TYPES = {
|
|
24
22
|
text: "String",
|
|
@@ -26,11 +24,10 @@ var COLUMN_TYPES = {
|
|
|
26
24
|
uuid: "String",
|
|
27
25
|
jsonb: "String",
|
|
28
26
|
integer: "Int64",
|
|
29
|
-
|
|
27
|
+
float: "Float64",
|
|
28
|
+
bigint: "Int64",
|
|
29
|
+
boolean: "Bool"
|
|
30
30
|
};
|
|
31
|
-
function transformRows(rows) {
|
|
32
|
-
return rows.map((row) => transformRow(row));
|
|
33
|
-
}
|
|
34
31
|
function transformRow(row) {
|
|
35
32
|
if (!row) {
|
|
36
33
|
return row;
|
|
@@ -41,31 +38,56 @@ function transformRow(row) {
|
|
|
41
38
|
if (row.updatedAt) {
|
|
42
39
|
row.updatedAt = new Date(row.updatedAt);
|
|
43
40
|
}
|
|
41
|
+
if (row.content && typeof row.content === "string") {
|
|
42
|
+
row.content = storage.safelyParseJSON(row.content);
|
|
43
|
+
}
|
|
44
44
|
return row;
|
|
45
45
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
// This is crucial
|
|
59
|
-
use_client_time_zone: 1,
|
|
60
|
-
output_format_json_quote_64bit_integers: 0
|
|
61
|
-
}
|
|
62
|
-
});
|
|
63
|
-
this.ttl = config.ttl;
|
|
46
|
+
function transformRows(rows) {
|
|
47
|
+
return rows.map((row) => transformRow(row));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
51
|
+
var LegacyEvalsStorageClickhouse = class extends storage.LegacyEvalsStorage {
|
|
52
|
+
client;
|
|
53
|
+
operations;
|
|
54
|
+
constructor({ client, operations }) {
|
|
55
|
+
super();
|
|
56
|
+
this.client = client;
|
|
57
|
+
this.operations = operations;
|
|
64
58
|
}
|
|
65
59
|
transformEvalRow(row) {
|
|
66
60
|
row = transformRow(row);
|
|
67
|
-
|
|
68
|
-
|
|
61
|
+
let resultValue;
|
|
62
|
+
try {
|
|
63
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
64
|
+
resultValue = JSON.parse(row.result);
|
|
65
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
66
|
+
resultValue = row.result;
|
|
67
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
68
|
+
resultValue = { score: 0 };
|
|
69
|
+
} else {
|
|
70
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
71
|
+
}
|
|
72
|
+
} catch (error$1) {
|
|
73
|
+
console.error("Error parsing result field:", row.result, error$1);
|
|
74
|
+
throw new error.MastraError({
|
|
75
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
76
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
77
|
+
domain: error.ErrorDomain.STORAGE,
|
|
78
|
+
category: error.ErrorCategory.USER
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
let testInfoValue;
|
|
82
|
+
try {
|
|
83
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
84
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
85
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
86
|
+
testInfoValue = row.test_info;
|
|
87
|
+
}
|
|
88
|
+
} catch {
|
|
89
|
+
testInfoValue = void 0;
|
|
90
|
+
}
|
|
69
91
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
70
92
|
throw new error.MastraError({
|
|
71
93
|
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
@@ -87,23 +109,11 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
87
109
|
createdAt: row.created_at
|
|
88
110
|
};
|
|
89
111
|
}
|
|
90
|
-
escape(value) {
|
|
91
|
-
if (typeof value === "string") {
|
|
92
|
-
return `'${value.replace(/'/g, "''")}'`;
|
|
93
|
-
}
|
|
94
|
-
if (value instanceof Date) {
|
|
95
|
-
return `'${value.toISOString()}'`;
|
|
96
|
-
}
|
|
97
|
-
if (value === null || value === void 0) {
|
|
98
|
-
return "NULL";
|
|
99
|
-
}
|
|
100
|
-
return value.toString();
|
|
101
|
-
}
|
|
102
112
|
async getEvalsByAgentName(agentName, type) {
|
|
103
113
|
try {
|
|
104
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
105
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
106
|
-
const result = await this.
|
|
114
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
115
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
116
|
+
const result = await this.client.query({
|
|
107
117
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
108
118
|
query_params: { var_agent_name: agentName },
|
|
109
119
|
clickhouse_settings: {
|
|
@@ -133,388 +143,438 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
133
143
|
);
|
|
134
144
|
}
|
|
135
145
|
}
|
|
136
|
-
async
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
values: records.map((record) => ({
|
|
141
|
-
...Object.fromEntries(
|
|
142
|
-
Object.entries(record).map(([key, value]) => [
|
|
143
|
-
key,
|
|
144
|
-
storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
145
|
-
])
|
|
146
|
-
)
|
|
147
|
-
})),
|
|
148
|
-
format: "JSONEachRow",
|
|
149
|
-
clickhouse_settings: {
|
|
150
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
151
|
-
date_time_input_format: "best_effort",
|
|
152
|
-
use_client_time_zone: 1,
|
|
153
|
-
output_format_json_quote_64bit_integers: 0
|
|
154
|
-
}
|
|
155
|
-
});
|
|
156
|
-
} catch (error$1) {
|
|
157
|
-
throw new error.MastraError(
|
|
158
|
-
{
|
|
159
|
-
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
160
|
-
domain: error.ErrorDomain.STORAGE,
|
|
161
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
162
|
-
details: { tableName }
|
|
163
|
-
},
|
|
164
|
-
error$1
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
async getTraces({
|
|
169
|
-
name,
|
|
170
|
-
scope,
|
|
171
|
-
page,
|
|
172
|
-
perPage,
|
|
173
|
-
attributes,
|
|
174
|
-
filters,
|
|
175
|
-
fromDate,
|
|
176
|
-
toDate
|
|
177
|
-
}) {
|
|
178
|
-
const limit = perPage;
|
|
179
|
-
const offset = page * perPage;
|
|
180
|
-
const args = {};
|
|
146
|
+
async getEvals(options = {}) {
|
|
147
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
148
|
+
const fromDate = dateRange?.start;
|
|
149
|
+
const toDate = dateRange?.end;
|
|
181
150
|
const conditions = [];
|
|
182
|
-
if (
|
|
183
|
-
conditions.push(`
|
|
184
|
-
args.var_name = name;
|
|
185
|
-
}
|
|
186
|
-
if (scope) {
|
|
187
|
-
conditions.push(`scope = {var_scope:String}`);
|
|
188
|
-
args.var_scope = scope;
|
|
151
|
+
if (agentName) {
|
|
152
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
189
153
|
}
|
|
190
|
-
if (
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
conditions.push(
|
|
199
|
-
`${key} = {var_col_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
|
|
200
|
-
);
|
|
201
|
-
args[`var_col_${key}`] = value;
|
|
202
|
-
});
|
|
154
|
+
if (type === "test") {
|
|
155
|
+
conditions.push(
|
|
156
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
157
|
+
);
|
|
158
|
+
} else if (type === "live") {
|
|
159
|
+
conditions.push(
|
|
160
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
161
|
+
);
|
|
203
162
|
}
|
|
204
163
|
if (fromDate) {
|
|
205
|
-
conditions.push(`
|
|
206
|
-
|
|
164
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
165
|
+
fromDate.toISOString();
|
|
207
166
|
}
|
|
208
167
|
if (toDate) {
|
|
209
|
-
conditions.push(`
|
|
210
|
-
|
|
168
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
169
|
+
toDate.toISOString();
|
|
211
170
|
}
|
|
212
171
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
213
172
|
try {
|
|
214
|
-
const
|
|
215
|
-
query: `SELECT
|
|
216
|
-
query_params:
|
|
173
|
+
const countResult = await this.client.query({
|
|
174
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_EVALS} ${whereClause}`,
|
|
175
|
+
query_params: {
|
|
176
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
177
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
178
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
179
|
+
},
|
|
217
180
|
clickhouse_settings: {
|
|
218
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
219
181
|
date_time_input_format: "best_effort",
|
|
220
182
|
date_time_output_format: "iso",
|
|
221
183
|
use_client_time_zone: 1,
|
|
222
184
|
output_format_json_quote_64bit_integers: 0
|
|
223
185
|
}
|
|
224
186
|
});
|
|
225
|
-
|
|
226
|
-
|
|
187
|
+
const countData = await countResult.json();
|
|
188
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
189
|
+
const currentOffset = page * perPage;
|
|
190
|
+
const hasMore = currentOffset + perPage < total;
|
|
191
|
+
if (total === 0) {
|
|
192
|
+
return {
|
|
193
|
+
evals: [],
|
|
194
|
+
total: 0,
|
|
195
|
+
page,
|
|
196
|
+
perPage,
|
|
197
|
+
hasMore: false
|
|
198
|
+
};
|
|
227
199
|
}
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
200
|
+
const dataResult = await this.client.query({
|
|
201
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
202
|
+
query_params: {
|
|
203
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
204
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
205
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
206
|
+
var_limit: perPage || 100,
|
|
207
|
+
var_offset: currentOffset || 0
|
|
208
|
+
},
|
|
209
|
+
clickhouse_settings: {
|
|
210
|
+
date_time_input_format: "best_effort",
|
|
211
|
+
date_time_output_format: "iso",
|
|
212
|
+
use_client_time_zone: 1,
|
|
213
|
+
output_format_json_quote_64bit_integers: 0
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
const rows = await dataResult.json();
|
|
217
|
+
return {
|
|
218
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
219
|
+
total,
|
|
220
|
+
page,
|
|
221
|
+
perPage,
|
|
222
|
+
hasMore
|
|
223
|
+
};
|
|
246
224
|
} catch (error$1) {
|
|
247
225
|
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
248
|
-
return
|
|
226
|
+
return {
|
|
227
|
+
evals: [],
|
|
228
|
+
total: 0,
|
|
229
|
+
page,
|
|
230
|
+
perPage,
|
|
231
|
+
hasMore: false
|
|
232
|
+
};
|
|
249
233
|
}
|
|
250
234
|
throw new error.MastraError(
|
|
251
235
|
{
|
|
252
|
-
id: "
|
|
236
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
253
237
|
domain: error.ErrorDomain.STORAGE,
|
|
254
238
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
255
|
-
details: {
|
|
256
|
-
name: name ?? null,
|
|
257
|
-
scope: scope ?? null,
|
|
258
|
-
page,
|
|
259
|
-
perPage,
|
|
260
|
-
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
261
|
-
filters: filters ? JSON.stringify(filters) : null,
|
|
262
|
-
fromDate: fromDate?.toISOString() ?? null,
|
|
263
|
-
toDate: toDate?.toISOString() ?? null
|
|
264
|
-
}
|
|
239
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
265
240
|
},
|
|
266
241
|
error$1
|
|
267
242
|
);
|
|
268
243
|
}
|
|
269
244
|
}
|
|
270
|
-
|
|
245
|
+
};
|
|
246
|
+
var MemoryStorageClickhouse = class extends storage.MemoryStorage {
|
|
247
|
+
client;
|
|
248
|
+
operations;
|
|
249
|
+
constructor({ client, operations }) {
|
|
250
|
+
super();
|
|
251
|
+
this.client = client;
|
|
252
|
+
this.operations = operations;
|
|
253
|
+
}
|
|
254
|
+
async getMessages({
|
|
255
|
+
threadId,
|
|
256
|
+
resourceId,
|
|
257
|
+
selectBy,
|
|
258
|
+
format
|
|
259
|
+
}) {
|
|
271
260
|
try {
|
|
272
|
-
|
|
273
|
-
|
|
261
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
262
|
+
const messages = [];
|
|
263
|
+
const limit = storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
264
|
+
const include = selectBy?.include || [];
|
|
265
|
+
if (include.length) {
|
|
266
|
+
const unionQueries = [];
|
|
267
|
+
const params = [];
|
|
268
|
+
let paramIdx = 1;
|
|
269
|
+
for (const inc of include) {
|
|
270
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
271
|
+
const searchId = inc.threadId || threadId;
|
|
272
|
+
unionQueries.push(`
|
|
273
|
+
SELECT * FROM (
|
|
274
|
+
WITH numbered_messages AS (
|
|
275
|
+
SELECT
|
|
276
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
277
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
278
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
279
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
280
|
+
),
|
|
281
|
+
target_positions AS (
|
|
282
|
+
SELECT row_num as target_pos
|
|
283
|
+
FROM numbered_messages
|
|
284
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
285
|
+
)
|
|
286
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
287
|
+
FROM numbered_messages m
|
|
288
|
+
CROSS JOIN target_positions t
|
|
289
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
290
|
+
) AS query_${paramIdx}
|
|
291
|
+
`);
|
|
292
|
+
params.push(
|
|
293
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
294
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
295
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
296
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
297
|
+
);
|
|
298
|
+
paramIdx++;
|
|
299
|
+
}
|
|
300
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
301
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
302
|
+
const includeResult = await this.client.query({
|
|
303
|
+
query: finalQuery,
|
|
304
|
+
query_params: mergedParams,
|
|
305
|
+
clickhouse_settings: {
|
|
306
|
+
date_time_input_format: "best_effort",
|
|
307
|
+
date_time_output_format: "iso",
|
|
308
|
+
use_client_time_zone: 1,
|
|
309
|
+
output_format_json_quote_64bit_integers: 0
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
const rows2 = await includeResult.json();
|
|
313
|
+
const includedMessages = transformRows(rows2.data);
|
|
314
|
+
const seen = /* @__PURE__ */ new Set();
|
|
315
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
316
|
+
if (seen.has(message.id)) return false;
|
|
317
|
+
seen.add(message.id);
|
|
318
|
+
return true;
|
|
319
|
+
});
|
|
320
|
+
messages.push(...dedupedMessages);
|
|
321
|
+
}
|
|
322
|
+
const result = await this.client.query({
|
|
323
|
+
query: `
|
|
324
|
+
SELECT
|
|
325
|
+
id,
|
|
326
|
+
content,
|
|
327
|
+
role,
|
|
328
|
+
type,
|
|
329
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
330
|
+
thread_id AS "threadId"
|
|
331
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
332
|
+
WHERE thread_id = {threadId:String}
|
|
333
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
334
|
+
ORDER BY "createdAt" DESC
|
|
335
|
+
LIMIT {limit:Int64}
|
|
336
|
+
`,
|
|
337
|
+
query_params: {
|
|
338
|
+
threadId,
|
|
339
|
+
exclude: messages.map((m) => m.id),
|
|
340
|
+
limit
|
|
341
|
+
},
|
|
342
|
+
clickhouse_settings: {
|
|
343
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
344
|
+
date_time_input_format: "best_effort",
|
|
345
|
+
date_time_output_format: "iso",
|
|
346
|
+
use_client_time_zone: 1,
|
|
347
|
+
output_format_json_quote_64bit_integers: 0
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
const rows = await result.json();
|
|
351
|
+
messages.push(...transformRows(rows.data));
|
|
352
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
353
|
+
messages.forEach((message) => {
|
|
354
|
+
if (typeof message.content === "string") {
|
|
355
|
+
try {
|
|
356
|
+
message.content = JSON.parse(message.content);
|
|
357
|
+
} catch {
|
|
358
|
+
}
|
|
359
|
+
}
|
|
274
360
|
});
|
|
361
|
+
const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
362
|
+
if (format === `v2`) return list.get.all.v2();
|
|
363
|
+
return list.get.all.v1();
|
|
275
364
|
} catch (error$1) {
|
|
276
365
|
throw new error.MastraError(
|
|
277
366
|
{
|
|
278
|
-
id: "
|
|
367
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
279
368
|
domain: error.ErrorDomain.STORAGE,
|
|
280
369
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
281
|
-
details: {
|
|
370
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
282
371
|
},
|
|
283
372
|
error$1
|
|
284
373
|
);
|
|
285
374
|
}
|
|
286
375
|
}
|
|
287
|
-
async
|
|
376
|
+
async getMessagesById({
|
|
377
|
+
messageIds,
|
|
378
|
+
format
|
|
379
|
+
}) {
|
|
380
|
+
if (messageIds.length === 0) return [];
|
|
288
381
|
try {
|
|
289
|
-
await this.
|
|
290
|
-
query: `
|
|
382
|
+
const result = await this.client.query({
|
|
383
|
+
query: `
|
|
384
|
+
SELECT
|
|
385
|
+
id,
|
|
386
|
+
content,
|
|
387
|
+
role,
|
|
388
|
+
type,
|
|
389
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
390
|
+
thread_id AS "threadId",
|
|
391
|
+
"resourceId"
|
|
392
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
393
|
+
WHERE id IN {messageIds:Array(String)}
|
|
394
|
+
ORDER BY "createdAt" DESC
|
|
395
|
+
`,
|
|
396
|
+
query_params: {
|
|
397
|
+
messageIds
|
|
398
|
+
},
|
|
399
|
+
clickhouse_settings: {
|
|
400
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
401
|
+
date_time_input_format: "best_effort",
|
|
402
|
+
date_time_output_format: "iso",
|
|
403
|
+
use_client_time_zone: 1,
|
|
404
|
+
output_format_json_quote_64bit_integers: 0
|
|
405
|
+
}
|
|
406
|
+
});
|
|
407
|
+
const rows = await result.json();
|
|
408
|
+
const messages = transformRows(rows.data);
|
|
409
|
+
messages.forEach((message) => {
|
|
410
|
+
if (typeof message.content === "string") {
|
|
411
|
+
try {
|
|
412
|
+
message.content = JSON.parse(message.content);
|
|
413
|
+
} catch {
|
|
414
|
+
}
|
|
415
|
+
}
|
|
291
416
|
});
|
|
417
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
418
|
+
if (format === `v1`) return list.get.all.v1();
|
|
419
|
+
return list.get.all.v2();
|
|
292
420
|
} catch (error$1) {
|
|
293
421
|
throw new error.MastraError(
|
|
294
422
|
{
|
|
295
|
-
id: "
|
|
423
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
|
|
296
424
|
domain: error.ErrorDomain.STORAGE,
|
|
297
425
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
298
|
-
details: {
|
|
426
|
+
details: { messageIds: JSON.stringify(messageIds) }
|
|
299
427
|
},
|
|
300
428
|
error$1
|
|
301
429
|
);
|
|
302
430
|
}
|
|
303
431
|
}
|
|
304
|
-
async
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
}
|
|
315
|
-
const
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
${["id String"].concat(columns)}
|
|
319
|
-
)
|
|
320
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
321
|
-
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
322
|
-
ORDER BY (createdAt, run_id, workflow_name)
|
|
323
|
-
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
324
|
-
SETTINGS index_granularity = 8192
|
|
325
|
-
` : `
|
|
326
|
-
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
327
|
-
${columns}
|
|
328
|
-
)
|
|
329
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
330
|
-
PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
331
|
-
ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
332
|
-
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
333
|
-
SETTINGS index_granularity = 8192
|
|
334
|
-
`;
|
|
335
|
-
await this.db.query({
|
|
336
|
-
query: sql,
|
|
337
|
-
clickhouse_settings: {
|
|
338
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
339
|
-
date_time_input_format: "best_effort",
|
|
340
|
-
date_time_output_format: "iso",
|
|
341
|
-
use_client_time_zone: 1,
|
|
342
|
-
output_format_json_quote_64bit_integers: 0
|
|
343
|
-
}
|
|
344
|
-
});
|
|
345
|
-
} catch (error$1) {
|
|
346
|
-
throw new error.MastraError(
|
|
347
|
-
{
|
|
348
|
-
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
349
|
-
domain: error.ErrorDomain.STORAGE,
|
|
350
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
351
|
-
details: { tableName }
|
|
352
|
-
},
|
|
353
|
-
error$1
|
|
354
|
-
);
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
getSqlType(type) {
|
|
358
|
-
switch (type) {
|
|
359
|
-
case "text":
|
|
360
|
-
return "String";
|
|
361
|
-
case "timestamp":
|
|
362
|
-
return "DateTime64(3)";
|
|
363
|
-
case "integer":
|
|
364
|
-
case "bigint":
|
|
365
|
-
return "Int64";
|
|
366
|
-
case "jsonb":
|
|
367
|
-
return "String";
|
|
368
|
-
default:
|
|
369
|
-
return super.getSqlType(type);
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
/**
|
|
373
|
-
* Alters table schema to add columns if they don't exist
|
|
374
|
-
* @param tableName Name of the table
|
|
375
|
-
* @param schema Schema of the table
|
|
376
|
-
* @param ifNotExists Array of column names to add if they don't exist
|
|
377
|
-
*/
|
|
378
|
-
async alterTable({
|
|
379
|
-
tableName,
|
|
380
|
-
schema,
|
|
381
|
-
ifNotExists
|
|
382
|
-
}) {
|
|
383
|
-
try {
|
|
384
|
-
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
385
|
-
const result = await this.db.query({
|
|
386
|
-
query: describeSql
|
|
387
|
-
});
|
|
388
|
-
const rows = await result.json();
|
|
389
|
-
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
390
|
-
for (const columnName of ifNotExists) {
|
|
391
|
-
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
392
|
-
const columnDef = schema[columnName];
|
|
393
|
-
let sqlType = this.getSqlType(columnDef.type);
|
|
394
|
-
if (columnDef.nullable !== false) {
|
|
395
|
-
sqlType = `Nullable(${sqlType})`;
|
|
396
|
-
}
|
|
397
|
-
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
398
|
-
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
399
|
-
await this.db.query({
|
|
400
|
-
query: alterSql
|
|
401
|
-
});
|
|
402
|
-
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
403
|
-
}
|
|
432
|
+
async saveMessages(args) {
|
|
433
|
+
const { messages, format = "v1" } = args;
|
|
434
|
+
if (messages.length === 0) return messages;
|
|
435
|
+
for (const message of messages) {
|
|
436
|
+
const resourceId = message.resourceId;
|
|
437
|
+
if (!resourceId) {
|
|
438
|
+
throw new Error("Resource ID is required");
|
|
439
|
+
}
|
|
440
|
+
if (!message.threadId) {
|
|
441
|
+
throw new Error("Thread ID is required");
|
|
442
|
+
}
|
|
443
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
444
|
+
if (!thread) {
|
|
445
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
404
446
|
}
|
|
405
|
-
} catch (error$1) {
|
|
406
|
-
throw new error.MastraError(
|
|
407
|
-
{
|
|
408
|
-
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
409
|
-
domain: error.ErrorDomain.STORAGE,
|
|
410
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
411
|
-
details: { tableName }
|
|
412
|
-
},
|
|
413
|
-
error$1
|
|
414
|
-
);
|
|
415
447
|
}
|
|
416
|
-
|
|
417
|
-
|
|
448
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
449
|
+
await Promise.all(
|
|
450
|
+
messages.map(async (m) => {
|
|
451
|
+
const resourceId = m.resourceId;
|
|
452
|
+
if (!resourceId) {
|
|
453
|
+
throw new Error("Resource ID is required");
|
|
454
|
+
}
|
|
455
|
+
if (!m.threadId) {
|
|
456
|
+
throw new Error("Thread ID is required");
|
|
457
|
+
}
|
|
458
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
459
|
+
if (!thread) {
|
|
460
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
461
|
+
}
|
|
462
|
+
threadIdSet.set(m.threadId, thread);
|
|
463
|
+
})
|
|
464
|
+
);
|
|
418
465
|
try {
|
|
419
|
-
await this.
|
|
420
|
-
query: `
|
|
466
|
+
const existingResult = await this.client.query({
|
|
467
|
+
query: `SELECT id, thread_id FROM ${storage.TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
468
|
+
query_params: {
|
|
469
|
+
ids: messages.map((m) => m.id)
|
|
470
|
+
},
|
|
421
471
|
clickhouse_settings: {
|
|
422
472
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
423
473
|
date_time_input_format: "best_effort",
|
|
424
474
|
date_time_output_format: "iso",
|
|
425
475
|
use_client_time_zone: 1,
|
|
426
476
|
output_format_json_quote_64bit_integers: 0
|
|
427
|
-
}
|
|
428
|
-
});
|
|
429
|
-
} catch (error$1) {
|
|
430
|
-
throw new error.MastraError(
|
|
431
|
-
{
|
|
432
|
-
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
433
|
-
domain: error.ErrorDomain.STORAGE,
|
|
434
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
435
|
-
details: { tableName }
|
|
436
477
|
},
|
|
437
|
-
|
|
438
|
-
);
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
478
|
+
format: "JSONEachRow"
|
|
479
|
+
});
|
|
480
|
+
const existingRows = await existingResult.json();
|
|
481
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
482
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
483
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
484
|
+
const toMove = messages.filter((m) => {
|
|
485
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
486
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
487
|
+
});
|
|
488
|
+
const deletePromises = toMove.map((message) => {
|
|
489
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
490
|
+
if (!existingRow) return Promise.resolve();
|
|
491
|
+
return this.client.command({
|
|
492
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
493
|
+
query_params: {
|
|
494
|
+
var_id: message.id,
|
|
495
|
+
var_old_thread_id: existingRow.thread_id
|
|
496
|
+
},
|
|
497
|
+
clickhouse_settings: {
|
|
498
|
+
date_time_input_format: "best_effort",
|
|
499
|
+
use_client_time_zone: 1,
|
|
500
|
+
output_format_json_quote_64bit_integers: 0
|
|
450
501
|
}
|
|
451
|
-
|
|
452
|
-
format: "JSONEachRow",
|
|
453
|
-
clickhouse_settings: {
|
|
454
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
455
|
-
output_format_json_quote_64bit_integers: 0,
|
|
456
|
-
date_time_input_format: "best_effort",
|
|
457
|
-
use_client_time_zone: 1
|
|
458
|
-
}
|
|
502
|
+
});
|
|
459
503
|
});
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
504
|
+
const updatePromises = toUpdate.map(
|
|
505
|
+
(message) => this.client.command({
|
|
506
|
+
query: `
|
|
507
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
508
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
509
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
510
|
+
`,
|
|
511
|
+
query_params: {
|
|
512
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
513
|
+
var_role: message.role,
|
|
514
|
+
var_type: message.type || "v2",
|
|
515
|
+
var_resourceId: message.resourceId,
|
|
516
|
+
var_id: message.id,
|
|
517
|
+
var_thread_id: message.threadId
|
|
518
|
+
},
|
|
519
|
+
clickhouse_settings: {
|
|
520
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
521
|
+
date_time_input_format: "best_effort",
|
|
522
|
+
use_client_time_zone: 1,
|
|
523
|
+
output_format_json_quote_64bit_integers: 0
|
|
524
|
+
}
|
|
525
|
+
})
|
|
469
526
|
);
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
527
|
+
await Promise.all([
|
|
528
|
+
// Insert new messages (including moved messages)
|
|
529
|
+
this.client.insert({
|
|
530
|
+
table: storage.TABLE_MESSAGES,
|
|
531
|
+
format: "JSONEachRow",
|
|
532
|
+
values: toInsert.map((message) => ({
|
|
533
|
+
id: message.id,
|
|
534
|
+
thread_id: message.threadId,
|
|
535
|
+
resourceId: message.resourceId,
|
|
536
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
537
|
+
createdAt: message.createdAt.toISOString(),
|
|
538
|
+
role: message.role,
|
|
539
|
+
type: message.type || "v2"
|
|
540
|
+
})),
|
|
541
|
+
clickhouse_settings: {
|
|
542
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
543
|
+
date_time_input_format: "best_effort",
|
|
544
|
+
use_client_time_zone: 1,
|
|
545
|
+
output_format_json_quote_64bit_integers: 0
|
|
546
|
+
}
|
|
547
|
+
}),
|
|
548
|
+
...updatePromises,
|
|
549
|
+
...deletePromises,
|
|
550
|
+
// Update thread's updatedAt timestamp
|
|
551
|
+
this.client.insert({
|
|
552
|
+
table: storage.TABLE_THREADS,
|
|
553
|
+
format: "JSONEachRow",
|
|
554
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
555
|
+
id: thread.id,
|
|
556
|
+
resourceId: thread.resourceId,
|
|
557
|
+
title: thread.title,
|
|
558
|
+
metadata: thread.metadata,
|
|
559
|
+
createdAt: thread.createdAt,
|
|
560
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
561
|
+
})),
|
|
562
|
+
clickhouse_settings: {
|
|
563
|
+
date_time_input_format: "best_effort",
|
|
564
|
+
use_client_time_zone: 1,
|
|
565
|
+
output_format_json_quote_64bit_integers: 0
|
|
566
|
+
}
|
|
567
|
+
})
|
|
568
|
+
]);
|
|
569
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
570
|
+
if (format === `v2`) return list.get.all.v2();
|
|
571
|
+
return list.get.all.v1();
|
|
511
572
|
} catch (error$1) {
|
|
512
573
|
throw new error.MastraError(
|
|
513
574
|
{
|
|
514
|
-
id: "
|
|
575
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
515
576
|
domain: error.ErrorDomain.STORAGE,
|
|
516
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
517
|
-
details: { tableName }
|
|
577
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
518
578
|
},
|
|
519
579
|
error$1
|
|
520
580
|
);
|
|
@@ -522,7 +582,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
522
582
|
}
|
|
523
583
|
async getThreadById({ threadId }) {
|
|
524
584
|
try {
|
|
525
|
-
const result = await this.
|
|
585
|
+
const result = await this.client.query({
|
|
526
586
|
query: `SELECT
|
|
527
587
|
id,
|
|
528
588
|
"resourceId",
|
|
@@ -567,7 +627,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
567
627
|
}
|
|
568
628
|
async getThreadsByResourceId({ resourceId }) {
|
|
569
629
|
try {
|
|
570
|
-
const result = await this.
|
|
630
|
+
const result = await this.client.query({
|
|
571
631
|
query: `SELECT
|
|
572
632
|
id,
|
|
573
633
|
"resourceId",
|
|
@@ -608,7 +668,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
608
668
|
}
|
|
609
669
|
async saveThread({ thread }) {
|
|
610
670
|
try {
|
|
611
|
-
await this.
|
|
671
|
+
await this.client.insert({
|
|
612
672
|
table: storage.TABLE_THREADS,
|
|
613
673
|
values: [
|
|
614
674
|
{
|
|
@@ -658,7 +718,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
658
718
|
metadata: mergedMetadata,
|
|
659
719
|
updatedAt: /* @__PURE__ */ new Date()
|
|
660
720
|
};
|
|
661
|
-
await this.
|
|
721
|
+
await this.client.insert({
|
|
662
722
|
table: storage.TABLE_THREADS,
|
|
663
723
|
format: "JSONEachRow",
|
|
664
724
|
values: [
|
|
@@ -692,14 +752,14 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
692
752
|
}
|
|
693
753
|
async deleteThread({ threadId }) {
|
|
694
754
|
try {
|
|
695
|
-
await this.
|
|
755
|
+
await this.client.command({
|
|
696
756
|
query: `DELETE FROM "${storage.TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
697
757
|
query_params: { var_thread_id: threadId },
|
|
698
758
|
clickhouse_settings: {
|
|
699
759
|
output_format_json_quote_64bit_integers: 0
|
|
700
760
|
}
|
|
701
761
|
});
|
|
702
|
-
await this.
|
|
762
|
+
await this.client.command({
|
|
703
763
|
query: `DELETE FROM "${storage.TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
704
764
|
query_params: { var_id: threadId },
|
|
705
765
|
clickhouse_settings: {
|
|
@@ -718,254 +778,1644 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
718
778
|
);
|
|
719
779
|
}
|
|
720
780
|
}
|
|
721
|
-
async
|
|
722
|
-
|
|
723
|
-
resourceId,
|
|
724
|
-
selectBy,
|
|
725
|
-
format
|
|
726
|
-
}) {
|
|
781
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
782
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
727
783
|
try {
|
|
728
|
-
const
|
|
729
|
-
const
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
m.updatedAt as updatedAt,
|
|
750
|
-
m.thread_id AS "threadId"
|
|
751
|
-
FROM ordered_messages m
|
|
752
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
753
|
-
OR EXISTS (
|
|
754
|
-
SELECT 1 FROM ordered_messages target
|
|
755
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
756
|
-
AND (
|
|
757
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
758
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
759
|
-
OR
|
|
760
|
-
-- Get next messages based on the max withNextMessages
|
|
761
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
762
|
-
)
|
|
763
|
-
)
|
|
764
|
-
ORDER BY m."createdAt" DESC
|
|
765
|
-
`,
|
|
766
|
-
query_params: {
|
|
767
|
-
var_thread_id: threadId,
|
|
768
|
-
var_include: include.map((i) => i.id),
|
|
769
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
770
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
771
|
-
},
|
|
772
|
-
clickhouse_settings: {
|
|
773
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
774
|
-
date_time_input_format: "best_effort",
|
|
775
|
-
date_time_output_format: "iso",
|
|
776
|
-
use_client_time_zone: 1,
|
|
777
|
-
output_format_json_quote_64bit_integers: 0
|
|
778
|
-
}
|
|
779
|
-
});
|
|
780
|
-
const rows2 = await includeResult.json();
|
|
781
|
-
messages.push(...transformRows(rows2.data));
|
|
784
|
+
const currentOffset = page * perPage;
|
|
785
|
+
const countResult = await this.client.query({
|
|
786
|
+
query: `SELECT count() as total FROM ${storage.TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
787
|
+
query_params: { resourceId },
|
|
788
|
+
clickhouse_settings: {
|
|
789
|
+
date_time_input_format: "best_effort",
|
|
790
|
+
date_time_output_format: "iso",
|
|
791
|
+
use_client_time_zone: 1,
|
|
792
|
+
output_format_json_quote_64bit_integers: 0
|
|
793
|
+
}
|
|
794
|
+
});
|
|
795
|
+
const countData = await countResult.json();
|
|
796
|
+
const total = countData.data[0].total;
|
|
797
|
+
if (total === 0) {
|
|
798
|
+
return {
|
|
799
|
+
threads: [],
|
|
800
|
+
total: 0,
|
|
801
|
+
page,
|
|
802
|
+
perPage,
|
|
803
|
+
hasMore: false
|
|
804
|
+
};
|
|
782
805
|
}
|
|
783
|
-
const
|
|
806
|
+
const dataResult = await this.client.query({
|
|
784
807
|
query: `
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
`,
|
|
808
|
+
SELECT
|
|
809
|
+
id,
|
|
810
|
+
resourceId,
|
|
811
|
+
title,
|
|
812
|
+
metadata,
|
|
813
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
814
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
815
|
+
FROM ${storage.TABLE_THREADS}
|
|
816
|
+
WHERE resourceId = {resourceId:String}
|
|
817
|
+
ORDER BY createdAt DESC
|
|
818
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
819
|
+
`,
|
|
798
820
|
query_params: {
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
821
|
+
resourceId,
|
|
822
|
+
limit: perPage,
|
|
823
|
+
offset: currentOffset
|
|
802
824
|
},
|
|
803
825
|
clickhouse_settings: {
|
|
804
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
805
826
|
date_time_input_format: "best_effort",
|
|
806
827
|
date_time_output_format: "iso",
|
|
807
828
|
use_client_time_zone: 1,
|
|
808
829
|
output_format_json_quote_64bit_integers: 0
|
|
809
830
|
}
|
|
810
831
|
});
|
|
811
|
-
const rows = await
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
}
|
|
821
|
-
});
|
|
822
|
-
const list = new agent.MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
823
|
-
if (format === `v2`) return list.get.all.v2();
|
|
824
|
-
return list.get.all.v1();
|
|
832
|
+
const rows = await dataResult.json();
|
|
833
|
+
const threads = transformRows(rows.data);
|
|
834
|
+
return {
|
|
835
|
+
threads,
|
|
836
|
+
total,
|
|
837
|
+
page,
|
|
838
|
+
perPage,
|
|
839
|
+
hasMore: currentOffset + threads.length < total
|
|
840
|
+
};
|
|
825
841
|
} catch (error$1) {
|
|
826
842
|
throw new error.MastraError(
|
|
827
843
|
{
|
|
828
|
-
id: "
|
|
844
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
829
845
|
domain: error.ErrorDomain.STORAGE,
|
|
830
846
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
831
|
-
details: {
|
|
847
|
+
details: { resourceId, page }
|
|
832
848
|
},
|
|
833
849
|
error$1
|
|
834
850
|
);
|
|
835
851
|
}
|
|
836
852
|
}
|
|
837
|
-
async
|
|
838
|
-
const {
|
|
839
|
-
|
|
853
|
+
async getMessagesPaginated(args) {
|
|
854
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
855
|
+
const page = selectBy?.pagination?.page || 0;
|
|
856
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
857
|
+
const perPage = perPageInput !== void 0 ? perPageInput : storage.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
840
858
|
try {
|
|
841
|
-
|
|
842
|
-
const
|
|
843
|
-
|
|
844
|
-
|
|
859
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
860
|
+
const offset = page * perPage;
|
|
861
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
862
|
+
const fromDate = dateRange?.start;
|
|
863
|
+
const toDate = dateRange?.end;
|
|
864
|
+
const messages = [];
|
|
865
|
+
if (selectBy?.include?.length) {
|
|
866
|
+
const include = selectBy.include;
|
|
867
|
+
const unionQueries = [];
|
|
868
|
+
const params = [];
|
|
869
|
+
let paramIdx = 1;
|
|
870
|
+
for (const inc of include) {
|
|
871
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
872
|
+
const searchId = inc.threadId || threadId;
|
|
873
|
+
unionQueries.push(`
|
|
874
|
+
SELECT * FROM (
|
|
875
|
+
WITH numbered_messages AS (
|
|
876
|
+
SELECT
|
|
877
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
878
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
879
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
880
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
881
|
+
),
|
|
882
|
+
target_positions AS (
|
|
883
|
+
SELECT row_num as target_pos
|
|
884
|
+
FROM numbered_messages
|
|
885
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
886
|
+
)
|
|
887
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
888
|
+
FROM numbered_messages m
|
|
889
|
+
CROSS JOIN target_positions t
|
|
890
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
891
|
+
) AS query_${paramIdx}
|
|
892
|
+
`);
|
|
893
|
+
params.push(
|
|
894
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
895
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
896
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
897
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
898
|
+
);
|
|
899
|
+
paramIdx++;
|
|
900
|
+
}
|
|
901
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
902
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
903
|
+
const includeResult = await this.client.query({
|
|
904
|
+
query: finalQuery,
|
|
905
|
+
query_params: mergedParams,
|
|
906
|
+
clickhouse_settings: {
|
|
907
|
+
date_time_input_format: "best_effort",
|
|
908
|
+
date_time_output_format: "iso",
|
|
909
|
+
use_client_time_zone: 1,
|
|
910
|
+
output_format_json_quote_64bit_integers: 0
|
|
911
|
+
}
|
|
912
|
+
});
|
|
913
|
+
const rows2 = await includeResult.json();
|
|
914
|
+
const includedMessages = transformRows(rows2.data);
|
|
915
|
+
const seen = /* @__PURE__ */ new Set();
|
|
916
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
917
|
+
if (seen.has(message.id)) return false;
|
|
918
|
+
seen.add(message.id);
|
|
919
|
+
return true;
|
|
920
|
+
});
|
|
921
|
+
messages.push(...dedupedMessages);
|
|
845
922
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
923
|
+
let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
924
|
+
const countParams = { threadId };
|
|
925
|
+
if (fromDate) {
|
|
926
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
927
|
+
countParams.fromDate = fromDate.toISOString();
|
|
849
928
|
}
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
929
|
+
if (toDate) {
|
|
930
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
931
|
+
countParams.toDate = toDate.toISOString();
|
|
932
|
+
}
|
|
933
|
+
const countResult = await this.client.query({
|
|
934
|
+
query: countQuery,
|
|
935
|
+
query_params: countParams,
|
|
936
|
+
clickhouse_settings: {
|
|
937
|
+
date_time_input_format: "best_effort",
|
|
938
|
+
date_time_output_format: "iso",
|
|
939
|
+
use_client_time_zone: 1,
|
|
940
|
+
output_format_json_quote_64bit_integers: 0
|
|
941
|
+
}
|
|
942
|
+
});
|
|
943
|
+
const countData = await countResult.json();
|
|
944
|
+
const total = countData.data[0].total;
|
|
945
|
+
if (total === 0 && messages.length === 0) {
|
|
946
|
+
return {
|
|
947
|
+
messages: [],
|
|
948
|
+
total: 0,
|
|
949
|
+
page,
|
|
950
|
+
perPage,
|
|
951
|
+
hasMore: false
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
const excludeIds = messages.map((m) => m.id);
|
|
955
|
+
let dataQuery = `
|
|
956
|
+
SELECT
|
|
957
|
+
id,
|
|
958
|
+
content,
|
|
959
|
+
role,
|
|
960
|
+
type,
|
|
961
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
962
|
+
thread_id AS "threadId",
|
|
963
|
+
resourceId
|
|
964
|
+
FROM ${storage.TABLE_MESSAGES}
|
|
965
|
+
WHERE thread_id = {threadId:String}
|
|
966
|
+
`;
|
|
967
|
+
const dataParams = { threadId };
|
|
968
|
+
if (fromDate) {
|
|
969
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
970
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
971
|
+
}
|
|
972
|
+
if (toDate) {
|
|
973
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
974
|
+
dataParams.toDate = toDate.toISOString();
|
|
975
|
+
}
|
|
976
|
+
if (excludeIds.length > 0) {
|
|
977
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
978
|
+
dataParams.excludeIds = excludeIds;
|
|
979
|
+
}
|
|
980
|
+
if (selectBy?.last) {
|
|
981
|
+
dataQuery += `
|
|
982
|
+
ORDER BY createdAt DESC
|
|
983
|
+
LIMIT {limit:Int64}
|
|
984
|
+
`;
|
|
985
|
+
dataParams.limit = perPage;
|
|
986
|
+
} else {
|
|
987
|
+
dataQuery += `
|
|
988
|
+
ORDER BY createdAt ASC
|
|
989
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
990
|
+
`;
|
|
991
|
+
dataParams.limit = perPage;
|
|
992
|
+
dataParams.offset = offset;
|
|
993
|
+
}
|
|
994
|
+
const result = await this.client.query({
|
|
995
|
+
query: dataQuery,
|
|
996
|
+
query_params: dataParams,
|
|
855
997
|
clickhouse_settings: {
|
|
856
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
857
998
|
date_time_input_format: "best_effort",
|
|
858
999
|
date_time_output_format: "iso",
|
|
859
1000
|
use_client_time_zone: 1,
|
|
860
1001
|
output_format_json_quote_64bit_integers: 0
|
|
1002
|
+
}
|
|
1003
|
+
});
|
|
1004
|
+
const rows = await result.json();
|
|
1005
|
+
const paginatedMessages = transformRows(rows.data);
|
|
1006
|
+
messages.push(...paginatedMessages);
|
|
1007
|
+
if (selectBy?.last) {
|
|
1008
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1009
|
+
}
|
|
1010
|
+
return {
|
|
1011
|
+
messages: format === "v2" ? messages : messages,
|
|
1012
|
+
total,
|
|
1013
|
+
page,
|
|
1014
|
+
perPage,
|
|
1015
|
+
hasMore: offset + perPage < total
|
|
1016
|
+
};
|
|
1017
|
+
} catch (error$1) {
|
|
1018
|
+
const mastraError = new error.MastraError(
|
|
1019
|
+
{
|
|
1020
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
1021
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1022
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1023
|
+
details: {
|
|
1024
|
+
threadId,
|
|
1025
|
+
resourceId: resourceId ?? ""
|
|
1026
|
+
}
|
|
861
1027
|
},
|
|
862
|
-
|
|
1028
|
+
error$1
|
|
1029
|
+
);
|
|
1030
|
+
this.logger?.trackException?.(mastraError);
|
|
1031
|
+
this.logger?.error?.(mastraError.toString());
|
|
1032
|
+
return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
async updateMessages(args) {
|
|
1036
|
+
const { messages } = args;
|
|
1037
|
+
if (messages.length === 0) {
|
|
1038
|
+
return [];
|
|
1039
|
+
}
|
|
1040
|
+
try {
|
|
1041
|
+
const messageIds = messages.map((m) => m.id);
|
|
1042
|
+
const existingResult = await this.client.query({
|
|
1043
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
1044
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
1045
|
+
clickhouse_settings: {
|
|
1046
|
+
date_time_input_format: "best_effort",
|
|
1047
|
+
date_time_output_format: "iso",
|
|
1048
|
+
use_client_time_zone: 1,
|
|
1049
|
+
output_format_json_quote_64bit_integers: 0
|
|
1050
|
+
}
|
|
863
1051
|
});
|
|
864
1052
|
const existingRows = await existingResult.json();
|
|
865
|
-
const
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
`,
|
|
875
|
-
query_params: {
|
|
876
|
-
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
877
|
-
var_role: message.role,
|
|
878
|
-
var_type: message.type || "v2",
|
|
879
|
-
var_id: message.id,
|
|
880
|
-
var_thread_id: threadId
|
|
881
|
-
},
|
|
882
|
-
clickhouse_settings: {
|
|
883
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
884
|
-
date_time_input_format: "best_effort",
|
|
885
|
-
use_client_time_zone: 1,
|
|
886
|
-
output_format_json_quote_64bit_integers: 0
|
|
1053
|
+
const existingMessages = transformRows(existingRows.data);
|
|
1054
|
+
if (existingMessages.length === 0) {
|
|
1055
|
+
return [];
|
|
1056
|
+
}
|
|
1057
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
1058
|
+
if (typeof msg.content === "string") {
|
|
1059
|
+
try {
|
|
1060
|
+
msg.content = JSON.parse(msg.content);
|
|
1061
|
+
} catch {
|
|
887
1062
|
}
|
|
888
|
-
}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1063
|
+
}
|
|
1064
|
+
return msg;
|
|
1065
|
+
});
|
|
1066
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
1067
|
+
const updatePromises = [];
|
|
1068
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1069
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1070
|
+
if (!updatePayload) continue;
|
|
1071
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1072
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1073
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1074
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1075
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1076
|
+
}
|
|
1077
|
+
const setClauses = [];
|
|
1078
|
+
const values = {};
|
|
1079
|
+
let paramIdx = 1;
|
|
1080
|
+
let newContent = null;
|
|
1081
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1082
|
+
if (updatableFields.content) {
|
|
1083
|
+
const existingContent = existingMessage.content || {};
|
|
1084
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1085
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1086
|
+
newContent = {
|
|
1087
|
+
...existingContent,
|
|
1088
|
+
...updatableFields.content,
|
|
1089
|
+
// Deep merge metadata
|
|
1090
|
+
metadata: {
|
|
1091
|
+
...existingMetadata,
|
|
1092
|
+
...updateMetadata
|
|
1093
|
+
}
|
|
1094
|
+
};
|
|
1095
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1096
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1097
|
+
paramIdx++;
|
|
1098
|
+
delete updatableFields.content;
|
|
1099
|
+
}
|
|
1100
|
+
for (const key in updatableFields) {
|
|
1101
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1102
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1103
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1104
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1105
|
+
paramIdx++;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
if (setClauses.length > 0) {
|
|
1109
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1110
|
+
const updateQuery = `
|
|
1111
|
+
ALTER TABLE ${storage.TABLE_MESSAGES}
|
|
1112
|
+
UPDATE ${setClauses.join(", ")}
|
|
1113
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1114
|
+
`;
|
|
1115
|
+
console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1116
|
+
updatePromises.push(
|
|
1117
|
+
this.client.command({
|
|
1118
|
+
query: updateQuery,
|
|
1119
|
+
query_params: values,
|
|
1120
|
+
clickhouse_settings: {
|
|
1121
|
+
date_time_input_format: "best_effort",
|
|
1122
|
+
use_client_time_zone: 1,
|
|
1123
|
+
output_format_json_quote_64bit_integers: 0
|
|
1124
|
+
}
|
|
1125
|
+
})
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
if (updatePromises.length > 0) {
|
|
1130
|
+
await Promise.all(updatePromises);
|
|
1131
|
+
}
|
|
1132
|
+
await this.client.command({
|
|
1133
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_MESSAGES} FINAL`,
|
|
1134
|
+
clickhouse_settings: {
|
|
1135
|
+
date_time_input_format: "best_effort",
|
|
1136
|
+
use_client_time_zone: 1,
|
|
1137
|
+
output_format_json_quote_64bit_integers: 0
|
|
1138
|
+
}
|
|
1139
|
+
});
|
|
1140
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1141
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1142
|
+
if (!updatePayload) continue;
|
|
1143
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1144
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1145
|
+
const verifyResult = await this.client.query({
|
|
1146
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1147
|
+
query_params: { messageId: id },
|
|
903
1148
|
clickhouse_settings: {
|
|
904
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
905
1149
|
date_time_input_format: "best_effort",
|
|
1150
|
+
date_time_output_format: "iso",
|
|
906
1151
|
use_client_time_zone: 1,
|
|
907
1152
|
output_format_json_quote_64bit_integers: 0
|
|
908
1153
|
}
|
|
909
|
-
})
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1154
|
+
});
|
|
1155
|
+
const verifyRows = await verifyResult.json();
|
|
1156
|
+
if (verifyRows.data.length > 0) {
|
|
1157
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1158
|
+
if (updatedMessage) {
|
|
1159
|
+
let needsRetry = false;
|
|
1160
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1161
|
+
if (key === "content") {
|
|
1162
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1163
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1164
|
+
if (actualContent !== expectedContent) {
|
|
1165
|
+
needsRetry = true;
|
|
1166
|
+
break;
|
|
1167
|
+
}
|
|
1168
|
+
} else if (updatedMessage[key] !== value) {
|
|
1169
|
+
needsRetry = true;
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
if (needsRetry) {
|
|
1174
|
+
console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1175
|
+
await this.client.command({
|
|
1176
|
+
query: `DELETE FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1177
|
+
query_params: { messageId: id },
|
|
1178
|
+
clickhouse_settings: {
|
|
1179
|
+
date_time_input_format: "best_effort",
|
|
1180
|
+
use_client_time_zone: 1,
|
|
1181
|
+
output_format_json_quote_64bit_integers: 0
|
|
1182
|
+
}
|
|
1183
|
+
});
|
|
1184
|
+
let updatedContent = existingMessage.content || {};
|
|
1185
|
+
if (fieldsToUpdate.content) {
|
|
1186
|
+
const existingContent = existingMessage.content || {};
|
|
1187
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1188
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1189
|
+
updatedContent = {
|
|
1190
|
+
...existingContent,
|
|
1191
|
+
...fieldsToUpdate.content,
|
|
1192
|
+
metadata: {
|
|
1193
|
+
...existingMetadata,
|
|
1194
|
+
...updateMetadata
|
|
1195
|
+
}
|
|
1196
|
+
};
|
|
1197
|
+
}
|
|
1198
|
+
const updatedMessageData = {
|
|
1199
|
+
...existingMessage,
|
|
1200
|
+
...fieldsToUpdate,
|
|
1201
|
+
content: updatedContent
|
|
1202
|
+
};
|
|
1203
|
+
await this.client.insert({
|
|
1204
|
+
table: storage.TABLE_MESSAGES,
|
|
1205
|
+
format: "JSONEachRow",
|
|
1206
|
+
values: [
|
|
1207
|
+
{
|
|
1208
|
+
id: updatedMessageData.id,
|
|
1209
|
+
thread_id: updatedMessageData.threadId,
|
|
1210
|
+
resourceId: updatedMessageData.resourceId,
|
|
1211
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1212
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1213
|
+
role: updatedMessageData.role,
|
|
1214
|
+
type: updatedMessageData.type || "v2"
|
|
1215
|
+
}
|
|
1216
|
+
],
|
|
1217
|
+
clickhouse_settings: {
|
|
1218
|
+
date_time_input_format: "best_effort",
|
|
1219
|
+
use_client_time_zone: 1,
|
|
1220
|
+
output_format_json_quote_64bit_integers: 0
|
|
1221
|
+
}
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1228
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1229
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1230
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1231
|
+
const threadResult = await this.client.query({
|
|
1232
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1233
|
+
query_params: { threadId },
|
|
1234
|
+
clickhouse_settings: {
|
|
1235
|
+
date_time_input_format: "best_effort",
|
|
1236
|
+
date_time_output_format: "iso",
|
|
1237
|
+
use_client_time_zone: 1,
|
|
1238
|
+
output_format_json_quote_64bit_integers: 0
|
|
923
1239
|
}
|
|
924
|
-
|
|
1240
|
+
});
|
|
1241
|
+
const threadRows = await threadResult.json();
|
|
1242
|
+
if (threadRows.data.length > 0) {
|
|
1243
|
+
const existingThread = threadRows.data[0];
|
|
1244
|
+
await this.client.command({
|
|
1245
|
+
query: `DELETE FROM ${storage.TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1246
|
+
query_params: { threadId },
|
|
1247
|
+
clickhouse_settings: {
|
|
1248
|
+
date_time_input_format: "best_effort",
|
|
1249
|
+
use_client_time_zone: 1,
|
|
1250
|
+
output_format_json_quote_64bit_integers: 0
|
|
1251
|
+
}
|
|
1252
|
+
});
|
|
1253
|
+
await this.client.insert({
|
|
1254
|
+
table: storage.TABLE_THREADS,
|
|
1255
|
+
format: "JSONEachRow",
|
|
1256
|
+
values: [
|
|
1257
|
+
{
|
|
1258
|
+
id: existingThread.id,
|
|
1259
|
+
resourceId: existingThread.resourceId,
|
|
1260
|
+
title: existingThread.title,
|
|
1261
|
+
metadata: existingThread.metadata,
|
|
1262
|
+
createdAt: existingThread.createdAt,
|
|
1263
|
+
updatedAt: now
|
|
1264
|
+
}
|
|
1265
|
+
],
|
|
1266
|
+
clickhouse_settings: {
|
|
1267
|
+
date_time_input_format: "best_effort",
|
|
1268
|
+
use_client_time_zone: 1,
|
|
1269
|
+
output_format_json_quote_64bit_integers: 0
|
|
1270
|
+
}
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
});
|
|
1274
|
+
await Promise.all(threadUpdatePromises);
|
|
1275
|
+
}
|
|
1276
|
+
const updatedMessages = [];
|
|
1277
|
+
for (const messageId of messageIds) {
|
|
1278
|
+
const updatedResult = await this.client.query({
|
|
1279
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${storage.TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1280
|
+
query_params: { messageId },
|
|
925
1281
|
clickhouse_settings: {
|
|
926
1282
|
date_time_input_format: "best_effort",
|
|
1283
|
+
date_time_output_format: "iso",
|
|
927
1284
|
use_client_time_zone: 1,
|
|
928
1285
|
output_format_json_quote_64bit_integers: 0
|
|
929
1286
|
}
|
|
930
|
-
})
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
1287
|
+
});
|
|
1288
|
+
const updatedRows = await updatedResult.json();
|
|
1289
|
+
if (updatedRows.data.length > 0) {
|
|
1290
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1291
|
+
if (message) {
|
|
1292
|
+
updatedMessages.push(message);
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
return updatedMessages.map((message) => {
|
|
1297
|
+
if (typeof message.content === "string") {
|
|
1298
|
+
try {
|
|
1299
|
+
message.content = JSON.parse(message.content);
|
|
1300
|
+
} catch {
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
return message;
|
|
1304
|
+
});
|
|
1305
|
+
} catch (error$1) {
|
|
1306
|
+
throw new error.MastraError(
|
|
1307
|
+
{
|
|
1308
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1309
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1310
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1311
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1312
|
+
},
|
|
1313
|
+
error$1
|
|
1314
|
+
);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
async getResourceById({ resourceId }) {
|
|
1318
|
+
try {
|
|
1319
|
+
const result = await this.client.query({
|
|
1320
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${storage.TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1321
|
+
query_params: { resourceId },
|
|
1322
|
+
clickhouse_settings: {
|
|
1323
|
+
date_time_input_format: "best_effort",
|
|
1324
|
+
date_time_output_format: "iso",
|
|
1325
|
+
use_client_time_zone: 1,
|
|
1326
|
+
output_format_json_quote_64bit_integers: 0
|
|
1327
|
+
}
|
|
1328
|
+
});
|
|
1329
|
+
const rows = await result.json();
|
|
1330
|
+
if (rows.data.length === 0) {
|
|
1331
|
+
return null;
|
|
1332
|
+
}
|
|
1333
|
+
const resource = rows.data[0];
|
|
1334
|
+
return {
|
|
1335
|
+
id: resource.id,
|
|
1336
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1337
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1338
|
+
createdAt: new Date(resource.createdAt),
|
|
1339
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1340
|
+
};
|
|
1341
|
+
} catch (error$1) {
|
|
1342
|
+
throw new error.MastraError(
|
|
1343
|
+
{
|
|
1344
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1345
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1346
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1347
|
+
details: { resourceId }
|
|
1348
|
+
},
|
|
1349
|
+
error$1
|
|
1350
|
+
);
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
async saveResource({ resource }) {
|
|
1354
|
+
try {
|
|
1355
|
+
await this.client.insert({
|
|
1356
|
+
table: storage.TABLE_RESOURCES,
|
|
1357
|
+
format: "JSONEachRow",
|
|
1358
|
+
values: [
|
|
1359
|
+
{
|
|
1360
|
+
id: resource.id,
|
|
1361
|
+
workingMemory: resource.workingMemory,
|
|
1362
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1363
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1364
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1365
|
+
}
|
|
1366
|
+
],
|
|
1367
|
+
clickhouse_settings: {
|
|
1368
|
+
date_time_input_format: "best_effort",
|
|
1369
|
+
use_client_time_zone: 1,
|
|
1370
|
+
output_format_json_quote_64bit_integers: 0
|
|
1371
|
+
}
|
|
1372
|
+
});
|
|
1373
|
+
return resource;
|
|
1374
|
+
} catch (error$1) {
|
|
1375
|
+
throw new error.MastraError(
|
|
1376
|
+
{
|
|
1377
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1378
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1379
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1380
|
+
details: { resourceId: resource.id }
|
|
1381
|
+
},
|
|
1382
|
+
error$1
|
|
1383
|
+
);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
async updateResource({
|
|
1387
|
+
resourceId,
|
|
1388
|
+
workingMemory,
|
|
1389
|
+
metadata
|
|
1390
|
+
}) {
|
|
1391
|
+
try {
|
|
1392
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1393
|
+
if (!existingResource) {
|
|
1394
|
+
const newResource = {
|
|
1395
|
+
id: resourceId,
|
|
1396
|
+
workingMemory,
|
|
1397
|
+
metadata: metadata || {},
|
|
1398
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1399
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1400
|
+
};
|
|
1401
|
+
return this.saveResource({ resource: newResource });
|
|
1402
|
+
}
|
|
1403
|
+
const updatedResource = {
|
|
1404
|
+
...existingResource,
|
|
1405
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1406
|
+
metadata: {
|
|
1407
|
+
...existingResource.metadata,
|
|
1408
|
+
...metadata
|
|
1409
|
+
},
|
|
1410
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1411
|
+
};
|
|
1412
|
+
const updateQuery = `
|
|
1413
|
+
ALTER TABLE ${storage.TABLE_RESOURCES}
|
|
1414
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1415
|
+
WHERE id = {resourceId:String}
|
|
1416
|
+
`;
|
|
1417
|
+
await this.client.command({
|
|
1418
|
+
query: updateQuery,
|
|
1419
|
+
query_params: {
|
|
1420
|
+
workingMemory: updatedResource.workingMemory,
|
|
1421
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1422
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1423
|
+
resourceId
|
|
1424
|
+
},
|
|
1425
|
+
clickhouse_settings: {
|
|
1426
|
+
date_time_input_format: "best_effort",
|
|
1427
|
+
use_client_time_zone: 1,
|
|
1428
|
+
output_format_json_quote_64bit_integers: 0
|
|
1429
|
+
}
|
|
1430
|
+
});
|
|
1431
|
+
await this.client.command({
|
|
1432
|
+
query: `OPTIMIZE TABLE ${storage.TABLE_RESOURCES} FINAL`,
|
|
1433
|
+
clickhouse_settings: {
|
|
1434
|
+
date_time_input_format: "best_effort",
|
|
1435
|
+
use_client_time_zone: 1,
|
|
1436
|
+
output_format_json_quote_64bit_integers: 0
|
|
1437
|
+
}
|
|
1438
|
+
});
|
|
1439
|
+
return updatedResource;
|
|
1440
|
+
} catch (error$1) {
|
|
1441
|
+
throw new error.MastraError(
|
|
1442
|
+
{
|
|
1443
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1444
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1445
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1446
|
+
details: { resourceId }
|
|
1447
|
+
},
|
|
1448
|
+
error$1
|
|
1449
|
+
);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
};
|
|
1453
|
+
var StoreOperationsClickhouse = class extends storage.StoreOperations {
|
|
1454
|
+
ttl;
|
|
1455
|
+
client;
|
|
1456
|
+
constructor({ client, ttl }) {
|
|
1457
|
+
super();
|
|
1458
|
+
this.ttl = ttl;
|
|
1459
|
+
this.client = client;
|
|
1460
|
+
}
|
|
1461
|
+
async hasColumn(table, column) {
|
|
1462
|
+
const result = await this.client.query({
|
|
1463
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1464
|
+
format: "JSONEachRow"
|
|
1465
|
+
});
|
|
1466
|
+
const columns = await result.json();
|
|
1467
|
+
return columns.some((c) => c.name === column);
|
|
1468
|
+
}
|
|
1469
|
+
getSqlType(type) {
|
|
1470
|
+
switch (type) {
|
|
1471
|
+
case "text":
|
|
1472
|
+
return "String";
|
|
1473
|
+
case "timestamp":
|
|
1474
|
+
return "DateTime64(3)";
|
|
1475
|
+
case "integer":
|
|
1476
|
+
case "bigint":
|
|
1477
|
+
return "Int64";
|
|
1478
|
+
case "jsonb":
|
|
1479
|
+
return "String";
|
|
1480
|
+
default:
|
|
1481
|
+
return super.getSqlType(type);
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
async createTable({
|
|
1485
|
+
tableName,
|
|
1486
|
+
schema
|
|
1487
|
+
}) {
|
|
1488
|
+
try {
|
|
1489
|
+
const columns = Object.entries(schema).map(([name, def]) => {
|
|
1490
|
+
const constraints = [];
|
|
1491
|
+
if (!def.nullable) constraints.push("NOT NULL");
|
|
1492
|
+
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
1493
|
+
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
|
|
1494
|
+
}).join(",\n");
|
|
1495
|
+
const rowTtl = this.ttl?.[tableName]?.row;
|
|
1496
|
+
const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
|
|
1497
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1498
|
+
${["id String"].concat(columns)}
|
|
1499
|
+
)
|
|
1500
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1501
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1502
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1503
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1504
|
+
SETTINGS index_granularity = 8192
|
|
1505
|
+
` : `
|
|
1506
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1507
|
+
${columns}
|
|
1508
|
+
)
|
|
1509
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1510
|
+
PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1511
|
+
ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
1512
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1513
|
+
SETTINGS index_granularity = 8192
|
|
1514
|
+
`;
|
|
1515
|
+
await this.client.query({
|
|
1516
|
+
query: sql,
|
|
1517
|
+
clickhouse_settings: {
|
|
1518
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1519
|
+
date_time_input_format: "best_effort",
|
|
1520
|
+
date_time_output_format: "iso",
|
|
1521
|
+
use_client_time_zone: 1,
|
|
1522
|
+
output_format_json_quote_64bit_integers: 0
|
|
1523
|
+
}
|
|
1524
|
+
});
|
|
1525
|
+
} catch (error$1) {
|
|
1526
|
+
throw new error.MastraError(
|
|
1527
|
+
{
|
|
1528
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1529
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1530
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1531
|
+
details: { tableName }
|
|
1532
|
+
},
|
|
1533
|
+
error$1
|
|
1534
|
+
);
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
async alterTable({
|
|
1538
|
+
tableName,
|
|
1539
|
+
schema,
|
|
1540
|
+
ifNotExists
|
|
1541
|
+
}) {
|
|
1542
|
+
try {
|
|
1543
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1544
|
+
const result = await this.client.query({
|
|
1545
|
+
query: describeSql
|
|
1546
|
+
});
|
|
1547
|
+
const rows = await result.json();
|
|
1548
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1549
|
+
for (const columnName of ifNotExists) {
|
|
1550
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1551
|
+
const columnDef = schema[columnName];
|
|
1552
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1553
|
+
if (columnDef.nullable !== false) {
|
|
1554
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1555
|
+
}
|
|
1556
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1557
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1558
|
+
await this.client.query({
|
|
1559
|
+
query: alterSql
|
|
1560
|
+
});
|
|
1561
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
} catch (error$1) {
|
|
1565
|
+
throw new error.MastraError(
|
|
1566
|
+
{
|
|
1567
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1568
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1569
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1570
|
+
details: { tableName }
|
|
1571
|
+
},
|
|
1572
|
+
error$1
|
|
1573
|
+
);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
async clearTable({ tableName }) {
|
|
1577
|
+
try {
|
|
1578
|
+
await this.client.query({
|
|
1579
|
+
query: `TRUNCATE TABLE ${tableName}`,
|
|
1580
|
+
clickhouse_settings: {
|
|
1581
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1582
|
+
date_time_input_format: "best_effort",
|
|
1583
|
+
date_time_output_format: "iso",
|
|
1584
|
+
use_client_time_zone: 1,
|
|
1585
|
+
output_format_json_quote_64bit_integers: 0
|
|
1586
|
+
}
|
|
1587
|
+
});
|
|
1588
|
+
} catch (error$1) {
|
|
1589
|
+
throw new error.MastraError(
|
|
1590
|
+
{
|
|
1591
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1592
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1593
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1594
|
+
details: { tableName }
|
|
1595
|
+
},
|
|
1596
|
+
error$1
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
async dropTable({ tableName }) {
|
|
1601
|
+
await this.client.query({
|
|
1602
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
async insert({ tableName, record }) {
|
|
1606
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1607
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
1608
|
+
try {
|
|
1609
|
+
const result = await this.client.insert({
|
|
1610
|
+
table: tableName,
|
|
1611
|
+
values: [
|
|
1612
|
+
{
|
|
1613
|
+
...record,
|
|
1614
|
+
createdAt,
|
|
1615
|
+
updatedAt
|
|
1616
|
+
}
|
|
1617
|
+
],
|
|
1618
|
+
format: "JSONEachRow",
|
|
1619
|
+
clickhouse_settings: {
|
|
1620
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1621
|
+
output_format_json_quote_64bit_integers: 0,
|
|
1622
|
+
date_time_input_format: "best_effort",
|
|
1623
|
+
use_client_time_zone: 1
|
|
1624
|
+
}
|
|
1625
|
+
});
|
|
1626
|
+
console.info("INSERT RESULT", result);
|
|
1627
|
+
} catch (error$1) {
|
|
1628
|
+
throw new error.MastraError(
|
|
1629
|
+
{
|
|
1630
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1631
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1632
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1633
|
+
details: { tableName }
|
|
1634
|
+
},
|
|
1635
|
+
error$1
|
|
1636
|
+
);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
async batchInsert({ tableName, records }) {
|
|
1640
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1641
|
+
...Object.fromEntries(
|
|
1642
|
+
Object.entries(record).map(([key, value]) => [
|
|
1643
|
+
key,
|
|
1644
|
+
storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1645
|
+
])
|
|
1646
|
+
)
|
|
1647
|
+
}));
|
|
1648
|
+
try {
|
|
1649
|
+
await this.client.insert({
|
|
1650
|
+
table: tableName,
|
|
1651
|
+
values: recordsToBeInserted,
|
|
1652
|
+
format: "JSONEachRow",
|
|
1653
|
+
clickhouse_settings: {
|
|
1654
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1655
|
+
date_time_input_format: "best_effort",
|
|
1656
|
+
use_client_time_zone: 1,
|
|
1657
|
+
output_format_json_quote_64bit_integers: 0
|
|
1658
|
+
}
|
|
1659
|
+
});
|
|
1660
|
+
} catch (error$1) {
|
|
1661
|
+
throw new error.MastraError(
|
|
1662
|
+
{
|
|
1663
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1664
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1665
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1666
|
+
details: { tableName }
|
|
1667
|
+
},
|
|
1668
|
+
error$1
|
|
1669
|
+
);
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
async load({ tableName, keys }) {
|
|
1673
|
+
try {
|
|
1674
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
1675
|
+
const keyEntries = Object.entries(keys);
|
|
1676
|
+
const conditions = keyEntries.map(
|
|
1677
|
+
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
1678
|
+
).join(" AND ");
|
|
1679
|
+
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
1680
|
+
return { ...acc, [`var_${key}`]: value };
|
|
1681
|
+
}, {});
|
|
1682
|
+
const hasUpdatedAt = storage.TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1683
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1684
|
+
const result = await this.client.query({
|
|
1685
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
1686
|
+
query_params: values,
|
|
1687
|
+
clickhouse_settings: {
|
|
1688
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1689
|
+
date_time_input_format: "best_effort",
|
|
1690
|
+
date_time_output_format: "iso",
|
|
1691
|
+
use_client_time_zone: 1,
|
|
1692
|
+
output_format_json_quote_64bit_integers: 0
|
|
1693
|
+
}
|
|
1694
|
+
});
|
|
1695
|
+
if (!result) {
|
|
1696
|
+
return null;
|
|
1697
|
+
}
|
|
1698
|
+
const rows = await result.json();
|
|
1699
|
+
if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
|
|
1700
|
+
const snapshot = rows.data[0];
|
|
1701
|
+
if (!snapshot) {
|
|
1702
|
+
return null;
|
|
1703
|
+
}
|
|
1704
|
+
if (typeof snapshot.snapshot === "string") {
|
|
1705
|
+
snapshot.snapshot = JSON.parse(snapshot.snapshot);
|
|
1706
|
+
}
|
|
1707
|
+
return transformRow(snapshot);
|
|
1708
|
+
}
|
|
1709
|
+
const data = transformRow(rows.data[0]);
|
|
1710
|
+
return data;
|
|
1711
|
+
} catch (error$1) {
|
|
1712
|
+
throw new error.MastraError(
|
|
1713
|
+
{
|
|
1714
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1715
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1716
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1717
|
+
details: { tableName }
|
|
1718
|
+
},
|
|
1719
|
+
error$1
|
|
1720
|
+
);
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
};
|
|
1724
|
+
var ScoresStorageClickhouse = class extends storage.ScoresStorage {
|
|
1725
|
+
client;
|
|
1726
|
+
operations;
|
|
1727
|
+
constructor({ client, operations }) {
|
|
1728
|
+
super();
|
|
1729
|
+
this.client = client;
|
|
1730
|
+
this.operations = operations;
|
|
1731
|
+
}
|
|
1732
|
+
transformScoreRow(row) {
|
|
1733
|
+
const scorer = storage.safelyParseJSON(row.scorer);
|
|
1734
|
+
const preprocessStepResult = storage.safelyParseJSON(row.preprocessStepResult);
|
|
1735
|
+
const analyzeStepResult = storage.safelyParseJSON(row.analyzeStepResult);
|
|
1736
|
+
const metadata = storage.safelyParseJSON(row.metadata);
|
|
1737
|
+
const input = storage.safelyParseJSON(row.input);
|
|
1738
|
+
const output = storage.safelyParseJSON(row.output);
|
|
1739
|
+
const additionalContext = storage.safelyParseJSON(row.additionalContext);
|
|
1740
|
+
const runtimeContext = storage.safelyParseJSON(row.runtimeContext);
|
|
1741
|
+
const entity = storage.safelyParseJSON(row.entity);
|
|
1742
|
+
return {
|
|
1743
|
+
...row,
|
|
1744
|
+
scorer,
|
|
1745
|
+
preprocessStepResult,
|
|
1746
|
+
analyzeStepResult,
|
|
1747
|
+
metadata,
|
|
1748
|
+
input,
|
|
1749
|
+
output,
|
|
1750
|
+
additionalContext,
|
|
1751
|
+
runtimeContext,
|
|
1752
|
+
entity,
|
|
1753
|
+
createdAt: new Date(row.createdAt),
|
|
1754
|
+
updatedAt: new Date(row.updatedAt)
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
async getScoreById({ id }) {
|
|
1758
|
+
try {
|
|
1759
|
+
const result = await this.client.query({
|
|
1760
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1761
|
+
query_params: { var_id: id },
|
|
1762
|
+
format: "JSONEachRow",
|
|
1763
|
+
clickhouse_settings: {
|
|
1764
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1765
|
+
date_time_input_format: "best_effort",
|
|
1766
|
+
date_time_output_format: "iso",
|
|
1767
|
+
use_client_time_zone: 1,
|
|
1768
|
+
output_format_json_quote_64bit_integers: 0
|
|
1769
|
+
}
|
|
1770
|
+
});
|
|
1771
|
+
const resultJson = await result.json();
|
|
1772
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
1773
|
+
return null;
|
|
1774
|
+
}
|
|
1775
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1776
|
+
} catch (error$1) {
|
|
1777
|
+
throw new error.MastraError(
|
|
1778
|
+
{
|
|
1779
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1780
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1781
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1782
|
+
details: { scoreId: id }
|
|
1783
|
+
},
|
|
1784
|
+
error$1
|
|
1785
|
+
);
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
async saveScore(score) {
|
|
1789
|
+
let parsedScore;
|
|
1790
|
+
try {
|
|
1791
|
+
parsedScore = scores.saveScorePayloadSchema.parse(score);
|
|
1792
|
+
} catch (error$1) {
|
|
1793
|
+
throw new error.MastraError(
|
|
1794
|
+
{
|
|
1795
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED_INVALID_SCORE_PAYLOAD",
|
|
1796
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1797
|
+
category: error.ErrorCategory.USER,
|
|
1798
|
+
details: { scoreId: score.id }
|
|
1799
|
+
},
|
|
1800
|
+
error$1
|
|
1801
|
+
);
|
|
1802
|
+
}
|
|
1803
|
+
try {
|
|
1804
|
+
const record = {
|
|
1805
|
+
...parsedScore
|
|
1806
|
+
};
|
|
1807
|
+
await this.client.insert({
|
|
1808
|
+
table: storage.TABLE_SCORERS,
|
|
1809
|
+
values: [record],
|
|
1810
|
+
format: "JSONEachRow",
|
|
1811
|
+
clickhouse_settings: {
|
|
1812
|
+
date_time_input_format: "best_effort",
|
|
1813
|
+
use_client_time_zone: 1,
|
|
1814
|
+
output_format_json_quote_64bit_integers: 0
|
|
1815
|
+
}
|
|
1816
|
+
});
|
|
1817
|
+
return { score };
|
|
1818
|
+
} catch (error$1) {
|
|
1819
|
+
throw new error.MastraError(
|
|
1820
|
+
{
|
|
1821
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1822
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1823
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1824
|
+
details: { scoreId: score.id }
|
|
1825
|
+
},
|
|
1826
|
+
error$1
|
|
1827
|
+
);
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
async getScoresByRunId({
|
|
1831
|
+
runId,
|
|
1832
|
+
pagination
|
|
1833
|
+
}) {
|
|
1834
|
+
try {
|
|
1835
|
+
const countResult = await this.client.query({
|
|
1836
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1837
|
+
query_params: { var_runId: runId },
|
|
1838
|
+
format: "JSONEachRow"
|
|
1839
|
+
});
|
|
1840
|
+
const countRows = await countResult.json();
|
|
1841
|
+
let total = 0;
|
|
1842
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1843
|
+
const countObj = countRows[0];
|
|
1844
|
+
total = Number(countObj.count);
|
|
1845
|
+
}
|
|
1846
|
+
if (!total) {
|
|
1847
|
+
return {
|
|
1848
|
+
pagination: {
|
|
1849
|
+
total: 0,
|
|
1850
|
+
page: pagination.page,
|
|
1851
|
+
perPage: pagination.perPage,
|
|
1852
|
+
hasMore: false
|
|
1853
|
+
},
|
|
1854
|
+
scores: []
|
|
1855
|
+
};
|
|
1856
|
+
}
|
|
1857
|
+
const offset = pagination.page * pagination.perPage;
|
|
1858
|
+
const result = await this.client.query({
|
|
1859
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1860
|
+
query_params: {
|
|
1861
|
+
var_runId: runId,
|
|
1862
|
+
var_limit: pagination.perPage,
|
|
1863
|
+
var_offset: offset
|
|
1864
|
+
},
|
|
1865
|
+
format: "JSONEachRow",
|
|
1866
|
+
clickhouse_settings: {
|
|
1867
|
+
date_time_input_format: "best_effort",
|
|
1868
|
+
date_time_output_format: "iso",
|
|
1869
|
+
use_client_time_zone: 1,
|
|
1870
|
+
output_format_json_quote_64bit_integers: 0
|
|
1871
|
+
}
|
|
1872
|
+
});
|
|
1873
|
+
const rows = await result.json();
|
|
1874
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1875
|
+
return {
|
|
1876
|
+
pagination: {
|
|
1877
|
+
total,
|
|
1878
|
+
page: pagination.page,
|
|
1879
|
+
perPage: pagination.perPage,
|
|
1880
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1881
|
+
},
|
|
1882
|
+
scores
|
|
1883
|
+
};
|
|
1884
|
+
} catch (error$1) {
|
|
1885
|
+
throw new error.MastraError(
|
|
1886
|
+
{
|
|
1887
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1888
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1889
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1890
|
+
details: { runId }
|
|
1891
|
+
},
|
|
1892
|
+
error$1
|
|
1893
|
+
);
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
async getScoresByScorerId({
|
|
1897
|
+
scorerId,
|
|
1898
|
+
entityId,
|
|
1899
|
+
entityType,
|
|
1900
|
+
source,
|
|
1901
|
+
pagination
|
|
1902
|
+
}) {
|
|
1903
|
+
let whereClause = `scorerId = {var_scorerId:String}`;
|
|
1904
|
+
if (entityId) {
|
|
1905
|
+
whereClause += ` AND entityId = {var_entityId:String}`;
|
|
1906
|
+
}
|
|
1907
|
+
if (entityType) {
|
|
1908
|
+
whereClause += ` AND entityType = {var_entityType:String}`;
|
|
1909
|
+
}
|
|
1910
|
+
if (source) {
|
|
1911
|
+
whereClause += ` AND source = {var_source:String}`;
|
|
1912
|
+
}
|
|
1913
|
+
try {
|
|
1914
|
+
const countResult = await this.client.query({
|
|
1915
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE ${whereClause}`,
|
|
1916
|
+
query_params: {
|
|
1917
|
+
var_scorerId: scorerId,
|
|
1918
|
+
var_entityId: entityId,
|
|
1919
|
+
var_entityType: entityType,
|
|
1920
|
+
var_source: source
|
|
1921
|
+
},
|
|
1922
|
+
format: "JSONEachRow"
|
|
1923
|
+
});
|
|
1924
|
+
const countRows = await countResult.json();
|
|
1925
|
+
let total = 0;
|
|
1926
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1927
|
+
const countObj = countRows[0];
|
|
1928
|
+
total = Number(countObj.count);
|
|
1929
|
+
}
|
|
1930
|
+
if (!total) {
|
|
1931
|
+
return {
|
|
1932
|
+
pagination: {
|
|
1933
|
+
total: 0,
|
|
1934
|
+
page: pagination.page,
|
|
1935
|
+
perPage: pagination.perPage,
|
|
1936
|
+
hasMore: false
|
|
1937
|
+
},
|
|
1938
|
+
scores: []
|
|
1939
|
+
};
|
|
1940
|
+
}
|
|
1941
|
+
const offset = pagination.page * pagination.perPage;
|
|
1942
|
+
const result = await this.client.query({
|
|
1943
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1944
|
+
query_params: {
|
|
1945
|
+
var_scorerId: scorerId,
|
|
1946
|
+
var_limit: pagination.perPage,
|
|
1947
|
+
var_offset: offset,
|
|
1948
|
+
var_entityId: entityId,
|
|
1949
|
+
var_entityType: entityType,
|
|
1950
|
+
var_source: source
|
|
1951
|
+
},
|
|
1952
|
+
format: "JSONEachRow",
|
|
1953
|
+
clickhouse_settings: {
|
|
1954
|
+
date_time_input_format: "best_effort",
|
|
1955
|
+
date_time_output_format: "iso",
|
|
1956
|
+
use_client_time_zone: 1,
|
|
1957
|
+
output_format_json_quote_64bit_integers: 0
|
|
1958
|
+
}
|
|
1959
|
+
});
|
|
1960
|
+
const rows = await result.json();
|
|
1961
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1962
|
+
return {
|
|
1963
|
+
pagination: {
|
|
1964
|
+
total,
|
|
1965
|
+
page: pagination.page,
|
|
1966
|
+
perPage: pagination.perPage,
|
|
1967
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1968
|
+
},
|
|
1969
|
+
scores
|
|
1970
|
+
};
|
|
1971
|
+
} catch (error$1) {
|
|
1972
|
+
throw new error.MastraError(
|
|
1973
|
+
{
|
|
1974
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1975
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1976
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1977
|
+
details: { scorerId }
|
|
1978
|
+
},
|
|
1979
|
+
error$1
|
|
1980
|
+
);
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
async getScoresByEntityId({
|
|
1984
|
+
entityId,
|
|
1985
|
+
entityType,
|
|
1986
|
+
pagination
|
|
1987
|
+
}) {
|
|
1988
|
+
try {
|
|
1989
|
+
const countResult = await this.client.query({
|
|
1990
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1991
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1992
|
+
format: "JSONEachRow"
|
|
1993
|
+
});
|
|
1994
|
+
const countRows = await countResult.json();
|
|
1995
|
+
let total = 0;
|
|
1996
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1997
|
+
const countObj = countRows[0];
|
|
1998
|
+
total = Number(countObj.count);
|
|
1999
|
+
}
|
|
2000
|
+
if (!total) {
|
|
2001
|
+
return {
|
|
2002
|
+
pagination: {
|
|
2003
|
+
total: 0,
|
|
2004
|
+
page: pagination.page,
|
|
2005
|
+
perPage: pagination.perPage,
|
|
2006
|
+
hasMore: false
|
|
2007
|
+
},
|
|
2008
|
+
scores: []
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
const offset = pagination.page * pagination.perPage;
|
|
2012
|
+
const result = await this.client.query({
|
|
2013
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
2014
|
+
query_params: {
|
|
2015
|
+
var_entityId: entityId,
|
|
2016
|
+
var_entityType: entityType,
|
|
2017
|
+
var_limit: pagination.perPage,
|
|
2018
|
+
var_offset: offset
|
|
2019
|
+
},
|
|
2020
|
+
format: "JSONEachRow",
|
|
2021
|
+
clickhouse_settings: {
|
|
2022
|
+
date_time_input_format: "best_effort",
|
|
2023
|
+
date_time_output_format: "iso",
|
|
2024
|
+
use_client_time_zone: 1,
|
|
2025
|
+
output_format_json_quote_64bit_integers: 0
|
|
2026
|
+
}
|
|
2027
|
+
});
|
|
2028
|
+
const rows = await result.json();
|
|
2029
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2030
|
+
return {
|
|
2031
|
+
pagination: {
|
|
2032
|
+
total,
|
|
2033
|
+
page: pagination.page,
|
|
2034
|
+
perPage: pagination.perPage,
|
|
2035
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
2036
|
+
},
|
|
2037
|
+
scores
|
|
2038
|
+
};
|
|
2039
|
+
} catch (error$1) {
|
|
2040
|
+
throw new error.MastraError(
|
|
2041
|
+
{
|
|
2042
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
2043
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2044
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2045
|
+
details: { entityId, entityType }
|
|
2046
|
+
},
|
|
2047
|
+
error$1
|
|
2048
|
+
);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
async getScoresBySpan({
|
|
2052
|
+
traceId,
|
|
2053
|
+
spanId,
|
|
2054
|
+
pagination
|
|
2055
|
+
}) {
|
|
2056
|
+
try {
|
|
2057
|
+
const countResult = await this.client.query({
|
|
2058
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String}`,
|
|
2059
|
+
query_params: {
|
|
2060
|
+
var_traceId: traceId,
|
|
2061
|
+
var_spanId: spanId
|
|
2062
|
+
},
|
|
2063
|
+
format: "JSONEachRow"
|
|
2064
|
+
});
|
|
2065
|
+
const countRows = await countResult.json();
|
|
2066
|
+
let total = 0;
|
|
2067
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
2068
|
+
const countObj = countRows[0];
|
|
2069
|
+
total = Number(countObj.count);
|
|
2070
|
+
}
|
|
2071
|
+
if (!total) {
|
|
2072
|
+
return {
|
|
2073
|
+
pagination: {
|
|
2074
|
+
total: 0,
|
|
2075
|
+
page: pagination.page,
|
|
2076
|
+
perPage: pagination.perPage,
|
|
2077
|
+
hasMore: false
|
|
2078
|
+
},
|
|
2079
|
+
scores: []
|
|
2080
|
+
};
|
|
2081
|
+
}
|
|
2082
|
+
const limit = pagination.perPage + 1;
|
|
2083
|
+
const offset = pagination.page * pagination.perPage;
|
|
2084
|
+
const result = await this.client.query({
|
|
2085
|
+
query: `SELECT * FROM ${storage.TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
2086
|
+
query_params: {
|
|
2087
|
+
var_traceId: traceId,
|
|
2088
|
+
var_spanId: spanId,
|
|
2089
|
+
var_limit: limit,
|
|
2090
|
+
var_offset: offset
|
|
2091
|
+
},
|
|
2092
|
+
format: "JSONEachRow",
|
|
2093
|
+
clickhouse_settings: {
|
|
2094
|
+
date_time_input_format: "best_effort",
|
|
2095
|
+
date_time_output_format: "iso",
|
|
2096
|
+
use_client_time_zone: 1,
|
|
2097
|
+
output_format_json_quote_64bit_integers: 0
|
|
2098
|
+
}
|
|
2099
|
+
});
|
|
2100
|
+
const rows = await result.json();
|
|
2101
|
+
const transformedRows = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2102
|
+
const hasMore = transformedRows.length > pagination.perPage;
|
|
2103
|
+
const scores = hasMore ? transformedRows.slice(0, pagination.perPage) : transformedRows;
|
|
2104
|
+
return {
|
|
2105
|
+
pagination: {
|
|
2106
|
+
total,
|
|
2107
|
+
page: pagination.page,
|
|
2108
|
+
perPage: pagination.perPage,
|
|
2109
|
+
hasMore
|
|
2110
|
+
},
|
|
2111
|
+
scores
|
|
2112
|
+
};
|
|
2113
|
+
} catch (error$1) {
|
|
2114
|
+
throw new error.MastraError(
|
|
2115
|
+
{
|
|
2116
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
|
|
2117
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2118
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2119
|
+
details: { traceId, spanId }
|
|
2120
|
+
},
|
|
2121
|
+
error$1
|
|
2122
|
+
);
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
};
|
|
2126
|
+
var TracesStorageClickhouse = class extends storage.TracesStorage {
|
|
2127
|
+
client;
|
|
2128
|
+
operations;
|
|
2129
|
+
constructor({ client, operations }) {
|
|
2130
|
+
super();
|
|
2131
|
+
this.client = client;
|
|
2132
|
+
this.operations = operations;
|
|
2133
|
+
}
|
|
2134
|
+
async getTracesPaginated(args) {
|
|
2135
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
2136
|
+
const fromDate = dateRange?.start;
|
|
2137
|
+
const toDate = dateRange?.end;
|
|
2138
|
+
const currentOffset = page * perPage;
|
|
2139
|
+
const queryArgs = {};
|
|
2140
|
+
const conditions = [];
|
|
2141
|
+
if (name) {
|
|
2142
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2143
|
+
queryArgs.var_name = name;
|
|
2144
|
+
}
|
|
2145
|
+
if (scope) {
|
|
2146
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2147
|
+
queryArgs.var_scope = scope;
|
|
2148
|
+
}
|
|
2149
|
+
if (attributes) {
|
|
2150
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2151
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2152
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
2153
|
+
});
|
|
2154
|
+
}
|
|
2155
|
+
if (filters) {
|
|
2156
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2157
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2158
|
+
queryArgs[`var_col_${key}`] = value;
|
|
2159
|
+
});
|
|
2160
|
+
}
|
|
2161
|
+
if (fromDate) {
|
|
2162
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
2163
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
2164
|
+
}
|
|
2165
|
+
if (toDate) {
|
|
2166
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
2167
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
2168
|
+
}
|
|
2169
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2170
|
+
try {
|
|
2171
|
+
const countResult = await this.client.query({
|
|
2172
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_TRACES} ${whereClause}`,
|
|
2173
|
+
query_params: queryArgs,
|
|
2174
|
+
clickhouse_settings: {
|
|
2175
|
+
date_time_input_format: "best_effort",
|
|
2176
|
+
date_time_output_format: "iso",
|
|
2177
|
+
use_client_time_zone: 1,
|
|
2178
|
+
output_format_json_quote_64bit_integers: 0
|
|
2179
|
+
}
|
|
2180
|
+
});
|
|
2181
|
+
const countData = await countResult.json();
|
|
2182
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2183
|
+
if (total === 0) {
|
|
2184
|
+
return {
|
|
2185
|
+
traces: [],
|
|
2186
|
+
total: 0,
|
|
2187
|
+
page,
|
|
2188
|
+
perPage,
|
|
2189
|
+
hasMore: false
|
|
2190
|
+
};
|
|
2191
|
+
}
|
|
2192
|
+
const result = await this.client.query({
|
|
2193
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2194
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2195
|
+
clickhouse_settings: {
|
|
2196
|
+
date_time_input_format: "best_effort",
|
|
2197
|
+
date_time_output_format: "iso",
|
|
2198
|
+
use_client_time_zone: 1,
|
|
2199
|
+
output_format_json_quote_64bit_integers: 0
|
|
2200
|
+
}
|
|
2201
|
+
});
|
|
2202
|
+
if (!result) {
|
|
2203
|
+
return {
|
|
2204
|
+
traces: [],
|
|
2205
|
+
total,
|
|
2206
|
+
page,
|
|
2207
|
+
perPage,
|
|
2208
|
+
hasMore: false
|
|
2209
|
+
};
|
|
2210
|
+
}
|
|
2211
|
+
const resp = await result.json();
|
|
2212
|
+
const rows = resp.data;
|
|
2213
|
+
const traces = rows.map((row) => ({
|
|
2214
|
+
id: row.id,
|
|
2215
|
+
parentSpanId: row.parentSpanId,
|
|
2216
|
+
traceId: row.traceId,
|
|
2217
|
+
name: row.name,
|
|
2218
|
+
scope: row.scope,
|
|
2219
|
+
kind: row.kind,
|
|
2220
|
+
status: storage.safelyParseJSON(row.status),
|
|
2221
|
+
events: storage.safelyParseJSON(row.events),
|
|
2222
|
+
links: storage.safelyParseJSON(row.links),
|
|
2223
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2224
|
+
startTime: row.startTime,
|
|
2225
|
+
endTime: row.endTime,
|
|
2226
|
+
other: storage.safelyParseJSON(row.other),
|
|
2227
|
+
createdAt: row.createdAt
|
|
2228
|
+
}));
|
|
2229
|
+
return {
|
|
2230
|
+
traces,
|
|
2231
|
+
total,
|
|
2232
|
+
page,
|
|
2233
|
+
perPage,
|
|
2234
|
+
hasMore: currentOffset + traces.length < total
|
|
2235
|
+
};
|
|
2236
|
+
} catch (error$1) {
|
|
2237
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2238
|
+
return {
|
|
2239
|
+
traces: [],
|
|
2240
|
+
total: 0,
|
|
2241
|
+
page,
|
|
2242
|
+
perPage,
|
|
2243
|
+
hasMore: false
|
|
2244
|
+
};
|
|
2245
|
+
}
|
|
2246
|
+
throw new error.MastraError(
|
|
2247
|
+
{
|
|
2248
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2249
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2250
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2251
|
+
details: {
|
|
2252
|
+
name: name ?? null,
|
|
2253
|
+
scope: scope ?? null,
|
|
2254
|
+
page,
|
|
2255
|
+
perPage,
|
|
2256
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2257
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2258
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2259
|
+
}
|
|
2260
|
+
},
|
|
2261
|
+
error$1
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
async getTraces({
|
|
2266
|
+
name,
|
|
2267
|
+
scope,
|
|
2268
|
+
page,
|
|
2269
|
+
perPage,
|
|
2270
|
+
attributes,
|
|
2271
|
+
filters,
|
|
2272
|
+
fromDate,
|
|
2273
|
+
toDate
|
|
2274
|
+
}) {
|
|
2275
|
+
const limit = perPage;
|
|
2276
|
+
const offset = page * perPage;
|
|
2277
|
+
const args = {};
|
|
2278
|
+
const conditions = [];
|
|
2279
|
+
if (name) {
|
|
2280
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2281
|
+
args.var_name = name;
|
|
2282
|
+
}
|
|
2283
|
+
if (scope) {
|
|
2284
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2285
|
+
args.var_scope = scope;
|
|
2286
|
+
}
|
|
2287
|
+
if (attributes) {
|
|
2288
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2289
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2290
|
+
args[`var_attr_${key}`] = value;
|
|
2291
|
+
});
|
|
2292
|
+
}
|
|
2293
|
+
if (filters) {
|
|
2294
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2295
|
+
conditions.push(`${key} = {var_col_${key}:${storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2296
|
+
args[`var_col_${key}`] = value;
|
|
2297
|
+
});
|
|
2298
|
+
}
|
|
2299
|
+
if (fromDate) {
|
|
2300
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2301
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2302
|
+
}
|
|
2303
|
+
if (toDate) {
|
|
2304
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2305
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2306
|
+
}
|
|
2307
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2308
|
+
try {
|
|
2309
|
+
const result = await this.client.query({
|
|
2310
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2311
|
+
query_params: args,
|
|
2312
|
+
clickhouse_settings: {
|
|
2313
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
2314
|
+
date_time_input_format: "best_effort",
|
|
2315
|
+
date_time_output_format: "iso",
|
|
2316
|
+
use_client_time_zone: 1,
|
|
2317
|
+
output_format_json_quote_64bit_integers: 0
|
|
2318
|
+
}
|
|
2319
|
+
});
|
|
2320
|
+
if (!result) {
|
|
2321
|
+
return [];
|
|
2322
|
+
}
|
|
2323
|
+
const resp = await result.json();
|
|
2324
|
+
const rows = resp.data;
|
|
2325
|
+
return rows.map((row) => ({
|
|
2326
|
+
id: row.id,
|
|
2327
|
+
parentSpanId: row.parentSpanId,
|
|
2328
|
+
traceId: row.traceId,
|
|
2329
|
+
name: row.name,
|
|
2330
|
+
scope: row.scope,
|
|
2331
|
+
kind: row.kind,
|
|
2332
|
+
status: storage.safelyParseJSON(row.status),
|
|
2333
|
+
events: storage.safelyParseJSON(row.events),
|
|
2334
|
+
links: storage.safelyParseJSON(row.links),
|
|
2335
|
+
attributes: storage.safelyParseJSON(row.attributes),
|
|
2336
|
+
startTime: row.startTime,
|
|
2337
|
+
endTime: row.endTime,
|
|
2338
|
+
other: storage.safelyParseJSON(row.other),
|
|
2339
|
+
createdAt: row.createdAt
|
|
2340
|
+
}));
|
|
935
2341
|
} catch (error$1) {
|
|
2342
|
+
if (error$1?.message?.includes("no such table") || error$1?.message?.includes("does not exist")) {
|
|
2343
|
+
return [];
|
|
2344
|
+
}
|
|
936
2345
|
throw new error.MastraError(
|
|
937
2346
|
{
|
|
938
|
-
id: "
|
|
2347
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
939
2348
|
domain: error.ErrorDomain.STORAGE,
|
|
940
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
2349
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2350
|
+
details: {
|
|
2351
|
+
name: name ?? null,
|
|
2352
|
+
scope: scope ?? null,
|
|
2353
|
+
page,
|
|
2354
|
+
perPage,
|
|
2355
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2356
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2357
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2358
|
+
toDate: toDate?.toISOString() ?? null
|
|
2359
|
+
}
|
|
941
2360
|
},
|
|
942
2361
|
error$1
|
|
943
2362
|
);
|
|
944
2363
|
}
|
|
945
2364
|
}
|
|
2365
|
+
async batchTraceInsert(args) {
|
|
2366
|
+
await this.operations.batchInsert({ tableName: storage.TABLE_TRACES, records: args.records });
|
|
2367
|
+
}
|
|
2368
|
+
};
|
|
2369
|
+
var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
|
|
2370
|
+
client;
|
|
2371
|
+
operations;
|
|
2372
|
+
constructor({ client, operations }) {
|
|
2373
|
+
super();
|
|
2374
|
+
this.operations = operations;
|
|
2375
|
+
this.client = client;
|
|
2376
|
+
}
|
|
2377
|
+
updateWorkflowResults({
|
|
2378
|
+
// workflowName,
|
|
2379
|
+
// runId,
|
|
2380
|
+
// stepId,
|
|
2381
|
+
// result,
|
|
2382
|
+
// runtimeContext,
|
|
2383
|
+
}) {
|
|
2384
|
+
throw new Error("Method not implemented.");
|
|
2385
|
+
}
|
|
2386
|
+
updateWorkflowState({
|
|
2387
|
+
// workflowName,
|
|
2388
|
+
// runId,
|
|
2389
|
+
// opts,
|
|
2390
|
+
}) {
|
|
2391
|
+
throw new Error("Method not implemented.");
|
|
2392
|
+
}
|
|
946
2393
|
async persistWorkflowSnapshot({
|
|
947
2394
|
workflowName,
|
|
948
2395
|
runId,
|
|
2396
|
+
resourceId,
|
|
949
2397
|
snapshot
|
|
950
2398
|
}) {
|
|
951
2399
|
try {
|
|
952
|
-
const currentSnapshot = await this.load({
|
|
2400
|
+
const currentSnapshot = await this.operations.load({
|
|
953
2401
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
954
2402
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
955
2403
|
});
|
|
956
2404
|
const now = /* @__PURE__ */ new Date();
|
|
957
2405
|
const persisting = currentSnapshot ? {
|
|
958
2406
|
...currentSnapshot,
|
|
2407
|
+
resourceId,
|
|
959
2408
|
snapshot: JSON.stringify(snapshot),
|
|
960
2409
|
updatedAt: now.toISOString()
|
|
961
2410
|
} : {
|
|
962
2411
|
workflow_name: workflowName,
|
|
963
2412
|
run_id: runId,
|
|
2413
|
+
resourceId,
|
|
964
2414
|
snapshot: JSON.stringify(snapshot),
|
|
965
2415
|
createdAt: now.toISOString(),
|
|
966
2416
|
updatedAt: now.toISOString()
|
|
967
2417
|
};
|
|
968
|
-
await this.
|
|
2418
|
+
await this.client.insert({
|
|
969
2419
|
table: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
970
2420
|
format: "JSONEachRow",
|
|
971
2421
|
values: [persisting],
|
|
@@ -993,7 +2443,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
993
2443
|
runId
|
|
994
2444
|
}) {
|
|
995
2445
|
try {
|
|
996
|
-
const result = await this.load({
|
|
2446
|
+
const result = await this.operations.load({
|
|
997
2447
|
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
998
2448
|
keys: {
|
|
999
2449
|
workflow_name: workflowName,
|
|
@@ -1050,7 +2500,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
1050
2500
|
values.var_workflow_name = workflowName;
|
|
1051
2501
|
}
|
|
1052
2502
|
if (resourceId) {
|
|
1053
|
-
const hasResourceId = await this.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2503
|
+
const hasResourceId = await this.operations.hasColumn(storage.TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
1054
2504
|
if (hasResourceId) {
|
|
1055
2505
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
1056
2506
|
values.var_resourceId = resourceId;
|
|
@@ -1071,7 +2521,7 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
1071
2521
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
1072
2522
|
let total = 0;
|
|
1073
2523
|
if (limit !== void 0 && offset !== void 0) {
|
|
1074
|
-
const countResult = await this.
|
|
2524
|
+
const countResult = await this.client.query({
|
|
1075
2525
|
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
1076
2526
|
query_params: values,
|
|
1077
2527
|
format: "JSONEachRow"
|
|
@@ -1079,21 +2529,21 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
1079
2529
|
const countRows = await countResult.json();
|
|
1080
2530
|
total = Number(countRows[0]?.count ?? 0);
|
|
1081
2531
|
}
|
|
1082
|
-
const result = await this.
|
|
2532
|
+
const result = await this.client.query({
|
|
1083
2533
|
query: `
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
2534
|
+
SELECT
|
|
2535
|
+
workflow_name,
|
|
2536
|
+
run_id,
|
|
2537
|
+
snapshot,
|
|
2538
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2539
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2540
|
+
resourceId
|
|
2541
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2542
|
+
${whereClause}
|
|
2543
|
+
ORDER BY createdAt DESC
|
|
2544
|
+
${limitClause}
|
|
2545
|
+
${offsetClause}
|
|
2546
|
+
`,
|
|
1097
2547
|
query_params: values,
|
|
1098
2548
|
format: "JSONEachRow"
|
|
1099
2549
|
});
|
|
@@ -1131,18 +2581,19 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
1131
2581
|
values.var_workflow_name = workflowName;
|
|
1132
2582
|
}
|
|
1133
2583
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1134
|
-
const result = await this.
|
|
2584
|
+
const result = await this.client.query({
|
|
1135
2585
|
query: `
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
2586
|
+
SELECT
|
|
2587
|
+
workflow_name,
|
|
2588
|
+
run_id,
|
|
2589
|
+
snapshot,
|
|
2590
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2591
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2592
|
+
resourceId
|
|
2593
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2594
|
+
${whereClause}
|
|
2595
|
+
ORDER BY createdAt DESC LIMIT 1
|
|
2596
|
+
`,
|
|
1146
2597
|
query_params: values,
|
|
1147
2598
|
format: "JSONEachRow"
|
|
1148
2599
|
});
|
|
@@ -1163,47 +2614,276 @@ var ClickhouseStore = class extends storage.MastraStorage {
|
|
|
1163
2614
|
);
|
|
1164
2615
|
}
|
|
1165
2616
|
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
2617
|
+
};
|
|
2618
|
+
|
|
2619
|
+
// src/storage/index.ts
|
|
2620
|
+
var ClickhouseStore = class extends storage.MastraStorage {
|
|
2621
|
+
db;
|
|
2622
|
+
ttl = {};
|
|
2623
|
+
stores;
|
|
2624
|
+
constructor(config) {
|
|
2625
|
+
super({ name: "ClickhouseStore" });
|
|
2626
|
+
this.db = client.createClient({
|
|
2627
|
+
url: config.url,
|
|
2628
|
+
username: config.username,
|
|
2629
|
+
password: config.password,
|
|
2630
|
+
clickhouse_settings: {
|
|
2631
|
+
date_time_input_format: "best_effort",
|
|
2632
|
+
date_time_output_format: "iso",
|
|
2633
|
+
// This is crucial
|
|
2634
|
+
use_client_time_zone: 1,
|
|
2635
|
+
output_format_json_quote_64bit_integers: 0
|
|
2636
|
+
}
|
|
1170
2637
|
});
|
|
1171
|
-
|
|
1172
|
-
|
|
2638
|
+
this.ttl = config.ttl;
|
|
2639
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2640
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2641
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2642
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2643
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2644
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2645
|
+
this.stores = {
|
|
2646
|
+
operations,
|
|
2647
|
+
workflows,
|
|
2648
|
+
scores,
|
|
2649
|
+
legacyEvals,
|
|
2650
|
+
traces,
|
|
2651
|
+
memory
|
|
2652
|
+
};
|
|
1173
2653
|
}
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
2654
|
+
get supports() {
|
|
2655
|
+
return {
|
|
2656
|
+
selectByIncludeResourceScope: true,
|
|
2657
|
+
resourceWorkingMemory: true,
|
|
2658
|
+
hasColumn: true,
|
|
2659
|
+
createTable: true,
|
|
2660
|
+
deleteMessages: false,
|
|
2661
|
+
getScoresBySpan: true
|
|
2662
|
+
};
|
|
1181
2663
|
}
|
|
1182
|
-
async
|
|
1183
|
-
|
|
1184
|
-
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
1185
|
-
domain: error.ErrorDomain.STORAGE,
|
|
1186
|
-
category: error.ErrorCategory.USER,
|
|
1187
|
-
text: "Method not implemented."
|
|
1188
|
-
});
|
|
2664
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2665
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
1189
2666
|
}
|
|
1190
|
-
async
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
2667
|
+
async getEvals(options) {
|
|
2668
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2669
|
+
}
|
|
2670
|
+
async batchInsert({ tableName, records }) {
|
|
2671
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2672
|
+
}
|
|
2673
|
+
async optimizeTable({ tableName }) {
|
|
2674
|
+
try {
|
|
2675
|
+
await this.db.command({
|
|
2676
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2677
|
+
});
|
|
2678
|
+
} catch (error$1) {
|
|
2679
|
+
throw new error.MastraError(
|
|
2680
|
+
{
|
|
2681
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2682
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2683
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2684
|
+
details: { tableName }
|
|
2685
|
+
},
|
|
2686
|
+
error$1
|
|
2687
|
+
);
|
|
2688
|
+
}
|
|
2689
|
+
}
|
|
2690
|
+
async materializeTtl({ tableName }) {
|
|
2691
|
+
try {
|
|
2692
|
+
await this.db.command({
|
|
2693
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2694
|
+
});
|
|
2695
|
+
} catch (error$1) {
|
|
2696
|
+
throw new error.MastraError(
|
|
2697
|
+
{
|
|
2698
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2699
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2700
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2701
|
+
details: { tableName }
|
|
2702
|
+
},
|
|
2703
|
+
error$1
|
|
2704
|
+
);
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
async createTable({
|
|
2708
|
+
tableName,
|
|
2709
|
+
schema
|
|
2710
|
+
}) {
|
|
2711
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2712
|
+
}
|
|
2713
|
+
async dropTable({ tableName }) {
|
|
2714
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2715
|
+
}
|
|
2716
|
+
async alterTable({
|
|
2717
|
+
tableName,
|
|
2718
|
+
schema,
|
|
2719
|
+
ifNotExists
|
|
2720
|
+
}) {
|
|
2721
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2722
|
+
}
|
|
2723
|
+
async clearTable({ tableName }) {
|
|
2724
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2725
|
+
}
|
|
2726
|
+
async insert({ tableName, record }) {
|
|
2727
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2728
|
+
}
|
|
2729
|
+
async load({ tableName, keys }) {
|
|
2730
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2731
|
+
}
|
|
2732
|
+
async updateWorkflowResults({
|
|
2733
|
+
workflowName,
|
|
2734
|
+
runId,
|
|
2735
|
+
stepId,
|
|
2736
|
+
result,
|
|
2737
|
+
runtimeContext
|
|
2738
|
+
}) {
|
|
2739
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2740
|
+
}
|
|
2741
|
+
async updateWorkflowState({
|
|
2742
|
+
workflowName,
|
|
2743
|
+
runId,
|
|
2744
|
+
opts
|
|
2745
|
+
}) {
|
|
2746
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2747
|
+
}
|
|
2748
|
+
async persistWorkflowSnapshot({
|
|
2749
|
+
workflowName,
|
|
2750
|
+
runId,
|
|
2751
|
+
resourceId,
|
|
2752
|
+
snapshot
|
|
2753
|
+
}) {
|
|
2754
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2755
|
+
}
|
|
2756
|
+
async loadWorkflowSnapshot({
|
|
2757
|
+
workflowName,
|
|
2758
|
+
runId
|
|
2759
|
+
}) {
|
|
2760
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2761
|
+
}
|
|
2762
|
+
async getWorkflowRuns({
|
|
2763
|
+
workflowName,
|
|
2764
|
+
fromDate,
|
|
2765
|
+
toDate,
|
|
2766
|
+
limit,
|
|
2767
|
+
offset,
|
|
2768
|
+
resourceId
|
|
2769
|
+
} = {}) {
|
|
2770
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2771
|
+
}
|
|
2772
|
+
async getWorkflowRunById({
|
|
2773
|
+
runId,
|
|
2774
|
+
workflowName
|
|
2775
|
+
}) {
|
|
2776
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2777
|
+
}
|
|
2778
|
+
async getTraces(args) {
|
|
2779
|
+
return this.stores.traces.getTraces(args);
|
|
2780
|
+
}
|
|
2781
|
+
async getTracesPaginated(args) {
|
|
2782
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2783
|
+
}
|
|
2784
|
+
async batchTraceInsert(args) {
|
|
2785
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2786
|
+
}
|
|
2787
|
+
async getThreadById({ threadId }) {
|
|
2788
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2789
|
+
}
|
|
2790
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2791
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2792
|
+
}
|
|
2793
|
+
async saveThread({ thread }) {
|
|
2794
|
+
return this.stores.memory.saveThread({ thread });
|
|
2795
|
+
}
|
|
2796
|
+
async updateThread({
|
|
2797
|
+
id,
|
|
2798
|
+
title,
|
|
2799
|
+
metadata
|
|
2800
|
+
}) {
|
|
2801
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2802
|
+
}
|
|
2803
|
+
async deleteThread({ threadId }) {
|
|
2804
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2805
|
+
}
|
|
2806
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2807
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2808
|
+
}
|
|
2809
|
+
async getMessages({
|
|
2810
|
+
threadId,
|
|
2811
|
+
resourceId,
|
|
2812
|
+
selectBy,
|
|
2813
|
+
format
|
|
2814
|
+
}) {
|
|
2815
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2816
|
+
}
|
|
2817
|
+
async getMessagesById({
|
|
2818
|
+
messageIds,
|
|
2819
|
+
format
|
|
2820
|
+
}) {
|
|
2821
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2822
|
+
}
|
|
2823
|
+
async saveMessages(args) {
|
|
2824
|
+
return this.stores.memory.saveMessages(args);
|
|
2825
|
+
}
|
|
2826
|
+
async getMessagesPaginated(args) {
|
|
2827
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2828
|
+
}
|
|
2829
|
+
async updateMessages(args) {
|
|
2830
|
+
return this.stores.memory.updateMessages(args);
|
|
2831
|
+
}
|
|
2832
|
+
async getResourceById({ resourceId }) {
|
|
2833
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2834
|
+
}
|
|
2835
|
+
async saveResource({ resource }) {
|
|
2836
|
+
return this.stores.memory.saveResource({ resource });
|
|
2837
|
+
}
|
|
2838
|
+
async updateResource({
|
|
2839
|
+
resourceId,
|
|
2840
|
+
workingMemory,
|
|
2841
|
+
metadata
|
|
2842
|
+
}) {
|
|
2843
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2844
|
+
}
|
|
2845
|
+
async getScoreById({ id }) {
|
|
2846
|
+
return this.stores.scores.getScoreById({ id });
|
|
2847
|
+
}
|
|
2848
|
+
async saveScore(_score) {
|
|
2849
|
+
return this.stores.scores.saveScore(_score);
|
|
2850
|
+
}
|
|
2851
|
+
async getScoresByRunId({
|
|
2852
|
+
runId,
|
|
2853
|
+
pagination
|
|
2854
|
+
}) {
|
|
2855
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2856
|
+
}
|
|
2857
|
+
async getScoresByEntityId({
|
|
2858
|
+
entityId,
|
|
2859
|
+
entityType,
|
|
2860
|
+
pagination
|
|
2861
|
+
}) {
|
|
2862
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2863
|
+
}
|
|
2864
|
+
async getScoresByScorerId({
|
|
2865
|
+
scorerId,
|
|
2866
|
+
pagination,
|
|
2867
|
+
entityId,
|
|
2868
|
+
entityType,
|
|
2869
|
+
source
|
|
2870
|
+
}) {
|
|
2871
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
|
|
2872
|
+
}
|
|
2873
|
+
async getScoresBySpan({
|
|
2874
|
+
traceId,
|
|
2875
|
+
spanId,
|
|
2876
|
+
pagination
|
|
2877
|
+
}) {
|
|
2878
|
+
return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
|
|
1197
2879
|
}
|
|
1198
2880
|
async close() {
|
|
1199
2881
|
await this.db.close();
|
|
1200
2882
|
}
|
|
1201
|
-
async updateMessages(_args) {
|
|
1202
|
-
this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
|
|
1203
|
-
throw new Error("Method not implemented");
|
|
1204
|
-
}
|
|
1205
2883
|
};
|
|
1206
2884
|
|
|
1207
2885
|
exports.COLUMN_TYPES = COLUMN_TYPES;
|
|
1208
2886
|
exports.ClickhouseStore = ClickhouseStore;
|
|
1209
2887
|
exports.TABLE_ENGINES = TABLE_ENGINES;
|
|
2888
|
+
//# sourceMappingURL=index.cjs.map
|
|
2889
|
+
//# sourceMappingURL=index.cjs.map
|