@mastra/clickhouse 0.0.0-tsconfig-compile-20250703214351 → 0.0.0-unified-sidebar-20251010130811
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +524 -4
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2278 -598
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2278 -598
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +87 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +54 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +55 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +245 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +25 -11
- package/dist/_tsup-dts-rollup.d.cts +0 -191
- package/dist/_tsup-dts-rollup.d.ts +0 -191
- package/dist/index.d.cts +0 -4
- package/docker-compose.yaml +0 -15
- package/eslint.config.js +0 -6
- package/src/index.ts +0 -1
- package/src/storage/index.test.ts +0 -1154
- package/src/storage/index.ts +0 -1464
- package/tsconfig.json +0 -5
- package/vitest.config.ts +0 -12
package/dist/index.js
CHANGED
|
@@ -1,22 +1,20 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import { MessageList } from '@mastra/core/agent';
|
|
3
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
4
|
-
import { TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, TABLE_SCHEMAS } from '@mastra/core/storage';
|
|
3
|
+
import { TABLE_AI_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, ScoresStorage, safelyParseJSON, LegacyEvalsStorage, TracesStorage, MemoryStorage, resolveMessageLimit } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { saveScorePayloadSchema } from '@mastra/core/scores';
|
|
5
6
|
|
|
6
7
|
// src/storage/index.ts
|
|
7
|
-
function safelyParseJSON(jsonString) {
|
|
8
|
-
try {
|
|
9
|
-
return JSON.parse(jsonString);
|
|
10
|
-
} catch {
|
|
11
|
-
return {};
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
8
|
var TABLE_ENGINES = {
|
|
15
9
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
16
10
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
17
11
|
[TABLE_TRACES]: `MergeTree()`,
|
|
18
12
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
19
|
-
[TABLE_EVALS]: `MergeTree()
|
|
13
|
+
[TABLE_EVALS]: `MergeTree()`,
|
|
14
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
15
|
+
[TABLE_RESOURCES]: `ReplacingMergeTree()`,
|
|
16
|
+
// TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
|
|
17
|
+
[TABLE_AI_SPANS]: `ReplacingMergeTree()`
|
|
20
18
|
};
|
|
21
19
|
var COLUMN_TYPES = {
|
|
22
20
|
text: "String",
|
|
@@ -24,11 +22,10 @@ var COLUMN_TYPES = {
|
|
|
24
22
|
uuid: "String",
|
|
25
23
|
jsonb: "String",
|
|
26
24
|
integer: "Int64",
|
|
27
|
-
|
|
25
|
+
float: "Float64",
|
|
26
|
+
bigint: "Int64",
|
|
27
|
+
boolean: "Bool"
|
|
28
28
|
};
|
|
29
|
-
function transformRows(rows) {
|
|
30
|
-
return rows.map((row) => transformRow(row));
|
|
31
|
-
}
|
|
32
29
|
function transformRow(row) {
|
|
33
30
|
if (!row) {
|
|
34
31
|
return row;
|
|
@@ -39,31 +36,56 @@ function transformRow(row) {
|
|
|
39
36
|
if (row.updatedAt) {
|
|
40
37
|
row.updatedAt = new Date(row.updatedAt);
|
|
41
38
|
}
|
|
39
|
+
if (row.content && typeof row.content === "string") {
|
|
40
|
+
row.content = safelyParseJSON(row.content);
|
|
41
|
+
}
|
|
42
42
|
return row;
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// This is crucial
|
|
57
|
-
use_client_time_zone: 1,
|
|
58
|
-
output_format_json_quote_64bit_integers: 0
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
this.ttl = config.ttl;
|
|
44
|
+
function transformRows(rows) {
|
|
45
|
+
return rows.map((row) => transformRow(row));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
49
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
50
|
+
client;
|
|
51
|
+
operations;
|
|
52
|
+
constructor({ client, operations }) {
|
|
53
|
+
super();
|
|
54
|
+
this.client = client;
|
|
55
|
+
this.operations = operations;
|
|
62
56
|
}
|
|
63
57
|
transformEvalRow(row) {
|
|
64
58
|
row = transformRow(row);
|
|
65
|
-
|
|
66
|
-
|
|
59
|
+
let resultValue;
|
|
60
|
+
try {
|
|
61
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
62
|
+
resultValue = JSON.parse(row.result);
|
|
63
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
64
|
+
resultValue = row.result;
|
|
65
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
66
|
+
resultValue = { score: 0 };
|
|
67
|
+
} else {
|
|
68
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
69
|
+
}
|
|
70
|
+
} catch (error) {
|
|
71
|
+
console.error("Error parsing result field:", row.result, error);
|
|
72
|
+
throw new MastraError({
|
|
73
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
74
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
75
|
+
domain: ErrorDomain.STORAGE,
|
|
76
|
+
category: ErrorCategory.USER
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
let testInfoValue;
|
|
80
|
+
try {
|
|
81
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
82
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
83
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
84
|
+
testInfoValue = row.test_info;
|
|
85
|
+
}
|
|
86
|
+
} catch {
|
|
87
|
+
testInfoValue = void 0;
|
|
88
|
+
}
|
|
67
89
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
90
|
throw new MastraError({
|
|
69
91
|
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
@@ -85,23 +107,11 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
85
107
|
createdAt: row.created_at
|
|
86
108
|
};
|
|
87
109
|
}
|
|
88
|
-
escape(value) {
|
|
89
|
-
if (typeof value === "string") {
|
|
90
|
-
return `'${value.replace(/'/g, "''")}'`;
|
|
91
|
-
}
|
|
92
|
-
if (value instanceof Date) {
|
|
93
|
-
return `'${value.toISOString()}'`;
|
|
94
|
-
}
|
|
95
|
-
if (value === null || value === void 0) {
|
|
96
|
-
return "NULL";
|
|
97
|
-
}
|
|
98
|
-
return value.toString();
|
|
99
|
-
}
|
|
100
110
|
async getEvalsByAgentName(agentName, type) {
|
|
101
111
|
try {
|
|
102
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
103
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
104
|
-
const result = await this.
|
|
112
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
113
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
114
|
+
const result = await this.client.query({
|
|
105
115
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
106
116
|
query_params: { var_agent_name: agentName },
|
|
107
117
|
clickhouse_settings: {
|
|
@@ -131,388 +141,438 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
131
141
|
);
|
|
132
142
|
}
|
|
133
143
|
}
|
|
134
|
-
async
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
values: records.map((record) => ({
|
|
139
|
-
...Object.fromEntries(
|
|
140
|
-
Object.entries(record).map(([key, value]) => [
|
|
141
|
-
key,
|
|
142
|
-
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
143
|
-
])
|
|
144
|
-
)
|
|
145
|
-
})),
|
|
146
|
-
format: "JSONEachRow",
|
|
147
|
-
clickhouse_settings: {
|
|
148
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
149
|
-
date_time_input_format: "best_effort",
|
|
150
|
-
use_client_time_zone: 1,
|
|
151
|
-
output_format_json_quote_64bit_integers: 0
|
|
152
|
-
}
|
|
153
|
-
});
|
|
154
|
-
} catch (error) {
|
|
155
|
-
throw new MastraError(
|
|
156
|
-
{
|
|
157
|
-
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
158
|
-
domain: ErrorDomain.STORAGE,
|
|
159
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
160
|
-
details: { tableName }
|
|
161
|
-
},
|
|
162
|
-
error
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
async getTraces({
|
|
167
|
-
name,
|
|
168
|
-
scope,
|
|
169
|
-
page,
|
|
170
|
-
perPage,
|
|
171
|
-
attributes,
|
|
172
|
-
filters,
|
|
173
|
-
fromDate,
|
|
174
|
-
toDate
|
|
175
|
-
}) {
|
|
176
|
-
const limit = perPage;
|
|
177
|
-
const offset = page * perPage;
|
|
178
|
-
const args = {};
|
|
144
|
+
async getEvals(options = {}) {
|
|
145
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
146
|
+
const fromDate = dateRange?.start;
|
|
147
|
+
const toDate = dateRange?.end;
|
|
179
148
|
const conditions = [];
|
|
180
|
-
if (
|
|
181
|
-
conditions.push(`
|
|
182
|
-
args.var_name = name;
|
|
183
|
-
}
|
|
184
|
-
if (scope) {
|
|
185
|
-
conditions.push(`scope = {var_scope:String}`);
|
|
186
|
-
args.var_scope = scope;
|
|
149
|
+
if (agentName) {
|
|
150
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
187
151
|
}
|
|
188
|
-
if (
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
conditions.push(
|
|
197
|
-
`${key} = {var_col_${key}:${COLUMN_TYPES[TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
|
|
198
|
-
);
|
|
199
|
-
args[`var_col_${key}`] = value;
|
|
200
|
-
});
|
|
152
|
+
if (type === "test") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
155
|
+
);
|
|
156
|
+
} else if (type === "live") {
|
|
157
|
+
conditions.push(
|
|
158
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
159
|
+
);
|
|
201
160
|
}
|
|
202
161
|
if (fromDate) {
|
|
203
|
-
conditions.push(`
|
|
204
|
-
|
|
162
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
163
|
+
fromDate.toISOString();
|
|
205
164
|
}
|
|
206
165
|
if (toDate) {
|
|
207
|
-
conditions.push(`
|
|
208
|
-
|
|
166
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
167
|
+
toDate.toISOString();
|
|
209
168
|
}
|
|
210
169
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
211
170
|
try {
|
|
212
|
-
const
|
|
213
|
-
query: `SELECT
|
|
214
|
-
query_params:
|
|
171
|
+
const countResult = await this.client.query({
|
|
172
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
173
|
+
query_params: {
|
|
174
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
175
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
176
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
177
|
+
},
|
|
215
178
|
clickhouse_settings: {
|
|
216
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
217
179
|
date_time_input_format: "best_effort",
|
|
218
180
|
date_time_output_format: "iso",
|
|
219
181
|
use_client_time_zone: 1,
|
|
220
182
|
output_format_json_quote_64bit_integers: 0
|
|
221
183
|
}
|
|
222
184
|
});
|
|
223
|
-
|
|
224
|
-
|
|
185
|
+
const countData = await countResult.json();
|
|
186
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
187
|
+
const currentOffset = page * perPage;
|
|
188
|
+
const hasMore = currentOffset + perPage < total;
|
|
189
|
+
if (total === 0) {
|
|
190
|
+
return {
|
|
191
|
+
evals: [],
|
|
192
|
+
total: 0,
|
|
193
|
+
page,
|
|
194
|
+
perPage,
|
|
195
|
+
hasMore: false
|
|
196
|
+
};
|
|
225
197
|
}
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
198
|
+
const dataResult = await this.client.query({
|
|
199
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
200
|
+
query_params: {
|
|
201
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
202
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
203
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
204
|
+
var_limit: perPage || 100,
|
|
205
|
+
var_offset: currentOffset || 0
|
|
206
|
+
},
|
|
207
|
+
clickhouse_settings: {
|
|
208
|
+
date_time_input_format: "best_effort",
|
|
209
|
+
date_time_output_format: "iso",
|
|
210
|
+
use_client_time_zone: 1,
|
|
211
|
+
output_format_json_quote_64bit_integers: 0
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
const rows = await dataResult.json();
|
|
215
|
+
return {
|
|
216
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
217
|
+
total,
|
|
218
|
+
page,
|
|
219
|
+
perPage,
|
|
220
|
+
hasMore
|
|
221
|
+
};
|
|
244
222
|
} catch (error) {
|
|
245
223
|
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
246
|
-
return
|
|
224
|
+
return {
|
|
225
|
+
evals: [],
|
|
226
|
+
total: 0,
|
|
227
|
+
page,
|
|
228
|
+
perPage,
|
|
229
|
+
hasMore: false
|
|
230
|
+
};
|
|
247
231
|
}
|
|
248
232
|
throw new MastraError(
|
|
249
233
|
{
|
|
250
|
-
id: "
|
|
234
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
251
235
|
domain: ErrorDomain.STORAGE,
|
|
252
236
|
category: ErrorCategory.THIRD_PARTY,
|
|
253
|
-
details: {
|
|
254
|
-
name: name ?? null,
|
|
255
|
-
scope: scope ?? null,
|
|
256
|
-
page,
|
|
257
|
-
perPage,
|
|
258
|
-
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
259
|
-
filters: filters ? JSON.stringify(filters) : null,
|
|
260
|
-
fromDate: fromDate?.toISOString() ?? null,
|
|
261
|
-
toDate: toDate?.toISOString() ?? null
|
|
262
|
-
}
|
|
237
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
263
238
|
},
|
|
264
239
|
error
|
|
265
240
|
);
|
|
266
241
|
}
|
|
267
242
|
}
|
|
268
|
-
|
|
243
|
+
};
|
|
244
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
245
|
+
client;
|
|
246
|
+
operations;
|
|
247
|
+
constructor({ client, operations }) {
|
|
248
|
+
super();
|
|
249
|
+
this.client = client;
|
|
250
|
+
this.operations = operations;
|
|
251
|
+
}
|
|
252
|
+
async getMessages({
|
|
253
|
+
threadId,
|
|
254
|
+
resourceId,
|
|
255
|
+
selectBy,
|
|
256
|
+
format
|
|
257
|
+
}) {
|
|
269
258
|
try {
|
|
270
|
-
|
|
271
|
-
|
|
259
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
260
|
+
const messages = [];
|
|
261
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
262
|
+
const include = selectBy?.include || [];
|
|
263
|
+
if (include.length) {
|
|
264
|
+
const unionQueries = [];
|
|
265
|
+
const params = [];
|
|
266
|
+
let paramIdx = 1;
|
|
267
|
+
for (const inc of include) {
|
|
268
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
269
|
+
const searchId = inc.threadId || threadId;
|
|
270
|
+
unionQueries.push(`
|
|
271
|
+
SELECT * FROM (
|
|
272
|
+
WITH numbered_messages AS (
|
|
273
|
+
SELECT
|
|
274
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
275
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
276
|
+
FROM "${TABLE_MESSAGES}"
|
|
277
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
278
|
+
),
|
|
279
|
+
target_positions AS (
|
|
280
|
+
SELECT row_num as target_pos
|
|
281
|
+
FROM numbered_messages
|
|
282
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
283
|
+
)
|
|
284
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
285
|
+
FROM numbered_messages m
|
|
286
|
+
CROSS JOIN target_positions t
|
|
287
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
288
|
+
) AS query_${paramIdx}
|
|
289
|
+
`);
|
|
290
|
+
params.push(
|
|
291
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
292
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
293
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
294
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
295
|
+
);
|
|
296
|
+
paramIdx++;
|
|
297
|
+
}
|
|
298
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
299
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
300
|
+
const includeResult = await this.client.query({
|
|
301
|
+
query: finalQuery,
|
|
302
|
+
query_params: mergedParams,
|
|
303
|
+
clickhouse_settings: {
|
|
304
|
+
date_time_input_format: "best_effort",
|
|
305
|
+
date_time_output_format: "iso",
|
|
306
|
+
use_client_time_zone: 1,
|
|
307
|
+
output_format_json_quote_64bit_integers: 0
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
const rows2 = await includeResult.json();
|
|
311
|
+
const includedMessages = transformRows(rows2.data);
|
|
312
|
+
const seen = /* @__PURE__ */ new Set();
|
|
313
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
314
|
+
if (seen.has(message.id)) return false;
|
|
315
|
+
seen.add(message.id);
|
|
316
|
+
return true;
|
|
317
|
+
});
|
|
318
|
+
messages.push(...dedupedMessages);
|
|
319
|
+
}
|
|
320
|
+
const result = await this.client.query({
|
|
321
|
+
query: `
|
|
322
|
+
SELECT
|
|
323
|
+
id,
|
|
324
|
+
content,
|
|
325
|
+
role,
|
|
326
|
+
type,
|
|
327
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
328
|
+
thread_id AS "threadId"
|
|
329
|
+
FROM "${TABLE_MESSAGES}"
|
|
330
|
+
WHERE thread_id = {threadId:String}
|
|
331
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
332
|
+
ORDER BY "createdAt" DESC
|
|
333
|
+
LIMIT {limit:Int64}
|
|
334
|
+
`,
|
|
335
|
+
query_params: {
|
|
336
|
+
threadId,
|
|
337
|
+
exclude: messages.map((m) => m.id),
|
|
338
|
+
limit
|
|
339
|
+
},
|
|
340
|
+
clickhouse_settings: {
|
|
341
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
342
|
+
date_time_input_format: "best_effort",
|
|
343
|
+
date_time_output_format: "iso",
|
|
344
|
+
use_client_time_zone: 1,
|
|
345
|
+
output_format_json_quote_64bit_integers: 0
|
|
346
|
+
}
|
|
347
|
+
});
|
|
348
|
+
const rows = await result.json();
|
|
349
|
+
messages.push(...transformRows(rows.data));
|
|
350
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
351
|
+
messages.forEach((message) => {
|
|
352
|
+
if (typeof message.content === "string") {
|
|
353
|
+
try {
|
|
354
|
+
message.content = JSON.parse(message.content);
|
|
355
|
+
} catch {
|
|
356
|
+
}
|
|
357
|
+
}
|
|
272
358
|
});
|
|
359
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
360
|
+
if (format === `v2`) return list.get.all.v2();
|
|
361
|
+
return list.get.all.v1();
|
|
273
362
|
} catch (error) {
|
|
274
363
|
throw new MastraError(
|
|
275
364
|
{
|
|
276
|
-
id: "
|
|
365
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
277
366
|
domain: ErrorDomain.STORAGE,
|
|
278
367
|
category: ErrorCategory.THIRD_PARTY,
|
|
279
|
-
details: {
|
|
368
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
280
369
|
},
|
|
281
370
|
error
|
|
282
371
|
);
|
|
283
372
|
}
|
|
284
373
|
}
|
|
285
|
-
async
|
|
374
|
+
async getMessagesById({
|
|
375
|
+
messageIds,
|
|
376
|
+
format
|
|
377
|
+
}) {
|
|
378
|
+
if (messageIds.length === 0) return [];
|
|
286
379
|
try {
|
|
287
|
-
await this.
|
|
288
|
-
query: `
|
|
380
|
+
const result = await this.client.query({
|
|
381
|
+
query: `
|
|
382
|
+
SELECT
|
|
383
|
+
id,
|
|
384
|
+
content,
|
|
385
|
+
role,
|
|
386
|
+
type,
|
|
387
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
388
|
+
thread_id AS "threadId",
|
|
389
|
+
"resourceId"
|
|
390
|
+
FROM "${TABLE_MESSAGES}"
|
|
391
|
+
WHERE id IN {messageIds:Array(String)}
|
|
392
|
+
ORDER BY "createdAt" DESC
|
|
393
|
+
`,
|
|
394
|
+
query_params: {
|
|
395
|
+
messageIds
|
|
396
|
+
},
|
|
397
|
+
clickhouse_settings: {
|
|
398
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
399
|
+
date_time_input_format: "best_effort",
|
|
400
|
+
date_time_output_format: "iso",
|
|
401
|
+
use_client_time_zone: 1,
|
|
402
|
+
output_format_json_quote_64bit_integers: 0
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
const rows = await result.json();
|
|
406
|
+
const messages = transformRows(rows.data);
|
|
407
|
+
messages.forEach((message) => {
|
|
408
|
+
if (typeof message.content === "string") {
|
|
409
|
+
try {
|
|
410
|
+
message.content = JSON.parse(message.content);
|
|
411
|
+
} catch {
|
|
412
|
+
}
|
|
413
|
+
}
|
|
289
414
|
});
|
|
415
|
+
const list = new MessageList().add(messages, "memory");
|
|
416
|
+
if (format === `v1`) return list.get.all.v1();
|
|
417
|
+
return list.get.all.v2();
|
|
290
418
|
} catch (error) {
|
|
291
419
|
throw new MastraError(
|
|
292
420
|
{
|
|
293
|
-
id: "
|
|
421
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
|
|
294
422
|
domain: ErrorDomain.STORAGE,
|
|
295
423
|
category: ErrorCategory.THIRD_PARTY,
|
|
296
|
-
details: {
|
|
424
|
+
details: { messageIds: JSON.stringify(messageIds) }
|
|
297
425
|
},
|
|
298
426
|
error
|
|
299
427
|
);
|
|
300
428
|
}
|
|
301
429
|
}
|
|
302
|
-
async
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
}
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
${["id String"].concat(columns)}
|
|
317
|
-
)
|
|
318
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
319
|
-
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
320
|
-
ORDER BY (createdAt, run_id, workflow_name)
|
|
321
|
-
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
322
|
-
SETTINGS index_granularity = 8192
|
|
323
|
-
` : `
|
|
324
|
-
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
325
|
-
${columns}
|
|
326
|
-
)
|
|
327
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
328
|
-
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
329
|
-
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
330
|
-
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
331
|
-
SETTINGS index_granularity = 8192
|
|
332
|
-
`;
|
|
333
|
-
await this.db.query({
|
|
334
|
-
query: sql,
|
|
335
|
-
clickhouse_settings: {
|
|
336
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
337
|
-
date_time_input_format: "best_effort",
|
|
338
|
-
date_time_output_format: "iso",
|
|
339
|
-
use_client_time_zone: 1,
|
|
340
|
-
output_format_json_quote_64bit_integers: 0
|
|
341
|
-
}
|
|
342
|
-
});
|
|
343
|
-
} catch (error) {
|
|
344
|
-
throw new MastraError(
|
|
345
|
-
{
|
|
346
|
-
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
347
|
-
domain: ErrorDomain.STORAGE,
|
|
348
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
349
|
-
details: { tableName }
|
|
350
|
-
},
|
|
351
|
-
error
|
|
352
|
-
);
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
getSqlType(type) {
|
|
356
|
-
switch (type) {
|
|
357
|
-
case "text":
|
|
358
|
-
return "String";
|
|
359
|
-
case "timestamp":
|
|
360
|
-
return "DateTime64(3)";
|
|
361
|
-
case "integer":
|
|
362
|
-
case "bigint":
|
|
363
|
-
return "Int64";
|
|
364
|
-
case "jsonb":
|
|
365
|
-
return "String";
|
|
366
|
-
default:
|
|
367
|
-
return super.getSqlType(type);
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
/**
|
|
371
|
-
* Alters table schema to add columns if they don't exist
|
|
372
|
-
* @param tableName Name of the table
|
|
373
|
-
* @param schema Schema of the table
|
|
374
|
-
* @param ifNotExists Array of column names to add if they don't exist
|
|
375
|
-
*/
|
|
376
|
-
async alterTable({
|
|
377
|
-
tableName,
|
|
378
|
-
schema,
|
|
379
|
-
ifNotExists
|
|
380
|
-
}) {
|
|
381
|
-
try {
|
|
382
|
-
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
383
|
-
const result = await this.db.query({
|
|
384
|
-
query: describeSql
|
|
385
|
-
});
|
|
386
|
-
const rows = await result.json();
|
|
387
|
-
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
388
|
-
for (const columnName of ifNotExists) {
|
|
389
|
-
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
390
|
-
const columnDef = schema[columnName];
|
|
391
|
-
let sqlType = this.getSqlType(columnDef.type);
|
|
392
|
-
if (columnDef.nullable !== false) {
|
|
393
|
-
sqlType = `Nullable(${sqlType})`;
|
|
394
|
-
}
|
|
395
|
-
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
396
|
-
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
397
|
-
await this.db.query({
|
|
398
|
-
query: alterSql
|
|
399
|
-
});
|
|
400
|
-
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
401
|
-
}
|
|
430
|
+
async saveMessages(args) {
|
|
431
|
+
const { messages, format = "v1" } = args;
|
|
432
|
+
if (messages.length === 0) return messages;
|
|
433
|
+
for (const message of messages) {
|
|
434
|
+
const resourceId = message.resourceId;
|
|
435
|
+
if (!resourceId) {
|
|
436
|
+
throw new Error("Resource ID is required");
|
|
437
|
+
}
|
|
438
|
+
if (!message.threadId) {
|
|
439
|
+
throw new Error("Thread ID is required");
|
|
440
|
+
}
|
|
441
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
442
|
+
if (!thread) {
|
|
443
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
402
444
|
}
|
|
403
|
-
} catch (error) {
|
|
404
|
-
throw new MastraError(
|
|
405
|
-
{
|
|
406
|
-
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
407
|
-
domain: ErrorDomain.STORAGE,
|
|
408
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
409
|
-
details: { tableName }
|
|
410
|
-
},
|
|
411
|
-
error
|
|
412
|
-
);
|
|
413
445
|
}
|
|
414
|
-
|
|
415
|
-
|
|
446
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
447
|
+
await Promise.all(
|
|
448
|
+
messages.map(async (m) => {
|
|
449
|
+
const resourceId = m.resourceId;
|
|
450
|
+
if (!resourceId) {
|
|
451
|
+
throw new Error("Resource ID is required");
|
|
452
|
+
}
|
|
453
|
+
if (!m.threadId) {
|
|
454
|
+
throw new Error("Thread ID is required");
|
|
455
|
+
}
|
|
456
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
457
|
+
if (!thread) {
|
|
458
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
459
|
+
}
|
|
460
|
+
threadIdSet.set(m.threadId, thread);
|
|
461
|
+
})
|
|
462
|
+
);
|
|
416
463
|
try {
|
|
417
|
-
await this.
|
|
418
|
-
query: `
|
|
464
|
+
const existingResult = await this.client.query({
|
|
465
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
466
|
+
query_params: {
|
|
467
|
+
ids: messages.map((m) => m.id)
|
|
468
|
+
},
|
|
419
469
|
clickhouse_settings: {
|
|
420
470
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
421
471
|
date_time_input_format: "best_effort",
|
|
422
472
|
date_time_output_format: "iso",
|
|
423
473
|
use_client_time_zone: 1,
|
|
424
474
|
output_format_json_quote_64bit_integers: 0
|
|
425
|
-
}
|
|
426
|
-
});
|
|
427
|
-
} catch (error) {
|
|
428
|
-
throw new MastraError(
|
|
429
|
-
{
|
|
430
|
-
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
431
|
-
domain: ErrorDomain.STORAGE,
|
|
432
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
433
|
-
details: { tableName }
|
|
434
475
|
},
|
|
435
|
-
|
|
436
|
-
);
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
476
|
+
format: "JSONEachRow"
|
|
477
|
+
});
|
|
478
|
+
const existingRows = await existingResult.json();
|
|
479
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
480
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
481
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
482
|
+
const toMove = messages.filter((m) => {
|
|
483
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
484
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
485
|
+
});
|
|
486
|
+
const deletePromises = toMove.map((message) => {
|
|
487
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
488
|
+
if (!existingRow) return Promise.resolve();
|
|
489
|
+
return this.client.command({
|
|
490
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
491
|
+
query_params: {
|
|
492
|
+
var_id: message.id,
|
|
493
|
+
var_old_thread_id: existingRow.thread_id
|
|
494
|
+
},
|
|
495
|
+
clickhouse_settings: {
|
|
496
|
+
date_time_input_format: "best_effort",
|
|
497
|
+
use_client_time_zone: 1,
|
|
498
|
+
output_format_json_quote_64bit_integers: 0
|
|
448
499
|
}
|
|
449
|
-
|
|
450
|
-
format: "JSONEachRow",
|
|
451
|
-
clickhouse_settings: {
|
|
452
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
453
|
-
output_format_json_quote_64bit_integers: 0,
|
|
454
|
-
date_time_input_format: "best_effort",
|
|
455
|
-
use_client_time_zone: 1
|
|
456
|
-
}
|
|
500
|
+
});
|
|
457
501
|
});
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
502
|
+
const updatePromises = toUpdate.map(
|
|
503
|
+
(message) => this.client.command({
|
|
504
|
+
query: `
|
|
505
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
506
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
507
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
508
|
+
`,
|
|
509
|
+
query_params: {
|
|
510
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
511
|
+
var_role: message.role,
|
|
512
|
+
var_type: message.type || "v2",
|
|
513
|
+
var_resourceId: message.resourceId,
|
|
514
|
+
var_id: message.id,
|
|
515
|
+
var_thread_id: message.threadId
|
|
516
|
+
},
|
|
517
|
+
clickhouse_settings: {
|
|
518
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
519
|
+
date_time_input_format: "best_effort",
|
|
520
|
+
use_client_time_zone: 1,
|
|
521
|
+
output_format_json_quote_64bit_integers: 0
|
|
522
|
+
}
|
|
523
|
+
})
|
|
467
524
|
);
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
525
|
+
await Promise.all([
|
|
526
|
+
// Insert new messages (including moved messages)
|
|
527
|
+
this.client.insert({
|
|
528
|
+
table: TABLE_MESSAGES,
|
|
529
|
+
format: "JSONEachRow",
|
|
530
|
+
values: toInsert.map((message) => ({
|
|
531
|
+
id: message.id,
|
|
532
|
+
thread_id: message.threadId,
|
|
533
|
+
resourceId: message.resourceId,
|
|
534
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
535
|
+
createdAt: message.createdAt.toISOString(),
|
|
536
|
+
role: message.role,
|
|
537
|
+
type: message.type || "v2"
|
|
538
|
+
})),
|
|
539
|
+
clickhouse_settings: {
|
|
540
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
541
|
+
date_time_input_format: "best_effort",
|
|
542
|
+
use_client_time_zone: 1,
|
|
543
|
+
output_format_json_quote_64bit_integers: 0
|
|
544
|
+
}
|
|
545
|
+
}),
|
|
546
|
+
...updatePromises,
|
|
547
|
+
...deletePromises,
|
|
548
|
+
// Update thread's updatedAt timestamp
|
|
549
|
+
this.client.insert({
|
|
550
|
+
table: TABLE_THREADS,
|
|
551
|
+
format: "JSONEachRow",
|
|
552
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
553
|
+
id: thread.id,
|
|
554
|
+
resourceId: thread.resourceId,
|
|
555
|
+
title: thread.title,
|
|
556
|
+
metadata: thread.metadata,
|
|
557
|
+
createdAt: thread.createdAt,
|
|
558
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
559
|
+
})),
|
|
560
|
+
clickhouse_settings: {
|
|
561
|
+
date_time_input_format: "best_effort",
|
|
562
|
+
use_client_time_zone: 1,
|
|
563
|
+
output_format_json_quote_64bit_integers: 0
|
|
564
|
+
}
|
|
565
|
+
})
|
|
566
|
+
]);
|
|
567
|
+
const list = new MessageList().add(messages, "memory");
|
|
568
|
+
if (format === `v2`) return list.get.all.v2();
|
|
569
|
+
return list.get.all.v1();
|
|
509
570
|
} catch (error) {
|
|
510
571
|
throw new MastraError(
|
|
511
572
|
{
|
|
512
|
-
id: "
|
|
573
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
513
574
|
domain: ErrorDomain.STORAGE,
|
|
514
|
-
category: ErrorCategory.THIRD_PARTY
|
|
515
|
-
details: { tableName }
|
|
575
|
+
category: ErrorCategory.THIRD_PARTY
|
|
516
576
|
},
|
|
517
577
|
error
|
|
518
578
|
);
|
|
@@ -520,7 +580,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
520
580
|
}
|
|
521
581
|
async getThreadById({ threadId }) {
|
|
522
582
|
try {
|
|
523
|
-
const result = await this.
|
|
583
|
+
const result = await this.client.query({
|
|
524
584
|
query: `SELECT
|
|
525
585
|
id,
|
|
526
586
|
"resourceId",
|
|
@@ -565,7 +625,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
565
625
|
}
|
|
566
626
|
async getThreadsByResourceId({ resourceId }) {
|
|
567
627
|
try {
|
|
568
|
-
const result = await this.
|
|
628
|
+
const result = await this.client.query({
|
|
569
629
|
query: `SELECT
|
|
570
630
|
id,
|
|
571
631
|
"resourceId",
|
|
@@ -606,7 +666,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
606
666
|
}
|
|
607
667
|
async saveThread({ thread }) {
|
|
608
668
|
try {
|
|
609
|
-
await this.
|
|
669
|
+
await this.client.insert({
|
|
610
670
|
table: TABLE_THREADS,
|
|
611
671
|
values: [
|
|
612
672
|
{
|
|
@@ -656,7 +716,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
656
716
|
metadata: mergedMetadata,
|
|
657
717
|
updatedAt: /* @__PURE__ */ new Date()
|
|
658
718
|
};
|
|
659
|
-
await this.
|
|
719
|
+
await this.client.insert({
|
|
660
720
|
table: TABLE_THREADS,
|
|
661
721
|
format: "JSONEachRow",
|
|
662
722
|
values: [
|
|
@@ -690,14 +750,14 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
690
750
|
}
|
|
691
751
|
async deleteThread({ threadId }) {
|
|
692
752
|
try {
|
|
693
|
-
await this.
|
|
753
|
+
await this.client.command({
|
|
694
754
|
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
695
755
|
query_params: { var_thread_id: threadId },
|
|
696
756
|
clickhouse_settings: {
|
|
697
757
|
output_format_json_quote_64bit_integers: 0
|
|
698
758
|
}
|
|
699
759
|
});
|
|
700
|
-
await this.
|
|
760
|
+
await this.client.command({
|
|
701
761
|
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
702
762
|
query_params: { var_id: threadId },
|
|
703
763
|
clickhouse_settings: {
|
|
@@ -716,254 +776,1644 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
716
776
|
);
|
|
717
777
|
}
|
|
718
778
|
}
|
|
719
|
-
async
|
|
720
|
-
|
|
721
|
-
resourceId,
|
|
722
|
-
selectBy,
|
|
723
|
-
format
|
|
724
|
-
}) {
|
|
779
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
780
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
725
781
|
try {
|
|
726
|
-
const
|
|
727
|
-
const
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
782
|
+
const currentOffset = page * perPage;
|
|
783
|
+
const countResult = await this.client.query({
|
|
784
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
785
|
+
query_params: { resourceId },
|
|
786
|
+
clickhouse_settings: {
|
|
787
|
+
date_time_input_format: "best_effort",
|
|
788
|
+
date_time_output_format: "iso",
|
|
789
|
+
use_client_time_zone: 1,
|
|
790
|
+
output_format_json_quote_64bit_integers: 0
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
const countData = await countResult.json();
|
|
794
|
+
const total = countData.data[0].total;
|
|
795
|
+
if (total === 0) {
|
|
796
|
+
return {
|
|
797
|
+
threads: [],
|
|
798
|
+
total: 0,
|
|
799
|
+
page,
|
|
800
|
+
perPage,
|
|
801
|
+
hasMore: false
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
const dataResult = await this.client.query({
|
|
805
|
+
query: `
|
|
806
|
+
SELECT
|
|
807
|
+
id,
|
|
808
|
+
resourceId,
|
|
809
|
+
title,
|
|
810
|
+
metadata,
|
|
811
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
812
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
813
|
+
FROM ${TABLE_THREADS}
|
|
814
|
+
WHERE resourceId = {resourceId:String}
|
|
815
|
+
ORDER BY createdAt DESC
|
|
816
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
817
|
+
`,
|
|
818
|
+
query_params: {
|
|
819
|
+
resourceId,
|
|
820
|
+
limit: perPage,
|
|
821
|
+
offset: currentOffset
|
|
822
|
+
},
|
|
823
|
+
clickhouse_settings: {
|
|
824
|
+
date_time_input_format: "best_effort",
|
|
825
|
+
date_time_output_format: "iso",
|
|
826
|
+
use_client_time_zone: 1,
|
|
827
|
+
output_format_json_quote_64bit_integers: 0
|
|
828
|
+
}
|
|
829
|
+
});
|
|
830
|
+
const rows = await dataResult.json();
|
|
831
|
+
const threads = transformRows(rows.data);
|
|
832
|
+
return {
|
|
833
|
+
threads,
|
|
834
|
+
total,
|
|
835
|
+
page,
|
|
836
|
+
perPage,
|
|
837
|
+
hasMore: currentOffset + threads.length < total
|
|
838
|
+
};
|
|
839
|
+
} catch (error) {
|
|
840
|
+
throw new MastraError(
|
|
841
|
+
{
|
|
842
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
843
|
+
domain: ErrorDomain.STORAGE,
|
|
844
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
845
|
+
details: { resourceId, page }
|
|
846
|
+
},
|
|
847
|
+
error
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
async getMessagesPaginated(args) {
|
|
852
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
853
|
+
const page = selectBy?.pagination?.page || 0;
|
|
854
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
855
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
856
|
+
try {
|
|
857
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
858
|
+
const offset = page * perPage;
|
|
859
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
860
|
+
const fromDate = dateRange?.start;
|
|
861
|
+
const toDate = dateRange?.end;
|
|
862
|
+
const messages = [];
|
|
863
|
+
if (selectBy?.include?.length) {
|
|
864
|
+
const include = selectBy.include;
|
|
865
|
+
const unionQueries = [];
|
|
866
|
+
const params = [];
|
|
867
|
+
let paramIdx = 1;
|
|
868
|
+
for (const inc of include) {
|
|
869
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
870
|
+
const searchId = inc.threadId || threadId;
|
|
871
|
+
unionQueries.push(`
|
|
872
|
+
SELECT * FROM (
|
|
873
|
+
WITH numbered_messages AS (
|
|
874
|
+
SELECT
|
|
875
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
876
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
877
|
+
FROM "${TABLE_MESSAGES}"
|
|
878
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
879
|
+
),
|
|
880
|
+
target_positions AS (
|
|
881
|
+
SELECT row_num as target_pos
|
|
882
|
+
FROM numbered_messages
|
|
883
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
884
|
+
)
|
|
885
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
886
|
+
FROM numbered_messages m
|
|
887
|
+
CROSS JOIN target_positions t
|
|
888
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
889
|
+
) AS query_${paramIdx}
|
|
890
|
+
`);
|
|
891
|
+
params.push(
|
|
892
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
893
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
894
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
895
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
896
|
+
);
|
|
897
|
+
paramIdx++;
|
|
898
|
+
}
|
|
899
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
900
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
901
|
+
const includeResult = await this.client.query({
|
|
902
|
+
query: finalQuery,
|
|
903
|
+
query_params: mergedParams,
|
|
904
|
+
clickhouse_settings: {
|
|
905
|
+
date_time_input_format: "best_effort",
|
|
773
906
|
date_time_output_format: "iso",
|
|
774
907
|
use_client_time_zone: 1,
|
|
775
908
|
output_format_json_quote_64bit_integers: 0
|
|
776
909
|
}
|
|
777
910
|
});
|
|
778
911
|
const rows2 = await includeResult.json();
|
|
779
|
-
|
|
912
|
+
const includedMessages = transformRows(rows2.data);
|
|
913
|
+
const seen = /* @__PURE__ */ new Set();
|
|
914
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
915
|
+
if (seen.has(message.id)) return false;
|
|
916
|
+
seen.add(message.id);
|
|
917
|
+
return true;
|
|
918
|
+
});
|
|
919
|
+
messages.push(...dedupedMessages);
|
|
780
920
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
LIMIT {limit:Int64}
|
|
795
|
-
`,
|
|
796
|
-
query_params: {
|
|
797
|
-
threadId,
|
|
798
|
-
exclude: messages.map((m) => m.id),
|
|
799
|
-
limit
|
|
800
|
-
},
|
|
921
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
922
|
+
const countParams = { threadId };
|
|
923
|
+
if (fromDate) {
|
|
924
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
925
|
+
countParams.fromDate = fromDate.toISOString();
|
|
926
|
+
}
|
|
927
|
+
if (toDate) {
|
|
928
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
929
|
+
countParams.toDate = toDate.toISOString();
|
|
930
|
+
}
|
|
931
|
+
const countResult = await this.client.query({
|
|
932
|
+
query: countQuery,
|
|
933
|
+
query_params: countParams,
|
|
801
934
|
clickhouse_settings: {
|
|
802
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
803
935
|
date_time_input_format: "best_effort",
|
|
804
936
|
date_time_output_format: "iso",
|
|
805
937
|
use_client_time_zone: 1,
|
|
806
938
|
output_format_json_quote_64bit_integers: 0
|
|
807
939
|
}
|
|
808
940
|
});
|
|
809
|
-
const
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
941
|
+
const countData = await countResult.json();
|
|
942
|
+
const total = countData.data[0].total;
|
|
943
|
+
if (total === 0 && messages.length === 0) {
|
|
944
|
+
return {
|
|
945
|
+
messages: [],
|
|
946
|
+
total: 0,
|
|
947
|
+
page,
|
|
948
|
+
perPage,
|
|
949
|
+
hasMore: false
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
const excludeIds = messages.map((m) => m.id);
|
|
953
|
+
let dataQuery = `
|
|
954
|
+
SELECT
|
|
955
|
+
id,
|
|
956
|
+
content,
|
|
957
|
+
role,
|
|
958
|
+
type,
|
|
959
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
960
|
+
thread_id AS "threadId",
|
|
961
|
+
resourceId
|
|
962
|
+
FROM ${TABLE_MESSAGES}
|
|
963
|
+
WHERE thread_id = {threadId:String}
|
|
964
|
+
`;
|
|
965
|
+
const dataParams = { threadId };
|
|
966
|
+
if (fromDate) {
|
|
967
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
968
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
969
|
+
}
|
|
970
|
+
if (toDate) {
|
|
971
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
972
|
+
dataParams.toDate = toDate.toISOString();
|
|
973
|
+
}
|
|
974
|
+
if (excludeIds.length > 0) {
|
|
975
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
976
|
+
dataParams.excludeIds = excludeIds;
|
|
977
|
+
}
|
|
978
|
+
if (selectBy?.last) {
|
|
979
|
+
dataQuery += `
|
|
980
|
+
ORDER BY createdAt DESC
|
|
981
|
+
LIMIT {limit:Int64}
|
|
982
|
+
`;
|
|
983
|
+
dataParams.limit = perPage;
|
|
984
|
+
} else {
|
|
985
|
+
dataQuery += `
|
|
986
|
+
ORDER BY createdAt ASC
|
|
987
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
988
|
+
`;
|
|
989
|
+
dataParams.limit = perPage;
|
|
990
|
+
dataParams.offset = offset;
|
|
991
|
+
}
|
|
992
|
+
const result = await this.client.query({
|
|
993
|
+
query: dataQuery,
|
|
994
|
+
query_params: dataParams,
|
|
995
|
+
clickhouse_settings: {
|
|
996
|
+
date_time_input_format: "best_effort",
|
|
997
|
+
date_time_output_format: "iso",
|
|
998
|
+
use_client_time_zone: 1,
|
|
999
|
+
output_format_json_quote_64bit_integers: 0
|
|
818
1000
|
}
|
|
819
1001
|
});
|
|
820
|
-
const
|
|
821
|
-
|
|
822
|
-
|
|
1002
|
+
const rows = await result.json();
|
|
1003
|
+
const paginatedMessages = transformRows(rows.data);
|
|
1004
|
+
messages.push(...paginatedMessages);
|
|
1005
|
+
if (selectBy?.last) {
|
|
1006
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1007
|
+
}
|
|
1008
|
+
return {
|
|
1009
|
+
messages: format === "v2" ? messages : messages,
|
|
1010
|
+
total,
|
|
1011
|
+
page,
|
|
1012
|
+
perPage,
|
|
1013
|
+
hasMore: offset + perPage < total
|
|
1014
|
+
};
|
|
823
1015
|
} catch (error) {
|
|
824
|
-
|
|
1016
|
+
const mastraError = new MastraError(
|
|
825
1017
|
{
|
|
826
|
-
id: "
|
|
1018
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
827
1019
|
domain: ErrorDomain.STORAGE,
|
|
828
1020
|
category: ErrorCategory.THIRD_PARTY,
|
|
829
|
-
details: {
|
|
1021
|
+
details: {
|
|
1022
|
+
threadId,
|
|
1023
|
+
resourceId: resourceId ?? ""
|
|
1024
|
+
}
|
|
830
1025
|
},
|
|
831
1026
|
error
|
|
832
1027
|
);
|
|
1028
|
+
this.logger?.trackException?.(mastraError);
|
|
1029
|
+
this.logger?.error?.(mastraError.toString());
|
|
1030
|
+
return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
|
|
833
1031
|
}
|
|
834
1032
|
}
|
|
835
|
-
async
|
|
836
|
-
const { messages
|
|
837
|
-
if (messages.length === 0)
|
|
1033
|
+
async updateMessages(args) {
|
|
1034
|
+
const { messages } = args;
|
|
1035
|
+
if (messages.length === 0) {
|
|
1036
|
+
return [];
|
|
1037
|
+
}
|
|
838
1038
|
try {
|
|
839
|
-
const
|
|
840
|
-
const
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
}
|
|
844
|
-
const thread = await this.getThreadById({ threadId });
|
|
845
|
-
if (!thread) {
|
|
846
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
847
|
-
}
|
|
848
|
-
const existingResult = await this.db.query({
|
|
849
|
-
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
850
|
-
query_params: {
|
|
851
|
-
ids: messages.map((m) => m.id)
|
|
852
|
-
},
|
|
1039
|
+
const messageIds = messages.map((m) => m.id);
|
|
1040
|
+
const existingResult = await this.client.query({
|
|
1041
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
1042
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
853
1043
|
clickhouse_settings: {
|
|
854
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
855
1044
|
date_time_input_format: "best_effort",
|
|
856
1045
|
date_time_output_format: "iso",
|
|
857
1046
|
use_client_time_zone: 1,
|
|
858
1047
|
output_format_json_quote_64bit_integers: 0
|
|
859
|
-
}
|
|
860
|
-
format: "JSONEachRow"
|
|
1048
|
+
}
|
|
861
1049
|
});
|
|
862
1050
|
const existingRows = await existingResult.json();
|
|
863
|
-
const
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
`,
|
|
873
|
-
query_params: {
|
|
874
|
-
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
875
|
-
var_role: message.role,
|
|
876
|
-
var_type: message.type || "v2",
|
|
877
|
-
var_id: message.id,
|
|
878
|
-
var_thread_id: threadId
|
|
879
|
-
},
|
|
880
|
-
clickhouse_settings: {
|
|
881
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
882
|
-
date_time_input_format: "best_effort",
|
|
883
|
-
use_client_time_zone: 1,
|
|
884
|
-
output_format_json_quote_64bit_integers: 0
|
|
1051
|
+
const existingMessages = transformRows(existingRows.data);
|
|
1052
|
+
if (existingMessages.length === 0) {
|
|
1053
|
+
return [];
|
|
1054
|
+
}
|
|
1055
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
1056
|
+
if (typeof msg.content === "string") {
|
|
1057
|
+
try {
|
|
1058
|
+
msg.content = JSON.parse(msg.content);
|
|
1059
|
+
} catch {
|
|
885
1060
|
}
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
1061
|
+
}
|
|
1062
|
+
return msg;
|
|
1063
|
+
});
|
|
1064
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
1065
|
+
const updatePromises = [];
|
|
1066
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1067
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1068
|
+
if (!updatePayload) continue;
|
|
1069
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1070
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1071
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1072
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1073
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1074
|
+
}
|
|
1075
|
+
const setClauses = [];
|
|
1076
|
+
const values = {};
|
|
1077
|
+
let paramIdx = 1;
|
|
1078
|
+
let newContent = null;
|
|
1079
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1080
|
+
if (updatableFields.content) {
|
|
1081
|
+
const existingContent = existingMessage.content || {};
|
|
1082
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1083
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1084
|
+
newContent = {
|
|
1085
|
+
...existingContent,
|
|
1086
|
+
...updatableFields.content,
|
|
1087
|
+
// Deep merge metadata
|
|
1088
|
+
metadata: {
|
|
1089
|
+
...existingMetadata,
|
|
1090
|
+
...updateMetadata
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1094
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1095
|
+
paramIdx++;
|
|
1096
|
+
delete updatableFields.content;
|
|
1097
|
+
}
|
|
1098
|
+
for (const key in updatableFields) {
|
|
1099
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1100
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1101
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1102
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1103
|
+
paramIdx++;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
if (setClauses.length > 0) {
|
|
1107
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1108
|
+
const updateQuery = `
|
|
1109
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1110
|
+
UPDATE ${setClauses.join(", ")}
|
|
1111
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1112
|
+
`;
|
|
1113
|
+
console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1114
|
+
updatePromises.push(
|
|
1115
|
+
this.client.command({
|
|
1116
|
+
query: updateQuery,
|
|
1117
|
+
query_params: values,
|
|
1118
|
+
clickhouse_settings: {
|
|
1119
|
+
date_time_input_format: "best_effort",
|
|
1120
|
+
use_client_time_zone: 1,
|
|
1121
|
+
output_format_json_quote_64bit_integers: 0
|
|
1122
|
+
}
|
|
1123
|
+
})
|
|
1124
|
+
);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
if (updatePromises.length > 0) {
|
|
1128
|
+
await Promise.all(updatePromises);
|
|
1129
|
+
}
|
|
1130
|
+
await this.client.command({
|
|
1131
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1132
|
+
clickhouse_settings: {
|
|
1133
|
+
date_time_input_format: "best_effort",
|
|
1134
|
+
use_client_time_zone: 1,
|
|
1135
|
+
output_format_json_quote_64bit_integers: 0
|
|
1136
|
+
}
|
|
1137
|
+
});
|
|
1138
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1139
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1140
|
+
if (!updatePayload) continue;
|
|
1141
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1142
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1143
|
+
const verifyResult = await this.client.query({
|
|
1144
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1145
|
+
query_params: { messageId: id },
|
|
901
1146
|
clickhouse_settings: {
|
|
902
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
903
1147
|
date_time_input_format: "best_effort",
|
|
1148
|
+
date_time_output_format: "iso",
|
|
904
1149
|
use_client_time_zone: 1,
|
|
905
1150
|
output_format_json_quote_64bit_integers: 0
|
|
906
1151
|
}
|
|
907
|
-
})
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1152
|
+
});
|
|
1153
|
+
const verifyRows = await verifyResult.json();
|
|
1154
|
+
if (verifyRows.data.length > 0) {
|
|
1155
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1156
|
+
if (updatedMessage) {
|
|
1157
|
+
let needsRetry = false;
|
|
1158
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1159
|
+
if (key === "content") {
|
|
1160
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1161
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1162
|
+
if (actualContent !== expectedContent) {
|
|
1163
|
+
needsRetry = true;
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
} else if (updatedMessage[key] !== value) {
|
|
1167
|
+
needsRetry = true;
|
|
1168
|
+
break;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
if (needsRetry) {
|
|
1172
|
+
console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1175
|
+
query_params: { messageId: id },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
let updatedContent = existingMessage.content || {};
|
|
1183
|
+
if (fieldsToUpdate.content) {
|
|
1184
|
+
const existingContent = existingMessage.content || {};
|
|
1185
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1186
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1187
|
+
updatedContent = {
|
|
1188
|
+
...existingContent,
|
|
1189
|
+
...fieldsToUpdate.content,
|
|
1190
|
+
metadata: {
|
|
1191
|
+
...existingMetadata,
|
|
1192
|
+
...updateMetadata
|
|
1193
|
+
}
|
|
1194
|
+
};
|
|
1195
|
+
}
|
|
1196
|
+
const updatedMessageData = {
|
|
1197
|
+
...existingMessage,
|
|
1198
|
+
...fieldsToUpdate,
|
|
1199
|
+
content: updatedContent
|
|
1200
|
+
};
|
|
1201
|
+
await this.client.insert({
|
|
1202
|
+
table: TABLE_MESSAGES,
|
|
1203
|
+
format: "JSONEachRow",
|
|
1204
|
+
values: [
|
|
1205
|
+
{
|
|
1206
|
+
id: updatedMessageData.id,
|
|
1207
|
+
thread_id: updatedMessageData.threadId,
|
|
1208
|
+
resourceId: updatedMessageData.resourceId,
|
|
1209
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1210
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1211
|
+
role: updatedMessageData.role,
|
|
1212
|
+
type: updatedMessageData.type || "v2"
|
|
1213
|
+
}
|
|
1214
|
+
],
|
|
1215
|
+
clickhouse_settings: {
|
|
1216
|
+
date_time_input_format: "best_effort",
|
|
1217
|
+
use_client_time_zone: 1,
|
|
1218
|
+
output_format_json_quote_64bit_integers: 0
|
|
1219
|
+
}
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1226
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1227
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1228
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1229
|
+
const threadResult = await this.client.query({
|
|
1230
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1231
|
+
query_params: { threadId },
|
|
1232
|
+
clickhouse_settings: {
|
|
1233
|
+
date_time_input_format: "best_effort",
|
|
1234
|
+
date_time_output_format: "iso",
|
|
1235
|
+
use_client_time_zone: 1,
|
|
1236
|
+
output_format_json_quote_64bit_integers: 0
|
|
921
1237
|
}
|
|
922
|
-
|
|
1238
|
+
});
|
|
1239
|
+
const threadRows = await threadResult.json();
|
|
1240
|
+
if (threadRows.data.length > 0) {
|
|
1241
|
+
const existingThread = threadRows.data[0];
|
|
1242
|
+
await this.client.command({
|
|
1243
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1244
|
+
query_params: { threadId },
|
|
1245
|
+
clickhouse_settings: {
|
|
1246
|
+
date_time_input_format: "best_effort",
|
|
1247
|
+
use_client_time_zone: 1,
|
|
1248
|
+
output_format_json_quote_64bit_integers: 0
|
|
1249
|
+
}
|
|
1250
|
+
});
|
|
1251
|
+
await this.client.insert({
|
|
1252
|
+
table: TABLE_THREADS,
|
|
1253
|
+
format: "JSONEachRow",
|
|
1254
|
+
values: [
|
|
1255
|
+
{
|
|
1256
|
+
id: existingThread.id,
|
|
1257
|
+
resourceId: existingThread.resourceId,
|
|
1258
|
+
title: existingThread.title,
|
|
1259
|
+
metadata: existingThread.metadata,
|
|
1260
|
+
createdAt: existingThread.createdAt,
|
|
1261
|
+
updatedAt: now
|
|
1262
|
+
}
|
|
1263
|
+
],
|
|
1264
|
+
clickhouse_settings: {
|
|
1265
|
+
date_time_input_format: "best_effort",
|
|
1266
|
+
use_client_time_zone: 1,
|
|
1267
|
+
output_format_json_quote_64bit_integers: 0
|
|
1268
|
+
}
|
|
1269
|
+
});
|
|
1270
|
+
}
|
|
1271
|
+
});
|
|
1272
|
+
await Promise.all(threadUpdatePromises);
|
|
1273
|
+
}
|
|
1274
|
+
const updatedMessages = [];
|
|
1275
|
+
for (const messageId of messageIds) {
|
|
1276
|
+
const updatedResult = await this.client.query({
|
|
1277
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1278
|
+
query_params: { messageId },
|
|
923
1279
|
clickhouse_settings: {
|
|
924
1280
|
date_time_input_format: "best_effort",
|
|
1281
|
+
date_time_output_format: "iso",
|
|
925
1282
|
use_client_time_zone: 1,
|
|
926
1283
|
output_format_json_quote_64bit_integers: 0
|
|
927
1284
|
}
|
|
928
|
-
})
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
1285
|
+
});
|
|
1286
|
+
const updatedRows = await updatedResult.json();
|
|
1287
|
+
if (updatedRows.data.length > 0) {
|
|
1288
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1289
|
+
if (message) {
|
|
1290
|
+
updatedMessages.push(message);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
return updatedMessages.map((message) => {
|
|
1295
|
+
if (typeof message.content === "string") {
|
|
1296
|
+
try {
|
|
1297
|
+
message.content = JSON.parse(message.content);
|
|
1298
|
+
} catch {
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
return message;
|
|
1302
|
+
});
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
async getResourceById({ resourceId }) {
|
|
1316
|
+
try {
|
|
1317
|
+
const result = await this.client.query({
|
|
1318
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1319
|
+
query_params: { resourceId },
|
|
1320
|
+
clickhouse_settings: {
|
|
1321
|
+
date_time_input_format: "best_effort",
|
|
1322
|
+
date_time_output_format: "iso",
|
|
1323
|
+
use_client_time_zone: 1,
|
|
1324
|
+
output_format_json_quote_64bit_integers: 0
|
|
1325
|
+
}
|
|
1326
|
+
});
|
|
1327
|
+
const rows = await result.json();
|
|
1328
|
+
if (rows.data.length === 0) {
|
|
1329
|
+
return null;
|
|
1330
|
+
}
|
|
1331
|
+
const resource = rows.data[0];
|
|
1332
|
+
return {
|
|
1333
|
+
id: resource.id,
|
|
1334
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1335
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1336
|
+
createdAt: new Date(resource.createdAt),
|
|
1337
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1338
|
+
};
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
throw new MastraError(
|
|
1341
|
+
{
|
|
1342
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1343
|
+
domain: ErrorDomain.STORAGE,
|
|
1344
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1345
|
+
details: { resourceId }
|
|
1346
|
+
},
|
|
1347
|
+
error
|
|
1348
|
+
);
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
async saveResource({ resource }) {
|
|
1352
|
+
try {
|
|
1353
|
+
await this.client.insert({
|
|
1354
|
+
table: TABLE_RESOURCES,
|
|
1355
|
+
format: "JSONEachRow",
|
|
1356
|
+
values: [
|
|
1357
|
+
{
|
|
1358
|
+
id: resource.id,
|
|
1359
|
+
workingMemory: resource.workingMemory,
|
|
1360
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1361
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1362
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1363
|
+
}
|
|
1364
|
+
],
|
|
1365
|
+
clickhouse_settings: {
|
|
1366
|
+
date_time_input_format: "best_effort",
|
|
1367
|
+
use_client_time_zone: 1,
|
|
1368
|
+
output_format_json_quote_64bit_integers: 0
|
|
1369
|
+
}
|
|
1370
|
+
});
|
|
1371
|
+
return resource;
|
|
1372
|
+
} catch (error) {
|
|
1373
|
+
throw new MastraError(
|
|
1374
|
+
{
|
|
1375
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1376
|
+
domain: ErrorDomain.STORAGE,
|
|
1377
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1378
|
+
details: { resourceId: resource.id }
|
|
1379
|
+
},
|
|
1380
|
+
error
|
|
1381
|
+
);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
async updateResource({
|
|
1385
|
+
resourceId,
|
|
1386
|
+
workingMemory,
|
|
1387
|
+
metadata
|
|
1388
|
+
}) {
|
|
1389
|
+
try {
|
|
1390
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1391
|
+
if (!existingResource) {
|
|
1392
|
+
const newResource = {
|
|
1393
|
+
id: resourceId,
|
|
1394
|
+
workingMemory,
|
|
1395
|
+
metadata: metadata || {},
|
|
1396
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1397
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1398
|
+
};
|
|
1399
|
+
return this.saveResource({ resource: newResource });
|
|
1400
|
+
}
|
|
1401
|
+
const updatedResource = {
|
|
1402
|
+
...existingResource,
|
|
1403
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1404
|
+
metadata: {
|
|
1405
|
+
...existingResource.metadata,
|
|
1406
|
+
...metadata
|
|
1407
|
+
},
|
|
1408
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1409
|
+
};
|
|
1410
|
+
const updateQuery = `
|
|
1411
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1412
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1413
|
+
WHERE id = {resourceId:String}
|
|
1414
|
+
`;
|
|
1415
|
+
await this.client.command({
|
|
1416
|
+
query: updateQuery,
|
|
1417
|
+
query_params: {
|
|
1418
|
+
workingMemory: updatedResource.workingMemory,
|
|
1419
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1420
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1421
|
+
resourceId
|
|
1422
|
+
},
|
|
1423
|
+
clickhouse_settings: {
|
|
1424
|
+
date_time_input_format: "best_effort",
|
|
1425
|
+
use_client_time_zone: 1,
|
|
1426
|
+
output_format_json_quote_64bit_integers: 0
|
|
1427
|
+
}
|
|
1428
|
+
});
|
|
1429
|
+
await this.client.command({
|
|
1430
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1431
|
+
clickhouse_settings: {
|
|
1432
|
+
date_time_input_format: "best_effort",
|
|
1433
|
+
use_client_time_zone: 1,
|
|
1434
|
+
output_format_json_quote_64bit_integers: 0
|
|
1435
|
+
}
|
|
1436
|
+
});
|
|
1437
|
+
return updatedResource;
|
|
1438
|
+
} catch (error) {
|
|
1439
|
+
throw new MastraError(
|
|
1440
|
+
{
|
|
1441
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1442
|
+
domain: ErrorDomain.STORAGE,
|
|
1443
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1444
|
+
details: { resourceId }
|
|
1445
|
+
},
|
|
1446
|
+
error
|
|
1447
|
+
);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
};
|
|
1451
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1452
|
+
ttl;
|
|
1453
|
+
client;
|
|
1454
|
+
constructor({ client, ttl }) {
|
|
1455
|
+
super();
|
|
1456
|
+
this.ttl = ttl;
|
|
1457
|
+
this.client = client;
|
|
1458
|
+
}
|
|
1459
|
+
async hasColumn(table, column) {
|
|
1460
|
+
const result = await this.client.query({
|
|
1461
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1462
|
+
format: "JSONEachRow"
|
|
1463
|
+
});
|
|
1464
|
+
const columns = await result.json();
|
|
1465
|
+
return columns.some((c) => c.name === column);
|
|
1466
|
+
}
|
|
1467
|
+
getSqlType(type) {
|
|
1468
|
+
switch (type) {
|
|
1469
|
+
case "text":
|
|
1470
|
+
return "String";
|
|
1471
|
+
case "timestamp":
|
|
1472
|
+
return "DateTime64(3)";
|
|
1473
|
+
case "integer":
|
|
1474
|
+
case "bigint":
|
|
1475
|
+
return "Int64";
|
|
1476
|
+
case "jsonb":
|
|
1477
|
+
return "String";
|
|
1478
|
+
default:
|
|
1479
|
+
return super.getSqlType(type);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
async createTable({
|
|
1483
|
+
tableName,
|
|
1484
|
+
schema
|
|
1485
|
+
}) {
|
|
1486
|
+
try {
|
|
1487
|
+
const columns = Object.entries(schema).map(([name, def]) => {
|
|
1488
|
+
const constraints = [];
|
|
1489
|
+
if (!def.nullable) constraints.push("NOT NULL");
|
|
1490
|
+
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
1491
|
+
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
|
|
1492
|
+
}).join(",\n");
|
|
1493
|
+
const rowTtl = this.ttl?.[tableName]?.row;
|
|
1494
|
+
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
1495
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1496
|
+
${["id String"].concat(columns)}
|
|
1497
|
+
)
|
|
1498
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1499
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1500
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1501
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1502
|
+
SETTINGS index_granularity = 8192
|
|
1503
|
+
` : `
|
|
1504
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1505
|
+
${columns}
|
|
1506
|
+
)
|
|
1507
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1508
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1509
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1510
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1511
|
+
SETTINGS index_granularity = 8192
|
|
1512
|
+
`;
|
|
1513
|
+
await this.client.query({
|
|
1514
|
+
query: sql,
|
|
1515
|
+
clickhouse_settings: {
|
|
1516
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1517
|
+
date_time_input_format: "best_effort",
|
|
1518
|
+
date_time_output_format: "iso",
|
|
1519
|
+
use_client_time_zone: 1,
|
|
1520
|
+
output_format_json_quote_64bit_integers: 0
|
|
1521
|
+
}
|
|
1522
|
+
});
|
|
1523
|
+
} catch (error) {
|
|
1524
|
+
throw new MastraError(
|
|
1525
|
+
{
|
|
1526
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1527
|
+
domain: ErrorDomain.STORAGE,
|
|
1528
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1529
|
+
details: { tableName }
|
|
1530
|
+
},
|
|
1531
|
+
error
|
|
1532
|
+
);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
async alterTable({
|
|
1536
|
+
tableName,
|
|
1537
|
+
schema,
|
|
1538
|
+
ifNotExists
|
|
1539
|
+
}) {
|
|
1540
|
+
try {
|
|
1541
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1542
|
+
const result = await this.client.query({
|
|
1543
|
+
query: describeSql
|
|
1544
|
+
});
|
|
1545
|
+
const rows = await result.json();
|
|
1546
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1547
|
+
for (const columnName of ifNotExists) {
|
|
1548
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1549
|
+
const columnDef = schema[columnName];
|
|
1550
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1551
|
+
if (columnDef.nullable !== false) {
|
|
1552
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1553
|
+
}
|
|
1554
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1555
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1556
|
+
await this.client.query({
|
|
1557
|
+
query: alterSql
|
|
1558
|
+
});
|
|
1559
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
} catch (error) {
|
|
1563
|
+
throw new MastraError(
|
|
1564
|
+
{
|
|
1565
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1566
|
+
domain: ErrorDomain.STORAGE,
|
|
1567
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1568
|
+
details: { tableName }
|
|
1569
|
+
},
|
|
1570
|
+
error
|
|
1571
|
+
);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
async clearTable({ tableName }) {
|
|
1575
|
+
try {
|
|
1576
|
+
await this.client.query({
|
|
1577
|
+
query: `TRUNCATE TABLE ${tableName}`,
|
|
1578
|
+
clickhouse_settings: {
|
|
1579
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1580
|
+
date_time_input_format: "best_effort",
|
|
1581
|
+
date_time_output_format: "iso",
|
|
1582
|
+
use_client_time_zone: 1,
|
|
1583
|
+
output_format_json_quote_64bit_integers: 0
|
|
1584
|
+
}
|
|
1585
|
+
});
|
|
1586
|
+
} catch (error) {
|
|
1587
|
+
throw new MastraError(
|
|
1588
|
+
{
|
|
1589
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1590
|
+
domain: ErrorDomain.STORAGE,
|
|
1591
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1592
|
+
details: { tableName }
|
|
1593
|
+
},
|
|
1594
|
+
error
|
|
1595
|
+
);
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
async dropTable({ tableName }) {
|
|
1599
|
+
await this.client.query({
|
|
1600
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1601
|
+
});
|
|
1602
|
+
}
|
|
1603
|
+
async insert({ tableName, record }) {
|
|
1604
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1605
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
1606
|
+
try {
|
|
1607
|
+
const result = await this.client.insert({
|
|
1608
|
+
table: tableName,
|
|
1609
|
+
values: [
|
|
1610
|
+
{
|
|
1611
|
+
...record,
|
|
1612
|
+
createdAt,
|
|
1613
|
+
updatedAt
|
|
1614
|
+
}
|
|
1615
|
+
],
|
|
1616
|
+
format: "JSONEachRow",
|
|
1617
|
+
clickhouse_settings: {
|
|
1618
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1619
|
+
output_format_json_quote_64bit_integers: 0,
|
|
1620
|
+
date_time_input_format: "best_effort",
|
|
1621
|
+
use_client_time_zone: 1
|
|
1622
|
+
}
|
|
1623
|
+
});
|
|
1624
|
+
console.info("INSERT RESULT", result);
|
|
1625
|
+
} catch (error) {
|
|
1626
|
+
throw new MastraError(
|
|
1627
|
+
{
|
|
1628
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1629
|
+
domain: ErrorDomain.STORAGE,
|
|
1630
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1631
|
+
details: { tableName }
|
|
1632
|
+
},
|
|
1633
|
+
error
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
async batchInsert({ tableName, records }) {
|
|
1638
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1639
|
+
...Object.fromEntries(
|
|
1640
|
+
Object.entries(record).map(([key, value]) => [
|
|
1641
|
+
key,
|
|
1642
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1643
|
+
])
|
|
1644
|
+
)
|
|
1645
|
+
}));
|
|
1646
|
+
try {
|
|
1647
|
+
await this.client.insert({
|
|
1648
|
+
table: tableName,
|
|
1649
|
+
values: recordsToBeInserted,
|
|
1650
|
+
format: "JSONEachRow",
|
|
1651
|
+
clickhouse_settings: {
|
|
1652
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1653
|
+
date_time_input_format: "best_effort",
|
|
1654
|
+
use_client_time_zone: 1,
|
|
1655
|
+
output_format_json_quote_64bit_integers: 0
|
|
1656
|
+
}
|
|
1657
|
+
});
|
|
1658
|
+
} catch (error) {
|
|
1659
|
+
throw new MastraError(
|
|
1660
|
+
{
|
|
1661
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1662
|
+
domain: ErrorDomain.STORAGE,
|
|
1663
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1664
|
+
details: { tableName }
|
|
1665
|
+
},
|
|
1666
|
+
error
|
|
1667
|
+
);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
async load({ tableName, keys }) {
|
|
1671
|
+
try {
|
|
1672
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
1673
|
+
const keyEntries = Object.entries(keys);
|
|
1674
|
+
const conditions = keyEntries.map(
|
|
1675
|
+
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
1676
|
+
).join(" AND ");
|
|
1677
|
+
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
1678
|
+
return { ...acc, [`var_${key}`]: value };
|
|
1679
|
+
}, {});
|
|
1680
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1681
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1682
|
+
const result = await this.client.query({
|
|
1683
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
|
|
1684
|
+
query_params: values,
|
|
1685
|
+
clickhouse_settings: {
|
|
1686
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1687
|
+
date_time_input_format: "best_effort",
|
|
1688
|
+
date_time_output_format: "iso",
|
|
1689
|
+
use_client_time_zone: 1,
|
|
1690
|
+
output_format_json_quote_64bit_integers: 0
|
|
1691
|
+
}
|
|
1692
|
+
});
|
|
1693
|
+
if (!result) {
|
|
1694
|
+
return null;
|
|
1695
|
+
}
|
|
1696
|
+
const rows = await result.json();
|
|
1697
|
+
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
1698
|
+
const snapshot = rows.data[0];
|
|
1699
|
+
if (!snapshot) {
|
|
1700
|
+
return null;
|
|
1701
|
+
}
|
|
1702
|
+
if (typeof snapshot.snapshot === "string") {
|
|
1703
|
+
snapshot.snapshot = JSON.parse(snapshot.snapshot);
|
|
1704
|
+
}
|
|
1705
|
+
return transformRow(snapshot);
|
|
1706
|
+
}
|
|
1707
|
+
const data = transformRow(rows.data[0]);
|
|
1708
|
+
return data;
|
|
1709
|
+
} catch (error) {
|
|
1710
|
+
throw new MastraError(
|
|
1711
|
+
{
|
|
1712
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1713
|
+
domain: ErrorDomain.STORAGE,
|
|
1714
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1715
|
+
details: { tableName }
|
|
1716
|
+
},
|
|
1717
|
+
error
|
|
1718
|
+
);
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
};
|
|
1722
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1723
|
+
client;
|
|
1724
|
+
operations;
|
|
1725
|
+
constructor({ client, operations }) {
|
|
1726
|
+
super();
|
|
1727
|
+
this.client = client;
|
|
1728
|
+
this.operations = operations;
|
|
1729
|
+
}
|
|
1730
|
+
transformScoreRow(row) {
|
|
1731
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1732
|
+
const preprocessStepResult = safelyParseJSON(row.preprocessStepResult);
|
|
1733
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1734
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1735
|
+
const input = safelyParseJSON(row.input);
|
|
1736
|
+
const output = safelyParseJSON(row.output);
|
|
1737
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1738
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1739
|
+
const entity = safelyParseJSON(row.entity);
|
|
1740
|
+
return {
|
|
1741
|
+
...row,
|
|
1742
|
+
scorer,
|
|
1743
|
+
preprocessStepResult,
|
|
1744
|
+
analyzeStepResult,
|
|
1745
|
+
metadata,
|
|
1746
|
+
input,
|
|
1747
|
+
output,
|
|
1748
|
+
additionalContext,
|
|
1749
|
+
runtimeContext,
|
|
1750
|
+
entity,
|
|
1751
|
+
createdAt: new Date(row.createdAt),
|
|
1752
|
+
updatedAt: new Date(row.updatedAt)
|
|
1753
|
+
};
|
|
1754
|
+
}
|
|
1755
|
+
async getScoreById({ id }) {
|
|
1756
|
+
try {
|
|
1757
|
+
const result = await this.client.query({
|
|
1758
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1759
|
+
query_params: { var_id: id },
|
|
1760
|
+
format: "JSONEachRow",
|
|
1761
|
+
clickhouse_settings: {
|
|
1762
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1763
|
+
date_time_input_format: "best_effort",
|
|
1764
|
+
date_time_output_format: "iso",
|
|
1765
|
+
use_client_time_zone: 1,
|
|
1766
|
+
output_format_json_quote_64bit_integers: 0
|
|
1767
|
+
}
|
|
1768
|
+
});
|
|
1769
|
+
const resultJson = await result.json();
|
|
1770
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
1771
|
+
return null;
|
|
1772
|
+
}
|
|
1773
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1774
|
+
} catch (error) {
|
|
1775
|
+
throw new MastraError(
|
|
1776
|
+
{
|
|
1777
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1778
|
+
domain: ErrorDomain.STORAGE,
|
|
1779
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1780
|
+
details: { scoreId: id }
|
|
1781
|
+
},
|
|
1782
|
+
error
|
|
1783
|
+
);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
async saveScore(score) {
|
|
1787
|
+
let parsedScore;
|
|
1788
|
+
try {
|
|
1789
|
+
parsedScore = saveScorePayloadSchema.parse(score);
|
|
1790
|
+
} catch (error) {
|
|
1791
|
+
throw new MastraError(
|
|
1792
|
+
{
|
|
1793
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED_INVALID_SCORE_PAYLOAD",
|
|
1794
|
+
domain: ErrorDomain.STORAGE,
|
|
1795
|
+
category: ErrorCategory.USER,
|
|
1796
|
+
details: { scoreId: score.id }
|
|
1797
|
+
},
|
|
1798
|
+
error
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
try {
|
|
1802
|
+
const record = {
|
|
1803
|
+
...parsedScore
|
|
1804
|
+
};
|
|
1805
|
+
await this.client.insert({
|
|
1806
|
+
table: TABLE_SCORERS,
|
|
1807
|
+
values: [record],
|
|
1808
|
+
format: "JSONEachRow",
|
|
1809
|
+
clickhouse_settings: {
|
|
1810
|
+
date_time_input_format: "best_effort",
|
|
1811
|
+
use_client_time_zone: 1,
|
|
1812
|
+
output_format_json_quote_64bit_integers: 0
|
|
1813
|
+
}
|
|
1814
|
+
});
|
|
1815
|
+
return { score };
|
|
1816
|
+
} catch (error) {
|
|
1817
|
+
throw new MastraError(
|
|
1818
|
+
{
|
|
1819
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1820
|
+
domain: ErrorDomain.STORAGE,
|
|
1821
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1822
|
+
details: { scoreId: score.id }
|
|
1823
|
+
},
|
|
1824
|
+
error
|
|
1825
|
+
);
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
async getScoresByRunId({
|
|
1829
|
+
runId,
|
|
1830
|
+
pagination
|
|
1831
|
+
}) {
|
|
1832
|
+
try {
|
|
1833
|
+
const countResult = await this.client.query({
|
|
1834
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1835
|
+
query_params: { var_runId: runId },
|
|
1836
|
+
format: "JSONEachRow"
|
|
1837
|
+
});
|
|
1838
|
+
const countRows = await countResult.json();
|
|
1839
|
+
let total = 0;
|
|
1840
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1841
|
+
const countObj = countRows[0];
|
|
1842
|
+
total = Number(countObj.count);
|
|
1843
|
+
}
|
|
1844
|
+
if (!total) {
|
|
1845
|
+
return {
|
|
1846
|
+
pagination: {
|
|
1847
|
+
total: 0,
|
|
1848
|
+
page: pagination.page,
|
|
1849
|
+
perPage: pagination.perPage,
|
|
1850
|
+
hasMore: false
|
|
1851
|
+
},
|
|
1852
|
+
scores: []
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
const offset = pagination.page * pagination.perPage;
|
|
1856
|
+
const result = await this.client.query({
|
|
1857
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1858
|
+
query_params: {
|
|
1859
|
+
var_runId: runId,
|
|
1860
|
+
var_limit: pagination.perPage,
|
|
1861
|
+
var_offset: offset
|
|
1862
|
+
},
|
|
1863
|
+
format: "JSONEachRow",
|
|
1864
|
+
clickhouse_settings: {
|
|
1865
|
+
date_time_input_format: "best_effort",
|
|
1866
|
+
date_time_output_format: "iso",
|
|
1867
|
+
use_client_time_zone: 1,
|
|
1868
|
+
output_format_json_quote_64bit_integers: 0
|
|
1869
|
+
}
|
|
1870
|
+
});
|
|
1871
|
+
const rows = await result.json();
|
|
1872
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1873
|
+
return {
|
|
1874
|
+
pagination: {
|
|
1875
|
+
total,
|
|
1876
|
+
page: pagination.page,
|
|
1877
|
+
perPage: pagination.perPage,
|
|
1878
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1879
|
+
},
|
|
1880
|
+
scores
|
|
1881
|
+
};
|
|
1882
|
+
} catch (error) {
|
|
1883
|
+
throw new MastraError(
|
|
1884
|
+
{
|
|
1885
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1886
|
+
domain: ErrorDomain.STORAGE,
|
|
1887
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1888
|
+
details: { runId }
|
|
1889
|
+
},
|
|
1890
|
+
error
|
|
1891
|
+
);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
async getScoresByScorerId({
|
|
1895
|
+
scorerId,
|
|
1896
|
+
entityId,
|
|
1897
|
+
entityType,
|
|
1898
|
+
source,
|
|
1899
|
+
pagination
|
|
1900
|
+
}) {
|
|
1901
|
+
let whereClause = `scorerId = {var_scorerId:String}`;
|
|
1902
|
+
if (entityId) {
|
|
1903
|
+
whereClause += ` AND entityId = {var_entityId:String}`;
|
|
1904
|
+
}
|
|
1905
|
+
if (entityType) {
|
|
1906
|
+
whereClause += ` AND entityType = {var_entityType:String}`;
|
|
1907
|
+
}
|
|
1908
|
+
if (source) {
|
|
1909
|
+
whereClause += ` AND source = {var_source:String}`;
|
|
1910
|
+
}
|
|
1911
|
+
try {
|
|
1912
|
+
const countResult = await this.client.query({
|
|
1913
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE ${whereClause}`,
|
|
1914
|
+
query_params: {
|
|
1915
|
+
var_scorerId: scorerId,
|
|
1916
|
+
var_entityId: entityId,
|
|
1917
|
+
var_entityType: entityType,
|
|
1918
|
+
var_source: source
|
|
1919
|
+
},
|
|
1920
|
+
format: "JSONEachRow"
|
|
1921
|
+
});
|
|
1922
|
+
const countRows = await countResult.json();
|
|
1923
|
+
let total = 0;
|
|
1924
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1925
|
+
const countObj = countRows[0];
|
|
1926
|
+
total = Number(countObj.count);
|
|
1927
|
+
}
|
|
1928
|
+
if (!total) {
|
|
1929
|
+
return {
|
|
1930
|
+
pagination: {
|
|
1931
|
+
total: 0,
|
|
1932
|
+
page: pagination.page,
|
|
1933
|
+
perPage: pagination.perPage,
|
|
1934
|
+
hasMore: false
|
|
1935
|
+
},
|
|
1936
|
+
scores: []
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
const offset = pagination.page * pagination.perPage;
|
|
1940
|
+
const result = await this.client.query({
|
|
1941
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1942
|
+
query_params: {
|
|
1943
|
+
var_scorerId: scorerId,
|
|
1944
|
+
var_limit: pagination.perPage,
|
|
1945
|
+
var_offset: offset,
|
|
1946
|
+
var_entityId: entityId,
|
|
1947
|
+
var_entityType: entityType,
|
|
1948
|
+
var_source: source
|
|
1949
|
+
},
|
|
1950
|
+
format: "JSONEachRow",
|
|
1951
|
+
clickhouse_settings: {
|
|
1952
|
+
date_time_input_format: "best_effort",
|
|
1953
|
+
date_time_output_format: "iso",
|
|
1954
|
+
use_client_time_zone: 1,
|
|
1955
|
+
output_format_json_quote_64bit_integers: 0
|
|
1956
|
+
}
|
|
1957
|
+
});
|
|
1958
|
+
const rows = await result.json();
|
|
1959
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1960
|
+
return {
|
|
1961
|
+
pagination: {
|
|
1962
|
+
total,
|
|
1963
|
+
page: pagination.page,
|
|
1964
|
+
perPage: pagination.perPage,
|
|
1965
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1966
|
+
},
|
|
1967
|
+
scores
|
|
1968
|
+
};
|
|
1969
|
+
} catch (error) {
|
|
1970
|
+
throw new MastraError(
|
|
1971
|
+
{
|
|
1972
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1973
|
+
domain: ErrorDomain.STORAGE,
|
|
1974
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1975
|
+
details: { scorerId }
|
|
1976
|
+
},
|
|
1977
|
+
error
|
|
1978
|
+
);
|
|
1979
|
+
}
|
|
1980
|
+
}
|
|
1981
|
+
async getScoresByEntityId({
|
|
1982
|
+
entityId,
|
|
1983
|
+
entityType,
|
|
1984
|
+
pagination
|
|
1985
|
+
}) {
|
|
1986
|
+
try {
|
|
1987
|
+
const countResult = await this.client.query({
|
|
1988
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1989
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1990
|
+
format: "JSONEachRow"
|
|
1991
|
+
});
|
|
1992
|
+
const countRows = await countResult.json();
|
|
1993
|
+
let total = 0;
|
|
1994
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1995
|
+
const countObj = countRows[0];
|
|
1996
|
+
total = Number(countObj.count);
|
|
1997
|
+
}
|
|
1998
|
+
if (!total) {
|
|
1999
|
+
return {
|
|
2000
|
+
pagination: {
|
|
2001
|
+
total: 0,
|
|
2002
|
+
page: pagination.page,
|
|
2003
|
+
perPage: pagination.perPage,
|
|
2004
|
+
hasMore: false
|
|
2005
|
+
},
|
|
2006
|
+
scores: []
|
|
2007
|
+
};
|
|
2008
|
+
}
|
|
2009
|
+
const offset = pagination.page * pagination.perPage;
|
|
2010
|
+
const result = await this.client.query({
|
|
2011
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
2012
|
+
query_params: {
|
|
2013
|
+
var_entityId: entityId,
|
|
2014
|
+
var_entityType: entityType,
|
|
2015
|
+
var_limit: pagination.perPage,
|
|
2016
|
+
var_offset: offset
|
|
2017
|
+
},
|
|
2018
|
+
format: "JSONEachRow",
|
|
2019
|
+
clickhouse_settings: {
|
|
2020
|
+
date_time_input_format: "best_effort",
|
|
2021
|
+
date_time_output_format: "iso",
|
|
2022
|
+
use_client_time_zone: 1,
|
|
2023
|
+
output_format_json_quote_64bit_integers: 0
|
|
2024
|
+
}
|
|
2025
|
+
});
|
|
2026
|
+
const rows = await result.json();
|
|
2027
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2028
|
+
return {
|
|
2029
|
+
pagination: {
|
|
2030
|
+
total,
|
|
2031
|
+
page: pagination.page,
|
|
2032
|
+
perPage: pagination.perPage,
|
|
2033
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
2034
|
+
},
|
|
2035
|
+
scores
|
|
2036
|
+
};
|
|
2037
|
+
} catch (error) {
|
|
2038
|
+
throw new MastraError(
|
|
2039
|
+
{
|
|
2040
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
2041
|
+
domain: ErrorDomain.STORAGE,
|
|
2042
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2043
|
+
details: { entityId, entityType }
|
|
2044
|
+
},
|
|
2045
|
+
error
|
|
2046
|
+
);
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
async getScoresBySpan({
|
|
2050
|
+
traceId,
|
|
2051
|
+
spanId,
|
|
2052
|
+
pagination
|
|
2053
|
+
}) {
|
|
2054
|
+
try {
|
|
2055
|
+
const countResult = await this.client.query({
|
|
2056
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String}`,
|
|
2057
|
+
query_params: {
|
|
2058
|
+
var_traceId: traceId,
|
|
2059
|
+
var_spanId: spanId
|
|
2060
|
+
},
|
|
2061
|
+
format: "JSONEachRow"
|
|
2062
|
+
});
|
|
2063
|
+
const countRows = await countResult.json();
|
|
2064
|
+
let total = 0;
|
|
2065
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
2066
|
+
const countObj = countRows[0];
|
|
2067
|
+
total = Number(countObj.count);
|
|
2068
|
+
}
|
|
2069
|
+
if (!total) {
|
|
2070
|
+
return {
|
|
2071
|
+
pagination: {
|
|
2072
|
+
total: 0,
|
|
2073
|
+
page: pagination.page,
|
|
2074
|
+
perPage: pagination.perPage,
|
|
2075
|
+
hasMore: false
|
|
2076
|
+
},
|
|
2077
|
+
scores: []
|
|
2078
|
+
};
|
|
2079
|
+
}
|
|
2080
|
+
const limit = pagination.perPage + 1;
|
|
2081
|
+
const offset = pagination.page * pagination.perPage;
|
|
2082
|
+
const result = await this.client.query({
|
|
2083
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
2084
|
+
query_params: {
|
|
2085
|
+
var_traceId: traceId,
|
|
2086
|
+
var_spanId: spanId,
|
|
2087
|
+
var_limit: limit,
|
|
2088
|
+
var_offset: offset
|
|
2089
|
+
},
|
|
2090
|
+
format: "JSONEachRow",
|
|
2091
|
+
clickhouse_settings: {
|
|
2092
|
+
date_time_input_format: "best_effort",
|
|
2093
|
+
date_time_output_format: "iso",
|
|
2094
|
+
use_client_time_zone: 1,
|
|
2095
|
+
output_format_json_quote_64bit_integers: 0
|
|
2096
|
+
}
|
|
2097
|
+
});
|
|
2098
|
+
const rows = await result.json();
|
|
2099
|
+
const transformedRows = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
2100
|
+
const hasMore = transformedRows.length > pagination.perPage;
|
|
2101
|
+
const scores = hasMore ? transformedRows.slice(0, pagination.perPage) : transformedRows;
|
|
2102
|
+
return {
|
|
2103
|
+
pagination: {
|
|
2104
|
+
total,
|
|
2105
|
+
page: pagination.page,
|
|
2106
|
+
perPage: pagination.perPage,
|
|
2107
|
+
hasMore
|
|
2108
|
+
},
|
|
2109
|
+
scores
|
|
2110
|
+
};
|
|
2111
|
+
} catch (error) {
|
|
2112
|
+
throw new MastraError(
|
|
2113
|
+
{
|
|
2114
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
|
|
2115
|
+
domain: ErrorDomain.STORAGE,
|
|
2116
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2117
|
+
details: { traceId, spanId }
|
|
2118
|
+
},
|
|
2119
|
+
error
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
};
|
|
2124
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
2125
|
+
client;
|
|
2126
|
+
operations;
|
|
2127
|
+
constructor({ client, operations }) {
|
|
2128
|
+
super();
|
|
2129
|
+
this.client = client;
|
|
2130
|
+
this.operations = operations;
|
|
2131
|
+
}
|
|
2132
|
+
async getTracesPaginated(args) {
|
|
2133
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
2134
|
+
const fromDate = dateRange?.start;
|
|
2135
|
+
const toDate = dateRange?.end;
|
|
2136
|
+
const currentOffset = page * perPage;
|
|
2137
|
+
const queryArgs = {};
|
|
2138
|
+
const conditions = [];
|
|
2139
|
+
if (name) {
|
|
2140
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2141
|
+
queryArgs.var_name = name;
|
|
2142
|
+
}
|
|
2143
|
+
if (scope) {
|
|
2144
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2145
|
+
queryArgs.var_scope = scope;
|
|
2146
|
+
}
|
|
2147
|
+
if (attributes) {
|
|
2148
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2149
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2150
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
2151
|
+
});
|
|
2152
|
+
}
|
|
2153
|
+
if (filters) {
|
|
2154
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2155
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2156
|
+
queryArgs[`var_col_${key}`] = value;
|
|
2157
|
+
});
|
|
2158
|
+
}
|
|
2159
|
+
if (fromDate) {
|
|
2160
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
2161
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
2162
|
+
}
|
|
2163
|
+
if (toDate) {
|
|
2164
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
2165
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
2166
|
+
}
|
|
2167
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2168
|
+
try {
|
|
2169
|
+
const countResult = await this.client.query({
|
|
2170
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
2171
|
+
query_params: queryArgs,
|
|
2172
|
+
clickhouse_settings: {
|
|
2173
|
+
date_time_input_format: "best_effort",
|
|
2174
|
+
date_time_output_format: "iso",
|
|
2175
|
+
use_client_time_zone: 1,
|
|
2176
|
+
output_format_json_quote_64bit_integers: 0
|
|
2177
|
+
}
|
|
2178
|
+
});
|
|
2179
|
+
const countData = await countResult.json();
|
|
2180
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2181
|
+
if (total === 0) {
|
|
2182
|
+
return {
|
|
2183
|
+
traces: [],
|
|
2184
|
+
total: 0,
|
|
2185
|
+
page,
|
|
2186
|
+
perPage,
|
|
2187
|
+
hasMore: false
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
const result = await this.client.query({
|
|
2191
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2192
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
2193
|
+
clickhouse_settings: {
|
|
2194
|
+
date_time_input_format: "best_effort",
|
|
2195
|
+
date_time_output_format: "iso",
|
|
2196
|
+
use_client_time_zone: 1,
|
|
2197
|
+
output_format_json_quote_64bit_integers: 0
|
|
2198
|
+
}
|
|
2199
|
+
});
|
|
2200
|
+
if (!result) {
|
|
2201
|
+
return {
|
|
2202
|
+
traces: [],
|
|
2203
|
+
total,
|
|
2204
|
+
page,
|
|
2205
|
+
perPage,
|
|
2206
|
+
hasMore: false
|
|
2207
|
+
};
|
|
2208
|
+
}
|
|
2209
|
+
const resp = await result.json();
|
|
2210
|
+
const rows = resp.data;
|
|
2211
|
+
const traces = rows.map((row) => ({
|
|
2212
|
+
id: row.id,
|
|
2213
|
+
parentSpanId: row.parentSpanId,
|
|
2214
|
+
traceId: row.traceId,
|
|
2215
|
+
name: row.name,
|
|
2216
|
+
scope: row.scope,
|
|
2217
|
+
kind: row.kind,
|
|
2218
|
+
status: safelyParseJSON(row.status),
|
|
2219
|
+
events: safelyParseJSON(row.events),
|
|
2220
|
+
links: safelyParseJSON(row.links),
|
|
2221
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2222
|
+
startTime: row.startTime,
|
|
2223
|
+
endTime: row.endTime,
|
|
2224
|
+
other: safelyParseJSON(row.other),
|
|
2225
|
+
createdAt: row.createdAt
|
|
2226
|
+
}));
|
|
2227
|
+
return {
|
|
2228
|
+
traces,
|
|
2229
|
+
total,
|
|
2230
|
+
page,
|
|
2231
|
+
perPage,
|
|
2232
|
+
hasMore: currentOffset + traces.length < total
|
|
2233
|
+
};
|
|
2234
|
+
} catch (error) {
|
|
2235
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2236
|
+
return {
|
|
2237
|
+
traces: [],
|
|
2238
|
+
total: 0,
|
|
2239
|
+
page,
|
|
2240
|
+
perPage,
|
|
2241
|
+
hasMore: false
|
|
2242
|
+
};
|
|
2243
|
+
}
|
|
2244
|
+
throw new MastraError(
|
|
2245
|
+
{
|
|
2246
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2247
|
+
domain: ErrorDomain.STORAGE,
|
|
2248
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2249
|
+
details: {
|
|
2250
|
+
name: name ?? null,
|
|
2251
|
+
scope: scope ?? null,
|
|
2252
|
+
page,
|
|
2253
|
+
perPage,
|
|
2254
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2255
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2256
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2257
|
+
}
|
|
2258
|
+
},
|
|
2259
|
+
error
|
|
2260
|
+
);
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
async getTraces({
|
|
2264
|
+
name,
|
|
2265
|
+
scope,
|
|
2266
|
+
page,
|
|
2267
|
+
perPage,
|
|
2268
|
+
attributes,
|
|
2269
|
+
filters,
|
|
2270
|
+
fromDate,
|
|
2271
|
+
toDate
|
|
2272
|
+
}) {
|
|
2273
|
+
const limit = perPage;
|
|
2274
|
+
const offset = page * perPage;
|
|
2275
|
+
const args = {};
|
|
2276
|
+
const conditions = [];
|
|
2277
|
+
if (name) {
|
|
2278
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2279
|
+
args.var_name = name;
|
|
2280
|
+
}
|
|
2281
|
+
if (scope) {
|
|
2282
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2283
|
+
args.var_scope = scope;
|
|
2284
|
+
}
|
|
2285
|
+
if (attributes) {
|
|
2286
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2287
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2288
|
+
args[`var_attr_${key}`] = value;
|
|
2289
|
+
});
|
|
2290
|
+
}
|
|
2291
|
+
if (filters) {
|
|
2292
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2293
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2294
|
+
args[`var_col_${key}`] = value;
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
if (fromDate) {
|
|
2298
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2299
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2300
|
+
}
|
|
2301
|
+
if (toDate) {
|
|
2302
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2303
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2304
|
+
}
|
|
2305
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2306
|
+
try {
|
|
2307
|
+
const result = await this.client.query({
|
|
2308
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2309
|
+
query_params: args,
|
|
2310
|
+
clickhouse_settings: {
|
|
2311
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
2312
|
+
date_time_input_format: "best_effort",
|
|
2313
|
+
date_time_output_format: "iso",
|
|
2314
|
+
use_client_time_zone: 1,
|
|
2315
|
+
output_format_json_quote_64bit_integers: 0
|
|
2316
|
+
}
|
|
2317
|
+
});
|
|
2318
|
+
if (!result) {
|
|
2319
|
+
return [];
|
|
2320
|
+
}
|
|
2321
|
+
const resp = await result.json();
|
|
2322
|
+
const rows = resp.data;
|
|
2323
|
+
return rows.map((row) => ({
|
|
2324
|
+
id: row.id,
|
|
2325
|
+
parentSpanId: row.parentSpanId,
|
|
2326
|
+
traceId: row.traceId,
|
|
2327
|
+
name: row.name,
|
|
2328
|
+
scope: row.scope,
|
|
2329
|
+
kind: row.kind,
|
|
2330
|
+
status: safelyParseJSON(row.status),
|
|
2331
|
+
events: safelyParseJSON(row.events),
|
|
2332
|
+
links: safelyParseJSON(row.links),
|
|
2333
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2334
|
+
startTime: row.startTime,
|
|
2335
|
+
endTime: row.endTime,
|
|
2336
|
+
other: safelyParseJSON(row.other),
|
|
2337
|
+
createdAt: row.createdAt
|
|
2338
|
+
}));
|
|
933
2339
|
} catch (error) {
|
|
2340
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2341
|
+
return [];
|
|
2342
|
+
}
|
|
934
2343
|
throw new MastraError(
|
|
935
2344
|
{
|
|
936
|
-
id: "
|
|
2345
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
937
2346
|
domain: ErrorDomain.STORAGE,
|
|
938
|
-
category: ErrorCategory.THIRD_PARTY
|
|
2347
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2348
|
+
details: {
|
|
2349
|
+
name: name ?? null,
|
|
2350
|
+
scope: scope ?? null,
|
|
2351
|
+
page,
|
|
2352
|
+
perPage,
|
|
2353
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2354
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2355
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2356
|
+
toDate: toDate?.toISOString() ?? null
|
|
2357
|
+
}
|
|
939
2358
|
},
|
|
940
2359
|
error
|
|
941
2360
|
);
|
|
942
2361
|
}
|
|
943
2362
|
}
|
|
2363
|
+
async batchTraceInsert(args) {
|
|
2364
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2365
|
+
}
|
|
2366
|
+
};
|
|
2367
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2368
|
+
client;
|
|
2369
|
+
operations;
|
|
2370
|
+
constructor({ client, operations }) {
|
|
2371
|
+
super();
|
|
2372
|
+
this.operations = operations;
|
|
2373
|
+
this.client = client;
|
|
2374
|
+
}
|
|
2375
|
+
updateWorkflowResults({
|
|
2376
|
+
// workflowName,
|
|
2377
|
+
// runId,
|
|
2378
|
+
// stepId,
|
|
2379
|
+
// result,
|
|
2380
|
+
// runtimeContext,
|
|
2381
|
+
}) {
|
|
2382
|
+
throw new Error("Method not implemented.");
|
|
2383
|
+
}
|
|
2384
|
+
updateWorkflowState({
|
|
2385
|
+
// workflowName,
|
|
2386
|
+
// runId,
|
|
2387
|
+
// opts,
|
|
2388
|
+
}) {
|
|
2389
|
+
throw new Error("Method not implemented.");
|
|
2390
|
+
}
|
|
944
2391
|
async persistWorkflowSnapshot({
|
|
945
2392
|
workflowName,
|
|
946
2393
|
runId,
|
|
2394
|
+
resourceId,
|
|
947
2395
|
snapshot
|
|
948
2396
|
}) {
|
|
949
2397
|
try {
|
|
950
|
-
const currentSnapshot = await this.load({
|
|
2398
|
+
const currentSnapshot = await this.operations.load({
|
|
951
2399
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
952
2400
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
953
2401
|
});
|
|
954
2402
|
const now = /* @__PURE__ */ new Date();
|
|
955
2403
|
const persisting = currentSnapshot ? {
|
|
956
2404
|
...currentSnapshot,
|
|
2405
|
+
resourceId,
|
|
957
2406
|
snapshot: JSON.stringify(snapshot),
|
|
958
2407
|
updatedAt: now.toISOString()
|
|
959
2408
|
} : {
|
|
960
2409
|
workflow_name: workflowName,
|
|
961
2410
|
run_id: runId,
|
|
2411
|
+
resourceId,
|
|
962
2412
|
snapshot: JSON.stringify(snapshot),
|
|
963
2413
|
createdAt: now.toISOString(),
|
|
964
2414
|
updatedAt: now.toISOString()
|
|
965
2415
|
};
|
|
966
|
-
await this.
|
|
2416
|
+
await this.client.insert({
|
|
967
2417
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
968
2418
|
format: "JSONEachRow",
|
|
969
2419
|
values: [persisting],
|
|
@@ -991,7 +2441,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
991
2441
|
runId
|
|
992
2442
|
}) {
|
|
993
2443
|
try {
|
|
994
|
-
const result = await this.load({
|
|
2444
|
+
const result = await this.operations.load({
|
|
995
2445
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
996
2446
|
keys: {
|
|
997
2447
|
workflow_name: workflowName,
|
|
@@ -1048,7 +2498,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1048
2498
|
values.var_workflow_name = workflowName;
|
|
1049
2499
|
}
|
|
1050
2500
|
if (resourceId) {
|
|
1051
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2501
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
1052
2502
|
if (hasResourceId) {
|
|
1053
2503
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
1054
2504
|
values.var_resourceId = resourceId;
|
|
@@ -1069,7 +2519,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1069
2519
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
1070
2520
|
let total = 0;
|
|
1071
2521
|
if (limit !== void 0 && offset !== void 0) {
|
|
1072
|
-
const countResult = await this.
|
|
2522
|
+
const countResult = await this.client.query({
|
|
1073
2523
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
1074
2524
|
query_params: values,
|
|
1075
2525
|
format: "JSONEachRow"
|
|
@@ -1077,21 +2527,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1077
2527
|
const countRows = await countResult.json();
|
|
1078
2528
|
total = Number(countRows[0]?.count ?? 0);
|
|
1079
2529
|
}
|
|
1080
|
-
const result = await this.
|
|
2530
|
+
const result = await this.client.query({
|
|
1081
2531
|
query: `
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
2532
|
+
SELECT
|
|
2533
|
+
workflow_name,
|
|
2534
|
+
run_id,
|
|
2535
|
+
snapshot,
|
|
2536
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2537
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2538
|
+
resourceId
|
|
2539
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2540
|
+
${whereClause}
|
|
2541
|
+
ORDER BY createdAt DESC
|
|
2542
|
+
${limitClause}
|
|
2543
|
+
${offsetClause}
|
|
2544
|
+
`,
|
|
1095
2545
|
query_params: values,
|
|
1096
2546
|
format: "JSONEachRow"
|
|
1097
2547
|
});
|
|
@@ -1129,18 +2579,19 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1129
2579
|
values.var_workflow_name = workflowName;
|
|
1130
2580
|
}
|
|
1131
2581
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1132
|
-
const result = await this.
|
|
2582
|
+
const result = await this.client.query({
|
|
1133
2583
|
query: `
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
2584
|
+
SELECT
|
|
2585
|
+
workflow_name,
|
|
2586
|
+
run_id,
|
|
2587
|
+
snapshot,
|
|
2588
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2589
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2590
|
+
resourceId
|
|
2591
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2592
|
+
${whereClause}
|
|
2593
|
+
ORDER BY createdAt DESC LIMIT 1
|
|
2594
|
+
`,
|
|
1144
2595
|
query_params: values,
|
|
1145
2596
|
format: "JSONEachRow"
|
|
1146
2597
|
});
|
|
@@ -1161,45 +2612,274 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
1161
2612
|
);
|
|
1162
2613
|
}
|
|
1163
2614
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
2615
|
+
};
|
|
2616
|
+
|
|
2617
|
+
// src/storage/index.ts
|
|
2618
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2619
|
+
db;
|
|
2620
|
+
ttl = {};
|
|
2621
|
+
stores;
|
|
2622
|
+
constructor(config) {
|
|
2623
|
+
super({ name: "ClickhouseStore" });
|
|
2624
|
+
this.db = createClient({
|
|
2625
|
+
url: config.url,
|
|
2626
|
+
username: config.username,
|
|
2627
|
+
password: config.password,
|
|
2628
|
+
clickhouse_settings: {
|
|
2629
|
+
date_time_input_format: "best_effort",
|
|
2630
|
+
date_time_output_format: "iso",
|
|
2631
|
+
// This is crucial
|
|
2632
|
+
use_client_time_zone: 1,
|
|
2633
|
+
output_format_json_quote_64bit_integers: 0
|
|
2634
|
+
}
|
|
1168
2635
|
});
|
|
1169
|
-
|
|
1170
|
-
|
|
2636
|
+
this.ttl = config.ttl;
|
|
2637
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2638
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2639
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2640
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2641
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2642
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2643
|
+
this.stores = {
|
|
2644
|
+
operations,
|
|
2645
|
+
workflows,
|
|
2646
|
+
scores,
|
|
2647
|
+
legacyEvals,
|
|
2648
|
+
traces,
|
|
2649
|
+
memory
|
|
2650
|
+
};
|
|
1171
2651
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
2652
|
+
get supports() {
|
|
2653
|
+
return {
|
|
2654
|
+
selectByIncludeResourceScope: true,
|
|
2655
|
+
resourceWorkingMemory: true,
|
|
2656
|
+
hasColumn: true,
|
|
2657
|
+
createTable: true,
|
|
2658
|
+
deleteMessages: false,
|
|
2659
|
+
getScoresBySpan: true
|
|
2660
|
+
};
|
|
1179
2661
|
}
|
|
1180
|
-
async
|
|
1181
|
-
|
|
1182
|
-
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
1183
|
-
domain: ErrorDomain.STORAGE,
|
|
1184
|
-
category: ErrorCategory.USER,
|
|
1185
|
-
text: "Method not implemented."
|
|
1186
|
-
});
|
|
2662
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2663
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
1187
2664
|
}
|
|
1188
|
-
async
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
2665
|
+
async getEvals(options) {
|
|
2666
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2667
|
+
}
|
|
2668
|
+
async batchInsert({ tableName, records }) {
|
|
2669
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2670
|
+
}
|
|
2671
|
+
async optimizeTable({ tableName }) {
|
|
2672
|
+
try {
|
|
2673
|
+
await this.db.command({
|
|
2674
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2675
|
+
});
|
|
2676
|
+
} catch (error) {
|
|
2677
|
+
throw new MastraError(
|
|
2678
|
+
{
|
|
2679
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2680
|
+
domain: ErrorDomain.STORAGE,
|
|
2681
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2682
|
+
details: { tableName }
|
|
2683
|
+
},
|
|
2684
|
+
error
|
|
2685
|
+
);
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
async materializeTtl({ tableName }) {
|
|
2689
|
+
try {
|
|
2690
|
+
await this.db.command({
|
|
2691
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2692
|
+
});
|
|
2693
|
+
} catch (error) {
|
|
2694
|
+
throw new MastraError(
|
|
2695
|
+
{
|
|
2696
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2697
|
+
domain: ErrorDomain.STORAGE,
|
|
2698
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2699
|
+
details: { tableName }
|
|
2700
|
+
},
|
|
2701
|
+
error
|
|
2702
|
+
);
|
|
2703
|
+
}
|
|
2704
|
+
}
|
|
2705
|
+
async createTable({
|
|
2706
|
+
tableName,
|
|
2707
|
+
schema
|
|
2708
|
+
}) {
|
|
2709
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2710
|
+
}
|
|
2711
|
+
async dropTable({ tableName }) {
|
|
2712
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2713
|
+
}
|
|
2714
|
+
async alterTable({
|
|
2715
|
+
tableName,
|
|
2716
|
+
schema,
|
|
2717
|
+
ifNotExists
|
|
2718
|
+
}) {
|
|
2719
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2720
|
+
}
|
|
2721
|
+
async clearTable({ tableName }) {
|
|
2722
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2723
|
+
}
|
|
2724
|
+
async insert({ tableName, record }) {
|
|
2725
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2726
|
+
}
|
|
2727
|
+
async load({ tableName, keys }) {
|
|
2728
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2729
|
+
}
|
|
2730
|
+
async updateWorkflowResults({
|
|
2731
|
+
workflowName,
|
|
2732
|
+
runId,
|
|
2733
|
+
stepId,
|
|
2734
|
+
result,
|
|
2735
|
+
runtimeContext
|
|
2736
|
+
}) {
|
|
2737
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2738
|
+
}
|
|
2739
|
+
async updateWorkflowState({
|
|
2740
|
+
workflowName,
|
|
2741
|
+
runId,
|
|
2742
|
+
opts
|
|
2743
|
+
}) {
|
|
2744
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2745
|
+
}
|
|
2746
|
+
async persistWorkflowSnapshot({
|
|
2747
|
+
workflowName,
|
|
2748
|
+
runId,
|
|
2749
|
+
resourceId,
|
|
2750
|
+
snapshot
|
|
2751
|
+
}) {
|
|
2752
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2753
|
+
}
|
|
2754
|
+
async loadWorkflowSnapshot({
|
|
2755
|
+
workflowName,
|
|
2756
|
+
runId
|
|
2757
|
+
}) {
|
|
2758
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2759
|
+
}
|
|
2760
|
+
async getWorkflowRuns({
|
|
2761
|
+
workflowName,
|
|
2762
|
+
fromDate,
|
|
2763
|
+
toDate,
|
|
2764
|
+
limit,
|
|
2765
|
+
offset,
|
|
2766
|
+
resourceId
|
|
2767
|
+
} = {}) {
|
|
2768
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2769
|
+
}
|
|
2770
|
+
async getWorkflowRunById({
|
|
2771
|
+
runId,
|
|
2772
|
+
workflowName
|
|
2773
|
+
}) {
|
|
2774
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2775
|
+
}
|
|
2776
|
+
async getTraces(args) {
|
|
2777
|
+
return this.stores.traces.getTraces(args);
|
|
2778
|
+
}
|
|
2779
|
+
async getTracesPaginated(args) {
|
|
2780
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2781
|
+
}
|
|
2782
|
+
async batchTraceInsert(args) {
|
|
2783
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2784
|
+
}
|
|
2785
|
+
async getThreadById({ threadId }) {
|
|
2786
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2787
|
+
}
|
|
2788
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2789
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2790
|
+
}
|
|
2791
|
+
async saveThread({ thread }) {
|
|
2792
|
+
return this.stores.memory.saveThread({ thread });
|
|
2793
|
+
}
|
|
2794
|
+
async updateThread({
|
|
2795
|
+
id,
|
|
2796
|
+
title,
|
|
2797
|
+
metadata
|
|
2798
|
+
}) {
|
|
2799
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2800
|
+
}
|
|
2801
|
+
async deleteThread({ threadId }) {
|
|
2802
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2803
|
+
}
|
|
2804
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2805
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2806
|
+
}
|
|
2807
|
+
async getMessages({
|
|
2808
|
+
threadId,
|
|
2809
|
+
resourceId,
|
|
2810
|
+
selectBy,
|
|
2811
|
+
format
|
|
2812
|
+
}) {
|
|
2813
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2814
|
+
}
|
|
2815
|
+
async getMessagesById({
|
|
2816
|
+
messageIds,
|
|
2817
|
+
format
|
|
2818
|
+
}) {
|
|
2819
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2820
|
+
}
|
|
2821
|
+
async saveMessages(args) {
|
|
2822
|
+
return this.stores.memory.saveMessages(args);
|
|
2823
|
+
}
|
|
2824
|
+
async getMessagesPaginated(args) {
|
|
2825
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2826
|
+
}
|
|
2827
|
+
async updateMessages(args) {
|
|
2828
|
+
return this.stores.memory.updateMessages(args);
|
|
2829
|
+
}
|
|
2830
|
+
async getResourceById({ resourceId }) {
|
|
2831
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2832
|
+
}
|
|
2833
|
+
async saveResource({ resource }) {
|
|
2834
|
+
return this.stores.memory.saveResource({ resource });
|
|
2835
|
+
}
|
|
2836
|
+
async updateResource({
|
|
2837
|
+
resourceId,
|
|
2838
|
+
workingMemory,
|
|
2839
|
+
metadata
|
|
2840
|
+
}) {
|
|
2841
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2842
|
+
}
|
|
2843
|
+
async getScoreById({ id }) {
|
|
2844
|
+
return this.stores.scores.getScoreById({ id });
|
|
2845
|
+
}
|
|
2846
|
+
async saveScore(_score) {
|
|
2847
|
+
return this.stores.scores.saveScore(_score);
|
|
2848
|
+
}
|
|
2849
|
+
async getScoresByRunId({
|
|
2850
|
+
runId,
|
|
2851
|
+
pagination
|
|
2852
|
+
}) {
|
|
2853
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2854
|
+
}
|
|
2855
|
+
async getScoresByEntityId({
|
|
2856
|
+
entityId,
|
|
2857
|
+
entityType,
|
|
2858
|
+
pagination
|
|
2859
|
+
}) {
|
|
2860
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2861
|
+
}
|
|
2862
|
+
async getScoresByScorerId({
|
|
2863
|
+
scorerId,
|
|
2864
|
+
pagination,
|
|
2865
|
+
entityId,
|
|
2866
|
+
entityType,
|
|
2867
|
+
source
|
|
2868
|
+
}) {
|
|
2869
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
|
|
2870
|
+
}
|
|
2871
|
+
async getScoresBySpan({
|
|
2872
|
+
traceId,
|
|
2873
|
+
spanId,
|
|
2874
|
+
pagination
|
|
2875
|
+
}) {
|
|
2876
|
+
return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
|
|
1195
2877
|
}
|
|
1196
2878
|
async close() {
|
|
1197
2879
|
await this.db.close();
|
|
1198
2880
|
}
|
|
1199
|
-
async updateMessages(_args) {
|
|
1200
|
-
this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
|
|
1201
|
-
throw new Error("Method not implemented");
|
|
1202
|
-
}
|
|
1203
2881
|
};
|
|
1204
2882
|
|
|
1205
2883
|
export { COLUMN_TYPES, ClickhouseStore, TABLE_ENGINES };
|
|
2884
|
+
//# sourceMappingURL=index.js.map
|
|
2885
|
+
//# sourceMappingURL=index.js.map
|