@mastra/clickhouse 0.0.0-working-memory-per-user-20250620163010 → 0.0.0-zod-v4-compat-part-2-20250822105954
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +168 -4
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2259 -566
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +2 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2245 -550
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +79 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +43 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +36 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/{_tsup-dts-rollup.d.cts → storage/index.d.ts} +106 -87
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +9 -9
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +15 -1013
- package/src/storage/index.ts +214 -1013
- package/tsconfig.build.json +9 -0
- package/tsconfig.json +1 -1
- package/tsup.config.ts +22 -0
- package/dist/_tsup-dts-rollup.d.ts +0 -187
- package/dist/index.d.cts +0 -4
package/dist/index.js
CHANGED
|
@@ -1,21 +1,16 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { MastraStorage, StoreOperations, TABLE_SCHEMAS, TABLE_WORKFLOW_SNAPSHOT, WorkflowsStorage, ScoresStorage, safelyParseJSON, TABLE_SCORERS, LegacyEvalsStorage, TABLE_EVALS, TracesStorage, TABLE_TRACES, MemoryStorage, resolveMessageLimit, TABLE_MESSAGES, TABLE_THREADS, TABLE_RESOURCES } from '@mastra/core/storage';
|
|
2
4
|
import { MessageList } from '@mastra/core/agent';
|
|
3
|
-
import { TABLE_RESOURCES, TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, TABLE_SCHEMAS } from '@mastra/core/storage';
|
|
4
5
|
|
|
5
6
|
// src/storage/index.ts
|
|
6
|
-
function safelyParseJSON(jsonString) {
|
|
7
|
-
try {
|
|
8
|
-
return JSON.parse(jsonString);
|
|
9
|
-
} catch {
|
|
10
|
-
return {};
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
7
|
var TABLE_ENGINES = {
|
|
14
8
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
15
9
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
16
10
|
[TABLE_TRACES]: `MergeTree()`,
|
|
17
11
|
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
18
12
|
[TABLE_EVALS]: `MergeTree()`,
|
|
13
|
+
[TABLE_SCORERS]: `MergeTree()`,
|
|
19
14
|
[TABLE_RESOURCES]: `ReplacingMergeTree()`
|
|
20
15
|
};
|
|
21
16
|
var COLUMN_TYPES = {
|
|
@@ -24,11 +19,9 @@ var COLUMN_TYPES = {
|
|
|
24
19
|
uuid: "String",
|
|
25
20
|
jsonb: "String",
|
|
26
21
|
integer: "Int64",
|
|
22
|
+
float: "Float64",
|
|
27
23
|
bigint: "Int64"
|
|
28
24
|
};
|
|
29
|
-
function transformRows(rows) {
|
|
30
|
-
return rows.map((row) => transformRow(row));
|
|
31
|
-
}
|
|
32
25
|
function transformRow(row) {
|
|
33
26
|
if (!row) {
|
|
34
27
|
return row;
|
|
@@ -39,33 +32,63 @@ function transformRow(row) {
|
|
|
39
32
|
if (row.updatedAt) {
|
|
40
33
|
row.updatedAt = new Date(row.updatedAt);
|
|
41
34
|
}
|
|
35
|
+
if (row.content && typeof row.content === "string") {
|
|
36
|
+
row.content = safelyParseJSON(row.content);
|
|
37
|
+
}
|
|
42
38
|
return row;
|
|
43
39
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// This is crucial
|
|
57
|
-
use_client_time_zone: 1,
|
|
58
|
-
output_format_json_quote_64bit_integers: 0
|
|
59
|
-
}
|
|
60
|
-
});
|
|
61
|
-
this.ttl = config.ttl;
|
|
40
|
+
function transformRows(rows) {
|
|
41
|
+
return rows.map((row) => transformRow(row));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// src/storage/domains/legacy-evals/index.ts
|
|
45
|
+
var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
|
|
46
|
+
client;
|
|
47
|
+
operations;
|
|
48
|
+
constructor({ client, operations }) {
|
|
49
|
+
super();
|
|
50
|
+
this.client = client;
|
|
51
|
+
this.operations = operations;
|
|
62
52
|
}
|
|
63
53
|
transformEvalRow(row) {
|
|
64
54
|
row = transformRow(row);
|
|
65
|
-
|
|
66
|
-
|
|
55
|
+
let resultValue;
|
|
56
|
+
try {
|
|
57
|
+
if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
|
|
58
|
+
resultValue = JSON.parse(row.result);
|
|
59
|
+
} else if (typeof row.result === "object" && row.result !== null) {
|
|
60
|
+
resultValue = row.result;
|
|
61
|
+
} else if (row.result === null || row.result === void 0 || row.result === "") {
|
|
62
|
+
resultValue = { score: 0 };
|
|
63
|
+
} else {
|
|
64
|
+
throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
console.error("Error parsing result field:", row.result, error);
|
|
68
|
+
throw new MastraError({
|
|
69
|
+
id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
|
|
70
|
+
text: `Invalid result format: ${JSON.stringify(row.result)}`,
|
|
71
|
+
domain: ErrorDomain.STORAGE,
|
|
72
|
+
category: ErrorCategory.USER
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
let testInfoValue;
|
|
76
|
+
try {
|
|
77
|
+
if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
|
|
78
|
+
testInfoValue = JSON.parse(row.test_info);
|
|
79
|
+
} else if (typeof row.test_info === "object" && row.test_info !== null) {
|
|
80
|
+
testInfoValue = row.test_info;
|
|
81
|
+
}
|
|
82
|
+
} catch {
|
|
83
|
+
testInfoValue = void 0;
|
|
84
|
+
}
|
|
67
85
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
|
-
throw new
|
|
86
|
+
throw new MastraError({
|
|
87
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
88
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
89
|
+
domain: ErrorDomain.STORAGE,
|
|
90
|
+
category: ErrorCategory.USER
|
|
91
|
+
});
|
|
69
92
|
}
|
|
70
93
|
return {
|
|
71
94
|
input: row.input,
|
|
@@ -82,9 +105,9 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
82
105
|
}
|
|
83
106
|
async getEvalsByAgentName(agentName, type) {
|
|
84
107
|
try {
|
|
85
|
-
const baseQuery = `SELECT *, toDateTime64(
|
|
86
|
-
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
87
|
-
const result = await this.
|
|
108
|
+
const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
109
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
|
|
110
|
+
const result = await this.client.query({
|
|
88
111
|
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
89
112
|
query_params: { var_agent_name: agentName },
|
|
90
113
|
clickhouse_settings: {
|
|
@@ -100,253 +123,1425 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
100
123
|
const rows = await result.json();
|
|
101
124
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
102
125
|
} catch (error) {
|
|
103
|
-
if (error
|
|
126
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
104
127
|
return [];
|
|
105
128
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
...Object.fromEntries(
|
|
116
|
-
Object.entries(record).map(([key, value]) => [
|
|
117
|
-
key,
|
|
118
|
-
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
119
|
-
])
|
|
120
|
-
)
|
|
121
|
-
})),
|
|
122
|
-
format: "JSONEachRow",
|
|
123
|
-
clickhouse_settings: {
|
|
124
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
125
|
-
date_time_input_format: "best_effort",
|
|
126
|
-
use_client_time_zone: 1,
|
|
127
|
-
output_format_json_quote_64bit_integers: 0
|
|
128
|
-
}
|
|
129
|
-
});
|
|
130
|
-
} catch (error) {
|
|
131
|
-
console.error(`Error inserting into ${tableName}:`, error);
|
|
132
|
-
throw error;
|
|
129
|
+
throw new MastraError(
|
|
130
|
+
{
|
|
131
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
132
|
+
domain: ErrorDomain.STORAGE,
|
|
133
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
134
|
+
details: { agentName, type: type ?? null }
|
|
135
|
+
},
|
|
136
|
+
error
|
|
137
|
+
);
|
|
133
138
|
}
|
|
134
139
|
}
|
|
135
|
-
async
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
perPage,
|
|
140
|
-
attributes,
|
|
141
|
-
filters,
|
|
142
|
-
fromDate,
|
|
143
|
-
toDate
|
|
144
|
-
}) {
|
|
145
|
-
const limit = perPage;
|
|
146
|
-
const offset = page * perPage;
|
|
147
|
-
const args = {};
|
|
140
|
+
async getEvals(options = {}) {
|
|
141
|
+
const { agentName, type, page = 0, perPage = 100, dateRange } = options;
|
|
142
|
+
const fromDate = dateRange?.start;
|
|
143
|
+
const toDate = dateRange?.end;
|
|
148
144
|
const conditions = [];
|
|
149
|
-
if (
|
|
150
|
-
conditions.push(`
|
|
151
|
-
args.var_name = name;
|
|
152
|
-
}
|
|
153
|
-
if (scope) {
|
|
154
|
-
conditions.push(`scope = {var_scope:String}`);
|
|
155
|
-
args.var_scope = scope;
|
|
156
|
-
}
|
|
157
|
-
if (attributes) {
|
|
158
|
-
Object.entries(attributes).forEach(([key, value]) => {
|
|
159
|
-
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
160
|
-
args[`var_attr_${key}`] = value;
|
|
161
|
-
});
|
|
145
|
+
if (agentName) {
|
|
146
|
+
conditions.push(`agent_name = {var_agent_name:String}`);
|
|
162
147
|
}
|
|
163
|
-
if (
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
148
|
+
if (type === "test") {
|
|
149
|
+
conditions.push(
|
|
150
|
+
`(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
|
|
151
|
+
);
|
|
152
|
+
} else if (type === "live") {
|
|
153
|
+
conditions.push(
|
|
154
|
+
`(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
|
|
155
|
+
);
|
|
170
156
|
}
|
|
171
157
|
if (fromDate) {
|
|
172
|
-
conditions.push(`
|
|
173
|
-
|
|
158
|
+
conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
159
|
+
fromDate.toISOString();
|
|
174
160
|
}
|
|
175
161
|
if (toDate) {
|
|
176
|
-
conditions.push(`
|
|
177
|
-
|
|
162
|
+
conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
163
|
+
toDate.toISOString();
|
|
178
164
|
}
|
|
179
165
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
180
|
-
const result = await this.db.query({
|
|
181
|
-
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
182
|
-
query_params: args,
|
|
183
|
-
clickhouse_settings: {
|
|
184
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
185
|
-
date_time_input_format: "best_effort",
|
|
186
|
-
date_time_output_format: "iso",
|
|
187
|
-
use_client_time_zone: 1,
|
|
188
|
-
output_format_json_quote_64bit_integers: 0
|
|
189
|
-
}
|
|
190
|
-
});
|
|
191
|
-
if (!result) {
|
|
192
|
-
return [];
|
|
193
|
-
}
|
|
194
|
-
const resp = await result.json();
|
|
195
|
-
const rows = resp.data;
|
|
196
|
-
return rows.map((row) => ({
|
|
197
|
-
id: row.id,
|
|
198
|
-
parentSpanId: row.parentSpanId,
|
|
199
|
-
traceId: row.traceId,
|
|
200
|
-
name: row.name,
|
|
201
|
-
scope: row.scope,
|
|
202
|
-
kind: row.kind,
|
|
203
|
-
status: safelyParseJSON(row.status),
|
|
204
|
-
events: safelyParseJSON(row.events),
|
|
205
|
-
links: safelyParseJSON(row.links),
|
|
206
|
-
attributes: safelyParseJSON(row.attributes),
|
|
207
|
-
startTime: row.startTime,
|
|
208
|
-
endTime: row.endTime,
|
|
209
|
-
other: safelyParseJSON(row.other),
|
|
210
|
-
createdAt: row.createdAt
|
|
211
|
-
}));
|
|
212
|
-
}
|
|
213
|
-
async optimizeTable({ tableName }) {
|
|
214
|
-
await this.db.command({
|
|
215
|
-
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
async materializeTtl({ tableName }) {
|
|
219
|
-
await this.db.command({
|
|
220
|
-
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
async createTable({
|
|
224
|
-
tableName,
|
|
225
|
-
schema
|
|
226
|
-
}) {
|
|
227
166
|
try {
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
167
|
+
const countResult = await this.client.query({
|
|
168
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
|
|
169
|
+
query_params: {
|
|
170
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
171
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
172
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {}
|
|
173
|
+
},
|
|
174
|
+
clickhouse_settings: {
|
|
175
|
+
date_time_input_format: "best_effort",
|
|
176
|
+
date_time_output_format: "iso",
|
|
177
|
+
use_client_time_zone: 1,
|
|
178
|
+
output_format_json_quote_64bit_integers: 0
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
const countData = await countResult.json();
|
|
182
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
183
|
+
const currentOffset = page * perPage;
|
|
184
|
+
const hasMore = currentOffset + perPage < total;
|
|
185
|
+
if (total === 0) {
|
|
186
|
+
return {
|
|
187
|
+
evals: [],
|
|
188
|
+
total: 0,
|
|
189
|
+
page,
|
|
190
|
+
perPage,
|
|
191
|
+
hasMore: false
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
const dataResult = await this.client.query({
|
|
195
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
196
|
+
query_params: {
|
|
197
|
+
...agentName ? { var_agent_name: agentName } : {},
|
|
198
|
+
...fromDate ? { var_from_date: fromDate.toISOString() } : {},
|
|
199
|
+
...toDate ? { var_to_date: toDate.toISOString() } : {},
|
|
200
|
+
var_limit: perPage || 100,
|
|
201
|
+
var_offset: currentOffset || 0
|
|
202
|
+
},
|
|
256
203
|
clickhouse_settings: {
|
|
257
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
258
204
|
date_time_input_format: "best_effort",
|
|
259
205
|
date_time_output_format: "iso",
|
|
260
206
|
use_client_time_zone: 1,
|
|
261
207
|
output_format_json_quote_64bit_integers: 0
|
|
262
208
|
}
|
|
263
209
|
});
|
|
210
|
+
const rows = await dataResult.json();
|
|
211
|
+
return {
|
|
212
|
+
evals: rows.data.map((row) => this.transformEvalRow(row)),
|
|
213
|
+
total,
|
|
214
|
+
page,
|
|
215
|
+
perPage,
|
|
216
|
+
hasMore
|
|
217
|
+
};
|
|
264
218
|
} catch (error) {
|
|
265
|
-
|
|
266
|
-
|
|
219
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
220
|
+
return {
|
|
221
|
+
evals: [],
|
|
222
|
+
total: 0,
|
|
223
|
+
page,
|
|
224
|
+
perPage,
|
|
225
|
+
hasMore: false
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
throw new MastraError(
|
|
229
|
+
{
|
|
230
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
|
|
231
|
+
domain: ErrorDomain.STORAGE,
|
|
232
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
233
|
+
details: { agentName: agentName ?? "all", type: type ?? "all" }
|
|
234
|
+
},
|
|
235
|
+
error
|
|
236
|
+
);
|
|
267
237
|
}
|
|
268
238
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
return "Int64";
|
|
278
|
-
case "jsonb":
|
|
279
|
-
return "String";
|
|
280
|
-
default:
|
|
281
|
-
return super.getSqlType(type);
|
|
282
|
-
}
|
|
239
|
+
};
|
|
240
|
+
var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
241
|
+
client;
|
|
242
|
+
operations;
|
|
243
|
+
constructor({ client, operations }) {
|
|
244
|
+
super();
|
|
245
|
+
this.client = client;
|
|
246
|
+
this.operations = operations;
|
|
283
247
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
*/
|
|
290
|
-
async alterTable({
|
|
291
|
-
tableName,
|
|
292
|
-
schema,
|
|
293
|
-
ifNotExists
|
|
248
|
+
async getMessages({
|
|
249
|
+
threadId,
|
|
250
|
+
resourceId,
|
|
251
|
+
selectBy,
|
|
252
|
+
format
|
|
294
253
|
}) {
|
|
295
254
|
try {
|
|
296
|
-
const
|
|
297
|
-
const
|
|
298
|
-
|
|
255
|
+
const messages = [];
|
|
256
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
257
|
+
const include = selectBy?.include || [];
|
|
258
|
+
if (include.length) {
|
|
259
|
+
const unionQueries = [];
|
|
260
|
+
const params = [];
|
|
261
|
+
let paramIdx = 1;
|
|
262
|
+
for (const inc of include) {
|
|
263
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
264
|
+
const searchId = inc.threadId || threadId;
|
|
265
|
+
unionQueries.push(`
|
|
266
|
+
SELECT * FROM (
|
|
267
|
+
WITH numbered_messages AS (
|
|
268
|
+
SELECT
|
|
269
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
270
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
271
|
+
FROM "${TABLE_MESSAGES}"
|
|
272
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
273
|
+
),
|
|
274
|
+
target_positions AS (
|
|
275
|
+
SELECT row_num as target_pos
|
|
276
|
+
FROM numbered_messages
|
|
277
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
278
|
+
)
|
|
279
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
280
|
+
FROM numbered_messages m
|
|
281
|
+
CROSS JOIN target_positions t
|
|
282
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
283
|
+
) AS query_${paramIdx}
|
|
284
|
+
`);
|
|
285
|
+
params.push(
|
|
286
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
287
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
288
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
289
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
290
|
+
);
|
|
291
|
+
paramIdx++;
|
|
292
|
+
}
|
|
293
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
294
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
295
|
+
const includeResult = await this.client.query({
|
|
296
|
+
query: finalQuery,
|
|
297
|
+
query_params: mergedParams,
|
|
298
|
+
clickhouse_settings: {
|
|
299
|
+
date_time_input_format: "best_effort",
|
|
300
|
+
date_time_output_format: "iso",
|
|
301
|
+
use_client_time_zone: 1,
|
|
302
|
+
output_format_json_quote_64bit_integers: 0
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
const rows2 = await includeResult.json();
|
|
306
|
+
const includedMessages = transformRows(rows2.data);
|
|
307
|
+
const seen = /* @__PURE__ */ new Set();
|
|
308
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
309
|
+
if (seen.has(message.id)) return false;
|
|
310
|
+
seen.add(message.id);
|
|
311
|
+
return true;
|
|
312
|
+
});
|
|
313
|
+
messages.push(...dedupedMessages);
|
|
314
|
+
}
|
|
315
|
+
const result = await this.client.query({
|
|
316
|
+
query: `
|
|
317
|
+
SELECT
|
|
318
|
+
id,
|
|
319
|
+
content,
|
|
320
|
+
role,
|
|
321
|
+
type,
|
|
322
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
323
|
+
thread_id AS "threadId"
|
|
324
|
+
FROM "${TABLE_MESSAGES}"
|
|
325
|
+
WHERE thread_id = {threadId:String}
|
|
326
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
327
|
+
ORDER BY "createdAt" DESC
|
|
328
|
+
LIMIT {limit:Int64}
|
|
329
|
+
`,
|
|
330
|
+
query_params: {
|
|
331
|
+
threadId,
|
|
332
|
+
exclude: messages.map((m) => m.id),
|
|
333
|
+
limit
|
|
334
|
+
},
|
|
335
|
+
clickhouse_settings: {
|
|
336
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
337
|
+
date_time_input_format: "best_effort",
|
|
338
|
+
date_time_output_format: "iso",
|
|
339
|
+
use_client_time_zone: 1,
|
|
340
|
+
output_format_json_quote_64bit_integers: 0
|
|
341
|
+
}
|
|
299
342
|
});
|
|
300
343
|
const rows = await result.json();
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
344
|
+
messages.push(...transformRows(rows.data));
|
|
345
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
346
|
+
messages.forEach((message) => {
|
|
347
|
+
if (typeof message.content === "string") {
|
|
348
|
+
try {
|
|
349
|
+
message.content = JSON.parse(message.content);
|
|
350
|
+
} catch {
|
|
308
351
|
}
|
|
309
|
-
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
310
|
-
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
311
|
-
await this.db.query({
|
|
312
|
-
query: alterSql
|
|
313
|
-
});
|
|
314
|
-
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
315
352
|
}
|
|
316
|
-
}
|
|
353
|
+
});
|
|
354
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
355
|
+
if (format === `v2`) return list.get.all.v2();
|
|
356
|
+
return list.get.all.v1();
|
|
317
357
|
} catch (error) {
|
|
318
|
-
|
|
319
|
-
|
|
358
|
+
throw new MastraError(
|
|
359
|
+
{
|
|
360
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
361
|
+
domain: ErrorDomain.STORAGE,
|
|
362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
363
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
364
|
+
},
|
|
365
|
+
error
|
|
320
366
|
);
|
|
321
|
-
throw new Error(`Failed to alter table ${tableName}: ${error}`);
|
|
322
367
|
}
|
|
323
368
|
}
|
|
324
|
-
async
|
|
369
|
+
async saveMessages(args) {
|
|
370
|
+
const { messages, format = "v1" } = args;
|
|
371
|
+
if (messages.length === 0) return messages;
|
|
372
|
+
for (const message of messages) {
|
|
373
|
+
const resourceId = message.resourceId;
|
|
374
|
+
if (!resourceId) {
|
|
375
|
+
throw new Error("Resource ID is required");
|
|
376
|
+
}
|
|
377
|
+
if (!message.threadId) {
|
|
378
|
+
throw new Error("Thread ID is required");
|
|
379
|
+
}
|
|
380
|
+
const thread = await this.getThreadById({ threadId: message.threadId });
|
|
381
|
+
if (!thread) {
|
|
382
|
+
throw new Error(`Thread ${message.threadId} not found`);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
const threadIdSet = /* @__PURE__ */ new Map();
|
|
386
|
+
await Promise.all(
|
|
387
|
+
messages.map(async (m) => {
|
|
388
|
+
const resourceId = m.resourceId;
|
|
389
|
+
if (!resourceId) {
|
|
390
|
+
throw new Error("Resource ID is required");
|
|
391
|
+
}
|
|
392
|
+
if (!m.threadId) {
|
|
393
|
+
throw new Error("Thread ID is required");
|
|
394
|
+
}
|
|
395
|
+
const thread = await this.getThreadById({ threadId: m.threadId });
|
|
396
|
+
if (!thread) {
|
|
397
|
+
throw new Error(`Thread ${m.threadId} not found`);
|
|
398
|
+
}
|
|
399
|
+
threadIdSet.set(m.threadId, thread);
|
|
400
|
+
})
|
|
401
|
+
);
|
|
325
402
|
try {
|
|
326
|
-
await this.
|
|
327
|
-
query: `
|
|
403
|
+
const existingResult = await this.client.query({
|
|
404
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
405
|
+
query_params: {
|
|
406
|
+
ids: messages.map((m) => m.id)
|
|
407
|
+
},
|
|
328
408
|
clickhouse_settings: {
|
|
329
409
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
330
410
|
date_time_input_format: "best_effort",
|
|
331
411
|
date_time_output_format: "iso",
|
|
332
412
|
use_client_time_zone: 1,
|
|
333
413
|
output_format_json_quote_64bit_integers: 0
|
|
334
|
-
}
|
|
414
|
+
},
|
|
415
|
+
format: "JSONEachRow"
|
|
335
416
|
});
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
417
|
+
const existingRows = await existingResult.json();
|
|
418
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
419
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
|
|
420
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
|
|
421
|
+
const toMove = messages.filter((m) => {
|
|
422
|
+
const existingRow = existingRows.find((row) => row.id === m.id);
|
|
423
|
+
return existingRow && existingRow.thread_id !== m.threadId;
|
|
424
|
+
});
|
|
425
|
+
const deletePromises = toMove.map((message) => {
|
|
426
|
+
const existingRow = existingRows.find((row) => row.id === message.id);
|
|
427
|
+
if (!existingRow) return Promise.resolve();
|
|
428
|
+
return this.client.command({
|
|
429
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
|
|
430
|
+
query_params: {
|
|
431
|
+
var_id: message.id,
|
|
432
|
+
var_old_thread_id: existingRow.thread_id
|
|
433
|
+
},
|
|
434
|
+
clickhouse_settings: {
|
|
435
|
+
date_time_input_format: "best_effort",
|
|
436
|
+
use_client_time_zone: 1,
|
|
437
|
+
output_format_json_quote_64bit_integers: 0
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
});
|
|
441
|
+
const updatePromises = toUpdate.map(
|
|
442
|
+
(message) => this.client.command({
|
|
443
|
+
query: `
|
|
444
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
445
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
|
|
446
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
447
|
+
`,
|
|
448
|
+
query_params: {
|
|
449
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
450
|
+
var_role: message.role,
|
|
451
|
+
var_type: message.type || "v2",
|
|
452
|
+
var_resourceId: message.resourceId,
|
|
453
|
+
var_id: message.id,
|
|
454
|
+
var_thread_id: message.threadId
|
|
455
|
+
},
|
|
456
|
+
clickhouse_settings: {
|
|
457
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
458
|
+
date_time_input_format: "best_effort",
|
|
459
|
+
use_client_time_zone: 1,
|
|
460
|
+
output_format_json_quote_64bit_integers: 0
|
|
461
|
+
}
|
|
462
|
+
})
|
|
463
|
+
);
|
|
464
|
+
await Promise.all([
|
|
465
|
+
// Insert new messages (including moved messages)
|
|
466
|
+
this.client.insert({
|
|
467
|
+
table: TABLE_MESSAGES,
|
|
468
|
+
format: "JSONEachRow",
|
|
469
|
+
values: toInsert.map((message) => ({
|
|
470
|
+
id: message.id,
|
|
471
|
+
thread_id: message.threadId,
|
|
472
|
+
resourceId: message.resourceId,
|
|
473
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
474
|
+
createdAt: message.createdAt.toISOString(),
|
|
475
|
+
role: message.role,
|
|
476
|
+
type: message.type || "v2"
|
|
477
|
+
})),
|
|
478
|
+
clickhouse_settings: {
|
|
479
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
480
|
+
date_time_input_format: "best_effort",
|
|
481
|
+
use_client_time_zone: 1,
|
|
482
|
+
output_format_json_quote_64bit_integers: 0
|
|
483
|
+
}
|
|
484
|
+
}),
|
|
485
|
+
...updatePromises,
|
|
486
|
+
...deletePromises,
|
|
487
|
+
// Update thread's updatedAt timestamp
|
|
488
|
+
this.client.insert({
|
|
489
|
+
table: TABLE_THREADS,
|
|
490
|
+
format: "JSONEachRow",
|
|
491
|
+
values: Array.from(threadIdSet.values()).map((thread) => ({
|
|
492
|
+
id: thread.id,
|
|
493
|
+
resourceId: thread.resourceId,
|
|
494
|
+
title: thread.title,
|
|
495
|
+
metadata: thread.metadata,
|
|
496
|
+
createdAt: thread.createdAt,
|
|
497
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
498
|
+
})),
|
|
499
|
+
clickhouse_settings: {
|
|
500
|
+
date_time_input_format: "best_effort",
|
|
501
|
+
use_client_time_zone: 1,
|
|
502
|
+
output_format_json_quote_64bit_integers: 0
|
|
503
|
+
}
|
|
504
|
+
})
|
|
505
|
+
]);
|
|
506
|
+
const list = new MessageList().add(messages, "memory");
|
|
507
|
+
if (format === `v2`) return list.get.all.v2();
|
|
508
|
+
return list.get.all.v1();
|
|
509
|
+
} catch (error) {
|
|
510
|
+
throw new MastraError(
|
|
511
|
+
{
|
|
512
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
513
|
+
domain: ErrorDomain.STORAGE,
|
|
514
|
+
category: ErrorCategory.THIRD_PARTY
|
|
515
|
+
},
|
|
516
|
+
error
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async getThreadById({ threadId }) {
|
|
521
|
+
try {
|
|
522
|
+
const result = await this.client.query({
|
|
523
|
+
query: `SELECT
|
|
524
|
+
id,
|
|
525
|
+
"resourceId",
|
|
526
|
+
title,
|
|
527
|
+
metadata,
|
|
528
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
529
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
530
|
+
FROM "${TABLE_THREADS}"
|
|
531
|
+
FINAL
|
|
532
|
+
WHERE id = {var_id:String}`,
|
|
533
|
+
query_params: { var_id: threadId },
|
|
534
|
+
clickhouse_settings: {
|
|
535
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
536
|
+
date_time_input_format: "best_effort",
|
|
537
|
+
date_time_output_format: "iso",
|
|
538
|
+
use_client_time_zone: 1,
|
|
539
|
+
output_format_json_quote_64bit_integers: 0
|
|
540
|
+
}
|
|
541
|
+
});
|
|
542
|
+
const rows = await result.json();
|
|
543
|
+
const thread = transformRow(rows.data[0]);
|
|
544
|
+
if (!thread) {
|
|
545
|
+
return null;
|
|
546
|
+
}
|
|
547
|
+
return {
|
|
548
|
+
...thread,
|
|
549
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
550
|
+
createdAt: thread.createdAt,
|
|
551
|
+
updatedAt: thread.updatedAt
|
|
552
|
+
};
|
|
553
|
+
} catch (error) {
|
|
554
|
+
throw new MastraError(
|
|
555
|
+
{
|
|
556
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
557
|
+
domain: ErrorDomain.STORAGE,
|
|
558
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
559
|
+
details: { threadId }
|
|
560
|
+
},
|
|
561
|
+
error
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
566
|
+
try {
|
|
567
|
+
const result = await this.client.query({
|
|
568
|
+
query: `SELECT
|
|
569
|
+
id,
|
|
570
|
+
"resourceId",
|
|
571
|
+
title,
|
|
572
|
+
metadata,
|
|
573
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
574
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
575
|
+
FROM "${TABLE_THREADS}"
|
|
576
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
577
|
+
query_params: { var_resourceId: resourceId },
|
|
578
|
+
clickhouse_settings: {
|
|
579
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
580
|
+
date_time_input_format: "best_effort",
|
|
581
|
+
date_time_output_format: "iso",
|
|
582
|
+
use_client_time_zone: 1,
|
|
583
|
+
output_format_json_quote_64bit_integers: 0
|
|
584
|
+
}
|
|
585
|
+
});
|
|
586
|
+
const rows = await result.json();
|
|
587
|
+
const threads = transformRows(rows.data);
|
|
588
|
+
return threads.map((thread) => ({
|
|
589
|
+
...thread,
|
|
590
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
591
|
+
createdAt: thread.createdAt,
|
|
592
|
+
updatedAt: thread.updatedAt
|
|
593
|
+
}));
|
|
594
|
+
} catch (error) {
|
|
595
|
+
throw new MastraError(
|
|
596
|
+
{
|
|
597
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
598
|
+
domain: ErrorDomain.STORAGE,
|
|
599
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
600
|
+
details: { resourceId }
|
|
601
|
+
},
|
|
602
|
+
error
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
async saveThread({ thread }) {
|
|
607
|
+
try {
|
|
608
|
+
await this.client.insert({
|
|
609
|
+
table: TABLE_THREADS,
|
|
610
|
+
values: [
|
|
611
|
+
{
|
|
612
|
+
...thread,
|
|
613
|
+
createdAt: thread.createdAt.toISOString(),
|
|
614
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
615
|
+
}
|
|
616
|
+
],
|
|
617
|
+
format: "JSONEachRow",
|
|
618
|
+
clickhouse_settings: {
|
|
619
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
620
|
+
date_time_input_format: "best_effort",
|
|
621
|
+
use_client_time_zone: 1,
|
|
622
|
+
output_format_json_quote_64bit_integers: 0
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
return thread;
|
|
626
|
+
} catch (error) {
|
|
627
|
+
throw new MastraError(
|
|
628
|
+
{
|
|
629
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
630
|
+
domain: ErrorDomain.STORAGE,
|
|
631
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
632
|
+
details: { threadId: thread.id }
|
|
633
|
+
},
|
|
634
|
+
error
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
async updateThread({
|
|
639
|
+
id,
|
|
640
|
+
title,
|
|
641
|
+
metadata
|
|
642
|
+
}) {
|
|
643
|
+
try {
|
|
644
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
645
|
+
if (!existingThread) {
|
|
646
|
+
throw new Error(`Thread ${id} not found`);
|
|
647
|
+
}
|
|
648
|
+
const mergedMetadata = {
|
|
649
|
+
...existingThread.metadata,
|
|
650
|
+
...metadata
|
|
651
|
+
};
|
|
652
|
+
const updatedThread = {
|
|
653
|
+
...existingThread,
|
|
654
|
+
title,
|
|
655
|
+
metadata: mergedMetadata,
|
|
656
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
657
|
+
};
|
|
658
|
+
await this.client.insert({
|
|
659
|
+
table: TABLE_THREADS,
|
|
660
|
+
format: "JSONEachRow",
|
|
661
|
+
values: [
|
|
662
|
+
{
|
|
663
|
+
id: updatedThread.id,
|
|
664
|
+
resourceId: updatedThread.resourceId,
|
|
665
|
+
title: updatedThread.title,
|
|
666
|
+
metadata: updatedThread.metadata,
|
|
667
|
+
createdAt: updatedThread.createdAt,
|
|
668
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
669
|
+
}
|
|
670
|
+
],
|
|
671
|
+
clickhouse_settings: {
|
|
672
|
+
date_time_input_format: "best_effort",
|
|
673
|
+
use_client_time_zone: 1,
|
|
674
|
+
output_format_json_quote_64bit_integers: 0
|
|
675
|
+
}
|
|
676
|
+
});
|
|
677
|
+
return updatedThread;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
throw new MastraError(
|
|
680
|
+
{
|
|
681
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
682
|
+
domain: ErrorDomain.STORAGE,
|
|
683
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
684
|
+
details: { threadId: id, title }
|
|
685
|
+
},
|
|
686
|
+
error
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
async deleteThread({ threadId }) {
|
|
691
|
+
try {
|
|
692
|
+
await this.client.command({
|
|
693
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
694
|
+
query_params: { var_thread_id: threadId },
|
|
695
|
+
clickhouse_settings: {
|
|
696
|
+
output_format_json_quote_64bit_integers: 0
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
await this.client.command({
|
|
700
|
+
query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
701
|
+
query_params: { var_id: threadId },
|
|
702
|
+
clickhouse_settings: {
|
|
703
|
+
output_format_json_quote_64bit_integers: 0
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
} catch (error) {
|
|
707
|
+
throw new MastraError(
|
|
708
|
+
{
|
|
709
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
710
|
+
domain: ErrorDomain.STORAGE,
|
|
711
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
712
|
+
details: { threadId }
|
|
713
|
+
},
|
|
714
|
+
error
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
719
|
+
const { resourceId, page = 0, perPage = 100 } = args;
|
|
720
|
+
try {
|
|
721
|
+
const currentOffset = page * perPage;
|
|
722
|
+
const countResult = await this.client.query({
|
|
723
|
+
query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
|
|
724
|
+
query_params: { resourceId },
|
|
725
|
+
clickhouse_settings: {
|
|
726
|
+
date_time_input_format: "best_effort",
|
|
727
|
+
date_time_output_format: "iso",
|
|
728
|
+
use_client_time_zone: 1,
|
|
729
|
+
output_format_json_quote_64bit_integers: 0
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
const countData = await countResult.json();
|
|
733
|
+
const total = countData.data[0].total;
|
|
734
|
+
if (total === 0) {
|
|
735
|
+
return {
|
|
736
|
+
threads: [],
|
|
737
|
+
total: 0,
|
|
738
|
+
page,
|
|
739
|
+
perPage,
|
|
740
|
+
hasMore: false
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
const dataResult = await this.client.query({
|
|
744
|
+
query: `
|
|
745
|
+
SELECT
|
|
746
|
+
id,
|
|
747
|
+
resourceId,
|
|
748
|
+
title,
|
|
749
|
+
metadata,
|
|
750
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
751
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
752
|
+
FROM ${TABLE_THREADS}
|
|
753
|
+
WHERE resourceId = {resourceId:String}
|
|
754
|
+
ORDER BY createdAt DESC
|
|
755
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
756
|
+
`,
|
|
757
|
+
query_params: {
|
|
758
|
+
resourceId,
|
|
759
|
+
limit: perPage,
|
|
760
|
+
offset: currentOffset
|
|
761
|
+
},
|
|
762
|
+
clickhouse_settings: {
|
|
763
|
+
date_time_input_format: "best_effort",
|
|
764
|
+
date_time_output_format: "iso",
|
|
765
|
+
use_client_time_zone: 1,
|
|
766
|
+
output_format_json_quote_64bit_integers: 0
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
const rows = await dataResult.json();
|
|
770
|
+
const threads = transformRows(rows.data);
|
|
771
|
+
return {
|
|
772
|
+
threads,
|
|
773
|
+
total,
|
|
774
|
+
page,
|
|
775
|
+
perPage,
|
|
776
|
+
hasMore: currentOffset + threads.length < total
|
|
777
|
+
};
|
|
778
|
+
} catch (error) {
|
|
779
|
+
throw new MastraError(
|
|
780
|
+
{
|
|
781
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
782
|
+
domain: ErrorDomain.STORAGE,
|
|
783
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
784
|
+
details: { resourceId, page }
|
|
785
|
+
},
|
|
786
|
+
error
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async getMessagesPaginated(args) {
|
|
791
|
+
try {
|
|
792
|
+
const { threadId, selectBy, format = "v1" } = args;
|
|
793
|
+
const page = selectBy?.pagination?.page || 0;
|
|
794
|
+
const perPageInput = selectBy?.pagination?.perPage;
|
|
795
|
+
const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
|
|
796
|
+
const offset = page * perPage;
|
|
797
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
798
|
+
const fromDate = dateRange?.start;
|
|
799
|
+
const toDate = dateRange?.end;
|
|
800
|
+
const messages = [];
|
|
801
|
+
if (selectBy?.include?.length) {
|
|
802
|
+
const include = selectBy.include;
|
|
803
|
+
const unionQueries = [];
|
|
804
|
+
const params = [];
|
|
805
|
+
let paramIdx = 1;
|
|
806
|
+
for (const inc of include) {
|
|
807
|
+
const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
|
|
808
|
+
const searchId = inc.threadId || threadId;
|
|
809
|
+
unionQueries.push(`
|
|
810
|
+
SELECT * FROM (
|
|
811
|
+
WITH numbered_messages AS (
|
|
812
|
+
SELECT
|
|
813
|
+
id, content, role, type, "createdAt", thread_id, "resourceId",
|
|
814
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
|
|
815
|
+
FROM "${TABLE_MESSAGES}"
|
|
816
|
+
WHERE thread_id = {var_thread_id_${paramIdx}:String}
|
|
817
|
+
),
|
|
818
|
+
target_positions AS (
|
|
819
|
+
SELECT row_num as target_pos
|
|
820
|
+
FROM numbered_messages
|
|
821
|
+
WHERE id = {var_include_id_${paramIdx}:String}
|
|
822
|
+
)
|
|
823
|
+
SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
|
|
824
|
+
FROM numbered_messages m
|
|
825
|
+
CROSS JOIN target_positions t
|
|
826
|
+
WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
|
|
827
|
+
) AS query_${paramIdx}
|
|
828
|
+
`);
|
|
829
|
+
params.push(
|
|
830
|
+
{ [`var_thread_id_${paramIdx}`]: searchId },
|
|
831
|
+
{ [`var_include_id_${paramIdx}`]: id },
|
|
832
|
+
{ [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
|
|
833
|
+
{ [`var_withNextMessages_${paramIdx}`]: withNextMessages }
|
|
834
|
+
);
|
|
835
|
+
paramIdx++;
|
|
836
|
+
}
|
|
837
|
+
const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
|
|
838
|
+
const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
|
|
839
|
+
const includeResult = await this.client.query({
|
|
840
|
+
query: finalQuery,
|
|
841
|
+
query_params: mergedParams,
|
|
842
|
+
clickhouse_settings: {
|
|
843
|
+
date_time_input_format: "best_effort",
|
|
844
|
+
date_time_output_format: "iso",
|
|
845
|
+
use_client_time_zone: 1,
|
|
846
|
+
output_format_json_quote_64bit_integers: 0
|
|
847
|
+
}
|
|
848
|
+
});
|
|
849
|
+
const rows2 = await includeResult.json();
|
|
850
|
+
const includedMessages = transformRows(rows2.data);
|
|
851
|
+
const seen = /* @__PURE__ */ new Set();
|
|
852
|
+
const dedupedMessages = includedMessages.filter((message) => {
|
|
853
|
+
if (seen.has(message.id)) return false;
|
|
854
|
+
seen.add(message.id);
|
|
855
|
+
return true;
|
|
856
|
+
});
|
|
857
|
+
messages.push(...dedupedMessages);
|
|
858
|
+
}
|
|
859
|
+
let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
|
|
860
|
+
const countParams = { threadId };
|
|
861
|
+
if (fromDate) {
|
|
862
|
+
countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
863
|
+
countParams.fromDate = fromDate.toISOString();
|
|
864
|
+
}
|
|
865
|
+
if (toDate) {
|
|
866
|
+
countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
867
|
+
countParams.toDate = toDate.toISOString();
|
|
868
|
+
}
|
|
869
|
+
const countResult = await this.client.query({
|
|
870
|
+
query: countQuery,
|
|
871
|
+
query_params: countParams,
|
|
872
|
+
clickhouse_settings: {
|
|
873
|
+
date_time_input_format: "best_effort",
|
|
874
|
+
date_time_output_format: "iso",
|
|
875
|
+
use_client_time_zone: 1,
|
|
876
|
+
output_format_json_quote_64bit_integers: 0
|
|
877
|
+
}
|
|
878
|
+
});
|
|
879
|
+
const countData = await countResult.json();
|
|
880
|
+
const total = countData.data[0].total;
|
|
881
|
+
if (total === 0 && messages.length === 0) {
|
|
882
|
+
return {
|
|
883
|
+
messages: [],
|
|
884
|
+
total: 0,
|
|
885
|
+
page,
|
|
886
|
+
perPage,
|
|
887
|
+
hasMore: false
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const excludeIds = messages.map((m) => m.id);
|
|
891
|
+
let dataQuery = `
|
|
892
|
+
SELECT
|
|
893
|
+
id,
|
|
894
|
+
content,
|
|
895
|
+
role,
|
|
896
|
+
type,
|
|
897
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
898
|
+
thread_id AS "threadId",
|
|
899
|
+
resourceId
|
|
900
|
+
FROM ${TABLE_MESSAGES}
|
|
901
|
+
WHERE thread_id = {threadId:String}
|
|
902
|
+
`;
|
|
903
|
+
const dataParams = { threadId };
|
|
904
|
+
if (fromDate) {
|
|
905
|
+
dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
|
|
906
|
+
dataParams.fromDate = fromDate.toISOString();
|
|
907
|
+
}
|
|
908
|
+
if (toDate) {
|
|
909
|
+
dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
|
|
910
|
+
dataParams.toDate = toDate.toISOString();
|
|
911
|
+
}
|
|
912
|
+
if (excludeIds.length > 0) {
|
|
913
|
+
dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
|
|
914
|
+
dataParams.excludeIds = excludeIds;
|
|
915
|
+
}
|
|
916
|
+
if (selectBy?.last) {
|
|
917
|
+
dataQuery += `
|
|
918
|
+
ORDER BY createdAt DESC
|
|
919
|
+
LIMIT {limit:Int64}
|
|
920
|
+
`;
|
|
921
|
+
dataParams.limit = perPage;
|
|
922
|
+
} else {
|
|
923
|
+
dataQuery += `
|
|
924
|
+
ORDER BY createdAt ASC
|
|
925
|
+
LIMIT {limit:Int64} OFFSET {offset:Int64}
|
|
926
|
+
`;
|
|
927
|
+
dataParams.limit = perPage;
|
|
928
|
+
dataParams.offset = offset;
|
|
929
|
+
}
|
|
930
|
+
const result = await this.client.query({
|
|
931
|
+
query: dataQuery,
|
|
932
|
+
query_params: dataParams,
|
|
933
|
+
clickhouse_settings: {
|
|
934
|
+
date_time_input_format: "best_effort",
|
|
935
|
+
date_time_output_format: "iso",
|
|
936
|
+
use_client_time_zone: 1,
|
|
937
|
+
output_format_json_quote_64bit_integers: 0
|
|
938
|
+
}
|
|
939
|
+
});
|
|
940
|
+
const rows = await result.json();
|
|
941
|
+
const paginatedMessages = transformRows(rows.data);
|
|
942
|
+
messages.push(...paginatedMessages);
|
|
943
|
+
if (selectBy?.last) {
|
|
944
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
945
|
+
}
|
|
946
|
+
return {
|
|
947
|
+
messages: format === "v2" ? messages : messages,
|
|
948
|
+
total,
|
|
949
|
+
page,
|
|
950
|
+
perPage,
|
|
951
|
+
hasMore: offset + perPage < total
|
|
952
|
+
};
|
|
953
|
+
} catch (error) {
|
|
954
|
+
throw new MastraError(
|
|
955
|
+
{
|
|
956
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
957
|
+
domain: ErrorDomain.STORAGE,
|
|
958
|
+
category: ErrorCategory.THIRD_PARTY
|
|
959
|
+
},
|
|
960
|
+
error
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
async updateMessages(args) {
|
|
965
|
+
const { messages } = args;
|
|
966
|
+
if (messages.length === 0) {
|
|
967
|
+
return [];
|
|
968
|
+
}
|
|
969
|
+
try {
|
|
970
|
+
const messageIds = messages.map((m) => m.id);
|
|
971
|
+
const existingResult = await this.client.query({
|
|
972
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
|
|
973
|
+
query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
|
|
974
|
+
clickhouse_settings: {
|
|
975
|
+
date_time_input_format: "best_effort",
|
|
976
|
+
date_time_output_format: "iso",
|
|
977
|
+
use_client_time_zone: 1,
|
|
978
|
+
output_format_json_quote_64bit_integers: 0
|
|
979
|
+
}
|
|
980
|
+
});
|
|
981
|
+
const existingRows = await existingResult.json();
|
|
982
|
+
const existingMessages = transformRows(existingRows.data);
|
|
983
|
+
if (existingMessages.length === 0) {
|
|
984
|
+
return [];
|
|
985
|
+
}
|
|
986
|
+
const parsedExistingMessages = existingMessages.map((msg) => {
|
|
987
|
+
if (typeof msg.content === "string") {
|
|
988
|
+
try {
|
|
989
|
+
msg.content = JSON.parse(msg.content);
|
|
990
|
+
} catch {
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return msg;
|
|
994
|
+
});
|
|
995
|
+
const threadIdsToUpdate = /* @__PURE__ */ new Set();
|
|
996
|
+
const updatePromises = [];
|
|
997
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
998
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
999
|
+
if (!updatePayload) continue;
|
|
1000
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1001
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1002
|
+
threadIdsToUpdate.add(existingMessage.threadId);
|
|
1003
|
+
if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
|
|
1004
|
+
threadIdsToUpdate.add(updatePayload.threadId);
|
|
1005
|
+
}
|
|
1006
|
+
const setClauses = [];
|
|
1007
|
+
const values = {};
|
|
1008
|
+
let paramIdx = 1;
|
|
1009
|
+
let newContent = null;
|
|
1010
|
+
const updatableFields = { ...fieldsToUpdate };
|
|
1011
|
+
if (updatableFields.content) {
|
|
1012
|
+
const existingContent = existingMessage.content || {};
|
|
1013
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1014
|
+
const updateMetadata = updatableFields.content.metadata || {};
|
|
1015
|
+
newContent = {
|
|
1016
|
+
...existingContent,
|
|
1017
|
+
...updatableFields.content,
|
|
1018
|
+
// Deep merge metadata
|
|
1019
|
+
metadata: {
|
|
1020
|
+
...existingMetadata,
|
|
1021
|
+
...updateMetadata
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
setClauses.push(`content = {var_content_${paramIdx}:String}`);
|
|
1025
|
+
values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
|
|
1026
|
+
paramIdx++;
|
|
1027
|
+
delete updatableFields.content;
|
|
1028
|
+
}
|
|
1029
|
+
for (const key in updatableFields) {
|
|
1030
|
+
if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
|
|
1031
|
+
const dbColumn = key === "threadId" ? "thread_id" : key;
|
|
1032
|
+
setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
|
|
1033
|
+
values[`var_${key}_${paramIdx}`] = updatableFields[key];
|
|
1034
|
+
paramIdx++;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
if (setClauses.length > 0) {
|
|
1038
|
+
values[`var_id_${paramIdx}`] = id;
|
|
1039
|
+
const updateQuery = `
|
|
1040
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1041
|
+
UPDATE ${setClauses.join(", ")}
|
|
1042
|
+
WHERE id = {var_id_${paramIdx}:String}
|
|
1043
|
+
`;
|
|
1044
|
+
console.log("Updating message:", id, "with query:", updateQuery, "values:", values);
|
|
1045
|
+
updatePromises.push(
|
|
1046
|
+
this.client.command({
|
|
1047
|
+
query: updateQuery,
|
|
1048
|
+
query_params: values,
|
|
1049
|
+
clickhouse_settings: {
|
|
1050
|
+
date_time_input_format: "best_effort",
|
|
1051
|
+
use_client_time_zone: 1,
|
|
1052
|
+
output_format_json_quote_64bit_integers: 0
|
|
1053
|
+
}
|
|
1054
|
+
})
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
if (updatePromises.length > 0) {
|
|
1059
|
+
await Promise.all(updatePromises);
|
|
1060
|
+
}
|
|
1061
|
+
await this.client.command({
|
|
1062
|
+
query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
|
|
1063
|
+
clickhouse_settings: {
|
|
1064
|
+
date_time_input_format: "best_effort",
|
|
1065
|
+
use_client_time_zone: 1,
|
|
1066
|
+
output_format_json_quote_64bit_integers: 0
|
|
1067
|
+
}
|
|
1068
|
+
});
|
|
1069
|
+
for (const existingMessage of parsedExistingMessages) {
|
|
1070
|
+
const updatePayload = messages.find((m) => m.id === existingMessage.id);
|
|
1071
|
+
if (!updatePayload) continue;
|
|
1072
|
+
const { id, ...fieldsToUpdate } = updatePayload;
|
|
1073
|
+
if (Object.keys(fieldsToUpdate).length === 0) continue;
|
|
1074
|
+
const verifyResult = await this.client.query({
|
|
1075
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1076
|
+
query_params: { messageId: id },
|
|
1077
|
+
clickhouse_settings: {
|
|
1078
|
+
date_time_input_format: "best_effort",
|
|
1079
|
+
date_time_output_format: "iso",
|
|
1080
|
+
use_client_time_zone: 1,
|
|
1081
|
+
output_format_json_quote_64bit_integers: 0
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
const verifyRows = await verifyResult.json();
|
|
1085
|
+
if (verifyRows.data.length > 0) {
|
|
1086
|
+
const updatedMessage = transformRows(verifyRows.data)[0];
|
|
1087
|
+
if (updatedMessage) {
|
|
1088
|
+
let needsRetry = false;
|
|
1089
|
+
for (const [key, value] of Object.entries(fieldsToUpdate)) {
|
|
1090
|
+
if (key === "content") {
|
|
1091
|
+
const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
|
|
1092
|
+
const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
|
|
1093
|
+
if (actualContent !== expectedContent) {
|
|
1094
|
+
needsRetry = true;
|
|
1095
|
+
break;
|
|
1096
|
+
}
|
|
1097
|
+
} else if (updatedMessage[key] !== value) {
|
|
1098
|
+
needsRetry = true;
|
|
1099
|
+
break;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
if (needsRetry) {
|
|
1103
|
+
console.log("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
|
|
1104
|
+
await this.client.command({
|
|
1105
|
+
query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1106
|
+
query_params: { messageId: id },
|
|
1107
|
+
clickhouse_settings: {
|
|
1108
|
+
date_time_input_format: "best_effort",
|
|
1109
|
+
use_client_time_zone: 1,
|
|
1110
|
+
output_format_json_quote_64bit_integers: 0
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
let updatedContent = existingMessage.content || {};
|
|
1114
|
+
if (fieldsToUpdate.content) {
|
|
1115
|
+
const existingContent = existingMessage.content || {};
|
|
1116
|
+
const existingMetadata = existingContent.metadata || {};
|
|
1117
|
+
const updateMetadata = fieldsToUpdate.content.metadata || {};
|
|
1118
|
+
updatedContent = {
|
|
1119
|
+
...existingContent,
|
|
1120
|
+
...fieldsToUpdate.content,
|
|
1121
|
+
metadata: {
|
|
1122
|
+
...existingMetadata,
|
|
1123
|
+
...updateMetadata
|
|
1124
|
+
}
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
const updatedMessageData = {
|
|
1128
|
+
...existingMessage,
|
|
1129
|
+
...fieldsToUpdate,
|
|
1130
|
+
content: updatedContent
|
|
1131
|
+
};
|
|
1132
|
+
await this.client.insert({
|
|
1133
|
+
table: TABLE_MESSAGES,
|
|
1134
|
+
format: "JSONEachRow",
|
|
1135
|
+
values: [
|
|
1136
|
+
{
|
|
1137
|
+
id: updatedMessageData.id,
|
|
1138
|
+
thread_id: updatedMessageData.threadId,
|
|
1139
|
+
resourceId: updatedMessageData.resourceId,
|
|
1140
|
+
content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
|
|
1141
|
+
createdAt: updatedMessageData.createdAt.toISOString(),
|
|
1142
|
+
role: updatedMessageData.role,
|
|
1143
|
+
type: updatedMessageData.type || "v2"
|
|
1144
|
+
}
|
|
1145
|
+
],
|
|
1146
|
+
clickhouse_settings: {
|
|
1147
|
+
date_time_input_format: "best_effort",
|
|
1148
|
+
use_client_time_zone: 1,
|
|
1149
|
+
output_format_json_quote_64bit_integers: 0
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
if (threadIdsToUpdate.size > 0) {
|
|
1157
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1158
|
+
const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
|
|
1159
|
+
const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
|
|
1160
|
+
const threadResult = await this.client.query({
|
|
1161
|
+
query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1162
|
+
query_params: { threadId },
|
|
1163
|
+
clickhouse_settings: {
|
|
1164
|
+
date_time_input_format: "best_effort",
|
|
1165
|
+
date_time_output_format: "iso",
|
|
1166
|
+
use_client_time_zone: 1,
|
|
1167
|
+
output_format_json_quote_64bit_integers: 0
|
|
1168
|
+
}
|
|
1169
|
+
});
|
|
1170
|
+
const threadRows = await threadResult.json();
|
|
1171
|
+
if (threadRows.data.length > 0) {
|
|
1172
|
+
const existingThread = threadRows.data[0];
|
|
1173
|
+
await this.client.command({
|
|
1174
|
+
query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
|
|
1175
|
+
query_params: { threadId },
|
|
1176
|
+
clickhouse_settings: {
|
|
1177
|
+
date_time_input_format: "best_effort",
|
|
1178
|
+
use_client_time_zone: 1,
|
|
1179
|
+
output_format_json_quote_64bit_integers: 0
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
await this.client.insert({
|
|
1183
|
+
table: TABLE_THREADS,
|
|
1184
|
+
format: "JSONEachRow",
|
|
1185
|
+
values: [
|
|
1186
|
+
{
|
|
1187
|
+
id: existingThread.id,
|
|
1188
|
+
resourceId: existingThread.resourceId,
|
|
1189
|
+
title: existingThread.title,
|
|
1190
|
+
metadata: existingThread.metadata,
|
|
1191
|
+
createdAt: existingThread.createdAt,
|
|
1192
|
+
updatedAt: now
|
|
1193
|
+
}
|
|
1194
|
+
],
|
|
1195
|
+
clickhouse_settings: {
|
|
1196
|
+
date_time_input_format: "best_effort",
|
|
1197
|
+
use_client_time_zone: 1,
|
|
1198
|
+
output_format_json_quote_64bit_integers: 0
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
});
|
|
1203
|
+
await Promise.all(threadUpdatePromises);
|
|
1204
|
+
}
|
|
1205
|
+
const updatedMessages = [];
|
|
1206
|
+
for (const messageId of messageIds) {
|
|
1207
|
+
const updatedResult = await this.client.query({
|
|
1208
|
+
query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
|
|
1209
|
+
query_params: { messageId },
|
|
1210
|
+
clickhouse_settings: {
|
|
1211
|
+
date_time_input_format: "best_effort",
|
|
1212
|
+
date_time_output_format: "iso",
|
|
1213
|
+
use_client_time_zone: 1,
|
|
1214
|
+
output_format_json_quote_64bit_integers: 0
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
const updatedRows = await updatedResult.json();
|
|
1218
|
+
if (updatedRows.data.length > 0) {
|
|
1219
|
+
const message = transformRows(updatedRows.data)[0];
|
|
1220
|
+
if (message) {
|
|
1221
|
+
updatedMessages.push(message);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return updatedMessages.map((message) => {
|
|
1226
|
+
if (typeof message.content === "string") {
|
|
1227
|
+
try {
|
|
1228
|
+
message.content = JSON.parse(message.content);
|
|
1229
|
+
} catch {
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
return message;
|
|
1233
|
+
});
|
|
1234
|
+
} catch (error) {
|
|
1235
|
+
throw new MastraError(
|
|
1236
|
+
{
|
|
1237
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
|
|
1238
|
+
domain: ErrorDomain.STORAGE,
|
|
1239
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1240
|
+
details: { messageIds: messages.map((m) => m.id).join(",") }
|
|
1241
|
+
},
|
|
1242
|
+
error
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async getResourceById({ resourceId }) {
|
|
1247
|
+
try {
|
|
1248
|
+
const result = await this.client.query({
|
|
1249
|
+
query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
|
|
1250
|
+
query_params: { resourceId },
|
|
1251
|
+
clickhouse_settings: {
|
|
1252
|
+
date_time_input_format: "best_effort",
|
|
1253
|
+
date_time_output_format: "iso",
|
|
1254
|
+
use_client_time_zone: 1,
|
|
1255
|
+
output_format_json_quote_64bit_integers: 0
|
|
1256
|
+
}
|
|
1257
|
+
});
|
|
1258
|
+
const rows = await result.json();
|
|
1259
|
+
if (rows.data.length === 0) {
|
|
1260
|
+
return null;
|
|
1261
|
+
}
|
|
1262
|
+
const resource = rows.data[0];
|
|
1263
|
+
return {
|
|
1264
|
+
id: resource.id,
|
|
1265
|
+
workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
|
|
1266
|
+
metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
|
|
1267
|
+
createdAt: new Date(resource.createdAt),
|
|
1268
|
+
updatedAt: new Date(resource.updatedAt)
|
|
1269
|
+
};
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
throw new MastraError(
|
|
1272
|
+
{
|
|
1273
|
+
id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
|
|
1274
|
+
domain: ErrorDomain.STORAGE,
|
|
1275
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1276
|
+
details: { resourceId }
|
|
1277
|
+
},
|
|
1278
|
+
error
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
async saveResource({ resource }) {
|
|
1283
|
+
try {
|
|
1284
|
+
await this.client.insert({
|
|
1285
|
+
table: TABLE_RESOURCES,
|
|
1286
|
+
format: "JSONEachRow",
|
|
1287
|
+
values: [
|
|
1288
|
+
{
|
|
1289
|
+
id: resource.id,
|
|
1290
|
+
workingMemory: resource.workingMemory,
|
|
1291
|
+
metadata: JSON.stringify(resource.metadata),
|
|
1292
|
+
createdAt: resource.createdAt.toISOString(),
|
|
1293
|
+
updatedAt: resource.updatedAt.toISOString()
|
|
1294
|
+
}
|
|
1295
|
+
],
|
|
1296
|
+
clickhouse_settings: {
|
|
1297
|
+
date_time_input_format: "best_effort",
|
|
1298
|
+
use_client_time_zone: 1,
|
|
1299
|
+
output_format_json_quote_64bit_integers: 0
|
|
1300
|
+
}
|
|
1301
|
+
});
|
|
1302
|
+
return resource;
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
throw new MastraError(
|
|
1305
|
+
{
|
|
1306
|
+
id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
|
|
1307
|
+
domain: ErrorDomain.STORAGE,
|
|
1308
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1309
|
+
details: { resourceId: resource.id }
|
|
1310
|
+
},
|
|
1311
|
+
error
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
async updateResource({
|
|
1316
|
+
resourceId,
|
|
1317
|
+
workingMemory,
|
|
1318
|
+
metadata
|
|
1319
|
+
}) {
|
|
1320
|
+
try {
|
|
1321
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1322
|
+
if (!existingResource) {
|
|
1323
|
+
const newResource = {
|
|
1324
|
+
id: resourceId,
|
|
1325
|
+
workingMemory,
|
|
1326
|
+
metadata: metadata || {},
|
|
1327
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1328
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1329
|
+
};
|
|
1330
|
+
return this.saveResource({ resource: newResource });
|
|
1331
|
+
}
|
|
1332
|
+
const updatedResource = {
|
|
1333
|
+
...existingResource,
|
|
1334
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1335
|
+
metadata: {
|
|
1336
|
+
...existingResource.metadata,
|
|
1337
|
+
...metadata
|
|
1338
|
+
},
|
|
1339
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1340
|
+
};
|
|
1341
|
+
const updateQuery = `
|
|
1342
|
+
ALTER TABLE ${TABLE_RESOURCES}
|
|
1343
|
+
UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
|
|
1344
|
+
WHERE id = {resourceId:String}
|
|
1345
|
+
`;
|
|
1346
|
+
await this.client.command({
|
|
1347
|
+
query: updateQuery,
|
|
1348
|
+
query_params: {
|
|
1349
|
+
workingMemory: updatedResource.workingMemory,
|
|
1350
|
+
metadata: JSON.stringify(updatedResource.metadata),
|
|
1351
|
+
updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
|
|
1352
|
+
resourceId
|
|
1353
|
+
},
|
|
1354
|
+
clickhouse_settings: {
|
|
1355
|
+
date_time_input_format: "best_effort",
|
|
1356
|
+
use_client_time_zone: 1,
|
|
1357
|
+
output_format_json_quote_64bit_integers: 0
|
|
1358
|
+
}
|
|
1359
|
+
});
|
|
1360
|
+
await this.client.command({
|
|
1361
|
+
query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
|
|
1362
|
+
clickhouse_settings: {
|
|
1363
|
+
date_time_input_format: "best_effort",
|
|
1364
|
+
use_client_time_zone: 1,
|
|
1365
|
+
output_format_json_quote_64bit_integers: 0
|
|
1366
|
+
}
|
|
1367
|
+
});
|
|
1368
|
+
return updatedResource;
|
|
1369
|
+
} catch (error) {
|
|
1370
|
+
throw new MastraError(
|
|
1371
|
+
{
|
|
1372
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
|
|
1373
|
+
domain: ErrorDomain.STORAGE,
|
|
1374
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1375
|
+
details: { resourceId }
|
|
1376
|
+
},
|
|
1377
|
+
error
|
|
1378
|
+
);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
};
|
|
1382
|
+
var StoreOperationsClickhouse = class extends StoreOperations {
|
|
1383
|
+
ttl;
|
|
1384
|
+
client;
|
|
1385
|
+
constructor({ client, ttl }) {
|
|
1386
|
+
super();
|
|
1387
|
+
this.ttl = ttl;
|
|
1388
|
+
this.client = client;
|
|
1389
|
+
}
|
|
1390
|
+
async hasColumn(table, column) {
|
|
1391
|
+
const result = await this.client.query({
|
|
1392
|
+
query: `DESCRIBE TABLE ${table}`,
|
|
1393
|
+
format: "JSONEachRow"
|
|
1394
|
+
});
|
|
1395
|
+
const columns = await result.json();
|
|
1396
|
+
return columns.some((c) => c.name === column);
|
|
1397
|
+
}
|
|
1398
|
+
getSqlType(type) {
|
|
1399
|
+
switch (type) {
|
|
1400
|
+
case "text":
|
|
1401
|
+
return "String";
|
|
1402
|
+
case "timestamp":
|
|
1403
|
+
return "DateTime64(3)";
|
|
1404
|
+
case "integer":
|
|
1405
|
+
case "bigint":
|
|
1406
|
+
return "Int64";
|
|
1407
|
+
case "jsonb":
|
|
1408
|
+
return "String";
|
|
1409
|
+
default:
|
|
1410
|
+
return super.getSqlType(type);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
async createTable({
|
|
1414
|
+
tableName,
|
|
1415
|
+
schema
|
|
1416
|
+
}) {
|
|
1417
|
+
try {
|
|
1418
|
+
const columns = Object.entries(schema).map(([name, def]) => {
|
|
1419
|
+
const constraints = [];
|
|
1420
|
+
if (!def.nullable) constraints.push("NOT NULL");
|
|
1421
|
+
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
1422
|
+
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
|
|
1423
|
+
}).join(",\n");
|
|
1424
|
+
const rowTtl = this.ttl?.[tableName]?.row;
|
|
1425
|
+
const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
|
|
1426
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1427
|
+
${["id String"].concat(columns)}
|
|
1428
|
+
)
|
|
1429
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1430
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
1431
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
1432
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
1433
|
+
SETTINGS index_granularity = 8192
|
|
1434
|
+
` : `
|
|
1435
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
1436
|
+
${columns}
|
|
1437
|
+
)
|
|
1438
|
+
ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
|
|
1439
|
+
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1440
|
+
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
1441
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
1442
|
+
SETTINGS index_granularity = 8192
|
|
1443
|
+
`;
|
|
1444
|
+
await this.client.query({
|
|
1445
|
+
query: sql,
|
|
1446
|
+
clickhouse_settings: {
|
|
1447
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1448
|
+
date_time_input_format: "best_effort",
|
|
1449
|
+
date_time_output_format: "iso",
|
|
1450
|
+
use_client_time_zone: 1,
|
|
1451
|
+
output_format_json_quote_64bit_integers: 0
|
|
1452
|
+
}
|
|
1453
|
+
});
|
|
1454
|
+
} catch (error) {
|
|
1455
|
+
throw new MastraError(
|
|
1456
|
+
{
|
|
1457
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
1458
|
+
domain: ErrorDomain.STORAGE,
|
|
1459
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1460
|
+
details: { tableName }
|
|
1461
|
+
},
|
|
1462
|
+
error
|
|
1463
|
+
);
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
async alterTable({
|
|
1467
|
+
tableName,
|
|
1468
|
+
schema,
|
|
1469
|
+
ifNotExists
|
|
1470
|
+
}) {
|
|
1471
|
+
try {
|
|
1472
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
1473
|
+
const result = await this.client.query({
|
|
1474
|
+
query: describeSql
|
|
1475
|
+
});
|
|
1476
|
+
const rows = await result.json();
|
|
1477
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
1478
|
+
for (const columnName of ifNotExists) {
|
|
1479
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
1480
|
+
const columnDef = schema[columnName];
|
|
1481
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
1482
|
+
if (columnDef.nullable !== false) {
|
|
1483
|
+
sqlType = `Nullable(${sqlType})`;
|
|
1484
|
+
}
|
|
1485
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
1486
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
1487
|
+
await this.client.query({
|
|
1488
|
+
query: alterSql
|
|
1489
|
+
});
|
|
1490
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
throw new MastraError(
|
|
1495
|
+
{
|
|
1496
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
1497
|
+
domain: ErrorDomain.STORAGE,
|
|
1498
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1499
|
+
details: { tableName }
|
|
1500
|
+
},
|
|
1501
|
+
error
|
|
1502
|
+
);
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
async clearTable({ tableName }) {
|
|
1506
|
+
try {
|
|
1507
|
+
await this.client.query({
|
|
1508
|
+
query: `TRUNCATE TABLE ${tableName}`,
|
|
1509
|
+
clickhouse_settings: {
|
|
1510
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1511
|
+
date_time_input_format: "best_effort",
|
|
1512
|
+
date_time_output_format: "iso",
|
|
1513
|
+
use_client_time_zone: 1,
|
|
1514
|
+
output_format_json_quote_64bit_integers: 0
|
|
1515
|
+
}
|
|
1516
|
+
});
|
|
1517
|
+
} catch (error) {
|
|
1518
|
+
throw new MastraError(
|
|
1519
|
+
{
|
|
1520
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1521
|
+
domain: ErrorDomain.STORAGE,
|
|
1522
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1523
|
+
details: { tableName }
|
|
1524
|
+
},
|
|
1525
|
+
error
|
|
1526
|
+
);
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
async dropTable({ tableName }) {
|
|
1530
|
+
await this.client.query({
|
|
1531
|
+
query: `DROP TABLE IF EXISTS ${tableName}`
|
|
1532
|
+
});
|
|
340
1533
|
}
|
|
341
1534
|
async insert({ tableName, record }) {
|
|
1535
|
+
const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
|
|
1536
|
+
const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
|
|
342
1537
|
try {
|
|
343
|
-
await this.
|
|
1538
|
+
const result = await this.client.insert({
|
|
344
1539
|
table: tableName,
|
|
345
1540
|
values: [
|
|
346
1541
|
{
|
|
347
1542
|
...record,
|
|
348
|
-
createdAt
|
|
349
|
-
updatedAt
|
|
1543
|
+
createdAt,
|
|
1544
|
+
updatedAt
|
|
350
1545
|
}
|
|
351
1546
|
],
|
|
352
1547
|
format: "JSONEachRow",
|
|
@@ -357,13 +1552,55 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
357
1552
|
use_client_time_zone: 1
|
|
358
1553
|
}
|
|
359
1554
|
});
|
|
1555
|
+
console.log("INSERT RESULT", result);
|
|
360
1556
|
} catch (error) {
|
|
361
|
-
|
|
362
|
-
|
|
1557
|
+
throw new MastraError(
|
|
1558
|
+
{
|
|
1559
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
1560
|
+
domain: ErrorDomain.STORAGE,
|
|
1561
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1562
|
+
details: { tableName }
|
|
1563
|
+
},
|
|
1564
|
+
error
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
async batchInsert({ tableName, records }) {
|
|
1569
|
+
const recordsToBeInserted = records.map((record) => ({
|
|
1570
|
+
...Object.fromEntries(
|
|
1571
|
+
Object.entries(record).map(([key, value]) => [
|
|
1572
|
+
key,
|
|
1573
|
+
TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
1574
|
+
])
|
|
1575
|
+
)
|
|
1576
|
+
}));
|
|
1577
|
+
try {
|
|
1578
|
+
await this.client.insert({
|
|
1579
|
+
table: tableName,
|
|
1580
|
+
values: recordsToBeInserted,
|
|
1581
|
+
format: "JSONEachRow",
|
|
1582
|
+
clickhouse_settings: {
|
|
1583
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1584
|
+
date_time_input_format: "best_effort",
|
|
1585
|
+
use_client_time_zone: 1,
|
|
1586
|
+
output_format_json_quote_64bit_integers: 0
|
|
1587
|
+
}
|
|
1588
|
+
});
|
|
1589
|
+
} catch (error) {
|
|
1590
|
+
throw new MastraError(
|
|
1591
|
+
{
|
|
1592
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
1593
|
+
domain: ErrorDomain.STORAGE,
|
|
1594
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1595
|
+
details: { tableName }
|
|
1596
|
+
},
|
|
1597
|
+
error
|
|
1598
|
+
);
|
|
363
1599
|
}
|
|
364
1600
|
}
|
|
365
1601
|
async load({ tableName, keys }) {
|
|
366
1602
|
try {
|
|
1603
|
+
const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
|
|
367
1604
|
const keyEntries = Object.entries(keys);
|
|
368
1605
|
const conditions = keyEntries.map(
|
|
369
1606
|
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
@@ -371,8 +1608,10 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
371
1608
|
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
372
1609
|
return { ...acc, [`var_${key}`]: value };
|
|
373
1610
|
}, {});
|
|
374
|
-
const
|
|
375
|
-
|
|
1611
|
+
const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
|
|
1612
|
+
const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
|
|
1613
|
+
const result = await this.client.query({
|
|
1614
|
+
query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
376
1615
|
query_params: values,
|
|
377
1616
|
clickhouse_settings: {
|
|
378
1617
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -399,24 +1638,57 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
399
1638
|
const data = transformRow(rows.data[0]);
|
|
400
1639
|
return data;
|
|
401
1640
|
} catch (error) {
|
|
402
|
-
|
|
403
|
-
|
|
1641
|
+
throw new MastraError(
|
|
1642
|
+
{
|
|
1643
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
1644
|
+
domain: ErrorDomain.STORAGE,
|
|
1645
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1646
|
+
details: { tableName }
|
|
1647
|
+
},
|
|
1648
|
+
error
|
|
1649
|
+
);
|
|
404
1650
|
}
|
|
405
1651
|
}
|
|
406
|
-
|
|
1652
|
+
};
|
|
1653
|
+
var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
1654
|
+
client;
|
|
1655
|
+
operations;
|
|
1656
|
+
constructor({ client, operations }) {
|
|
1657
|
+
super();
|
|
1658
|
+
this.client = client;
|
|
1659
|
+
this.operations = operations;
|
|
1660
|
+
}
|
|
1661
|
+
transformScoreRow(row) {
|
|
1662
|
+
const scorer = safelyParseJSON(row.scorer);
|
|
1663
|
+
const extractStepResult = safelyParseJSON(row.extractStepResult);
|
|
1664
|
+
const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
|
|
1665
|
+
const metadata = safelyParseJSON(row.metadata);
|
|
1666
|
+
const input = safelyParseJSON(row.input);
|
|
1667
|
+
const output = safelyParseJSON(row.output);
|
|
1668
|
+
const additionalContext = safelyParseJSON(row.additionalContext);
|
|
1669
|
+
const runtimeContext = safelyParseJSON(row.runtimeContext);
|
|
1670
|
+
const entity = safelyParseJSON(row.entity);
|
|
1671
|
+
return {
|
|
1672
|
+
...row,
|
|
1673
|
+
scorer,
|
|
1674
|
+
extractStepResult,
|
|
1675
|
+
analyzeStepResult,
|
|
1676
|
+
metadata,
|
|
1677
|
+
input,
|
|
1678
|
+
output,
|
|
1679
|
+
additionalContext,
|
|
1680
|
+
runtimeContext,
|
|
1681
|
+
entity,
|
|
1682
|
+
createdAt: new Date(row.createdAt),
|
|
1683
|
+
updatedAt: new Date(row.updatedAt)
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
async getScoreById({ id }) {
|
|
407
1687
|
try {
|
|
408
|
-
const result = await this.
|
|
409
|
-
query: `SELECT
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
title,
|
|
413
|
-
metadata,
|
|
414
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
415
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
416
|
-
FROM "${TABLE_THREADS}"
|
|
417
|
-
FINAL
|
|
418
|
-
WHERE id = {var_id:String}`,
|
|
419
|
-
query_params: { var_id: threadId },
|
|
1688
|
+
const result = await this.client.query({
|
|
1689
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
|
|
1690
|
+
query_params: { var_id: id },
|
|
1691
|
+
format: "JSONEachRow",
|
|
420
1692
|
clickhouse_settings: {
|
|
421
1693
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
422
1694
|
date_time_input_format: "best_effort",
|
|
@@ -425,37 +1697,88 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
425
1697
|
output_format_json_quote_64bit_integers: 0
|
|
426
1698
|
}
|
|
427
1699
|
});
|
|
428
|
-
const
|
|
429
|
-
|
|
430
|
-
if (!thread) {
|
|
1700
|
+
const resultJson = await result.json();
|
|
1701
|
+
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
431
1702
|
return null;
|
|
432
1703
|
}
|
|
433
|
-
return
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
1704
|
+
return this.transformScoreRow(resultJson[0]);
|
|
1705
|
+
} catch (error) {
|
|
1706
|
+
throw new MastraError(
|
|
1707
|
+
{
|
|
1708
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1709
|
+
domain: ErrorDomain.STORAGE,
|
|
1710
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1711
|
+
details: { scoreId: id }
|
|
1712
|
+
},
|
|
1713
|
+
error
|
|
1714
|
+
);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
async saveScore(score) {
|
|
1718
|
+
try {
|
|
1719
|
+
const record = {
|
|
1720
|
+
...score
|
|
438
1721
|
};
|
|
1722
|
+
await this.client.insert({
|
|
1723
|
+
table: TABLE_SCORERS,
|
|
1724
|
+
values: [record],
|
|
1725
|
+
format: "JSONEachRow",
|
|
1726
|
+
clickhouse_settings: {
|
|
1727
|
+
date_time_input_format: "best_effort",
|
|
1728
|
+
use_client_time_zone: 1,
|
|
1729
|
+
output_format_json_quote_64bit_integers: 0
|
|
1730
|
+
}
|
|
1731
|
+
});
|
|
1732
|
+
return { score };
|
|
439
1733
|
} catch (error) {
|
|
440
|
-
|
|
441
|
-
|
|
1734
|
+
throw new MastraError(
|
|
1735
|
+
{
|
|
1736
|
+
id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
|
|
1737
|
+
domain: ErrorDomain.STORAGE,
|
|
1738
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1739
|
+
details: { scoreId: score.id }
|
|
1740
|
+
},
|
|
1741
|
+
error
|
|
1742
|
+
);
|
|
442
1743
|
}
|
|
443
1744
|
}
|
|
444
|
-
async
|
|
1745
|
+
async getScoresByRunId({
|
|
1746
|
+
runId,
|
|
1747
|
+
pagination
|
|
1748
|
+
}) {
|
|
445
1749
|
try {
|
|
446
|
-
const
|
|
447
|
-
query: `SELECT
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
1750
|
+
const countResult = await this.client.query({
|
|
1751
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
|
|
1752
|
+
query_params: { var_runId: runId },
|
|
1753
|
+
format: "JSONEachRow"
|
|
1754
|
+
});
|
|
1755
|
+
const countRows = await countResult.json();
|
|
1756
|
+
let total = 0;
|
|
1757
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1758
|
+
const countObj = countRows[0];
|
|
1759
|
+
total = Number(countObj.count);
|
|
1760
|
+
}
|
|
1761
|
+
if (!total) {
|
|
1762
|
+
return {
|
|
1763
|
+
pagination: {
|
|
1764
|
+
total: 0,
|
|
1765
|
+
page: pagination.page,
|
|
1766
|
+
perPage: pagination.perPage,
|
|
1767
|
+
hasMore: false
|
|
1768
|
+
},
|
|
1769
|
+
scores: []
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
const offset = pagination.page * pagination.perPage;
|
|
1773
|
+
const result = await this.client.query({
|
|
1774
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1775
|
+
query_params: {
|
|
1776
|
+
var_runId: runId,
|
|
1777
|
+
var_limit: pagination.perPage,
|
|
1778
|
+
var_offset: offset
|
|
1779
|
+
},
|
|
1780
|
+
format: "JSONEachRow",
|
|
457
1781
|
clickhouse_settings: {
|
|
458
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
459
1782
|
date_time_input_format: "best_effort",
|
|
460
1783
|
date_time_output_format: "iso",
|
|
461
1784
|
use_client_time_zone: 1,
|
|
@@ -463,191 +1786,349 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
463
1786
|
}
|
|
464
1787
|
});
|
|
465
1788
|
const rows = await result.json();
|
|
466
|
-
const
|
|
467
|
-
return
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
1789
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1790
|
+
return {
|
|
1791
|
+
pagination: {
|
|
1792
|
+
total,
|
|
1793
|
+
page: pagination.page,
|
|
1794
|
+
perPage: pagination.perPage,
|
|
1795
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1796
|
+
},
|
|
1797
|
+
scores
|
|
1798
|
+
};
|
|
473
1799
|
} catch (error) {
|
|
474
|
-
|
|
475
|
-
|
|
1800
|
+
throw new MastraError(
|
|
1801
|
+
{
|
|
1802
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1803
|
+
domain: ErrorDomain.STORAGE,
|
|
1804
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1805
|
+
details: { runId }
|
|
1806
|
+
},
|
|
1807
|
+
error
|
|
1808
|
+
);
|
|
476
1809
|
}
|
|
477
1810
|
}
|
|
478
|
-
async
|
|
1811
|
+
async getScoresByScorerId({
|
|
1812
|
+
scorerId,
|
|
1813
|
+
pagination
|
|
1814
|
+
}) {
|
|
479
1815
|
try {
|
|
480
|
-
await this.
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
]
|
|
1816
|
+
const countResult = await this.client.query({
|
|
1817
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String}`,
|
|
1818
|
+
query_params: { var_scorerId: scorerId },
|
|
1819
|
+
format: "JSONEachRow"
|
|
1820
|
+
});
|
|
1821
|
+
const countRows = await countResult.json();
|
|
1822
|
+
let total = 0;
|
|
1823
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1824
|
+
const countObj = countRows[0];
|
|
1825
|
+
total = Number(countObj.count);
|
|
1826
|
+
}
|
|
1827
|
+
if (!total) {
|
|
1828
|
+
return {
|
|
1829
|
+
pagination: {
|
|
1830
|
+
total: 0,
|
|
1831
|
+
page: pagination.page,
|
|
1832
|
+
perPage: pagination.perPage,
|
|
1833
|
+
hasMore: false
|
|
1834
|
+
},
|
|
1835
|
+
scores: []
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
const offset = pagination.page * pagination.perPage;
|
|
1839
|
+
const result = await this.client.query({
|
|
1840
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE scorerId = {var_scorerId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1841
|
+
query_params: {
|
|
1842
|
+
var_scorerId: scorerId,
|
|
1843
|
+
var_limit: pagination.perPage,
|
|
1844
|
+
var_offset: offset
|
|
1845
|
+
},
|
|
489
1846
|
format: "JSONEachRow",
|
|
490
1847
|
clickhouse_settings: {
|
|
491
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
492
1848
|
date_time_input_format: "best_effort",
|
|
1849
|
+
date_time_output_format: "iso",
|
|
493
1850
|
use_client_time_zone: 1,
|
|
494
1851
|
output_format_json_quote_64bit_integers: 0
|
|
495
1852
|
}
|
|
496
1853
|
});
|
|
497
|
-
|
|
1854
|
+
const rows = await result.json();
|
|
1855
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1856
|
+
return {
|
|
1857
|
+
pagination: {
|
|
1858
|
+
total,
|
|
1859
|
+
page: pagination.page,
|
|
1860
|
+
perPage: pagination.perPage,
|
|
1861
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1862
|
+
},
|
|
1863
|
+
scores
|
|
1864
|
+
};
|
|
498
1865
|
} catch (error) {
|
|
499
|
-
|
|
500
|
-
|
|
1866
|
+
throw new MastraError(
|
|
1867
|
+
{
|
|
1868
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1869
|
+
domain: ErrorDomain.STORAGE,
|
|
1870
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1871
|
+
details: { scorerId }
|
|
1872
|
+
},
|
|
1873
|
+
error
|
|
1874
|
+
);
|
|
501
1875
|
}
|
|
502
1876
|
}
|
|
503
|
-
async
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
1877
|
+
async getScoresByEntityId({
|
|
1878
|
+
entityId,
|
|
1879
|
+
entityType,
|
|
1880
|
+
pagination
|
|
507
1881
|
}) {
|
|
508
1882
|
try {
|
|
509
|
-
const
|
|
510
|
-
|
|
511
|
-
|
|
1883
|
+
const countResult = await this.client.query({
|
|
1884
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
|
|
1885
|
+
query_params: { var_entityId: entityId, var_entityType: entityType },
|
|
1886
|
+
format: "JSONEachRow"
|
|
1887
|
+
});
|
|
1888
|
+
const countRows = await countResult.json();
|
|
1889
|
+
let total = 0;
|
|
1890
|
+
if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
|
|
1891
|
+
const countObj = countRows[0];
|
|
1892
|
+
total = Number(countObj.count);
|
|
512
1893
|
}
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
1894
|
+
if (!total) {
|
|
1895
|
+
return {
|
|
1896
|
+
pagination: {
|
|
1897
|
+
total: 0,
|
|
1898
|
+
page: pagination.page,
|
|
1899
|
+
perPage: pagination.perPage,
|
|
1900
|
+
hasMore: false
|
|
1901
|
+
},
|
|
1902
|
+
scores: []
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
const offset = pagination.page * pagination.perPage;
|
|
1906
|
+
const result = await this.client.query({
|
|
1907
|
+
query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
|
|
1908
|
+
query_params: {
|
|
1909
|
+
var_entityId: entityId,
|
|
1910
|
+
var_entityType: entityType,
|
|
1911
|
+
var_limit: pagination.perPage,
|
|
1912
|
+
var_offset: offset
|
|
1913
|
+
},
|
|
525
1914
|
format: "JSONEachRow",
|
|
526
|
-
values: [
|
|
527
|
-
{
|
|
528
|
-
id: updatedThread.id,
|
|
529
|
-
resourceId: updatedThread.resourceId,
|
|
530
|
-
title: updatedThread.title,
|
|
531
|
-
metadata: updatedThread.metadata,
|
|
532
|
-
createdAt: updatedThread.createdAt,
|
|
533
|
-
updatedAt: updatedThread.updatedAt.toISOString()
|
|
534
|
-
}
|
|
535
|
-
],
|
|
536
1915
|
clickhouse_settings: {
|
|
537
1916
|
date_time_input_format: "best_effort",
|
|
1917
|
+
date_time_output_format: "iso",
|
|
538
1918
|
use_client_time_zone: 1,
|
|
539
1919
|
output_format_json_quote_64bit_integers: 0
|
|
540
1920
|
}
|
|
541
1921
|
});
|
|
542
|
-
|
|
1922
|
+
const rows = await result.json();
|
|
1923
|
+
const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
|
|
1924
|
+
return {
|
|
1925
|
+
pagination: {
|
|
1926
|
+
total,
|
|
1927
|
+
page: pagination.page,
|
|
1928
|
+
perPage: pagination.perPage,
|
|
1929
|
+
hasMore: total > (pagination.page + 1) * pagination.perPage
|
|
1930
|
+
},
|
|
1931
|
+
scores
|
|
1932
|
+
};
|
|
543
1933
|
} catch (error) {
|
|
544
|
-
|
|
545
|
-
|
|
1934
|
+
throw new MastraError(
|
|
1935
|
+
{
|
|
1936
|
+
id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1937
|
+
domain: ErrorDomain.STORAGE,
|
|
1938
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1939
|
+
details: { entityId, entityType }
|
|
1940
|
+
},
|
|
1941
|
+
error
|
|
1942
|
+
);
|
|
546
1943
|
}
|
|
547
1944
|
}
|
|
548
|
-
|
|
1945
|
+
};
|
|
1946
|
+
var TracesStorageClickhouse = class extends TracesStorage {
|
|
1947
|
+
client;
|
|
1948
|
+
operations;
|
|
1949
|
+
constructor({ client, operations }) {
|
|
1950
|
+
super();
|
|
1951
|
+
this.client = client;
|
|
1952
|
+
this.operations = operations;
|
|
1953
|
+
}
|
|
1954
|
+
async getTracesPaginated(args) {
|
|
1955
|
+
const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
|
|
1956
|
+
const fromDate = dateRange?.start;
|
|
1957
|
+
const toDate = dateRange?.end;
|
|
1958
|
+
const currentOffset = page * perPage;
|
|
1959
|
+
const queryArgs = {};
|
|
1960
|
+
const conditions = [];
|
|
1961
|
+
if (name) {
|
|
1962
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
1963
|
+
queryArgs.var_name = name;
|
|
1964
|
+
}
|
|
1965
|
+
if (scope) {
|
|
1966
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
1967
|
+
queryArgs.var_scope = scope;
|
|
1968
|
+
}
|
|
1969
|
+
if (attributes) {
|
|
1970
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
1971
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
1972
|
+
queryArgs[`var_attr_${key}`] = value;
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
if (filters) {
|
|
1976
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
1977
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
1978
|
+
queryArgs[`var_col_${key}`] = value;
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
if (fromDate) {
|
|
1982
|
+
conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
|
|
1983
|
+
queryArgs.var_from_date = fromDate.toISOString();
|
|
1984
|
+
}
|
|
1985
|
+
if (toDate) {
|
|
1986
|
+
conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
|
|
1987
|
+
queryArgs.var_to_date = toDate.toISOString();
|
|
1988
|
+
}
|
|
1989
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
549
1990
|
try {
|
|
550
|
-
await this.
|
|
551
|
-
query: `
|
|
552
|
-
query_params:
|
|
1991
|
+
const countResult = await this.client.query({
|
|
1992
|
+
query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
|
|
1993
|
+
query_params: queryArgs,
|
|
553
1994
|
clickhouse_settings: {
|
|
1995
|
+
date_time_input_format: "best_effort",
|
|
1996
|
+
date_time_output_format: "iso",
|
|
1997
|
+
use_client_time_zone: 1,
|
|
554
1998
|
output_format_json_quote_64bit_integers: 0
|
|
555
1999
|
}
|
|
556
2000
|
});
|
|
557
|
-
await
|
|
558
|
-
|
|
559
|
-
|
|
2001
|
+
const countData = await countResult.json();
|
|
2002
|
+
const total = Number(countData.data?.[0]?.count ?? 0);
|
|
2003
|
+
if (total === 0) {
|
|
2004
|
+
return {
|
|
2005
|
+
traces: [],
|
|
2006
|
+
total: 0,
|
|
2007
|
+
page,
|
|
2008
|
+
perPage,
|
|
2009
|
+
hasMore: false
|
|
2010
|
+
};
|
|
2011
|
+
}
|
|
2012
|
+
const result = await this.client.query({
|
|
2013
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
|
|
2014
|
+
query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
|
|
560
2015
|
clickhouse_settings: {
|
|
2016
|
+
date_time_input_format: "best_effort",
|
|
2017
|
+
date_time_output_format: "iso",
|
|
2018
|
+
use_client_time_zone: 1,
|
|
561
2019
|
output_format_json_quote_64bit_integers: 0
|
|
562
2020
|
}
|
|
563
2021
|
});
|
|
2022
|
+
if (!result) {
|
|
2023
|
+
return {
|
|
2024
|
+
traces: [],
|
|
2025
|
+
total,
|
|
2026
|
+
page,
|
|
2027
|
+
perPage,
|
|
2028
|
+
hasMore: false
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
const resp = await result.json();
|
|
2032
|
+
const rows = resp.data;
|
|
2033
|
+
const traces = rows.map((row) => ({
|
|
2034
|
+
id: row.id,
|
|
2035
|
+
parentSpanId: row.parentSpanId,
|
|
2036
|
+
traceId: row.traceId,
|
|
2037
|
+
name: row.name,
|
|
2038
|
+
scope: row.scope,
|
|
2039
|
+
kind: row.kind,
|
|
2040
|
+
status: safelyParseJSON(row.status),
|
|
2041
|
+
events: safelyParseJSON(row.events),
|
|
2042
|
+
links: safelyParseJSON(row.links),
|
|
2043
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2044
|
+
startTime: row.startTime,
|
|
2045
|
+
endTime: row.endTime,
|
|
2046
|
+
other: safelyParseJSON(row.other),
|
|
2047
|
+
createdAt: row.createdAt
|
|
2048
|
+
}));
|
|
2049
|
+
return {
|
|
2050
|
+
traces,
|
|
2051
|
+
total,
|
|
2052
|
+
page,
|
|
2053
|
+
perPage,
|
|
2054
|
+
hasMore: currentOffset + traces.length < total
|
|
2055
|
+
};
|
|
564
2056
|
} catch (error) {
|
|
565
|
-
|
|
566
|
-
|
|
2057
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2058
|
+
return {
|
|
2059
|
+
traces: [],
|
|
2060
|
+
total: 0,
|
|
2061
|
+
page,
|
|
2062
|
+
perPage,
|
|
2063
|
+
hasMore: false
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
throw new MastraError(
|
|
2067
|
+
{
|
|
2068
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
2069
|
+
domain: ErrorDomain.STORAGE,
|
|
2070
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2071
|
+
details: {
|
|
2072
|
+
name: name ?? null,
|
|
2073
|
+
scope: scope ?? null,
|
|
2074
|
+
page,
|
|
2075
|
+
perPage,
|
|
2076
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2077
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2078
|
+
dateRange: dateRange ? JSON.stringify(dateRange) : null
|
|
2079
|
+
}
|
|
2080
|
+
},
|
|
2081
|
+
error
|
|
2082
|
+
);
|
|
567
2083
|
}
|
|
568
2084
|
}
|
|
569
|
-
async
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
2085
|
+
async getTraces({
|
|
2086
|
+
name,
|
|
2087
|
+
scope,
|
|
2088
|
+
page,
|
|
2089
|
+
perPage,
|
|
2090
|
+
attributes,
|
|
2091
|
+
filters,
|
|
2092
|
+
fromDate,
|
|
2093
|
+
toDate
|
|
574
2094
|
}) {
|
|
2095
|
+
const limit = perPage;
|
|
2096
|
+
const offset = page * perPage;
|
|
2097
|
+
const args = {};
|
|
2098
|
+
const conditions = [];
|
|
2099
|
+
if (name) {
|
|
2100
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
2101
|
+
args.var_name = name;
|
|
2102
|
+
}
|
|
2103
|
+
if (scope) {
|
|
2104
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
2105
|
+
args.var_scope = scope;
|
|
2106
|
+
}
|
|
2107
|
+
if (attributes) {
|
|
2108
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
2109
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
2110
|
+
args[`var_attr_${key}`] = value;
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
if (filters) {
|
|
2114
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
2115
|
+
conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
|
|
2116
|
+
args[`var_col_${key}`] = value;
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
if (fromDate) {
|
|
2120
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
2121
|
+
args.var_from_date = fromDate.getTime() / 1e3;
|
|
2122
|
+
}
|
|
2123
|
+
if (toDate) {
|
|
2124
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
2125
|
+
args.var_to_date = toDate.getTime() / 1e3;
|
|
2126
|
+
}
|
|
2127
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
575
2128
|
try {
|
|
576
|
-
const
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
if (include.length) {
|
|
580
|
-
const includeResult = await this.db.query({
|
|
581
|
-
query: `
|
|
582
|
-
WITH ordered_messages AS (
|
|
583
|
-
SELECT
|
|
584
|
-
*,
|
|
585
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
586
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
587
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
588
|
-
FROM "${TABLE_MESSAGES}"
|
|
589
|
-
WHERE thread_id = {var_thread_id:String}
|
|
590
|
-
)
|
|
591
|
-
SELECT
|
|
592
|
-
m.id AS id,
|
|
593
|
-
m.content as content,
|
|
594
|
-
m.role as role,
|
|
595
|
-
m.type as type,
|
|
596
|
-
m.createdAt as createdAt,
|
|
597
|
-
m.updatedAt as updatedAt,
|
|
598
|
-
m.thread_id AS "threadId"
|
|
599
|
-
FROM ordered_messages m
|
|
600
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
601
|
-
OR EXISTS (
|
|
602
|
-
SELECT 1 FROM ordered_messages target
|
|
603
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
604
|
-
AND (
|
|
605
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
606
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
607
|
-
OR
|
|
608
|
-
-- Get next messages based on the max withNextMessages
|
|
609
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
610
|
-
)
|
|
611
|
-
)
|
|
612
|
-
ORDER BY m."createdAt" DESC
|
|
613
|
-
`,
|
|
614
|
-
query_params: {
|
|
615
|
-
var_thread_id: threadId,
|
|
616
|
-
var_include: include.map((i) => i.id),
|
|
617
|
-
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
618
|
-
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
619
|
-
},
|
|
620
|
-
clickhouse_settings: {
|
|
621
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
622
|
-
date_time_input_format: "best_effort",
|
|
623
|
-
date_time_output_format: "iso",
|
|
624
|
-
use_client_time_zone: 1,
|
|
625
|
-
output_format_json_quote_64bit_integers: 0
|
|
626
|
-
}
|
|
627
|
-
});
|
|
628
|
-
const rows2 = await includeResult.json();
|
|
629
|
-
messages.push(...transformRows(rows2.data));
|
|
630
|
-
}
|
|
631
|
-
const result = await this.db.query({
|
|
632
|
-
query: `
|
|
633
|
-
SELECT
|
|
634
|
-
id,
|
|
635
|
-
content,
|
|
636
|
-
role,
|
|
637
|
-
type,
|
|
638
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
639
|
-
thread_id AS "threadId"
|
|
640
|
-
FROM "${TABLE_MESSAGES}"
|
|
641
|
-
WHERE thread_id = {threadId:String}
|
|
642
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
643
|
-
ORDER BY "createdAt" DESC
|
|
644
|
-
LIMIT {limit:Int64}
|
|
645
|
-
`,
|
|
646
|
-
query_params: {
|
|
647
|
-
threadId,
|
|
648
|
-
exclude: messages.map((m) => m.id),
|
|
649
|
-
limit
|
|
650
|
-
},
|
|
2129
|
+
const result = await this.client.query({
|
|
2130
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
2131
|
+
query_params: args,
|
|
651
2132
|
clickhouse_settings: {
|
|
652
2133
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
653
2134
|
date_time_input_format: "best_effort",
|
|
@@ -656,94 +2137,70 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
656
2137
|
output_format_json_quote_64bit_integers: 0
|
|
657
2138
|
}
|
|
658
2139
|
});
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
662
|
-
messages.forEach((message) => {
|
|
663
|
-
if (typeof message.content === "string") {
|
|
664
|
-
try {
|
|
665
|
-
message.content = JSON.parse(message.content);
|
|
666
|
-
} catch {
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
});
|
|
670
|
-
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
671
|
-
if (format === `v2`) return list.get.all.v2();
|
|
672
|
-
return list.get.all.v1();
|
|
673
|
-
} catch (error) {
|
|
674
|
-
console.error("Error getting messages:", error);
|
|
675
|
-
throw error;
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
async saveMessages(args) {
|
|
679
|
-
const { messages, format = "v1" } = args;
|
|
680
|
-
if (messages.length === 0) return messages;
|
|
681
|
-
try {
|
|
682
|
-
const threadId = messages[0]?.threadId;
|
|
683
|
-
const resourceId = messages[0]?.resourceId;
|
|
684
|
-
if (!threadId) {
|
|
685
|
-
throw new Error("Thread ID is required");
|
|
2140
|
+
if (!result) {
|
|
2141
|
+
return [];
|
|
686
2142
|
}
|
|
687
|
-
const
|
|
688
|
-
|
|
689
|
-
|
|
2143
|
+
const resp = await result.json();
|
|
2144
|
+
const rows = resp.data;
|
|
2145
|
+
return rows.map((row) => ({
|
|
2146
|
+
id: row.id,
|
|
2147
|
+
parentSpanId: row.parentSpanId,
|
|
2148
|
+
traceId: row.traceId,
|
|
2149
|
+
name: row.name,
|
|
2150
|
+
scope: row.scope,
|
|
2151
|
+
kind: row.kind,
|
|
2152
|
+
status: safelyParseJSON(row.status),
|
|
2153
|
+
events: safelyParseJSON(row.events),
|
|
2154
|
+
links: safelyParseJSON(row.links),
|
|
2155
|
+
attributes: safelyParseJSON(row.attributes),
|
|
2156
|
+
startTime: row.startTime,
|
|
2157
|
+
endTime: row.endTime,
|
|
2158
|
+
other: safelyParseJSON(row.other),
|
|
2159
|
+
createdAt: row.createdAt
|
|
2160
|
+
}));
|
|
2161
|
+
} catch (error) {
|
|
2162
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
2163
|
+
return [];
|
|
690
2164
|
}
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
706
|
-
date_time_input_format: "best_effort",
|
|
707
|
-
use_client_time_zone: 1,
|
|
708
|
-
output_format_json_quote_64bit_integers: 0
|
|
709
|
-
}
|
|
710
|
-
}),
|
|
711
|
-
// Update thread's updatedAt timestamp
|
|
712
|
-
this.db.insert({
|
|
713
|
-
table: TABLE_THREADS,
|
|
714
|
-
format: "JSONEachRow",
|
|
715
|
-
values: [
|
|
716
|
-
{
|
|
717
|
-
id: thread.id,
|
|
718
|
-
resourceId: thread.resourceId,
|
|
719
|
-
title: thread.title,
|
|
720
|
-
metadata: thread.metadata,
|
|
721
|
-
createdAt: thread.createdAt,
|
|
722
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
723
|
-
}
|
|
724
|
-
],
|
|
725
|
-
clickhouse_settings: {
|
|
726
|
-
date_time_input_format: "best_effort",
|
|
727
|
-
use_client_time_zone: 1,
|
|
728
|
-
output_format_json_quote_64bit_integers: 0
|
|
2165
|
+
throw new MastraError(
|
|
2166
|
+
{
|
|
2167
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
2168
|
+
domain: ErrorDomain.STORAGE,
|
|
2169
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2170
|
+
details: {
|
|
2171
|
+
name: name ?? null,
|
|
2172
|
+
scope: scope ?? null,
|
|
2173
|
+
page,
|
|
2174
|
+
perPage,
|
|
2175
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
2176
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
2177
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
2178
|
+
toDate: toDate?.toISOString() ?? null
|
|
729
2179
|
}
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
if (format === `v2`) return list.get.all.v2();
|
|
734
|
-
return list.get.all.v1();
|
|
735
|
-
} catch (error) {
|
|
736
|
-
console.error("Error saving messages:", error);
|
|
737
|
-
throw error;
|
|
2180
|
+
},
|
|
2181
|
+
error
|
|
2182
|
+
);
|
|
738
2183
|
}
|
|
739
2184
|
}
|
|
2185
|
+
async batchTraceInsert(args) {
|
|
2186
|
+
await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
|
|
2187
|
+
}
|
|
2188
|
+
};
|
|
2189
|
+
var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
2190
|
+
client;
|
|
2191
|
+
operations;
|
|
2192
|
+
constructor({ client, operations }) {
|
|
2193
|
+
super();
|
|
2194
|
+
this.operations = operations;
|
|
2195
|
+
this.client = client;
|
|
2196
|
+
}
|
|
740
2197
|
async persistWorkflowSnapshot({
|
|
741
2198
|
workflowName,
|
|
742
2199
|
runId,
|
|
743
2200
|
snapshot
|
|
744
2201
|
}) {
|
|
745
2202
|
try {
|
|
746
|
-
const currentSnapshot = await this.load({
|
|
2203
|
+
const currentSnapshot = await this.operations.load({
|
|
747
2204
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
748
2205
|
keys: { workflow_name: workflowName, run_id: runId }
|
|
749
2206
|
});
|
|
@@ -759,7 +2216,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
759
2216
|
createdAt: now.toISOString(),
|
|
760
2217
|
updatedAt: now.toISOString()
|
|
761
2218
|
};
|
|
762
|
-
await this.
|
|
2219
|
+
await this.client.insert({
|
|
763
2220
|
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
764
2221
|
format: "JSONEachRow",
|
|
765
2222
|
values: [persisting],
|
|
@@ -771,8 +2228,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
771
2228
|
}
|
|
772
2229
|
});
|
|
773
2230
|
} catch (error) {
|
|
774
|
-
|
|
775
|
-
|
|
2231
|
+
throw new MastraError(
|
|
2232
|
+
{
|
|
2233
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
2234
|
+
domain: ErrorDomain.STORAGE,
|
|
2235
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2236
|
+
details: { workflowName, runId }
|
|
2237
|
+
},
|
|
2238
|
+
error
|
|
2239
|
+
);
|
|
776
2240
|
}
|
|
777
2241
|
}
|
|
778
2242
|
async loadWorkflowSnapshot({
|
|
@@ -780,7 +2244,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
780
2244
|
runId
|
|
781
2245
|
}) {
|
|
782
2246
|
try {
|
|
783
|
-
const result = await this.load({
|
|
2247
|
+
const result = await this.operations.load({
|
|
784
2248
|
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
785
2249
|
keys: {
|
|
786
2250
|
workflow_name: workflowName,
|
|
@@ -792,8 +2256,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
792
2256
|
}
|
|
793
2257
|
return result.snapshot;
|
|
794
2258
|
} catch (error) {
|
|
795
|
-
|
|
796
|
-
|
|
2259
|
+
throw new MastraError(
|
|
2260
|
+
{
|
|
2261
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
2262
|
+
domain: ErrorDomain.STORAGE,
|
|
2263
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2264
|
+
details: { workflowName, runId }
|
|
2265
|
+
},
|
|
2266
|
+
error
|
|
2267
|
+
);
|
|
797
2268
|
}
|
|
798
2269
|
}
|
|
799
2270
|
parseWorkflowRun(row) {
|
|
@@ -830,7 +2301,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
830
2301
|
values.var_workflow_name = workflowName;
|
|
831
2302
|
}
|
|
832
2303
|
if (resourceId) {
|
|
833
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
2304
|
+
const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
|
|
834
2305
|
if (hasResourceId) {
|
|
835
2306
|
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
836
2307
|
values.var_resourceId = resourceId;
|
|
@@ -851,7 +2322,7 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
851
2322
|
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
852
2323
|
let total = 0;
|
|
853
2324
|
if (limit !== void 0 && offset !== void 0) {
|
|
854
|
-
const countResult = await this.
|
|
2325
|
+
const countResult = await this.client.query({
|
|
855
2326
|
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
856
2327
|
query_params: values,
|
|
857
2328
|
format: "JSONEachRow"
|
|
@@ -859,21 +2330,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
859
2330
|
const countRows = await countResult.json();
|
|
860
2331
|
total = Number(countRows[0]?.count ?? 0);
|
|
861
2332
|
}
|
|
862
|
-
const result = await this.
|
|
2333
|
+
const result = await this.client.query({
|
|
863
2334
|
query: `
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
2335
|
+
SELECT
|
|
2336
|
+
workflow_name,
|
|
2337
|
+
run_id,
|
|
2338
|
+
snapshot,
|
|
2339
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2340
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2341
|
+
resourceId
|
|
2342
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2343
|
+
${whereClause}
|
|
2344
|
+
ORDER BY createdAt DESC
|
|
2345
|
+
${limitClause}
|
|
2346
|
+
${offsetClause}
|
|
2347
|
+
`,
|
|
877
2348
|
query_params: values,
|
|
878
2349
|
format: "JSONEachRow"
|
|
879
2350
|
});
|
|
@@ -884,8 +2355,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
884
2355
|
});
|
|
885
2356
|
return { runs, total: total || runs.length };
|
|
886
2357
|
} catch (error) {
|
|
887
|
-
|
|
888
|
-
|
|
2358
|
+
throw new MastraError(
|
|
2359
|
+
{
|
|
2360
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
2361
|
+
domain: ErrorDomain.STORAGE,
|
|
2362
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2363
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
2364
|
+
},
|
|
2365
|
+
error
|
|
2366
|
+
);
|
|
889
2367
|
}
|
|
890
2368
|
}
|
|
891
2369
|
async getWorkflowRunById({
|
|
@@ -904,18 +2382,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
904
2382
|
values.var_workflow_name = workflowName;
|
|
905
2383
|
}
|
|
906
2384
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
907
|
-
const result = await this.
|
|
2385
|
+
const result = await this.client.query({
|
|
908
2386
|
query: `
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
2387
|
+
SELECT
|
|
2388
|
+
workflow_name,
|
|
2389
|
+
run_id,
|
|
2390
|
+
snapshot,
|
|
2391
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
2392
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
2393
|
+
resourceId
|
|
2394
|
+
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2395
|
+
${whereClause}
|
|
2396
|
+
`,
|
|
919
2397
|
query_params: values,
|
|
920
2398
|
format: "JSONEachRow"
|
|
921
2399
|
});
|
|
@@ -925,34 +2403,251 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
925
2403
|
}
|
|
926
2404
|
return this.parseWorkflowRun(resultJson[0]);
|
|
927
2405
|
} catch (error) {
|
|
928
|
-
|
|
929
|
-
|
|
2406
|
+
throw new MastraError(
|
|
2407
|
+
{
|
|
2408
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
2409
|
+
domain: ErrorDomain.STORAGE,
|
|
2410
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2411
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
2412
|
+
},
|
|
2413
|
+
error
|
|
2414
|
+
);
|
|
930
2415
|
}
|
|
931
2416
|
}
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
2417
|
+
};
|
|
2418
|
+
|
|
2419
|
+
// src/storage/index.ts
|
|
2420
|
+
var ClickhouseStore = class extends MastraStorage {
|
|
2421
|
+
db;
|
|
2422
|
+
ttl = {};
|
|
2423
|
+
stores;
|
|
2424
|
+
constructor(config) {
|
|
2425
|
+
super({ name: "ClickhouseStore" });
|
|
2426
|
+
this.db = createClient({
|
|
2427
|
+
url: config.url,
|
|
2428
|
+
username: config.username,
|
|
2429
|
+
password: config.password,
|
|
2430
|
+
clickhouse_settings: {
|
|
2431
|
+
date_time_input_format: "best_effort",
|
|
2432
|
+
date_time_output_format: "iso",
|
|
2433
|
+
// This is crucial
|
|
2434
|
+
use_client_time_zone: 1,
|
|
2435
|
+
output_format_json_quote_64bit_integers: 0
|
|
2436
|
+
}
|
|
936
2437
|
});
|
|
937
|
-
|
|
938
|
-
|
|
2438
|
+
this.ttl = config.ttl;
|
|
2439
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
2440
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
2441
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
2442
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
2443
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
2444
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
2445
|
+
this.stores = {
|
|
2446
|
+
operations,
|
|
2447
|
+
workflows,
|
|
2448
|
+
scores,
|
|
2449
|
+
legacyEvals,
|
|
2450
|
+
traces,
|
|
2451
|
+
memory
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
get supports() {
|
|
2455
|
+
return {
|
|
2456
|
+
selectByIncludeResourceScope: true,
|
|
2457
|
+
resourceWorkingMemory: true,
|
|
2458
|
+
hasColumn: true,
|
|
2459
|
+
createTable: true,
|
|
2460
|
+
deleteMessages: false
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
2463
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2464
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2465
|
+
}
|
|
2466
|
+
async getEvals(options) {
|
|
2467
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2468
|
+
}
|
|
2469
|
+
async batchInsert({ tableName, records }) {
|
|
2470
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
2471
|
+
}
|
|
2472
|
+
async optimizeTable({ tableName }) {
|
|
2473
|
+
try {
|
|
2474
|
+
await this.db.command({
|
|
2475
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
2476
|
+
});
|
|
2477
|
+
} catch (error) {
|
|
2478
|
+
throw new MastraError(
|
|
2479
|
+
{
|
|
2480
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
2481
|
+
domain: ErrorDomain.STORAGE,
|
|
2482
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2483
|
+
details: { tableName }
|
|
2484
|
+
},
|
|
2485
|
+
error
|
|
2486
|
+
);
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
async materializeTtl({ tableName }) {
|
|
2490
|
+
try {
|
|
2491
|
+
await this.db.command({
|
|
2492
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
2493
|
+
});
|
|
2494
|
+
} catch (error) {
|
|
2495
|
+
throw new MastraError(
|
|
2496
|
+
{
|
|
2497
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
2498
|
+
domain: ErrorDomain.STORAGE,
|
|
2499
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2500
|
+
details: { tableName }
|
|
2501
|
+
},
|
|
2502
|
+
error
|
|
2503
|
+
);
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2506
|
+
async createTable({
|
|
2507
|
+
tableName,
|
|
2508
|
+
schema
|
|
2509
|
+
}) {
|
|
2510
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2511
|
+
}
|
|
2512
|
+
async dropTable({ tableName }) {
|
|
2513
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2514
|
+
}
|
|
2515
|
+
async alterTable({
|
|
2516
|
+
tableName,
|
|
2517
|
+
schema,
|
|
2518
|
+
ifNotExists
|
|
2519
|
+
}) {
|
|
2520
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2521
|
+
}
|
|
2522
|
+
async clearTable({ tableName }) {
|
|
2523
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2524
|
+
}
|
|
2525
|
+
async insert({ tableName, record }) {
|
|
2526
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2527
|
+
}
|
|
2528
|
+
async load({ tableName, keys }) {
|
|
2529
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2530
|
+
}
|
|
2531
|
+
async persistWorkflowSnapshot({
|
|
2532
|
+
workflowName,
|
|
2533
|
+
runId,
|
|
2534
|
+
snapshot
|
|
2535
|
+
}) {
|
|
2536
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2537
|
+
}
|
|
2538
|
+
async loadWorkflowSnapshot({
|
|
2539
|
+
workflowName,
|
|
2540
|
+
runId
|
|
2541
|
+
}) {
|
|
2542
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2543
|
+
}
|
|
2544
|
+
async getWorkflowRuns({
|
|
2545
|
+
workflowName,
|
|
2546
|
+
fromDate,
|
|
2547
|
+
toDate,
|
|
2548
|
+
limit,
|
|
2549
|
+
offset,
|
|
2550
|
+
resourceId
|
|
2551
|
+
} = {}) {
|
|
2552
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
2553
|
+
}
|
|
2554
|
+
async getWorkflowRunById({
|
|
2555
|
+
runId,
|
|
2556
|
+
workflowName
|
|
2557
|
+
}) {
|
|
2558
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
2559
|
+
}
|
|
2560
|
+
async getTraces(args) {
|
|
2561
|
+
return this.stores.traces.getTraces(args);
|
|
2562
|
+
}
|
|
2563
|
+
async getTracesPaginated(args) {
|
|
2564
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2565
|
+
}
|
|
2566
|
+
async batchTraceInsert(args) {
|
|
2567
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
2568
|
+
}
|
|
2569
|
+
async getThreadById({ threadId }) {
|
|
2570
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2571
|
+
}
|
|
2572
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2573
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2574
|
+
}
|
|
2575
|
+
async saveThread({ thread }) {
|
|
2576
|
+
return this.stores.memory.saveThread({ thread });
|
|
2577
|
+
}
|
|
2578
|
+
async updateThread({
|
|
2579
|
+
id,
|
|
2580
|
+
title,
|
|
2581
|
+
metadata
|
|
2582
|
+
}) {
|
|
2583
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2584
|
+
}
|
|
2585
|
+
async deleteThread({ threadId }) {
|
|
2586
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2587
|
+
}
|
|
2588
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2589
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2590
|
+
}
|
|
2591
|
+
async getMessages({
|
|
2592
|
+
threadId,
|
|
2593
|
+
resourceId,
|
|
2594
|
+
selectBy,
|
|
2595
|
+
format
|
|
2596
|
+
}) {
|
|
2597
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
2598
|
+
}
|
|
2599
|
+
async saveMessages(args) {
|
|
2600
|
+
return this.stores.memory.saveMessages(args);
|
|
2601
|
+
}
|
|
2602
|
+
async getMessagesPaginated(args) {
|
|
2603
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2604
|
+
}
|
|
2605
|
+
async updateMessages(args) {
|
|
2606
|
+
return this.stores.memory.updateMessages(args);
|
|
2607
|
+
}
|
|
2608
|
+
async getResourceById({ resourceId }) {
|
|
2609
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2610
|
+
}
|
|
2611
|
+
async saveResource({ resource }) {
|
|
2612
|
+
return this.stores.memory.saveResource({ resource });
|
|
2613
|
+
}
|
|
2614
|
+
async updateResource({
|
|
2615
|
+
resourceId,
|
|
2616
|
+
workingMemory,
|
|
2617
|
+
metadata
|
|
2618
|
+
}) {
|
|
2619
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2620
|
+
}
|
|
2621
|
+
async getScoreById({ id }) {
|
|
2622
|
+
return this.stores.scores.getScoreById({ id });
|
|
939
2623
|
}
|
|
940
|
-
async
|
|
941
|
-
|
|
2624
|
+
async saveScore(_score) {
|
|
2625
|
+
return this.stores.scores.saveScore(_score);
|
|
2626
|
+
}
|
|
2627
|
+
async getScoresByRunId({
|
|
2628
|
+
runId,
|
|
2629
|
+
pagination
|
|
2630
|
+
}) {
|
|
2631
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
942
2632
|
}
|
|
943
|
-
async
|
|
944
|
-
|
|
2633
|
+
async getScoresByEntityId({
|
|
2634
|
+
entityId,
|
|
2635
|
+
entityType,
|
|
2636
|
+
pagination
|
|
2637
|
+
}) {
|
|
2638
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
945
2639
|
}
|
|
946
|
-
async
|
|
947
|
-
|
|
2640
|
+
async getScoresByScorerId({
|
|
2641
|
+
scorerId,
|
|
2642
|
+
pagination
|
|
2643
|
+
}) {
|
|
2644
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
948
2645
|
}
|
|
949
2646
|
async close() {
|
|
950
2647
|
await this.db.close();
|
|
951
2648
|
}
|
|
952
|
-
async updateMessages(_args) {
|
|
953
|
-
this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
|
|
954
|
-
throw new Error("Method not implemented");
|
|
955
|
-
}
|
|
956
2649
|
};
|
|
957
2650
|
|
|
958
|
-
export {
|
|
2651
|
+
export { ClickhouseStore };
|
|
2652
|
+
//# sourceMappingURL=index.js.map
|
|
2653
|
+
//# sourceMappingURL=index.js.map
|