@mastra/clickhouse 0.0.0-commonjs-20250414101718
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +174 -0
- package/LICENSE.md +46 -0
- package/README.md +122 -0
- package/dist/_tsup-dts-rollup.d.cts +127 -0
- package/dist/_tsup-dts-rollup.d.ts +127 -0
- package/dist/index.cjs +781 -0
- package/dist/index.d.cts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +779 -0
- package/docker-compose.yaml +15 -0
- package/eslint.config.js +6 -0
- package/package.json +46 -0
- package/src/index.ts +1 -0
- package/src/storage/index.test.ts +672 -0
- package/src/storage/index.ts +968 -0
- package/tsconfig.json +5 -0
- package/vitest.config.ts +12 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,781 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var client = require('@clickhouse/client');
|
|
4
|
+
var storage = require('@mastra/core/storage');
|
|
5
|
+
|
|
6
|
+
// src/storage/index.ts
|
|
7
|
+
function safelyParseJSON(jsonString) {
|
|
8
|
+
try {
|
|
9
|
+
return JSON.parse(jsonString);
|
|
10
|
+
} catch {
|
|
11
|
+
return {};
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
var TABLE_ENGINES = {
|
|
15
|
+
[storage.TABLE_MESSAGES]: `MergeTree()`,
|
|
16
|
+
[storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
17
|
+
[storage.TABLE_TRACES]: `MergeTree()`,
|
|
18
|
+
[storage.TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
19
|
+
[storage.TABLE_EVALS]: `MergeTree()`
|
|
20
|
+
};
|
|
21
|
+
var COLUMN_TYPES = {
|
|
22
|
+
text: "String",
|
|
23
|
+
timestamp: "DateTime64(3)",
|
|
24
|
+
uuid: "String",
|
|
25
|
+
jsonb: "String",
|
|
26
|
+
integer: "Int64",
|
|
27
|
+
bigint: "Int64"
|
|
28
|
+
};
|
|
29
|
+
function transformRows(rows) {
|
|
30
|
+
return rows.map((row) => transformRow(row));
|
|
31
|
+
}
|
|
32
|
+
function transformRow(row) {
|
|
33
|
+
if (!row) {
|
|
34
|
+
return row;
|
|
35
|
+
}
|
|
36
|
+
if (row.createdAt) {
|
|
37
|
+
row.createdAt = new Date(row.createdAt);
|
|
38
|
+
}
|
|
39
|
+
if (row.updatedAt) {
|
|
40
|
+
row.updatedAt = new Date(row.updatedAt);
|
|
41
|
+
}
|
|
42
|
+
return row;
|
|
43
|
+
}
|
|
44
|
+
var ClickhouseStore = class extends storage.MastraStorage {
|
|
45
|
+
db;
|
|
46
|
+
ttl = {};
|
|
47
|
+
constructor(config) {
|
|
48
|
+
super({ name: "ClickhouseStore" });
|
|
49
|
+
this.db = client.createClient({
|
|
50
|
+
url: config.url,
|
|
51
|
+
username: config.username,
|
|
52
|
+
password: config.password,
|
|
53
|
+
clickhouse_settings: {
|
|
54
|
+
date_time_input_format: "best_effort",
|
|
55
|
+
date_time_output_format: "iso",
|
|
56
|
+
// This is crucial
|
|
57
|
+
use_client_time_zone: 1,
|
|
58
|
+
output_format_json_quote_64bit_integers: 0
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
this.ttl = config.ttl;
|
|
62
|
+
}
|
|
63
|
+
transformEvalRow(row) {
|
|
64
|
+
row = transformRow(row);
|
|
65
|
+
const resultValue = JSON.parse(row.result);
|
|
66
|
+
const testInfoValue = row.test_info ? JSON.parse(row.test_info) : void 0;
|
|
67
|
+
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
68
|
+
throw new Error(`Invalid MetricResult format: ${JSON.stringify(resultValue)}`);
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
input: row.input,
|
|
72
|
+
output: row.output,
|
|
73
|
+
result: resultValue,
|
|
74
|
+
agentName: row.agent_name,
|
|
75
|
+
metricName: row.metric_name,
|
|
76
|
+
instructions: row.instructions,
|
|
77
|
+
testInfo: testInfoValue,
|
|
78
|
+
globalRunId: row.global_run_id,
|
|
79
|
+
runId: row.run_id,
|
|
80
|
+
createdAt: row.created_at
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
async getEvalsByAgentName(agentName, type) {
|
|
84
|
+
try {
|
|
85
|
+
const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
86
|
+
const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
|
|
87
|
+
const result = await this.db.query({
|
|
88
|
+
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
89
|
+
query_params: { var_agent_name: agentName },
|
|
90
|
+
clickhouse_settings: {
|
|
91
|
+
date_time_input_format: "best_effort",
|
|
92
|
+
date_time_output_format: "iso",
|
|
93
|
+
use_client_time_zone: 1,
|
|
94
|
+
output_format_json_quote_64bit_integers: 0
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
if (!result) {
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
const rows = await result.json();
|
|
101
|
+
return rows.data.map((row) => this.transformEvalRow(row));
|
|
102
|
+
} catch (error) {
|
|
103
|
+
if (error instanceof Error && error.message.includes("no such table")) {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
this.logger.error("Failed to get evals for the specified agent: " + error?.message);
|
|
107
|
+
throw error;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
async batchInsert({ tableName, records }) {
|
|
111
|
+
try {
|
|
112
|
+
await this.db.insert({
|
|
113
|
+
table: tableName,
|
|
114
|
+
values: records.map((record) => ({
|
|
115
|
+
...Object.fromEntries(
|
|
116
|
+
Object.entries(record).map(([key, value]) => [
|
|
117
|
+
key,
|
|
118
|
+
storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
|
|
119
|
+
])
|
|
120
|
+
)
|
|
121
|
+
})),
|
|
122
|
+
format: "JSONEachRow",
|
|
123
|
+
clickhouse_settings: {
|
|
124
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
125
|
+
date_time_input_format: "best_effort",
|
|
126
|
+
use_client_time_zone: 1,
|
|
127
|
+
output_format_json_quote_64bit_integers: 0
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
} catch (error) {
|
|
131
|
+
console.error(`Error inserting into ${tableName}:`, error);
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
async getTraces({
|
|
136
|
+
name,
|
|
137
|
+
scope,
|
|
138
|
+
page,
|
|
139
|
+
perPage,
|
|
140
|
+
attributes,
|
|
141
|
+
filters
|
|
142
|
+
}) {
|
|
143
|
+
const limit = perPage;
|
|
144
|
+
const offset = page * perPage;
|
|
145
|
+
const args = {};
|
|
146
|
+
const conditions = [];
|
|
147
|
+
if (name) {
|
|
148
|
+
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
149
|
+
args.var_name = name;
|
|
150
|
+
}
|
|
151
|
+
if (scope) {
|
|
152
|
+
conditions.push(`scope = {var_scope:String}`);
|
|
153
|
+
args.var_scope = scope;
|
|
154
|
+
}
|
|
155
|
+
if (attributes) {
|
|
156
|
+
Object.entries(attributes).forEach(([key, value]) => {
|
|
157
|
+
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
158
|
+
args[`var_attr_${key}`] = value;
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
if (filters) {
|
|
162
|
+
Object.entries(filters).forEach(([key, value]) => {
|
|
163
|
+
conditions.push(
|
|
164
|
+
`${key} = {var_col_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
|
|
165
|
+
);
|
|
166
|
+
args[`var_col_${key}`] = value;
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
170
|
+
const result = await this.db.query({
|
|
171
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${storage.TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
172
|
+
query_params: args,
|
|
173
|
+
clickhouse_settings: {
|
|
174
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
175
|
+
date_time_input_format: "best_effort",
|
|
176
|
+
date_time_output_format: "iso",
|
|
177
|
+
use_client_time_zone: 1,
|
|
178
|
+
output_format_json_quote_64bit_integers: 0
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
if (!result) {
|
|
182
|
+
return [];
|
|
183
|
+
}
|
|
184
|
+
const resp = await result.json();
|
|
185
|
+
const rows = resp.data;
|
|
186
|
+
return rows.map((row) => ({
|
|
187
|
+
id: row.id,
|
|
188
|
+
parentSpanId: row.parentSpanId,
|
|
189
|
+
traceId: row.traceId,
|
|
190
|
+
name: row.name,
|
|
191
|
+
scope: row.scope,
|
|
192
|
+
kind: row.kind,
|
|
193
|
+
status: safelyParseJSON(row.status),
|
|
194
|
+
events: safelyParseJSON(row.events),
|
|
195
|
+
links: safelyParseJSON(row.links),
|
|
196
|
+
attributes: safelyParseJSON(row.attributes),
|
|
197
|
+
startTime: row.startTime,
|
|
198
|
+
endTime: row.endTime,
|
|
199
|
+
other: safelyParseJSON(row.other),
|
|
200
|
+
createdAt: row.createdAt
|
|
201
|
+
}));
|
|
202
|
+
}
|
|
203
|
+
async optimizeTable({ tableName }) {
|
|
204
|
+
await this.db.command({
|
|
205
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
async materializeTtl({ tableName }) {
|
|
209
|
+
await this.db.command({
|
|
210
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
async createTable({
|
|
214
|
+
tableName,
|
|
215
|
+
schema
|
|
216
|
+
}) {
|
|
217
|
+
try {
|
|
218
|
+
const columns = Object.entries(schema).map(([name, def]) => {
|
|
219
|
+
const constraints = [];
|
|
220
|
+
if (!def.nullable) constraints.push("NOT NULL");
|
|
221
|
+
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
222
|
+
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
|
|
223
|
+
}).join(",\n");
|
|
224
|
+
const rowTtl = this.ttl?.[tableName]?.row;
|
|
225
|
+
const sql = tableName === storage.TABLE_WORKFLOW_SNAPSHOT ? `
|
|
226
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
227
|
+
${["id String"].concat(columns)}
|
|
228
|
+
)
|
|
229
|
+
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
230
|
+
PARTITION BY "createdAt"
|
|
231
|
+
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
232
|
+
ORDER BY (createdAt, run_id, workflow_name)
|
|
233
|
+
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
234
|
+
SETTINGS index_granularity = 8192
|
|
235
|
+
` : `
|
|
236
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
237
|
+
${columns}
|
|
238
|
+
)
|
|
239
|
+
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
240
|
+
PARTITION BY "createdAt"
|
|
241
|
+
PRIMARY KEY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
242
|
+
ORDER BY (createdAt, ${tableName === storage.TABLE_EVALS ? "run_id" : "id"})
|
|
243
|
+
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
244
|
+
SETTINGS index_granularity = 8192
|
|
245
|
+
`;
|
|
246
|
+
await this.db.query({
|
|
247
|
+
query: sql,
|
|
248
|
+
clickhouse_settings: {
|
|
249
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
250
|
+
date_time_input_format: "best_effort",
|
|
251
|
+
date_time_output_format: "iso",
|
|
252
|
+
use_client_time_zone: 1,
|
|
253
|
+
output_format_json_quote_64bit_integers: 0
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
} catch (error) {
|
|
257
|
+
console.error(`Error creating table ${tableName}:`, error);
|
|
258
|
+
throw error;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async clearTable({ tableName }) {
|
|
262
|
+
try {
|
|
263
|
+
await this.db.query({
|
|
264
|
+
query: `TRUNCATE TABLE ${tableName}`,
|
|
265
|
+
clickhouse_settings: {
|
|
266
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
267
|
+
date_time_input_format: "best_effort",
|
|
268
|
+
date_time_output_format: "iso",
|
|
269
|
+
use_client_time_zone: 1,
|
|
270
|
+
output_format_json_quote_64bit_integers: 0
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
} catch (error) {
|
|
274
|
+
console.error(`Error clearing table ${tableName}:`, error);
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
async insert({ tableName, record }) {
|
|
279
|
+
try {
|
|
280
|
+
await this.db.insert({
|
|
281
|
+
table: tableName,
|
|
282
|
+
values: [
|
|
283
|
+
{
|
|
284
|
+
...record,
|
|
285
|
+
createdAt: record.createdAt.toISOString(),
|
|
286
|
+
updatedAt: record.updatedAt.toISOString()
|
|
287
|
+
}
|
|
288
|
+
],
|
|
289
|
+
format: "JSONEachRow",
|
|
290
|
+
clickhouse_settings: {
|
|
291
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
292
|
+
output_format_json_quote_64bit_integers: 0,
|
|
293
|
+
date_time_input_format: "best_effort",
|
|
294
|
+
use_client_time_zone: 1
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
} catch (error) {
|
|
298
|
+
console.error(`Error inserting into ${tableName}:`, error);
|
|
299
|
+
throw error;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
async load({ tableName, keys }) {
|
|
303
|
+
try {
|
|
304
|
+
const keyEntries = Object.entries(keys);
|
|
305
|
+
const conditions = keyEntries.map(
|
|
306
|
+
([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[storage.TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
|
|
307
|
+
).join(" AND ");
|
|
308
|
+
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
309
|
+
return { ...acc, [`var_${key}`]: value };
|
|
310
|
+
}, {});
|
|
311
|
+
const result = await this.db.query({
|
|
312
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName].startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
|
|
313
|
+
query_params: values,
|
|
314
|
+
clickhouse_settings: {
|
|
315
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
316
|
+
date_time_input_format: "best_effort",
|
|
317
|
+
date_time_output_format: "iso",
|
|
318
|
+
use_client_time_zone: 1,
|
|
319
|
+
output_format_json_quote_64bit_integers: 0
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
if (!result) {
|
|
323
|
+
return null;
|
|
324
|
+
}
|
|
325
|
+
const rows = await result.json();
|
|
326
|
+
if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
|
|
327
|
+
const snapshot = rows.data[0];
|
|
328
|
+
if (!snapshot) {
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
if (typeof snapshot.snapshot === "string") {
|
|
332
|
+
snapshot.snapshot = JSON.parse(snapshot.snapshot);
|
|
333
|
+
}
|
|
334
|
+
return transformRow(snapshot);
|
|
335
|
+
}
|
|
336
|
+
const data = transformRow(rows.data[0]);
|
|
337
|
+
return data;
|
|
338
|
+
} catch (error) {
|
|
339
|
+
console.error(`Error loading from ${tableName}:`, error);
|
|
340
|
+
throw error;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
async getThreadById({ threadId }) {
|
|
344
|
+
try {
|
|
345
|
+
const result = await this.db.query({
|
|
346
|
+
query: `SELECT
|
|
347
|
+
id,
|
|
348
|
+
"resourceId",
|
|
349
|
+
title,
|
|
350
|
+
metadata,
|
|
351
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
352
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
353
|
+
FROM "${storage.TABLE_THREADS}"
|
|
354
|
+
FINAL
|
|
355
|
+
WHERE id = {var_id:String}`,
|
|
356
|
+
query_params: { var_id: threadId },
|
|
357
|
+
clickhouse_settings: {
|
|
358
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
359
|
+
date_time_input_format: "best_effort",
|
|
360
|
+
date_time_output_format: "iso",
|
|
361
|
+
use_client_time_zone: 1,
|
|
362
|
+
output_format_json_quote_64bit_integers: 0
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
const rows = await result.json();
|
|
366
|
+
const thread = transformRow(rows.data[0]);
|
|
367
|
+
if (!thread) {
|
|
368
|
+
return null;
|
|
369
|
+
}
|
|
370
|
+
return {
|
|
371
|
+
...thread,
|
|
372
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
373
|
+
createdAt: thread.createdAt,
|
|
374
|
+
updatedAt: thread.updatedAt
|
|
375
|
+
};
|
|
376
|
+
} catch (error) {
|
|
377
|
+
console.error(`Error getting thread ${threadId}:`, error);
|
|
378
|
+
throw error;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
382
|
+
try {
|
|
383
|
+
const result = await this.db.query({
|
|
384
|
+
query: `SELECT
|
|
385
|
+
id,
|
|
386
|
+
"resourceId",
|
|
387
|
+
title,
|
|
388
|
+
metadata,
|
|
389
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
390
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
391
|
+
FROM "${storage.TABLE_THREADS}"
|
|
392
|
+
WHERE "resourceId" = {var_resourceId:String}`,
|
|
393
|
+
query_params: { var_resourceId: resourceId },
|
|
394
|
+
clickhouse_settings: {
|
|
395
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
396
|
+
date_time_input_format: "best_effort",
|
|
397
|
+
date_time_output_format: "iso",
|
|
398
|
+
use_client_time_zone: 1,
|
|
399
|
+
output_format_json_quote_64bit_integers: 0
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
const rows = await result.json();
|
|
403
|
+
const threads = transformRows(rows.data);
|
|
404
|
+
return threads.map((thread) => ({
|
|
405
|
+
...thread,
|
|
406
|
+
metadata: typeof thread.metadata === "string" ? JSON.parse(thread.metadata) : thread.metadata,
|
|
407
|
+
createdAt: thread.createdAt,
|
|
408
|
+
updatedAt: thread.updatedAt
|
|
409
|
+
}));
|
|
410
|
+
} catch (error) {
|
|
411
|
+
console.error(`Error getting threads for resource ${resourceId}:`, error);
|
|
412
|
+
throw error;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async saveThread({ thread }) {
|
|
416
|
+
try {
|
|
417
|
+
await this.db.insert({
|
|
418
|
+
table: storage.TABLE_THREADS,
|
|
419
|
+
values: [
|
|
420
|
+
{
|
|
421
|
+
...thread,
|
|
422
|
+
createdAt: thread.createdAt.toISOString(),
|
|
423
|
+
updatedAt: thread.updatedAt.toISOString()
|
|
424
|
+
}
|
|
425
|
+
],
|
|
426
|
+
format: "JSONEachRow",
|
|
427
|
+
clickhouse_settings: {
|
|
428
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
429
|
+
date_time_input_format: "best_effort",
|
|
430
|
+
use_client_time_zone: 1,
|
|
431
|
+
output_format_json_quote_64bit_integers: 0
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
return thread;
|
|
435
|
+
} catch (error) {
|
|
436
|
+
console.error("Error saving thread:", error);
|
|
437
|
+
throw error;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
async updateThread({
|
|
441
|
+
id,
|
|
442
|
+
title,
|
|
443
|
+
metadata
|
|
444
|
+
}) {
|
|
445
|
+
try {
|
|
446
|
+
const existingThread = await this.getThreadById({ threadId: id });
|
|
447
|
+
if (!existingThread) {
|
|
448
|
+
throw new Error(`Thread ${id} not found`);
|
|
449
|
+
}
|
|
450
|
+
const mergedMetadata = {
|
|
451
|
+
...existingThread.metadata,
|
|
452
|
+
...metadata
|
|
453
|
+
};
|
|
454
|
+
const updatedThread = {
|
|
455
|
+
...existingThread,
|
|
456
|
+
title,
|
|
457
|
+
metadata: mergedMetadata,
|
|
458
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
459
|
+
};
|
|
460
|
+
await this.db.insert({
|
|
461
|
+
table: storage.TABLE_THREADS,
|
|
462
|
+
values: [
|
|
463
|
+
{
|
|
464
|
+
...updatedThread,
|
|
465
|
+
updatedAt: updatedThread.updatedAt.toISOString()
|
|
466
|
+
}
|
|
467
|
+
],
|
|
468
|
+
format: "JSONEachRow",
|
|
469
|
+
clickhouse_settings: {
|
|
470
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
471
|
+
date_time_input_format: "best_effort",
|
|
472
|
+
use_client_time_zone: 1,
|
|
473
|
+
output_format_json_quote_64bit_integers: 0
|
|
474
|
+
}
|
|
475
|
+
});
|
|
476
|
+
return updatedThread;
|
|
477
|
+
} catch (error) {
|
|
478
|
+
console.error("Error updating thread:", error);
|
|
479
|
+
throw error;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
async deleteThread({ threadId }) {
|
|
483
|
+
try {
|
|
484
|
+
await this.db.command({
|
|
485
|
+
query: `DELETE FROM "${storage.TABLE_MESSAGES}" WHERE thread_id = '${threadId}';`,
|
|
486
|
+
query_params: { var_thread_id: threadId },
|
|
487
|
+
clickhouse_settings: {
|
|
488
|
+
output_format_json_quote_64bit_integers: 0
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
await this.db.command({
|
|
492
|
+
query: `DELETE FROM "${storage.TABLE_THREADS}" WHERE id = {var_id:String};`,
|
|
493
|
+
query_params: { var_id: threadId },
|
|
494
|
+
clickhouse_settings: {
|
|
495
|
+
output_format_json_quote_64bit_integers: 0
|
|
496
|
+
}
|
|
497
|
+
});
|
|
498
|
+
} catch (error) {
|
|
499
|
+
console.error("Error deleting thread:", error);
|
|
500
|
+
throw error;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
async getMessages({ threadId, selectBy }) {
|
|
504
|
+
try {
|
|
505
|
+
const messages = [];
|
|
506
|
+
const limit = typeof selectBy?.last === `number` ? selectBy.last : 40;
|
|
507
|
+
const include = selectBy?.include || [];
|
|
508
|
+
if (include.length) {
|
|
509
|
+
const includeResult = await this.db.query({
|
|
510
|
+
query: `
|
|
511
|
+
WITH ordered_messages AS (
|
|
512
|
+
SELECT
|
|
513
|
+
*,
|
|
514
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
515
|
+
toDateTime64(updatedAt, 3) as updatedAt,
|
|
516
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
517
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
518
|
+
WHERE thread_id = {var_thread_id:String}
|
|
519
|
+
)
|
|
520
|
+
SELECT
|
|
521
|
+
m.id AS id,
|
|
522
|
+
m.content as content,
|
|
523
|
+
m.role as role,
|
|
524
|
+
m.type as type,
|
|
525
|
+
m.createdAt as createdAt,
|
|
526
|
+
m.updatedAt as updatedAt,
|
|
527
|
+
m.thread_id AS "threadId"
|
|
528
|
+
FROM ordered_messages m
|
|
529
|
+
WHERE m.id = ANY({var_include:Array(String)})
|
|
530
|
+
OR EXISTS (
|
|
531
|
+
SELECT 1 FROM ordered_messages target
|
|
532
|
+
WHERE target.id = ANY({var_include:Array(String)})
|
|
533
|
+
AND (
|
|
534
|
+
-- Get previous messages based on the max withPreviousMessages
|
|
535
|
+
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
536
|
+
OR
|
|
537
|
+
-- Get next messages based on the max withNextMessages
|
|
538
|
+
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
539
|
+
)
|
|
540
|
+
)
|
|
541
|
+
ORDER BY m."createdAt" DESC
|
|
542
|
+
`,
|
|
543
|
+
query_params: {
|
|
544
|
+
var_thread_id: threadId,
|
|
545
|
+
var_include: include.map((i) => i.id),
|
|
546
|
+
var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
|
|
547
|
+
var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
|
|
548
|
+
},
|
|
549
|
+
clickhouse_settings: {
|
|
550
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
551
|
+
date_time_input_format: "best_effort",
|
|
552
|
+
date_time_output_format: "iso",
|
|
553
|
+
use_client_time_zone: 1,
|
|
554
|
+
output_format_json_quote_64bit_integers: 0
|
|
555
|
+
}
|
|
556
|
+
});
|
|
557
|
+
const rows2 = await includeResult.json();
|
|
558
|
+
messages.push(...transformRows(rows2.data));
|
|
559
|
+
}
|
|
560
|
+
const result = await this.db.query({
|
|
561
|
+
query: `
|
|
562
|
+
SELECT
|
|
563
|
+
id,
|
|
564
|
+
content,
|
|
565
|
+
role,
|
|
566
|
+
type,
|
|
567
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
568
|
+
thread_id AS "threadId"
|
|
569
|
+
FROM "${storage.TABLE_MESSAGES}"
|
|
570
|
+
WHERE thread_id = {threadId:String}
|
|
571
|
+
AND id NOT IN ({exclude:Array(String)})
|
|
572
|
+
ORDER BY "createdAt" DESC
|
|
573
|
+
LIMIT {limit:Int64}
|
|
574
|
+
`,
|
|
575
|
+
query_params: {
|
|
576
|
+
threadId,
|
|
577
|
+
exclude: messages.map((m) => m.id),
|
|
578
|
+
limit
|
|
579
|
+
},
|
|
580
|
+
clickhouse_settings: {
|
|
581
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
582
|
+
date_time_input_format: "best_effort",
|
|
583
|
+
date_time_output_format: "iso",
|
|
584
|
+
use_client_time_zone: 1,
|
|
585
|
+
output_format_json_quote_64bit_integers: 0
|
|
586
|
+
}
|
|
587
|
+
});
|
|
588
|
+
const rows = await result.json();
|
|
589
|
+
messages.push(...transformRows(rows.data));
|
|
590
|
+
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
591
|
+
messages.forEach((message) => {
|
|
592
|
+
if (typeof message.content === "string") {
|
|
593
|
+
try {
|
|
594
|
+
message.content = JSON.parse(message.content);
|
|
595
|
+
} catch {
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
});
|
|
599
|
+
return messages;
|
|
600
|
+
} catch (error) {
|
|
601
|
+
console.error("Error getting messages:", error);
|
|
602
|
+
throw error;
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
async saveMessages({ messages }) {
|
|
606
|
+
if (messages.length === 0) return messages;
|
|
607
|
+
try {
|
|
608
|
+
const threadId = messages[0]?.threadId;
|
|
609
|
+
if (!threadId) {
|
|
610
|
+
throw new Error("Thread ID is required");
|
|
611
|
+
}
|
|
612
|
+
const thread = await this.getThreadById({ threadId });
|
|
613
|
+
if (!thread) {
|
|
614
|
+
throw new Error(`Thread ${threadId} not found`);
|
|
615
|
+
}
|
|
616
|
+
await this.db.insert({
|
|
617
|
+
table: storage.TABLE_MESSAGES,
|
|
618
|
+
format: "JSONEachRow",
|
|
619
|
+
values: messages.map((message) => ({
|
|
620
|
+
id: message.id,
|
|
621
|
+
thread_id: threadId,
|
|
622
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
623
|
+
createdAt: message.createdAt.toISOString(),
|
|
624
|
+
role: message.role,
|
|
625
|
+
type: message.type
|
|
626
|
+
})),
|
|
627
|
+
clickhouse_settings: {
|
|
628
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
629
|
+
date_time_input_format: "best_effort",
|
|
630
|
+
use_client_time_zone: 1,
|
|
631
|
+
output_format_json_quote_64bit_integers: 0
|
|
632
|
+
}
|
|
633
|
+
});
|
|
634
|
+
return messages;
|
|
635
|
+
} catch (error) {
|
|
636
|
+
console.error("Error saving messages:", error);
|
|
637
|
+
throw error;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
async persistWorkflowSnapshot({
|
|
641
|
+
workflowName,
|
|
642
|
+
runId,
|
|
643
|
+
snapshot
|
|
644
|
+
}) {
|
|
645
|
+
try {
|
|
646
|
+
const currentSnapshot = await this.load({
|
|
647
|
+
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
648
|
+
keys: { workflow_name: workflowName, run_id: runId }
|
|
649
|
+
});
|
|
650
|
+
const now = /* @__PURE__ */ new Date();
|
|
651
|
+
const persisting = currentSnapshot ? {
|
|
652
|
+
...currentSnapshot,
|
|
653
|
+
snapshot: JSON.stringify(snapshot),
|
|
654
|
+
updatedAt: now.toISOString()
|
|
655
|
+
} : {
|
|
656
|
+
workflow_name: workflowName,
|
|
657
|
+
run_id: runId,
|
|
658
|
+
snapshot: JSON.stringify(snapshot),
|
|
659
|
+
createdAt: now.toISOString(),
|
|
660
|
+
updatedAt: now.toISOString()
|
|
661
|
+
};
|
|
662
|
+
await this.db.insert({
|
|
663
|
+
table: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
664
|
+
format: "JSONEachRow",
|
|
665
|
+
values: [persisting],
|
|
666
|
+
clickhouse_settings: {
|
|
667
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
668
|
+
date_time_input_format: "best_effort",
|
|
669
|
+
use_client_time_zone: 1,
|
|
670
|
+
output_format_json_quote_64bit_integers: 0
|
|
671
|
+
}
|
|
672
|
+
});
|
|
673
|
+
} catch (error) {
|
|
674
|
+
console.error("Error persisting workflow snapshot:", error);
|
|
675
|
+
throw error;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
async loadWorkflowSnapshot({
|
|
679
|
+
workflowName,
|
|
680
|
+
runId
|
|
681
|
+
}) {
|
|
682
|
+
try {
|
|
683
|
+
const result = await this.load({
|
|
684
|
+
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
685
|
+
keys: {
|
|
686
|
+
workflow_name: workflowName,
|
|
687
|
+
run_id: runId
|
|
688
|
+
}
|
|
689
|
+
});
|
|
690
|
+
if (!result) {
|
|
691
|
+
return null;
|
|
692
|
+
}
|
|
693
|
+
return result.snapshot;
|
|
694
|
+
} catch (error) {
|
|
695
|
+
console.error("Error loading workflow snapshot:", error);
|
|
696
|
+
throw error;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
async getWorkflowRuns({
|
|
700
|
+
workflowName,
|
|
701
|
+
fromDate,
|
|
702
|
+
toDate,
|
|
703
|
+
limit,
|
|
704
|
+
offset
|
|
705
|
+
} = {}) {
|
|
706
|
+
try {
|
|
707
|
+
const conditions = [];
|
|
708
|
+
const values = {};
|
|
709
|
+
if (workflowName) {
|
|
710
|
+
conditions.push(`workflow_name = {var_workflow_name:String}`);
|
|
711
|
+
values.var_workflow_name = workflowName;
|
|
712
|
+
}
|
|
713
|
+
if (fromDate) {
|
|
714
|
+
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
715
|
+
values.var_from_date = fromDate.getTime() / 1e3;
|
|
716
|
+
}
|
|
717
|
+
if (toDate) {
|
|
718
|
+
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
719
|
+
values.var_to_date = toDate.getTime() / 1e3;
|
|
720
|
+
}
|
|
721
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
722
|
+
const limitClause = limit !== void 0 ? `LIMIT ${limit}` : "";
|
|
723
|
+
const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
|
|
724
|
+
let total = 0;
|
|
725
|
+
if (limit !== void 0 && offset !== void 0) {
|
|
726
|
+
const countResult = await this.db.query({
|
|
727
|
+
query: `SELECT COUNT(*) as count FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
|
|
728
|
+
query_params: values,
|
|
729
|
+
format: "JSONEachRow"
|
|
730
|
+
});
|
|
731
|
+
const countRows = await countResult.json();
|
|
732
|
+
total = Number(countRows[0]?.count ?? 0);
|
|
733
|
+
}
|
|
734
|
+
const result = await this.db.query({
|
|
735
|
+
query: `
|
|
736
|
+
SELECT
|
|
737
|
+
workflow_name,
|
|
738
|
+
run_id,
|
|
739
|
+
snapshot,
|
|
740
|
+
toDateTime64(createdAt, 3) as createdAt,
|
|
741
|
+
toDateTime64(updatedAt, 3) as updatedAt
|
|
742
|
+
FROM ${storage.TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[storage.TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
743
|
+
${whereClause}
|
|
744
|
+
ORDER BY createdAt DESC
|
|
745
|
+
${limitClause}
|
|
746
|
+
${offsetClause}
|
|
747
|
+
`,
|
|
748
|
+
query_params: values,
|
|
749
|
+
format: "JSONEachRow"
|
|
750
|
+
});
|
|
751
|
+
const resultJson = await result.json();
|
|
752
|
+
const rows = resultJson;
|
|
753
|
+
const runs = rows.map((row) => {
|
|
754
|
+
let parsedSnapshot = row.snapshot;
|
|
755
|
+
if (typeof parsedSnapshot === "string") {
|
|
756
|
+
try {
|
|
757
|
+
parsedSnapshot = JSON.parse(row.snapshot);
|
|
758
|
+
} catch (e) {
|
|
759
|
+
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
return {
|
|
763
|
+
workflowName: row.workflow_name,
|
|
764
|
+
runId: row.run_id,
|
|
765
|
+
snapshot: parsedSnapshot,
|
|
766
|
+
createdAt: new Date(row.createdAt),
|
|
767
|
+
updatedAt: new Date(row.updatedAt)
|
|
768
|
+
};
|
|
769
|
+
});
|
|
770
|
+
return { runs, total: total || runs.length };
|
|
771
|
+
} catch (error) {
|
|
772
|
+
console.error("Error getting workflow runs:", error);
|
|
773
|
+
throw error;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
async close() {
|
|
777
|
+
await this.db.close();
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
|
|
781
|
+
exports.ClickhouseStore = ClickhouseStore;
|