@mastra/clickhouse 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +27 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-clickhouse.md +52 -7
- package/dist/docs/references/reference-storage-composite.md +34 -2
- package/dist/index.cjs +243 -38
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +244 -39
- package/dist/index.js.map +1 -1
- package/dist/storage/db/index.d.ts +7 -1
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/db/replication.d.ts +52 -0
- package/dist/storage/db/replication.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts +2 -0
- package/dist/storage/db/utils.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +8 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/package.json +8 -7
package/dist/index.cjs
CHANGED
|
@@ -10,6 +10,139 @@ var features = require('@mastra/core/features');
|
|
|
10
10
|
var evals = require('@mastra/core/evals');
|
|
11
11
|
|
|
12
12
|
// src/storage/index.ts
|
|
13
|
+
var DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
|
|
14
|
+
var DEFAULT_REPLICA_NAME = "{replica}";
|
|
15
|
+
var REPLICATED_ENGINE_NAMES = /* @__PURE__ */ new Set(["ReplicatedMergeTree", "ReplicatedReplacingMergeTree"]);
|
|
16
|
+
var SUPPORTED_ENGINE_NAMES = /* @__PURE__ */ new Set(["MergeTree", "ReplacingMergeTree", ...REPLICATED_ENGINE_NAMES]);
|
|
17
|
+
function isReplicationConfigured(replication) {
|
|
18
|
+
return replication !== void 0;
|
|
19
|
+
}
|
|
20
|
+
function isReplicatedOrSharedEngine(engine) {
|
|
21
|
+
if (!engine) return false;
|
|
22
|
+
return engine.startsWith("Replicated") || engine.startsWith("Shared");
|
|
23
|
+
}
|
|
24
|
+
function assertReplicationConfigField(fieldName, value, errorCode) {
|
|
25
|
+
if (value === void 0) return;
|
|
26
|
+
if (value.trim() === "") {
|
|
27
|
+
throw new error.MastraError({
|
|
28
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION_CONFIG", errorCode),
|
|
29
|
+
domain: error.ErrorDomain.STORAGE,
|
|
30
|
+
category: error.ErrorCategory.USER,
|
|
31
|
+
text: `ClickHouse replication.${fieldName} must be a non-empty string when provided.`
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
if (/\s/.test(value) || value.includes("'") || value.includes('"') || value.includes("\\")) {
|
|
35
|
+
throw new error.MastraError({
|
|
36
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION_CONFIG", errorCode),
|
|
37
|
+
domain: error.ErrorDomain.STORAGE,
|
|
38
|
+
category: error.ErrorCategory.USER,
|
|
39
|
+
text: `ClickHouse replication.${fieldName} must not contain whitespace or quote characters.`
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function validateReplicationConfig(replication) {
|
|
44
|
+
if (!replication) return;
|
|
45
|
+
assertReplicationConfigField("cluster", replication.cluster, "INVALID_CLUSTER");
|
|
46
|
+
assertReplicationConfigField("zookeeperPath", replication.zookeeperPath, "INVALID_ZOOKEEPER_PATH");
|
|
47
|
+
assertReplicationConfigField("replicaName", replication.replicaName, "INVALID_REPLICA_NAME");
|
|
48
|
+
}
|
|
49
|
+
function quoteClickhouseString(value) {
|
|
50
|
+
return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
51
|
+
}
|
|
52
|
+
function getEngineNameAndArgs(engine) {
|
|
53
|
+
const trimmed = engine.trim();
|
|
54
|
+
const match = trimmed.match(/^(\w+)\s*(?:\((.*)\))?$/s);
|
|
55
|
+
if (!match) return null;
|
|
56
|
+
const name = match[1];
|
|
57
|
+
if (!name) return null;
|
|
58
|
+
const args = match[2]?.trim() ?? "";
|
|
59
|
+
if (args && !hasBalancedParens(args)) return null;
|
|
60
|
+
return { name, args };
|
|
61
|
+
}
|
|
62
|
+
function hasBalancedParens(s) {
|
|
63
|
+
let depth = 0;
|
|
64
|
+
for (const c of s) {
|
|
65
|
+
if (c === "(") depth++;
|
|
66
|
+
else if (c === ")") {
|
|
67
|
+
depth--;
|
|
68
|
+
if (depth < 0) return false;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return depth === 0;
|
|
72
|
+
}
|
|
73
|
+
function buildReplicatedTableEngine(engine, replication) {
|
|
74
|
+
if (!replication) return engine;
|
|
75
|
+
const parsed = getEngineNameAndArgs(engine);
|
|
76
|
+
if (!parsed || !SUPPORTED_ENGINE_NAMES.has(parsed.name)) return engine;
|
|
77
|
+
if (isReplicatedOrSharedEngine(parsed.name)) return engine;
|
|
78
|
+
const zookeeperPath = quoteClickhouseString(replication.zookeeperPath ?? DEFAULT_ZOOKEEPER_PATH);
|
|
79
|
+
const replicaName = quoteClickhouseString(replication.replicaName ?? DEFAULT_REPLICA_NAME);
|
|
80
|
+
const replicatedName = parsed.name === "ReplacingMergeTree" ? "ReplicatedReplacingMergeTree" : "ReplicatedMergeTree";
|
|
81
|
+
const args = [zookeeperPath, replicaName, parsed.args].filter(Boolean).join(", ");
|
|
82
|
+
return `${replicatedName}(${args})`;
|
|
83
|
+
}
|
|
84
|
+
function addOnClusterToDDL(sql, replication) {
|
|
85
|
+
const cluster = replication?.cluster?.trim();
|
|
86
|
+
if (!cluster) return sql;
|
|
87
|
+
const quotedCluster = quoteClickhouseString(cluster);
|
|
88
|
+
const onClusterSuffix = ` ON CLUSTER ${quotedCluster}`;
|
|
89
|
+
const rewrite = (input, pattern) => {
|
|
90
|
+
return input.replace(pattern, (...args) => {
|
|
91
|
+
const match = args[0];
|
|
92
|
+
const source = args[args.length - 1];
|
|
93
|
+
const offset = args[args.length - 2];
|
|
94
|
+
const tail = source.slice(offset + match.length);
|
|
95
|
+
if (/^\s+ON\s+CLUSTER\s/i.test(tail)) return match;
|
|
96
|
+
return match + onClusterSuffix;
|
|
97
|
+
});
|
|
98
|
+
};
|
|
99
|
+
let out = sql;
|
|
100
|
+
out = rewrite(out, /\bCREATE\s+TABLE\s+(IF\s+NOT\s+EXISTS\s+)?[^\s(]+/gi);
|
|
101
|
+
out = rewrite(out, /\bCREATE\s+MATERIALIZED\s+VIEW\s+(IF\s+NOT\s+EXISTS\s+)?[^\s(]+/gi);
|
|
102
|
+
out = rewrite(out, /\bALTER\s+TABLE\s+[^\s]+/gi);
|
|
103
|
+
out = rewrite(out, /\bDROP\s+(TABLE|VIEW)\s+(IF\s+EXISTS\s+)?[^\s]+/gi);
|
|
104
|
+
out = rewrite(out, /\bTRUNCATE\s+TABLE\s+(IF\s+EXISTS\s+)?[^\s]+/gi);
|
|
105
|
+
out = rewrite(out, /\bOPTIMIZE\s+TABLE\s+[^\s]+/gi);
|
|
106
|
+
out = rewrite(out, /\bSYSTEM\s+(REFRESH|WAIT)\s+VIEW\s+[^\s;]+/gi);
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
function rewriteEngineClauses(sql, replication) {
|
|
110
|
+
return sql.replace(/ENGINE\s*=\s*(\w+)\s*/gi, (match, engineName, offset, source) => {
|
|
111
|
+
const argsStart = offset + match.length;
|
|
112
|
+
if (source[argsStart] !== "(") {
|
|
113
|
+
return `ENGINE = ${buildReplicatedTableEngine(engineName, replication)}`;
|
|
114
|
+
}
|
|
115
|
+
let depth = 0;
|
|
116
|
+
for (let i = argsStart; i < source.length; i++) {
|
|
117
|
+
const char = source[i];
|
|
118
|
+
if (char === "(") depth++;
|
|
119
|
+
else if (char === ")") {
|
|
120
|
+
depth--;
|
|
121
|
+
if (depth === 0) {
|
|
122
|
+
const engine = `${engineName}${source.slice(argsStart, i + 1)}`;
|
|
123
|
+
return `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return match;
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
function applyReplicationToDDL(sql, replication) {
|
|
131
|
+
const withReplicatedEngine = replication ? rewriteEngineClauses(sql, replication) : sql;
|
|
132
|
+
return addOnClusterToDDL(withReplicatedEngine, replication);
|
|
133
|
+
}
|
|
134
|
+
function buildLocalTableReplicationError(tables) {
|
|
135
|
+
const tableList = tables.map((table) => ` - ${table.name} (${table.engine})`).join("\n");
|
|
136
|
+
return new error.MastraError({
|
|
137
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "LOCAL_TABLES_UNSUPPORTED"),
|
|
138
|
+
domain: error.ErrorDomain.STORAGE,
|
|
139
|
+
category: error.ErrorCategory.USER,
|
|
140
|
+
text: `ClickHouse replication is enabled, but existing Mastra tables use non-replicated local engines.
|
|
141
|
+
Mastra will not automatically convert existing local tables to replicated tables.
|
|
142
|
+
Please migrate or recreate these tables manually before enabling replication:
|
|
143
|
+
${tableList}`
|
|
144
|
+
});
|
|
145
|
+
}
|
|
13
146
|
var TABLE_ENGINES = {
|
|
14
147
|
[storage.TABLE_MESSAGES]: `MergeTree()`,
|
|
15
148
|
[storage.TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
@@ -46,8 +179,11 @@ var TABLE_ENGINES = {
|
|
|
46
179
|
mastra_background_tasks: `ReplacingMergeTree()`,
|
|
47
180
|
[storage.TABLE_SCHEDULES]: `ReplacingMergeTree()`,
|
|
48
181
|
[storage.TABLE_SCHEDULE_TRIGGERS]: `MergeTree()`,
|
|
182
|
+
[storage.TABLE_NOTIFICATIONS]: `ReplacingMergeTree()`,
|
|
183
|
+
[storage.TABLE_HARNESS_SESSIONS]: `ReplacingMergeTree()`,
|
|
49
184
|
mastra_channel_installations: `ReplacingMergeTree()`,
|
|
50
|
-
mastra_channel_config: `ReplacingMergeTree()
|
|
185
|
+
mastra_channel_config: `ReplacingMergeTree()`,
|
|
186
|
+
[storage.TABLE_THREAD_STATE]: `ReplacingMergeTree()`
|
|
51
187
|
};
|
|
52
188
|
var COLUMN_TYPES = {
|
|
53
189
|
text: "String",
|
|
@@ -95,8 +231,9 @@ function transformRows(rows) {
|
|
|
95
231
|
|
|
96
232
|
// src/storage/db/index.ts
|
|
97
233
|
function resolveClickhouseConfig(config) {
|
|
234
|
+
validateReplicationConfig(config.replication);
|
|
98
235
|
if ("client" in config) {
|
|
99
|
-
return { client: config.client, ttl: config.ttl };
|
|
236
|
+
return { client: config.client, ttl: config.ttl, replication: config.replication };
|
|
100
237
|
}
|
|
101
238
|
const client$1 = client.createClient({
|
|
102
239
|
url: config.url,
|
|
@@ -109,18 +246,25 @@ function resolveClickhouseConfig(config) {
|
|
|
109
246
|
output_format_json_quote_64bit_integers: 0
|
|
110
247
|
}
|
|
111
248
|
});
|
|
112
|
-
return { client: client$1, ttl: config.ttl };
|
|
249
|
+
return { client: client$1, ttl: config.ttl, replication: config.replication };
|
|
113
250
|
}
|
|
114
251
|
var ClickhouseDB = class extends base.MastraBase {
|
|
115
252
|
ttl;
|
|
253
|
+
replication;
|
|
116
254
|
client;
|
|
117
255
|
/** Cache of actual table columns: tableName -> Promise<Set<columnName>> (stores in-flight promise to coalesce concurrent calls) */
|
|
118
256
|
tableColumnsCache = /* @__PURE__ */ new Map();
|
|
119
|
-
constructor({
|
|
257
|
+
constructor({
|
|
258
|
+
client,
|
|
259
|
+
ttl,
|
|
260
|
+
replication
|
|
261
|
+
}) {
|
|
120
262
|
super({
|
|
121
263
|
name: "CLICKHOUSE_DB"
|
|
122
264
|
});
|
|
265
|
+
validateReplicationConfig(replication);
|
|
123
266
|
this.ttl = ttl;
|
|
267
|
+
this.replication = replication;
|
|
124
268
|
this.client = client;
|
|
125
269
|
}
|
|
126
270
|
/**
|
|
@@ -188,6 +332,19 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
188
332
|
return false;
|
|
189
333
|
}
|
|
190
334
|
}
|
|
335
|
+
async assertExistingTableCompatibleWithReplication(tableName) {
|
|
336
|
+
if (!isReplicationConfigured(this.replication)) return;
|
|
337
|
+
const result = await this.client.query({
|
|
338
|
+
query: `SELECT name, engine FROM system.tables WHERE database = currentDatabase() AND name = {tableName:String}`,
|
|
339
|
+
query_params: { tableName },
|
|
340
|
+
format: "JSONEachRow"
|
|
341
|
+
});
|
|
342
|
+
const rows = await result.json();
|
|
343
|
+
const localTables = rows.filter((row) => row.engine && !isReplicatedOrSharedEngine(row.engine));
|
|
344
|
+
if (localTables.length > 0) {
|
|
345
|
+
throw buildLocalTableReplicationError(localTables);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
191
348
|
/**
|
|
192
349
|
* Gets the sorting key (ORDER BY columns) for a table.
|
|
193
350
|
* Returns null if the table doesn't exist.
|
|
@@ -291,6 +448,14 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
291
448
|
this.logger?.debug?.(`Spans table already has correct sorting key: ${currentSortingKey}`);
|
|
292
449
|
return false;
|
|
293
450
|
}
|
|
451
|
+
if (isReplicationConfigured(this.replication)) {
|
|
452
|
+
throw new error.MastraError({
|
|
453
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "SPANS_SORTING_KEY_MIGRATION_UNSUPPORTED"),
|
|
454
|
+
domain: error.ErrorDomain.STORAGE,
|
|
455
|
+
category: error.ErrorCategory.USER,
|
|
456
|
+
text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap spans table migrations automatically. Migrate the existing spans table manually before enabling replication."
|
|
457
|
+
});
|
|
458
|
+
}
|
|
294
459
|
this.logger?.info?.(`Migrating spans table from sorting key "${currentSortingKey}" to "(traceId, spanId)"`);
|
|
295
460
|
const backupTableName = `${tableName}_backup_${Date.now()}`;
|
|
296
461
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
@@ -341,7 +506,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
341
506
|
SELECT ${selectExpressions}
|
|
342
507
|
FROM ${backupTableName}
|
|
343
508
|
ORDER BY traceId, spanId,
|
|
344
|
-
(endedAt IS NOT NULL
|
|
509
|
+
(endedAt IS NOT NULL) DESC,
|
|
345
510
|
COALESCE(updatedAt, createdAt) DESC,
|
|
346
511
|
createdAt DESC
|
|
347
512
|
LIMIT 1 BY traceId, spanId`
|
|
@@ -404,6 +569,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
404
569
|
schema
|
|
405
570
|
}) {
|
|
406
571
|
try {
|
|
572
|
+
await this.assertExistingTableCompatibleWithReplication(tableName);
|
|
407
573
|
const columns = Object.entries(schema).map(([name, def]) => {
|
|
408
574
|
let sqlType = this.getSqlType(def.type);
|
|
409
575
|
let isNullable = def.nullable === true;
|
|
@@ -457,7 +623,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
457
623
|
`;
|
|
458
624
|
}
|
|
459
625
|
await this.client.query({
|
|
460
|
-
query: sql,
|
|
626
|
+
query: applyReplicationToDDL(sql, this.replication),
|
|
461
627
|
clickhouse_settings: {
|
|
462
628
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
463
629
|
date_time_input_format: "best_effort",
|
|
@@ -502,7 +668,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
502
668
|
const defaultValue = columnDef.nullable === false ? storage.getDefaultValue(columnDef.type) : "";
|
|
503
669
|
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
504
670
|
await this.client.query({
|
|
505
|
-
query: alterSql
|
|
671
|
+
query: addOnClusterToDDL(alterSql, this.replication)
|
|
506
672
|
});
|
|
507
673
|
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
508
674
|
}
|
|
@@ -525,7 +691,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
525
691
|
try {
|
|
526
692
|
await this.client.command({ query: `SYSTEM STOP MERGES ${tableName}` });
|
|
527
693
|
await this.client.command({
|
|
528
|
-
query: `TRUNCATE TABLE ${tableName}
|
|
694
|
+
query: addOnClusterToDDL(`TRUNCATE TABLE ${tableName}`, this.replication)
|
|
529
695
|
});
|
|
530
696
|
await this.client.command({ query: `SYSTEM START MERGES ${tableName}` });
|
|
531
697
|
} catch (error$1) {
|
|
@@ -543,7 +709,7 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
543
709
|
async dropTable({ tableName }) {
|
|
544
710
|
try {
|
|
545
711
|
await this.client.query({
|
|
546
|
-
query: `DROP TABLE IF EXISTS ${tableName}
|
|
712
|
+
query: addOnClusterToDDL(`DROP TABLE IF EXISTS ${tableName}`, this.replication)
|
|
547
713
|
});
|
|
548
714
|
} catch (error$1) {
|
|
549
715
|
throw new error.MastraError(
|
|
@@ -737,9 +903,9 @@ var BackgroundTasksStorageClickhouse = class extends storage.BackgroundTasksStor
|
|
|
737
903
|
#db;
|
|
738
904
|
constructor(config) {
|
|
739
905
|
super();
|
|
740
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
906
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
741
907
|
this.client = client;
|
|
742
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
908
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
743
909
|
}
|
|
744
910
|
async init() {
|
|
745
911
|
await this.#db.createTable({ tableName: storage.TABLE_BACKGROUND_TASKS, schema: storage.TABLE_SCHEMAS[storage.TABLE_BACKGROUND_TASKS] });
|
|
@@ -931,9 +1097,9 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
|
|
|
931
1097
|
#db;
|
|
932
1098
|
constructor(config) {
|
|
933
1099
|
super();
|
|
934
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
1100
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
935
1101
|
this.client = client;
|
|
936
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
1102
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
937
1103
|
}
|
|
938
1104
|
async init() {
|
|
939
1105
|
await this.#db.createTable({ tableName: storage.TABLE_THREADS, schema: storage.TABLE_SCHEMAS[storage.TABLE_THREADS] });
|
|
@@ -2194,9 +2360,9 @@ var ObservabilityStorageClickhouse = class extends storage.ObservabilityStorage
|
|
|
2194
2360
|
#db;
|
|
2195
2361
|
constructor(config) {
|
|
2196
2362
|
super();
|
|
2197
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
2363
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
2198
2364
|
this.client = client;
|
|
2199
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
2365
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
2200
2366
|
}
|
|
2201
2367
|
async init() {
|
|
2202
2368
|
const migrationStatus = await this.#db.checkSpansMigrationStatus(storage.TABLE_SPANS);
|
|
@@ -6990,7 +7156,20 @@ async function filterAppliedRetention(client, entries) {
|
|
|
6990
7156
|
return current.column !== e.column || current.days !== e.days;
|
|
6991
7157
|
});
|
|
6992
7158
|
}
|
|
6993
|
-
async function
|
|
7159
|
+
async function assertExistingTablesCompatibleWithReplication(client, replication) {
|
|
7160
|
+
if (!isReplicationConfigured(replication)) return;
|
|
7161
|
+
const result = await client.query({
|
|
7162
|
+
query: `SELECT name, engine FROM system.tables WHERE database = currentDatabase() AND name IN ({tables:Array(String)})`,
|
|
7163
|
+
query_params: { tables: [...ALL_TABLE_NAMES] },
|
|
7164
|
+
format: "JSONEachRow"
|
|
7165
|
+
});
|
|
7166
|
+
const rows = await result.json();
|
|
7167
|
+
const localTable = rows.find((row) => !isReplicatedOrSharedEngine(row.engine));
|
|
7168
|
+
if (localTable) {
|
|
7169
|
+
throw buildLocalTableReplicationError([{ name: localTable.name, engine: localTable.engine }]);
|
|
7170
|
+
}
|
|
7171
|
+
}
|
|
7172
|
+
async function reconcileDiscoveryTables(client, replication) {
|
|
6994
7173
|
let engines;
|
|
6995
7174
|
try {
|
|
6996
7175
|
const result = await client.query({
|
|
@@ -7010,8 +7189,8 @@ async function reconcileDiscoveryTables(client) {
|
|
|
7010
7189
|
for (const { table, mv } of targets) {
|
|
7011
7190
|
const engine = engines.get(table);
|
|
7012
7191
|
if (!engine || isReplacingMergeTreeEngine(engine)) continue;
|
|
7013
|
-
await client.command({ query: `DROP VIEW IF EXISTS ${mv}
|
|
7014
|
-
await client.command({ query: `DROP TABLE IF EXISTS ${table}
|
|
7192
|
+
await client.command({ query: addOnClusterToDDL(`DROP VIEW IF EXISTS ${mv}`, replication) });
|
|
7193
|
+
await client.command({ query: addOnClusterToDDL(`DROP TABLE IF EXISTS ${table}`, replication) });
|
|
7015
7194
|
}
|
|
7016
7195
|
}
|
|
7017
7196
|
async function queryNamesByTable(client, query, tables) {
|
|
@@ -7090,12 +7269,14 @@ async function detectExistingDeltaCursorStrategy(client) {
|
|
|
7090
7269
|
var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilityStorage {
|
|
7091
7270
|
#client;
|
|
7092
7271
|
#retention;
|
|
7272
|
+
#replication;
|
|
7093
7273
|
#deltaCursorStrategyOverride;
|
|
7094
7274
|
#deltaCursorStrategy = "fallback";
|
|
7095
7275
|
constructor(config) {
|
|
7096
7276
|
super();
|
|
7097
|
-
const { client } = resolveClickhouseConfig(config);
|
|
7277
|
+
const { client, replication } = resolveClickhouseConfig(config);
|
|
7098
7278
|
this.#client = client;
|
|
7279
|
+
this.#replication = replication;
|
|
7099
7280
|
this.#retention = config.retention;
|
|
7100
7281
|
this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
|
|
7101
7282
|
}
|
|
@@ -7116,6 +7297,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7116
7297
|
});
|
|
7117
7298
|
}
|
|
7118
7299
|
try {
|
|
7300
|
+
await assertExistingTablesCompatibleWithReplication(this.#client, this.#replication);
|
|
7119
7301
|
const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
|
|
7120
7302
|
if (existingStrategy === "mixed") {
|
|
7121
7303
|
this.#deltaCursorStrategy = null;
|
|
@@ -7129,19 +7311,19 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7129
7311
|
} else {
|
|
7130
7312
|
this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
|
|
7131
7313
|
}
|
|
7132
|
-
await reconcileDiscoveryTables(this.#client);
|
|
7314
|
+
await reconcileDiscoveryTables(this.#client, this.#replication);
|
|
7133
7315
|
const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
|
|
7134
7316
|
for (const ddl of coreDdl) {
|
|
7135
|
-
await this.#client.command({ query: ddl });
|
|
7317
|
+
await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
|
|
7136
7318
|
}
|
|
7137
7319
|
const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
|
|
7138
7320
|
for (const migration of pendingMigrations) {
|
|
7139
|
-
await this.#client.command({ query: migration.sql });
|
|
7321
|
+
await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
|
|
7140
7322
|
}
|
|
7141
7323
|
if (this.#retention) {
|
|
7142
7324
|
const pendingRetention = await filterAppliedRetention(this.#client, buildRetentionEntries(this.#retention));
|
|
7143
7325
|
for (const entry of pendingRetention) {
|
|
7144
|
-
await this.#client.command({ query: entry.sql });
|
|
7326
|
+
await this.#client.command({ query: addOnClusterToDDL(entry.sql, this.#replication) });
|
|
7145
7327
|
}
|
|
7146
7328
|
}
|
|
7147
7329
|
if (this.#deltaCursorStrategy === "serial") {
|
|
@@ -7170,12 +7352,20 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7170
7352
|
}
|
|
7171
7353
|
try {
|
|
7172
7354
|
for (const ddl of DISCOVERY_MV_DDL) {
|
|
7173
|
-
await this.#client.command({ query: ddl });
|
|
7355
|
+
await this.#client.command({ query: addOnClusterToDDL(ddl, this.#replication) });
|
|
7174
7356
|
}
|
|
7175
|
-
await this.#client.command({
|
|
7176
|
-
|
|
7177
|
-
|
|
7178
|
-
await this.#client.command({
|
|
7357
|
+
await this.#client.command({
|
|
7358
|
+
query: addOnClusterToDDL(`SYSTEM REFRESH VIEW ${MV_DISCOVERY_VALUES}`, this.#replication)
|
|
7359
|
+
});
|
|
7360
|
+
await this.#client.command({
|
|
7361
|
+
query: addOnClusterToDDL(`SYSTEM WAIT VIEW ${MV_DISCOVERY_VALUES}`, this.#replication)
|
|
7362
|
+
});
|
|
7363
|
+
await this.#client.command({
|
|
7364
|
+
query: addOnClusterToDDL(`SYSTEM REFRESH VIEW ${MV_DISCOVERY_PAIRS}`, this.#replication)
|
|
7365
|
+
});
|
|
7366
|
+
await this.#client.command({
|
|
7367
|
+
query: addOnClusterToDDL(`SYSTEM WAIT VIEW ${MV_DISCOVERY_PAIRS}`, this.#replication)
|
|
7368
|
+
});
|
|
7179
7369
|
} catch {
|
|
7180
7370
|
}
|
|
7181
7371
|
}
|
|
@@ -7194,6 +7384,14 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7194
7384
|
message: "Migration already complete. Signal tables already use signal-ID dedupe keys."
|
|
7195
7385
|
};
|
|
7196
7386
|
}
|
|
7387
|
+
if (isReplicationConfigured(this.#replication)) {
|
|
7388
|
+
throw new error.MastraError({
|
|
7389
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "SIGNAL_TABLES_MIGRATION_UNSUPPORTED"),
|
|
7390
|
+
domain: error.ErrorDomain.STORAGE,
|
|
7391
|
+
category: error.ErrorCategory.USER,
|
|
7392
|
+
text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap signal table migrations automatically. Migrate existing local signal tables manually before enabling replication."
|
|
7393
|
+
});
|
|
7394
|
+
}
|
|
7197
7395
|
await migrateSignalTables(this.#client, this.logger);
|
|
7198
7396
|
return {
|
|
7199
7397
|
success: true,
|
|
@@ -7890,7 +8088,11 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7890
8088
|
async dangerouslyClearAll() {
|
|
7891
8089
|
try {
|
|
7892
8090
|
await Promise.all(
|
|
7893
|
-
ALL_TABLE_NAMES.map(
|
|
8091
|
+
ALL_TABLE_NAMES.map(
|
|
8092
|
+
(table) => this.#client.command({
|
|
8093
|
+
query: addOnClusterToDDL(`TRUNCATE TABLE IF EXISTS ${table}`, this.#replication)
|
|
8094
|
+
})
|
|
8095
|
+
)
|
|
7894
8096
|
);
|
|
7895
8097
|
} catch (error$1) {
|
|
7896
8098
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
@@ -7910,9 +8112,9 @@ var ScoresStorageClickhouse = class extends storage.ScoresStorage {
|
|
|
7910
8112
|
#db;
|
|
7911
8113
|
constructor(config) {
|
|
7912
8114
|
super();
|
|
7913
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
8115
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
7914
8116
|
this.client = client;
|
|
7915
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
8117
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
7916
8118
|
}
|
|
7917
8119
|
async init() {
|
|
7918
8120
|
await this.#db.createTable({ tableName: storage.TABLE_SCORERS, schema: storage.TABLE_SCHEMAS[storage.TABLE_SCORERS] });
|
|
@@ -8337,9 +8539,9 @@ var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
|
|
|
8337
8539
|
#db;
|
|
8338
8540
|
constructor(config) {
|
|
8339
8541
|
super();
|
|
8340
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
8542
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
8341
8543
|
this.client = client;
|
|
8342
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
8544
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
8343
8545
|
}
|
|
8344
8546
|
supportsConcurrentUpdates() {
|
|
8345
8547
|
return false;
|
|
@@ -8631,9 +8833,11 @@ var isClientConfig = (config) => {
|
|
|
8631
8833
|
var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
8632
8834
|
db;
|
|
8633
8835
|
ttl = {};
|
|
8836
|
+
replication;
|
|
8634
8837
|
stores;
|
|
8635
8838
|
constructor(config) {
|
|
8636
8839
|
super({ id: config.id, name: "ClickhouseStore", disableInit: config.disableInit });
|
|
8840
|
+
validateReplicationConfig(config.replication);
|
|
8637
8841
|
if (isClientConfig(config)) {
|
|
8638
8842
|
this.db = config.client;
|
|
8639
8843
|
} else {
|
|
@@ -8646,7 +8850,7 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
8646
8850
|
if (typeof config.password !== "string") {
|
|
8647
8851
|
throw new Error("ClickhouseStore: password must be a string.");
|
|
8648
8852
|
}
|
|
8649
|
-
const { id, ttl, disableInit, clickhouse_settings, ...clientOptions } = config;
|
|
8853
|
+
const { id, ttl, disableInit, replication, clickhouse_settings, ...clientOptions } = config;
|
|
8650
8854
|
this.db = client.createClient({
|
|
8651
8855
|
...clientOptions,
|
|
8652
8856
|
clickhouse_settings: {
|
|
@@ -8660,7 +8864,8 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
8660
8864
|
});
|
|
8661
8865
|
}
|
|
8662
8866
|
this.ttl = config.ttl;
|
|
8663
|
-
|
|
8867
|
+
this.replication = config.replication;
|
|
8868
|
+
const domainConfig = { client: this.db, ttl: this.ttl, replication: config.replication };
|
|
8664
8869
|
const workflows = new WorkflowsStorageClickhouse(domainConfig);
|
|
8665
8870
|
const scores = new ScoresStorageClickhouse(domainConfig);
|
|
8666
8871
|
const memory = new MemoryStorageClickhouse(domainConfig);
|
|
@@ -8676,7 +8881,7 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
8676
8881
|
async optimizeTable({ tableName }) {
|
|
8677
8882
|
try {
|
|
8678
8883
|
await this.db.command({
|
|
8679
|
-
query: `OPTIMIZE TABLE ${tableName} FINAL
|
|
8884
|
+
query: addOnClusterToDDL(`OPTIMIZE TABLE ${tableName} FINAL`, this.replication)
|
|
8680
8885
|
});
|
|
8681
8886
|
} catch (error$1) {
|
|
8682
8887
|
throw new error.MastraError(
|
|
@@ -8693,7 +8898,7 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
8693
8898
|
async materializeTtl({ tableName }) {
|
|
8694
8899
|
try {
|
|
8695
8900
|
await this.db.command({
|
|
8696
|
-
query: `ALTER TABLE ${tableName} MATERIALIZE TTL
|
|
8901
|
+
query: addOnClusterToDDL(`ALTER TABLE ${tableName} MATERIALIZE TTL`, this.replication) + ";"
|
|
8697
8902
|
});
|
|
8698
8903
|
} catch (error$1) {
|
|
8699
8904
|
throw new error.MastraError(
|
|
@@ -8732,7 +8937,7 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
|
|
|
8732
8937
|
constructor(config) {
|
|
8733
8938
|
super(config);
|
|
8734
8939
|
this.name = "ClickhouseStoreVNext";
|
|
8735
|
-
const observability = new ObservabilityStorageClickhouseVNext({ client: this.db });
|
|
8940
|
+
const observability = new ObservabilityStorageClickhouseVNext({ client: this.db, replication: config.replication });
|
|
8736
8941
|
this.stores = {
|
|
8737
8942
|
...this.stores,
|
|
8738
8943
|
observability
|