@mastra/clickhouse 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +27 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-clickhouse.md +52 -7
- package/dist/docs/references/reference-storage-composite.md +34 -2
- package/dist/index.cjs +243 -38
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +244 -39
- package/dist/index.js.map +1 -1
- package/dist/storage/db/index.d.ts +7 -1
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/db/replication.d.ts +52 -0
- package/dist/storage/db/replication.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts +2 -0
- package/dist/storage/db/utils.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +8 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/package.json +8 -7
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
-
import { BRANCH_SPAN_TYPES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEDULES, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_FAVORITES, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SKILLS, TABLE_WORKSPACE_VERSIONS, TABLE_WORKSPACES, TABLE_MCP_SERVER_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_CLIENTS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SCORER_DEFINITIONS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_PROMPT_BLOCKS, TABLE_EXPERIMENT_RESULTS, TABLE_EXPERIMENTS, TABLE_DATASET_VERSIONS, TABLE_DATASET_ITEMS, TABLE_DATASETS, TABLE_AGENT_VERSIONS, TABLE_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, BackgroundTasksStorage, TABLE_SCHEMAS, TABLE_BACKGROUND_TASKS, MemoryStorage, createStorageErrorId, normalizePerPage, calculatePagination, ObservabilityStorage, SPAN_SCHEMA, listTracesArgsSchema, toTraceSpans, ScoresStorage, transformScoreRow, SCORERS_SCHEMA, WorkflowsStorage, MastraCompositeStore, getSqlType, getDefaultValue, safelyParseJSON, listBranchesArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listFeedbackArgsSchema, EntityType, TraceStatus, METRIC_DISTINCT_COLUMNS } from '@mastra/core/storage';
|
|
3
|
+
import { BRANCH_SPAN_TYPES, TABLE_THREAD_STATE, TABLE_HARNESS_SESSIONS, TABLE_NOTIFICATIONS, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEDULES, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_FAVORITES, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SKILLS, TABLE_WORKSPACE_VERSIONS, TABLE_WORKSPACES, TABLE_MCP_SERVER_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_CLIENTS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SCORER_DEFINITIONS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_PROMPT_BLOCKS, TABLE_EXPERIMENT_RESULTS, TABLE_EXPERIMENTS, TABLE_DATASET_VERSIONS, TABLE_DATASET_ITEMS, TABLE_DATASETS, TABLE_AGENT_VERSIONS, TABLE_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, BackgroundTasksStorage, TABLE_SCHEMAS, TABLE_BACKGROUND_TASKS, MemoryStorage, createStorageErrorId, normalizePerPage, calculatePagination, ObservabilityStorage, SPAN_SCHEMA, listTracesArgsSchema, toTraceSpans, ScoresStorage, transformScoreRow, SCORERS_SCHEMA, WorkflowsStorage, MastraCompositeStore, getSqlType, getDefaultValue, safelyParseJSON, listBranchesArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listFeedbackArgsSchema, EntityType, TraceStatus, METRIC_DISTINCT_COLUMNS } from '@mastra/core/storage';
|
|
4
4
|
import { MastraBase } from '@mastra/core/base';
|
|
5
5
|
import { MessageList } from '@mastra/core/agent';
|
|
6
6
|
import { parseFieldKey } from '@mastra/core/utils';
|
|
@@ -8,6 +8,139 @@ import { coreFeatures } from '@mastra/core/features';
|
|
|
8
8
|
import { saveScorePayloadSchema } from '@mastra/core/evals';
|
|
9
9
|
|
|
10
10
|
// src/storage/index.ts
|
|
11
|
+
var DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
|
|
12
|
+
var DEFAULT_REPLICA_NAME = "{replica}";
|
|
13
|
+
var REPLICATED_ENGINE_NAMES = /* @__PURE__ */ new Set(["ReplicatedMergeTree", "ReplicatedReplacingMergeTree"]);
|
|
14
|
+
var SUPPORTED_ENGINE_NAMES = /* @__PURE__ */ new Set(["MergeTree", "ReplacingMergeTree", ...REPLICATED_ENGINE_NAMES]);
|
|
15
|
+
function isReplicationConfigured(replication) {
|
|
16
|
+
return replication !== void 0;
|
|
17
|
+
}
|
|
18
|
+
function isReplicatedOrSharedEngine(engine) {
|
|
19
|
+
if (!engine) return false;
|
|
20
|
+
return engine.startsWith("Replicated") || engine.startsWith("Shared");
|
|
21
|
+
}
|
|
22
|
+
function assertReplicationConfigField(fieldName, value, errorCode) {
|
|
23
|
+
if (value === void 0) return;
|
|
24
|
+
if (value.trim() === "") {
|
|
25
|
+
throw new MastraError({
|
|
26
|
+
id: createStorageErrorId("CLICKHOUSE", "REPLICATION_CONFIG", errorCode),
|
|
27
|
+
domain: ErrorDomain.STORAGE,
|
|
28
|
+
category: ErrorCategory.USER,
|
|
29
|
+
text: `ClickHouse replication.${fieldName} must be a non-empty string when provided.`
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
if (/\s/.test(value) || value.includes("'") || value.includes('"') || value.includes("\\")) {
|
|
33
|
+
throw new MastraError({
|
|
34
|
+
id: createStorageErrorId("CLICKHOUSE", "REPLICATION_CONFIG", errorCode),
|
|
35
|
+
domain: ErrorDomain.STORAGE,
|
|
36
|
+
category: ErrorCategory.USER,
|
|
37
|
+
text: `ClickHouse replication.${fieldName} must not contain whitespace or quote characters.`
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function validateReplicationConfig(replication) {
|
|
42
|
+
if (!replication) return;
|
|
43
|
+
assertReplicationConfigField("cluster", replication.cluster, "INVALID_CLUSTER");
|
|
44
|
+
assertReplicationConfigField("zookeeperPath", replication.zookeeperPath, "INVALID_ZOOKEEPER_PATH");
|
|
45
|
+
assertReplicationConfigField("replicaName", replication.replicaName, "INVALID_REPLICA_NAME");
|
|
46
|
+
}
|
|
47
|
+
function quoteClickhouseString(value) {
|
|
48
|
+
return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
49
|
+
}
|
|
50
|
+
function getEngineNameAndArgs(engine) {
|
|
51
|
+
const trimmed = engine.trim();
|
|
52
|
+
const match = trimmed.match(/^(\w+)\s*(?:\((.*)\))?$/s);
|
|
53
|
+
if (!match) return null;
|
|
54
|
+
const name = match[1];
|
|
55
|
+
if (!name) return null;
|
|
56
|
+
const args = match[2]?.trim() ?? "";
|
|
57
|
+
if (args && !hasBalancedParens(args)) return null;
|
|
58
|
+
return { name, args };
|
|
59
|
+
}
|
|
60
|
+
function hasBalancedParens(s) {
|
|
61
|
+
let depth = 0;
|
|
62
|
+
for (const c of s) {
|
|
63
|
+
if (c === "(") depth++;
|
|
64
|
+
else if (c === ")") {
|
|
65
|
+
depth--;
|
|
66
|
+
if (depth < 0) return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return depth === 0;
|
|
70
|
+
}
|
|
71
|
+
function buildReplicatedTableEngine(engine, replication) {
|
|
72
|
+
if (!replication) return engine;
|
|
73
|
+
const parsed = getEngineNameAndArgs(engine);
|
|
74
|
+
if (!parsed || !SUPPORTED_ENGINE_NAMES.has(parsed.name)) return engine;
|
|
75
|
+
if (isReplicatedOrSharedEngine(parsed.name)) return engine;
|
|
76
|
+
const zookeeperPath = quoteClickhouseString(replication.zookeeperPath ?? DEFAULT_ZOOKEEPER_PATH);
|
|
77
|
+
const replicaName = quoteClickhouseString(replication.replicaName ?? DEFAULT_REPLICA_NAME);
|
|
78
|
+
const replicatedName = parsed.name === "ReplacingMergeTree" ? "ReplicatedReplacingMergeTree" : "ReplicatedMergeTree";
|
|
79
|
+
const args = [zookeeperPath, replicaName, parsed.args].filter(Boolean).join(", ");
|
|
80
|
+
return `${replicatedName}(${args})`;
|
|
81
|
+
}
|
|
82
|
+
function addOnClusterToDDL(sql, replication) {
|
|
83
|
+
const cluster = replication?.cluster?.trim();
|
|
84
|
+
if (!cluster) return sql;
|
|
85
|
+
const quotedCluster = quoteClickhouseString(cluster);
|
|
86
|
+
const onClusterSuffix = ` ON CLUSTER ${quotedCluster}`;
|
|
87
|
+
const rewrite = (input, pattern) => {
|
|
88
|
+
return input.replace(pattern, (...args) => {
|
|
89
|
+
const match = args[0];
|
|
90
|
+
const source = args[args.length - 1];
|
|
91
|
+
const offset = args[args.length - 2];
|
|
92
|
+
const tail = source.slice(offset + match.length);
|
|
93
|
+
if (/^\s+ON\s+CLUSTER\s/i.test(tail)) return match;
|
|
94
|
+
return match + onClusterSuffix;
|
|
95
|
+
});
|
|
96
|
+
};
|
|
97
|
+
let out = sql;
|
|
98
|
+
out = rewrite(out, /\bCREATE\s+TABLE\s+(IF\s+NOT\s+EXISTS\s+)?[^\s(]+/gi);
|
|
99
|
+
out = rewrite(out, /\bCREATE\s+MATERIALIZED\s+VIEW\s+(IF\s+NOT\s+EXISTS\s+)?[^\s(]+/gi);
|
|
100
|
+
out = rewrite(out, /\bALTER\s+TABLE\s+[^\s]+/gi);
|
|
101
|
+
out = rewrite(out, /\bDROP\s+(TABLE|VIEW)\s+(IF\s+EXISTS\s+)?[^\s]+/gi);
|
|
102
|
+
out = rewrite(out, /\bTRUNCATE\s+TABLE\s+(IF\s+EXISTS\s+)?[^\s]+/gi);
|
|
103
|
+
out = rewrite(out, /\bOPTIMIZE\s+TABLE\s+[^\s]+/gi);
|
|
104
|
+
out = rewrite(out, /\bSYSTEM\s+(REFRESH|WAIT)\s+VIEW\s+[^\s;]+/gi);
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
function rewriteEngineClauses(sql, replication) {
|
|
108
|
+
return sql.replace(/ENGINE\s*=\s*(\w+)\s*/gi, (match, engineName, offset, source) => {
|
|
109
|
+
const argsStart = offset + match.length;
|
|
110
|
+
if (source[argsStart] !== "(") {
|
|
111
|
+
return `ENGINE = ${buildReplicatedTableEngine(engineName, replication)}`;
|
|
112
|
+
}
|
|
113
|
+
let depth = 0;
|
|
114
|
+
for (let i = argsStart; i < source.length; i++) {
|
|
115
|
+
const char = source[i];
|
|
116
|
+
if (char === "(") depth++;
|
|
117
|
+
else if (char === ")") {
|
|
118
|
+
depth--;
|
|
119
|
+
if (depth === 0) {
|
|
120
|
+
const engine = `${engineName}${source.slice(argsStart, i + 1)}`;
|
|
121
|
+
return `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return match;
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
function applyReplicationToDDL(sql, replication) {
|
|
129
|
+
const withReplicatedEngine = replication ? rewriteEngineClauses(sql, replication) : sql;
|
|
130
|
+
return addOnClusterToDDL(withReplicatedEngine, replication);
|
|
131
|
+
}
|
|
132
|
+
function buildLocalTableReplicationError(tables) {
|
|
133
|
+
const tableList = tables.map((table) => ` - ${table.name} (${table.engine})`).join("\n");
|
|
134
|
+
return new MastraError({
|
|
135
|
+
id: createStorageErrorId("CLICKHOUSE", "REPLICATION", "LOCAL_TABLES_UNSUPPORTED"),
|
|
136
|
+
domain: ErrorDomain.STORAGE,
|
|
137
|
+
category: ErrorCategory.USER,
|
|
138
|
+
text: `ClickHouse replication is enabled, but existing Mastra tables use non-replicated local engines.
|
|
139
|
+
Mastra will not automatically convert existing local tables to replicated tables.
|
|
140
|
+
Please migrate or recreate these tables manually before enabling replication:
|
|
141
|
+
${tableList}`
|
|
142
|
+
});
|
|
143
|
+
}
|
|
11
144
|
var TABLE_ENGINES = {
|
|
12
145
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
13
146
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
@@ -44,8 +177,11 @@ var TABLE_ENGINES = {
|
|
|
44
177
|
mastra_background_tasks: `ReplacingMergeTree()`,
|
|
45
178
|
[TABLE_SCHEDULES]: `ReplacingMergeTree()`,
|
|
46
179
|
[TABLE_SCHEDULE_TRIGGERS]: `MergeTree()`,
|
|
180
|
+
[TABLE_NOTIFICATIONS]: `ReplacingMergeTree()`,
|
|
181
|
+
[TABLE_HARNESS_SESSIONS]: `ReplacingMergeTree()`,
|
|
47
182
|
mastra_channel_installations: `ReplacingMergeTree()`,
|
|
48
|
-
mastra_channel_config: `ReplacingMergeTree()
|
|
183
|
+
mastra_channel_config: `ReplacingMergeTree()`,
|
|
184
|
+
[TABLE_THREAD_STATE]: `ReplacingMergeTree()`
|
|
49
185
|
};
|
|
50
186
|
var COLUMN_TYPES = {
|
|
51
187
|
text: "String",
|
|
@@ -93,8 +229,9 @@ function transformRows(rows) {
|
|
|
93
229
|
|
|
94
230
|
// src/storage/db/index.ts
|
|
95
231
|
function resolveClickhouseConfig(config) {
|
|
232
|
+
validateReplicationConfig(config.replication);
|
|
96
233
|
if ("client" in config) {
|
|
97
|
-
return { client: config.client, ttl: config.ttl };
|
|
234
|
+
return { client: config.client, ttl: config.ttl, replication: config.replication };
|
|
98
235
|
}
|
|
99
236
|
const client = createClient({
|
|
100
237
|
url: config.url,
|
|
@@ -107,18 +244,25 @@ function resolveClickhouseConfig(config) {
|
|
|
107
244
|
output_format_json_quote_64bit_integers: 0
|
|
108
245
|
}
|
|
109
246
|
});
|
|
110
|
-
return { client, ttl: config.ttl };
|
|
247
|
+
return { client, ttl: config.ttl, replication: config.replication };
|
|
111
248
|
}
|
|
112
249
|
var ClickhouseDB = class extends MastraBase {
|
|
113
250
|
ttl;
|
|
251
|
+
replication;
|
|
114
252
|
client;
|
|
115
253
|
/** Cache of actual table columns: tableName -> Promise<Set<columnName>> (stores in-flight promise to coalesce concurrent calls) */
|
|
116
254
|
tableColumnsCache = /* @__PURE__ */ new Map();
|
|
117
|
-
constructor({
|
|
255
|
+
constructor({
|
|
256
|
+
client,
|
|
257
|
+
ttl,
|
|
258
|
+
replication
|
|
259
|
+
}) {
|
|
118
260
|
super({
|
|
119
261
|
name: "CLICKHOUSE_DB"
|
|
120
262
|
});
|
|
263
|
+
validateReplicationConfig(replication);
|
|
121
264
|
this.ttl = ttl;
|
|
265
|
+
this.replication = replication;
|
|
122
266
|
this.client = client;
|
|
123
267
|
}
|
|
124
268
|
/**
|
|
@@ -186,6 +330,19 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
186
330
|
return false;
|
|
187
331
|
}
|
|
188
332
|
}
|
|
333
|
+
async assertExistingTableCompatibleWithReplication(tableName) {
|
|
334
|
+
if (!isReplicationConfigured(this.replication)) return;
|
|
335
|
+
const result = await this.client.query({
|
|
336
|
+
query: `SELECT name, engine FROM system.tables WHERE database = currentDatabase() AND name = {tableName:String}`,
|
|
337
|
+
query_params: { tableName },
|
|
338
|
+
format: "JSONEachRow"
|
|
339
|
+
});
|
|
340
|
+
const rows = await result.json();
|
|
341
|
+
const localTables = rows.filter((row) => row.engine && !isReplicatedOrSharedEngine(row.engine));
|
|
342
|
+
if (localTables.length > 0) {
|
|
343
|
+
throw buildLocalTableReplicationError(localTables);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
189
346
|
/**
|
|
190
347
|
* Gets the sorting key (ORDER BY columns) for a table.
|
|
191
348
|
* Returns null if the table doesn't exist.
|
|
@@ -289,6 +446,14 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
289
446
|
this.logger?.debug?.(`Spans table already has correct sorting key: ${currentSortingKey}`);
|
|
290
447
|
return false;
|
|
291
448
|
}
|
|
449
|
+
if (isReplicationConfigured(this.replication)) {
|
|
450
|
+
throw new MastraError({
|
|
451
|
+
id: createStorageErrorId("CLICKHOUSE", "REPLICATION", "SPANS_SORTING_KEY_MIGRATION_UNSUPPORTED"),
|
|
452
|
+
domain: ErrorDomain.STORAGE,
|
|
453
|
+
category: ErrorCategory.USER,
|
|
454
|
+
text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap spans table migrations automatically. Migrate the existing spans table manually before enabling replication."
|
|
455
|
+
});
|
|
456
|
+
}
|
|
292
457
|
this.logger?.info?.(`Migrating spans table from sorting key "${currentSortingKey}" to "(traceId, spanId)"`);
|
|
293
458
|
const backupTableName = `${tableName}_backup_${Date.now()}`;
|
|
294
459
|
const rowTtl = this.ttl?.[tableName]?.row;
|
|
@@ -339,7 +504,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
339
504
|
SELECT ${selectExpressions}
|
|
340
505
|
FROM ${backupTableName}
|
|
341
506
|
ORDER BY traceId, spanId,
|
|
342
|
-
(endedAt IS NOT NULL
|
|
507
|
+
(endedAt IS NOT NULL) DESC,
|
|
343
508
|
COALESCE(updatedAt, createdAt) DESC,
|
|
344
509
|
createdAt DESC
|
|
345
510
|
LIMIT 1 BY traceId, spanId`
|
|
@@ -402,6 +567,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
402
567
|
schema
|
|
403
568
|
}) {
|
|
404
569
|
try {
|
|
570
|
+
await this.assertExistingTableCompatibleWithReplication(tableName);
|
|
405
571
|
const columns = Object.entries(schema).map(([name, def]) => {
|
|
406
572
|
let sqlType = this.getSqlType(def.type);
|
|
407
573
|
let isNullable = def.nullable === true;
|
|
@@ -455,7 +621,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
455
621
|
`;
|
|
456
622
|
}
|
|
457
623
|
await this.client.query({
|
|
458
|
-
query: sql,
|
|
624
|
+
query: applyReplicationToDDL(sql, this.replication),
|
|
459
625
|
clickhouse_settings: {
|
|
460
626
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
461
627
|
date_time_input_format: "best_effort",
|
|
@@ -500,7 +666,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
500
666
|
const defaultValue = columnDef.nullable === false ? getDefaultValue(columnDef.type) : "";
|
|
501
667
|
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
502
668
|
await this.client.query({
|
|
503
|
-
query: alterSql
|
|
669
|
+
query: addOnClusterToDDL(alterSql, this.replication)
|
|
504
670
|
});
|
|
505
671
|
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
506
672
|
}
|
|
@@ -523,7 +689,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
523
689
|
try {
|
|
524
690
|
await this.client.command({ query: `SYSTEM STOP MERGES ${tableName}` });
|
|
525
691
|
await this.client.command({
|
|
526
|
-
query: `TRUNCATE TABLE ${tableName}
|
|
692
|
+
query: addOnClusterToDDL(`TRUNCATE TABLE ${tableName}`, this.replication)
|
|
527
693
|
});
|
|
528
694
|
await this.client.command({ query: `SYSTEM START MERGES ${tableName}` });
|
|
529
695
|
} catch (error) {
|
|
@@ -541,7 +707,7 @@ var ClickhouseDB = class extends MastraBase {
|
|
|
541
707
|
async dropTable({ tableName }) {
|
|
542
708
|
try {
|
|
543
709
|
await this.client.query({
|
|
544
|
-
query: `DROP TABLE IF EXISTS ${tableName}
|
|
710
|
+
query: addOnClusterToDDL(`DROP TABLE IF EXISTS ${tableName}`, this.replication)
|
|
545
711
|
});
|
|
546
712
|
} catch (error) {
|
|
547
713
|
throw new MastraError(
|
|
@@ -735,9 +901,9 @@ var BackgroundTasksStorageClickhouse = class extends BackgroundTasksStorage {
|
|
|
735
901
|
#db;
|
|
736
902
|
constructor(config) {
|
|
737
903
|
super();
|
|
738
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
904
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
739
905
|
this.client = client;
|
|
740
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
906
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
741
907
|
}
|
|
742
908
|
async init() {
|
|
743
909
|
await this.#db.createTable({ tableName: TABLE_BACKGROUND_TASKS, schema: TABLE_SCHEMAS[TABLE_BACKGROUND_TASKS] });
|
|
@@ -929,9 +1095,9 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
|
929
1095
|
#db;
|
|
930
1096
|
constructor(config) {
|
|
931
1097
|
super();
|
|
932
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
1098
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
933
1099
|
this.client = client;
|
|
934
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
1100
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
935
1101
|
}
|
|
936
1102
|
async init() {
|
|
937
1103
|
await this.#db.createTable({ tableName: TABLE_THREADS, schema: TABLE_SCHEMAS[TABLE_THREADS] });
|
|
@@ -2192,9 +2358,9 @@ var ObservabilityStorageClickhouse = class extends ObservabilityStorage {
|
|
|
2192
2358
|
#db;
|
|
2193
2359
|
constructor(config) {
|
|
2194
2360
|
super();
|
|
2195
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
2361
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
2196
2362
|
this.client = client;
|
|
2197
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
2363
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
2198
2364
|
}
|
|
2199
2365
|
async init() {
|
|
2200
2366
|
const migrationStatus = await this.#db.checkSpansMigrationStatus(TABLE_SPANS);
|
|
@@ -6988,7 +7154,20 @@ async function filterAppliedRetention(client, entries) {
|
|
|
6988
7154
|
return current.column !== e.column || current.days !== e.days;
|
|
6989
7155
|
});
|
|
6990
7156
|
}
|
|
6991
|
-
async function
|
|
7157
|
+
async function assertExistingTablesCompatibleWithReplication(client, replication) {
|
|
7158
|
+
if (!isReplicationConfigured(replication)) return;
|
|
7159
|
+
const result = await client.query({
|
|
7160
|
+
query: `SELECT name, engine FROM system.tables WHERE database = currentDatabase() AND name IN ({tables:Array(String)})`,
|
|
7161
|
+
query_params: { tables: [...ALL_TABLE_NAMES] },
|
|
7162
|
+
format: "JSONEachRow"
|
|
7163
|
+
});
|
|
7164
|
+
const rows = await result.json();
|
|
7165
|
+
const localTable = rows.find((row) => !isReplicatedOrSharedEngine(row.engine));
|
|
7166
|
+
if (localTable) {
|
|
7167
|
+
throw buildLocalTableReplicationError([{ name: localTable.name, engine: localTable.engine }]);
|
|
7168
|
+
}
|
|
7169
|
+
}
|
|
7170
|
+
async function reconcileDiscoveryTables(client, replication) {
|
|
6992
7171
|
let engines;
|
|
6993
7172
|
try {
|
|
6994
7173
|
const result = await client.query({
|
|
@@ -7008,8 +7187,8 @@ async function reconcileDiscoveryTables(client) {
|
|
|
7008
7187
|
for (const { table, mv } of targets) {
|
|
7009
7188
|
const engine = engines.get(table);
|
|
7010
7189
|
if (!engine || isReplacingMergeTreeEngine(engine)) continue;
|
|
7011
|
-
await client.command({ query: `DROP VIEW IF EXISTS ${mv}
|
|
7012
|
-
await client.command({ query: `DROP TABLE IF EXISTS ${table}
|
|
7190
|
+
await client.command({ query: addOnClusterToDDL(`DROP VIEW IF EXISTS ${mv}`, replication) });
|
|
7191
|
+
await client.command({ query: addOnClusterToDDL(`DROP TABLE IF EXISTS ${table}`, replication) });
|
|
7013
7192
|
}
|
|
7014
7193
|
}
|
|
7015
7194
|
async function queryNamesByTable(client, query, tables) {
|
|
@@ -7088,12 +7267,14 @@ async function detectExistingDeltaCursorStrategy(client) {
|
|
|
7088
7267
|
var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
7089
7268
|
#client;
|
|
7090
7269
|
#retention;
|
|
7270
|
+
#replication;
|
|
7091
7271
|
#deltaCursorStrategyOverride;
|
|
7092
7272
|
#deltaCursorStrategy = "fallback";
|
|
7093
7273
|
constructor(config) {
|
|
7094
7274
|
super();
|
|
7095
|
-
const { client } = resolveClickhouseConfig(config);
|
|
7275
|
+
const { client, replication } = resolveClickhouseConfig(config);
|
|
7096
7276
|
this.#client = client;
|
|
7277
|
+
this.#replication = replication;
|
|
7097
7278
|
this.#retention = config.retention;
|
|
7098
7279
|
this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
|
|
7099
7280
|
}
|
|
@@ -7114,6 +7295,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
7114
7295
|
});
|
|
7115
7296
|
}
|
|
7116
7297
|
try {
|
|
7298
|
+
await assertExistingTablesCompatibleWithReplication(this.#client, this.#replication);
|
|
7117
7299
|
const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
|
|
7118
7300
|
if (existingStrategy === "mixed") {
|
|
7119
7301
|
this.#deltaCursorStrategy = null;
|
|
@@ -7127,19 +7309,19 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
7127
7309
|
} else {
|
|
7128
7310
|
this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
|
|
7129
7311
|
}
|
|
7130
|
-
await reconcileDiscoveryTables(this.#client);
|
|
7312
|
+
await reconcileDiscoveryTables(this.#client, this.#replication);
|
|
7131
7313
|
const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
|
|
7132
7314
|
for (const ddl of coreDdl) {
|
|
7133
|
-
await this.#client.command({ query: ddl });
|
|
7315
|
+
await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
|
|
7134
7316
|
}
|
|
7135
7317
|
const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
|
|
7136
7318
|
for (const migration of pendingMigrations) {
|
|
7137
|
-
await this.#client.command({ query: migration.sql });
|
|
7319
|
+
await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
|
|
7138
7320
|
}
|
|
7139
7321
|
if (this.#retention) {
|
|
7140
7322
|
const pendingRetention = await filterAppliedRetention(this.#client, buildRetentionEntries(this.#retention));
|
|
7141
7323
|
for (const entry of pendingRetention) {
|
|
7142
|
-
await this.#client.command({ query: entry.sql });
|
|
7324
|
+
await this.#client.command({ query: addOnClusterToDDL(entry.sql, this.#replication) });
|
|
7143
7325
|
}
|
|
7144
7326
|
}
|
|
7145
7327
|
if (this.#deltaCursorStrategy === "serial") {
|
|
@@ -7168,12 +7350,20 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
7168
7350
|
}
|
|
7169
7351
|
try {
|
|
7170
7352
|
for (const ddl of DISCOVERY_MV_DDL) {
|
|
7171
|
-
await this.#client.command({ query: ddl });
|
|
7353
|
+
await this.#client.command({ query: addOnClusterToDDL(ddl, this.#replication) });
|
|
7172
7354
|
}
|
|
7173
|
-
await this.#client.command({
|
|
7174
|
-
|
|
7175
|
-
|
|
7176
|
-
await this.#client.command({
|
|
7355
|
+
await this.#client.command({
|
|
7356
|
+
query: addOnClusterToDDL(`SYSTEM REFRESH VIEW ${MV_DISCOVERY_VALUES}`, this.#replication)
|
|
7357
|
+
});
|
|
7358
|
+
await this.#client.command({
|
|
7359
|
+
query: addOnClusterToDDL(`SYSTEM WAIT VIEW ${MV_DISCOVERY_VALUES}`, this.#replication)
|
|
7360
|
+
});
|
|
7361
|
+
await this.#client.command({
|
|
7362
|
+
query: addOnClusterToDDL(`SYSTEM REFRESH VIEW ${MV_DISCOVERY_PAIRS}`, this.#replication)
|
|
7363
|
+
});
|
|
7364
|
+
await this.#client.command({
|
|
7365
|
+
query: addOnClusterToDDL(`SYSTEM WAIT VIEW ${MV_DISCOVERY_PAIRS}`, this.#replication)
|
|
7366
|
+
});
|
|
7177
7367
|
} catch {
|
|
7178
7368
|
}
|
|
7179
7369
|
}
|
|
@@ -7192,6 +7382,14 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
7192
7382
|
message: "Migration already complete. Signal tables already use signal-ID dedupe keys."
|
|
7193
7383
|
};
|
|
7194
7384
|
}
|
|
7385
|
+
if (isReplicationConfigured(this.#replication)) {
|
|
7386
|
+
throw new MastraError({
|
|
7387
|
+
id: createStorageErrorId("CLICKHOUSE", "REPLICATION", "SIGNAL_TABLES_MIGRATION_UNSUPPORTED"),
|
|
7388
|
+
domain: ErrorDomain.STORAGE,
|
|
7389
|
+
category: ErrorCategory.USER,
|
|
7390
|
+
text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap signal table migrations automatically. Migrate existing local signal tables manually before enabling replication."
|
|
7391
|
+
});
|
|
7392
|
+
}
|
|
7195
7393
|
await migrateSignalTables(this.#client, this.logger);
|
|
7196
7394
|
return {
|
|
7197
7395
|
success: true,
|
|
@@ -7888,7 +8086,11 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
7888
8086
|
async dangerouslyClearAll() {
|
|
7889
8087
|
try {
|
|
7890
8088
|
await Promise.all(
|
|
7891
|
-
ALL_TABLE_NAMES.map(
|
|
8089
|
+
ALL_TABLE_NAMES.map(
|
|
8090
|
+
(table) => this.#client.command({
|
|
8091
|
+
query: addOnClusterToDDL(`TRUNCATE TABLE IF EXISTS ${table}`, this.#replication)
|
|
8092
|
+
})
|
|
8093
|
+
)
|
|
7892
8094
|
);
|
|
7893
8095
|
} catch (error) {
|
|
7894
8096
|
if (error instanceof MastraError) throw error;
|
|
@@ -7908,9 +8110,9 @@ var ScoresStorageClickhouse = class extends ScoresStorage {
|
|
|
7908
8110
|
#db;
|
|
7909
8111
|
constructor(config) {
|
|
7910
8112
|
super();
|
|
7911
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
8113
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
7912
8114
|
this.client = client;
|
|
7913
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
8115
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
7914
8116
|
}
|
|
7915
8117
|
async init() {
|
|
7916
8118
|
await this.#db.createTable({ tableName: TABLE_SCORERS, schema: TABLE_SCHEMAS[TABLE_SCORERS] });
|
|
@@ -8335,9 +8537,9 @@ var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
|
8335
8537
|
#db;
|
|
8336
8538
|
constructor(config) {
|
|
8337
8539
|
super();
|
|
8338
|
-
const { client, ttl } = resolveClickhouseConfig(config);
|
|
8540
|
+
const { client, ttl, replication } = resolveClickhouseConfig(config);
|
|
8339
8541
|
this.client = client;
|
|
8340
|
-
this.#db = new ClickhouseDB({ client, ttl });
|
|
8542
|
+
this.#db = new ClickhouseDB({ client, ttl, replication });
|
|
8341
8543
|
}
|
|
8342
8544
|
supportsConcurrentUpdates() {
|
|
8343
8545
|
return false;
|
|
@@ -8629,9 +8831,11 @@ var isClientConfig = (config) => {
|
|
|
8629
8831
|
var ClickhouseStore = class extends MastraCompositeStore {
|
|
8630
8832
|
db;
|
|
8631
8833
|
ttl = {};
|
|
8834
|
+
replication;
|
|
8632
8835
|
stores;
|
|
8633
8836
|
constructor(config) {
|
|
8634
8837
|
super({ id: config.id, name: "ClickhouseStore", disableInit: config.disableInit });
|
|
8838
|
+
validateReplicationConfig(config.replication);
|
|
8635
8839
|
if (isClientConfig(config)) {
|
|
8636
8840
|
this.db = config.client;
|
|
8637
8841
|
} else {
|
|
@@ -8644,7 +8848,7 @@ var ClickhouseStore = class extends MastraCompositeStore {
|
|
|
8644
8848
|
if (typeof config.password !== "string") {
|
|
8645
8849
|
throw new Error("ClickhouseStore: password must be a string.");
|
|
8646
8850
|
}
|
|
8647
|
-
const { id, ttl, disableInit, clickhouse_settings, ...clientOptions } = config;
|
|
8851
|
+
const { id, ttl, disableInit, replication, clickhouse_settings, ...clientOptions } = config;
|
|
8648
8852
|
this.db = createClient({
|
|
8649
8853
|
...clientOptions,
|
|
8650
8854
|
clickhouse_settings: {
|
|
@@ -8658,7 +8862,8 @@ var ClickhouseStore = class extends MastraCompositeStore {
|
|
|
8658
8862
|
});
|
|
8659
8863
|
}
|
|
8660
8864
|
this.ttl = config.ttl;
|
|
8661
|
-
|
|
8865
|
+
this.replication = config.replication;
|
|
8866
|
+
const domainConfig = { client: this.db, ttl: this.ttl, replication: config.replication };
|
|
8662
8867
|
const workflows = new WorkflowsStorageClickhouse(domainConfig);
|
|
8663
8868
|
const scores = new ScoresStorageClickhouse(domainConfig);
|
|
8664
8869
|
const memory = new MemoryStorageClickhouse(domainConfig);
|
|
@@ -8674,7 +8879,7 @@ var ClickhouseStore = class extends MastraCompositeStore {
|
|
|
8674
8879
|
async optimizeTable({ tableName }) {
|
|
8675
8880
|
try {
|
|
8676
8881
|
await this.db.command({
|
|
8677
|
-
query: `OPTIMIZE TABLE ${tableName} FINAL
|
|
8882
|
+
query: addOnClusterToDDL(`OPTIMIZE TABLE ${tableName} FINAL`, this.replication)
|
|
8678
8883
|
});
|
|
8679
8884
|
} catch (error) {
|
|
8680
8885
|
throw new MastraError(
|
|
@@ -8691,7 +8896,7 @@ var ClickhouseStore = class extends MastraCompositeStore {
|
|
|
8691
8896
|
async materializeTtl({ tableName }) {
|
|
8692
8897
|
try {
|
|
8693
8898
|
await this.db.command({
|
|
8694
|
-
query: `ALTER TABLE ${tableName} MATERIALIZE TTL
|
|
8899
|
+
query: addOnClusterToDDL(`ALTER TABLE ${tableName} MATERIALIZE TTL`, this.replication) + ";"
|
|
8695
8900
|
});
|
|
8696
8901
|
} catch (error) {
|
|
8697
8902
|
throw new MastraError(
|
|
@@ -8730,7 +8935,7 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
|
|
|
8730
8935
|
constructor(config) {
|
|
8731
8936
|
super(config);
|
|
8732
8937
|
this.name = "ClickhouseStoreVNext";
|
|
8733
|
-
const observability = new ObservabilityStorageClickhouseVNext({ client: this.db });
|
|
8938
|
+
const observability = new ObservabilityStorageClickhouseVNext({ client: this.db, replication: config.replication });
|
|
8734
8939
|
this.stores = {
|
|
8735
8940
|
...this.stores,
|
|
8736
8941
|
observability
|