@mastra/clickhouse 1.17.0-alpha.0 → 1.17.0

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-clickhouse
 description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
 metadata:
   package: "@mastra/clickhouse"
-  version: "1.17.0-alpha.0"
+  version: "1.17.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.17.0-alpha.0",
+  "version": "1.17.0",
   "package": "@mastra/clickhouse",
   "exports": {},
   "modules": {}

package/dist/docs/references/integrations-databases-clickhouse.md

@@ -87,6 +87,12 @@ export const mastra = new Mastra({
 
 `MastraStorageExporter` automatically selects the `insert-only` strategy when ClickHouse is the observability backend, which gives the highest write throughput. See [tracing strategies](https://mastra.ai/docs/observability/integrations/exporters/mastra-storage) for details.
 
+Trace deletion cascades to spans, trace roots and branches, metrics, logs, scores, and feedback linked by trace ID. Signals without a trace ID are preserved. Mastra records the deletion predicate, then waits for ClickHouse lightweight delete masks to be applied. Normal reads no longer return the rows matched by that operation when the call resolves.
+
+Lightweight deletion is a hide-only operation that marks rows with ClickHouse's `_row_exists` mask. Physical removal depends on merges and deployment-configured retention TTLs. `ObservabilityStorageClickhouseVNext` applies retention only when you provide a `RetentionConfig`; Mastra OSS doesn't configure a default retention TTL.
+
+Deletion requests aren't purged automatically in Mastra OSS. Automatic retirement will be introduced with future database-agnostic retention configuration.
+
 ### Observability with the legacy domain
 
 `ObservabilityStorageClickhouse` is the original observability adapter and remains supported for projects that haven't migrated to the vNext schema. The configuration shape is the same as the vNext class.

package/dist/index.cjs

@@ -29,6 +29,7 @@ let _mastra_core_agent = require("@mastra/core/agent");
 let _mastra_core_base = require("@mastra/core/base");
 let _mastra_core_utils = require("@mastra/core/utils");
 let _mastra_core_features = require("@mastra/core/features");
+let crypto$1 = require("crypto");
 let _mastra_core_evals = require("@mastra/core/evals");
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -94,12 +95,22 @@ function getEngineNameAndArgs(engine) {
 }
 function hasBalancedParens(s) {
 	let depth = 0;
-	for (const c of s) if (c === "(") depth++;
-	else if (c === ")") {
-		depth--;
-		if (depth < 0) return false;
+	let quote;
+	for (let i = 0; i < s.length; i++) {
+		const c = s[i];
+		if (quote) {
+			if (c === "\\") i++;
+			else if (c === quote) quote = void 0;
+			continue;
+		}
+		if (c === "`" || c === "\"" || c === "'") quote = c;
+		else if (c === "(") depth++;
+		else if (c === ")") {
+			depth--;
+			if (depth < 0) return false;
+		}
 	}
-	return depth === 0;
+	return depth === 0 && quote === void 0;
 }
 function buildReplicatedTableEngine(engine, replication) {
 	if (!replication) return engine;
@@ -171,20 +182,51 @@ function addOnClusterToDDL(sql, replication) {
 	return out;
 }
 function rewriteEngineClauses(sql, replication) {
-	return sql.replace(/ENGINE\s*=\s*(\w+)\s*/gi, (match, engineName, offset, source) => {
-		const argsStart = offset + match.length;
-		if (source[argsStart] !== "(") return `ENGINE = ${buildReplicatedTableEngine(engineName, replication)}`;
-		let depth = 0;
-		for (let i = argsStart; i < source.length; i++) {
-			const char = source[i];
-			if (char === "(") depth++;
-			else if (char === ")") {
-				depth--;
-				if (depth === 0) return `ENGINE = ${buildReplicatedTableEngine(`${engineName}${source.slice(argsStart, i + 1)}`, replication)}`;
+	const enginePattern = /ENGINE\s*=\s*(\w+)/gi;
+	let result = "";
+	let consumedUntil = 0;
+	for (let match = enginePattern.exec(sql); match; match = enginePattern.exec(sql)) {
+		const engineName = match[1];
+		if (!engineName) continue;
+		let engineEnd = enginePattern.lastIndex;
+		let argsStart = engineEnd;
+		while (/\s/.test(sql[argsStart] ?? "")) argsStart++;
+		let engine = engineName;
+		if (sql[argsStart] === "(") {
+			let depth = 0;
+			let closingParenEnd;
+			let quote;
+			for (let i = argsStart; i < sql.length; i++) {
+				const char = sql[i];
+				if (quote) {
+					if (char === "\\") i++;
+					else if (char === quote) quote = void 0;
+					continue;
+				}
+				if (char === "`" || char === "\"" || char === "'") quote = char;
+				else if (char === "(") depth++;
+				else if (char === ")") {
+					depth--;
+					if (depth === 0) {
+						closingParenEnd = i + 1;
+						break;
+					}
+				}
 			}
+			if (closingParenEnd === void 0) {
+				result += sql.slice(consumedUntil, enginePattern.lastIndex);
+				consumedUntil = enginePattern.lastIndex;
+				continue;
+			}
+			engineEnd = closingParenEnd;
+			engine = `${engineName}${sql.slice(argsStart, engineEnd)}`;
 		}
-		return match;
-	});
+		result += sql.slice(consumedUntil, match.index);
+		result += `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
+		consumedUntil = engineEnd;
+		enginePattern.lastIndex = engineEnd;
+	}
+	return result + sql.slice(consumedUntil);
 }
 function applyReplicationToDDL(sql, replication) {
 	return addOnClusterToDDL(replication ? rewriteEngineClauses(sql, replication) : sql, replication);
@@ -2501,6 +2543,7 @@ var ObservabilityStorageClickhouse = class extends _mastra_core_storage.Observab
 		}
 	}
 	async batchDeleteTraces(args) {
+		this.assertUnscopedBatchDeleteTraces(args);
 		try {
 			if (args.traceIds.length === 0) return;
 			await this.client.command({
@@ -2527,6 +2570,7 @@ const TABLE_METRIC_EVENTS = "mastra_metric_events";
 const TABLE_LOG_EVENTS = "mastra_log_events";
 const TABLE_SCORE_EVENTS = "mastra_score_events";
 const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
+const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
 const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
 const TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
 const TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
@@ -3222,6 +3266,23 @@ PARTITION BY toDate(timestamp)
 ORDER BY (traceId, timestamp, feedbackId)
 SETTINGS allow_nullable_key = 1
 `;
+const DELETION_REQUESTS_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_DELETION_REQUESTS} (
+  requestId       String,
+  organizationId  String DEFAULT '',
+  resourceId      String DEFAULT '',
+  signal          LowCardinality(String),
+  predicateType   LowCardinality(String),
+  predicateValues Array(String),
+  requestedAt     DateTime64(3),
+  requestedBy     String DEFAULT '',
+  lastAppliedAt   DateTime64(3) DEFAULT 0,
+  purgeVerifiedAt DateTime64(3) DEFAULT 0,
+  updatedAt       DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updatedAt)
+ORDER BY (organizationId, resourceId, requestId)
+`;
 function buildFeedbackEventsDeltaDDL() {
 	return `
 CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
@@ -3371,6 +3432,7 @@ const BASE_TABLE_DDL = [
 	LOG_EVENTS_DDL,
 	SCORE_EVENTS_DDL,
 	FEEDBACK_EVENTS_DDL,
+	DELETION_REQUESTS_DDL,
 	DISCOVERY_VALUES_DDL,
 	DISCOVERY_PAIRS_DDL
 ];
@@ -3455,6 +3517,7 @@ const ALL_TABLE_NAMES = [
 	TABLE_LOG_EVENTS,
 	TABLE_SCORE_EVENTS,
 	TABLE_FEEDBACK_EVENTS,
+	TABLE_DELETION_REQUESTS,
 	TABLE_METRIC_EVENTS_DELTA,
 	TABLE_LOG_EVENTS_DELTA,
 	TABLE_SCORE_EVENTS_DELTA,
@@ -4112,6 +4175,35 @@ function feedbackRecordToRow(feedback) {
 	};
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/deletion-requests.ts
+const EPOCH = "1970-01-01T00:00:00.000Z";
+async function recordDeletionRequest(client, args) {
+	const row = {
+		requestId: args.requestId,
+		organizationId: args.organizationId ?? "",
+		resourceId: args.resourceId ?? "",
+		signal: args.signal,
+		predicateType: args.predicateType,
+		predicateValues: args.predicateValues,
+		requestedAt: args.requestedAt,
+		requestedBy: args.requestedBy ?? "",
+		lastAppliedAt: EPOCH,
+		purgeVerifiedAt: EPOCH,
+		updatedAt: args.requestedAt
+	};
+	await client.insert({
+		table: TABLE_DELETION_REQUESTS,
+		values: [row],
+		format: "JSONEachRow",
+		clickhouse_settings: isReplicationConfigured(args.replication) ? {
+			...CH_INSERT_SETTINGS,
+			insert_quorum: "auto",
+			insert_quorum_parallel: 1
+		} : CH_INSERT_SETTINGS
+	});
+	return row;
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/discovery.ts
 async function queryJson$3(client, query, params = {}) {
 	return await (await client.query({
@@ -6614,6 +6706,17 @@ function buildTraceCursor(row) {
 }
 //#endregion
 //#region src/storage/domains/observability/v-next/tracing.ts
+/**
+* Tracing operations for ClickHouse v-next observability.
+*
+* Owns: batchCreateSpans, getSpan, getSpans, getTrace, getTraceLight,
+*       listBranches, batchDeleteTraces, dangerouslyClearSpanEvents.
+* Delegates to trace-roots.ts: listTraces, getRootSpan.
+*
+* `listBranches` reads from the MV-fed `mastra_trace_branches` table (one row
+* per branch anchor span). It lives here -- alongside the other read paths
+* over the trace data -- since branches are conceptually a subset of traces.
+*/
 const BRANCH_SPAN_TYPE_SQL_LIST = _mastra_core_storage.BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
 /** Insert a single completed span. */
 async function createSpan(client, args) {
@@ -6753,16 +6856,40 @@ async function getTraceLight(client, args) {
 	};
 }
 /**
-* Delete traces by traceId.
-* Issues lightweight DELETE against both span_events and trace_roots.
+* Delete traces by traceId, cascading to trace-derived tables and trace-linked
+* signal events (metrics, logs, scores, feedback). Signal rows with a NULL
+* traceId are never affected.
+*
+* On the tracing tables, rows are targeted by tracing identity: traceId +
+* dedupeKey (which starts with traceId). The dedupeKey condition is redundant
+* for correctness (dedupeKey = traceId:spanId) but satisfies the design-doc
+* requirement that trace deletes reference dedupeKey and helps the engine
+* narrow within the sorted ORDER BY key. Signal tables key on their own event
+* ids, so they are targeted by traceId alone.
+*
+* `trace_branches` must be deleted explicitly: its MV fires on insert only,
+* so span deletes never propagate to it. Delta tables self-expire via TTL and
+* discovery tables self-heal, so neither needs explicit deletes.
 *
-* Targets rows by tracing identity: traceId + dedupeKey (which starts with traceId).
-* The dedupeKey condition is redundant for correctness (dedupeKey = traceId:spanId)
-* but satisfies the design-doc requirement that trace deletes reference dedupeKey
-* and helps the engine narrow within the sorted ORDER BY key.
+* Records the predicate before using lightweight DELETE FROM on every table.
+* Lightweight deletes hide rows through ClickHouse's delete mask; physical
+* removal depends on the deployment's configured retention and merge policy.
+*
+* When the optional tenant scope (`organizationId` / `resourceId`) is set,
+* every DELETE additionally requires the row's tenant columns to match.
 */
-async function batchDeleteTraces(client, args) {
+async function batchDeleteTraces(client, args, replication) {
 	if (args.traceIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: (0, crypto$1.randomUUID)(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "traces",
+		predicateType: "traceIds",
+		predicateValues: [...args.traceIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
 	const params = {};
 	const traceInPlaceholders = [];
 	const dedupeOrParts = [];
@@ -6776,13 +6903,35 @@ async function batchDeleteTraces(client, args) {
 	}
 	const traceInList = traceInPlaceholders.join(", ");
 	const dedupeCondition = dedupeOrParts.length === 1 ? dedupeOrParts[0] : `(${dedupeOrParts.join(" OR ")})`;
-	await Promise.all([client.command({
-		query: `DELETE FROM ${TABLE_SPAN_EVENTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	}), client.command({
-		query: `DELETE FROM ${TABLE_TRACE_ROOTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	})]);
+	let scopeCondition = "";
+	if (args.organizationId !== void 0) {
+		params.scope_org = args.organizationId;
+		scopeCondition += ` AND organizationId = {scope_org:String}`;
+	}
+	if (args.resourceId !== void 0) {
+		params.scope_res = args.resourceId;
+		scopeCondition += ` AND resourceId = {scope_res:String}`;
+	}
+	const tracingTables = [
+		TABLE_SPAN_EVENTS,
+		TABLE_TRACE_ROOTS,
+		TABLE_TRACE_BRANCHES
+	];
+	const signalTables = [
+		TABLE_METRIC_EVENTS,
+		TABLE_LOG_EVENTS,
+		TABLE_SCORE_EVENTS,
+		TABLE_FEEDBACK_EVENTS
+	];
+	await Promise.all([...tracingTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	})), ...signalTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList})${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	}))]);
 }
 /**
 * List trace branches with optional filtering, pagination, and ordering.
@@ -7985,7 +8134,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
 	}
 	async batchDeleteTraces(args) {
 		try {
-			await batchDeleteTraces(this.#client, args);
+			await batchDeleteTraces(this.#client, args, this.#replication);
 		} catch (error) {
 			if (error instanceof _mastra_core_error.MastraError) throw error;
 			throw new _mastra_core_error.MastraError({
@@ -8862,7 +9011,9 @@ exports.MemoryStorageClickhouse = MemoryStorageClickhouse;
 exports.ObservabilityStorageClickhouse = ObservabilityStorageClickhouse;
 exports.ObservabilityStorageClickhouseVNext = ObservabilityStorageClickhouseVNext;
 exports.ScoresStorageClickhouse = ScoresStorageClickhouse;
+exports.TABLE_DELETION_REQUESTS = TABLE_DELETION_REQUESTS;
 exports.TABLE_ENGINES = TABLE_ENGINES;
 exports.WorkflowsStorageClickhouse = WorkflowsStorageClickhouse;
+exports.recordDeletionRequest = recordDeletionRequest;
 
 //# sourceMappingURL=index.cjs.map