@mastra/clickhouse 1.17.0-alpha.0 → 1.17.0

package/dist/index.js

@@ -6,6 +6,7 @@ import { MessageList } from "@mastra/core/agent";
 import { MastraBase } from "@mastra/core/base";
 import { parseFieldKey } from "@mastra/core/utils";
 import { coreFeatures } from "@mastra/core/features";
+import { randomUUID } from "crypto";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -71,12 +72,22 @@ function getEngineNameAndArgs(engine) {
 }
 function hasBalancedParens(s) {
 	let depth = 0;
-	for (const c of s) if (c === "(") depth++;
-	else if (c === ")") {
-		depth--;
-		if (depth < 0) return false;
+	let quote;
+	for (let i = 0; i < s.length; i++) {
+		const c = s[i];
+		if (quote) {
+			if (c === "\\") i++;
+			else if (c === quote) quote = void 0;
+			continue;
+		}
+		if (c === "`" || c === "\"" || c === "'") quote = c;
+		else if (c === "(") depth++;
+		else if (c === ")") {
+			depth--;
+			if (depth < 0) return false;
+		}
 	}
-	return depth === 0;
+	return depth === 0 && quote === void 0;
 }
 function buildReplicatedTableEngine(engine, replication) {
 	if (!replication) return engine;
@@ -148,20 +159,51 @@ function addOnClusterToDDL(sql, replication) {
 	return out;
 }
 function rewriteEngineClauses(sql, replication) {
-	return sql.replace(/ENGINE\s*=\s*(\w+)\s*/gi, (match, engineName, offset, source) => {
-		const argsStart = offset + match.length;
-		if (source[argsStart] !== "(") return `ENGINE = ${buildReplicatedTableEngine(engineName, replication)}`;
-		let depth = 0;
-		for (let i = argsStart; i < source.length; i++) {
-			const char = source[i];
-			if (char === "(") depth++;
-			else if (char === ")") {
-				depth--;
-				if (depth === 0) return `ENGINE = ${buildReplicatedTableEngine(`${engineName}${source.slice(argsStart, i + 1)}`, replication)}`;
+	const enginePattern = /ENGINE\s*=\s*(\w+)/gi;
+	let result = "";
+	let consumedUntil = 0;
+	for (let match = enginePattern.exec(sql); match; match = enginePattern.exec(sql)) {
+		const engineName = match[1];
+		if (!engineName) continue;
+		let engineEnd = enginePattern.lastIndex;
+		let argsStart = engineEnd;
+		while (/\s/.test(sql[argsStart] ?? "")) argsStart++;
+		let engine = engineName;
+		if (sql[argsStart] === "(") {
+			let depth = 0;
+			let closingParenEnd;
+			let quote;
+			for (let i = argsStart; i < sql.length; i++) {
+				const char = sql[i];
+				if (quote) {
+					if (char === "\\") i++;
+					else if (char === quote) quote = void 0;
+					continue;
+				}
+				if (char === "`" || char === "\"" || char === "'") quote = char;
+				else if (char === "(") depth++;
+				else if (char === ")") {
+					depth--;
+					if (depth === 0) {
+						closingParenEnd = i + 1;
+						break;
+					}
+				}
 			}
+			if (closingParenEnd === void 0) {
+				result += sql.slice(consumedUntil, enginePattern.lastIndex);
+				consumedUntil = enginePattern.lastIndex;
+				continue;
+			}
+			engineEnd = closingParenEnd;
+			engine = `${engineName}${sql.slice(argsStart, engineEnd)}`;
 		}
-		return match;
-	});
+		result += sql.slice(consumedUntil, match.index);
+		result += `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
+		consumedUntil = engineEnd;
+		enginePattern.lastIndex = engineEnd;
+	}
+	return result + sql.slice(consumedUntil);
 }
 function applyReplicationToDDL(sql, replication) {
 	return addOnClusterToDDL(replication ? rewriteEngineClauses(sql, replication) : sql, replication);
@@ -2478,6 +2520,7 @@ var ObservabilityStorageClickhouse = class extends ObservabilityStorage {
 		}
 	}
 	async batchDeleteTraces(args) {
+		this.assertUnscopedBatchDeleteTraces(args);
 		try {
 			if (args.traceIds.length === 0) return;
 			await this.client.command({
@@ -2504,6 +2547,7 @@ const TABLE_METRIC_EVENTS = "mastra_metric_events";
 const TABLE_LOG_EVENTS = "mastra_log_events";
 const TABLE_SCORE_EVENTS = "mastra_score_events";
 const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
+const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
 const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
 const TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
 const TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
@@ -3199,6 +3243,23 @@ PARTITION BY toDate(timestamp)
 ORDER BY (traceId, timestamp, feedbackId)
 SETTINGS allow_nullable_key = 1
 `;
+const DELETION_REQUESTS_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_DELETION_REQUESTS} (
+  requestId       String,
+  organizationId  String DEFAULT '',
+  resourceId      String DEFAULT '',
+  signal          LowCardinality(String),
+  predicateType   LowCardinality(String),
+  predicateValues Array(String),
+  requestedAt     DateTime64(3),
+  requestedBy     String DEFAULT '',
+  lastAppliedAt   DateTime64(3) DEFAULT 0,
+  purgeVerifiedAt DateTime64(3) DEFAULT 0,
+  updatedAt       DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updatedAt)
+ORDER BY (organizationId, resourceId, requestId)
+`;
 function buildFeedbackEventsDeltaDDL() {
 	return `
 CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
@@ -3348,6 +3409,7 @@ const BASE_TABLE_DDL = [
 	LOG_EVENTS_DDL,
 	SCORE_EVENTS_DDL,
 	FEEDBACK_EVENTS_DDL,
+	DELETION_REQUESTS_DDL,
 	DISCOVERY_VALUES_DDL,
 	DISCOVERY_PAIRS_DDL
 ];
@@ -3432,6 +3494,7 @@ const ALL_TABLE_NAMES = [
 	TABLE_LOG_EVENTS,
 	TABLE_SCORE_EVENTS,
 	TABLE_FEEDBACK_EVENTS,
+	TABLE_DELETION_REQUESTS,
 	TABLE_METRIC_EVENTS_DELTA,
 	TABLE_LOG_EVENTS_DELTA,
 	TABLE_SCORE_EVENTS_DELTA,
@@ -4089,6 +4152,35 @@ function feedbackRecordToRow(feedback) {
 	};
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/deletion-requests.ts
+const EPOCH = "1970-01-01T00:00:00.000Z";
+async function recordDeletionRequest(client, args) {
+	const row = {
+		requestId: args.requestId,
+		organizationId: args.organizationId ?? "",
+		resourceId: args.resourceId ?? "",
+		signal: args.signal,
+		predicateType: args.predicateType,
+		predicateValues: args.predicateValues,
+		requestedAt: args.requestedAt,
+		requestedBy: args.requestedBy ?? "",
+		lastAppliedAt: EPOCH,
+		purgeVerifiedAt: EPOCH,
+		updatedAt: args.requestedAt
+	};
+	await client.insert({
+		table: TABLE_DELETION_REQUESTS,
+		values: [row],
+		format: "JSONEachRow",
+		clickhouse_settings: isReplicationConfigured(args.replication) ? {
+			...CH_INSERT_SETTINGS,
+			insert_quorum: "auto",
+			insert_quorum_parallel: 1
+		} : CH_INSERT_SETTINGS
+	});
+	return row;
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/discovery.ts
 async function queryJson$3(client, query, params = {}) {
 	return await (await client.query({
@@ -6591,6 +6683,17 @@ function buildTraceCursor(row) {
 }
 //#endregion
 //#region src/storage/domains/observability/v-next/tracing.ts
+/**
+* Tracing operations for ClickHouse v-next observability.
+*
+* Owns: batchCreateSpans, getSpan, getSpans, getTrace, getTraceLight,
+*       listBranches, batchDeleteTraces, dangerouslyClearSpanEvents.
+* Delegates to trace-roots.ts: listTraces, getRootSpan.
+*
+* `listBranches` reads from the MV-fed `mastra_trace_branches` table (one row
+* per branch anchor span). It lives here -- alongside the other read paths
+* over the trace data -- since branches are conceptually a subset of traces.
+*/
 const BRANCH_SPAN_TYPE_SQL_LIST = BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
 /** Insert a single completed span. */
 async function createSpan(client, args) {
@@ -6730,16 +6833,40 @@ async function getTraceLight(client, args) {
 	};
 }
 /**
-* Delete traces by traceId.
-* Issues lightweight DELETE against both span_events and trace_roots.
+* Delete traces by traceId, cascading to trace-derived tables and trace-linked
+* signal events (metrics, logs, scores, feedback). Signal rows with a NULL
+* traceId are never affected.
+*
+* On the tracing tables, rows are targeted by tracing identity: traceId +
+* dedupeKey (which starts with traceId). The dedupeKey condition is redundant
+* for correctness (dedupeKey = traceId:spanId) but satisfies the design-doc
+* requirement that trace deletes reference dedupeKey and helps the engine
+* narrow within the sorted ORDER BY key. Signal tables key on their own event
+* ids, so they are targeted by traceId alone.
+*
+* `trace_branches` must be deleted explicitly: its MV fires on insert only,
+* so span deletes never propagate to it. Delta tables self-expire via TTL and
+* discovery tables self-heal, so neither needs explicit deletes.
 *
-* Targets rows by tracing identity: traceId + dedupeKey (which starts with traceId).
-* The dedupeKey condition is redundant for correctness (dedupeKey = traceId:spanId)
-* but satisfies the design-doc requirement that trace deletes reference dedupeKey
-* and helps the engine narrow within the sorted ORDER BY key.
+* Records the predicate before using lightweight DELETE FROM on every table.
+* Lightweight deletes hide rows through ClickHouse's delete mask; physical
+* removal depends on the deployment's configured retention and merge policy.
+*
+* When the optional tenant scope (`organizationId` / `resourceId`) is set,
+* every DELETE additionally requires the row's tenant columns to match.
 */
-async function batchDeleteTraces(client, args) {
+async function batchDeleteTraces(client, args, replication) {
 	if (args.traceIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "traces",
+		predicateType: "traceIds",
+		predicateValues: [...args.traceIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
 	const params = {};
 	const traceInPlaceholders = [];
 	const dedupeOrParts = [];
@@ -6753,13 +6880,35 @@ async function batchDeleteTraces(client, args) {
 	}
 	const traceInList = traceInPlaceholders.join(", ");
 	const dedupeCondition = dedupeOrParts.length === 1 ? dedupeOrParts[0] : `(${dedupeOrParts.join(" OR ")})`;
-	await Promise.all([client.command({
-		query: `DELETE FROM ${TABLE_SPAN_EVENTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	}), client.command({
-		query: `DELETE FROM ${TABLE_TRACE_ROOTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	})]);
+	let scopeCondition = "";
+	if (args.organizationId !== void 0) {
+		params.scope_org = args.organizationId;
+		scopeCondition += ` AND organizationId = {scope_org:String}`;
+	}
+	if (args.resourceId !== void 0) {
+		params.scope_res = args.resourceId;
+		scopeCondition += ` AND resourceId = {scope_res:String}`;
+	}
+	const tracingTables = [
+		TABLE_SPAN_EVENTS,
+		TABLE_TRACE_ROOTS,
+		TABLE_TRACE_BRANCHES
+	];
+	const signalTables = [
+		TABLE_METRIC_EVENTS,
+		TABLE_LOG_EVENTS,
+		TABLE_SCORE_EVENTS,
+		TABLE_FEEDBACK_EVENTS
+	];
+	await Promise.all([...tracingTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	})), ...signalTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList})${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	}))]);
 }
 /**
 * List trace branches with optional filtering, pagination, and ordering.
@@ -7962,7 +8111,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	}
 	async batchDeleteTraces(args) {
 		try {
-			await batchDeleteTraces(this.#client, args);
+			await batchDeleteTraces(this.#client, args, this.#replication);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({
@@ -8832,6 +8981,6 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
 	}
 };
 //#endregion
-export { COLUMN_TYPES, ClickhouseStore, ClickhouseStoreVNext, MemoryStorageClickhouse, ObservabilityStorageClickhouse, ObservabilityStorageClickhouseVNext, ScoresStorageClickhouse, TABLE_ENGINES, WorkflowsStorageClickhouse };
+export { COLUMN_TYPES, ClickhouseStore, ClickhouseStoreVNext, MemoryStorageClickhouse, ObservabilityStorageClickhouse, ObservabilityStorageClickhouseVNext, ScoresStorageClickhouse, TABLE_DELETION_REQUESTS, TABLE_ENGINES, WorkflowsStorageClickhouse, recordDeletionRequest };
 
 //# sourceMappingURL=index.js.map