@mastra/clickhouse 1.16.1 → 1.17.0-alpha.1

package/dist/index.js

@@ -1,10 +1,12 @@
 import { createClient } from "@clickhouse/client";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
+import * as coreStorage from "@mastra/core/storage";
 import { BRANCH_SPAN_TYPES, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_CURSORS, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, buildInputPreview, calculatePagination, computeTraceStatus, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MessageList } from "@mastra/core/agent";
 import { MastraBase } from "@mastra/core/base";
 import { parseFieldKey } from "@mastra/core/utils";
 import { coreFeatures } from "@mastra/core/features";
+import { randomUUID } from "crypto";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -70,12 +72,22 @@ function getEngineNameAndArgs(engine) {
 }
 function hasBalancedParens(s) {
 	let depth = 0;
-	for (const c of s) if (c === "(") depth++;
-	else if (c === ")") {
-		depth--;
-		if (depth < 0) return false;
+	let quote;
+	for (let i = 0; i < s.length; i++) {
+		const c = s[i];
+		if (quote) {
+			if (c === "\\") i++;
+			else if (c === quote) quote = void 0;
+			continue;
+		}
+		if (c === "`" || c === "\"" || c === "'") quote = c;
+		else if (c === "(") depth++;
+		else if (c === ")") {
+			depth--;
+			if (depth < 0) return false;
+		}
 	}
-	return depth === 0;
+	return depth === 0 && quote === void 0;
 }
 function buildReplicatedTableEngine(engine, replication) {
 	if (!replication) return engine;
@@ -147,20 +159,51 @@ function addOnClusterToDDL(sql, replication) {
 	return out;
 }
 function rewriteEngineClauses(sql, replication) {
-	return sql.replace(/ENGINE\s*=\s*(\w+)\s*/gi, (match, engineName, offset, source) => {
-		const argsStart = offset + match.length;
-		if (source[argsStart] !== "(") return `ENGINE = ${buildReplicatedTableEngine(engineName, replication)}`;
-		let depth = 0;
-		for (let i = argsStart; i < source.length; i++) {
-			const char = source[i];
-			if (char === "(") depth++;
-			else if (char === ")") {
-				depth--;
-				if (depth === 0) return `ENGINE = ${buildReplicatedTableEngine(`${engineName}${source.slice(argsStart, i + 1)}`, replication)}`;
+	const enginePattern = /ENGINE\s*=\s*(\w+)/gi;
+	let result = "";
+	let consumedUntil = 0;
+	for (let match = enginePattern.exec(sql); match; match = enginePattern.exec(sql)) {
+		const engineName = match[1];
+		if (!engineName) continue;
+		let engineEnd = enginePattern.lastIndex;
+		let argsStart = engineEnd;
+		while (/\s/.test(sql[argsStart] ?? "")) argsStart++;
+		let engine = engineName;
+		if (sql[argsStart] === "(") {
+			let depth = 0;
+			let closingParenEnd;
+			let quote;
+			for (let i = argsStart; i < sql.length; i++) {
+				const char = sql[i];
+				if (quote) {
+					if (char === "\\") i++;
+					else if (char === quote) quote = void 0;
+					continue;
+				}
+				if (char === "`" || char === "\"" || char === "'") quote = char;
+				else if (char === "(") depth++;
+				else if (char === ")") {
+					depth--;
+					if (depth === 0) {
+						closingParenEnd = i + 1;
+						break;
+					}
+				}
+			}
+			if (closingParenEnd === void 0) {
+				result += sql.slice(consumedUntil, enginePattern.lastIndex);
+				consumedUntil = enginePattern.lastIndex;
+				continue;
 			}
+			engineEnd = closingParenEnd;
+			engine = `${engineName}${sql.slice(argsStart, engineEnd)}`;
 		}
-		return match;
-	});
+		result += sql.slice(consumedUntil, match.index);
+		result += `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
+		consumedUntil = engineEnd;
+		enginePattern.lastIndex = engineEnd;
+	}
+	return result + sql.slice(consumedUntil);
 }
 function applyReplicationToDDL(sql, replication) {
 	return addOnClusterToDDL(replication ? rewriteEngineClauses(sql, replication) : sql, replication);
@@ -2477,6 +2520,7 @@ var ObservabilityStorageClickhouse = class extends ObservabilityStorage {
 		}
 	}
 	async batchDeleteTraces(args) {
+		this.assertUnscopedBatchDeleteTraces(args);
 		try {
 			if (args.traceIds.length === 0) return;
 			await this.client.command({
@@ -2503,6 +2547,7 @@ const TABLE_METRIC_EVENTS = "mastra_metric_events";
 const TABLE_LOG_EVENTS = "mastra_log_events";
 const TABLE_SCORE_EVENTS = "mastra_score_events";
 const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
+const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
 const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
 const TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
 const TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
@@ -3198,6 +3243,23 @@ PARTITION BY toDate(timestamp)
 ORDER BY (traceId, timestamp, feedbackId)
 SETTINGS allow_nullable_key = 1
 `;
+const DELETION_REQUESTS_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_DELETION_REQUESTS} (
+  requestId       String,
+  organizationId  String DEFAULT '',
+  resourceId      String DEFAULT '',
+  signal          LowCardinality(String),
+  predicateType   LowCardinality(String),
+  predicateValues Array(String),
+  requestedAt     DateTime64(3),
+  requestedBy     String DEFAULT '',
+  lastAppliedAt   DateTime64(3) DEFAULT 0,
+  purgeVerifiedAt DateTime64(3) DEFAULT 0,
+  updatedAt       DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updatedAt)
+ORDER BY (organizationId, resourceId, requestId)
+`;
 function buildFeedbackEventsDeltaDDL() {
 	return `
 CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
@@ -3347,6 +3409,7 @@ const BASE_TABLE_DDL = [
 	LOG_EVENTS_DDL,
 	SCORE_EVENTS_DDL,
 	FEEDBACK_EVENTS_DDL,
+	DELETION_REQUESTS_DDL,
 	DISCOVERY_VALUES_DDL,
 	DISCOVERY_PAIRS_DDL
 ];
@@ -3431,6 +3494,7 @@ const ALL_TABLE_NAMES = [
 	TABLE_LOG_EVENTS,
 	TABLE_SCORE_EVENTS,
 	TABLE_FEEDBACK_EVENTS,
+	TABLE_DELETION_REQUESTS,
 	TABLE_METRIC_EVENTS_DELTA,
 	TABLE_LOG_EVENTS_DELTA,
 	TABLE_SCORE_EVENTS_DELTA,
@@ -4088,6 +4152,35 @@ function feedbackRecordToRow(feedback) {
 	};
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/deletion-requests.ts
+const EPOCH = "1970-01-01T00:00:00.000Z";
+async function recordDeletionRequest(client, args) {
+	const row = {
+		requestId: args.requestId,
+		organizationId: args.organizationId ?? "",
+		resourceId: args.resourceId ?? "",
+		signal: args.signal,
+		predicateType: args.predicateType,
+		predicateValues: args.predicateValues,
+		requestedAt: args.requestedAt,
+		requestedBy: args.requestedBy ?? "",
+		lastAppliedAt: EPOCH,
+		purgeVerifiedAt: EPOCH,
+		updatedAt: args.requestedAt
+	};
+	await client.insert({
+		table: TABLE_DELETION_REQUESTS,
+		values: [row],
+		format: "JSONEachRow",
+		clickhouse_settings: isReplicationConfigured(args.replication) ? {
+			...CH_INSERT_SETTINGS,
+			insert_quorum: "auto",
+			insert_quorum_parallel: 1
+		} : CH_INSERT_SETTINGS
+	});
+	return row;
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/discovery.ts
 async function queryJson$3(client, query, params = {}) {
 	return await (await client.query({
@@ -6091,6 +6184,269 @@ async function getScorePercentiles(client, args) {
 	return { series };
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/trace-query.ts
+const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
+const TRACE_FIELDS = {
+	traceId: {
+		sql: "r.traceId",
+		parameterType: "String"
+	},
+	threadId: {
+		sql: "r.threadId",
+		parameterType: "String"
+	},
+	resourceId: {
+		sql: "r.resourceId",
+		parameterType: "String"
+	},
+	startedAt: {
+		sql: "r.startedAt",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	endedAt: {
+		sql: "r.endedAt",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	entityName: {
+		sql: "r.entityName",
+		parameterType: "String"
+	},
+	entityType: {
+		sql: "r.entityType",
+		parameterType: "String"
+	},
+	environment: {
+		sql: "r.environment",
+		parameterType: "String"
+	},
+	status: {
+		sql: TRACE_STATUS_SQL,
+		parameterType: "String"
+	}
+};
+const SPAN_FIELDS = {
+	spanType: {
+		sql: "s.spanType",
+		parameterType: "String"
+	},
+	error: {
+		sql: "s.error",
+		parameterType: "String"
+	}
+};
+const SCORE_FIELDS = {
+	scorerId: {
+		sql: "s.scorerId",
+		parameterType: "String"
+	},
+	score: {
+		sql: "s.score",
+		parameterType: "Float64"
+	}
+};
+const TRACE_SELECT = `
+  r.traceId AS traceId,
+  r.spanId AS rootSpanId,
+  r.threadId AS threadId,
+  r.resourceId AS resourceId,
+  r.startedAt AS startedAt,
+  r.endedAt AS endedAt,
+  r.entityName AS entityName,
+  r.entityType AS entityType,
+  r.environment AS environment,
+  ${TRACE_STATUS_SQL} AS status`;
+var ParameterBuilder = class {
+	params = {};
+	#next = 1;
+	add(value, type) {
+		const name = `trace_query_${this.#next++}`;
+		this.params[name] = type === "DateTime64(3, 'UTC')" ? new Date(value).toISOString().replace("T", " ").replace(/Z$/, "") : value;
+		return `{${name}:${type}}`;
+	}
+};
+function fieldDefinition(registry, field) {
+	const definition = registry[field];
+	if (definition === void 0) throw new Error(`Unsupported trusted trace-query field: ${field}`);
+	return definition;
+}
+function resolveOrderField(field) {
+	if (field === "startedAt" || field === "endedAt") return field;
+	throw new Error(`Unsupported trusted trace-query field: ${field}`);
+}
+function compileScalarPredicate(predicate, registry, parameters) {
+	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compileScalarPredicate(arg, registry, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
+	if (predicate.type === "not") return `NOT (${compileScalarPredicate(predicate.arg, registry, parameters)})`;
+	const field = fieldDefinition(registry, predicate.field);
+	if (predicate.type === "presence") return `${predicate.operator === "exists" ? "isNotNull" : "isNull"}(${field.sql})`;
+	if (predicate.type === "membership") {
+		const values = predicate.values.map((value) => parameters.add(value, field.parameterType)).join(", ");
+		return `ifNull(${`${field.sql} ${predicate.operator === "in" ? "IN" : "NOT IN"} (${values})`}, ${predicate.operator === "in" ? "0" : "1"})`;
+	}
+	const parameter = parameters.add(predicate.value, field.parameterType);
+	const operator = {
+		eq: "=",
+		ne: "!=",
+		lt: "<",
+		lte: "<=",
+		gt: ">",
+		gte: ">="
+	}[predicate.operator];
+	if (operator === void 0) throw new Error(`Unsupported trusted trace-query operator: ${predicate.operator}`);
+	return `ifNull(${field.sql} ${operator} ${parameter}, ${predicate.operator === "ne" ? "1" : "0"})`;
+}
+function collectRelationCollections(predicate, collections = /* @__PURE__ */ new Set()) {
+	if (!predicate) return collections;
+	if (predicate.type === "relation") collections.add(predicate.collection);
+	else if (predicate.type === "boolean") for (const arg of predicate.args) collectRelationCollections(arg, collections);
+	else if (predicate.type === "not") collectRelationCollections(predicate.arg, collections);
+	return collections;
+}
+function compilePredicate(predicate, parameters) {
+	if (predicate.type === "relation") {
+		const registry = predicate.collection === "spans" ? SPAN_FIELDS : SCORE_FIELDS;
+		const existence = `EXISTS (
+      SELECT 1 FROM ${predicate.collection === "spans" ? "current_spans" : "current_scores"} s
+      WHERE s.traceId = r.traceId
+        AND (${compileScalarPredicate(predicate.predicate, registry, parameters)})
+    )`;
+		return predicate.quantifier === "some" ? existence : `NOT ${existence}`;
+	}
+	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compilePredicate(arg, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
+	if (predicate.type === "not") return `NOT (${compilePredicate(predicate.arg, parameters)})`;
+	return compileScalarPredicate(predicate, TRACE_FIELDS, parameters);
+}
+function compileClickHouseTraceQuery(plan) {
+	const parameters = new ParameterBuilder();
+	const from = parameters.add(plan.timeRange.from, "DateTime64(3, 'UTC')");
+	const to = parameters.add(plan.timeRange.to, "DateTime64(3, 'UTC')");
+	const relationCollections = collectRelationCollections(plan.where);
+	const ctes = [`current_roots AS (
+    SELECT * FROM (
+      SELECT *
+      FROM ${TABLE_TRACE_ROOTS}
+      ORDER BY dedupeKey
+      LIMIT 1 BY dedupeKey
+    )
+    ORDER BY traceId, dedupeKey
+    LIMIT 1 BY traceId
+  )`, `root_scope AS (
+    SELECT *
+    FROM current_roots
+    WHERE startedAt >= ${from}
+      AND startedAt < ${to}
+  )`];
+	if (relationCollections.has("spans")) ctes.push(`current_spans AS (
+    SELECT traceId, spanType, error
+    FROM ${TABLE_SPAN_EVENTS}
+    WHERE traceId IN (SELECT traceId FROM root_scope)
+    ORDER BY dedupeKey
+    LIMIT 1 BY dedupeKey
+  )`);
+	if (relationCollections.has("scores")) ctes.push(`current_scores AS (
+    SELECT traceId, scorerId, score
+    FROM ${TABLE_SCORE_EVENTS}
+    WHERE isNotNull(traceId)
+      AND traceId IN (SELECT traceId FROM root_scope)
+  )`);
+	const predicate = plan.where ? compilePredicate(plan.where, parameters) : "1";
+	ctes.push(`candidates AS (
+    SELECT ${TRACE_SELECT}
+    FROM root_scope r
+    WHERE ${predicate}
+  )`);
+	const candidates = `WITH ${ctes.join(",\n")}`;
+	if (plan.result === "groups") return {
+		query: `${candidates}
+SELECT threadId
+FROM candidates
+WHERE isNotNull(threadId) ${plan.cursor ? `AND threadId > ${parameters.add(plan.cursor.threadId, "String")}` : ""}
+GROUP BY threadId
+ORDER BY threadId ASC
+LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
+		query_params: parameters.params
+	};
+	const orderField = resolveOrderField(plan.orderBy.field);
+	const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
+	let pageCondition = "";
+	if (plan.cursor) {
+		const comparison = plan.orderBy.direction === "asc" ? ">" : "<";
+		const sortValue = parameters.add(plan.cursor.sortValue, "DateTime64(3, 'UTC')");
+		pageCondition = `WHERE (${orderField} ${comparison} ${sortValue} OR (${orderField} = ${sortValue} AND traceId > ${parameters.add(plan.cursor.traceId, "String")}))`;
+	}
+	const limit = parameters.add(plan.limit + 1, "UInt64");
+	return {
+		query: `${candidates}
+SELECT *
+FROM candidates
+${pageCondition}
+ORDER BY ${orderField} ${direction}, traceId ASC
+LIMIT ${limit}`,
+		query_params: parameters.params
+	};
+}
+function asIsoTimestamp(value) {
+	return new Date(value).toISOString();
+}
+function isClickHouseExecutionTimeout(error) {
+	if (!error || typeof error !== "object") return false;
+	const candidate = error;
+	return String(candidate.code ?? "") === "159" || candidate.type === "TIMEOUT_EXCEEDED";
+}
+async function runWithClickHouseTraceQueryTimeout(client, timeoutMs, compiled, queryId) {
+	const resolvedTimeoutMs = coreStorage.resolveTraceQueryTimeoutMs(timeoutMs);
+	try {
+		return await (await client.query({
+			query: compiled.query,
+			query_params: compiled.query_params,
+			query_id: queryId,
+			format: "JSONEachRow",
+			clickhouse_settings: {
+				...CH_SETTINGS,
+				max_execution_time: resolvedTimeoutMs / 1e3
+			}
+		})).json();
+	} catch (error) {
+		if (isClickHouseExecutionTimeout(error)) throw new coreStorage.TraceQueryExecutionError();
+		throw error;
+	}
+}
+async function queryTraces(client, plan, timeoutMs) {
+	const rows = await runWithClickHouseTraceQueryTimeout(client, timeoutMs, compileClickHouseTraceQuery(plan));
+	const visibleRows = rows.slice(0, plan.limit);
+	if (plan.result === "groups") {
+		const groups = visibleRows.map((row) => ({ threadId: String(row.threadId) }));
+		const last = groups.at(-1);
+		return coreStorage.traceQueryResponseSchema.parse({
+			groups,
+			page: { next: rows.length > plan.limit && last ? coreStorage.encodeTraceQueryCursor(plan, {
+				result: "groups",
+				threadId: last.threadId
+			}) : null }
+		});
+	}
+	const traces = visibleRows.map((row) => ({
+		traceId: String(row.traceId),
+		rootSpanId: String(row.rootSpanId),
+		threadId: row.threadId == null ? null : String(row.threadId),
+		resourceId: row.resourceId == null ? null : String(row.resourceId),
+		startedAt: asIsoTimestamp(row.startedAt),
+		endedAt: asIsoTimestamp(row.endedAt),
+		entityName: row.entityName == null ? null : String(row.entityName),
+		entityType: row.entityType == null ? null : String(row.entityType),
+		environment: row.environment == null ? null : String(row.environment),
+		status: row.status
+	}));
+	const last = traces.at(-1);
+	return coreStorage.traceQueryResponseSchema.parse({
+		traces,
+		page: { next: rows.length > plan.limit && last ? coreStorage.encodeTraceQueryCursor(plan, {
+			result: "traces",
+			sortValue: last[plan.orderBy.field],
+			traceId: last.traceId
+		}) : null }
+	});
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/trace-roots.ts
 /**
 * Get the root span for a trace, reading from trace_roots as compatibility path.
@@ -6327,6 +6683,17 @@ function buildTraceCursor(row) {
 }
 //#endregion
 //#region src/storage/domains/observability/v-next/tracing.ts
+/**
+* Tracing operations for ClickHouse v-next observability.
+*
+* Owns: batchCreateSpans, getSpan, getSpans, getTrace, getTraceLight,
+*       listBranches, batchDeleteTraces, dangerouslyClearSpanEvents.
+* Delegates to trace-roots.ts: listTraces, getRootSpan.
+*
+* `listBranches` reads from the MV-fed `mastra_trace_branches` table (one row
+* per branch anchor span). It lives here -- alongside the other read paths
+* over the trace data -- since branches are conceptually a subset of traces.
+*/
 const BRANCH_SPAN_TYPE_SQL_LIST = BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
 /** Insert a single completed span. */
 async function createSpan(client, args) {
@@ -6466,16 +6833,40 @@ async function getTraceLight(client, args) {
 	};
 }
 /**
-* Delete traces by traceId.
-* Issues lightweight DELETE against both span_events and trace_roots.
+* Delete traces by traceId, cascading to trace-derived tables and trace-linked
+* signal events (metrics, logs, scores, feedback). Signal rows with a NULL
+* traceId are never affected.
+*
+* On the tracing tables, rows are targeted by tracing identity: traceId +
+* dedupeKey (which starts with traceId). The dedupeKey condition is redundant
+* for correctness (dedupeKey = traceId:spanId) but satisfies the design-doc
+* requirement that trace deletes reference dedupeKey and helps the engine
+* narrow within the sorted ORDER BY key. Signal tables key on their own event
+* ids, so they are targeted by traceId alone.
+*
+* `trace_branches` must be deleted explicitly: its MV fires on insert only,
+* so span deletes never propagate to it. Delta tables self-expire via TTL and
+* discovery tables self-heal, so neither needs explicit deletes.
 *
-* Targets rows by tracing identity: traceId + dedupeKey (which starts with traceId).
-* The dedupeKey condition is redundant for correctness (dedupeKey = traceId:spanId)
-* but satisfies the design-doc requirement that trace deletes reference dedupeKey
-* and helps the engine narrow within the sorted ORDER BY key.
+* Records the predicate before using lightweight DELETE FROM on every table.
+* Lightweight deletes hide rows through ClickHouse's delete mask; physical
+* removal depends on the deployment's configured retention and merge policy.
+*
+* When the optional tenant scope (`organizationId` / `resourceId`) is set,
+* every DELETE additionally requires the row's tenant columns to match.
 */
-async function batchDeleteTraces(client, args) {
+async function batchDeleteTraces(client, args, replication) {
 	if (args.traceIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "traces",
+		predicateType: "traceIds",
+		predicateValues: [...args.traceIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
 	const params = {};
 	const traceInPlaceholders = [];
 	const dedupeOrParts = [];
@@ -6489,13 +6880,35 @@ async function batchDeleteTraces(client, args) {
 	}
 	const traceInList = traceInPlaceholders.join(", ");
 	const dedupeCondition = dedupeOrParts.length === 1 ? dedupeOrParts[0] : `(${dedupeOrParts.join(" OR ")})`;
-	await Promise.all([client.command({
-		query: `DELETE FROM ${TABLE_SPAN_EVENTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	}), client.command({
-		query: `DELETE FROM ${TABLE_TRACE_ROOTS} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}`,
-		query_params: params
-	})]);
+	let scopeCondition = "";
+	if (args.organizationId !== void 0) {
+		params.scope_org = args.organizationId;
+		scopeCondition += ` AND organizationId = {scope_org:String}`;
+	}
+	if (args.resourceId !== void 0) {
+		params.scope_res = args.resourceId;
+		scopeCondition += ` AND resourceId = {scope_res:String}`;
+	}
+	const tracingTables = [
+		TABLE_SPAN_EVENTS,
+		TABLE_TRACE_ROOTS,
+		TABLE_TRACE_BRANCHES
+	];
+	const signalTables = [
+		TABLE_METRIC_EVENTS,
+		TABLE_LOG_EVENTS,
+		TABLE_SCORE_EVENTS,
+		TABLE_FEEDBACK_EVENTS
+	];
+	await Promise.all([...tracingTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	})), ...signalTables.map((table) => client.command({
+		query: `DELETE FROM ${table} WHERE traceId IN (${traceInList})${scopeCondition}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: "2" }
+	}))]);
 }
 /**
 * List trace branches with optional filtering, pagination, and ordering.
@@ -7033,6 +7446,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	#retention;
 	#replication;
 	#deltaCursorStrategyOverride;
+	#traceQueryTimeoutMs;
 	#deltaCursorStrategy = "fallback";
 	constructor(config) {
 		super();
@@ -7041,6 +7455,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 		this.#replication = replication;
 		this.#retention = config.retention;
 		this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
+		this.#traceQueryTimeoutMs = coreStorage.resolveTraceQueryTimeoutMs(config.traceQueryTimeoutMs);
 	}
 	async init() {
 		const migrationStatus = await checkSignalTablesMigrationStatus(this.#client);
@@ -7144,11 +7559,16 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 		};
 	}
 	getFeatures() {
-		if (!deltaPollingSupported(this.#deltaCursorStrategy)) return ["metrics", "logs"];
+		if (!deltaPollingSupported(this.#deltaCursorStrategy)) return [
+			"metrics",
+			"logs",
+			"trace-query"
+		];
 		return [
 			"metrics",
 			"logs",
-			"delta-polling"
+			"delta-polling",
+			"trace-query"
 		];
 	}
 	async createSpan(args) {
@@ -7263,6 +7683,18 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async queryTraces(plan) {
+		try {
+			return await queryTraces(this.#client, plan, this.#traceQueryTimeoutMs);
+		} catch (error) {
+			if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "QUERY_TRACES", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY
+			}, error);
+		}
+	}
 	async listTracesLight(args) {
 		try {
 			return await listTracesLight(this.#client, args, this.#deltaCursorStrategy);
@@ -7679,7 +8111,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	}
 	async batchDeleteTraces(args) {
 		try {
-			await batchDeleteTraces(this.#client, args);
+			await batchDeleteTraces(this.#client, args, this.#replication);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({
@@ -8539,7 +8971,8 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
 		this.name = "ClickhouseStoreVNext";
 		const observability = new ObservabilityStorageClickhouseVNext({
 			client: this.db,
-			replication: config.replication
+			replication: config.replication,
+			traceQueryTimeoutMs: config.traceQueryTimeoutMs
 		});
 		this.stores = {
 			...this.stores,
@@ -8548,6 +8981,6 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
 	}
 };
 //#endregion
-export { COLUMN_TYPES, ClickhouseStore, ClickhouseStoreVNext, MemoryStorageClickhouse, ObservabilityStorageClickhouse, ObservabilityStorageClickhouseVNext, ScoresStorageClickhouse, TABLE_ENGINES, WorkflowsStorageClickhouse };
+export { COLUMN_TYPES, ClickhouseStore, ClickhouseStoreVNext, MemoryStorageClickhouse, ObservabilityStorageClickhouse, ObservabilityStorageClickhouseVNext, ScoresStorageClickhouse, TABLE_DELETION_REQUESTS, TABLE_ENGINES, WorkflowsStorageClickhouse, recordDeletionRequest };
 
 //# sourceMappingURL=index.js.map