@mastra/clickhouse 1.20.0-alpha.2 → 1.20.0-alpha.4

package/dist/index.js

@@ -7,6 +7,7 @@ import { MastraBase } from "@mastra/core/base";
 import { randomUUID } from "crypto";
 import { parseFieldKey } from "@mastra/core/utils";
 import { coreFeatures } from "@mastra/core/features";
+import { z } from "zod/v4";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -2546,6 +2547,8 @@ const TABLE_TRACE_BRANCHES_DELTA = "mastra_trace_branches_delta";
 const TABLE_METRIC_EVENTS = "mastra_metric_events";
 const TABLE_LOG_EVENTS = "mastra_log_events";
 const TABLE_SCORE_EVENTS = "mastra_score_events";
+const TABLE_SCORE_EVENTS_CURRENT = "mastra_score_events_current";
+const TABLE_SCORE_EVENTS_CURRENT_BACKFILL = "mastra_score_events_current_backfill";
 const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
 const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
 const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
@@ -2561,6 +2564,7 @@ const MV_TRACE_BRANCHES_DELTA = "mastra_mv_trace_branches_delta";
 const MV_METRIC_EVENTS_DELTA = "mastra_mv_metric_events_delta";
 const MV_LOG_EVENTS_DELTA = "mastra_mv_log_events_delta";
 const MV_SCORE_EVENTS_DELTA = "mastra_mv_score_events_delta";
+const MV_SCORE_EVENTS_CURRENT = "mastra_mv_score_events_current";
 const MV_FEEDBACK_EVENTS_DELTA = "mastra_mv_feedback_events_delta";
 const MV_DISCOVERY_VALUES = "mastra_mv_discovery_values";
 const MV_DISCOVERY_PAIRS = "mastra_mv_discovery_pairs";
@@ -3078,19 +3082,53 @@ FROM (
 )
 `;
 }
-const SCORE_EVENTS_DDL = `
-CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
-  -- Timestamp
+const SCORE_EVENT_COLUMN_NAMES = [
+	"timestamp",
+	"scoreId",
+	"writeVersion",
+	"traceId",
+	"spanId",
+	"experimentId",
+	"scoreTraceId",
+	"entityType",
+	"entityId",
+	"entityName",
+	"entityVersionId",
+	"parentEntityVersionId",
+	"parentEntityType",
+	"parentEntityId",
+	"parentEntityName",
+	"rootEntityVersionId",
+	"rootEntityType",
+	"rootEntityId",
+	"rootEntityName",
+	"userId",
+	"organizationId",
+	"resourceId",
+	"runId",
+	"sessionId",
+	"threadId",
+	"requestId",
+	"environment",
+	"executionSource",
+	"serviceName",
+	"scorerId",
+	"scorerVersion",
+	"scoreSource",
+	"score",
+	"reason",
+	"tags",
+	"metadata",
+	"scope"
+];
+const SCORE_EVENT_COLUMNS_DDL = `
   timestamp          DateTime64(3, 'UTC'),
-
-  -- IDs
   scoreId            String,
+  writeVersion       UInt64 DEFAULT 0,
   traceId            Nullable(String),
   spanId             Nullable(String),
   experimentId       Nullable(String),
   scoreTraceId       Nullable(String),
-
-  -- Entity hierarchy
   entityType         LowCardinality(Nullable(String)),
   entityId           Nullable(String),
   entityName         Nullable(String),
@@ -3103,8 +3141,6 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
   rootEntityType     LowCardinality(Nullable(String)),
   rootEntityId       Nullable(String),
   rootEntityName     Nullable(String),
-
-  -- Context
   userId             Nullable(String),
   organizationId     Nullable(String),
   resourceId         Nullable(String),
@@ -3115,30 +3151,48 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
   environment        LowCardinality(Nullable(String)),
   executionSource    LowCardinality(Nullable(String)),
   serviceName        LowCardinality(Nullable(String)),
-
-  -- Scorer identity
   scorerId           LowCardinality(String),
   scorerVersion      LowCardinality(Nullable(String)),
   scoreSource        LowCardinality(Nullable(String)),
-
-  -- Score value
   score              Float64,
-
-  -- Information-only
   reason             Nullable(String),
-
-  -- Query-relevant flexible fields
   tags               Array(LowCardinality(String)) DEFAULT [],
-
-  -- Information-only JSON payloads
   metadata           Nullable(String),
   scope              Nullable(String)
+`;
+const SCORE_EVENTS_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
+${SCORE_EVENT_COLUMNS_DDL},
+  INDEX idx_scoreId scoreId TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree
 PARTITION BY toDate(timestamp)
 ORDER BY (traceId, timestamp, scoreId)
 SETTINGS allow_nullable_key = 1
 `;
+const SCORE_EVENTS_CURRENT_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_CURRENT} (
+${SCORE_EVENT_COLUMNS_DDL}
+)
+ENGINE = ReplacingMergeTree(writeVersion)
+PARTITION BY cityHash64(scoreId) % 64
+ORDER BY scoreId
+`;
+const SCORE_EVENTS_CURRENT_BACKFILL_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} (
+  marker String,
+  completedAt DateTime64(3, 'UTC')
+)
+ENGINE = ReplacingMergeTree(completedAt)
+ORDER BY marker
+`;
+const SCORE_EVENTS_CURRENT_MV_DDL = `
+CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_CURRENT}
+TO ${TABLE_SCORE_EVENTS_CURRENT}
+AS
+SELECT ${SCORE_EVENT_COLUMN_NAMES.join(", ")}
+FROM ${TABLE_SCORE_EVENTS}
+`;
 function buildScoreEventsDeltaDDL() {
 	return `
 CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
@@ -3412,6 +3466,8 @@ const BASE_TABLE_DDL = [
 	METRIC_EVENTS_DDL,
 	LOG_EVENTS_DDL,
 	SCORE_EVENTS_DDL,
+	SCORE_EVENTS_CURRENT_DDL,
+	SCORE_EVENTS_CURRENT_BACKFILL_DDL,
 	FEEDBACK_EVENTS_DDL,
 	DELETION_REQUESTS_DDL,
 	DISCOVERY_VALUES_DDL,
@@ -3430,7 +3486,11 @@ function buildDeltaTableDDL() {
 function buildAllTableDDL() {
 	return [...BASE_TABLE_DDL, ...buildDeltaTableDDL()];
 }
-const BASE_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
+const BASE_MV_DDL = [
+	TRACE_ROOTS_MV_DDL,
+	TRACE_BRANCHES_MV_DDL,
+	SCORE_EVENTS_CURRENT_MV_DDL
+];
 function buildDeltaMvDDL(strategy) {
 	return [
 		buildTraceRootsDeltaMvDDL(strategy),
@@ -3452,11 +3512,11 @@ const addColumn = (table, name, type) => ({
 	name,
 	sql: `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${name} ${type}`
 });
-const addBloomIndex = (table, name, column) => ({
+const addBloomIndex = (table, name, column, granularity = 2) => ({
 	kind: "index",
 	table,
 	name,
-	sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY 2`
+	sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY ${granularity}`
 });
 const ALL_MIGRATIONS = [
 	addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
@@ -3471,9 +3531,11 @@ const ALL_MIGRATIONS = [
 	addColumn(TABLE_LOG_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_LOG_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_LOG_EVENTS, "rootEntityVersionId", "Nullable(String)"),
+	addColumn(TABLE_SCORE_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_SCORE_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
+	addBloomIndex(TABLE_SCORE_EVENTS, "idx_scoreId", "scoreId", 1),
 	addColumn(TABLE_FEEDBACK_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "reviewStatus", "LowCardinality(String) DEFAULT 'needs-review'"),
@@ -3499,6 +3561,8 @@ const ALL_TABLE_NAMES = [
 	TABLE_METRIC_EVENTS,
 	TABLE_LOG_EVENTS,
 	TABLE_SCORE_EVENTS,
+	TABLE_SCORE_EVENTS_CURRENT,
+	TABLE_SCORE_EVENTS_CURRENT_BACKFILL,
 	TABLE_FEEDBACK_EVENTS,
 	TABLE_DELETION_REQUESTS,
 	TABLE_METRIC_EVENTS_DELTA,
@@ -3516,6 +3580,7 @@ const SIGNAL_TTL_COLUMNS = {
 	[TABLE_METRIC_EVENTS]: "timestamp",
 	[TABLE_LOG_EVENTS]: "timestamp",
 	[TABLE_SCORE_EVENTS]: "timestamp",
+	[TABLE_SCORE_EVENTS_CURRENT]: "timestamp",
 	[TABLE_FEEDBACK_EVENTS]: "timestamp"
 };
 /** Maps each signal key to the table(s) it controls. */
@@ -3527,7 +3592,7 @@ const SIGNAL_TO_TABLES = {
 	],
 	logs: [TABLE_LOG_EVENTS],
 	metrics: [TABLE_METRIC_EVENTS],
-	scores: [TABLE_SCORE_EVENTS],
+	scores: [TABLE_SCORE_EVENTS, TABLE_SCORE_EVENTS_CURRENT],
 	feedback: [TABLE_FEEDBACK_EVENTS]
 };
 const RETENTION_MANAGED_TABLES = [...Object.keys(SIGNAL_TTL_COLUMNS), TABLE_DELETION_REQUESTS];
@@ -5947,6 +6012,36 @@ async function migrateSignalTables(client, logger) {
 	}
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/score-current.ts
+const SCORE_CURRENT_BACKFILL_MARKER = "v1";
+async function backfillCurrentScores(client) {
+	const markerRows = await (await client.query({
+		query: `SELECT count() AS count
+            FROM ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} FINAL
+            WHERE marker = {marker:String}`,
+		query_params: { marker: SCORE_CURRENT_BACKFILL_MARKER },
+		format: "JSONEachRow"
+	})).json();
+	if (Number(markerRows[0]?.count ?? 0) > 0) return;
+	const columns = SCORE_EVENT_COLUMN_NAMES.join(", ");
+	await client.command({
+		query: `INSERT INTO ${TABLE_SCORE_EVENTS_CURRENT} (${columns})
+            SELECT ${columns}
+            FROM (
+              SELECT *, cityHash64(tuple(*)) AS _currentScoreFingerprint
+              FROM ${TABLE_SCORE_EVENTS} FINAL
+              ORDER BY scoreId, writeVersion DESC, timestamp DESC, _currentScoreFingerprint DESC
+              LIMIT 1 BY scoreId
+            )`,
+		clickhouse_settings: { max_bytes_ratio_before_external_sort: .5 }
+	});
+	await client.command({
+		query: `INSERT INTO ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} (marker, completedAt)
+            VALUES ({marker:String}, now64(3, 'UTC'))`,
+		query_params: { marker: SCORE_CURRENT_BACKFILL_MARKER }
+	});
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/scores.ts
 const SCORE_TYPED_COLUMNS = /* @__PURE__ */ new Set([
 	"timestamp",
@@ -6023,6 +6118,13 @@ function mergeFilters(...parts) {
 function toWhereClause(filter) {
 	return filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
 }
+function currentScoresRelation(sourcePredicate) {
+	return `(
+    SELECT *
+    FROM ${TABLE_SCORE_EVENTS_CURRENT} FINAL
+    ${sourcePredicate ? `WHERE ${sourcePredicate}` : ""}
+  )`;
+}
 function buildScoreIdentityFilter(args) {
 	const conditions = ["scorerId = {olapScorerId:String}"];
 	const params = { olapScorerId: args.scorerId };
@@ -6058,6 +6160,24 @@ async function queryJson(client, query, params) {
 		clickhouse_settings: CH_SETTINGS
 	})).json();
 }
+async function scoreRowsWithWriteVersions(client, scores) {
+	const scoreIds = [...new Set(scores.flatMap((score) => score.scoreId ? [score.scoreId] : []))];
+	const existingVersions = [];
+	const lookupBatchSize = 1e3;
+	for (let offset = 0; offset < scoreIds.length; offset += lookupBatchSize) existingVersions.push(...await queryJson(client, `SELECT scoreId, toString(max(writeVersion)) AS writeVersion
+         FROM ${TABLE_SCORE_EVENTS}
+         WHERE scoreId IN ({scoreIds:Array(String)})
+         GROUP BY scoreId`, { scoreIds: scoreIds.slice(offset, offset + lookupBatchSize) }));
+	const versions = new Map(existingVersions.map((row) => [row.scoreId, BigInt(row.writeVersion)]));
+	return scores.map((score) => {
+		const writeVersion = score.scoreId ? (versions.get(score.scoreId) ?? 0n) + 1n : 1n;
+		if (score.scoreId) versions.set(score.scoreId, writeVersion);
+		return {
+			...scoreRecordToRow(score),
+			writeVersion: writeVersion.toString()
+		};
+	});
+}
 async function createScore(client, args) {
 	await batchCreateScores(client, { scores: [args.score] });
 }
@@ -6065,7 +6185,7 @@ async function batchCreateScores(client, args) {
 	if (args.scores.length === 0) return;
 	await client.insert({
 		table: TABLE_SCORE_EVENTS,
-		values: args.scores.map(scoreRecordToRow),
+		values: await scoreRowsWithWriteVersions(client, args.scores),
 		format: "JSONEachRow",
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
@@ -6108,10 +6228,11 @@ async function deleteScores(client, args, replication) {
 		conditions.push("resourceId = {delResourceId:String}");
 		params.delResourceId = args.resourceId;
 	}
-	await client.command({
-		query: `DELETE FROM ${TABLE_SCORE_EVENTS} WHERE ${conditions.join(" AND ")}`,
+	const clickhouse_settings = { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" };
+	for (const table of [TABLE_SCORE_EVENTS_CURRENT, TABLE_SCORE_EVENTS]) await client.command({
+		query: `DELETE FROM ${table} WHERE ${conditions.join(" AND ")}`,
 		query_params: params,
-		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+		clickhouse_settings
 	});
 }
 async function listScores(client, args, strategy) {
@@ -6145,8 +6266,9 @@ async function listScores(client, args, strategy) {
 		};
 	}
 	const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor$2(client, whereClause, filter.params) : void 0;
-	const countResult = await queryJson(client, `SELECT count() AS total FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause}`, filter.params);
-	const rows = await queryJson(client, `SELECT * FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause} ORDER BY ${orderBy} LIMIT {limit:UInt32} OFFSET {offset:UInt32}`, {
+	const scoresRelation = currentScoresRelation();
+	const countResult = await queryJson(client, `SELECT count() AS total FROM ${scoresRelation} AS s ${whereClause}`, filter.params);
+	const rows = await queryJson(client, `SELECT * FROM ${scoresRelation} AS s ${whereClause} ORDER BY ${orderBy} LIMIT {limit:UInt32} OFFSET {offset:UInt32}`, {
 		...filter.params,
 		limit: pagination.limit,
 		offset: pagination.offset
@@ -6205,14 +6327,15 @@ function buildScoresCursor(row) {
 	return row.cursorId ?? "0";
 }
 async function getScoreById(client, scoreId) {
-	const rows = await queryJson(client, `SELECT * FROM ${TABLE_SCORE_EVENTS} WHERE scoreId = {scoreId:String} LIMIT 1`, { scoreId });
+	const rows = await queryJson(client, `SELECT * FROM ${currentScoresRelation("scoreId = {scoreId:String}")} LIMIT 1`, { scoreId });
 	return rows[0] ? rowToScoreRecord(rows[0]) : null;
 }
 async function getScoreAggregate(client, args) {
 	const aggSql = getAggregationSql(args.aggregation);
 	const identity = buildScoreIdentityFilter(args);
 	const combined = mergeFilters(identity, buildScoresFilterConditions(args.filters));
-	const result = await queryJson(client, `SELECT ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${toWhereClause(combined)}`, combined.params);
+	const whereClause = toWhereClause(combined);
+	const result = await queryJson(client, `SELECT ${aggSql} AS value FROM ${currentScoresRelation()} ${whereClause}`, combined.params);
 	const value = result[0]?.value == null ? null : Number(result[0]?.value);
 	if (args.comparePeriod && args.filters?.timestamp) {
 		const ts = args.filters.timestamp;
@@ -6246,7 +6369,8 @@ async function getScoreAggregate(client, args) {
 					endExclusive: ts.endExclusive
 				}
 			}));
-			const prevResult = await queryJson(client, `SELECT ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${toWhereClause(prevCombined)}`, prevCombined.params);
+			const prevWhereClause = toWhereClause(prevCombined);
+			const prevResult = await queryJson(client, `SELECT ${aggSql} AS value FROM ${currentScoresRelation()} ${prevWhereClause}`, prevCombined.params);
 			const previousValue = prevResult[0]?.value == null ? null : Number(prevResult[0]?.value);
 			let changePercent = null;
 			if (previousValue !== null && previousValue !== 0 && value !== null) changePercent = (value - previousValue) / Math.abs(previousValue) * 100;
@@ -6264,7 +6388,7 @@ async function getScoreBreakdown(client, args) {
 	const combined = mergeFilters(buildScoreIdentityFilter(args), buildScoresFilterConditions(args.filters));
 	const whereClause = toWhereClause(combined);
 	const resolved = resolveScoreGroupBy(args.groupBy);
-	return { groups: (await queryJson(client, `SELECT ${resolved.map((e) => e.selectSql).join(", ")}, ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${whereClause} GROUP BY ${resolved.map((e) => e.groupSql).join(", ")} ORDER BY value DESC`, combined.params)).map((row) => ({
+	return { groups: (await queryJson(client, `SELECT ${resolved.map((e) => e.selectSql).join(", ")}, ${aggSql} AS value FROM ${currentScoresRelation()} ${whereClause} GROUP BY ${resolved.map((e) => e.groupSql).join(", ")} ORDER BY value DESC`, combined.params)).map((row) => ({
 		dimensions: Object.fromEntries(resolved.map((entry, index) => {
 			const v = row[`group_by_${index}`];
 			return [entry.key, v == null ? null : String(v)];
@@ -6283,7 +6407,7 @@ async function getScoreTimeSeries(client, args) {
       SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
              ${resolved.map((e) => e.selectSql).join(", ")},
              ${aggSql} AS value
-      FROM ${TABLE_SCORE_EVENTS} ${whereClause}
+      FROM ${currentScoresRelation()} ${whereClause}
       GROUP BY bucket, ${resolved.map((e) => e.groupSql).join(", ")}
       ORDER BY bucket
     `, combined.params);
@@ -6305,7 +6429,7 @@ async function getScoreTimeSeries(client, args) {
 	const rows = await queryJson(client, `
     SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
            ${aggSql} AS value
-    FROM ${TABLE_SCORE_EVENTS} ${whereClause}
+    FROM ${currentScoresRelation()} ${whereClause}
     GROUP BY bucket
     ORDER BY bucket
   `, combined.params);
@@ -6328,7 +6452,7 @@ async function getScorePercentiles(client, args) {
 		const rows = await queryJson(client, `
       SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
              quantile(${p})(score) AS pvalue
-      FROM ${TABLE_SCORE_EVENTS}
+      FROM ${currentScoresRelation()}
       ${whereClause}
       GROUP BY bucket
       ORDER BY bucket
@@ -6697,11 +6821,9 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters)
       entityVersionId,
       parentEntityVersionId,
       rootEntityVersionId
-    FROM ${TABLE_SCORE_EVENTS}
-    WHERE isNotNull(traceId)
-      AND traceId IN (SELECT traceId FROM root_scope)
-    ORDER BY scoreId, timestamp DESC
-    LIMIT 1 BY scoreId
+    FROM ${currentScoresRelation()} AS current
+    WHERE isNotNull(current.traceId)
+      AND current.traceId IN (SELECT traceId FROM root_scope)
   )`);
 	if (relationCollections.has("feedback")) ctes.push(`current_feedback AS (
     SELECT
@@ -6728,7 +6850,18 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters)
   )`);
 	return ctes;
 }
-function compileClickHouseTraceQuery(plan) {
+const deltaWatermarkSchema = z.object({
+	cursorId: z.string().regex(/^\d+$/).refine((value) => BigInt(value) <= 18446744073709551615n),
+	traceId: z.string()
+}).strict();
+function parseDeltaWatermark(value) {
+	try {
+		return deltaWatermarkSchema.parse(JSON.parse(value));
+	} catch {
+		throw new coreStorage.TraceQueryCursorError("TRACE_QUERY_CURSOR_MALFORMED");
+	}
+}
+function compileClickHouseTraceQuery(plan, deltaHead) {
 	const parameters = new ParameterBuilder();
 	const ctes = compileClickHouseTraceScope(plan, collectRelationCollections(plan.where), parameters);
 	const predicate = plan.where ? compilePredicate(plan.where, parameters) : "1";
@@ -6748,8 +6881,70 @@ ORDER BY threadId ASC
 LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
 		query_params: parameters.params
 	};
+	if (plan.paginationMode === "delta") {
+		const watermark = coreStorage.getTraceQueryDeltaWatermark(plan, "clickhouse");
+		const after = watermark ? parseDeltaWatermark(watermark) : {
+			cursorId: "0",
+			traceId: ""
+		};
+		return {
+			query: `${candidates}, delta_candidates AS (
+  SELECT traceId, max(cursorId) AS latestCursorId
+  FROM ${TABLE_TRACE_ROOTS_DELTA}
+  WHERE tuple(cursorId, traceId) > ${`tuple(${parameters.add(after.cursorId, "UInt64")}, ${parameters.add(after.traceId, "String")})`} ${deltaHead ? `AND tuple(cursorId, traceId) <= tuple(${parameters.add(deltaHead.cursorId, "UInt64")}, ${parameters.add(deltaHead.traceId, "String")})` : ""}
+  GROUP BY traceId
+)
+SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
+FROM candidates c
+INNER JOIN delta_candidates d ON c.traceId = d.traceId
+ORDER BY d.latestCursorId ASC, c.traceId ASC
+LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
+			query_params: parameters.params,
+			sharedSnapshot: true
+		};
+	}
 	const orderField = resolveOrderField(plan.orderBy.field);
 	const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
+	if (plan.paginationMode === "page") return {
+		query: `${candidates},
+page_rows AS (
+  SELECT *, row_number() OVER (ORDER BY ${orderField} ${direction}, traceId ASC) AS __row_position
+  FROM candidates
+  ORDER BY ${orderField} ${direction}, traceId ASC
+  LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}
+),
+page_total AS (
+  SELECT count() AS total
+  FROM candidates
+)
+SELECT page_rows.*, page_total.total, 0 AS __metadata
+FROM page_rows
+CROSS JOIN page_total
+UNION ALL
+SELECT
+  '' AS traceId,
+  '' AS rootSpanId,
+  '' AS name,
+  CAST(NULL, 'Nullable(String)') AS entityId,
+  CAST(NULL, 'Nullable(String)') AS parentSpanId,
+  CAST(NULL, 'Nullable(String)') AS metadata,
+  CAST(NULL, 'Nullable(String)') AS input,
+  CAST(NULL, 'Nullable(String)') AS threadId,
+  CAST(NULL, 'Nullable(String)') AS resourceId,
+  toDateTime64(0, 3, 'UTC') AS startedAt,
+  toDateTime64(0, 3, 'UTC') AS endedAt,
+  CAST(NULL, 'Nullable(String)') AS entityName,
+  CAST(NULL, 'Nullable(String)') AS entityType,
+  CAST(NULL, 'Nullable(String)') AS environment,
+  '' AS status,
+  0 AS __row_position,
+  page_total.total AS total,
+  1 AS __metadata
+FROM page_total
+ORDER BY __metadata ASC, __row_position ASC`,
+		query_params: parameters.params,
+		sharedSnapshot: true
+	};
 	let pageCondition = "";
 	if (plan.cursor) {
 		const comparison = plan.orderBy.direction === "asc" ? ">" : "<";
@@ -6896,7 +7091,8 @@ async function runWithClickHouseTraceQueryTimeout(client, limits, compiled, quer
 			clickhouse_settings: {
 				...CH_SETTINGS,
 				max_execution_time: resolvedTimeoutMs / 1e3,
-				...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) }
+				...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) },
+				...compiled.sharedSnapshot ? { enable_shared_storage_snapshot_in_query: 1 } : {}
 			}
 		})).json();
 	} catch (error) {
@@ -6926,8 +7122,70 @@ async function getTraceQueryValues(client, plan, limits) {
 		valuesTruncated: rows.length > plan.limit
 	});
 }
-async function queryTraces(client, plan, timeoutMs) {
-	const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs }, compileClickHouseTraceQuery(plan));
+async function queryTraces(client, plan, timeoutMs, strategy = null) {
+	const deadline = performance.now() + coreStorage.resolveTraceQueryTimeoutMs(timeoutMs);
+	const remaining = () => {
+		const value = Math.floor(deadline - performance.now());
+		if (value <= 0) throw new coreStorage.TraceQueryExecutionError();
+		return value;
+	};
+	let deltaHead;
+	if (plan.paginationMode === "delta") {
+		assertDeltaPollingSupported(strategy);
+		const watermark = coreStorage.getTraceQueryDeltaWatermark(plan, "clickhouse");
+		if (watermark) parseDeltaWatermark(watermark);
+	}
+	if (plan.paginationMode === "delta" || plan.paginationMode === "page" && deltaPollingSupported(strategy) && typeof coreStorage.encodeTraceQueryDeltaCursor === "function") {
+		const head = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: remaining() }, {
+			query: `SELECT toString(cursorId) AS cursorId, traceId FROM ${TABLE_TRACE_ROOTS_DELTA} ORDER BY cursorId DESC, traceId DESC LIMIT 1`,
+			query_params: {}
+		});
+		deltaHead = head[0] ? parseDeltaWatermark(JSON.stringify(head[0])) : {
+			cursorId: "0",
+			traceId: ""
+		};
+		if (plan.paginationMode === "delta" && !plan.deltaCursor) return coreStorage.traceQueryResponseSchema.parse({
+			traces: [],
+			delta: {
+				limit: plan.limit,
+				hasMore: false
+			},
+			deltaCursor: coreStorage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead))
+		});
+	}
+	if (plan.paginationMode === "page") {
+		const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: deltaHead ? remaining() : timeoutMs }, compileClickHouseTraceQuery(plan, deltaHead));
+		const total = Number(rows.at(-1)?.total ?? 0);
+		const traces = rows.filter((row) => Number(row.__metadata) === 0).map((row) => ({
+			traceId: String(row.traceId),
+			rootSpanId: String(row.rootSpanId),
+			name: row.name,
+			entityId: row.entityId ?? null,
+			parentSpanId: row.parentSpanId ?? null,
+			createdAt: asIsoTimestamp(row.startedAt),
+			metadata: parseJson(row.metadata) ?? null,
+			inputPreview: coreStorage.buildInputPreview(row.input) ?? null,
+			threadId: row.threadId == null ? null : String(row.threadId),
+			resourceId: row.resourceId == null ? null : String(row.resourceId),
+			startedAt: asIsoTimestamp(row.startedAt),
+			endedAt: asIsoTimestamp(row.endedAt),
+			entityName: row.entityName == null ? null : String(row.entityName),
+			entityType: row.entityType == null ? null : String(row.entityType),
+			environment: row.environment == null ? null : String(row.environment),
+			status: row.status
+		}));
+		return coreStorage.traceQueryResponseSchema.parse({
+			traces,
+			...deltaHead ? { deltaCursor: coreStorage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead)) } : {},
+			pagination: {
+				total,
+				page: plan.page,
+				perPage: plan.perPage,
+				hasMore: (plan.page + 1) * plan.perPage < total
+			}
+		});
+	}
+	const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: deltaHead ? remaining() : timeoutMs }, compileClickHouseTraceQuery(plan, deltaHead));
 	const visibleRows = rows.slice(0, plan.limit);
 	if (plan.result === "groups") {
 		const groups = visibleRows.map((row) => ({ threadId: String(row.threadId) }));
@@ -6959,6 +7217,23 @@ async function queryTraces(client, plan, timeoutMs) {
 		status: row.status
 	}));
 	const last = traces.at(-1);
+	if (plan.paginationMode === "delta") {
+		const lastRow = visibleRows.at(-1);
+		let watermark = lastRow ? {
+			cursorId: String(lastRow.__delta_cursor),
+			traceId: String(lastRow.traceId)
+		} : deltaHead;
+		const previous = parseDeltaWatermark(coreStorage.getTraceQueryDeltaWatermark(plan, "clickhouse"));
+		if (BigInt(previous.cursorId) > BigInt(watermark.cursorId) || previous.cursorId === watermark.cursorId && previous.traceId > watermark.traceId) watermark = previous;
+		return coreStorage.traceQueryResponseSchema.parse({
+			traces,
+			delta: {
+				limit: plan.limit,
+				hasMore: rows.length > plan.limit
+			},
+			deltaCursor: coreStorage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(watermark))
+		});
+	}
 	return coreStorage.traceQueryResponseSchema.parse({
 		traces,
 		page: { next: rows.length > plan.limit && last ? coreStorage.encodeTraceQueryCursor(plan, {
@@ -7432,6 +7707,7 @@ async function batchDeleteTraces(client, args, replication) {
 		TABLE_METRIC_EVENTS,
 		TABLE_LOG_EVENTS,
 		TABLE_SCORE_EVENTS,
+		TABLE_SCORE_EVENTS_CURRENT,
 		TABLE_FEEDBACK_EVENTS
 	];
 	await Promise.all([...tracingTables.map((table) => client.command({
@@ -8111,10 +8387,13 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			else if (existingStrategy) this.#deltaCursorStrategy = existingStrategy;
 			else this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
 			await reconcileDiscoveryTables(this.#client, this.#replication);
-			const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
-			for (const ddl of coreDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			const coreTableDdl = this.#deltaCursorStrategy === null ? BASE_TABLE_DDL : buildAllTableDDL();
+			for (const ddl of coreTableDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
 			const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
 			for (const migration of pendingMigrations) await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
+			const coreMvDdl = this.#deltaCursorStrategy === null ? BASE_MV_DDL : buildAllMvDDL(this.#deltaCursorStrategy);
+			for (const ddl of coreMvDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			await backfillCurrentScores(this.#client);
 			if (this.#retention) await this.applyRetention();
 			if (this.#deltaCursorStrategy === "serial") for (const counterName of DELTA_CURSOR_COUNTER_NAMES) await this.#client.query({
 				query: `SELECT generateSerialID({counterName:String}) AS cursorId`,
@@ -8313,9 +8592,9 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	}
 	async queryTraces(plan) {
 		try {
-			return await queryTraces(this.#client, plan, this.#traceQueryTimeoutMs);
+			return await queryTraces(this.#client, plan, this.#traceQueryTimeoutMs, this.#deltaCursorStrategy);
 		} catch (error) {
-			if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError || error instanceof coreStorage.TraceQueryResourceLimitError) throw error;
+			if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError || error instanceof coreStorage.TraceQueryCursorError || error instanceof coreStorage.TraceQueryResourceLimitError) throw error;
 			throw new MastraError({
 				id: createStorageErrorId("CLICKHOUSE", "QUERY_TRACES", "FAILED"),
 				domain: ErrorDomain.STORAGE,