@mastra/clickhouse 1.20.0-alpha.2 → 1.20.0-alpha.3

package/dist/index.js

@@ -2546,6 +2546,8 @@ const TABLE_TRACE_BRANCHES_DELTA = "mastra_trace_branches_delta";
 const TABLE_METRIC_EVENTS = "mastra_metric_events";
 const TABLE_LOG_EVENTS = "mastra_log_events";
 const TABLE_SCORE_EVENTS = "mastra_score_events";
+const TABLE_SCORE_EVENTS_CURRENT = "mastra_score_events_current";
+const TABLE_SCORE_EVENTS_CURRENT_BACKFILL = "mastra_score_events_current_backfill";
 const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
 const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
 const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
@@ -2561,6 +2563,7 @@ const MV_TRACE_BRANCHES_DELTA = "mastra_mv_trace_branches_delta";
 const MV_METRIC_EVENTS_DELTA = "mastra_mv_metric_events_delta";
 const MV_LOG_EVENTS_DELTA = "mastra_mv_log_events_delta";
 const MV_SCORE_EVENTS_DELTA = "mastra_mv_score_events_delta";
+const MV_SCORE_EVENTS_CURRENT = "mastra_mv_score_events_current";
 const MV_FEEDBACK_EVENTS_DELTA = "mastra_mv_feedback_events_delta";
 const MV_DISCOVERY_VALUES = "mastra_mv_discovery_values";
 const MV_DISCOVERY_PAIRS = "mastra_mv_discovery_pairs";
@@ -3078,19 +3081,53 @@ FROM (
 )
 `;
 }
-const SCORE_EVENTS_DDL = `
-CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
-  -- Timestamp
+const SCORE_EVENT_COLUMN_NAMES = [
+	"timestamp",
+	"scoreId",
+	"writeVersion",
+	"traceId",
+	"spanId",
+	"experimentId",
+	"scoreTraceId",
+	"entityType",
+	"entityId",
+	"entityName",
+	"entityVersionId",
+	"parentEntityVersionId",
+	"parentEntityType",
+	"parentEntityId",
+	"parentEntityName",
+	"rootEntityVersionId",
+	"rootEntityType",
+	"rootEntityId",
+	"rootEntityName",
+	"userId",
+	"organizationId",
+	"resourceId",
+	"runId",
+	"sessionId",
+	"threadId",
+	"requestId",
+	"environment",
+	"executionSource",
+	"serviceName",
+	"scorerId",
+	"scorerVersion",
+	"scoreSource",
+	"score",
+	"reason",
+	"tags",
+	"metadata",
+	"scope"
+];
+const SCORE_EVENT_COLUMNS_DDL = `
   timestamp          DateTime64(3, 'UTC'),
-
-  -- IDs
   scoreId            String,
+  writeVersion       UInt64 DEFAULT 0,
   traceId            Nullable(String),
   spanId             Nullable(String),
   experimentId       Nullable(String),
   scoreTraceId       Nullable(String),
-
-  -- Entity hierarchy
   entityType         LowCardinality(Nullable(String)),
   entityId           Nullable(String),
   entityName         Nullable(String),
@@ -3103,8 +3140,6 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
   rootEntityType     LowCardinality(Nullable(String)),
   rootEntityId       Nullable(String),
   rootEntityName     Nullable(String),
-
-  -- Context
   userId             Nullable(String),
   organizationId     Nullable(String),
   resourceId         Nullable(String),
@@ -3115,30 +3150,48 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
   environment        LowCardinality(Nullable(String)),
   executionSource    LowCardinality(Nullable(String)),
   serviceName        LowCardinality(Nullable(String)),
-
-  -- Scorer identity
   scorerId           LowCardinality(String),
   scorerVersion      LowCardinality(Nullable(String)),
   scoreSource        LowCardinality(Nullable(String)),
-
-  -- Score value
   score              Float64,
-
-  -- Information-only
   reason             Nullable(String),
-
-  -- Query-relevant flexible fields
   tags               Array(LowCardinality(String)) DEFAULT [],
-
-  -- Information-only JSON payloads
   metadata           Nullable(String),
   scope              Nullable(String)
+`;
+const SCORE_EVENTS_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
+${SCORE_EVENT_COLUMNS_DDL},
+  INDEX idx_scoreId scoreId TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = ReplacingMergeTree
 PARTITION BY toDate(timestamp)
 ORDER BY (traceId, timestamp, scoreId)
 SETTINGS allow_nullable_key = 1
 `;
+const SCORE_EVENTS_CURRENT_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_CURRENT} (
+${SCORE_EVENT_COLUMNS_DDL}
+)
+ENGINE = ReplacingMergeTree(writeVersion)
+PARTITION BY cityHash64(scoreId) % 64
+ORDER BY scoreId
+`;
+const SCORE_EVENTS_CURRENT_BACKFILL_DDL = `
+CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} (
+  marker String,
+  completedAt DateTime64(3, 'UTC')
+)
+ENGINE = ReplacingMergeTree(completedAt)
+ORDER BY marker
+`;
+const SCORE_EVENTS_CURRENT_MV_DDL = `
+CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_CURRENT}
+TO ${TABLE_SCORE_EVENTS_CURRENT}
+AS
+SELECT ${SCORE_EVENT_COLUMN_NAMES.join(", ")}
+FROM ${TABLE_SCORE_EVENTS}
+`;
 function buildScoreEventsDeltaDDL() {
 	return `
 CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
@@ -3412,6 +3465,8 @@ const BASE_TABLE_DDL = [
 	METRIC_EVENTS_DDL,
 	LOG_EVENTS_DDL,
 	SCORE_EVENTS_DDL,
+	SCORE_EVENTS_CURRENT_DDL,
+	SCORE_EVENTS_CURRENT_BACKFILL_DDL,
 	FEEDBACK_EVENTS_DDL,
 	DELETION_REQUESTS_DDL,
 	DISCOVERY_VALUES_DDL,
@@ -3430,7 +3485,11 @@ function buildDeltaTableDDL() {
 function buildAllTableDDL() {
 	return [...BASE_TABLE_DDL, ...buildDeltaTableDDL()];
 }
-const BASE_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
+const BASE_MV_DDL = [
+	TRACE_ROOTS_MV_DDL,
+	TRACE_BRANCHES_MV_DDL,
+	SCORE_EVENTS_CURRENT_MV_DDL
+];
 function buildDeltaMvDDL(strategy) {
 	return [
 		buildTraceRootsDeltaMvDDL(strategy),
@@ -3452,11 +3511,11 @@ const addColumn = (table, name, type) => ({
 	name,
 	sql: `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${name} ${type}`
 });
-const addBloomIndex = (table, name, column) => ({
+const addBloomIndex = (table, name, column, granularity = 2) => ({
 	kind: "index",
 	table,
 	name,
-	sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY 2`
+	sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY ${granularity}`
 });
 const ALL_MIGRATIONS = [
 	addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
@@ -3471,9 +3530,11 @@ const ALL_MIGRATIONS = [
 	addColumn(TABLE_LOG_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_LOG_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_LOG_EVENTS, "rootEntityVersionId", "Nullable(String)"),
+	addColumn(TABLE_SCORE_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_SCORE_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
+	addBloomIndex(TABLE_SCORE_EVENTS, "idx_scoreId", "scoreId", 1),
 	addColumn(TABLE_FEEDBACK_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "reviewStatus", "LowCardinality(String) DEFAULT 'needs-review'"),
@@ -3499,6 +3560,8 @@ const ALL_TABLE_NAMES = [
 	TABLE_METRIC_EVENTS,
 	TABLE_LOG_EVENTS,
 	TABLE_SCORE_EVENTS,
+	TABLE_SCORE_EVENTS_CURRENT,
+	TABLE_SCORE_EVENTS_CURRENT_BACKFILL,
 	TABLE_FEEDBACK_EVENTS,
 	TABLE_DELETION_REQUESTS,
 	TABLE_METRIC_EVENTS_DELTA,
@@ -3516,6 +3579,7 @@ const SIGNAL_TTL_COLUMNS = {
 	[TABLE_METRIC_EVENTS]: "timestamp",
 	[TABLE_LOG_EVENTS]: "timestamp",
 	[TABLE_SCORE_EVENTS]: "timestamp",
+	[TABLE_SCORE_EVENTS_CURRENT]: "timestamp",
 	[TABLE_FEEDBACK_EVENTS]: "timestamp"
 };
 /** Maps each signal key to the table(s) it controls. */
@@ -3527,7 +3591,7 @@ const SIGNAL_TO_TABLES = {
 	],
 	logs: [TABLE_LOG_EVENTS],
 	metrics: [TABLE_METRIC_EVENTS],
-	scores: [TABLE_SCORE_EVENTS],
+	scores: [TABLE_SCORE_EVENTS, TABLE_SCORE_EVENTS_CURRENT],
 	feedback: [TABLE_FEEDBACK_EVENTS]
 };
 const RETENTION_MANAGED_TABLES = [...Object.keys(SIGNAL_TTL_COLUMNS), TABLE_DELETION_REQUESTS];
@@ -5947,6 +6011,36 @@ async function migrateSignalTables(client, logger) {
 	}
 }
 //#endregion
+//#region src/storage/domains/observability/v-next/score-current.ts
+const SCORE_CURRENT_BACKFILL_MARKER = "v1";
+async function backfillCurrentScores(client) {
+	const markerRows = await (await client.query({
+		query: `SELECT count() AS count
+            FROM ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} FINAL
+            WHERE marker = {marker:String}`,
+		query_params: { marker: SCORE_CURRENT_BACKFILL_MARKER },
+		format: "JSONEachRow"
+	})).json();
+	if (Number(markerRows[0]?.count ?? 0) > 0) return;
+	const columns = SCORE_EVENT_COLUMN_NAMES.join(", ");
+	await client.command({
+		query: `INSERT INTO ${TABLE_SCORE_EVENTS_CURRENT} (${columns})
+            SELECT ${columns}
+            FROM (
+              SELECT *, cityHash64(tuple(*)) AS _currentScoreFingerprint
+              FROM ${TABLE_SCORE_EVENTS} FINAL
+              ORDER BY scoreId, writeVersion DESC, timestamp DESC, _currentScoreFingerprint DESC
+              LIMIT 1 BY scoreId
+            )`,
+		clickhouse_settings: { max_bytes_ratio_before_external_sort: .5 }
+	});
+	await client.command({
+		query: `INSERT INTO ${TABLE_SCORE_EVENTS_CURRENT_BACKFILL} (marker, completedAt)
+            VALUES ({marker:String}, now64(3, 'UTC'))`,
+		query_params: { marker: SCORE_CURRENT_BACKFILL_MARKER }
+	});
+}
+//#endregion
 //#region src/storage/domains/observability/v-next/scores.ts
 const SCORE_TYPED_COLUMNS = /* @__PURE__ */ new Set([
 	"timestamp",
@@ -6023,6 +6117,13 @@ function mergeFilters(...parts) {
 function toWhereClause(filter) {
 	return filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
 }
+function currentScoresRelation(sourcePredicate) {
+	return `(
+    SELECT *
+    FROM ${TABLE_SCORE_EVENTS_CURRENT} FINAL
+    ${sourcePredicate ? `WHERE ${sourcePredicate}` : ""}
+  )`;
+}
 function buildScoreIdentityFilter(args) {
 	const conditions = ["scorerId = {olapScorerId:String}"];
 	const params = { olapScorerId: args.scorerId };
@@ -6058,6 +6159,24 @@ async function queryJson(client, query, params) {
 		clickhouse_settings: CH_SETTINGS
 	})).json();
 }
+async function scoreRowsWithWriteVersions(client, scores) {
+	const scoreIds = [...new Set(scores.flatMap((score) => score.scoreId ? [score.scoreId] : []))];
+	const existingVersions = [];
+	const lookupBatchSize = 1e3;
+	for (let offset = 0; offset < scoreIds.length; offset += lookupBatchSize) existingVersions.push(...await queryJson(client, `SELECT scoreId, toString(max(writeVersion)) AS writeVersion
+         FROM ${TABLE_SCORE_EVENTS}
+         WHERE scoreId IN ({scoreIds:Array(String)})
+         GROUP BY scoreId`, { scoreIds: scoreIds.slice(offset, offset + lookupBatchSize) }));
+	const versions = new Map(existingVersions.map((row) => [row.scoreId, BigInt(row.writeVersion)]));
+	return scores.map((score) => {
+		const writeVersion = score.scoreId ? (versions.get(score.scoreId) ?? 0n) + 1n : 1n;
+		if (score.scoreId) versions.set(score.scoreId, writeVersion);
+		return {
+			...scoreRecordToRow(score),
+			writeVersion: writeVersion.toString()
+		};
+	});
+}
 async function createScore(client, args) {
 	await batchCreateScores(client, { scores: [args.score] });
 }
@@ -6065,7 +6184,7 @@ async function batchCreateScores(client, args) {
 	if (args.scores.length === 0) return;
 	await client.insert({
 		table: TABLE_SCORE_EVENTS,
-		values: args.scores.map(scoreRecordToRow),
+		values: await scoreRowsWithWriteVersions(client, args.scores),
 		format: "JSONEachRow",
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
@@ -6108,10 +6227,11 @@ async function deleteScores(client, args, replication) {
 		conditions.push("resourceId = {delResourceId:String}");
 		params.delResourceId = args.resourceId;
 	}
-	await client.command({
-		query: `DELETE FROM ${TABLE_SCORE_EVENTS} WHERE ${conditions.join(" AND ")}`,
+	const clickhouse_settings = { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" };
+	for (const table of [TABLE_SCORE_EVENTS_CURRENT, TABLE_SCORE_EVENTS]) await client.command({
+		query: `DELETE FROM ${table} WHERE ${conditions.join(" AND ")}`,
 		query_params: params,
-		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+		clickhouse_settings
 	});
 }
 async function listScores(client, args, strategy) {
@@ -6145,8 +6265,9 @@ async function listScores(client, args, strategy) {
 		};
 	}
 	const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor$2(client, whereClause, filter.params) : void 0;
-	const countResult = await queryJson(client, `SELECT count() AS total FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause}`, filter.params);
-	const rows = await queryJson(client, `SELECT * FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause} ORDER BY ${orderBy} LIMIT {limit:UInt32} OFFSET {offset:UInt32}`, {
+	const scoresRelation = currentScoresRelation();
+	const countResult = await queryJson(client, `SELECT count() AS total FROM ${scoresRelation} AS s ${whereClause}`, filter.params);
+	const rows = await queryJson(client, `SELECT * FROM ${scoresRelation} AS s ${whereClause} ORDER BY ${orderBy} LIMIT {limit:UInt32} OFFSET {offset:UInt32}`, {
 		...filter.params,
 		limit: pagination.limit,
 		offset: pagination.offset
@@ -6205,14 +6326,15 @@ function buildScoresCursor(row) {
 	return row.cursorId ?? "0";
 }
 async function getScoreById(client, scoreId) {
-	const rows = await queryJson(client, `SELECT * FROM ${TABLE_SCORE_EVENTS} WHERE scoreId = {scoreId:String} LIMIT 1`, { scoreId });
+	const rows = await queryJson(client, `SELECT * FROM ${currentScoresRelation("scoreId = {scoreId:String}")} LIMIT 1`, { scoreId });
 	return rows[0] ? rowToScoreRecord(rows[0]) : null;
 }
 async function getScoreAggregate(client, args) {
 	const aggSql = getAggregationSql(args.aggregation);
 	const identity = buildScoreIdentityFilter(args);
 	const combined = mergeFilters(identity, buildScoresFilterConditions(args.filters));
-	const result = await queryJson(client, `SELECT ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${toWhereClause(combined)}`, combined.params);
+	const whereClause = toWhereClause(combined);
+	const result = await queryJson(client, `SELECT ${aggSql} AS value FROM ${currentScoresRelation()} ${whereClause}`, combined.params);
 	const value = result[0]?.value == null ? null : Number(result[0]?.value);
 	if (args.comparePeriod && args.filters?.timestamp) {
 		const ts = args.filters.timestamp;
@@ -6246,7 +6368,8 @@ async function getScoreAggregate(client, args) {
 					endExclusive: ts.endExclusive
 				}
 			}));
-			const prevResult = await queryJson(client, `SELECT ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${toWhereClause(prevCombined)}`, prevCombined.params);
+			const prevWhereClause = toWhereClause(prevCombined);
+			const prevResult = await queryJson(client, `SELECT ${aggSql} AS value FROM ${currentScoresRelation()} ${prevWhereClause}`, prevCombined.params);
 			const previousValue = prevResult[0]?.value == null ? null : Number(prevResult[0]?.value);
 			let changePercent = null;
 			if (previousValue !== null && previousValue !== 0 && value !== null) changePercent = (value - previousValue) / Math.abs(previousValue) * 100;
@@ -6264,7 +6387,7 @@ async function getScoreBreakdown(client, args) {
 	const combined = mergeFilters(buildScoreIdentityFilter(args), buildScoresFilterConditions(args.filters));
 	const whereClause = toWhereClause(combined);
 	const resolved = resolveScoreGroupBy(args.groupBy);
-	return { groups: (await queryJson(client, `SELECT ${resolved.map((e) => e.selectSql).join(", ")}, ${aggSql} AS value FROM ${TABLE_SCORE_EVENTS} ${whereClause} GROUP BY ${resolved.map((e) => e.groupSql).join(", ")} ORDER BY value DESC`, combined.params)).map((row) => ({
+	return { groups: (await queryJson(client, `SELECT ${resolved.map((e) => e.selectSql).join(", ")}, ${aggSql} AS value FROM ${currentScoresRelation()} ${whereClause} GROUP BY ${resolved.map((e) => e.groupSql).join(", ")} ORDER BY value DESC`, combined.params)).map((row) => ({
 		dimensions: Object.fromEntries(resolved.map((entry, index) => {
 			const v = row[`group_by_${index}`];
 			return [entry.key, v == null ? null : String(v)];
@@ -6283,7 +6406,7 @@ async function getScoreTimeSeries(client, args) {
       SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
              ${resolved.map((e) => e.selectSql).join(", ")},
              ${aggSql} AS value
-      FROM ${TABLE_SCORE_EVENTS} ${whereClause}
+      FROM ${currentScoresRelation()} ${whereClause}
       GROUP BY bucket, ${resolved.map((e) => e.groupSql).join(", ")}
       ORDER BY bucket
     `, combined.params);
@@ -6305,7 +6428,7 @@ async function getScoreTimeSeries(client, args) {
 	const rows = await queryJson(client, `
     SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
            ${aggSql} AS value
-    FROM ${TABLE_SCORE_EVENTS} ${whereClause}
+    FROM ${currentScoresRelation()} ${whereClause}
     GROUP BY bucket
     ORDER BY bucket
   `, combined.params);
@@ -6328,7 +6451,7 @@ async function getScorePercentiles(client, args) {
 		const rows = await queryJson(client, `
       SELECT toStartOfInterval(timestamp, ${intervalSql}) AS bucket,
              quantile(${p})(score) AS pvalue
-      FROM ${TABLE_SCORE_EVENTS}
+      FROM ${currentScoresRelation()}
       ${whereClause}
       GROUP BY bucket
       ORDER BY bucket
@@ -6697,11 +6820,9 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters)
       entityVersionId,
       parentEntityVersionId,
       rootEntityVersionId
-    FROM ${TABLE_SCORE_EVENTS}
-    WHERE isNotNull(traceId)
-      AND traceId IN (SELECT traceId FROM root_scope)
-    ORDER BY scoreId, timestamp DESC
-    LIMIT 1 BY scoreId
+    FROM ${currentScoresRelation()} AS current
+    WHERE isNotNull(current.traceId)
+      AND current.traceId IN (SELECT traceId FROM root_scope)
   )`);
 	if (relationCollections.has("feedback")) ctes.push(`current_feedback AS (
     SELECT
@@ -6750,6 +6871,46 @@ LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
 	};
 	const orderField = resolveOrderField(plan.orderBy.field);
 	const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
+	if (plan.paginationMode === "page") return {
+		query: `${candidates},
+page_rows AS (
+  SELECT *, row_number() OVER (ORDER BY ${orderField} ${direction}, traceId ASC) AS __row_position
+  FROM candidates
+  ORDER BY ${orderField} ${direction}, traceId ASC
+  LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}
+),
+page_total AS (
+  SELECT count() AS total
+  FROM candidates
+)
+SELECT page_rows.*, page_total.total, 0 AS __metadata
+FROM page_rows
+CROSS JOIN page_total
+UNION ALL
+SELECT
+  '' AS traceId,
+  '' AS rootSpanId,
+  '' AS name,
+  CAST(NULL, 'Nullable(String)') AS entityId,
+  CAST(NULL, 'Nullable(String)') AS parentSpanId,
+  CAST(NULL, 'Nullable(String)') AS metadata,
+  CAST(NULL, 'Nullable(String)') AS input,
+  CAST(NULL, 'Nullable(String)') AS threadId,
+  CAST(NULL, 'Nullable(String)') AS resourceId,
+  toDateTime64(0, 3, 'UTC') AS startedAt,
+  toDateTime64(0, 3, 'UTC') AS endedAt,
+  CAST(NULL, 'Nullable(String)') AS entityName,
+  CAST(NULL, 'Nullable(String)') AS entityType,
+  CAST(NULL, 'Nullable(String)') AS environment,
+  '' AS status,
+  0 AS __row_position,
+  page_total.total AS total,
+  1 AS __metadata
+FROM page_total
+ORDER BY __metadata ASC, __row_position ASC`,
+		query_params: parameters.params,
+		sharedSnapshot: true
+	};
 	let pageCondition = "";
 	if (plan.cursor) {
 		const comparison = plan.orderBy.direction === "asc" ? ">" : "<";
@@ -6896,7 +7057,8 @@ async function runWithClickHouseTraceQueryTimeout(client, limits, compiled, quer
 			clickhouse_settings: {
 				...CH_SETTINGS,
 				max_execution_time: resolvedTimeoutMs / 1e3,
-				...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) }
+				...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) },
+				...compiled.sharedSnapshot ? { enable_shared_storage_snapshot_in_query: 1 } : {}
 			}
 		})).json();
 	} catch (error) {
@@ -6927,6 +7089,37 @@ async function getTraceQueryValues(client, plan, limits) {
 	});
 }
 async function queryTraces(client, plan, timeoutMs) {
+	if (plan.paginationMode === "page") {
+		const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs }, compileClickHouseTraceQuery(plan));
+		const total = Number(rows.at(-1)?.total ?? 0);
+		const traces = rows.filter((row) => Number(row.__metadata) === 0).map((row) => ({
+			traceId: String(row.traceId),
+			rootSpanId: String(row.rootSpanId),
+			name: row.name,
+			entityId: row.entityId ?? null,
+			parentSpanId: row.parentSpanId ?? null,
+			createdAt: asIsoTimestamp(row.startedAt),
+			metadata: parseJson(row.metadata) ?? null,
+			inputPreview: coreStorage.buildInputPreview(row.input) ?? null,
+			threadId: row.threadId == null ? null : String(row.threadId),
+			resourceId: row.resourceId == null ? null : String(row.resourceId),
+			startedAt: asIsoTimestamp(row.startedAt),
+			endedAt: asIsoTimestamp(row.endedAt),
+			entityName: row.entityName == null ? null : String(row.entityName),
+			entityType: row.entityType == null ? null : String(row.entityType),
+			environment: row.environment == null ? null : String(row.environment),
+			status: row.status
+		}));
+		return coreStorage.traceQueryResponseSchema.parse({
+			traces,
+			pagination: {
+				total,
+				page: plan.page,
+				perPage: plan.perPage,
+				hasMore: (plan.page + 1) * plan.perPage < total
+			}
+		});
+	}
 	const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs }, compileClickHouseTraceQuery(plan));
 	const visibleRows = rows.slice(0, plan.limit);
 	if (plan.result === "groups") {
@@ -7432,6 +7625,7 @@ async function batchDeleteTraces(client, args, replication) {
 		TABLE_METRIC_EVENTS,
 		TABLE_LOG_EVENTS,
 		TABLE_SCORE_EVENTS,
+		TABLE_SCORE_EVENTS_CURRENT,
 		TABLE_FEEDBACK_EVENTS
 	];
 	await Promise.all([...tracingTables.map((table) => client.command({
@@ -8111,10 +8305,13 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			else if (existingStrategy) this.#deltaCursorStrategy = existingStrategy;
 			else this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
 			await reconcileDiscoveryTables(this.#client, this.#replication);
-			const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
-			for (const ddl of coreDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			const coreTableDdl = this.#deltaCursorStrategy === null ? BASE_TABLE_DDL : buildAllTableDDL();
+			for (const ddl of coreTableDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
 			const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
 			for (const migration of pendingMigrations) await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
+			const coreMvDdl = this.#deltaCursorStrategy === null ? BASE_MV_DDL : buildAllMvDDL(this.#deltaCursorStrategy);
+			for (const ddl of coreMvDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			await backfillCurrentScores(this.#client);
 			if (this.#retention) await this.applyRetention();
 			if (this.#deltaCursorStrategy === "serial") for (const counterName of DELTA_CURSOR_COUNTER_NAMES) await this.#client.query({
 				query: `SELECT generateSerialID({counterName:String}) AS cursorId`,