@mastra/clickhouse 1.20.0 → 1.20.1-alpha.0

package/dist/index.js

@@ -3200,7 +3200,8 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
   ingestedAt         ${DELTA_INGESTED_AT_TYPE},
   traceId            Nullable(String),
   timestamp          DateTime64(3, 'UTC'),
-  scoreId            String
+  scoreId            String,
+  INDEX idx_scoreId scoreId TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = MergeTree
 PARTITION BY toDate(ingestedAt)
@@ -3209,12 +3210,8 @@ TTL ingestedAt + toIntervalDay(2)
 SETTINGS allow_nullable_key = 1
 `;
 }
-function buildScoreEventsDeltaMvDDL(strategy) {
-	return `
-CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
-TO ${TABLE_SCORE_EVENTS_DELTA}
-AS
-SELECT
+function buildScoreEventsDeltaMvQuery(strategy) {
+	return `SELECT
   ${buildDeltaCursorExpr(strategy, "mastra_score_events_delta_cursor", "scoreId")} AS cursorId,
   ingestedAt,
   traceId,
@@ -3227,7 +3224,19 @@ FROM (
     timestamp,
     scoreId
   FROM ${TABLE_SCORE_EVENTS}
-)
+  WHERE scoreId NOT IN (
+    SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA}
+    WHERE scoreId IN (SELECT scoreId FROM ${TABLE_SCORE_EVENTS})
+  )
+  LIMIT 1 BY scoreId
+)`;
+}
+function buildScoreEventsDeltaMvDDL(strategy) {
+	return `
+CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
+TO ${TABLE_SCORE_EVENTS_DELTA}
+AS
+${buildScoreEventsDeltaMvQuery(strategy)}
 `;
 }
 const FEEDBACK_EVENTS_DDL = `
@@ -3536,6 +3545,7 @@ const ALL_MIGRATIONS = [
 	addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
 	addBloomIndex(TABLE_SCORE_EVENTS, "idx_scoreId", "scoreId", 1),
+	addBloomIndex(TABLE_SCORE_EVENTS_DELTA, "idx_scoreId", "scoreId", 1),
 	addColumn(TABLE_FEEDBACK_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "reviewStatus", "LowCardinality(String) DEFAULT 'needs-review'"),
@@ -6285,6 +6295,17 @@ async function listScores(client, args, strategy) {
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
 }
+/**
+* Delta reads join the delta stream to the current-state table by scoreId, so
+* a poll or replay always returns the latest write of a score (a rewrite may
+* change traceId or timestamp, which the delta row does not track).
+*
+* A score's first delta row is its only cursor: rows after `afterCursor` are
+* dropped when the same scoreId already has a row at or before it, and
+* `LIMIT 1 BY` keeps the lowest cursor within the page. Duplicate delta rows
+* (a concurrent-insert race past the MV check) therefore never surface, in
+* this poll or a later one.
+*/
 async function queryScoresAfterCursor(client, whereClause, params, limit, cursorId) {
 	return await queryJson(client, `
       SELECT
@@ -6294,12 +6315,16 @@ async function queryScoresAfterCursor(client, whereClause, params, limit, cursor
         s.scoreId AS scoreId,
         toString(d.cursorId) AS cursorId
       FROM ${TABLE_SCORE_EVENTS_DELTA} d
-      INNER JOIN ${TABLE_SCORE_EVENTS} s
-        ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
-       AND s.timestamp = d.timestamp
-       AND s.scoreId = d.scoreId
-      ${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
+      INNER JOIN ${TABLE_SCORE_EVENTS_CURRENT} s FINAL
+        ON s.scoreId = d.scoreId
+      ${whereClause ? `${whereClause} AND` : "WHERE"} d.cursorId > {afterCursor:UInt64}
+        AND d.scoreId NOT IN (
+          SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA}
+          WHERE cursorId <= {afterCursor:UInt64}
+            AND scoreId IN (SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA} WHERE cursorId > {afterCursor:UInt64})
+        )
       ORDER BY d.cursorId ASC
+      LIMIT 1 BY s.scoreId
       LIMIT {fetchLimit:UInt32}
     `, {
 		...params,
@@ -6311,10 +6336,8 @@ async function getDeltaCursor$2(client, whereClause, params) {
 	const cursorId = (await queryJson(client, `
       SELECT toString(max(d.cursorId)) AS cursorId
       FROM mastra_score_events_delta d
-      INNER JOIN mastra_score_events s
-        ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
-       AND s.timestamp = d.timestamp
-       AND s.scoreId = d.scoreId
+      INNER JOIN mastra_score_events_current s FINAL
+        ON s.scoreId = d.scoreId
       ${whereClause}
     `, params))[0]?.cursorId ?? null;
 	if (cursorId) return cursorId;
@@ -8284,6 +8307,37 @@ async function queryNamesByTable(client, query, tables) {
 	}
 	return out;
 }
+/**
+* Upgrades a score delta MV created before per-scoreId deduplication with
+* `ALTER TABLE ... MODIFY QUERY`. The view is changed in place, so there is no
+* window without a view in which score writes would miss their delta row (a
+* drop-and-recreate would leave such writes out of delta polling for good).
+*
+* With a cluster configured the definition is read from every replica via
+* `clusterAllReplicas`, since each host stores its own copy of the view; a
+* single legacy copy is enough to alter `ON CLUSTER`.
+*
+* Introspection failures are logged and skipped, like the other schema checks
+* in init(): the existing view keeps working and the next boot retries.
+*/
+async function reconcileScoreDeltaMv(client, replication, strategy, logger) {
+	const cluster = replication?.cluster?.trim();
+	let createQueries;
+	try {
+		createQueries = (await (await client.query({
+			query: cluster ? `SELECT create_table_query FROM clusterAllReplicas({cluster:String}, system.tables) WHERE database = currentDatabase() AND name = {name:String}` : `SELECT create_table_query FROM system.tables WHERE database = currentDatabase() AND name = {name:String}`,
+			query_params: cluster ? {
+				cluster,
+				name: MV_SCORE_EVENTS_DELTA
+			} : { name: MV_SCORE_EVENTS_DELTA },
+			format: "JSONEachRow"
+		})).json()).map((row) => row.create_table_query ?? "");
+	} catch (error) {
+		logger?.warn?.(`Could not verify the ${MV_SCORE_EVENTS_DELTA} definition; leaving the existing view in place: ${error instanceof Error ? error.message : String(error)}`);
+		return;
+	}
+	if (createQueries.some((createQuery) => createQuery.length > 0 && !/NOT IN/i.test(createQuery))) await client.command({ query: addOnClusterToDDL(`ALTER TABLE ${MV_SCORE_EVENTS_DELTA} MODIFY QUERY ${buildScoreEventsDeltaMvQuery(strategy)}`, replication) });
+}
 async function detectDeltaCursorStrategy(client, override, existingStrategy) {
 	if (override) return override;
 	if (existingStrategy && existingStrategy !== "mixed") return existingStrategy;
@@ -8393,6 +8447,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			for (const migration of pendingMigrations) await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
 			const coreMvDdl = this.#deltaCursorStrategy === null ? BASE_MV_DDL : buildAllMvDDL(this.#deltaCursorStrategy);
 			for (const ddl of coreMvDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			if (this.#deltaCursorStrategy !== null) await reconcileScoreDeltaMv(this.#client, this.#replication, this.#deltaCursorStrategy, this.logger);
 			await backfillCurrentScores(this.#client);
 			if (this.#retention) await this.applyRetention();
 			if (this.#deltaCursorStrategy === "serial") for (const counterName of DELTA_CURSOR_COUNTER_NAMES) await this.#client.query({