@mastra/clickhouse 1.20.0 → 1.20.1-alpha.0

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-clickhouse
 description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
 metadata:
   package: "@mastra/clickhouse"
-  version: "1.20.0"
+  version: "1.20.1-alpha.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.20.0",
+  "version": "1.20.1-alpha.0",
   "package": "@mastra/clickhouse",
   "exports": {},
   "modules": {}

package/dist/index.cjs

@@ -3223,7 +3223,8 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
   ingestedAt         ${DELTA_INGESTED_AT_TYPE},
   traceId            Nullable(String),
   timestamp          DateTime64(3, 'UTC'),
-  scoreId            String
+  scoreId            String,
+  INDEX idx_scoreId scoreId TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = MergeTree
 PARTITION BY toDate(ingestedAt)
@@ -3232,12 +3233,8 @@ TTL ingestedAt + toIntervalDay(2)
 SETTINGS allow_nullable_key = 1
 `;
 }
-function buildScoreEventsDeltaMvDDL(strategy) {
-	return `
-CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
-TO ${TABLE_SCORE_EVENTS_DELTA}
-AS
-SELECT
+function buildScoreEventsDeltaMvQuery(strategy) {
+	return `SELECT
   ${buildDeltaCursorExpr(strategy, "mastra_score_events_delta_cursor", "scoreId")} AS cursorId,
   ingestedAt,
   traceId,
@@ -3250,7 +3247,19 @@ FROM (
     timestamp,
     scoreId
   FROM ${TABLE_SCORE_EVENTS}
-)
+  WHERE scoreId NOT IN (
+    SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA}
+    WHERE scoreId IN (SELECT scoreId FROM ${TABLE_SCORE_EVENTS})
+  )
+  LIMIT 1 BY scoreId
+)`;
+}
+function buildScoreEventsDeltaMvDDL(strategy) {
+	return `
+CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
+TO ${TABLE_SCORE_EVENTS_DELTA}
+AS
+${buildScoreEventsDeltaMvQuery(strategy)}
 `;
 }
 const FEEDBACK_EVENTS_DDL = `
@@ -3559,6 +3568,7 @@ const ALL_MIGRATIONS = [
 	addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
 	addBloomIndex(TABLE_SCORE_EVENTS, "idx_scoreId", "scoreId", 1),
+	addBloomIndex(TABLE_SCORE_EVENTS_DELTA, "idx_scoreId", "scoreId", 1),
 	addColumn(TABLE_FEEDBACK_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "reviewStatus", "LowCardinality(String) DEFAULT 'needs-review'"),
@@ -6308,6 +6318,17 @@ async function listScores(client, args, strategy) {
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
 }
+/**
+* Delta reads join the delta stream to the current-state table by scoreId, so
+* a poll or replay always returns the latest write of a score (a rewrite may
+* change traceId or timestamp, which the delta row does not track).
+*
+* A score's first delta row is its only cursor: rows after `afterCursor` are
+* dropped when the same scoreId already has a row at or before it, and
+* `LIMIT 1 BY` keeps the lowest cursor within the page. Duplicate delta rows
+* (a concurrent-insert race past the MV check) therefore never surface, in
+* this poll or a later one.
+*/
 async function queryScoresAfterCursor(client, whereClause, params, limit, cursorId) {
 	return await queryJson(client, `
       SELECT
@@ -6317,12 +6338,16 @@ async function queryScoresAfterCursor(client, whereClause, params, limit, cursor
         s.scoreId AS scoreId,
         toString(d.cursorId) AS cursorId
       FROM ${TABLE_SCORE_EVENTS_DELTA} d
-      INNER JOIN ${TABLE_SCORE_EVENTS} s
-        ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
-       AND s.timestamp = d.timestamp
-       AND s.scoreId = d.scoreId
-      ${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
+      INNER JOIN ${TABLE_SCORE_EVENTS_CURRENT} s FINAL
+        ON s.scoreId = d.scoreId
+      ${whereClause ? `${whereClause} AND` : "WHERE"} d.cursorId > {afterCursor:UInt64}
+        AND d.scoreId NOT IN (
+          SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA}
+          WHERE cursorId <= {afterCursor:UInt64}
+            AND scoreId IN (SELECT scoreId FROM ${TABLE_SCORE_EVENTS_DELTA} WHERE cursorId > {afterCursor:UInt64})
+        )
       ORDER BY d.cursorId ASC
+      LIMIT 1 BY s.scoreId
       LIMIT {fetchLimit:UInt32}
     `, {
 		...params,
@@ -6334,10 +6359,8 @@ async function getDeltaCursor$2(client, whereClause, params) {
 	const cursorId = (await queryJson(client, `
       SELECT toString(max(d.cursorId)) AS cursorId
       FROM mastra_score_events_delta d
-      INNER JOIN mastra_score_events s
-        ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
-       AND s.timestamp = d.timestamp
-       AND s.scoreId = d.scoreId
+      INNER JOIN mastra_score_events_current s FINAL
+        ON s.scoreId = d.scoreId
       ${whereClause}
     `, params))[0]?.cursorId ?? null;
 	if (cursorId) return cursorId;
@@ -8307,6 +8330,37 @@ async function queryNamesByTable(client, query, tables) {
 	}
 	return out;
 }
+/**
+* Upgrades a score delta MV created before per-scoreId deduplication with
+* `ALTER TABLE ... MODIFY QUERY`. The view is changed in place, so there is no
+* window without a view in which score writes would miss their delta row (a
+* drop-and-recreate would leave such writes out of delta polling for good).
+*
+* With a cluster configured the definition is read from every replica via
+* `clusterAllReplicas`, since each host stores its own copy of the view; a
+* single legacy copy is enough to alter `ON CLUSTER`.
+*
+* Introspection failures are logged and skipped, like the other schema checks
+* in init(): the existing view keeps working and the next boot retries.
+*/
+async function reconcileScoreDeltaMv(client, replication, strategy, logger) {
+	const cluster = replication?.cluster?.trim();
+	let createQueries;
+	try {
+		createQueries = (await (await client.query({
+			query: cluster ? `SELECT create_table_query FROM clusterAllReplicas({cluster:String}, system.tables) WHERE database = currentDatabase() AND name = {name:String}` : `SELECT create_table_query FROM system.tables WHERE database = currentDatabase() AND name = {name:String}`,
+			query_params: cluster ? {
+				cluster,
+				name: MV_SCORE_EVENTS_DELTA
+			} : { name: MV_SCORE_EVENTS_DELTA },
+			format: "JSONEachRow"
+		})).json()).map((row) => row.create_table_query ?? "");
+	} catch (error) {
+		logger?.warn?.(`Could not verify the ${MV_SCORE_EVENTS_DELTA} definition; leaving the existing view in place: ${error instanceof Error ? error.message : String(error)}`);
+		return;
+	}
+	if (createQueries.some((createQuery) => createQuery.length > 0 && !/NOT IN/i.test(createQuery))) await client.command({ query: addOnClusterToDDL(`ALTER TABLE ${MV_SCORE_EVENTS_DELTA} MODIFY QUERY ${buildScoreEventsDeltaMvQuery(strategy)}`, replication) });
+}
 async function detectDeltaCursorStrategy(client, override, existingStrategy) {
 	if (override) return override;
 	if (existingStrategy && existingStrategy !== "mixed") return existingStrategy;
@@ -8416,6 +8470,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
 			for (const migration of pendingMigrations) await this.#client.command({ query: addOnClusterToDDL(migration.sql, this.#replication) });
 			const coreMvDdl = this.#deltaCursorStrategy === null ? BASE_MV_DDL : buildAllMvDDL(this.#deltaCursorStrategy);
 			for (const ddl of coreMvDdl) await this.#client.command({ query: applyReplicationToDDL(ddl, this.#replication) });
+			if (this.#deltaCursorStrategy !== null) await reconcileScoreDeltaMv(this.#client, this.#replication, this.#deltaCursorStrategy, this.logger);
 			await backfillCurrentScores(this.#client);
 			if (this.#retention) await this.applyRetention();
 			if (this.#deltaCursorStrategy === "serial") for (const counterName of DELTA_CURSOR_COUNTER_NAMES) await this.#client.query({