@mastra/clickhouse 1.21.0-alpha.1 → 1.21.0-alpha.3

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-clickhouse
 description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
 metadata:
   package: "@mastra/clickhouse"
-  version: "1.21.0-alpha.1"
+  version: "1.21.0-alpha.3"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.21.0-alpha.1",
+  "version": "1.21.0-alpha.3",
   "package": "@mastra/clickhouse",
   "exports": {},
   "modules": {}

package/dist/docs/references/integrations-databases-clickhouse.md

@@ -93,6 +93,10 @@ Lightweight deletion is a hide-only operation that marks rows with ClickHouse's
 
 When all five observability signals have finite retention, Mastra also applies a TTL to deletion requests so they outlive the signal rows they protect. If any signal is unbounded, deletion requests remain unbounded. See [storage retention](https://mastra.ai/reference/storage/retention) for how the deletion-request TTL is calculated.
 
+Feedback review-status updates require `ALTER UPDATE` permission on `mastra_feedback_events` and, when delta polling is enabled, `INSERT` permission on `mastra_feedback_events_delta`. These updates wait for the ClickHouse mutation to finish on the server that receives the write, so latency depends on that server's mutation queue. Other replicas apply the mutation through the replication log, so a reader on a lagging replica can briefly see the previous status, and an inactive replica doesn't block the update. They modify the existing feedback row and preserve deletion masks: a concurrent review update cannot recreate deleted feedback. Successful updates remain available through delta polling. If a newer version of the same feedback event is ingested while a review update is in flight, the update is re-applied to that newer version; after repeated conflicts it fails with a conflict error (HTTP 409) and the caller retries.
+
+The status mutation and the delta insert are separate operations. If the delta insert fails, the API returns an error even though the status may have changed, and continued delta polling doesn't recover that notification: later polls from the same cursor never return it, and starting delta mode without a cursor subscribes at the current head. To recover, retry the review update after resolving the error, or reread the feedback with a regular list query.
+
 ### Observability with the legacy domain
 
 `ObservabilityStorageClickhouse` is the original observability adapter and remains supported for projects that haven't migrated to the vNext schema. The configuration shape is the same as the vNext class.
@@ -368,7 +372,16 @@ const observability = new ObservabilityStorageClickhouseVNext({
 await observability.init()
 ```
 
-In CI/CD pipelines, set `disableInit: true` on `ClickhouseStore` and run `init()` from a deployment step that uses elevated credentials. Runtime application credentials can then be limited to read and insert.
+In CI/CD pipelines, set `disableInit: true` on `ClickhouseStore` and run `init()` from a deployment step that uses elevated credentials. Runtime application credentials still need more than read and insert:
+
+- `SELECT` and `INSERT` on the Mastra tables.
+- `ALTER DELETE` on the observability tables you delete from. On ClickHouse 26.6 and earlier, lightweight deletes also require `ALTER UPDATE` on those tables; ClickHouse 26.7 removed that requirement.
+- For feedback review updates, `ALTER UPDATE(reviewStatus)` on `mastra_feedback_events` and `INSERT` on `mastra_feedback_events_delta`.
+
+```sql
+GRANT ALTER UPDATE(reviewStatus) ON <database>.mastra_feedback_events TO <runtime_user>;
+GRANT INSERT ON <database>.mastra_feedback_events_delta TO <runtime_user>;
+```
 
 ## Observability
 

package/dist/index.cjs

@@ -4295,17 +4295,46 @@ async function recordDeletionRequest(client, args) {
 		purgeVerifiedAt: EPOCH,
 		updatedAt: args.requestedAt
 	};
+	await insertDeletionRequest(client, row, args.replication);
+	return row;
+}
+/**
+* Marks a recorded deletion request as applied after its lightweight DELETEs
+* succeeded. The table is `ReplacingMergeTree(updatedAt)`, so re-inserting the
+* row with a newer `updatedAt` supersedes the pending version on merge and
+* under `FINAL`. Requests whose DELETE failed keep `lastAppliedAt` at the
+* epoch; mutation guards ignore them, and re-invoking the delete API records a
+* new request and converges.
+*/
+async function markDeletionRequestApplied(client, row, replication) {
+	const pendingAt = Date.parse(row.updatedAt);
+	const appliedAt = new Date(Number.isFinite(pendingAt) ? Math.max(Date.now(), pendingAt + 1) : Date.now()).toISOString();
+	const applied = {
+		...row,
+		lastAppliedAt: appliedAt,
+		updatedAt: appliedAt
+	};
+	await insertDeletionRequest(client, applied, replication);
+	return applied;
+}
+/**
+* Replicated clusters insert with parallel quorum, the same as every other
+* deletion-request write on main, so concurrent deletes never reject each other.
+* The guard that reads these rows is advisory: review updates mutate feedback
+* in place and preserve the delete mask, so a replica that has not yet received
+* a marker can only mis-report a status, never bring deleted feedback back.
+*/
+async function insertDeletionRequest(client, row, replication) {
 	await client.insert({
 		table: TABLE_DELETION_REQUESTS,
 		values: [row],
 		format: "JSONEachRow",
-		clickhouse_settings: isReplicationConfigured(args.replication) ? {
+		clickhouse_settings: isReplicationConfigured(replication) ? {
 			...CH_INSERT_SETTINGS,
 			insert_quorum: "auto",
 			insert_quorum_parallel: 1
 		} : CH_INSERT_SETTINGS
 	});
-	return row;
 }
 //#endregion
 //#region src/storage/domains/observability/v-next/discovery.ts
@@ -4797,14 +4826,18 @@ async function batchCreateFeedback(client, args) {
 * `organizationId` and `resourceId` values are ANDed into the predicate to
 * restrict deletion to records with matching scope fields.
 *
-* A durable deletion request is recorded before the lightweight delete. The
-* delete is immediately visible to subsequent reads; physical purge depends on
-* the table's configured retention TTL. The delta table is intentionally not
+* A durable deletion request is recorded before the lightweight delete and
+* marked applied once the delete succeeds. If the delete fails, the request
+* stays unapplied and does not block updates to the still-visible rows; retry
+* by calling this function again.
+*
+* The delete is immediately visible to subsequent reads; physical purge depends
+* on the table's configured retention TTL. The delta table is intentionally not
 * touched and expires through its fixed two-day TTL.
 */
 async function deleteFeedback(client, args, replication) {
 	if (args.feedbackIds.length === 0) return;
-	await recordDeletionRequest(client, {
+	const request = await recordDeletionRequest(client, {
 		requestId: (0, crypto$1.randomUUID)(),
 		organizationId: args.organizationId,
 		resourceId: args.resourceId,
@@ -4835,6 +4868,7 @@ async function deleteFeedback(client, args, replication) {
 		query_params: params,
 		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
 	});
+	await markDeletionRequestApplied(client, request, replication);
 }
 function feedbackNotFoundError(feedbackId) {
 	return new _mastra_core_error.MastraError({
@@ -4845,11 +4879,29 @@ function feedbackNotFoundError(feedbackId) {
 		details: { feedbackId }
 	});
 }
+function feedbackConflictError(feedbackId) {
+	return new _mastra_core_error.MastraError({
+		id: "OBSERVABILITY_UPDATE_FEEDBACK_REVIEW_STATUS_CONFLICT",
+		domain: _mastra_core_error.ErrorDomain.MASTRA_OBSERVABILITY,
+		category: _mastra_core_error.ErrorCategory.USER,
+		text: "Feedback record changed while its review status was being updated; retry the update",
+		details: { feedbackId }
+	});
+}
+/** Re-reads before giving up when ingestion keeps superseding the observed row. */
+const REVIEW_UPDATE_ATTEMPTS = 3;
+/**
+* Only applied requests block a review update. The guard decides the reply,
+* not data safety: the update mutates the row in place and preserves the
+* delete mask, so a replica that has not received a marker yet can only
+* mis-report a status, never bring deleted feedback back.
+*/
 async function hasFeedbackDeletionRequest(client, feedbackId, organizationId, resourceId) {
 	return (await queryJson$2(client, `SELECT 1 AS found FROM ${TABLE_DELETION_REQUESTS} FINAL
      WHERE signal = 'feedback'
        AND predicateType = 'itemIds'
        AND has(predicateValues, {feedbackId:String})
+       AND lastAppliedAt > toDateTime64(0, 3)
        AND (organizationId = '' OR organizationId = {organizationId:String})
        AND (resourceId = '' OR resourceId = {resourceId:String})
      LIMIT 1`, {
@@ -4858,28 +4910,63 @@ async function hasFeedbackDeletionRequest(client, feedbackId, organizationId, re
 		resourceId: resourceId ?? ""
 	})).length > 0;
 }
-async function updateFeedbackReviewStatus(client, args, replication) {
+async function updateFeedbackReviewStatus(client, args, strategy = null) {
 	const { feedbackId, reviewStatus } = parseUpdateFeedbackReviewStatusArgs(args);
-	const existingRow = (await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL
+	for (let attempt = 1;; attempt++) {
+		const updated = await applyReviewStatus(client, feedbackId, reviewStatus, strategy);
+		if (updated) return updated;
+		if (attempt >= REVIEW_UPDATE_ATTEMPTS) throw feedbackConflictError(feedbackId);
+	}
+}
+/**
+* One read-guard-mutate-verify pass. Returns `null` when the read-back does not
+* show the new status: a newer version superseded the observed row, a delete
+* hid it after the guard read (the next pass then reports not found), or the
+* mutation left the row untouched.
+*/
+async function applyReviewStatus(client, feedbackId, reviewStatus, strategy) {
+	const existingRow = (await queryJson$2(client, `SELECT *, toString(writeVersion) AS reviewWriteVersion FROM ${TABLE_FEEDBACK_EVENTS} FINAL
      WHERE feedbackId = {feedbackId:String}
      ORDER BY writeVersion DESC, timestamp DESC
      LIMIT 1`, { feedbackId }))[0];
 	if (!existingRow) throw feedbackNotFoundError(feedbackId);
 	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) throw feedbackNotFoundError(feedbackId);
-	const updated = rowToFeedbackRecord({
-		...existingRow,
+	const identity = `feedbackId = {feedbackId:String}
+    AND timestamp = parseDateTime64BestEffort({timestamp:String}, 3, 'UTC')
+    AND (traceId = {traceId:Nullable(String)} OR (isNull(traceId) AND isNull({traceId:Nullable(String)})))
+    AND writeVersion = {writeVersion:UInt64}`;
+	const params = {
+		feedbackId,
+		timestamp: existingRow.timestamp,
+		traceId: existingRow.traceId,
+		writeVersion: existingRow.reviewWriteVersion,
 		reviewStatus
+	};
+	await client.command({
+		query: `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} UPDATE reviewStatus = {reviewStatus:String} WHERE ${identity}`,
+		query_params: params,
+		clickhouse_settings: {
+			...CH_SETTINGS,
+			mutations_sync: "1"
+		}
 	});
-	await batchCreateFeedback(client, { feedbacks: [updated] });
-	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) {
-		await deleteFeedback(client, {
-			feedbackIds: [feedbackId],
-			organizationId: existingRow.organizationId ?? void 0,
-			resourceId: existingRow.resourceId ?? void 0
-		}, replication);
-		throw feedbackNotFoundError(feedbackId);
-	}
-	return updated;
+	if (strategy !== null) await client.command({
+		query: `INSERT INTO ${TABLE_FEEDBACK_EVENTS_DELTA}
+        SELECT ${buildDeltaCursorExpr(strategy, "mastra_feedback_events_delta_cursor", "feedbackId")} AS cursorId,
+          ingestedAt, traceId, timestamp, feedbackId
+        FROM (
+          SELECT now64(9, 'UTC') AS ingestedAt, traceId, timestamp, feedbackId
+          FROM ${TABLE_FEEDBACK_EVENTS} FINAL WHERE ${identity}
+        )`,
+		query_params: params,
+		clickhouse_settings: {
+			...CH_INSERT_SETTINGS,
+			async_insert: 0
+		}
+	});
+	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) throw feedbackNotFoundError(feedbackId);
+	const current = await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL WHERE ${identity} LIMIT 1`, params);
+	return current[0]?.reviewStatus === reviewStatus ? rowToFeedbackRecord(current[0]) : null;
 }
 async function listFeedback(client, args, strategy) {
 	const parsed = _mastra_core_storage.listFeedbackArgsSchema.parse(args);
@@ -6228,14 +6315,16 @@ async function batchCreateScores(client, args) {
 * `organizationId` and `resourceId` values are ANDed into the predicate to
 * restrict deletion to records with matching scope fields.
 *
-* A durable deletion request is recorded before the lightweight delete. The
-* delete is immediately visible to subsequent reads; physical purge depends on
-* the table's configured retention TTL. The delta table is intentionally not
+* A durable deletion request is recorded before the lightweight delete and
+* marked applied once the delete succeeds. If the delete fails, the request
+* stays unapplied; retry by calling this function again. The delete is
+* immediately visible to subsequent reads; physical purge depends on the
+* table's configured retention TTL. The delta table is intentionally not
 * touched and expires through its fixed two-day TTL.
 */
 async function deleteScores(client, args, replication) {
 	if (args.scoreIds.length === 0) return;
-	await recordDeletionRequest(client, {
+	const request = await recordDeletionRequest(client, {
 		requestId: (0, crypto$1.randomUUID)(),
 		organizationId: args.organizationId,
 		resourceId: args.resourceId,
@@ -6267,6 +6356,7 @@ async function deleteScores(client, args, replication) {
 		query_params: params,
 		clickhouse_settings
 	});
+	await markDeletionRequestApplied(client, request, replication);
 }
 async function listScores(client, args, strategy) {
 	const parsed = _mastra_core_storage.listScoresArgsSchema.parse(args);
@@ -6516,6 +6606,9 @@ async function getScorePercentiles(client, args) {
 //#endregion
 //#region src/storage/domains/observability/v-next/trace-query.ts
 const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
+function durationMsSql(startedAt, endedAt) {
+	return `dateDiff('millisecond', ${startedAt}, ${endedAt})`;
+}
 const TRACE_FIELDS = {
 	traceId: {
 		sql: "r.traceId",
@@ -6537,6 +6630,10 @@ const TRACE_FIELDS = {
 		sql: "r.endedAt",
 		parameterType: "DateTime64(3, 'UTC')"
 	},
+	durationMs: {
+		sql: durationMsSql("r.startedAt", "r.endedAt"),
+		parameterType: "Float64"
+	},
 	entityName: {
 		sql: "r.entityName",
 		parameterType: "String"
@@ -6861,7 +6958,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
       if(JSONType(attributes, 'provider') = 'String', JSONExtractString(attributes, 'provider'), NULL) AS provider,
       startedAt,
       endedAt,
-      dateDiff('millisecond', startedAt, endedAt) AS durationMs,
+      ${durationMsSql("startedAt", "endedAt")} AS durationMs,
       if(isNotNull(error), 'error', 'success') AS status,
       error,
       entityType,
@@ -7725,7 +7822,9 @@ async function getTraceLight(client, args) {
 * so span deletes never propagate to it. Delta tables self-expire via TTL and
 * discovery tables self-heal, so neither needs explicit deletes.
 *
-* Records the predicate before using lightweight DELETE FROM on every table.
+* Records the predicate before using lightweight DELETE FROM on every table
+* and marks the request applied once every delete succeeds. If any delete
+* fails, the request stays unapplied; retry by calling this function again.
 * Lightweight deletes hide rows through ClickHouse's delete mask; physical
 * removal depends on the deployment's configured retention and merge policy.
 *
@@ -7734,7 +7833,7 @@ async function getTraceLight(client, args) {
 */
 async function batchDeleteTraces(client, args, replication) {
 	if (args.traceIds.length === 0) return;
-	await recordDeletionRequest(client, {
+	const request = await recordDeletionRequest(client, {
 		requestId: (0, crypto$1.randomUUID)(),
 		organizationId: args.organizationId,
 		resourceId: args.resourceId,
@@ -7787,6 +7886,7 @@ async function batchDeleteTraces(client, args, replication) {
 		query_params: params,
 		clickhouse_settings: { lightweight_deletes_sync: "2" }
 	}))]);
+	await markDeletionRequestApplied(client, request, replication);
 }
 /**
 * List trace branches with optional filtering, pagination, and ordering.
@@ -8566,6 +8666,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
 			"metrics",
 			"logs",
 			"trace-query",
+			"trace-query-root-duration",
 			"trace-query-discovery",
 			"thread-query",
 			"trace-query-tenant-scope"
@@ -8575,6 +8676,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
 			"logs",
 			"delta-polling",
 			"trace-query",
+			"trace-query-root-duration",
 			"trace-query-discovery",
 			"thread-query",
 			"trace-query-tenant-scope"
@@ -8917,7 +9019,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
 	}
 	async updateFeedbackReviewStatus(args) {
 		try {
-			return await updateFeedbackReviewStatus(this.#client, args, this.#replication);
+			return await updateFeedbackReviewStatus(this.#client, args, deltaPollingSupported(this.#deltaCursorStrategy) ? this.#deltaCursorStrategy : null);
 		} catch (error) {
 			if (error instanceof _mastra_core_error.MastraError) throw error;
 			throw new _mastra_core_error.MastraError({
@@ -10067,6 +10169,7 @@ exports.TABLE_DELETION_REQUESTS = TABLE_DELETION_REQUESTS;
 exports.TABLE_ENGINES = TABLE_ENGINES;
 exports.WorkflowsStorageClickhouse = WorkflowsStorageClickhouse;
 exports.applyClickHouseRetention = applyClickHouseRetention;
+exports.markDeletionRequestApplied = markDeletionRequestApplied;
 exports.recordDeletionRequest = recordDeletionRequest;
 
 //# sourceMappingURL=index.cjs.map