@mastra/clickhouse 1.18.0-alpha.0 → 1.18.0-alpha.2

package/dist/index.js

@@ -4,9 +4,9 @@ import * as coreStorage from "@mastra/core/storage";
 import { BRANCH_SPAN_TYPES, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_CURSORS, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, buildInputPreview, calculatePagination, computeTraceStatus, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MessageList } from "@mastra/core/agent";
 import { MastraBase } from "@mastra/core/base";
+import { randomUUID } from "crypto";
 import { parseFieldKey } from "@mastra/core/utils";
 import { coreFeatures } from "@mastra/core/features";
-import { randomUUID } from "crypto";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -3183,6 +3183,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS} (
 
   -- IDs
   feedbackId         String,
+  writeVersion       UInt64 DEFAULT 0,
   traceId            Nullable(String),
   spanId             Nullable(String),
   experimentId       Nullable(String),
@@ -3470,6 +3471,7 @@ const ALL_MIGRATIONS = [
 	addColumn(TABLE_SCORE_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
 	addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
+	addColumn(TABLE_FEEDBACK_EVENTS, "writeVersion", "UInt64 DEFAULT 0"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "reviewStatus", "LowCardinality(String) DEFAULT 'needs-review'"),
 	addColumn(TABLE_FEEDBACK_EVENTS, "parentEntityVersionId", "Nullable(String)"),
@@ -4633,6 +4635,26 @@ async function queryJson$2(client, query, params) {
 		clickhouse_settings: CH_SETTINGS
 	})).json();
 }
+async function feedbackRowsWithWriteVersions(client, feedbacks) {
+	const identifiedFeedback = feedbacks.map((feedback) => ({
+		...feedback,
+		feedbackId: feedback.feedbackId ?? ""
+	}));
+	const feedbackIds = [...new Set(identifiedFeedback.map((feedback) => feedback.feedbackId))];
+	const existingVersions = await queryJson$2(client, `SELECT feedbackId, toString(max(writeVersion)) AS writeVersion
+     FROM ${TABLE_FEEDBACK_EVENTS}
+     WHERE feedbackId IN ({feedbackIds:Array(String)})
+     GROUP BY feedbackId`, { feedbackIds });
+	const versions = new Map(existingVersions.map((row) => [row.feedbackId, BigInt(row.writeVersion)]));
+	return identifiedFeedback.map((feedback) => {
+		const writeVersion = (versions.get(feedback.feedbackId) ?? 0n) + 1n;
+		versions.set(feedback.feedbackId, writeVersion);
+		return {
+			...feedbackRecordToRow(feedback),
+			writeVersion: writeVersion.toString()
+		};
+	});
+}
 async function createFeedback(client, args) {
 	await batchCreateFeedback(client, { feedbacks: [args.feedback] });
 }
@@ -4640,31 +4662,98 @@ async function batchCreateFeedback(client, args) {
 	if (args.feedbacks.length === 0) return;
 	await client.insert({
 		table: TABLE_FEEDBACK_EVENTS,
-		values: args.feedbacks.map(feedbackRecordToRow),
+		values: await feedbackRowsWithWriteVersions(client, args.feedbacks),
 		format: "JSONEachRow",
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
 }
-async function updateFeedbackReviewStatus(client, args) {
-	const { feedbackId, reviewStatus } = parseUpdateFeedbackReviewStatusArgs(args);
-	const existing = await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL WHERE feedbackId = {feedbackId:String} LIMIT 1`, { feedbackId });
-	if (!existing[0]) throw new MastraError({
+/**
+* Delete feedback events by feedbackId via lightweight DELETE. Optional
+* `organizationId` and `resourceId` values are ANDed into the predicate to
+* restrict deletion to records with matching scope fields.
+*
+* A durable deletion request is recorded before the lightweight delete. The
+* delete is immediately visible to subsequent reads; physical purge depends on
+* the table's configured retention TTL. The delta table is intentionally not
+* touched and expires through its fixed two-day TTL.
+*/
+async function deleteFeedback(client, args, replication) {
+	if (args.feedbackIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "feedback",
+		predicateType: "itemIds",
+		predicateValues: [...args.feedbackIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
+	const params = {};
+	const idPlaceholders = [];
+	for (let i = 0; i < args.feedbackIds.length; i++) {
+		const name = `fid_${i}`;
+		params[name] = args.feedbackIds[i];
+		idPlaceholders.push(`{${name}:String}`);
+	}
+	const conditions = [`feedbackId IN (${idPlaceholders.join(", ")})`];
+	if (args.organizationId !== void 0) {
+		conditions.push("organizationId = {delOrganizationId:String}");
+		params.delOrganizationId = args.organizationId;
+	}
+	if (args.resourceId !== void 0) {
+		conditions.push("resourceId = {delResourceId:String}");
+		params.delResourceId = args.resourceId;
+	}
+	await client.command({
+		query: `DELETE FROM ${TABLE_FEEDBACK_EVENTS} WHERE ${conditions.join(" AND ")}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+	});
+}
+function feedbackNotFoundError(feedbackId) {
+	return new MastraError({
 		id: "OBSERVABILITY_UPDATE_FEEDBACK_REVIEW_STATUS_NOT_FOUND",
 		domain: ErrorDomain.MASTRA_OBSERVABILITY,
 		category: ErrorCategory.USER,
 		text: "Feedback record not found",
 		details: { feedbackId }
 	});
+}
+async function hasFeedbackDeletionRequest(client, feedbackId, organizationId, resourceId) {
+	return (await queryJson$2(client, `SELECT 1 AS found FROM ${TABLE_DELETION_REQUESTS} FINAL
+     WHERE signal = 'feedback'
+       AND predicateType = 'itemIds'
+       AND has(predicateValues, {feedbackId:String})
+       AND (organizationId = '' OR organizationId = {organizationId:String})
+       AND (resourceId = '' OR resourceId = {resourceId:String})
+     LIMIT 1`, {
+		feedbackId,
+		organizationId: organizationId ?? "",
+		resourceId: resourceId ?? ""
+	})).length > 0;
+}
+async function updateFeedbackReviewStatus(client, args, replication) {
+	const { feedbackId, reviewStatus } = parseUpdateFeedbackReviewStatusArgs(args);
+	const existingRow = (await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL
+     WHERE feedbackId = {feedbackId:String}
+     ORDER BY writeVersion DESC, timestamp DESC
+     LIMIT 1`, { feedbackId }))[0];
+	if (!existingRow) throw feedbackNotFoundError(feedbackId);
+	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) throw feedbackNotFoundError(feedbackId);
 	const updated = rowToFeedbackRecord({
-		...existing[0],
+		...existingRow,
 		reviewStatus
 	});
-	await client.insert({
-		table: TABLE_FEEDBACK_EVENTS,
-		values: [feedbackRecordToRow(updated)],
-		format: "JSONEachRow",
-		clickhouse_settings: CH_INSERT_SETTINGS
-	});
+	await batchCreateFeedback(client, { feedbacks: [updated] });
+	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) {
+		await deleteFeedback(client, {
+			feedbackIds: [feedbackId],
+			organizationId: existingRow.organizationId ?? void 0,
+			resourceId: existingRow.resourceId ?? void 0
+		}, replication);
+		throw feedbackNotFoundError(feedbackId);
+	}
 	return updated;
 }
 async function listFeedback(client, args, strategy) {
@@ -5954,6 +6043,50 @@ async function batchCreateScores(client, args) {
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
 }
+/**
+* Delete score events by scoreId via lightweight DELETE. Optional
+* `organizationId` and `resourceId` values are ANDed into the predicate to
+* restrict deletion to records with matching scope fields.
+*
+* A durable deletion request is recorded before the lightweight delete. The
+* delete is immediately visible to subsequent reads; physical purge depends on
+* the table's configured retention TTL. The delta table is intentionally not
+* touched and expires through its fixed two-day TTL.
+*/
+async function deleteScores(client, args, replication) {
+	if (args.scoreIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "scores",
+		predicateType: "itemIds",
+		predicateValues: [...args.scoreIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
+	const params = {};
+	const idPlaceholders = [];
+	for (let i = 0; i < args.scoreIds.length; i++) {
+		const name = `sid_${i}`;
+		params[name] = args.scoreIds[i];
+		idPlaceholders.push(`{${name}:String}`);
+	}
+	const conditions = [`scoreId IN (${idPlaceholders.join(", ")})`];
+	if (args.organizationId !== void 0) {
+		conditions.push("organizationId = {delOrganizationId:String}");
+		params.delOrganizationId = args.organizationId;
+	}
+	if (args.resourceId !== void 0) {
+		conditions.push("resourceId = {delResourceId:String}");
+		params.delResourceId = args.resourceId;
+	}
+	await client.command({
+		query: `DELETE FROM ${TABLE_SCORE_EVENTS} WHERE ${conditions.join(" AND ")}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+	});
+}
 async function listScores(client, args, strategy) {
 	const parsed = listScoresArgsSchema.parse(args);
 	const deltaCursorEnabled = deltaPollingSupported(strategy);
@@ -6324,6 +6457,44 @@ const SCORE_FIELDS = {
 		parameterType: "String"
 	}
 };
+const FEEDBACK_FIELDS = {
+	feedbackType: {
+		sql: "s.feedbackType",
+		parameterType: "String"
+	},
+	feedbackSource: {
+		sql: "s.feedbackSource",
+		parameterType: "String"
+	},
+	feedbackUserId: {
+		sql: "s.feedbackUserId",
+		parameterType: "String"
+	},
+	sourceId: {
+		sql: "s.sourceId",
+		parameterType: "String"
+	},
+	entityVersionId: {
+		sql: "s.entityVersionId",
+		parameterType: "String"
+	},
+	parentEntityVersionId: {
+		sql: "s.parentEntityVersionId",
+		parameterType: "String"
+	},
+	rootEntityVersionId: {
+		sql: "s.rootEntityVersionId",
+		parameterType: "String"
+	},
+	timestamp: {
+		sql: "s.timestamp",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	comment: {
+		sql: "s.comment",
+		parameterType: "String"
+	}
+};
 const TRACE_SELECT = `
   r.traceId AS traceId,
   r.spanId AS rootSpanId,
@@ -6384,6 +6555,22 @@ function compileScalarPredicate(predicate, registry, parameters, allowMetadata =
 	if (operator === void 0) throw new Error(`Unsupported trusted trace-query operator: ${predicate.operator}`);
 	return `ifNull(${field.sql} ${operator} ${parameter}, ${predicate.operator === "ne" ? "1" : "0"})`;
 }
+function compileFeedbackScalarPredicate(predicate, parameters) {
+	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compileFeedbackScalarPredicate(arg, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
+	if (predicate.type === "not") return `NOT (${compileFeedbackScalarPredicate(predicate.arg, parameters)})`;
+	if (predicate.field !== "value") return compileScalarPredicate(predicate, FEEDBACK_FIELDS, parameters);
+	if (predicate.type === "presence") {
+		const present = `(isNotNull(s.valueString) OR isNotNull(s.valueNumber))`;
+		return predicate.operator === "exists" ? present : `NOT ${present}`;
+	}
+	return compileScalarPredicate(predicate, typeof (predicate.type === "membership" ? predicate.values[0] : predicate.value) === "number" ? { value: {
+		sql: "s.valueNumber",
+		parameterType: "Float64"
+	} } : { value: {
+		sql: "s.valueString",
+		parameterType: "String"
+	} }, parameters);
+}
 function collectRelationCollections(predicate, collections = /* @__PURE__ */ new Set()) {
 	if (!predicate) return collections;
 	if (predicate.type === "relation") collections.add(predicate.collection);
@@ -6393,11 +6580,11 @@ function collectRelationCollections(predicate, collections = /* @__PURE__ */ new
 }
 function compilePredicate(predicate, parameters) {
 	if (predicate.type === "relation") {
-		const registry = predicate.collection === "spans" ? SPAN_FIELDS : SCORE_FIELDS;
 		const existence = `EXISTS (
-      SELECT 1 FROM ${predicate.collection === "spans" ? "current_spans" : "current_scores"} s
-      WHERE s.traceId = r.traceId
-        AND (${compileScalarPredicate(predicate.predicate, registry, parameters)})
+      SELECT 1 FROM ${predicate.collection === "spans" ? "current_spans" : predicate.collection === "scores" ? "current_scores" : "current_feedback"} s
+      WHERE isNotNull(s.traceId)
+        AND s.traceId = r.traceId
+        AND (${predicate.collection === "feedback" ? compileFeedbackScalarPredicate(predicate.predicate, parameters) : compileScalarPredicate(predicate.predicate, predicate.collection === "spans" ? SPAN_FIELDS : SCORE_FIELDS, parameters)})
     )`;
 		return predicate.quantifier === "some" ? existence : `NOT ${existence}`;
 	}
@@ -6466,6 +6653,29 @@ function compileClickHouseTraceQuery(plan) {
       AND traceId IN (SELECT traceId FROM root_scope)
     ORDER BY scoreId, timestamp DESC
     LIMIT 1 BY scoreId
+  )`);
+	if (relationCollections.has("feedback")) ctes.push(`current_feedback AS (
+    SELECT
+      traceId,
+      feedbackType,
+      feedbackSource,
+      feedbackUserId,
+      sourceId,
+      valueString,
+      valueNumber,
+      comment,
+      timestamp,
+      entityVersionId,
+      parentEntityVersionId,
+      rootEntityVersionId
+    FROM (
+      SELECT *
+      FROM ${TABLE_FEEDBACK_EVENTS} FINAL
+      ORDER BY feedbackId, writeVersion DESC, timestamp DESC
+      LIMIT 1 BY feedbackId
+    ) AS current
+    WHERE isNotNull(traceId)
+      AND traceId IN (SELECT traceId FROM root_scope)
   )`);
 	const predicate = plan.where ? compilePredicate(plan.where, parameters) : "1";
 	ctes.push(`candidates AS (
@@ -7925,6 +8135,19 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async deleteScores(args) {
+		try {
+			await deleteScores(this.#client, args, this.#replication);
+		} catch (error) {
+			if (error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "DELETE_SCORES", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { count: args.scoreIds.length }
+			}, error);
+		}
+	}
 	async getScoreById(scoreId) {
 		try {
 			return await getScoreById(this.#client, scoreId);
@@ -7963,9 +8186,22 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async deleteFeedback(args) {
+		try {
+			await deleteFeedback(this.#client, args, this.#replication);
+		} catch (error) {
+			if (error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "DELETE_FEEDBACK", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { count: args.feedbackIds.length }
+			}, error);
+		}
+	}
 	async updateFeedbackReviewStatus(args) {
 		try {
-			return await updateFeedbackReviewStatus(this.#client, args);
+			return await updateFeedbackReviewStatus(this.#client, args, this.#replication);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({