@mastra/clickhouse 1.17.0 → 1.18.0-alpha.1

package/dist/index.js

@@ -4,9 +4,9 @@ import * as coreStorage from "@mastra/core/storage";
 import { BRANCH_SPAN_TYPES, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_KNOWLEDGE_ACTIVITY, TABLE_KNOWLEDGE_CURSORS, TABLE_KNOWLEDGE_MENTIONS, TABLE_KNOWLEDGE_NODES, TABLE_KNOWLEDGE_RECORDS, TABLE_KNOWLEDGE_SEMANTIC_OUTBOX, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, buildInputPreview, calculatePagination, computeTraceStatus, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MessageList } from "@mastra/core/agent";
 import { MastraBase } from "@mastra/core/base";
+import { randomUUID } from "crypto";
 import { parseFieldKey } from "@mastra/core/utils";
 import { coreFeatures } from "@mastra/core/features";
-import { randomUUID } from "crypto";
 import { saveScorePayloadSchema } from "@mastra/core/evals";
 //#region src/storage/db/replication.ts
 const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
@@ -4645,18 +4645,79 @@ async function batchCreateFeedback(client, args) {
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
 }
-async function updateFeedbackReviewStatus(client, args) {
-	const { feedbackId, reviewStatus } = parseUpdateFeedbackReviewStatusArgs(args);
-	const existing = await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL WHERE feedbackId = {feedbackId:String} LIMIT 1`, { feedbackId });
-	if (!existing[0]) throw new MastraError({
+/**
+* Delete feedback events by feedbackId via lightweight DELETE. Optional
+* `organizationId` and `resourceId` values are ANDed into the predicate to
+* restrict deletion to records with matching scope fields.
+*
+* A durable deletion request is recorded before the lightweight delete. The
+* delete is immediately visible to subsequent reads; physical purge depends on
+* the table's configured retention TTL. The delta table is intentionally not
+* touched and expires through its fixed two-day TTL.
+*/
+async function deleteFeedback(client, args, replication) {
+	if (args.feedbackIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "feedback",
+		predicateType: "itemIds",
+		predicateValues: [...args.feedbackIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
+	const params = {};
+	const idPlaceholders = [];
+	for (let i = 0; i < args.feedbackIds.length; i++) {
+		const name = `fid_${i}`;
+		params[name] = args.feedbackIds[i];
+		idPlaceholders.push(`{${name}:String}`);
+	}
+	const conditions = [`feedbackId IN (${idPlaceholders.join(", ")})`];
+	if (args.organizationId !== void 0) {
+		conditions.push("organizationId = {delOrganizationId:String}");
+		params.delOrganizationId = args.organizationId;
+	}
+	if (args.resourceId !== void 0) {
+		conditions.push("resourceId = {delResourceId:String}");
+		params.delResourceId = args.resourceId;
+	}
+	await client.command({
+		query: `DELETE FROM ${TABLE_FEEDBACK_EVENTS} WHERE ${conditions.join(" AND ")}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+	});
+}
+function feedbackNotFoundError(feedbackId) {
+	return new MastraError({
 		id: "OBSERVABILITY_UPDATE_FEEDBACK_REVIEW_STATUS_NOT_FOUND",
 		domain: ErrorDomain.MASTRA_OBSERVABILITY,
 		category: ErrorCategory.USER,
 		text: "Feedback record not found",
 		details: { feedbackId }
 	});
+}
+async function hasFeedbackDeletionRequest(client, feedbackId, organizationId, resourceId) {
+	return (await queryJson$2(client, `SELECT 1 AS found FROM ${TABLE_DELETION_REQUESTS} FINAL
+     WHERE signal = 'feedback'
+       AND predicateType = 'itemIds'
+       AND has(predicateValues, {feedbackId:String})
+       AND (organizationId = '' OR organizationId = {organizationId:String})
+       AND (resourceId = '' OR resourceId = {resourceId:String})
+     LIMIT 1`, {
+		feedbackId,
+		organizationId: organizationId ?? "",
+		resourceId: resourceId ?? ""
+	})).length > 0;
+}
+async function updateFeedbackReviewStatus(client, args, replication) {
+	const { feedbackId, reviewStatus } = parseUpdateFeedbackReviewStatusArgs(args);
+	const existingRow = (await queryJson$2(client, `SELECT * FROM ${TABLE_FEEDBACK_EVENTS} FINAL WHERE feedbackId = {feedbackId:String} LIMIT 1`, { feedbackId }))[0];
+	if (!existingRow) throw feedbackNotFoundError(feedbackId);
+	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) throw feedbackNotFoundError(feedbackId);
 	const updated = rowToFeedbackRecord({
-		...existing[0],
+		...existingRow,
 		reviewStatus
 	});
 	await client.insert({
@@ -4665,6 +4726,14 @@ async function updateFeedbackReviewStatus(client, args) {
 		format: "JSONEachRow",
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
+	if (await hasFeedbackDeletionRequest(client, feedbackId, existingRow.organizationId, existingRow.resourceId)) {
+		await deleteFeedback(client, {
+			feedbackIds: [feedbackId],
+			organizationId: existingRow.organizationId ?? void 0,
+			resourceId: existingRow.resourceId ?? void 0
+		}, replication);
+		throw feedbackNotFoundError(feedbackId);
+	}
 	return updated;
 }
 async function listFeedback(client, args, strategy) {
@@ -5954,6 +6023,50 @@ async function batchCreateScores(client, args) {
 		clickhouse_settings: CH_INSERT_SETTINGS
 	});
 }
+/**
+* Delete score events by scoreId via lightweight DELETE. Optional
+* `organizationId` and `resourceId` values are ANDed into the predicate to
+* restrict deletion to records with matching scope fields.
+*
+* A durable deletion request is recorded before the lightweight delete. The
+* delete is immediately visible to subsequent reads; physical purge depends on
+* the table's configured retention TTL. The delta table is intentionally not
+* touched and expires through its fixed two-day TTL.
+*/
+async function deleteScores(client, args, replication) {
+	if (args.scoreIds.length === 0) return;
+	await recordDeletionRequest(client, {
+		requestId: randomUUID(),
+		organizationId: args.organizationId,
+		resourceId: args.resourceId,
+		signal: "scores",
+		predicateType: "itemIds",
+		predicateValues: [...args.scoreIds],
+		requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		replication
+	});
+	const params = {};
+	const idPlaceholders = [];
+	for (let i = 0; i < args.scoreIds.length; i++) {
+		const name = `sid_${i}`;
+		params[name] = args.scoreIds[i];
+		idPlaceholders.push(`{${name}:String}`);
+	}
+	const conditions = [`scoreId IN (${idPlaceholders.join(", ")})`];
+	if (args.organizationId !== void 0) {
+		conditions.push("organizationId = {delOrganizationId:String}");
+		params.delOrganizationId = args.organizationId;
+	}
+	if (args.resourceId !== void 0) {
+		conditions.push("resourceId = {delResourceId:String}");
+		params.delResourceId = args.resourceId;
+	}
+	await client.command({
+		query: `DELETE FROM ${TABLE_SCORE_EVENTS} WHERE ${conditions.join(" AND ")}`,
+		query_params: params,
+		clickhouse_settings: { lightweight_deletes_sync: isReplicationConfigured(replication) ? "2" : "1" }
+	});
+}
 async function listScores(client, args, strategy) {
 	const parsed = listScoresArgsSchema.parse(args);
 	const deltaCursorEnabled = deltaPollingSupported(strategy);
@@ -6225,13 +6338,65 @@ const TRACE_FIELDS = {
 	}
 };
 const SPAN_FIELDS = {
+	name: {
+		sql: "s.name",
+		parameterType: "String"
+	},
 	spanType: {
 		sql: "s.spanType",
 		parameterType: "String"
 	},
+	model: {
+		sql: "s.model",
+		parameterType: "String"
+	},
+	provider: {
+		sql: "s.provider",
+		parameterType: "String"
+	},
+	startedAt: {
+		sql: "s.startedAt",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	endedAt: {
+		sql: "s.endedAt",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	durationMs: {
+		sql: "s.durationMs",
+		parameterType: "Float64"
+	},
+	status: {
+		sql: "s.status",
+		parameterType: "String"
+	},
 	error: {
 		sql: "s.error",
 		parameterType: "String"
+	},
+	entityType: {
+		sql: "s.entityType",
+		parameterType: "String"
+	},
+	entityId: {
+		sql: "s.entityId",
+		parameterType: "String"
+	},
+	entityName: {
+		sql: "s.entityName",
+		parameterType: "String"
+	},
+	entityVersionId: {
+		sql: "s.entityVersionId",
+		parameterType: "String"
+	},
+	parentEntityVersionId: {
+		sql: "s.parentEntityVersionId",
+		parameterType: "String"
+	},
+	rootEntityVersionId: {
+		sql: "s.rootEntityVersionId",
+		parameterType: "String"
 	}
 };
 const SCORE_FIELDS = {
@@ -6239,9 +6404,37 @@ const SCORE_FIELDS = {
 		sql: "s.scorerId",
 		parameterType: "String"
 	},
+	scorerVersion: {
+		sql: "s.scorerVersion",
+		parameterType: "String"
+	},
+	scoreSource: {
+		sql: "s.scoreSource",
+		parameterType: "String"
+	},
 	score: {
 		sql: "s.score",
 		parameterType: "Float64"
+	},
+	timestamp: {
+		sql: "s.timestamp",
+		parameterType: "DateTime64(3, 'UTC')"
+	},
+	spanId: {
+		sql: "s.spanId",
+		parameterType: "String"
+	},
+	entityVersionId: {
+		sql: "s.entityVersionId",
+		parameterType: "String"
+	},
+	parentEntityVersionId: {
+		sql: "s.parentEntityVersionId",
+		parameterType: "String"
+	},
+	rootEntityVersionId: {
+		sql: "s.rootEntityVersionId",
+		parameterType: "String"
 	}
 };
 const TRACE_SELECT = `
@@ -6273,10 +6466,20 @@ function resolveOrderField(field) {
 	if (field === "startedAt" || field === "endedAt") return field;
 	throw new Error(`Unsupported trusted trace-query field: ${field}`);
 }
-function compileScalarPredicate(predicate, registry, parameters) {
-	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compileScalarPredicate(arg, registry, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
-	if (predicate.type === "not") return `NOT (${compileScalarPredicate(predicate.arg, registry, parameters)})`;
-	const field = fieldDefinition(registry, predicate.field);
+function isMetadataField(field) {
+	return field.startsWith("metadata.");
+}
+function compileScalarPredicate(predicate, registry, parameters, allowMetadata = false) {
+	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compileScalarPredicate(arg, registry, parameters, allowMetadata)})`).join(predicate.operator === "and" ? " AND " : " OR ");
+	if (predicate.type === "not") return `NOT (${compileScalarPredicate(predicate.arg, registry, parameters, allowMetadata)})`;
+	const field = isMetadataField(predicate.field) ? (() => {
+		if (!allowMetadata) throw new Error(`Unsupported trusted trace-query field: ${predicate.field}`);
+		const key = parameters.add(predicate.field.slice(9), "String");
+		return {
+			sql: `coalesce(if(mapContains(r.metadataSearch, ${key}), r.metadataSearch[${key}], NULL), nullIf(trim(JSONExtractString(r.metadataRaw, ${key})), ''))`,
+			parameterType: "String"
+		};
+	})() : fieldDefinition(registry, predicate.field);
 	if (predicate.type === "presence") return `${predicate.operator === "exists" ? "isNotNull" : "isNull"}(${field.sql})`;
 	if (predicate.type === "membership") {
 		const values = predicate.values.map((value) => parameters.add(value, field.parameterType)).join(", ");
@@ -6313,7 +6516,7 @@ function compilePredicate(predicate, parameters) {
 	}
 	if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compilePredicate(arg, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
 	if (predicate.type === "not") return `NOT (${compilePredicate(predicate.arg, parameters)})`;
-	return compileScalarPredicate(predicate, TRACE_FIELDS, parameters);
+	return compileScalarPredicate(predicate, TRACE_FIELDS, parameters, true);
 }
 function compileClickHouseTraceQuery(plan) {
 	const parameters = new ParameterBuilder();
@@ -6336,17 +6539,46 @@ function compileClickHouseTraceQuery(plan) {
       AND startedAt < ${to}
   )`];
 	if (relationCollections.has("spans")) ctes.push(`current_spans AS (
-    SELECT traceId, spanType, error
+    SELECT
+      traceId,
+      name,
+      spanType,
+      if(JSONType(attributes, 'model') = 'String', JSONExtractString(attributes, 'model'), NULL) AS model,
+      if(JSONType(attributes, 'provider') = 'String', JSONExtractString(attributes, 'provider'), NULL) AS provider,
+      startedAt,
+      endedAt,
+      dateDiff('millisecond', startedAt, endedAt) AS durationMs,
+      if(isNotNull(error), 'error', 'success') AS status,
+      error,
+      entityType,
+      entityId,
+      entityName,
+      entityVersionId,
+      parentEntityVersionId,
+      rootEntityVersionId
     FROM ${TABLE_SPAN_EVENTS}
-    WHERE traceId IN (SELECT traceId FROM root_scope)
+    WHERE isNotNull(traceId)
+      AND traceId IN (SELECT traceId FROM root_scope)
     ORDER BY dedupeKey
     LIMIT 1 BY dedupeKey
   )`);
 	if (relationCollections.has("scores")) ctes.push(`current_scores AS (
-    SELECT traceId, scorerId, score
+    SELECT
+      traceId,
+      spanId,
+      timestamp,
+      scorerId,
+      scorerVersion,
+      scoreSource,
+      score,
+      entityVersionId,
+      parentEntityVersionId,
+      rootEntityVersionId
     FROM ${TABLE_SCORE_EVENTS}
     WHERE isNotNull(traceId)
       AND traceId IN (SELECT traceId FROM root_scope)
+    ORDER BY scoreId, timestamp DESC
+    LIMIT 1 BY scoreId
   )`);
 	const predicate = plan.where ? compilePredicate(plan.where, parameters) : "1";
 	ctes.push(`candidates AS (
@@ -7806,6 +8038,19 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async deleteScores(args) {
+		try {
+			await deleteScores(this.#client, args, this.#replication);
+		} catch (error) {
+			if (error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "DELETE_SCORES", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { count: args.scoreIds.length }
+			}, error);
+		}
+	}
 	async getScoreById(scoreId) {
 		try {
 			return await getScoreById(this.#client, scoreId);
@@ -7844,9 +8089,22 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async deleteFeedback(args) {
+		try {
+			await deleteFeedback(this.#client, args, this.#replication);
+		} catch (error) {
+			if (error instanceof MastraError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "DELETE_FEEDBACK", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY,
+				details: { count: args.feedbackIds.length }
+			}, error);
+		}
+	}
 	async updateFeedbackReviewStatus(args) {
 		try {
-			return await updateFeedbackReviewStatus(this.#client, args);
+			return await updateFeedbackReviewStatus(this.#client, args, this.#replication);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({