@mastra/clickhouse 1.14.0 → 1.15.0-alpha.0

package/dist/index.js

@@ -1,6 +1,6 @@
 import { createClient } from "@clickhouse/client";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
-import { BRANCH_SPAN_TYPES, BackgroundTasksStorage, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, calculatePagination, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
+import { BRANCH_SPAN_TYPES, BackgroundTasksStorage, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, buildInputPreview, calculatePagination, computeTraceStatus, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MastraBase } from "@mastra/core/base";
 import { MessageList } from "@mastra/core/agent";
 import { parseFieldKey } from "@mastra/core/utils";
@@ -1269,6 +1269,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 				hasMore: metadataFilter && perPageForResponse !== false ? offset + paginatedCount < total : perPageForResponse === false ? false : allThreadMessagesReturned ? false : offset + paginatedCount < total
 			};
 		} catch (error) {
+			if (error instanceof MastraError && error.category === ErrorCategory.USER) throw error;
 			const mastraError = new MastraError({
 				id: createStorageErrorId("CLICKHOUSE", "LIST_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1280,13 +1281,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 			}, error);
 			this.logger?.error?.(mastraError.toString());
 			this.logger?.trackException?.(mastraError);
-			return {
-				messages: [],
-				total: 0,
-				page,
-				perPage: perPageForResponse,
-				hasMore: false
-			};
+			throw mastraError;
 		}
 	}
 	_sortMessages(messages, field, direction) {
@@ -3872,6 +3867,7 @@ function rowToSpanRecord(row) {
 function rowToLightSpanRecord(row) {
 	const startedAt = toDate(row.startedAt);
 	const endedAt = row.isEvent ? startedAt : toDateOrNull(row.endedAt);
+	const error = parseJson(row.error) ?? void 0;
 	return {
 		traceId: row.traceId,
 		spanId: row.spanId,
@@ -3884,7 +3880,13 @@ function rowToLightSpanRecord(row) {
 		entityType: nullableEntityType(row.entityType),
 		entityId: nullableString(row.entityId),
 		entityName: nullableString(row.entityName),
-		error: parseJson(row.error) ?? void 0,
+		error,
+		status: computeTraceStatus({
+			error,
+			endedAt
+		}),
+		metadata: parseJson(row.metadataRaw) ?? void 0,
+		inputPreview: buildInputPreview(row.input ?? null),
 		createdAt: startedAt,
 		updatedAt: null
 	};
@@ -6207,93 +6209,48 @@ async function getRootSpan(client, args) {
 	return { span: rowToSpanRecord(rows[0]) };
 }
 /**
-* Shared page-mode helper used by listTracesLight.
+* Columns a trace list renders. `input` is selected only so the row mapper can
+* derive a short `inputPreview` at read time (see `rowToLightSpanRecord`); the
+* raw blob itself never leaves the store.
 */
-async function listTraceRows(client, args, selectClause, mapRows) {
-	const { filters, pagination, orderBy } = listTracesArgsSchema.parse(args);
-	const page = pagination?.page ?? 0;
-	const perPage = pagination?.perPage ?? 10;
-	const { conditions, params } = buildTraceFilterConditions(filters, "r");
-	if (filters?.hasChildError != null) if (filters.hasChildError) conditions.push(`EXISTS (
-        SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
-        WHERE c.traceId = r.traceId
-          AND c.parentSpanId IS NOT NULL
-          AND c.error IS NOT NULL
-      )`);
-	else conditions.push(`NOT EXISTS (
-        SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
-        WHERE c.traceId = r.traceId
-          AND c.parentSpanId IS NOT NULL
-          AND c.error IS NOT NULL
-      )`);
-	const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-	const orderClause = buildTraceOrderByClause(orderBy);
-	const countRows = await (await client.query({
-		query: `
-      SELECT count() as cnt FROM (
-        SELECT dedupeKey
-        FROM ${TABLE_TRACE_ROOTS} r
-        ${whereClause}
-        ORDER BY dedupeKey
-        LIMIT 1 BY dedupeKey
-      )
-    `,
-		query_params: params,
-		format: "JSONEachRow",
-		clickhouse_settings: CH_SETTINGS
-	})).json();
-	const total = Number(countRows[0]?.cnt ?? 0);
-	if (total === 0) return {
-		pagination: {
-			total: 0,
-			page,
-			perPage,
-			hasMore: false
-		},
-		spans: []
-	};
-	const rows = await (await client.query({
-		query: `
-      SELECT ${selectClause} FROM (
-        SELECT ${selectClause}
-        FROM ${TABLE_TRACE_ROOTS} r
-        ${whereClause}
-        ORDER BY dedupeKey
-        LIMIT 1 BY dedupeKey
-      )
-      ORDER BY ${orderClause}
-      LIMIT {limit:UInt32}
-      OFFSET {offset:UInt32}
-    `,
-		query_params: {
-			...params,
-			limit: perPage,
-			offset: page * perPage
-		},
-		format: "JSONEachRow",
-		clickhouse_settings: CH_SETTINGS
-	})).json();
-	return {
-		pagination: {
-			total,
-			page,
-			perPage,
-			hasMore: (page + 1) * perPage < total
-		},
-		spans: mapRows(rows)
-	};
-}
+const LIGHT_TRACE_ROOT_FIELDS = [
+	"traceId",
+	"spanId",
+	"parentSpanId",
+	"name",
+	"spanType",
+	"isEvent",
+	"startedAt",
+	"endedAt",
+	"entityType",
+	"entityId",
+	"entityName",
+	"error",
+	"metadataRaw",
+	"input"
+];
+const FULL_PROJECTION = {
+	innerSelect: "*",
+	outerSelect: "*",
+	deltaSelect: "r.* EXCEPT(startedAt, traceId, dedupeKey)"
+};
+const LIGHT_PROJECTION = {
+	innerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
+	outerSelect: LIGHT_TRACE_ROOT_FIELDS.join(", "),
+	deltaSelect: LIGHT_TRACE_ROOT_FIELDS.filter((field) => field !== "startedAt" && field !== "traceId").map((field) => `r.${field}`).join(", ")
+};
 /**
-* List traces with optional filtering, pagination, and ordering.
+* Shared implementation behind listTraces and listTracesLight.
 *
 * Reads from trace_roots (root spans only).
-* Uses two-stage query for ReplacingMergeTree deduplication:
+* Page mode uses a two-stage query for ReplacingMergeTree deduplication:
 *   Inner: filter + deterministic ORDER BY + LIMIT 1 BY dedupeKey
 *   Outer: final ordering + pagination
+* Delta mode joins the append-only delta table and returns only rows past the cursor.
 *
 * hasChildError is handled via EXISTS subquery against span_events.
 */
-async function listTraces(client, args, strategy) {
+async function listTraceRows(client, args, strategy, projection, mapRows) {
 	const { mode, filters, pagination, orderBy, after, limit } = listTracesArgsSchema.parse(args);
 	const page = pagination?.page ?? 0;
 	const perPage = pagination?.perPage ?? 10;
@@ -6323,10 +6280,11 @@ async function listTraces(client, args, strategy) {
 			},
 			deltaCursor: streamHeadCursor
 		};
-		const rows = await queryTracesAfterCursor(client, whereClause, params, limit, validateCursorId(after));
+		const afterCursor = validateCursorId(after);
+		const rows = await queryTracesAfterCursor(client, projection.deltaSelect, whereClause, params, limit, afterCursor);
 		const visibleRows = rows.slice(0, limit);
 		return {
-			spans: toTraceSpans(visibleRows.map(rowToSpanRecord)),
+			spans: mapRows(visibleRows),
 			delta: {
 				limit,
 				hasMore: rows.length > limit
@@ -6361,10 +6319,10 @@ async function listTraces(client, args, strategy) {
 		spans: [],
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
-	const spans = (await (await client.query({
+	const rows = await (await client.query({
 		query: `
-      SELECT * FROM (
-        SELECT *
+      SELECT ${projection.outerSelect} FROM (
+        SELECT ${projection.innerSelect}
         FROM ${TABLE_TRACE_ROOTS} r
         ${whereClause}
         ORDER BY dedupeKey
@@ -6381,7 +6339,7 @@ async function listTraces(client, args, strategy) {
 		},
 		format: "JSONEachRow",
 		clickhouse_settings: CH_SETTINGS
-	})).json()).map(rowToSpanRecord);
+	})).json();
 	return {
 		pagination: {
 			total,
@@ -6389,32 +6347,27 @@ async function listTraces(client, args, strategy) {
 			perPage,
 			hasMore: (page + 1) * perPage < total
 		},
-		spans: toTraceSpans(spans),
+		spans: mapRows(rows),
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
 }
-const LIGHT_TRACE_ROOT_COLUMNS = [
-	"traceId",
-	"spanId",
-	"parentSpanId",
-	"name",
-	"spanType",
-	"isEvent",
-	"startedAt",
-	"endedAt",
-	"entityType",
-	"entityId",
-	"entityName",
-	"error"
-].join(", ");
-async function listTracesLight(client, args) {
-	return listTraceRows(client, args, LIGHT_TRACE_ROOT_COLUMNS, (rows) => rows.map(rowToLightSpanRecord));
+/** List traces with optional filtering, pagination, and ordering. */
+async function listTraces(client, args, strategy) {
+	return listTraceRows(client, args, strategy, FULL_PROJECTION, (rows) => toTraceSpans(rows.map(rowToSpanRecord)));
+}
+/**
+* List traces projecting only the columns a trace list renders.
+* Skips the attributes/output blobs and reduces `input` to a short preview in
+* the mapper, so the response payload stays flat as traces grow.
+*/
+async function listTracesLight(client, args, strategy) {
+	return listTraceRows(client, args, strategy, LIGHT_PROJECTION, (rows) => rows.map(rowToLightSpanRecord));
 }
-async function queryTracesAfterCursor(client, whereClause, params, limit, cursorId) {
+async function queryTracesAfterCursor(client, deltaSelect, whereClause, params, limit, cursorId) {
 	return await (await client.query({
 		query: `
         SELECT
-          r.* EXCEPT(startedAt, traceId, dedupeKey),
+          ${deltaSelect},
           r.startedAt AS startedAt,
           r.traceId AS traceId,
           r.dedupeKey AS dedupeKey,
@@ -7391,7 +7344,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	}
 	async listTracesLight(args) {
 		try {
-			return await listTracesLight(this.#client, args);
+			return await listTracesLight(this.#client, args, this.#deltaCursorStrategy);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({