@mastra/clickhouse 1.14.0 → 1.15.0-alpha.1

package/dist/index.js

@@ -1,6 +1,6 @@
 import { createClient } from "@clickhouse/client";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
-import { BRANCH_SPAN_TYPES, BackgroundTasksStorage, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, calculatePagination, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
+import { BRANCH_SPAN_TYPES, BackgroundTasksStorage, EntityType, METRIC_DISTINCT_COLUMNS, MastraCompositeStore, MemoryStorage, ObservabilityStorage, SCORERS_SCHEMA, SPAN_SCHEMA, ScoresStorage, TABLE_AGENT_VERSIONS, TABLE_BACKGROUND_TASKS, TABLE_DATASETS, TABLE_DATASET_ITEMS, TABLE_DATASET_VERSIONS, TABLE_EXPERIMENTS, TABLE_EXPERIMENT_RESULTS, TABLE_FAVORITES, TABLE_HARNESS_SESSIONS, TABLE_MCP_CLIENTS, TABLE_MCP_CLIENT_VERSIONS, TABLE_MCP_SERVERS, TABLE_MCP_SERVER_VERSIONS, TABLE_MESSAGES, TABLE_NOTIFICATIONS, TABLE_PROMPT_BLOCKS, TABLE_PROMPT_BLOCK_VERSIONS, TABLE_RESOURCES, TABLE_SCHEDULES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEMAS, TABLE_SCORERS, TABLE_SCORER_DEFINITIONS, TABLE_SCORER_DEFINITION_VERSIONS, TABLE_SKILLS, TABLE_SKILL_BLOBS, TABLE_SKILL_VERSIONS, TABLE_SPANS, TABLE_THREADS, TABLE_THREAD_STATE, TABLE_TOOL_PROVIDER_CONNECTIONS, TABLE_TRACES, TABLE_WORKFLOW_DEFINITIONS, TABLE_WORKFLOW_SNAPSHOT, TABLE_WORKSPACES, TABLE_WORKSPACE_VERSIONS, TraceStatus, WorkflowsStorage, buildInputPreview, calculatePagination, computeTraceStatus, createStorageErrorId, getDefaultValue, getSqlType, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, normalizePerPage, safelyParseJSON, toTraceSpans, transformScoreRow, validateStorageMetadataFilter } from "@mastra/core/storage";
 import { MastraBase } from "@mastra/core/base";
 import { MessageList } from "@mastra/core/agent";
 import { parseFieldKey } from "@mastra/core/utils";
@@ -1184,7 +1184,10 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 				hasMore: false
 			};
 			if (perPageForQuery === 0 && include && include.length > 0) {
-				const includeResult = await this._getIncludedMessages({ include });
+				const includeResult = await this._getIncludedMessages({
+					include,
+					resourceId
+				});
 				const list = new MessageList().add(includeResult, "memory");
 				return {
 					messages: this._sortMessages(list.get.all.db(), field, direction),
@@ -1251,7 +1254,10 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 			};
 			const messageIds = new Set(paginatedMessages.map((m) => m.id));
 			if (include && include.length > 0) {
-				const includeMessages = await this._getIncludedMessages({ include });
+				const includeMessages = await this._getIncludedMessages({
+					include,
+					resourceId
+				});
 				for (const includeMsg of includeMessages) if (!messageIds.has(includeMsg.id)) {
 					paginatedMessages.push(includeMsg);
 					messageIds.add(includeMsg.id);
@@ -1269,6 +1275,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 				hasMore: metadataFilter && perPageForResponse !== false ? offset + paginatedCount < total : perPageForResponse === false ? false : allThreadMessagesReturned ? false : offset + paginatedCount < total
 			};
 		} catch (error) {
+			if (error instanceof MastraError && error.category === ErrorCategory.USER) throw error;
 			const mastraError = new MastraError({
 				id: createStorageErrorId("CLICKHOUSE", "LIST_MESSAGES", "FAILED"),
 				domain: ErrorDomain.STORAGE,
@@ -1280,13 +1287,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 			}, error);
 			this.logger?.error?.(mastraError.toString());
 			this.logger?.trackException?.(mastraError);
-			return {
-				messages: [],
-				total: 0,
-				page,
-				perPage: perPageForResponse,
-				hasMore: false
-			};
+			throw mastraError;
 		}
 	}
 	_sortMessages(messages, field, direction) {
@@ -1299,15 +1300,24 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 			return direction === "ASC" ? String(aValue).localeCompare(String(bValue)) : String(bValue).localeCompare(String(aValue));
 		});
 	}
-	async _getIncludedMessages({ include }) {
+	/**
+	* Fetches the messages named by `include` together with their surrounding context.
+	*
+	* @param include - Message ids to pin, each with an optional before/after window.
+	* @param resourceId - When set, restricts both the pinned messages and their context
+	* to that resource so an id from another resource returns nothing.
+	*/
+	async _getIncludedMessages({ include, resourceId }) {
 		if (!include || include.length === 0) return [];
 		const targetIds = include.map((inc) => inc.id).filter(Boolean);
 		if (targetIds.length === 0) return [];
 		const { messages: targetDocs } = await this.listMessagesById({ messageIds: targetIds });
-		const targetMap = new Map(targetDocs.map((msg) => [msg.id, {
+		const scopedTargetDocs = resourceId ? targetDocs.filter((msg) => msg.resourceId === resourceId) : targetDocs;
+		const targetMap = new Map(scopedTargetDocs.map((msg) => [msg.id, {
 			threadId: msg.threadId,
 			createdAt: msg.createdAt
 		}]));
+		const resourceCondition = resourceId ? ` AND "resourceId" = {var_resource:String}` : "";
 		const unionQueries = [];
 		const params = {};
 		let paramIdx = 1;
@@ -1322,7 +1332,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
         SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId"
         FROM "${TABLE_MESSAGES}"
         WHERE thread_id = {${threadParam}:String}
-          AND createdAt <= parseDateTime64BestEffort({${createdAtParam}:String}, 3)
+          AND createdAt <= parseDateTime64BestEffort({${createdAtParam}:String}, 3)${resourceCondition}
         ORDER BY createdAt DESC, id DESC
         LIMIT {${limitParam}:Int64}
       `);
@@ -1338,7 +1348,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
           SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId"
           FROM "${TABLE_MESSAGES}"
           WHERE thread_id = {${threadParam2}:String}
-            AND createdAt > parseDateTime64BestEffort({${createdAtParam2}:String}, 3)
+            AND createdAt > parseDateTime64BestEffort({${createdAtParam2}:String}, 3)${resourceCondition}
           ORDER BY createdAt ASC, id ASC
           LIMIT {${limitParam2}:Int64}
         `);
@@ -1349,6 +1359,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
 			}
 		}
 		if (unionQueries.length === 0) return [];
+		if (resourceId) params.var_resource = resourceId;
 		let finalQuery;
 		if (unionQueries.length === 1) finalQuery = unionQueries[0];
 		else finalQuery = `SELECT * FROM (${unionQueries.join(" UNION ALL ")}) ORDER BY "createdAt" ASC, id ASC`;
@@ -3872,6 +3883,7 @@ function rowToSpanRecord(row) {
 function rowToLightSpanRecord(row) {
 	const startedAt = toDate(row.startedAt);
 	const endedAt = row.isEvent ? startedAt : toDateOrNull(row.endedAt);
+	const error = parseJson(row.error) ?? void 0;
 	return {
 		traceId: row.traceId,
 		spanId: row.spanId,
@@ -3884,7 +3896,13 @@ function rowToLightSpanRecord(row) {
 		entityType: nullableEntityType(row.entityType),
 		entityId: nullableString(row.entityId),
 		entityName: nullableString(row.entityName),
-		error: parseJson(row.error) ?? void 0,
+		error,
+		status: computeTraceStatus({
+			error,
+			endedAt
+		}),
+		metadata: parseJson(row.metadataRaw) ?? void 0,
+		inputPreview: buildInputPreview(row.input ?? null),
 		createdAt: startedAt,
 		updatedAt: null
 	};
@@ -6207,93 +6225,48 @@ async function getRootSpan(client, args) {
 	return { span: rowToSpanRecord(rows[0]) };
 }
 /**
-* Shared page-mode helper used by listTracesLight.
+* Columns a trace list renders. `input` is selected only so the row mapper can
+* derive a short `inputPreview` at read time (see `rowToLightSpanRecord`); the
+* raw blob itself never leaves the store.
 */
-async function listTraceRows(client, args, selectClause, mapRows) {
-	const { filters, pagination, orderBy } = listTracesArgsSchema.parse(args);
-	const page = pagination?.page ?? 0;
-	const perPage = pagination?.perPage ?? 10;
-	const { conditions, params } = buildTraceFilterConditions(filters, "r");
-	if (filters?.hasChildError != null) if (filters.hasChildError) conditions.push(`EXISTS (
-        SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
-        WHERE c.traceId = r.traceId
-          AND c.parentSpanId IS NOT NULL
-          AND c.error IS NOT NULL
-      )`);
-	else conditions.push(`NOT EXISTS (
-        SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
-        WHERE c.traceId = r.traceId
-          AND c.parentSpanId IS NOT NULL
-          AND c.error IS NOT NULL
-      )`);
-	const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-	const orderClause = buildTraceOrderByClause(orderBy);
-	const countRows = await (await client.query({
-		query: `
-      SELECT count() as cnt FROM (
-        SELECT dedupeKey
-        FROM ${TABLE_TRACE_ROOTS} r
-        ${whereClause}
-        ORDER BY dedupeKey
-        LIMIT 1 BY dedupeKey
-      )
-    `,
-		query_params: params,
-		format: "JSONEachRow",
-		clickhouse_settings: CH_SETTINGS
-	})).json();
-	const total = Number(countRows[0]?.cnt ?? 0);
-	if (total === 0) return {
-		pagination: {
-			total: 0,
-			page,
-			perPage,
-			hasMore: false
-		},
-		spans: []
-	};
-	const rows = await (await client.query({
-		query: `
-      SELECT ${selectClause} FROM (
-        SELECT ${selectClause}
-        FROM ${TABLE_TRACE_ROOTS} r
-        ${whereClause}
-        ORDER BY dedupeKey
-        LIMIT 1 BY dedupeKey
-      )
-      ORDER BY ${orderClause}
-      LIMIT {limit:UInt32}
-      OFFSET {offset:UInt32}
-    `,
-		query_params: {
-			...params,
-			limit: perPage,
-			offset: page * perPage
-		},
-		format: "JSONEachRow",
-		clickhouse_settings: CH_SETTINGS
-	})).json();
-	return {
-		pagination: {
-			total,
-			page,
-			perPage,
-			hasMore: (page + 1) * perPage < total
-		},
-		spans: mapRows(rows)
-	};
-}
+const LIGHT_TRACE_ROOT_FIELDS = [
+	"traceId",
+	"spanId",
+	"parentSpanId",
+	"name",
+	"spanType",
+	"isEvent",
+	"startedAt",
+	"endedAt",
+	"entityType",
+	"entityId",
+	"entityName",
+	"error",
+	"metadataRaw",
+	"input"
+];
+const FULL_PROJECTION = {
+	innerSelect: "*",
+	outerSelect: "*",
+	deltaSelect: "r.* EXCEPT(startedAt, traceId, dedupeKey)"
+};
+const LIGHT_PROJECTION = {
+	innerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
+	outerSelect: LIGHT_TRACE_ROOT_FIELDS.join(", "),
+	deltaSelect: LIGHT_TRACE_ROOT_FIELDS.filter((field) => field !== "startedAt" && field !== "traceId").map((field) => `r.${field}`).join(", ")
+};
 /**
-* List traces with optional filtering, pagination, and ordering.
+* Shared implementation behind listTraces and listTracesLight.
 *
 * Reads from trace_roots (root spans only).
-* Uses two-stage query for ReplacingMergeTree deduplication:
+* Page mode uses a two-stage query for ReplacingMergeTree deduplication:
 *   Inner: filter + deterministic ORDER BY + LIMIT 1 BY dedupeKey
 *   Outer: final ordering + pagination
+* Delta mode joins the append-only delta table and returns only rows past the cursor.
 *
 * hasChildError is handled via EXISTS subquery against span_events.
 */
-async function listTraces(client, args, strategy) {
+async function listTraceRows(client, args, strategy, projection, mapRows) {
 	const { mode, filters, pagination, orderBy, after, limit } = listTracesArgsSchema.parse(args);
 	const page = pagination?.page ?? 0;
 	const perPage = pagination?.perPage ?? 10;
@@ -6323,10 +6296,11 @@ async function listTraces(client, args, strategy) {
 			},
 			deltaCursor: streamHeadCursor
 		};
-		const rows = await queryTracesAfterCursor(client, whereClause, params, limit, validateCursorId(after));
+		const afterCursor = validateCursorId(after);
+		const rows = await queryTracesAfterCursor(client, projection.deltaSelect, whereClause, params, limit, afterCursor);
 		const visibleRows = rows.slice(0, limit);
 		return {
-			spans: toTraceSpans(visibleRows.map(rowToSpanRecord)),
+			spans: mapRows(visibleRows),
 			delta: {
 				limit,
 				hasMore: rows.length > limit
@@ -6361,10 +6335,10 @@ async function listTraces(client, args, strategy) {
 		spans: [],
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
-	const spans = (await (await client.query({
+	const rows = await (await client.query({
 		query: `
-      SELECT * FROM (
-        SELECT *
+      SELECT ${projection.outerSelect} FROM (
+        SELECT ${projection.innerSelect}
         FROM ${TABLE_TRACE_ROOTS} r
         ${whereClause}
         ORDER BY dedupeKey
@@ -6381,7 +6355,7 @@ async function listTraces(client, args, strategy) {
 		},
 		format: "JSONEachRow",
 		clickhouse_settings: CH_SETTINGS
-	})).json()).map(rowToSpanRecord);
+	})).json();
 	return {
 		pagination: {
 			total,
@@ -6389,32 +6363,27 @@ async function listTraces(client, args, strategy) {
 			perPage,
 			hasMore: (page + 1) * perPage < total
 		},
-		spans: toTraceSpans(spans),
+		spans: mapRows(rows),
 		...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
 	};
 }
-const LIGHT_TRACE_ROOT_COLUMNS = [
-	"traceId",
-	"spanId",
-	"parentSpanId",
-	"name",
-	"spanType",
-	"isEvent",
-	"startedAt",
-	"endedAt",
-	"entityType",
-	"entityId",
-	"entityName",
-	"error"
-].join(", ");
-async function listTracesLight(client, args) {
-	return listTraceRows(client, args, LIGHT_TRACE_ROOT_COLUMNS, (rows) => rows.map(rowToLightSpanRecord));
+/** List traces with optional filtering, pagination, and ordering. */
+async function listTraces(client, args, strategy) {
+	return listTraceRows(client, args, strategy, FULL_PROJECTION, (rows) => toTraceSpans(rows.map(rowToSpanRecord)));
+}
+/**
+* List traces projecting only the columns a trace list renders.
+* Skips the attributes/output blobs and reduces `input` to a short preview in
+* the mapper, so the response payload stays flat as traces grow.
+*/
+async function listTracesLight(client, args, strategy) {
+	return listTraceRows(client, args, strategy, LIGHT_PROJECTION, (rows) => rows.map(rowToLightSpanRecord));
 }
-async function queryTracesAfterCursor(client, whereClause, params, limit, cursorId) {
+async function queryTracesAfterCursor(client, deltaSelect, whereClause, params, limit, cursorId) {
 	return await (await client.query({
 		query: `
         SELECT
-          r.* EXCEPT(startedAt, traceId, dedupeKey),
+          ${deltaSelect},
           r.startedAt AS startedAt,
           r.traceId AS traceId,
           r.dedupeKey AS dedupeKey,
@@ -7391,7 +7360,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 	}
 	async listTracesLight(args) {
 		try {
-			return await listTracesLight(this.#client, args);
+			return await listTracesLight(this.#client, args, this.#deltaCursorStrategy);
 		} catch (error) {
 			if (error instanceof MastraError) throw error;
 			throw new MastraError({