@mastra/clickhouse 1.26.0-alpha.4 → 1.26.0-alpha.5

package/dist/index.js

@@ -6901,6 +6901,10 @@ const TRACE_FIELDS = {
 		sql: "r.timeToFirstChunkMs",
 		parameterType: "Float64"
 	},
+	entityId: {
+		sql: "r.entityId",
+		parameterType: "String"
+	},
 	entityName: {
 		sql: "r.entityName",
 		parameterType: "String"
@@ -7299,6 +7303,28 @@ function compileTenantScope(scope, parameters) {
 	if (scope.resourceId !== void 0) sql += `\n      AND resourceId = ${parameters.add(scope.resourceId, "String")}`;
 	return sql;
 }
+/**
+* Order of the versions of one root span, current version first. `mastra_trace_roots` has no
+* version column, so until a background merge runs, every export of a root is its own row with the
+* same sort key, and merges never meet versions that ended on different days (it is partitioned by
+* `endedAt`). The version that ended last is current, as a resumed or completed root is exported
+* again with a later `endedAt`; a pending copy stores `endedAt = startedAt` and can lack `threadId`,
+* so a copy with a thread wins an equal-time tie. Remaining ties are broken by insertion order, last
+* insert first: the part's first block number, then the row's offset in the part. Tied versions
+* share a partition and sort key, so a merge meets them and keeps the last insert too, and reads
+* pick the same version before and after it. Both are virtual columns, so the order reads nothing
+* beyond `endedAt` and `threadId`.
+*
+* Merges don't follow this order across `endedAt`: within one day's partition a merge keeps the last
+* insert, not the version that ended last. If an outdated export of a root is redelivered after its
+* current version, on the same day, reads pick the current version until a merge, and the outdated
+* one afterwards (last write wins, as in the PG and DuckDB stores). Making merges keep the version
+* that ended last needs `ReplacingMergeTree(endedAt)`, a schema change.
+*
+* Every root collapse uses this order: trace rows, aggregates, thread rows and the discovery reads.
+*/
+const ROOT_VERSION_ORDER = `endedAt DESC, isNotNull(threadId) DESC, toUInt64(splitByChar('_', _part)[2]) DESC,
+      _part_offset DESC`;
 function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = [], options = {}) {
 	const from = parameters.add(selection.timeRange.from, "DateTime64(3, 'UTC')");
 	const to = parameters.add(selection.timeRange.to, "DateTime64(3, 'UTC')");
@@ -7322,7 +7348,15 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
     SELECT *
     FROM ${TABLE_TRACE_ROOTS} r
     WHERE startedAt >= ${from}
-      AND startedAt < ${to}${tenant}${seedFilter}${bound}
+      AND startedAt < ${to}${tenant}${bound}${seedFilter ? `
+      AND traceId IN (
+        SELECT traceId
+        FROM ${TABLE_TRACE_ROOTS} r
+        WHERE startedAt >= ${from}
+          AND startedAt < ${to}${tenant}${seedFilter}
+      )` : ""}
+    ORDER BY startedAt, traceId, spanId, ${ROOT_VERSION_ORDER}
+    LIMIT 1 BY traceId
   )`] : [`current_roots AS (
     SELECT *
     FROM ${TABLE_TRACE_ROOTS}
@@ -7332,7 +7366,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
       WHERE startedAt >= ${from}
         AND startedAt < ${to}${tenant}${seedFilter}
     )${tenant}
-    ORDER BY traceId, dedupeKey
+    ORDER BY traceId, startedAt, spanId, ${ROOT_VERSION_ORDER}
     LIMIT 1 BY traceId
   )`, `root_scope AS (
     SELECT *
@@ -7462,11 +7496,7 @@ function traceOrder(plan) {
 		direction: plan.orderBy.direction === "asc" ? "ASC" : "DESC"
 	};
 }
-/**
-* Keyset cursor on a trace's sort value. Applied after each trace is reduced to one sort value:
-* filtering root rows first would let a trace with several roots (or unmerged copies with
-* different `endedAt`) come back on a later page under a root that sorts after the cursor.
-*/
+/** Keyset cursor on a trace's sort value (the current root's), with traceId breaking ties. */
 function keysetCursorCondition(plan, order, parameters) {
 	if (plan.paginationMode !== "keyset" || !plan.cursor) return "";
 	if (plan.cursor.sortValue === null) return `(isNull(sortValue) AND traceId > ${parameters.add(plan.cursor.traceId, "String")})`;
@@ -7477,9 +7507,8 @@ function keysetCursorCondition(plan, order, parameters) {
 }
 /**
 * First statement of keyset and page mode: the page's traces (after the keyset cursor, `limit + 1`
-* for keyset mode, `perPage` at the page offset for page mode) as root keys, and the
-* number of matching traces on every row. Per trace, the root row that sorts first gives the sort
-* value and the key, so the key always names a row with that sort value. No blob columns are read.
+* for keyset mode, `perPage` at the page offset for page mode) as keys of their current roots, and
+* the number of matching traces on every row. No blob columns are read.
 */
 function compileClickHouseTraceQueryKeys(plan) {
 	if (plan.paginationMode === "delta") throw new Error("Delta trace queries have no keys statement");
@@ -7487,19 +7516,12 @@ function compileClickHouseTraceQueryKeys(plan) {
 	const order = traceOrder(plan);
 	const ctes = compileClickHouseTraceCandidates(plan, "r.traceId AS traceId, r.dedupeKey AS dedupeKey, r.startedAt AS startedAt, r.endedAt AS endedAt, r.timeToFirstChunkMs AS timeToFirstChunkMs", parameters, QUERY_TRACES_SCOPE);
 	const cursor = keysetCursorCondition(plan, order, parameters);
-	const pick = order.direction === "DESC" ? "max" : "min";
-	const rootOrder = order.field === "timeToFirstChunkMs" ? `ifNull(${order.field}, ${pick === "max" ? "-inf" : "inf"})` : order.field;
 	const limit = plan.paginationMode === "page" ? `LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}` : `LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`;
 	return {
-		query: `WITH ${ctes.join(",\n")},
-per_trace AS (
-  SELECT traceId, ${pick}(${order.field}) AS sortValue,
-    ${pick === "max" ? "argMax" : "argMin"}((dedupeKey, startedAt), ${rootOrder}) AS root
-  FROM candidates
-  GROUP BY traceId${cursor ? `\n  HAVING ${cursor}` : ""}
-)
-SELECT traceId, root.1 AS dedupeKey, sortValue, root.2 AS rootStartedAt, count() OVER () AS __total
-FROM per_trace
+		query: `WITH ${ctes.join(",\n")}
+SELECT traceId, dedupeKey, ${order.field} AS sortValue, startedAt AS rootStartedAt, endedAt AS rootEndedAt,
+  count() OVER () AS __total
+FROM candidates${cursor ? `\nWHERE ${cursor}` : ""}
 ORDER BY sortValue ${order.direction} NULLS LAST, traceId ASC
 ${limit}`,
 		query_params: parameters.params,
@@ -7507,27 +7529,28 @@ ${limit}`,
 	};
 }
 /**
-* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata`
-* and the stored previews, read by sort-key tuple. Unmerged copies of a root key are redeliveries of one
-* span; the relation-free `where` conjuncts are re-applied so a copy that doesn't match isn't
-* returned, and the copy that sorts first is kept, as in the keys statement.
+* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata` and the
+* stored previews, read by sort key and `endedAt`, so only the keyed version's rows match. Versions
+* that end at the same instant are ordered as in the keys statement (ROOT_VERSION_ORDER). The
+* relation-free `where` conjuncts are re-applied, so a version written between the two statements
+* that doesn't match isn't returned.
+*
+* Trade-off of matching on `endedAt`: if a newer version is written between the two statements and
+* a same-day merge then drops the keyed version, the trace has no row and drops off the page. Without
+* `endedAt` the lookup would find the replacement, but could also return a version other than the
+* keyed one whenever both are unmerged. The window is the gap between the two statements.
 */
 function compileClickHouseTraceRowsByKey(plan, keys) {
 	const parameters = new ParameterBuilder();
 	const order = traceOrder(plan);
 	const ts = (value) => parameters.add(value, "DateTime64(3, 'UTC')");
-	const byEnd = order.field === "endedAt";
-	const tuples = keys.map((key) => {
-		const started = ts(key.rootStartedAt);
-		const id = `${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}`;
-		return byEnd ? `(${started}, ${id}, ${ts(String(key.sortValue))})` : `(${started}, ${id})`;
-	});
+	const tuples = keys.map((key) => `(${ts(key.rootStartedAt)}, ${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}, ${ts(key.rootEndedAt)})`);
 	const predicate = (plan.where ? (plan.where.type === "boolean" && plan.where.operator === "and" ? plan.where.args : [plan.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : []).map((conjunct) => `\n  AND (${compilePredicate(conjunct, parameters)})`).join("");
 	return {
 		query: `SELECT ${TRACE_SELECT}
 FROM ${TABLE_TRACE_ROOTS} r
-WHERE (r.startedAt, r.traceId, r.dedupeKey${byEnd ? ", r.endedAt" : ""}) IN (${tuples.join(", ")})${compileTenantScope(plan.scope, parameters)}${predicate}
-ORDER BY ${order.field} ${order.direction} NULLS LAST, traceId ASC
+WHERE (r.startedAt, r.traceId, r.dedupeKey, r.endedAt) IN (${tuples.join(", ")})${compileTenantScope(plan.scope, parameters)}${predicate}
+ORDER BY ${order.field} ${order.direction} NULLS LAST, traceId ASC, ${ROOT_VERSION_ORDER}
 LIMIT 1 BY traceId`,
 		query_params: parameters.params
 	};
@@ -7546,9 +7569,7 @@ FROM candidates`,
 /**
 * The list statement of groups and delta mode (rows without `metadata` and the previews). Keyset
 * and page mode list with compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey.
-* A trace with several candidate rows (several roots, or unmerged copies of one) is listed in delta
-* mode under the row that sorts first in the list order, as in the keys statement; ties go to the
-* lowest dedupeKey, then the latest endedAt, so every poll picks the same row.
+* Candidates hold one current root per trace, as in the keys statement.
 */
 function compileClickHouseTraceQuery(plan, deltaHead) {
 	const parameters = new ParameterBuilder();
@@ -7569,7 +7590,6 @@ LIMIT ${limit}`,
 		};
 	}
 	if (plan.paginationMode !== "delta") throw new Error("Keyset and page trace queries use compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey");
-	const order = traceOrder(plan);
 	const watermark = coreStorage.getTraceQueryDeltaWatermark(plan, "clickhouse");
 	const after = watermark ? parseDeltaWatermark(watermark) : {
 		cursorId: "0",
@@ -7588,8 +7608,7 @@ LIMIT ${limit}`,
 SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
 FROM candidates c
 INNER JOIN delta_candidates d ON c.traceId = d.traceId
-ORDER BY d.latestCursorId ASC, c.traceId ASC, c.${order.field} ${order.direction}, c.dedupeKey ASC, c.endedAt DESC
-LIMIT 1 BY c.traceId
+ORDER BY d.latestCursorId ASC, c.traceId ASC
 LIMIT ${limit}`,
 		query_params: parameters.params,
 		sharedSnapshot: true
@@ -7633,6 +7652,7 @@ function compileClickHouseTraceRootPayloads(keys, scope, columns = "payload") {
 		query: `SELECT traceId, spanId AS rootSpanId, dedupeKey, organizationId, resourceId, ${columns === "payload" ? `metadataRaw AS metadata, ${PREVIEW_COLUMNS.join(", ")}` : "input, output, error"}
 FROM ${TABLE_TRACE_ROOTS}
 WHERE (startedAt, traceId, dedupeKey, endedAt, ifNull(organizationId, ''), isNull(organizationId), ifNull(resourceId, ''), isNull(resourceId)) IN (${tuples.join(", ")})${compileTenantScope(scope, parameters)}
+ORDER BY ${ROOT_VERSION_ORDER}
 LIMIT 1 BY traceId, dedupeKey, organizationId, resourceId`,
 		query_params: parameters.params
 	};
@@ -7929,7 +7949,8 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
 		traceId: String(key.traceId),
 		dedupeKey: String(key.dedupeKey),
 		sortValue: sortValue(key.sortValue),
-		rootStartedAt: asIsoTimestamp(key.rootStartedAt)
+		rootStartedAt: asIsoTimestamp(key.rootStartedAt),
+		rootEndedAt: asIsoTimestamp(key.rootEndedAt)
 	})))) : [];
 	if (plan.paginationMode === "page") return coreStorage.traceQueryResponseSchema.parse({
 		traces: await toTraceRecords(rows),
@@ -8338,7 +8359,7 @@ function compileClickHouseThreadRowsQuery(plan, strategy = "full") {
 	const sortedRoots = `roots AS (
     SELECT organizationId, traceId, ${rootDataFields.map((field) => `${rootExpression(field)} AS ${field}`).join(", ")}
     FROM ${TABLE_TRACE_ROOTS} ${scope.length || strategy === "candidates" ? `WHERE ${[...scope, ...strategy === "candidates" ? [candidateTracesFilter] : []].join(" AND ")}` : ""}
-    ORDER BY organizationId, traceId, startedAt ASC, spanId, endedAt DESC, isNotNull(threadId) DESC, dedupeKey
+    ORDER BY organizationId, traceId, startedAt ASC, spanId, ${ROOT_VERSION_ORDER}
     LIMIT 1 BY organizationId, traceId
   )`;
 	return {
@@ -8440,7 +8461,7 @@ async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
       ${contextFields.join(", ")}, if(spanType = 'agent_run', timeToFirstChunkMs, NULL) AS timeToFirstChunkMs,
       if(isNull(error), 'success', 'error') AS status, ${PREVIEW_COLUMNS.join(", ")}
       FROM ${TABLE_TRACE_ROOTS} WHERE ${rootWhere}
-      ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
+      ORDER BY ${ROOT_VERSION_ORDER} LIMIT 1 BY organizationId, traceId, spanId`,
 		query_params: { roots: keys(batch) }
 	}));
 	const byRoot = new Map(stored.map((row) => [rootKey(row), row]));
@@ -8448,7 +8469,7 @@ async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
 	for (const batch of batches(missing, 32)) {
 		const payloads = await run({
 			query: `SELECT organizationId, traceId, spanId, input, output, error FROM ${TABLE_TRACE_ROOTS}
-        WHERE ${rootWhere} ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
+        WHERE ${rootWhere} ORDER BY ${ROOT_VERSION_ORDER} LIMIT 1 BY organizationId, traceId, spanId`,
 			query_params: { roots: keys(batch) }
 		});
 		for (const payload of payloads) {
@@ -8517,6 +8538,7 @@ const DIMENSION_SQL = {
 	entityName: "r.entityName",
 	name: "r.name",
 	spanType: "r.spanType",
+	entityId: "r.entityId",
 	environment: "r.environment",
 	status: TRACE_STATUS_SQL,
 	serviceName: "r.serviceName",
@@ -8555,12 +8577,6 @@ const COMPARISON_SQL = {
 function compileClickHouseTraceAggregate(plan) {
 	const parameters = new ParameterBuilder();
 	const ctes = compileClickHouseTraceCandidates(plan, "*", parameters, { oneRootPerTrace: true });
-	ctes.push(`trace_roots AS (
-    SELECT *
-    FROM candidates
-    ORDER BY traceId, dedupeKey
-    LIMIT 1 BY traceId
-  )`);
 	const dimensionSql = (field) => {
 		if (field.startsWith("metadata.")) return traceMetadataValueSql(parameters.add(field.slice(9), "String"));
 		const sql = Object.hasOwn(DIMENSION_SQL, field) ? DIMENSION_SQL[field] : void 0;
@@ -8594,7 +8610,7 @@ function compileClickHouseTraceAggregate(plan) {
 	}
 	ctes.push(`facts AS (
     SELECT ${factColumns.join(",\n      ")}
-    FROM trace_roots r${usageJoin}
+    FROM candidates r${usageJoin}
   )`);
 	const mixedUnitsSql = "minOrNull(unitMin) != maxOrNull(unitMax)";
 	const measureSql = (name) => {
@@ -10100,6 +10116,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			"trace-query-context-ids",
 			...timeToFirstChunkFeatures,
 			"trace-query-root-span-fields",
+			"trace-query-root-entity-id",
 			"trace-aggregate",
 			...spanQueryFeatures,
 			...threadQueryFeatures
@@ -10123,6 +10140,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			"trace-query-context-ids",
 			...timeToFirstChunkFeatures,
 			"trace-query-root-span-fields",
+			"trace-query-root-entity-id",
 			"trace-aggregate",
 			...spanQueryFeatures,
 			...threadQueryFeatures