@mastra/clickhouse 1.19.0 → 1.20.0-alpha.1

package/dist/index.js

@@ -3256,7 +3256,10 @@ CREATE TABLE IF NOT EXISTS ${TABLE_DELETION_REQUESTS} (
   requestedBy     String DEFAULT '',
   lastAppliedAt   DateTime64(3) DEFAULT 0,
   purgeVerifiedAt DateTime64(3) DEFAULT 0,
-  updatedAt       DateTime64(3)
+  updatedAt       DateTime64(3),
+  -- Bloom-filter skip index so has(predicateValues, id) mutation guards
+  -- can skip granules instead of scanning every request in a tenant scope.
+  INDEX idx_predicateValues predicateValues TYPE bloom_filter(0.01) GRANULARITY 2
 )
 ENGINE = ReplacingMergeTree(updatedAt)
 ORDER BY (organizationId, resourceId, requestId)
@@ -3484,7 +3487,8 @@ const ALL_MIGRATIONS = [
 	addBloomIndex(TABLE_METRIC_EVENTS, "idx_experimentId", "experimentId"),
 	addBloomIndex(TABLE_METRIC_EVENTS, "idx_runId", "runId"),
 	addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
-	addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId")
+	addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId"),
+	addBloomIndex(TABLE_DELETION_REQUESTS, "idx_predicateValues", "predicateValues")
 ];
 const ALL_TABLE_NAMES = [
 	TABLE_SPAN_EVENTS,
@@ -6770,6 +6774,76 @@ LIMIT ${limit}`,
 		query_params: parameters.params
 	};
 }
+function discoveryRegistry(scope) {
+	if (scope === "trace") return TRACE_FIELDS;
+	if (scope === "spans") return SPAN_FIELDS;
+	if (scope === "scores") return SCORE_FIELDS;
+	return FEEDBACK_FIELDS;
+}
+function discoverySource(scope) {
+	if (scope === "trace") return "root_scope r";
+	if (scope === "spans") return "current_spans s";
+	if (scope === "scores") return "current_scores s";
+	return "current_feedback s";
+}
+function discoveryCollections(scope) {
+	return scope === "trace" ? /* @__PURE__ */ new Set() : /* @__PURE__ */ new Set([scope]);
+}
+function compileClickHouseTraceQueryObservedFields(plan) {
+	const parameters = new ParameterBuilder();
+	const ctes = compileClickHouseTraceScope(plan, /* @__PURE__ */ new Set(), parameters);
+	const search = plan.search ? `AND positionCaseInsensitiveUTF8(concat('metadata.', key), ${parameters.add(plan.search, "String")}) > 0` : "";
+	const limit = parameters.add(plan.limit + 1, "UInt64");
+	ctes.push(`metadata_entries AS (
+    SELECT
+      entry.1 AS key,
+      entry.2 AS rawValue,
+      JSONExtractString(entry.2) AS value
+    FROM root_scope r
+    ARRAY JOIN JSONExtractKeysAndValuesRaw(ifNull(r.metadataRaw, '{}')) AS entry
+  )`);
+	return {
+		query: `WITH ${ctes.join(",\n")}
+SELECT concat('metadata.', key) AS path, count() AS occurrences
+FROM metadata_entries
+WHERE JSONType(rawValue) = 'String'
+  AND trim(value) != ''
+  AND key != ''
+  AND position(key, '.') = 0
+  AND length(concat('metadata.', key)) <= ${coreStorage.TRACE_QUERY_MAX_PATH_BYTES}
+  AND length(value) <= ${coreStorage.TRACE_QUERY_MAX_STRING_BYTES}
+  ${search}
+GROUP BY key
+ORDER BY occurrences DESC, path ASC
+LIMIT ${limit}`,
+		query_params: parameters.params
+	};
+}
+function compileClickHouseTraceQueryValues(plan) {
+	const parameters = new ParameterBuilder();
+	const ctes = compileClickHouseTraceScope(plan, discoveryCollections(plan.predicateScope), parameters);
+	let field;
+	if (plan.predicateScope === "trace" && plan.path.startsWith("metadata.")) {
+		const key = parameters.add(plan.path.slice(9), "String");
+		field = `coalesce(if(mapContains(r.metadataSearch, ${key}), r.metadataSearch[${key}], NULL), nullIf(trim(JSONExtractString(r.metadataRaw, ${key})), ''))`;
+	} else field = fieldDefinition(discoveryRegistry(plan.predicateScope), plan.path).sql;
+	const search = plan.search ? `AND positionCaseInsensitiveUTF8(value, ${parameters.add(plan.search, "String")}) > 0` : "";
+	const limit = parameters.add(plan.limit + 1, "UInt64");
+	return {
+		query: `WITH ${ctes.join(",\n")}, extracted AS (
+  SELECT toString(${field}) AS value FROM ${discoverySource(plan.predicateScope)}
+)
+SELECT value, count() AS count
+FROM extracted
+WHERE value IS NOT NULL
+  AND length(value) <= ${coreStorage.TRACE_QUERY_MAX_STRING_BYTES}
+  ${search}
+GROUP BY value
+ORDER BY count DESC, value ASC
+LIMIT ${limit}`,
+		query_params: parameters.params
+	};
+}
 function asIsoTimestamp(value) {
 	return new Date(value).toISOString();
 }
@@ -6796,6 +6870,27 @@ async function runWithClickHouseTraceQueryTimeout(client, timeoutMs, compiled, q
 		throw error;
 	}
 }
+async function getTraceQueryObservedFields(client, plan, timeoutMs) {
+	if (plan.predicateScope !== "trace") return {
+		observedFields: [],
+		observedFieldsTruncated: false
+	};
+	const rows = await runWithClickHouseTraceQueryTimeout(client, timeoutMs, compileClickHouseTraceQueryObservedFields(plan));
+	return {
+		observedFields: rows.slice(0, plan.limit).map((row) => coreStorage.createTraceQueryObservedFieldDescriptor(String(row.path), Number(row.occurrences))),
+		observedFieldsTruncated: rows.length > plan.limit
+	};
+}
+async function getTraceQueryValues(client, plan, timeoutMs) {
+	const rows = await runWithClickHouseTraceQueryTimeout(client, timeoutMs, compileClickHouseTraceQueryValues(plan));
+	return coreStorage.getTraceQueryValuesResponseSchema.parse({
+		values: rows.slice(0, plan.limit).map((row) => ({
+			value: String(row.value),
+			count: Number(row.count)
+		})),
+		valuesTruncated: rows.length > plan.limit
+	});
+}
 async function queryTraces(client, plan, timeoutMs) {
 	const rows = await runWithClickHouseTraceQueryTimeout(client, timeoutMs, compileClickHouseTraceQuery(plan));
 	const visibleRows = rows.slice(0, plan.limit);
@@ -7961,6 +8056,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			"metrics",
 			"logs",
 			"trace-query",
+			"trace-query-discovery",
 			"thread-query"
 		];
 		return [
@@ -7968,6 +8064,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			"logs",
 			"delta-polling",
 			"trace-query",
+			"trace-query-discovery",
 			"thread-query"
 		];
 	}
@@ -8095,6 +8192,30 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
 			}, error);
 		}
 	}
+	async getTraceQueryObservedFields(plan) {
+		try {
+			return await getTraceQueryObservedFields(this.#client, plan, this.#traceQueryTimeoutMs);
+		} catch (error) {
+			if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "GET_TRACE_QUERY_OBSERVED_FIELDS", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY
+			}, error);
+		}
+	}
+	async getTraceQueryValues(plan) {
+		try {
+			return await getTraceQueryValues(this.#client, plan, this.#traceQueryTimeoutMs);
+		} catch (error) {
+			if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError) throw error;
+			throw new MastraError({
+				id: createStorageErrorId("CLICKHOUSE", "GET_TRACE_QUERY_VALUES", "FAILED"),
+				domain: ErrorDomain.STORAGE,
+				category: ErrorCategory.THIRD_PARTY
+			}, error);
+		}
+	}
 	async queryThreads(plan) {
 		try {
 			return await queryThreads(this.#client, plan, this.#traceQueryTimeoutMs);