@mastra/clickhouse 1.26.0-alpha.1 → 1.26.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +370 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +370 -15
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +3 -3
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/helpers.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +4 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/span-query.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/thread-query.d.ts +55 -0
- package/dist/storage/domains/observability/v-next/thread-query.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/trace-aggregate.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +11 -0
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/package.json +5 -6
package/dist/docs/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: mastra-clickhouse
|
|
|
3
3
|
description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
|
|
4
4
|
metadata:
|
|
5
5
|
package: "@mastra/clickhouse"
|
|
6
|
-
version: "1.26.0-alpha.
|
|
6
|
+
version: "1.26.0-alpha.3"
|
|
7
7
|
---
|
|
8
8
|
|
|
9
9
|
## When to use
|
package/dist/index.cjs
CHANGED
|
@@ -2743,6 +2743,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SPAN_EVENTS} (
|
|
|
2743
2743
|
isEvent Bool DEFAULT false,
|
|
2744
2744
|
startedAt DateTime64(3, 'UTC'),
|
|
2745
2745
|
endedAt DateTime64(3, 'UTC'),
|
|
2746
|
+
timeToFirstChunkMs Nullable(Float64),
|
|
2746
2747
|
|
|
2747
2748
|
-- Query-relevant flexible fields
|
|
2748
2749
|
tags Array(LowCardinality(String)) DEFAULT [],
|
|
@@ -2816,6 +2817,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_ROOTS} (
|
|
|
2816
2817
|
isEvent Bool DEFAULT false,
|
|
2817
2818
|
startedAt DateTime64(3, 'UTC'),
|
|
2818
2819
|
endedAt DateTime64(3, 'UTC'),
|
|
2820
|
+
timeToFirstChunkMs Nullable(Float64),
|
|
2819
2821
|
|
|
2820
2822
|
-- Query-relevant flexible fields
|
|
2821
2823
|
tags Array(LowCardinality(String)) DEFAULT [],
|
|
@@ -2901,6 +2903,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_BRANCHES} (
|
|
|
2901
2903
|
isEvent Bool DEFAULT false,
|
|
2902
2904
|
startedAt DateTime64(3, 'UTC'),
|
|
2903
2905
|
endedAt DateTime64(3, 'UTC'),
|
|
2906
|
+
timeToFirstChunkMs Nullable(Float64),
|
|
2904
2907
|
|
|
2905
2908
|
-- Query-relevant flexible fields
|
|
2906
2909
|
tags Array(LowCardinality(String)) DEFAULT [],
|
|
@@ -3706,7 +3709,10 @@ const ALL_MIGRATIONS = [
|
|
|
3706
3709
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
|
|
3707
3710
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId"),
|
|
3708
3711
|
...LOG_SKIP_INDEX_COLUMNS.map((column) => addBloomIndex(TABLE_LOG_EVENTS, `idx_${column}`, column)),
|
|
3709
|
-
addBloomIndex(TABLE_DELETION_REQUESTS, "idx_predicateValues", "predicateValues")
|
|
3712
|
+
addBloomIndex(TABLE_DELETION_REQUESTS, "idx_predicateValues", "predicateValues"),
|
|
3713
|
+
addColumn(TABLE_TRACE_ROOTS, "timeToFirstChunkMs", "Nullable(Float64)"),
|
|
3714
|
+
addColumn(TABLE_TRACE_BRANCHES, "timeToFirstChunkMs", "Nullable(Float64)"),
|
|
3715
|
+
addColumn(TABLE_SPAN_EVENTS, "timeToFirstChunkMs", "Nullable(Float64)")
|
|
3710
3716
|
];
|
|
3711
3717
|
const ALL_TABLE_NAMES = [
|
|
3712
3718
|
TABLE_SPAN_EVENTS,
|
|
@@ -4155,6 +4161,7 @@ function spanRecordToRow(span) {
|
|
|
4155
4161
|
isEvent: span.isEvent,
|
|
4156
4162
|
startedAt: toISOString(span.startedAt),
|
|
4157
4163
|
endedAt: toISOString(endedAt),
|
|
4164
|
+
timeToFirstChunkMs: span.timeToFirstChunkMs ?? null,
|
|
4158
4165
|
tags: normalizeTags(span.tags),
|
|
4159
4166
|
metadataSearch: buildMetadataSearch(metadata),
|
|
4160
4167
|
metadataRaw: jsonEncode(metadata),
|
|
@@ -6857,6 +6864,12 @@ async function getScorePercentiles(client, args) {
|
|
|
6857
6864
|
//#endregion
|
|
6858
6865
|
//#region src/storage/domains/observability/v-next/trace-query.ts
|
|
6859
6866
|
const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
|
|
6867
|
+
/**
|
|
6868
|
+
* Core versions without the field reject it in strict query results, so it is only returned when core knows it.
|
|
6869
|
+
* Core before 1.68 has no field registry at all.
|
|
6870
|
+
*/
|
|
6871
|
+
const timeToFirstChunkSupported = "timeToFirstChunkMs" in (_mastra_core_storage.TRACE_QUERY_FIELD_REGISTRY?.trace ?? {});
|
|
6872
|
+
const timeToFirstChunkFeatures = timeToFirstChunkSupported ? ["trace-query-time-to-first-chunk"] : [];
|
|
6860
6873
|
function durationMsSql(startedAt, endedAt) {
|
|
6861
6874
|
return `dateDiff('millisecond', ${startedAt}, ${endedAt})`;
|
|
6862
6875
|
}
|
|
@@ -6905,6 +6918,10 @@ const TRACE_FIELDS = {
|
|
|
6905
6918
|
sql: durationMsSql("r.startedAt", "r.endedAt"),
|
|
6906
6919
|
parameterType: "Float64"
|
|
6907
6920
|
},
|
|
6921
|
+
timeToFirstChunkMs: {
|
|
6922
|
+
sql: "r.timeToFirstChunkMs",
|
|
6923
|
+
parameterType: "Float64"
|
|
6924
|
+
},
|
|
6908
6925
|
entityName: {
|
|
6909
6926
|
sql: "r.entityName",
|
|
6910
6927
|
parameterType: "String"
|
|
@@ -6913,6 +6930,14 @@ const TRACE_FIELDS = {
|
|
|
6913
6930
|
sql: "r.entityType",
|
|
6914
6931
|
parameterType: "String"
|
|
6915
6932
|
},
|
|
6933
|
+
name: {
|
|
6934
|
+
sql: "r.name",
|
|
6935
|
+
parameterType: "String"
|
|
6936
|
+
},
|
|
6937
|
+
spanType: {
|
|
6938
|
+
sql: "r.spanType",
|
|
6939
|
+
parameterType: "String"
|
|
6940
|
+
},
|
|
6916
6941
|
environment: {
|
|
6917
6942
|
sql: "r.environment",
|
|
6918
6943
|
parameterType: "String"
|
|
@@ -6955,6 +6980,10 @@ const SPAN_FIELDS = {
|
|
|
6955
6980
|
sql: "s.durationMs",
|
|
6956
6981
|
parameterType: "Float64"
|
|
6957
6982
|
},
|
|
6983
|
+
timeToFirstChunkMs: {
|
|
6984
|
+
sql: "s.timeToFirstChunkMs",
|
|
6985
|
+
parameterType: "Float64"
|
|
6986
|
+
},
|
|
6958
6987
|
status: {
|
|
6959
6988
|
sql: "s.status",
|
|
6960
6989
|
parameterType: "String"
|
|
@@ -7166,6 +7195,7 @@ const TRACE_SELECT = `
|
|
|
7166
7195
|
r.entityName AS entityName,
|
|
7167
7196
|
r.entityType AS entityType,
|
|
7168
7197
|
r.environment AS environment,
|
|
7198
|
+
r.timeToFirstChunkMs AS timeToFirstChunkMs,
|
|
7169
7199
|
${TRACE_STATUS_SQL} AS status`;
|
|
7170
7200
|
var ParameterBuilder = class {
|
|
7171
7201
|
params = {};
|
|
@@ -7341,6 +7371,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7341
7371
|
startedAt,
|
|
7342
7372
|
endedAt,
|
|
7343
7373
|
${durationMsSql("startedAt", "endedAt")} AS durationMs,
|
|
7374
|
+
timeToFirstChunkMs,
|
|
7344
7375
|
if(isNotNull(error), 'error', 'success') AS status,
|
|
7345
7376
|
error,
|
|
7346
7377
|
entityType,
|
|
@@ -7753,6 +7784,12 @@ async function runWithClickHouseTraceQueryTimeout(client, limits, compiled, quer
|
|
|
7753
7784
|
...CH_SETTINGS,
|
|
7754
7785
|
max_execution_time: resolvedTimeoutMs / 1e3,
|
|
7755
7786
|
...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) },
|
|
7787
|
+
...limits.maxThreads === void 0 ? {} : { max_threads: limits.maxThreads },
|
|
7788
|
+
...limits.maxBlockSize === void 0 ? {} : { max_block_size: String(limits.maxBlockSize) },
|
|
7789
|
+
...limits.spillThresholdBytes === void 0 ? {} : {
|
|
7790
|
+
max_bytes_before_external_sort: String(limits.spillThresholdBytes),
|
|
7791
|
+
max_bytes_before_external_group_by: String(limits.spillThresholdBytes)
|
|
7792
|
+
},
|
|
7756
7793
|
...compiled.sharedSnapshot ? { enable_shared_storage_snapshot_in_query: 1 } : {}
|
|
7757
7794
|
}
|
|
7758
7795
|
})).json();
|
|
@@ -7875,7 +7912,8 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7875
7912
|
entityName: row.entityName == null ? null : String(row.entityName),
|
|
7876
7913
|
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7877
7914
|
environment: row.environment == null ? null : String(row.environment),
|
|
7878
|
-
status: row.status
|
|
7915
|
+
status: row.status,
|
|
7916
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: row.timeToFirstChunkMs == null ? null : Number(row.timeToFirstChunkMs) }
|
|
7879
7917
|
});
|
|
7880
7918
|
if (plan.paginationMode === "delta") {
|
|
7881
7919
|
const rows = await run(compileClickHouseTraceQuery(plan, deltaHead));
|
|
@@ -7950,6 +7988,16 @@ function compileSpanQueryPredicate(predicate) {
|
|
|
7950
7988
|
params: parameters.params
|
|
7951
7989
|
};
|
|
7952
7990
|
}
|
|
7991
|
+
/** Thread filters see full-history representative values, never individual member traces. */
|
|
7992
|
+
function compileThreadQueryPredicate(predicate, parameters = new ParameterBuilder()) {
|
|
7993
|
+
return {
|
|
7994
|
+
sql: compileScalarPredicate(predicate, Object.fromEntries(Object.entries(_mastra_core_storage.THREAD_QUERY_FIELD_REGISTRY).map(([field, rule]) => [field, {
|
|
7995
|
+
sql: `s.${field}`,
|
|
7996
|
+
parameterType: rule.valueKind === "timestamp" ? "DateTime64(3, 'UTC')" : rule.valueKind === "number" ? "Float64" : "String"
|
|
7997
|
+
}])), parameters),
|
|
7998
|
+
params: parameters.params
|
|
7999
|
+
};
|
|
8000
|
+
}
|
|
7953
8001
|
//#endregion
|
|
7954
8002
|
//#region src/storage/domains/observability/v-next/span-query.ts
|
|
7955
8003
|
const identityFields = [
|
|
@@ -7990,6 +8038,7 @@ const displayFields = [
|
|
|
7990
8038
|
"model",
|
|
7991
8039
|
"provider",
|
|
7992
8040
|
"durationMs",
|
|
8041
|
+
"timeToFirstChunkMs",
|
|
7993
8042
|
"status"
|
|
7994
8043
|
];
|
|
7995
8044
|
var Parameters = class {
|
|
@@ -8010,7 +8059,7 @@ function identityValues(row) {
|
|
|
8010
8059
|
row.spanId
|
|
8011
8060
|
];
|
|
8012
8061
|
}
|
|
8013
|
-
function afterCursor(plan, parameters) {
|
|
8062
|
+
function afterCursor$1(plan, parameters) {
|
|
8014
8063
|
if (!plan.cursor) return "1";
|
|
8015
8064
|
const cursor = plan.cursor;
|
|
8016
8065
|
const timestamp = parameters.add(cursor.sortValue, true);
|
|
@@ -8028,10 +8077,10 @@ function afterCursor(plan, parameters) {
|
|
|
8028
8077
|
}
|
|
8029
8078
|
return `(${terms.join(" OR ")})`;
|
|
8030
8079
|
}
|
|
8031
|
-
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8080
|
+
function predicateFields$1(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8032
8081
|
if (!predicate) return fields;
|
|
8033
|
-
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8034
|
-
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8082
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields$1(arg, fields);
|
|
8083
|
+
else if (predicate.type === "not") predicateFields$1(predicate.arg, fields);
|
|
8035
8084
|
else fields.add(predicate.field);
|
|
8036
8085
|
return fields;
|
|
8037
8086
|
}
|
|
@@ -8058,10 +8107,10 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8058
8107
|
const selectionFields = [.../* @__PURE__ */ new Set([
|
|
8059
8108
|
"startedAt",
|
|
8060
8109
|
"endedAt",
|
|
8061
|
-
...predicateFields(plan.where)
|
|
8110
|
+
...predicateFields$1(plan.where)
|
|
8062
8111
|
])].filter((field) => !new Set(identityFields).has(field));
|
|
8063
8112
|
const expressions = selectionFields.map(fieldExpression);
|
|
8064
|
-
const after = afterCursor(plan, parameters);
|
|
8113
|
+
const after = afterCursor$1(plan, parameters);
|
|
8065
8114
|
return {
|
|
8066
8115
|
query: `WITH candidates AS (
|
|
8067
8116
|
SELECT ${identities}, ${selectionFields.map((field, index) => `${expressions[index]} AS ${field}`).join(", ")}
|
|
@@ -8086,7 +8135,7 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8086
8135
|
};
|
|
8087
8136
|
}
|
|
8088
8137
|
const LEGACY_PAYLOAD_BATCH_ROWS = 32;
|
|
8089
|
-
function iso(value) {
|
|
8138
|
+
function iso$1(value) {
|
|
8090
8139
|
return new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8091
8140
|
}
|
|
8092
8141
|
async function querySpans(client, plan, timeoutMs) {
|
|
@@ -8101,8 +8150,8 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8101
8150
|
};
|
|
8102
8151
|
const selected = (await run(compileClickHouseSpanQuery(plan))).map((row) => ({
|
|
8103
8152
|
...row,
|
|
8104
|
-
startedAt: iso(row.startedAt),
|
|
8105
|
-
endedAt: iso(row.endedAt)
|
|
8153
|
+
startedAt: iso$1(row.startedAt),
|
|
8154
|
+
endedAt: iso$1(row.endedAt)
|
|
8106
8155
|
}));
|
|
8107
8156
|
const page = selected.slice(0, plan.limit);
|
|
8108
8157
|
if (!page.length) return {
|
|
@@ -8140,11 +8189,12 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8140
8189
|
const hydrated = page.flatMap((identity) => {
|
|
8141
8190
|
const row = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(identity));
|
|
8142
8191
|
if (!row) return [];
|
|
8143
|
-
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, ...metadata } = row;
|
|
8192
|
+
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, timeToFirstChunkMs, ...metadata } = row;
|
|
8144
8193
|
return [{
|
|
8145
8194
|
...metadata,
|
|
8146
8195
|
...identity,
|
|
8147
|
-
durationMs: Number(row.durationMs)
|
|
8196
|
+
durationMs: Number(row.durationMs),
|
|
8197
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: timeToFirstChunkMs == null ? null : Number(timeToFirstChunkMs) }
|
|
8148
8198
|
}];
|
|
8149
8199
|
});
|
|
8150
8200
|
let metrics = [];
|
|
@@ -8173,11 +8223,308 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8173
8223
|
return result;
|
|
8174
8224
|
}
|
|
8175
8225
|
//#endregion
|
|
8226
|
+
//#region src/storage/domains/observability/v-next/thread-query.ts
|
|
8227
|
+
const contextFields = [
|
|
8228
|
+
"resourceId",
|
|
8229
|
+
"userId",
|
|
8230
|
+
"entityType",
|
|
8231
|
+
"entityId",
|
|
8232
|
+
"entityName",
|
|
8233
|
+
"entityVersionId",
|
|
8234
|
+
"runId",
|
|
8235
|
+
"sessionId",
|
|
8236
|
+
"requestId",
|
|
8237
|
+
"serviceName",
|
|
8238
|
+
"experimentId",
|
|
8239
|
+
"environment",
|
|
8240
|
+
"executionSource"
|
|
8241
|
+
];
|
|
8242
|
+
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8243
|
+
if (!predicate) return fields;
|
|
8244
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8245
|
+
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8246
|
+
else fields.add(predicate.field);
|
|
8247
|
+
return fields;
|
|
8248
|
+
}
|
|
8249
|
+
function afterCursor(plan, parameters) {
|
|
8250
|
+
if (!plan.cursor) return "1";
|
|
8251
|
+
const { cursor } = plan;
|
|
8252
|
+
const stamp = parameters.add(cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
8253
|
+
const field = `s.${plan.orderBy.field}`;
|
|
8254
|
+
const terms = [`${field} ${plan.orderBy.direction === "asc" ? ">" : "<"} ${stamp}`];
|
|
8255
|
+
const equal = [`${field} = ${stamp}`];
|
|
8256
|
+
for (const key of ["organizationId", "threadId"]) if (cursor[key] === null) equal.push(`isNull(s.${key})`);
|
|
8257
|
+
else {
|
|
8258
|
+
const value = parameters.add(cursor[key], "String");
|
|
8259
|
+
terms.push(`(${equal.join(" AND ")} AND (s.${key} > ${value} OR isNull(s.${key})))`);
|
|
8260
|
+
equal.push(`s.${key} = ${value}`);
|
|
8261
|
+
}
|
|
8262
|
+
return `(${terms.join(" OR ")})`;
|
|
8263
|
+
}
|
|
8264
|
+
const THREAD_QUERY_EXECUTION_LIMITS = {
|
|
8265
|
+
memoryLimitBytes: 512 * 1024 * 1024,
|
|
8266
|
+
maxThreads: 2,
|
|
8267
|
+
maxBlockSize: 4096,
|
|
8268
|
+
spillThresholdBytes: 32 * 1024 * 1024
|
|
8269
|
+
};
|
|
8270
|
+
const MAX_CANDIDATE_THREADS = 16384;
|
|
8271
|
+
const MAX_CANDIDATE_TRACES = 1e5;
|
|
8272
|
+
const MAX_CANDIDATE_ID_BYTES = 4 * 1024 * 1024;
|
|
8273
|
+
const MAX_FULL_HISTORY_ROOTS = 25e4;
|
|
8274
|
+
function canUseThreadCandidates(probe) {
|
|
8275
|
+
return Number(probe.threadCount) <= MAX_CANDIDATE_THREADS && Number(probe.traceCount) <= MAX_CANDIDATE_TRACES && Number(probe.threadBytes) + Number(probe.traceBytes) <= MAX_CANDIDATE_ID_BYTES;
|
|
8276
|
+
}
|
|
8277
|
+
const identity = (field) => `tuple(isNull(organizationId), ifNull(organizationId, ''), ${field})`;
|
|
8278
|
+
const candidateThreadsFilter = `${identity("threadId")} IN (SELECT ${identity("threadId")} FROM candidate_threads)`;
|
|
8279
|
+
const candidateTracesFilter = `${identity("traceId")} IN (SELECT ${identity("traceId")} FROM candidate_traces)`;
|
|
8280
|
+
function candidateCtes(scope, from, to, plan, probe = false) {
|
|
8281
|
+
const timestamp = plan.timeRange.mode === "threads" ? plan.timeRange.field : "startedAt";
|
|
8282
|
+
const scoped = scope.length ? `${scope.join(" AND ")} AND ` : "";
|
|
8283
|
+
return `candidate_threads AS (
|
|
8284
|
+
SELECT DISTINCT organizationId, threadId FROM ${TABLE_TRACE_ROOTS}
|
|
8285
|
+
WHERE ${scoped}threadId IS NOT NULL AND ${timestamp} >= ${from} AND ${timestamp} < ${to}
|
|
8286
|
+
${probe ? `LIMIT 16385` : ""}
|
|
8287
|
+
), candidate_traces AS (
|
|
8288
|
+
SELECT DISTINCT organizationId, traceId FROM ${TABLE_TRACE_ROOTS}
|
|
8289
|
+
WHERE ${scoped}${candidateThreadsFilter}
|
|
8290
|
+
${probe ? `LIMIT 100001` : ""}
|
|
8291
|
+
)`;
|
|
8292
|
+
}
|
|
8293
|
+
function compileClickHouseThreadRowsQuery(plan, strategy = "full") {
|
|
8294
|
+
const parameters = new ParameterBuilder();
|
|
8295
|
+
const from = parameters.add(plan.timeRange.from, "DateTime64(3, 'UTC')");
|
|
8296
|
+
const to = parameters.add(plan.timeRange.to, "DateTime64(3, 'UTC')");
|
|
8297
|
+
const scope = [];
|
|
8298
|
+
if (plan.scope) {
|
|
8299
|
+
scope.push(`organizationId = ${parameters.add(plan.scope.organizationId, "String")}`);
|
|
8300
|
+
if (plan.scope.resourceId !== void 0) scope.push(`resourceId = ${parameters.add(plan.scope.resourceId, "String")}`);
|
|
8301
|
+
}
|
|
8302
|
+
if (strategy === "historyBudget") return {
|
|
8303
|
+
query: `SELECT count() AS rootCount FROM (
|
|
8304
|
+
SELECT 1 FROM ${TABLE_TRACE_ROOTS} ${scope.length ? `WHERE ${scope.join(" AND ")}` : ""}
|
|
8305
|
+
LIMIT 250001
|
|
8306
|
+
)`,
|
|
8307
|
+
query_params: parameters.params
|
|
8308
|
+
};
|
|
8309
|
+
if (strategy === "probe") return {
|
|
8310
|
+
query: `WITH ${candidateCtes(scope, from, to, plan, true)}
|
|
8311
|
+
SELECT (SELECT count() FROM candidate_threads) AS threadCount,
|
|
8312
|
+
(SELECT sum(length(ifNull(organizationId, '')) + length(threadId)) FROM candidate_threads) AS threadBytes,
|
|
8313
|
+
count() AS traceCount, sum(length(ifNull(organizationId, '')) + length(traceId)) AS traceBytes
|
|
8314
|
+
FROM candidate_traces`,
|
|
8315
|
+
query_params: parameters.params
|
|
8316
|
+
};
|
|
8317
|
+
const predicate = plan.where ? compileThreadQueryPredicate(plan.where, parameters).sql : "1";
|
|
8318
|
+
const after = afterCursor(plan, parameters);
|
|
8319
|
+
const qualify = plan.timeRange.mode === "traces" ? "s.inWindow > 0" : `s.${plan.timeRange.field} >= ${from} AND s.${plan.timeRange.field} < ${to}`;
|
|
8320
|
+
const filtered = predicateFields(plan.where);
|
|
8321
|
+
const firstContext = [...contextFields, "timeToFirstChunkMs"].filter((field) => field === "resourceId" || filtered.has(field));
|
|
8322
|
+
const firstFields = [
|
|
8323
|
+
"startedAt",
|
|
8324
|
+
"traceId",
|
|
8325
|
+
"spanId",
|
|
8326
|
+
"endedAt",
|
|
8327
|
+
...firstContext
|
|
8328
|
+
];
|
|
8329
|
+
const lastFields = [
|
|
8330
|
+
"startedAt",
|
|
8331
|
+
"traceId",
|
|
8332
|
+
"spanId",
|
|
8333
|
+
"endedAt",
|
|
8334
|
+
"resourceId",
|
|
8335
|
+
...filtered.has("status") ? ["status"] : []
|
|
8336
|
+
];
|
|
8337
|
+
const rootDataFields = [
|
|
8338
|
+
"threadId",
|
|
8339
|
+
"spanId",
|
|
8340
|
+
"startedAt",
|
|
8341
|
+
"endedAt",
|
|
8342
|
+
...firstContext,
|
|
8343
|
+
...filtered.has("status") ? ["status"] : []
|
|
8344
|
+
];
|
|
8345
|
+
const rootExpression = (field) => field === "timeToFirstChunkMs" ? "if(spanType = 'agent_run', timeToFirstChunkMs, NULL)" : field === "status" ? "if(isNull(error), 'success', 'error')" : field;
|
|
8346
|
+
const sortedRoots = `roots AS (
|
|
8347
|
+
SELECT organizationId, traceId, ${rootDataFields.map((field) => `${rootExpression(field)} AS ${field}`).join(", ")}
|
|
8348
|
+
FROM ${TABLE_TRACE_ROOTS} ${scope.length || strategy === "candidates" ? `WHERE ${[...scope, ...strategy === "candidates" ? [candidateTracesFilter] : []].join(" AND ")}` : ""}
|
|
8349
|
+
ORDER BY organizationId, traceId, startedAt ASC, spanId, endedAt DESC, isNotNull(threadId) DESC, dedupeKey
|
|
8350
|
+
LIMIT 1 BY organizationId, traceId
|
|
8351
|
+
)`;
|
|
8352
|
+
return {
|
|
8353
|
+
query: `WITH ${strategy === "candidates" ? `${candidateCtes(scope, from, to, plan)}, ` : ""}${sortedRoots}, grouped AS (
|
|
8354
|
+
SELECT organizationId, threadId, min(startedAt) AS firstActivity, max(endedAt) AS lastActivity,
|
|
8355
|
+
count() AS traceCount, countIf(startedAt >= ${from} AND startedAt < ${to}) AS inWindow,
|
|
8356
|
+
min(tuple(${firstFields.join(", ")})) AS first,
|
|
8357
|
+
max(tuple(${lastFields.join(", ")})) AS last
|
|
8358
|
+
FROM roots WHERE threadId IS NOT NULL ${strategy === "candidates" ? `AND ${candidateThreadsFilter}` : ""} GROUP BY organizationId, threadId
|
|
8359
|
+
), rows AS (
|
|
8360
|
+
SELECT organizationId, threadId, firstActivity AS startedAt, lastActivity AS endedAt, traceCount, inWindow,
|
|
8361
|
+
${firstFields.map((field, i) => `first.${i + 1} AS ${i < 4 ? `first${field[0].toUpperCase()}${field.slice(1)}` : field}`).join(", ")},
|
|
8362
|
+
${lastFields.map((field, i) => `last.${i + 1} AS ${field === "status" ? "status" : `last${field[0].toUpperCase()}${field.slice(1)}`}`).join(", ")}
|
|
8363
|
+
FROM grouped
|
|
8364
|
+
) SELECT * FROM rows s WHERE ${qualify} AND (${predicate}) AND ${after}
|
|
8365
|
+
ORDER BY s.${plan.orderBy.field} ${plan.orderBy.direction === "asc" ? "ASC" : "DESC"}, s.organizationId ASC NULLS LAST, s.threadId ASC
|
|
8366
|
+
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
8367
|
+
query_params: parameters.params
|
|
8368
|
+
};
|
|
8369
|
+
}
|
|
8370
|
+
const rootKey = (row) => JSON.stringify([
|
|
8371
|
+
row.organizationId,
|
|
8372
|
+
row.traceId,
|
|
8373
|
+
row.spanId
|
|
8374
|
+
]);
|
|
8375
|
+
const iso = (value) => new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8376
|
+
/** Bound HTTP form parameter size as well as payload rows. Long IDs must not break large pages. */
|
|
8377
|
+
function* batches(rows, maxRows = 256) {
|
|
8378
|
+
let batch = [];
|
|
8379
|
+
let bytes = 0;
|
|
8380
|
+
for (const row of rows) {
|
|
8381
|
+
const size = Buffer.byteLength(JSON.stringify(row), "utf8") * 2;
|
|
8382
|
+
if (batch.length && (batch.length >= maxRows || bytes + size > 48 * 1024)) {
|
|
8383
|
+
yield batch;
|
|
8384
|
+
batch = [];
|
|
8385
|
+
bytes = 0;
|
|
8386
|
+
}
|
|
8387
|
+
batch.push(row);
|
|
8388
|
+
bytes += size;
|
|
8389
|
+
}
|
|
8390
|
+
if (batch.length) yield batch;
|
|
8391
|
+
}
|
|
8392
|
+
async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
|
|
8393
|
+
const deadline = performance.now() + _mastra_core_storage.resolveTraceQueryTimeoutMs(timeoutMs);
|
|
8394
|
+
const run = async (compiled) => {
|
|
8395
|
+
const remaining = Math.floor(deadline - performance.now());
|
|
8396
|
+
if (remaining <= 0) throw new _mastra_core_storage.TraceQueryExecutionError();
|
|
8397
|
+
return await runWithClickHouseTraceQueryTimeout(client, {
|
|
8398
|
+
timeoutMs: remaining,
|
|
8399
|
+
...THREAD_QUERY_EXECUTION_LIMITS
|
|
8400
|
+
}, compiled);
|
|
8401
|
+
};
|
|
8402
|
+
if (strategy === "auto") {
|
|
8403
|
+
const [probe] = await run(compileClickHouseThreadRowsQuery(plan, "probe"));
|
|
8404
|
+
if (Number(probe.threadCount) === 0) return {
|
|
8405
|
+
threads: [],
|
|
8406
|
+
page: { next: null }
|
|
8407
|
+
};
|
|
8408
|
+
strategy = canUseThreadCandidates(probe) ? "candidates" : "full";
|
|
8409
|
+
}
|
|
8410
|
+
if (strategy === "full") {
|
|
8411
|
+
const [budget] = await run(compileClickHouseThreadRowsQuery(plan, "historyBudget"));
|
|
8412
|
+
if (Number(budget.rootCount) > MAX_FULL_HISTORY_ROOTS) throw new _mastra_core_storage.TraceQueryResourceLimitError();
|
|
8413
|
+
}
|
|
8414
|
+
const selected = await run(compileClickHouseThreadRowsQuery(plan, strategy));
|
|
8415
|
+
const page = selected.slice(0, plan.limit);
|
|
8416
|
+
if (!page.length) return {
|
|
8417
|
+
threads: [],
|
|
8418
|
+
page: { next: null }
|
|
8419
|
+
};
|
|
8420
|
+
const roots = /* @__PURE__ */ new Map();
|
|
8421
|
+
const representative = (row, which) => ({
|
|
8422
|
+
organizationId: row.organizationId,
|
|
8423
|
+
resourceId: which === "first" ? row.resourceId : row.lastResourceId,
|
|
8424
|
+
traceId: row[`${which}TraceId`],
|
|
8425
|
+
spanId: row[`${which}SpanId`],
|
|
8426
|
+
startedAt: row[`${which}StartedAt`],
|
|
8427
|
+
endedAt: row[`${which}EndedAt`]
|
|
8428
|
+
});
|
|
8429
|
+
for (const row of page) for (const which of ["first", "last"]) {
|
|
8430
|
+
const root = representative(row, which);
|
|
8431
|
+
roots.set(rootKey(root), root);
|
|
8432
|
+
}
|
|
8433
|
+
const keys = (batch) => batch.map((root) => new _clickhouse_client.TupleParam([
|
|
8434
|
+
root.organizationId === null ? 1 : 0,
|
|
8435
|
+
root.organizationId ?? "",
|
|
8436
|
+
root.resourceId === null ? 1 : 0,
|
|
8437
|
+
root.resourceId ?? "",
|
|
8438
|
+
root.traceId,
|
|
8439
|
+
root.spanId,
|
|
8440
|
+
iso(root.startedAt).replace("T", " ").replace(/Z$/, ""),
|
|
8441
|
+
iso(root.endedAt).replace("T", " ").replace(/Z$/, "")
|
|
8442
|
+
]));
|
|
8443
|
+
const rootWhere = `tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''),
|
|
8444
|
+
traceId, spanId, startedAt, endedAt) IN {roots:Array(Tuple(UInt8, String, UInt8, String, String, String, DateTime64(3, 'UTC'), DateTime64(3, 'UTC')))}`;
|
|
8445
|
+
const stored = [];
|
|
8446
|
+
for (const batch of batches([...roots.values()])) stored.push(...await run({
|
|
8447
|
+
query: `SELECT organizationId, traceId, spanId, startedAt, endedAt, spanType,
|
|
8448
|
+
${contextFields.join(", ")}, if(spanType = 'agent_run', timeToFirstChunkMs, NULL) AS timeToFirstChunkMs,
|
|
8449
|
+
if(isNull(error), 'success', 'error') AS status, ${PREVIEW_COLUMNS.join(", ")}
|
|
8450
|
+
FROM ${TABLE_TRACE_ROOTS} WHERE ${rootWhere}
|
|
8451
|
+
ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8452
|
+
query_params: { roots: keys(batch) }
|
|
8453
|
+
}));
|
|
8454
|
+
const byRoot = new Map(stored.map((row) => [rootKey(row), row]));
|
|
8455
|
+
const missing = stored.filter(needsPreviewPayloads);
|
|
8456
|
+
for (const batch of batches(missing, 32)) {
|
|
8457
|
+
const payloads = await run({
|
|
8458
|
+
query: `SELECT organizationId, traceId, spanId, input, output, error FROM ${TABLE_TRACE_ROOTS}
|
|
8459
|
+
WHERE ${rootWhere} ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8460
|
+
query_params: { roots: keys(batch) }
|
|
8461
|
+
});
|
|
8462
|
+
for (const payload of payloads) {
|
|
8463
|
+
const row = byRoot.get(rootKey(payload));
|
|
8464
|
+
if (row) Object.assign(row, resolveStoredPreviews(row, payload));
|
|
8465
|
+
}
|
|
8466
|
+
}
|
|
8467
|
+
const firstRoots = page.map((row) => representative(row, "first")).filter((root) => byRoot.get(rootKey(root))?.spanType === "agent_run");
|
|
8468
|
+
const models = [];
|
|
8469
|
+
for (const batch of batches(firstRoots)) models.push(...await run({
|
|
8470
|
+
query: `SELECT organizationId, traceId,
|
|
8471
|
+
if(JSONType(attributes, 'model') = 'String', JSONExtractString(attributes, 'model'), NULL) AS model,
|
|
8472
|
+
if(JSONType(attributes, 'provider') = 'String', JSONExtractString(attributes, 'provider'), NULL) AS provider
|
|
8473
|
+
FROM ${TABLE_SPAN_EVENTS} WHERE spanType = 'model_generation' AND
|
|
8474
|
+
tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''), traceId, parentSpanId)
|
|
8475
|
+
IN {agents:Array(Tuple(UInt8, String, UInt8, String, String, String))}
|
|
8476
|
+
ORDER BY organizationId, traceId, startedAt, spanId LIMIT 1 BY organizationId, traceId`,
|
|
8477
|
+
query_params: { agents: batch.map((root) => new _clickhouse_client.TupleParam([
|
|
8478
|
+
root.organizationId === null ? 1 : 0,
|
|
8479
|
+
root.organizationId ?? "",
|
|
8480
|
+
root.resourceId === null ? 1 : 0,
|
|
8481
|
+
root.resourceId ?? "",
|
|
8482
|
+
root.traceId,
|
|
8483
|
+
root.spanId
|
|
8484
|
+
])) }
|
|
8485
|
+
}));
|
|
8486
|
+
const byTrace = new Map(models.map((row) => [JSON.stringify([row.organizationId, row.traceId]), row]));
|
|
8487
|
+
const threads = page.flatMap((row) => {
|
|
8488
|
+
const first = byRoot.get(rootKey(representative(row, "first")));
|
|
8489
|
+
const last = byRoot.get(rootKey(representative(row, "last")));
|
|
8490
|
+
if (!first || !last) return [];
|
|
8491
|
+
const model = byTrace.get(JSON.stringify([row.organizationId, row.firstTraceId]));
|
|
8492
|
+
return [_mastra_core_storage.threadQueryRowSchema.parse({
|
|
8493
|
+
organizationId: row.organizationId,
|
|
8494
|
+
threadId: row.threadId,
|
|
8495
|
+
firstTraceId: row.firstTraceId,
|
|
8496
|
+
lastTraceId: row.lastTraceId,
|
|
8497
|
+
...Object.fromEntries(contextFields.map((field) => [field, first[field]])),
|
|
8498
|
+
status: last.status,
|
|
8499
|
+
startedAt: iso(row.startedAt),
|
|
8500
|
+
endedAt: iso(row.endedAt),
|
|
8501
|
+
traceCount: Number(row.traceCount),
|
|
8502
|
+
timeToFirstChunkMs: first.timeToFirstChunkMs,
|
|
8503
|
+
model: model?.model ?? null,
|
|
8504
|
+
provider: model?.provider ?? null,
|
|
8505
|
+
inputPreview: resolveStoredPreviews(first).inputPreview,
|
|
8506
|
+
outputPreview: resolveStoredPreviews(last).outputPreview,
|
|
8507
|
+
errorPreview: resolveStoredPreviews(last).errorPreview
|
|
8508
|
+
})];
|
|
8509
|
+
});
|
|
8510
|
+
const last = page.at(-1);
|
|
8511
|
+
return {
|
|
8512
|
+
threads,
|
|
8513
|
+
page: { next: selected.length > plan.limit ? _mastra_core_storage.encodeThreadQueryCursor(plan, {
|
|
8514
|
+
organizationId: last.organizationId,
|
|
8515
|
+
threadId: last.threadId,
|
|
8516
|
+
sortValue: iso(last[plan.orderBy.field])
|
|
8517
|
+
}) : null }
|
|
8518
|
+
};
|
|
8519
|
+
}
|
|
8520
|
+
//#endregion
|
|
8176
8521
|
//#region src/storage/domains/observability/v-next/trace-aggregate.ts
|
|
8177
8522
|
/** Trace-root column expressions (alias `r`) for each groupable dimension; same values `where` sees. */
|
|
8178
8523
|
const DIMENSION_SQL = {
|
|
8179
8524
|
entityType: "r.entityType",
|
|
8180
8525
|
entityName: "r.entityName",
|
|
8526
|
+
name: "r.name",
|
|
8527
|
+
spanType: "r.spanType",
|
|
8181
8528
|
environment: "r.environment",
|
|
8182
8529
|
status: TRACE_STATUS_SQL,
|
|
8183
8530
|
serviceName: "r.serviceName",
|
|
@@ -9321,6 +9668,7 @@ function buildBranchCursor(row) {
|
|
|
9321
9668
|
//#endregion
|
|
9322
9669
|
//#region src/storage/domains/observability/v-next/index.ts
|
|
9323
9670
|
const spanQueryFeatures = typeof _mastra_core_storage.planSpanQuery === "function" ? ["span-query", "span-query-context-fields"] : [];
|
|
9671
|
+
const threadQueryFeatures = typeof _mastra_core_storage.planThreadRowsQuery === "function" ? ["thread-query-rows"] : [];
|
|
9324
9672
|
function buildSignalMigrationRequiredMessage(args) {
|
|
9325
9673
|
const tableList = args.tables.map((table) => ` - ${table.table} (${table.engine})`).join("\n");
|
|
9326
9674
|
return `
|
|
@@ -9758,8 +10106,11 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9758
10106
|
"trace-query-tenant-scope",
|
|
9759
10107
|
"feedback",
|
|
9760
10108
|
"trace-query-context-ids",
|
|
10109
|
+
...timeToFirstChunkFeatures,
|
|
10110
|
+
"trace-query-root-span-fields",
|
|
9761
10111
|
"trace-aggregate",
|
|
9762
|
-
...spanQueryFeatures
|
|
10112
|
+
...spanQueryFeatures,
|
|
10113
|
+
...threadQueryFeatures
|
|
9763
10114
|
];
|
|
9764
10115
|
return [
|
|
9765
10116
|
"metrics",
|
|
@@ -9778,8 +10129,11 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9778
10129
|
"trace-query-tenant-scope",
|
|
9779
10130
|
"feedback",
|
|
9780
10131
|
"trace-query-context-ids",
|
|
10132
|
+
...timeToFirstChunkFeatures,
|
|
10133
|
+
"trace-query-root-span-fields",
|
|
9781
10134
|
"trace-aggregate",
|
|
9782
|
-
...spanQueryFeatures
|
|
10135
|
+
...spanQueryFeatures,
|
|
10136
|
+
...threadQueryFeatures
|
|
9783
10137
|
];
|
|
9784
10138
|
}
|
|
9785
10139
|
async createSpan(args) {
|
|
@@ -9956,6 +10310,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9956
10310
|
}
|
|
9957
10311
|
async queryThreads(plan) {
|
|
9958
10312
|
try {
|
|
10313
|
+
if (plan.result === "threadRows") return await queryThreadRows(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9959
10314
|
return await queryThreads(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9960
10315
|
} catch (error) {
|
|
9961
10316
|
if (error instanceof _mastra_core_error.MastraError || error instanceof _mastra_core_storage.TraceQueryExecutionError || error instanceof _mastra_core_storage.TraceQueryResourceLimitError) throw error;
|