@mastra/clickhouse 1.26.0-alpha.2 → 1.26.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +362 -14
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +362 -14
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +4 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/span-query.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/thread-query.d.ts +55 -0
- package/dist/storage/domains/observability/v-next/thread-query.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/trace-aggregate.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +11 -0
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/package.json +5 -6
package/dist/docs/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: mastra-clickhouse
|
|
|
3
3
|
description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
|
|
4
4
|
metadata:
|
|
5
5
|
package: "@mastra/clickhouse"
|
|
6
|
-
version: "1.26.0-alpha.
|
|
6
|
+
version: "1.26.0-alpha.3"
|
|
7
7
|
---
|
|
8
8
|
|
|
9
9
|
## When to use
|
package/dist/index.cjs
CHANGED
|
@@ -6864,6 +6864,12 @@ async function getScorePercentiles(client, args) {
|
|
|
6864
6864
|
//#endregion
|
|
6865
6865
|
//#region src/storage/domains/observability/v-next/trace-query.ts
|
|
6866
6866
|
const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
|
|
6867
|
+
/**
|
|
6868
|
+
* Core versions without the field reject it in strict query results, so it is only returned when core knows it.
|
|
6869
|
+
* Core before 1.68 has no field registry at all.
|
|
6870
|
+
*/
|
|
6871
|
+
const timeToFirstChunkSupported = "timeToFirstChunkMs" in (_mastra_core_storage.TRACE_QUERY_FIELD_REGISTRY?.trace ?? {});
|
|
6872
|
+
const timeToFirstChunkFeatures = timeToFirstChunkSupported ? ["trace-query-time-to-first-chunk"] : [];
|
|
6867
6873
|
function durationMsSql(startedAt, endedAt) {
|
|
6868
6874
|
return `dateDiff('millisecond', ${startedAt}, ${endedAt})`;
|
|
6869
6875
|
}
|
|
@@ -6912,6 +6918,10 @@ const TRACE_FIELDS = {
|
|
|
6912
6918
|
sql: durationMsSql("r.startedAt", "r.endedAt"),
|
|
6913
6919
|
parameterType: "Float64"
|
|
6914
6920
|
},
|
|
6921
|
+
timeToFirstChunkMs: {
|
|
6922
|
+
sql: "r.timeToFirstChunkMs",
|
|
6923
|
+
parameterType: "Float64"
|
|
6924
|
+
},
|
|
6915
6925
|
entityName: {
|
|
6916
6926
|
sql: "r.entityName",
|
|
6917
6927
|
parameterType: "String"
|
|
@@ -6920,6 +6930,14 @@ const TRACE_FIELDS = {
|
|
|
6920
6930
|
sql: "r.entityType",
|
|
6921
6931
|
parameterType: "String"
|
|
6922
6932
|
},
|
|
6933
|
+
name: {
|
|
6934
|
+
sql: "r.name",
|
|
6935
|
+
parameterType: "String"
|
|
6936
|
+
},
|
|
6937
|
+
spanType: {
|
|
6938
|
+
sql: "r.spanType",
|
|
6939
|
+
parameterType: "String"
|
|
6940
|
+
},
|
|
6923
6941
|
environment: {
|
|
6924
6942
|
sql: "r.environment",
|
|
6925
6943
|
parameterType: "String"
|
|
@@ -6962,6 +6980,10 @@ const SPAN_FIELDS = {
|
|
|
6962
6980
|
sql: "s.durationMs",
|
|
6963
6981
|
parameterType: "Float64"
|
|
6964
6982
|
},
|
|
6983
|
+
timeToFirstChunkMs: {
|
|
6984
|
+
sql: "s.timeToFirstChunkMs",
|
|
6985
|
+
parameterType: "Float64"
|
|
6986
|
+
},
|
|
6965
6987
|
status: {
|
|
6966
6988
|
sql: "s.status",
|
|
6967
6989
|
parameterType: "String"
|
|
@@ -7173,6 +7195,7 @@ const TRACE_SELECT = `
|
|
|
7173
7195
|
r.entityName AS entityName,
|
|
7174
7196
|
r.entityType AS entityType,
|
|
7175
7197
|
r.environment AS environment,
|
|
7198
|
+
r.timeToFirstChunkMs AS timeToFirstChunkMs,
|
|
7176
7199
|
${TRACE_STATUS_SQL} AS status`;
|
|
7177
7200
|
var ParameterBuilder = class {
|
|
7178
7201
|
params = {};
|
|
@@ -7348,6 +7371,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7348
7371
|
startedAt,
|
|
7349
7372
|
endedAt,
|
|
7350
7373
|
${durationMsSql("startedAt", "endedAt")} AS durationMs,
|
|
7374
|
+
timeToFirstChunkMs,
|
|
7351
7375
|
if(isNotNull(error), 'error', 'success') AS status,
|
|
7352
7376
|
error,
|
|
7353
7377
|
entityType,
|
|
@@ -7760,6 +7784,12 @@ async function runWithClickHouseTraceQueryTimeout(client, limits, compiled, quer
|
|
|
7760
7784
|
...CH_SETTINGS,
|
|
7761
7785
|
max_execution_time: resolvedTimeoutMs / 1e3,
|
|
7762
7786
|
...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) },
|
|
7787
|
+
...limits.maxThreads === void 0 ? {} : { max_threads: limits.maxThreads },
|
|
7788
|
+
...limits.maxBlockSize === void 0 ? {} : { max_block_size: String(limits.maxBlockSize) },
|
|
7789
|
+
...limits.spillThresholdBytes === void 0 ? {} : {
|
|
7790
|
+
max_bytes_before_external_sort: String(limits.spillThresholdBytes),
|
|
7791
|
+
max_bytes_before_external_group_by: String(limits.spillThresholdBytes)
|
|
7792
|
+
},
|
|
7763
7793
|
...compiled.sharedSnapshot ? { enable_shared_storage_snapshot_in_query: 1 } : {}
|
|
7764
7794
|
}
|
|
7765
7795
|
})).json();
|
|
@@ -7882,7 +7912,8 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7882
7912
|
entityName: row.entityName == null ? null : String(row.entityName),
|
|
7883
7913
|
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7884
7914
|
environment: row.environment == null ? null : String(row.environment),
|
|
7885
|
-
status: row.status
|
|
7915
|
+
status: row.status,
|
|
7916
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: row.timeToFirstChunkMs == null ? null : Number(row.timeToFirstChunkMs) }
|
|
7886
7917
|
});
|
|
7887
7918
|
if (plan.paginationMode === "delta") {
|
|
7888
7919
|
const rows = await run(compileClickHouseTraceQuery(plan, deltaHead));
|
|
@@ -7957,6 +7988,16 @@ function compileSpanQueryPredicate(predicate) {
|
|
|
7957
7988
|
params: parameters.params
|
|
7958
7989
|
};
|
|
7959
7990
|
}
|
|
7991
|
+
/** Thread filters see full-history representative values, never individual member traces. */
|
|
7992
|
+
function compileThreadQueryPredicate(predicate, parameters = new ParameterBuilder()) {
|
|
7993
|
+
return {
|
|
7994
|
+
sql: compileScalarPredicate(predicate, Object.fromEntries(Object.entries(_mastra_core_storage.THREAD_QUERY_FIELD_REGISTRY).map(([field, rule]) => [field, {
|
|
7995
|
+
sql: `s.${field}`,
|
|
7996
|
+
parameterType: rule.valueKind === "timestamp" ? "DateTime64(3, 'UTC')" : rule.valueKind === "number" ? "Float64" : "String"
|
|
7997
|
+
}])), parameters),
|
|
7998
|
+
params: parameters.params
|
|
7999
|
+
};
|
|
8000
|
+
}
|
|
7960
8001
|
//#endregion
|
|
7961
8002
|
//#region src/storage/domains/observability/v-next/span-query.ts
|
|
7962
8003
|
const identityFields = [
|
|
@@ -7997,6 +8038,7 @@ const displayFields = [
|
|
|
7997
8038
|
"model",
|
|
7998
8039
|
"provider",
|
|
7999
8040
|
"durationMs",
|
|
8041
|
+
"timeToFirstChunkMs",
|
|
8000
8042
|
"status"
|
|
8001
8043
|
];
|
|
8002
8044
|
var Parameters = class {
|
|
@@ -8017,7 +8059,7 @@ function identityValues(row) {
|
|
|
8017
8059
|
row.spanId
|
|
8018
8060
|
];
|
|
8019
8061
|
}
|
|
8020
|
-
function afterCursor(plan, parameters) {
|
|
8062
|
+
function afterCursor$1(plan, parameters) {
|
|
8021
8063
|
if (!plan.cursor) return "1";
|
|
8022
8064
|
const cursor = plan.cursor;
|
|
8023
8065
|
const timestamp = parameters.add(cursor.sortValue, true);
|
|
@@ -8035,10 +8077,10 @@ function afterCursor(plan, parameters) {
|
|
|
8035
8077
|
}
|
|
8036
8078
|
return `(${terms.join(" OR ")})`;
|
|
8037
8079
|
}
|
|
8038
|
-
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8080
|
+
function predicateFields$1(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8039
8081
|
if (!predicate) return fields;
|
|
8040
|
-
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8041
|
-
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8082
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields$1(arg, fields);
|
|
8083
|
+
else if (predicate.type === "not") predicateFields$1(predicate.arg, fields);
|
|
8042
8084
|
else fields.add(predicate.field);
|
|
8043
8085
|
return fields;
|
|
8044
8086
|
}
|
|
@@ -8065,10 +8107,10 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8065
8107
|
const selectionFields = [.../* @__PURE__ */ new Set([
|
|
8066
8108
|
"startedAt",
|
|
8067
8109
|
"endedAt",
|
|
8068
|
-
...predicateFields(plan.where)
|
|
8110
|
+
...predicateFields$1(plan.where)
|
|
8069
8111
|
])].filter((field) => !new Set(identityFields).has(field));
|
|
8070
8112
|
const expressions = selectionFields.map(fieldExpression);
|
|
8071
|
-
const after = afterCursor(plan, parameters);
|
|
8113
|
+
const after = afterCursor$1(plan, parameters);
|
|
8072
8114
|
return {
|
|
8073
8115
|
query: `WITH candidates AS (
|
|
8074
8116
|
SELECT ${identities}, ${selectionFields.map((field, index) => `${expressions[index]} AS ${field}`).join(", ")}
|
|
@@ -8093,7 +8135,7 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8093
8135
|
};
|
|
8094
8136
|
}
|
|
8095
8137
|
const LEGACY_PAYLOAD_BATCH_ROWS = 32;
|
|
8096
|
-
function iso(value) {
|
|
8138
|
+
function iso$1(value) {
|
|
8097
8139
|
return new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8098
8140
|
}
|
|
8099
8141
|
async function querySpans(client, plan, timeoutMs) {
|
|
@@ -8108,8 +8150,8 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8108
8150
|
};
|
|
8109
8151
|
const selected = (await run(compileClickHouseSpanQuery(plan))).map((row) => ({
|
|
8110
8152
|
...row,
|
|
8111
|
-
startedAt: iso(row.startedAt),
|
|
8112
|
-
endedAt: iso(row.endedAt)
|
|
8153
|
+
startedAt: iso$1(row.startedAt),
|
|
8154
|
+
endedAt: iso$1(row.endedAt)
|
|
8113
8155
|
}));
|
|
8114
8156
|
const page = selected.slice(0, plan.limit);
|
|
8115
8157
|
if (!page.length) return {
|
|
@@ -8147,11 +8189,12 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8147
8189
|
const hydrated = page.flatMap((identity) => {
|
|
8148
8190
|
const row = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(identity));
|
|
8149
8191
|
if (!row) return [];
|
|
8150
|
-
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, ...metadata } = row;
|
|
8192
|
+
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, timeToFirstChunkMs, ...metadata } = row;
|
|
8151
8193
|
return [{
|
|
8152
8194
|
...metadata,
|
|
8153
8195
|
...identity,
|
|
8154
|
-
durationMs: Number(row.durationMs)
|
|
8196
|
+
durationMs: Number(row.durationMs),
|
|
8197
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: timeToFirstChunkMs == null ? null : Number(timeToFirstChunkMs) }
|
|
8155
8198
|
}];
|
|
8156
8199
|
});
|
|
8157
8200
|
let metrics = [];
|
|
@@ -8180,11 +8223,308 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8180
8223
|
return result;
|
|
8181
8224
|
}
|
|
8182
8225
|
//#endregion
|
|
8226
|
+
//#region src/storage/domains/observability/v-next/thread-query.ts
|
|
8227
|
+
const contextFields = [
|
|
8228
|
+
"resourceId",
|
|
8229
|
+
"userId",
|
|
8230
|
+
"entityType",
|
|
8231
|
+
"entityId",
|
|
8232
|
+
"entityName",
|
|
8233
|
+
"entityVersionId",
|
|
8234
|
+
"runId",
|
|
8235
|
+
"sessionId",
|
|
8236
|
+
"requestId",
|
|
8237
|
+
"serviceName",
|
|
8238
|
+
"experimentId",
|
|
8239
|
+
"environment",
|
|
8240
|
+
"executionSource"
|
|
8241
|
+
];
|
|
8242
|
+
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8243
|
+
if (!predicate) return fields;
|
|
8244
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8245
|
+
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8246
|
+
else fields.add(predicate.field);
|
|
8247
|
+
return fields;
|
|
8248
|
+
}
|
|
8249
|
+
function afterCursor(plan, parameters) {
|
|
8250
|
+
if (!plan.cursor) return "1";
|
|
8251
|
+
const { cursor } = plan;
|
|
8252
|
+
const stamp = parameters.add(cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
8253
|
+
const field = `s.${plan.orderBy.field}`;
|
|
8254
|
+
const terms = [`${field} ${plan.orderBy.direction === "asc" ? ">" : "<"} ${stamp}`];
|
|
8255
|
+
const equal = [`${field} = ${stamp}`];
|
|
8256
|
+
for (const key of ["organizationId", "threadId"]) if (cursor[key] === null) equal.push(`isNull(s.${key})`);
|
|
8257
|
+
else {
|
|
8258
|
+
const value = parameters.add(cursor[key], "String");
|
|
8259
|
+
terms.push(`(${equal.join(" AND ")} AND (s.${key} > ${value} OR isNull(s.${key})))`);
|
|
8260
|
+
equal.push(`s.${key} = ${value}`);
|
|
8261
|
+
}
|
|
8262
|
+
return `(${terms.join(" OR ")})`;
|
|
8263
|
+
}
|
|
8264
|
+
const THREAD_QUERY_EXECUTION_LIMITS = {
|
|
8265
|
+
memoryLimitBytes: 512 * 1024 * 1024,
|
|
8266
|
+
maxThreads: 2,
|
|
8267
|
+
maxBlockSize: 4096,
|
|
8268
|
+
spillThresholdBytes: 32 * 1024 * 1024
|
|
8269
|
+
};
|
|
8270
|
+
const MAX_CANDIDATE_THREADS = 16384;
|
|
8271
|
+
const MAX_CANDIDATE_TRACES = 1e5;
|
|
8272
|
+
const MAX_CANDIDATE_ID_BYTES = 4 * 1024 * 1024;
|
|
8273
|
+
const MAX_FULL_HISTORY_ROOTS = 25e4;
|
|
8274
|
+
function canUseThreadCandidates(probe) {
|
|
8275
|
+
return Number(probe.threadCount) <= MAX_CANDIDATE_THREADS && Number(probe.traceCount) <= MAX_CANDIDATE_TRACES && Number(probe.threadBytes) + Number(probe.traceBytes) <= MAX_CANDIDATE_ID_BYTES;
|
|
8276
|
+
}
|
|
8277
|
+
const identity = (field) => `tuple(isNull(organizationId), ifNull(organizationId, ''), ${field})`;
|
|
8278
|
+
const candidateThreadsFilter = `${identity("threadId")} IN (SELECT ${identity("threadId")} FROM candidate_threads)`;
|
|
8279
|
+
const candidateTracesFilter = `${identity("traceId")} IN (SELECT ${identity("traceId")} FROM candidate_traces)`;
|
|
8280
|
+
function candidateCtes(scope, from, to, plan, probe = false) {
|
|
8281
|
+
const timestamp = plan.timeRange.mode === "threads" ? plan.timeRange.field : "startedAt";
|
|
8282
|
+
const scoped = scope.length ? `${scope.join(" AND ")} AND ` : "";
|
|
8283
|
+
return `candidate_threads AS (
|
|
8284
|
+
SELECT DISTINCT organizationId, threadId FROM ${TABLE_TRACE_ROOTS}
|
|
8285
|
+
WHERE ${scoped}threadId IS NOT NULL AND ${timestamp} >= ${from} AND ${timestamp} < ${to}
|
|
8286
|
+
${probe ? `LIMIT 16385` : ""}
|
|
8287
|
+
), candidate_traces AS (
|
|
8288
|
+
SELECT DISTINCT organizationId, traceId FROM ${TABLE_TRACE_ROOTS}
|
|
8289
|
+
WHERE ${scoped}${candidateThreadsFilter}
|
|
8290
|
+
${probe ? `LIMIT 100001` : ""}
|
|
8291
|
+
)`;
|
|
8292
|
+
}
|
|
8293
|
+
function compileClickHouseThreadRowsQuery(plan, strategy = "full") {
|
|
8294
|
+
const parameters = new ParameterBuilder();
|
|
8295
|
+
const from = parameters.add(plan.timeRange.from, "DateTime64(3, 'UTC')");
|
|
8296
|
+
const to = parameters.add(plan.timeRange.to, "DateTime64(3, 'UTC')");
|
|
8297
|
+
const scope = [];
|
|
8298
|
+
if (plan.scope) {
|
|
8299
|
+
scope.push(`organizationId = ${parameters.add(plan.scope.organizationId, "String")}`);
|
|
8300
|
+
if (plan.scope.resourceId !== void 0) scope.push(`resourceId = ${parameters.add(plan.scope.resourceId, "String")}`);
|
|
8301
|
+
}
|
|
8302
|
+
if (strategy === "historyBudget") return {
|
|
8303
|
+
query: `SELECT count() AS rootCount FROM (
|
|
8304
|
+
SELECT 1 FROM ${TABLE_TRACE_ROOTS} ${scope.length ? `WHERE ${scope.join(" AND ")}` : ""}
|
|
8305
|
+
LIMIT 250001
|
|
8306
|
+
)`,
|
|
8307
|
+
query_params: parameters.params
|
|
8308
|
+
};
|
|
8309
|
+
if (strategy === "probe") return {
|
|
8310
|
+
query: `WITH ${candidateCtes(scope, from, to, plan, true)}
|
|
8311
|
+
SELECT (SELECT count() FROM candidate_threads) AS threadCount,
|
|
8312
|
+
(SELECT sum(length(ifNull(organizationId, '')) + length(threadId)) FROM candidate_threads) AS threadBytes,
|
|
8313
|
+
count() AS traceCount, sum(length(ifNull(organizationId, '')) + length(traceId)) AS traceBytes
|
|
8314
|
+
FROM candidate_traces`,
|
|
8315
|
+
query_params: parameters.params
|
|
8316
|
+
};
|
|
8317
|
+
const predicate = plan.where ? compileThreadQueryPredicate(plan.where, parameters).sql : "1";
|
|
8318
|
+
const after = afterCursor(plan, parameters);
|
|
8319
|
+
const qualify = plan.timeRange.mode === "traces" ? "s.inWindow > 0" : `s.${plan.timeRange.field} >= ${from} AND s.${plan.timeRange.field} < ${to}`;
|
|
8320
|
+
const filtered = predicateFields(plan.where);
|
|
8321
|
+
const firstContext = [...contextFields, "timeToFirstChunkMs"].filter((field) => field === "resourceId" || filtered.has(field));
|
|
8322
|
+
const firstFields = [
|
|
8323
|
+
"startedAt",
|
|
8324
|
+
"traceId",
|
|
8325
|
+
"spanId",
|
|
8326
|
+
"endedAt",
|
|
8327
|
+
...firstContext
|
|
8328
|
+
];
|
|
8329
|
+
const lastFields = [
|
|
8330
|
+
"startedAt",
|
|
8331
|
+
"traceId",
|
|
8332
|
+
"spanId",
|
|
8333
|
+
"endedAt",
|
|
8334
|
+
"resourceId",
|
|
8335
|
+
...filtered.has("status") ? ["status"] : []
|
|
8336
|
+
];
|
|
8337
|
+
const rootDataFields = [
|
|
8338
|
+
"threadId",
|
|
8339
|
+
"spanId",
|
|
8340
|
+
"startedAt",
|
|
8341
|
+
"endedAt",
|
|
8342
|
+
...firstContext,
|
|
8343
|
+
...filtered.has("status") ? ["status"] : []
|
|
8344
|
+
];
|
|
8345
|
+
const rootExpression = (field) => field === "timeToFirstChunkMs" ? "if(spanType = 'agent_run', timeToFirstChunkMs, NULL)" : field === "status" ? "if(isNull(error), 'success', 'error')" : field;
|
|
8346
|
+
const sortedRoots = `roots AS (
|
|
8347
|
+
SELECT organizationId, traceId, ${rootDataFields.map((field) => `${rootExpression(field)} AS ${field}`).join(", ")}
|
|
8348
|
+
FROM ${TABLE_TRACE_ROOTS} ${scope.length || strategy === "candidates" ? `WHERE ${[...scope, ...strategy === "candidates" ? [candidateTracesFilter] : []].join(" AND ")}` : ""}
|
|
8349
|
+
ORDER BY organizationId, traceId, startedAt ASC, spanId, endedAt DESC, isNotNull(threadId) DESC, dedupeKey
|
|
8350
|
+
LIMIT 1 BY organizationId, traceId
|
|
8351
|
+
)`;
|
|
8352
|
+
return {
|
|
8353
|
+
query: `WITH ${strategy === "candidates" ? `${candidateCtes(scope, from, to, plan)}, ` : ""}${sortedRoots}, grouped AS (
|
|
8354
|
+
SELECT organizationId, threadId, min(startedAt) AS firstActivity, max(endedAt) AS lastActivity,
|
|
8355
|
+
count() AS traceCount, countIf(startedAt >= ${from} AND startedAt < ${to}) AS inWindow,
|
|
8356
|
+
min(tuple(${firstFields.join(", ")})) AS first,
|
|
8357
|
+
max(tuple(${lastFields.join(", ")})) AS last
|
|
8358
|
+
FROM roots WHERE threadId IS NOT NULL ${strategy === "candidates" ? `AND ${candidateThreadsFilter}` : ""} GROUP BY organizationId, threadId
|
|
8359
|
+
), rows AS (
|
|
8360
|
+
SELECT organizationId, threadId, firstActivity AS startedAt, lastActivity AS endedAt, traceCount, inWindow,
|
|
8361
|
+
${firstFields.map((field, i) => `first.${i + 1} AS ${i < 4 ? `first${field[0].toUpperCase()}${field.slice(1)}` : field}`).join(", ")},
|
|
8362
|
+
${lastFields.map((field, i) => `last.${i + 1} AS ${field === "status" ? "status" : `last${field[0].toUpperCase()}${field.slice(1)}`}`).join(", ")}
|
|
8363
|
+
FROM grouped
|
|
8364
|
+
) SELECT * FROM rows s WHERE ${qualify} AND (${predicate}) AND ${after}
|
|
8365
|
+
ORDER BY s.${plan.orderBy.field} ${plan.orderBy.direction === "asc" ? "ASC" : "DESC"}, s.organizationId ASC NULLS LAST, s.threadId ASC
|
|
8366
|
+
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
8367
|
+
query_params: parameters.params
|
|
8368
|
+
};
|
|
8369
|
+
}
|
|
8370
|
+
const rootKey = (row) => JSON.stringify([
|
|
8371
|
+
row.organizationId,
|
|
8372
|
+
row.traceId,
|
|
8373
|
+
row.spanId
|
|
8374
|
+
]);
|
|
8375
|
+
const iso = (value) => new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8376
|
+
/** Bound HTTP form parameter size as well as payload rows. Long IDs must not break large pages. */
|
|
8377
|
+
function* batches(rows, maxRows = 256) {
|
|
8378
|
+
let batch = [];
|
|
8379
|
+
let bytes = 0;
|
|
8380
|
+
for (const row of rows) {
|
|
8381
|
+
const size = Buffer.byteLength(JSON.stringify(row), "utf8") * 2;
|
|
8382
|
+
if (batch.length && (batch.length >= maxRows || bytes + size > 48 * 1024)) {
|
|
8383
|
+
yield batch;
|
|
8384
|
+
batch = [];
|
|
8385
|
+
bytes = 0;
|
|
8386
|
+
}
|
|
8387
|
+
batch.push(row);
|
|
8388
|
+
bytes += size;
|
|
8389
|
+
}
|
|
8390
|
+
if (batch.length) yield batch;
|
|
8391
|
+
}
|
|
8392
|
+
async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
|
|
8393
|
+
const deadline = performance.now() + _mastra_core_storage.resolveTraceQueryTimeoutMs(timeoutMs);
|
|
8394
|
+
const run = async (compiled) => {
|
|
8395
|
+
const remaining = Math.floor(deadline - performance.now());
|
|
8396
|
+
if (remaining <= 0) throw new _mastra_core_storage.TraceQueryExecutionError();
|
|
8397
|
+
return await runWithClickHouseTraceQueryTimeout(client, {
|
|
8398
|
+
timeoutMs: remaining,
|
|
8399
|
+
...THREAD_QUERY_EXECUTION_LIMITS
|
|
8400
|
+
}, compiled);
|
|
8401
|
+
};
|
|
8402
|
+
if (strategy === "auto") {
|
|
8403
|
+
const [probe] = await run(compileClickHouseThreadRowsQuery(plan, "probe"));
|
|
8404
|
+
if (Number(probe.threadCount) === 0) return {
|
|
8405
|
+
threads: [],
|
|
8406
|
+
page: { next: null }
|
|
8407
|
+
};
|
|
8408
|
+
strategy = canUseThreadCandidates(probe) ? "candidates" : "full";
|
|
8409
|
+
}
|
|
8410
|
+
if (strategy === "full") {
|
|
8411
|
+
const [budget] = await run(compileClickHouseThreadRowsQuery(plan, "historyBudget"));
|
|
8412
|
+
if (Number(budget.rootCount) > MAX_FULL_HISTORY_ROOTS) throw new _mastra_core_storage.TraceQueryResourceLimitError();
|
|
8413
|
+
}
|
|
8414
|
+
const selected = await run(compileClickHouseThreadRowsQuery(plan, strategy));
|
|
8415
|
+
const page = selected.slice(0, plan.limit);
|
|
8416
|
+
if (!page.length) return {
|
|
8417
|
+
threads: [],
|
|
8418
|
+
page: { next: null }
|
|
8419
|
+
};
|
|
8420
|
+
const roots = /* @__PURE__ */ new Map();
|
|
8421
|
+
const representative = (row, which) => ({
|
|
8422
|
+
organizationId: row.organizationId,
|
|
8423
|
+
resourceId: which === "first" ? row.resourceId : row.lastResourceId,
|
|
8424
|
+
traceId: row[`${which}TraceId`],
|
|
8425
|
+
spanId: row[`${which}SpanId`],
|
|
8426
|
+
startedAt: row[`${which}StartedAt`],
|
|
8427
|
+
endedAt: row[`${which}EndedAt`]
|
|
8428
|
+
});
|
|
8429
|
+
for (const row of page) for (const which of ["first", "last"]) {
|
|
8430
|
+
const root = representative(row, which);
|
|
8431
|
+
roots.set(rootKey(root), root);
|
|
8432
|
+
}
|
|
8433
|
+
const keys = (batch) => batch.map((root) => new _clickhouse_client.TupleParam([
|
|
8434
|
+
root.organizationId === null ? 1 : 0,
|
|
8435
|
+
root.organizationId ?? "",
|
|
8436
|
+
root.resourceId === null ? 1 : 0,
|
|
8437
|
+
root.resourceId ?? "",
|
|
8438
|
+
root.traceId,
|
|
8439
|
+
root.spanId,
|
|
8440
|
+
iso(root.startedAt).replace("T", " ").replace(/Z$/, ""),
|
|
8441
|
+
iso(root.endedAt).replace("T", " ").replace(/Z$/, "")
|
|
8442
|
+
]));
|
|
8443
|
+
const rootWhere = `tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''),
|
|
8444
|
+
traceId, spanId, startedAt, endedAt) IN {roots:Array(Tuple(UInt8, String, UInt8, String, String, String, DateTime64(3, 'UTC'), DateTime64(3, 'UTC')))}`;
|
|
8445
|
+
const stored = [];
|
|
8446
|
+
for (const batch of batches([...roots.values()])) stored.push(...await run({
|
|
8447
|
+
query: `SELECT organizationId, traceId, spanId, startedAt, endedAt, spanType,
|
|
8448
|
+
${contextFields.join(", ")}, if(spanType = 'agent_run', timeToFirstChunkMs, NULL) AS timeToFirstChunkMs,
|
|
8449
|
+
if(isNull(error), 'success', 'error') AS status, ${PREVIEW_COLUMNS.join(", ")}
|
|
8450
|
+
FROM ${TABLE_TRACE_ROOTS} WHERE ${rootWhere}
|
|
8451
|
+
ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8452
|
+
query_params: { roots: keys(batch) }
|
|
8453
|
+
}));
|
|
8454
|
+
const byRoot = new Map(stored.map((row) => [rootKey(row), row]));
|
|
8455
|
+
const missing = stored.filter(needsPreviewPayloads);
|
|
8456
|
+
for (const batch of batches(missing, 32)) {
|
|
8457
|
+
const payloads = await run({
|
|
8458
|
+
query: `SELECT organizationId, traceId, spanId, input, output, error FROM ${TABLE_TRACE_ROOTS}
|
|
8459
|
+
WHERE ${rootWhere} ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8460
|
+
query_params: { roots: keys(batch) }
|
|
8461
|
+
});
|
|
8462
|
+
for (const payload of payloads) {
|
|
8463
|
+
const row = byRoot.get(rootKey(payload));
|
|
8464
|
+
if (row) Object.assign(row, resolveStoredPreviews(row, payload));
|
|
8465
|
+
}
|
|
8466
|
+
}
|
|
8467
|
+
const firstRoots = page.map((row) => representative(row, "first")).filter((root) => byRoot.get(rootKey(root))?.spanType === "agent_run");
|
|
8468
|
+
const models = [];
|
|
8469
|
+
for (const batch of batches(firstRoots)) models.push(...await run({
|
|
8470
|
+
query: `SELECT organizationId, traceId,
|
|
8471
|
+
if(JSONType(attributes, 'model') = 'String', JSONExtractString(attributes, 'model'), NULL) AS model,
|
|
8472
|
+
if(JSONType(attributes, 'provider') = 'String', JSONExtractString(attributes, 'provider'), NULL) AS provider
|
|
8473
|
+
FROM ${TABLE_SPAN_EVENTS} WHERE spanType = 'model_generation' AND
|
|
8474
|
+
tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''), traceId, parentSpanId)
|
|
8475
|
+
IN {agents:Array(Tuple(UInt8, String, UInt8, String, String, String))}
|
|
8476
|
+
ORDER BY organizationId, traceId, startedAt, spanId LIMIT 1 BY organizationId, traceId`,
|
|
8477
|
+
query_params: { agents: batch.map((root) => new _clickhouse_client.TupleParam([
|
|
8478
|
+
root.organizationId === null ? 1 : 0,
|
|
8479
|
+
root.organizationId ?? "",
|
|
8480
|
+
root.resourceId === null ? 1 : 0,
|
|
8481
|
+
root.resourceId ?? "",
|
|
8482
|
+
root.traceId,
|
|
8483
|
+
root.spanId
|
|
8484
|
+
])) }
|
|
8485
|
+
}));
|
|
8486
|
+
const byTrace = new Map(models.map((row) => [JSON.stringify([row.organizationId, row.traceId]), row]));
|
|
8487
|
+
const threads = page.flatMap((row) => {
|
|
8488
|
+
const first = byRoot.get(rootKey(representative(row, "first")));
|
|
8489
|
+
const last = byRoot.get(rootKey(representative(row, "last")));
|
|
8490
|
+
if (!first || !last) return [];
|
|
8491
|
+
const model = byTrace.get(JSON.stringify([row.organizationId, row.firstTraceId]));
|
|
8492
|
+
return [_mastra_core_storage.threadQueryRowSchema.parse({
|
|
8493
|
+
organizationId: row.organizationId,
|
|
8494
|
+
threadId: row.threadId,
|
|
8495
|
+
firstTraceId: row.firstTraceId,
|
|
8496
|
+
lastTraceId: row.lastTraceId,
|
|
8497
|
+
...Object.fromEntries(contextFields.map((field) => [field, first[field]])),
|
|
8498
|
+
status: last.status,
|
|
8499
|
+
startedAt: iso(row.startedAt),
|
|
8500
|
+
endedAt: iso(row.endedAt),
|
|
8501
|
+
traceCount: Number(row.traceCount),
|
|
8502
|
+
timeToFirstChunkMs: first.timeToFirstChunkMs,
|
|
8503
|
+
model: model?.model ?? null,
|
|
8504
|
+
provider: model?.provider ?? null,
|
|
8505
|
+
inputPreview: resolveStoredPreviews(first).inputPreview,
|
|
8506
|
+
outputPreview: resolveStoredPreviews(last).outputPreview,
|
|
8507
|
+
errorPreview: resolveStoredPreviews(last).errorPreview
|
|
8508
|
+
})];
|
|
8509
|
+
});
|
|
8510
|
+
const last = page.at(-1);
|
|
8511
|
+
return {
|
|
8512
|
+
threads,
|
|
8513
|
+
page: { next: selected.length > plan.limit ? _mastra_core_storage.encodeThreadQueryCursor(plan, {
|
|
8514
|
+
organizationId: last.organizationId,
|
|
8515
|
+
threadId: last.threadId,
|
|
8516
|
+
sortValue: iso(last[plan.orderBy.field])
|
|
8517
|
+
}) : null }
|
|
8518
|
+
};
|
|
8519
|
+
}
|
|
8520
|
+
//#endregion
|
|
8183
8521
|
//#region src/storage/domains/observability/v-next/trace-aggregate.ts
|
|
8184
8522
|
/** Trace-root column expressions (alias `r`) for each groupable dimension; same values `where` sees. */
|
|
8185
8523
|
const DIMENSION_SQL = {
|
|
8186
8524
|
entityType: "r.entityType",
|
|
8187
8525
|
entityName: "r.entityName",
|
|
8526
|
+
name: "r.name",
|
|
8527
|
+
spanType: "r.spanType",
|
|
8188
8528
|
environment: "r.environment",
|
|
8189
8529
|
status: TRACE_STATUS_SQL,
|
|
8190
8530
|
serviceName: "r.serviceName",
|
|
@@ -9328,6 +9668,7 @@ function buildBranchCursor(row) {
|
|
|
9328
9668
|
//#endregion
|
|
9329
9669
|
//#region src/storage/domains/observability/v-next/index.ts
|
|
9330
9670
|
const spanQueryFeatures = typeof _mastra_core_storage.planSpanQuery === "function" ? ["span-query", "span-query-context-fields"] : [];
|
|
9671
|
+
const threadQueryFeatures = typeof _mastra_core_storage.planThreadRowsQuery === "function" ? ["thread-query-rows"] : [];
|
|
9331
9672
|
function buildSignalMigrationRequiredMessage(args) {
|
|
9332
9673
|
const tableList = args.tables.map((table) => ` - ${table.table} (${table.engine})`).join("\n");
|
|
9333
9674
|
return `
|
|
@@ -9765,8 +10106,11 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9765
10106
|
"trace-query-tenant-scope",
|
|
9766
10107
|
"feedback",
|
|
9767
10108
|
"trace-query-context-ids",
|
|
10109
|
+
...timeToFirstChunkFeatures,
|
|
10110
|
+
"trace-query-root-span-fields",
|
|
9768
10111
|
"trace-aggregate",
|
|
9769
|
-
...spanQueryFeatures
|
|
10112
|
+
...spanQueryFeatures,
|
|
10113
|
+
...threadQueryFeatures
|
|
9770
10114
|
];
|
|
9771
10115
|
return [
|
|
9772
10116
|
"metrics",
|
|
@@ -9785,8 +10129,11 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9785
10129
|
"trace-query-tenant-scope",
|
|
9786
10130
|
"feedback",
|
|
9787
10131
|
"trace-query-context-ids",
|
|
10132
|
+
...timeToFirstChunkFeatures,
|
|
10133
|
+
"trace-query-root-span-fields",
|
|
9788
10134
|
"trace-aggregate",
|
|
9789
|
-
...spanQueryFeatures
|
|
10135
|
+
...spanQueryFeatures,
|
|
10136
|
+
...threadQueryFeatures
|
|
9790
10137
|
];
|
|
9791
10138
|
}
|
|
9792
10139
|
async createSpan(args) {
|
|
@@ -9963,6 +10310,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
9963
10310
|
}
|
|
9964
10311
|
async queryThreads(plan) {
|
|
9965
10312
|
try {
|
|
10313
|
+
if (plan.result === "threadRows") return await queryThreadRows(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9966
10314
|
return await queryThreads(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9967
10315
|
} catch (error) {
|
|
9968
10316
|
if (error instanceof _mastra_core_error.MastraError || error instanceof _mastra_core_storage.TraceQueryExecutionError || error instanceof _mastra_core_storage.TraceQueryResourceLimitError) throw error;
|