@mastra/clickhouse 1.26.0-alpha.2 → 1.26.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +362 -14
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +362 -14
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +4 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/span-query.d.ts +1 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/thread-query.d.ts +55 -0
- package/dist/storage/domains/observability/v-next/thread-query.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/trace-aggregate.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +11 -0
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/package.json +5 -6
package/dist/index.js
CHANGED
|
@@ -6841,6 +6841,12 @@ async function getScorePercentiles(client, args) {
|
|
|
6841
6841
|
//#endregion
|
|
6842
6842
|
//#region src/storage/domains/observability/v-next/trace-query.ts
|
|
6843
6843
|
const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
|
|
6844
|
+
/**
|
|
6845
|
+
* Core versions without the field reject it in strict query results, so it is only returned when core knows it.
|
|
6846
|
+
* Core before 1.68 has no field registry at all.
|
|
6847
|
+
*/
|
|
6848
|
+
const timeToFirstChunkSupported = "timeToFirstChunkMs" in (coreStorage.TRACE_QUERY_FIELD_REGISTRY?.trace ?? {});
|
|
6849
|
+
const timeToFirstChunkFeatures = timeToFirstChunkSupported ? ["trace-query-time-to-first-chunk"] : [];
|
|
6844
6850
|
function durationMsSql(startedAt, endedAt) {
|
|
6845
6851
|
return `dateDiff('millisecond', ${startedAt}, ${endedAt})`;
|
|
6846
6852
|
}
|
|
@@ -6889,6 +6895,10 @@ const TRACE_FIELDS = {
|
|
|
6889
6895
|
sql: durationMsSql("r.startedAt", "r.endedAt"),
|
|
6890
6896
|
parameterType: "Float64"
|
|
6891
6897
|
},
|
|
6898
|
+
timeToFirstChunkMs: {
|
|
6899
|
+
sql: "r.timeToFirstChunkMs",
|
|
6900
|
+
parameterType: "Float64"
|
|
6901
|
+
},
|
|
6892
6902
|
entityName: {
|
|
6893
6903
|
sql: "r.entityName",
|
|
6894
6904
|
parameterType: "String"
|
|
@@ -6897,6 +6907,14 @@ const TRACE_FIELDS = {
|
|
|
6897
6907
|
sql: "r.entityType",
|
|
6898
6908
|
parameterType: "String"
|
|
6899
6909
|
},
|
|
6910
|
+
name: {
|
|
6911
|
+
sql: "r.name",
|
|
6912
|
+
parameterType: "String"
|
|
6913
|
+
},
|
|
6914
|
+
spanType: {
|
|
6915
|
+
sql: "r.spanType",
|
|
6916
|
+
parameterType: "String"
|
|
6917
|
+
},
|
|
6900
6918
|
environment: {
|
|
6901
6919
|
sql: "r.environment",
|
|
6902
6920
|
parameterType: "String"
|
|
@@ -6939,6 +6957,10 @@ const SPAN_FIELDS = {
|
|
|
6939
6957
|
sql: "s.durationMs",
|
|
6940
6958
|
parameterType: "Float64"
|
|
6941
6959
|
},
|
|
6960
|
+
timeToFirstChunkMs: {
|
|
6961
|
+
sql: "s.timeToFirstChunkMs",
|
|
6962
|
+
parameterType: "Float64"
|
|
6963
|
+
},
|
|
6942
6964
|
status: {
|
|
6943
6965
|
sql: "s.status",
|
|
6944
6966
|
parameterType: "String"
|
|
@@ -7150,6 +7172,7 @@ const TRACE_SELECT = `
|
|
|
7150
7172
|
r.entityName AS entityName,
|
|
7151
7173
|
r.entityType AS entityType,
|
|
7152
7174
|
r.environment AS environment,
|
|
7175
|
+
r.timeToFirstChunkMs AS timeToFirstChunkMs,
|
|
7153
7176
|
${TRACE_STATUS_SQL} AS status`;
|
|
7154
7177
|
var ParameterBuilder = class {
|
|
7155
7178
|
params = {};
|
|
@@ -7325,6 +7348,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7325
7348
|
startedAt,
|
|
7326
7349
|
endedAt,
|
|
7327
7350
|
${durationMsSql("startedAt", "endedAt")} AS durationMs,
|
|
7351
|
+
timeToFirstChunkMs,
|
|
7328
7352
|
if(isNotNull(error), 'error', 'success') AS status,
|
|
7329
7353
|
error,
|
|
7330
7354
|
entityType,
|
|
@@ -7737,6 +7761,12 @@ async function runWithClickHouseTraceQueryTimeout(client, limits, compiled, quer
|
|
|
7737
7761
|
...CH_SETTINGS,
|
|
7738
7762
|
max_execution_time: resolvedTimeoutMs / 1e3,
|
|
7739
7763
|
...limits.memoryLimitBytes === void 0 ? {} : { max_memory_usage: String(limits.memoryLimitBytes) },
|
|
7764
|
+
...limits.maxThreads === void 0 ? {} : { max_threads: limits.maxThreads },
|
|
7765
|
+
...limits.maxBlockSize === void 0 ? {} : { max_block_size: String(limits.maxBlockSize) },
|
|
7766
|
+
...limits.spillThresholdBytes === void 0 ? {} : {
|
|
7767
|
+
max_bytes_before_external_sort: String(limits.spillThresholdBytes),
|
|
7768
|
+
max_bytes_before_external_group_by: String(limits.spillThresholdBytes)
|
|
7769
|
+
},
|
|
7740
7770
|
...compiled.sharedSnapshot ? { enable_shared_storage_snapshot_in_query: 1 } : {}
|
|
7741
7771
|
}
|
|
7742
7772
|
})).json();
|
|
@@ -7859,7 +7889,8 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7859
7889
|
entityName: row.entityName == null ? null : String(row.entityName),
|
|
7860
7890
|
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7861
7891
|
environment: row.environment == null ? null : String(row.environment),
|
|
7862
|
-
status: row.status
|
|
7892
|
+
status: row.status,
|
|
7893
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: row.timeToFirstChunkMs == null ? null : Number(row.timeToFirstChunkMs) }
|
|
7863
7894
|
});
|
|
7864
7895
|
if (plan.paginationMode === "delta") {
|
|
7865
7896
|
const rows = await run(compileClickHouseTraceQuery(plan, deltaHead));
|
|
@@ -7934,6 +7965,16 @@ function compileSpanQueryPredicate(predicate) {
|
|
|
7934
7965
|
params: parameters.params
|
|
7935
7966
|
};
|
|
7936
7967
|
}
|
|
7968
|
+
/** Thread filters see full-history representative values, never individual member traces. */
|
|
7969
|
+
function compileThreadQueryPredicate(predicate, parameters = new ParameterBuilder()) {
|
|
7970
|
+
return {
|
|
7971
|
+
sql: compileScalarPredicate(predicate, Object.fromEntries(Object.entries(coreStorage.THREAD_QUERY_FIELD_REGISTRY).map(([field, rule]) => [field, {
|
|
7972
|
+
sql: `s.${field}`,
|
|
7973
|
+
parameterType: rule.valueKind === "timestamp" ? "DateTime64(3, 'UTC')" : rule.valueKind === "number" ? "Float64" : "String"
|
|
7974
|
+
}])), parameters),
|
|
7975
|
+
params: parameters.params
|
|
7976
|
+
};
|
|
7977
|
+
}
|
|
7937
7978
|
//#endregion
|
|
7938
7979
|
//#region src/storage/domains/observability/v-next/span-query.ts
|
|
7939
7980
|
const identityFields = [
|
|
@@ -7974,6 +8015,7 @@ const displayFields = [
|
|
|
7974
8015
|
"model",
|
|
7975
8016
|
"provider",
|
|
7976
8017
|
"durationMs",
|
|
8018
|
+
"timeToFirstChunkMs",
|
|
7977
8019
|
"status"
|
|
7978
8020
|
];
|
|
7979
8021
|
var Parameters = class {
|
|
@@ -7994,7 +8036,7 @@ function identityValues(row) {
|
|
|
7994
8036
|
row.spanId
|
|
7995
8037
|
];
|
|
7996
8038
|
}
|
|
7997
|
-
function afterCursor(plan, parameters) {
|
|
8039
|
+
function afterCursor$1(plan, parameters) {
|
|
7998
8040
|
if (!plan.cursor) return "1";
|
|
7999
8041
|
const cursor = plan.cursor;
|
|
8000
8042
|
const timestamp = parameters.add(cursor.sortValue, true);
|
|
@@ -8012,10 +8054,10 @@ function afterCursor(plan, parameters) {
|
|
|
8012
8054
|
}
|
|
8013
8055
|
return `(${terms.join(" OR ")})`;
|
|
8014
8056
|
}
|
|
8015
|
-
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8057
|
+
function predicateFields$1(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8016
8058
|
if (!predicate) return fields;
|
|
8017
|
-
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8018
|
-
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8059
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields$1(arg, fields);
|
|
8060
|
+
else if (predicate.type === "not") predicateFields$1(predicate.arg, fields);
|
|
8019
8061
|
else fields.add(predicate.field);
|
|
8020
8062
|
return fields;
|
|
8021
8063
|
}
|
|
@@ -8042,10 +8084,10 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8042
8084
|
const selectionFields = [.../* @__PURE__ */ new Set([
|
|
8043
8085
|
"startedAt",
|
|
8044
8086
|
"endedAt",
|
|
8045
|
-
...predicateFields(plan.where)
|
|
8087
|
+
...predicateFields$1(plan.where)
|
|
8046
8088
|
])].filter((field) => !new Set(identityFields).has(field));
|
|
8047
8089
|
const expressions = selectionFields.map(fieldExpression);
|
|
8048
|
-
const after = afterCursor(plan, parameters);
|
|
8090
|
+
const after = afterCursor$1(plan, parameters);
|
|
8049
8091
|
return {
|
|
8050
8092
|
query: `WITH candidates AS (
|
|
8051
8093
|
SELECT ${identities}, ${selectionFields.map((field, index) => `${expressions[index]} AS ${field}`).join(", ")}
|
|
@@ -8070,7 +8112,7 @@ function compileClickHouseSpanQuery(plan) {
|
|
|
8070
8112
|
};
|
|
8071
8113
|
}
|
|
8072
8114
|
const LEGACY_PAYLOAD_BATCH_ROWS = 32;
|
|
8073
|
-
function iso(value) {
|
|
8115
|
+
function iso$1(value) {
|
|
8074
8116
|
return new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8075
8117
|
}
|
|
8076
8118
|
async function querySpans(client, plan, timeoutMs) {
|
|
@@ -8085,8 +8127,8 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8085
8127
|
};
|
|
8086
8128
|
const selected = (await run(compileClickHouseSpanQuery(plan))).map((row) => ({
|
|
8087
8129
|
...row,
|
|
8088
|
-
startedAt: iso(row.startedAt),
|
|
8089
|
-
endedAt: iso(row.endedAt)
|
|
8130
|
+
startedAt: iso$1(row.startedAt),
|
|
8131
|
+
endedAt: iso$1(row.endedAt)
|
|
8090
8132
|
}));
|
|
8091
8133
|
const page = selected.slice(0, plan.limit);
|
|
8092
8134
|
if (!page.length) return {
|
|
@@ -8124,11 +8166,12 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8124
8166
|
const hydrated = page.flatMap((identity) => {
|
|
8125
8167
|
const row = byIdentity.get(coreStorage.spanQueryIdentityKey(identity));
|
|
8126
8168
|
if (!row) return [];
|
|
8127
|
-
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, ...metadata } = row;
|
|
8169
|
+
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, timeToFirstChunkMs, ...metadata } = row;
|
|
8128
8170
|
return [{
|
|
8129
8171
|
...metadata,
|
|
8130
8172
|
...identity,
|
|
8131
|
-
durationMs: Number(row.durationMs)
|
|
8173
|
+
durationMs: Number(row.durationMs),
|
|
8174
|
+
...timeToFirstChunkSupported && { timeToFirstChunkMs: timeToFirstChunkMs == null ? null : Number(timeToFirstChunkMs) }
|
|
8132
8175
|
}];
|
|
8133
8176
|
});
|
|
8134
8177
|
let metrics = [];
|
|
@@ -8157,11 +8200,308 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8157
8200
|
return result;
|
|
8158
8201
|
}
|
|
8159
8202
|
//#endregion
|
|
8203
|
+
//#region src/storage/domains/observability/v-next/thread-query.ts
|
|
8204
|
+
const contextFields = [
|
|
8205
|
+
"resourceId",
|
|
8206
|
+
"userId",
|
|
8207
|
+
"entityType",
|
|
8208
|
+
"entityId",
|
|
8209
|
+
"entityName",
|
|
8210
|
+
"entityVersionId",
|
|
8211
|
+
"runId",
|
|
8212
|
+
"sessionId",
|
|
8213
|
+
"requestId",
|
|
8214
|
+
"serviceName",
|
|
8215
|
+
"experimentId",
|
|
8216
|
+
"environment",
|
|
8217
|
+
"executionSource"
|
|
8218
|
+
];
|
|
8219
|
+
function predicateFields(predicate, fields = /* @__PURE__ */ new Set()) {
|
|
8220
|
+
if (!predicate) return fields;
|
|
8221
|
+
if (predicate.type === "boolean") for (const arg of predicate.args) predicateFields(arg, fields);
|
|
8222
|
+
else if (predicate.type === "not") predicateFields(predicate.arg, fields);
|
|
8223
|
+
else fields.add(predicate.field);
|
|
8224
|
+
return fields;
|
|
8225
|
+
}
|
|
8226
|
+
function afterCursor(plan, parameters) {
|
|
8227
|
+
if (!plan.cursor) return "1";
|
|
8228
|
+
const { cursor } = plan;
|
|
8229
|
+
const stamp = parameters.add(cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
8230
|
+
const field = `s.${plan.orderBy.field}`;
|
|
8231
|
+
const terms = [`${field} ${plan.orderBy.direction === "asc" ? ">" : "<"} ${stamp}`];
|
|
8232
|
+
const equal = [`${field} = ${stamp}`];
|
|
8233
|
+
for (const key of ["organizationId", "threadId"]) if (cursor[key] === null) equal.push(`isNull(s.${key})`);
|
|
8234
|
+
else {
|
|
8235
|
+
const value = parameters.add(cursor[key], "String");
|
|
8236
|
+
terms.push(`(${equal.join(" AND ")} AND (s.${key} > ${value} OR isNull(s.${key})))`);
|
|
8237
|
+
equal.push(`s.${key} = ${value}`);
|
|
8238
|
+
}
|
|
8239
|
+
return `(${terms.join(" OR ")})`;
|
|
8240
|
+
}
|
|
8241
|
+
const THREAD_QUERY_EXECUTION_LIMITS = {
|
|
8242
|
+
memoryLimitBytes: 512 * 1024 * 1024,
|
|
8243
|
+
maxThreads: 2,
|
|
8244
|
+
maxBlockSize: 4096,
|
|
8245
|
+
spillThresholdBytes: 32 * 1024 * 1024
|
|
8246
|
+
};
|
|
8247
|
+
const MAX_CANDIDATE_THREADS = 16384;
|
|
8248
|
+
const MAX_CANDIDATE_TRACES = 1e5;
|
|
8249
|
+
const MAX_CANDIDATE_ID_BYTES = 4 * 1024 * 1024;
|
|
8250
|
+
const MAX_FULL_HISTORY_ROOTS = 25e4;
|
|
8251
|
+
function canUseThreadCandidates(probe) {
|
|
8252
|
+
return Number(probe.threadCount) <= MAX_CANDIDATE_THREADS && Number(probe.traceCount) <= MAX_CANDIDATE_TRACES && Number(probe.threadBytes) + Number(probe.traceBytes) <= MAX_CANDIDATE_ID_BYTES;
|
|
8253
|
+
}
|
|
8254
|
+
const identity = (field) => `tuple(isNull(organizationId), ifNull(organizationId, ''), ${field})`;
|
|
8255
|
+
const candidateThreadsFilter = `${identity("threadId")} IN (SELECT ${identity("threadId")} FROM candidate_threads)`;
|
|
8256
|
+
const candidateTracesFilter = `${identity("traceId")} IN (SELECT ${identity("traceId")} FROM candidate_traces)`;
|
|
8257
|
+
function candidateCtes(scope, from, to, plan, probe = false) {
|
|
8258
|
+
const timestamp = plan.timeRange.mode === "threads" ? plan.timeRange.field : "startedAt";
|
|
8259
|
+
const scoped = scope.length ? `${scope.join(" AND ")} AND ` : "";
|
|
8260
|
+
return `candidate_threads AS (
|
|
8261
|
+
SELECT DISTINCT organizationId, threadId FROM ${TABLE_TRACE_ROOTS}
|
|
8262
|
+
WHERE ${scoped}threadId IS NOT NULL AND ${timestamp} >= ${from} AND ${timestamp} < ${to}
|
|
8263
|
+
${probe ? `LIMIT 16385` : ""}
|
|
8264
|
+
), candidate_traces AS (
|
|
8265
|
+
SELECT DISTINCT organizationId, traceId FROM ${TABLE_TRACE_ROOTS}
|
|
8266
|
+
WHERE ${scoped}${candidateThreadsFilter}
|
|
8267
|
+
${probe ? `LIMIT 100001` : ""}
|
|
8268
|
+
)`;
|
|
8269
|
+
}
|
|
8270
|
+
function compileClickHouseThreadRowsQuery(plan, strategy = "full") {
|
|
8271
|
+
const parameters = new ParameterBuilder();
|
|
8272
|
+
const from = parameters.add(plan.timeRange.from, "DateTime64(3, 'UTC')");
|
|
8273
|
+
const to = parameters.add(plan.timeRange.to, "DateTime64(3, 'UTC')");
|
|
8274
|
+
const scope = [];
|
|
8275
|
+
if (plan.scope) {
|
|
8276
|
+
scope.push(`organizationId = ${parameters.add(plan.scope.organizationId, "String")}`);
|
|
8277
|
+
if (plan.scope.resourceId !== void 0) scope.push(`resourceId = ${parameters.add(plan.scope.resourceId, "String")}`);
|
|
8278
|
+
}
|
|
8279
|
+
if (strategy === "historyBudget") return {
|
|
8280
|
+
query: `SELECT count() AS rootCount FROM (
|
|
8281
|
+
SELECT 1 FROM ${TABLE_TRACE_ROOTS} ${scope.length ? `WHERE ${scope.join(" AND ")}` : ""}
|
|
8282
|
+
LIMIT 250001
|
|
8283
|
+
)`,
|
|
8284
|
+
query_params: parameters.params
|
|
8285
|
+
};
|
|
8286
|
+
if (strategy === "probe") return {
|
|
8287
|
+
query: `WITH ${candidateCtes(scope, from, to, plan, true)}
|
|
8288
|
+
SELECT (SELECT count() FROM candidate_threads) AS threadCount,
|
|
8289
|
+
(SELECT sum(length(ifNull(organizationId, '')) + length(threadId)) FROM candidate_threads) AS threadBytes,
|
|
8290
|
+
count() AS traceCount, sum(length(ifNull(organizationId, '')) + length(traceId)) AS traceBytes
|
|
8291
|
+
FROM candidate_traces`,
|
|
8292
|
+
query_params: parameters.params
|
|
8293
|
+
};
|
|
8294
|
+
const predicate = plan.where ? compileThreadQueryPredicate(plan.where, parameters).sql : "1";
|
|
8295
|
+
const after = afterCursor(plan, parameters);
|
|
8296
|
+
const qualify = plan.timeRange.mode === "traces" ? "s.inWindow > 0" : `s.${plan.timeRange.field} >= ${from} AND s.${plan.timeRange.field} < ${to}`;
|
|
8297
|
+
const filtered = predicateFields(plan.where);
|
|
8298
|
+
const firstContext = [...contextFields, "timeToFirstChunkMs"].filter((field) => field === "resourceId" || filtered.has(field));
|
|
8299
|
+
const firstFields = [
|
|
8300
|
+
"startedAt",
|
|
8301
|
+
"traceId",
|
|
8302
|
+
"spanId",
|
|
8303
|
+
"endedAt",
|
|
8304
|
+
...firstContext
|
|
8305
|
+
];
|
|
8306
|
+
const lastFields = [
|
|
8307
|
+
"startedAt",
|
|
8308
|
+
"traceId",
|
|
8309
|
+
"spanId",
|
|
8310
|
+
"endedAt",
|
|
8311
|
+
"resourceId",
|
|
8312
|
+
...filtered.has("status") ? ["status"] : []
|
|
8313
|
+
];
|
|
8314
|
+
const rootDataFields = [
|
|
8315
|
+
"threadId",
|
|
8316
|
+
"spanId",
|
|
8317
|
+
"startedAt",
|
|
8318
|
+
"endedAt",
|
|
8319
|
+
...firstContext,
|
|
8320
|
+
...filtered.has("status") ? ["status"] : []
|
|
8321
|
+
];
|
|
8322
|
+
const rootExpression = (field) => field === "timeToFirstChunkMs" ? "if(spanType = 'agent_run', timeToFirstChunkMs, NULL)" : field === "status" ? "if(isNull(error), 'success', 'error')" : field;
|
|
8323
|
+
const sortedRoots = `roots AS (
|
|
8324
|
+
SELECT organizationId, traceId, ${rootDataFields.map((field) => `${rootExpression(field)} AS ${field}`).join(", ")}
|
|
8325
|
+
FROM ${TABLE_TRACE_ROOTS} ${scope.length || strategy === "candidates" ? `WHERE ${[...scope, ...strategy === "candidates" ? [candidateTracesFilter] : []].join(" AND ")}` : ""}
|
|
8326
|
+
ORDER BY organizationId, traceId, startedAt ASC, spanId, endedAt DESC, isNotNull(threadId) DESC, dedupeKey
|
|
8327
|
+
LIMIT 1 BY organizationId, traceId
|
|
8328
|
+
)`;
|
|
8329
|
+
return {
|
|
8330
|
+
query: `WITH ${strategy === "candidates" ? `${candidateCtes(scope, from, to, plan)}, ` : ""}${sortedRoots}, grouped AS (
|
|
8331
|
+
SELECT organizationId, threadId, min(startedAt) AS firstActivity, max(endedAt) AS lastActivity,
|
|
8332
|
+
count() AS traceCount, countIf(startedAt >= ${from} AND startedAt < ${to}) AS inWindow,
|
|
8333
|
+
min(tuple(${firstFields.join(", ")})) AS first,
|
|
8334
|
+
max(tuple(${lastFields.join(", ")})) AS last
|
|
8335
|
+
FROM roots WHERE threadId IS NOT NULL ${strategy === "candidates" ? `AND ${candidateThreadsFilter}` : ""} GROUP BY organizationId, threadId
|
|
8336
|
+
), rows AS (
|
|
8337
|
+
SELECT organizationId, threadId, firstActivity AS startedAt, lastActivity AS endedAt, traceCount, inWindow,
|
|
8338
|
+
${firstFields.map((field, i) => `first.${i + 1} AS ${i < 4 ? `first${field[0].toUpperCase()}${field.slice(1)}` : field}`).join(", ")},
|
|
8339
|
+
${lastFields.map((field, i) => `last.${i + 1} AS ${field === "status" ? "status" : `last${field[0].toUpperCase()}${field.slice(1)}`}`).join(", ")}
|
|
8340
|
+
FROM grouped
|
|
8341
|
+
) SELECT * FROM rows s WHERE ${qualify} AND (${predicate}) AND ${after}
|
|
8342
|
+
ORDER BY s.${plan.orderBy.field} ${plan.orderBy.direction === "asc" ? "ASC" : "DESC"}, s.organizationId ASC NULLS LAST, s.threadId ASC
|
|
8343
|
+
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
8344
|
+
query_params: parameters.params
|
|
8345
|
+
};
|
|
8346
|
+
}
|
|
8347
|
+
const rootKey = (row) => JSON.stringify([
|
|
8348
|
+
row.organizationId,
|
|
8349
|
+
row.traceId,
|
|
8350
|
+
row.spanId
|
|
8351
|
+
]);
|
|
8352
|
+
const iso = (value) => new Date(value.includes("T") ? value : `${value.replace(" ", "T")}Z`).toISOString();
|
|
8353
|
+
/** Bound HTTP form parameter size as well as payload rows. Long IDs must not break large pages. */
|
|
8354
|
+
function* batches(rows, maxRows = 256) {
|
|
8355
|
+
let batch = [];
|
|
8356
|
+
let bytes = 0;
|
|
8357
|
+
for (const row of rows) {
|
|
8358
|
+
const size = Buffer.byteLength(JSON.stringify(row), "utf8") * 2;
|
|
8359
|
+
if (batch.length && (batch.length >= maxRows || bytes + size > 48 * 1024)) {
|
|
8360
|
+
yield batch;
|
|
8361
|
+
batch = [];
|
|
8362
|
+
bytes = 0;
|
|
8363
|
+
}
|
|
8364
|
+
batch.push(row);
|
|
8365
|
+
bytes += size;
|
|
8366
|
+
}
|
|
8367
|
+
if (batch.length) yield batch;
|
|
8368
|
+
}
|
|
8369
|
+
async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
|
|
8370
|
+
const deadline = performance.now() + coreStorage.resolveTraceQueryTimeoutMs(timeoutMs);
|
|
8371
|
+
const run = async (compiled) => {
|
|
8372
|
+
const remaining = Math.floor(deadline - performance.now());
|
|
8373
|
+
if (remaining <= 0) throw new coreStorage.TraceQueryExecutionError();
|
|
8374
|
+
return await runWithClickHouseTraceQueryTimeout(client, {
|
|
8375
|
+
timeoutMs: remaining,
|
|
8376
|
+
...THREAD_QUERY_EXECUTION_LIMITS
|
|
8377
|
+
}, compiled);
|
|
8378
|
+
};
|
|
8379
|
+
if (strategy === "auto") {
|
|
8380
|
+
const [probe] = await run(compileClickHouseThreadRowsQuery(plan, "probe"));
|
|
8381
|
+
if (Number(probe.threadCount) === 0) return {
|
|
8382
|
+
threads: [],
|
|
8383
|
+
page: { next: null }
|
|
8384
|
+
};
|
|
8385
|
+
strategy = canUseThreadCandidates(probe) ? "candidates" : "full";
|
|
8386
|
+
}
|
|
8387
|
+
if (strategy === "full") {
|
|
8388
|
+
const [budget] = await run(compileClickHouseThreadRowsQuery(plan, "historyBudget"));
|
|
8389
|
+
if (Number(budget.rootCount) > MAX_FULL_HISTORY_ROOTS) throw new coreStorage.TraceQueryResourceLimitError();
|
|
8390
|
+
}
|
|
8391
|
+
const selected = await run(compileClickHouseThreadRowsQuery(plan, strategy));
|
|
8392
|
+
const page = selected.slice(0, plan.limit);
|
|
8393
|
+
if (!page.length) return {
|
|
8394
|
+
threads: [],
|
|
8395
|
+
page: { next: null }
|
|
8396
|
+
};
|
|
8397
|
+
const roots = /* @__PURE__ */ new Map();
|
|
8398
|
+
const representative = (row, which) => ({
|
|
8399
|
+
organizationId: row.organizationId,
|
|
8400
|
+
resourceId: which === "first" ? row.resourceId : row.lastResourceId,
|
|
8401
|
+
traceId: row[`${which}TraceId`],
|
|
8402
|
+
spanId: row[`${which}SpanId`],
|
|
8403
|
+
startedAt: row[`${which}StartedAt`],
|
|
8404
|
+
endedAt: row[`${which}EndedAt`]
|
|
8405
|
+
});
|
|
8406
|
+
for (const row of page) for (const which of ["first", "last"]) {
|
|
8407
|
+
const root = representative(row, which);
|
|
8408
|
+
roots.set(rootKey(root), root);
|
|
8409
|
+
}
|
|
8410
|
+
const keys = (batch) => batch.map((root) => new TupleParam([
|
|
8411
|
+
root.organizationId === null ? 1 : 0,
|
|
8412
|
+
root.organizationId ?? "",
|
|
8413
|
+
root.resourceId === null ? 1 : 0,
|
|
8414
|
+
root.resourceId ?? "",
|
|
8415
|
+
root.traceId,
|
|
8416
|
+
root.spanId,
|
|
8417
|
+
iso(root.startedAt).replace("T", " ").replace(/Z$/, ""),
|
|
8418
|
+
iso(root.endedAt).replace("T", " ").replace(/Z$/, "")
|
|
8419
|
+
]));
|
|
8420
|
+
const rootWhere = `tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''),
|
|
8421
|
+
traceId, spanId, startedAt, endedAt) IN {roots:Array(Tuple(UInt8, String, UInt8, String, String, String, DateTime64(3, 'UTC'), DateTime64(3, 'UTC')))}`;
|
|
8422
|
+
const stored = [];
|
|
8423
|
+
for (const batch of batches([...roots.values()])) stored.push(...await run({
|
|
8424
|
+
query: `SELECT organizationId, traceId, spanId, startedAt, endedAt, spanType,
|
|
8425
|
+
${contextFields.join(", ")}, if(spanType = 'agent_run', timeToFirstChunkMs, NULL) AS timeToFirstChunkMs,
|
|
8426
|
+
if(isNull(error), 'success', 'error') AS status, ${PREVIEW_COLUMNS.join(", ")}
|
|
8427
|
+
FROM ${TABLE_TRACE_ROOTS} WHERE ${rootWhere}
|
|
8428
|
+
ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8429
|
+
query_params: { roots: keys(batch) }
|
|
8430
|
+
}));
|
|
8431
|
+
const byRoot = new Map(stored.map((row) => [rootKey(row), row]));
|
|
8432
|
+
const missing = stored.filter(needsPreviewPayloads);
|
|
8433
|
+
for (const batch of batches(missing, 32)) {
|
|
8434
|
+
const payloads = await run({
|
|
8435
|
+
query: `SELECT organizationId, traceId, spanId, input, output, error FROM ${TABLE_TRACE_ROOTS}
|
|
8436
|
+
WHERE ${rootWhere} ORDER BY isNotNull(threadId) DESC LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8437
|
+
query_params: { roots: keys(batch) }
|
|
8438
|
+
});
|
|
8439
|
+
for (const payload of payloads) {
|
|
8440
|
+
const row = byRoot.get(rootKey(payload));
|
|
8441
|
+
if (row) Object.assign(row, resolveStoredPreviews(row, payload));
|
|
8442
|
+
}
|
|
8443
|
+
}
|
|
8444
|
+
const firstRoots = page.map((row) => representative(row, "first")).filter((root) => byRoot.get(rootKey(root))?.spanType === "agent_run");
|
|
8445
|
+
const models = [];
|
|
8446
|
+
for (const batch of batches(firstRoots)) models.push(...await run({
|
|
8447
|
+
query: `SELECT organizationId, traceId,
|
|
8448
|
+
if(JSONType(attributes, 'model') = 'String', JSONExtractString(attributes, 'model'), NULL) AS model,
|
|
8449
|
+
if(JSONType(attributes, 'provider') = 'String', JSONExtractString(attributes, 'provider'), NULL) AS provider
|
|
8450
|
+
FROM ${TABLE_SPAN_EVENTS} WHERE spanType = 'model_generation' AND
|
|
8451
|
+
tuple(isNull(organizationId), ifNull(organizationId, ''), isNull(resourceId), ifNull(resourceId, ''), traceId, parentSpanId)
|
|
8452
|
+
IN {agents:Array(Tuple(UInt8, String, UInt8, String, String, String))}
|
|
8453
|
+
ORDER BY organizationId, traceId, startedAt, spanId LIMIT 1 BY organizationId, traceId`,
|
|
8454
|
+
query_params: { agents: batch.map((root) => new TupleParam([
|
|
8455
|
+
root.organizationId === null ? 1 : 0,
|
|
8456
|
+
root.organizationId ?? "",
|
|
8457
|
+
root.resourceId === null ? 1 : 0,
|
|
8458
|
+
root.resourceId ?? "",
|
|
8459
|
+
root.traceId,
|
|
8460
|
+
root.spanId
|
|
8461
|
+
])) }
|
|
8462
|
+
}));
|
|
8463
|
+
const byTrace = new Map(models.map((row) => [JSON.stringify([row.organizationId, row.traceId]), row]));
|
|
8464
|
+
const threads = page.flatMap((row) => {
|
|
8465
|
+
const first = byRoot.get(rootKey(representative(row, "first")));
|
|
8466
|
+
const last = byRoot.get(rootKey(representative(row, "last")));
|
|
8467
|
+
if (!first || !last) return [];
|
|
8468
|
+
const model = byTrace.get(JSON.stringify([row.organizationId, row.firstTraceId]));
|
|
8469
|
+
return [coreStorage.threadQueryRowSchema.parse({
|
|
8470
|
+
organizationId: row.organizationId,
|
|
8471
|
+
threadId: row.threadId,
|
|
8472
|
+
firstTraceId: row.firstTraceId,
|
|
8473
|
+
lastTraceId: row.lastTraceId,
|
|
8474
|
+
...Object.fromEntries(contextFields.map((field) => [field, first[field]])),
|
|
8475
|
+
status: last.status,
|
|
8476
|
+
startedAt: iso(row.startedAt),
|
|
8477
|
+
endedAt: iso(row.endedAt),
|
|
8478
|
+
traceCount: Number(row.traceCount),
|
|
8479
|
+
timeToFirstChunkMs: first.timeToFirstChunkMs,
|
|
8480
|
+
model: model?.model ?? null,
|
|
8481
|
+
provider: model?.provider ?? null,
|
|
8482
|
+
inputPreview: resolveStoredPreviews(first).inputPreview,
|
|
8483
|
+
outputPreview: resolveStoredPreviews(last).outputPreview,
|
|
8484
|
+
errorPreview: resolveStoredPreviews(last).errorPreview
|
|
8485
|
+
})];
|
|
8486
|
+
});
|
|
8487
|
+
const last = page.at(-1);
|
|
8488
|
+
return {
|
|
8489
|
+
threads,
|
|
8490
|
+
page: { next: selected.length > plan.limit ? coreStorage.encodeThreadQueryCursor(plan, {
|
|
8491
|
+
organizationId: last.organizationId,
|
|
8492
|
+
threadId: last.threadId,
|
|
8493
|
+
sortValue: iso(last[plan.orderBy.field])
|
|
8494
|
+
}) : null }
|
|
8495
|
+
};
|
|
8496
|
+
}
|
|
8497
|
+
//#endregion
|
|
8160
8498
|
//#region src/storage/domains/observability/v-next/trace-aggregate.ts
|
|
8161
8499
|
/** Trace-root column expressions (alias `r`) for each groupable dimension; same values `where` sees. */
|
|
8162
8500
|
const DIMENSION_SQL = {
|
|
8163
8501
|
entityType: "r.entityType",
|
|
8164
8502
|
entityName: "r.entityName",
|
|
8503
|
+
name: "r.name",
|
|
8504
|
+
spanType: "r.spanType",
|
|
8165
8505
|
environment: "r.environment",
|
|
8166
8506
|
status: TRACE_STATUS_SQL,
|
|
8167
8507
|
serviceName: "r.serviceName",
|
|
@@ -9305,6 +9645,7 @@ function buildBranchCursor(row) {
|
|
|
9305
9645
|
//#endregion
|
|
9306
9646
|
//#region src/storage/domains/observability/v-next/index.ts
|
|
9307
9647
|
const spanQueryFeatures = typeof coreStorage.planSpanQuery === "function" ? ["span-query", "span-query-context-fields"] : [];
|
|
9648
|
+
const threadQueryFeatures = typeof coreStorage.planThreadRowsQuery === "function" ? ["thread-query-rows"] : [];
|
|
9308
9649
|
function buildSignalMigrationRequiredMessage(args) {
|
|
9309
9650
|
const tableList = args.tables.map((table) => ` - ${table.table} (${table.engine})`).join("\n");
|
|
9310
9651
|
return `
|
|
@@ -9742,8 +10083,11 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
9742
10083
|
"trace-query-tenant-scope",
|
|
9743
10084
|
"feedback",
|
|
9744
10085
|
"trace-query-context-ids",
|
|
10086
|
+
...timeToFirstChunkFeatures,
|
|
10087
|
+
"trace-query-root-span-fields",
|
|
9745
10088
|
"trace-aggregate",
|
|
9746
|
-
...spanQueryFeatures
|
|
10089
|
+
...spanQueryFeatures,
|
|
10090
|
+
...threadQueryFeatures
|
|
9747
10091
|
];
|
|
9748
10092
|
return [
|
|
9749
10093
|
"metrics",
|
|
@@ -9762,8 +10106,11 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
9762
10106
|
"trace-query-tenant-scope",
|
|
9763
10107
|
"feedback",
|
|
9764
10108
|
"trace-query-context-ids",
|
|
10109
|
+
...timeToFirstChunkFeatures,
|
|
10110
|
+
"trace-query-root-span-fields",
|
|
9765
10111
|
"trace-aggregate",
|
|
9766
|
-
...spanQueryFeatures
|
|
10112
|
+
...spanQueryFeatures,
|
|
10113
|
+
...threadQueryFeatures
|
|
9767
10114
|
];
|
|
9768
10115
|
}
|
|
9769
10116
|
async createSpan(args) {
|
|
@@ -9940,6 +10287,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
9940
10287
|
}
|
|
9941
10288
|
async queryThreads(plan) {
|
|
9942
10289
|
try {
|
|
10290
|
+
if (plan.result === "threadRows") return await queryThreadRows(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9943
10291
|
return await queryThreads(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
9944
10292
|
} catch (error) {
|
|
9945
10293
|
if (error instanceof MastraError || error instanceof coreStorage.TraceQueryExecutionError || error instanceof coreStorage.TraceQueryResourceLimitError) throw error;
|