@mastra/clickhouse 1.25.1-alpha.0 → 1.25.1-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/integrations-databases-clickhouse.md +1 -1
- package/dist/docs/references/reference-storage-retention.md +15 -12
- package/dist/index.cjs +451 -187
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +452 -188
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +3 -3
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/helpers.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts +45 -0
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/previews.d.ts +16 -0
- package/dist/storage/domains/observability/v-next/previews.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts +2 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-aggregate.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +66 -11
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts +2 -2
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -2756,7 +2756,14 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SPAN_EVENTS} (
|
|
|
2756
2756
|
output Nullable(String),
|
|
2757
2757
|
error Nullable(String),
|
|
2758
2758
|
metadataRaw Nullable(String),
|
|
2759
|
-
requestContext Nullable(String)
|
|
2759
|
+
requestContext Nullable(String),
|
|
2760
|
+
|
|
2761
|
+
-- Previews of \`input\` / \`output\` / \`error\` in core's \`buildPreview\` format, written
|
|
2762
|
+
-- on every span ('' when there is nothing to preview). NULL on rows written before
|
|
2763
|
+
-- the columns existed; readers build those from the payload columns.
|
|
2764
|
+
inputPreview Nullable(String),
|
|
2765
|
+
outputPreview Nullable(String),
|
|
2766
|
+
errorPreview Nullable(String)
|
|
2760
2767
|
)
|
|
2761
2768
|
ENGINE = ReplacingMergeTree
|
|
2762
2769
|
PARTITION BY toDate(endedAt)
|
|
@@ -2824,6 +2831,12 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_ROOTS} (
|
|
|
2824
2831
|
metadataRaw Nullable(String),
|
|
2825
2832
|
requestContext Nullable(String),
|
|
2826
2833
|
|
|
2834
|
+
-- Previews of \`input\` / \`output\` / \`error\` ('' when there is nothing to preview).
|
|
2835
|
+
-- NULL on rows written before the columns existed.
|
|
2836
|
+
inputPreview Nullable(String),
|
|
2837
|
+
outputPreview Nullable(String),
|
|
2838
|
+
errorPreview Nullable(String),
|
|
2839
|
+
|
|
2827
2840
|
-- traceId is second in the sort key, behind startedAt, so point lookups by
|
|
2828
2841
|
-- traceId (getRootSpan, trace deletes, trace-query root dedupe) need a skip
|
|
2829
2842
|
-- index to avoid scanning every granule.
|
|
@@ -2902,6 +2915,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_BRANCHES} (
|
|
|
2902
2915
|
error Nullable(String),
|
|
2903
2916
|
metadataRaw Nullable(String),
|
|
2904
2917
|
requestContext Nullable(String),
|
|
2918
|
+
inputPreview Nullable(String),
|
|
2919
|
+
outputPreview Nullable(String),
|
|
2920
|
+
errorPreview Nullable(String),
|
|
2905
2921
|
|
|
2906
2922
|
INDEX idx_traceId traceId TYPE bloom_filter(0.01) GRANULARITY 1
|
|
2907
2923
|
)
|
|
@@ -3633,7 +3649,26 @@ const addBloomIndex = (table, name, column, granularity = 2) => ({
|
|
|
3633
3649
|
name,
|
|
3634
3650
|
sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY ${granularity}`
|
|
3635
3651
|
});
|
|
3652
|
+
const LOG_SKIP_INDEX_COLUMNS = [
|
|
3653
|
+
"traceId",
|
|
3654
|
+
"threadId",
|
|
3655
|
+
"resourceId",
|
|
3656
|
+
"userId",
|
|
3657
|
+
"organizationId",
|
|
3658
|
+
"experimentId",
|
|
3659
|
+
"runId",
|
|
3660
|
+
"sessionId",
|
|
3661
|
+
"requestId"
|
|
3662
|
+
];
|
|
3636
3663
|
const ALL_MIGRATIONS = [
|
|
3664
|
+
...[
|
|
3665
|
+
"inputPreview",
|
|
3666
|
+
"outputPreview",
|
|
3667
|
+
"errorPreview"
|
|
3668
|
+
].flatMap((column) => [addColumn(TABLE_TRACE_ROOTS, column, "Nullable(String)"), addColumn(TABLE_TRACE_BRANCHES, column, "Nullable(String)")]),
|
|
3669
|
+
addColumn(TABLE_SPAN_EVENTS, "inputPreview", "Nullable(String)"),
|
|
3670
|
+
addColumn(TABLE_SPAN_EVENTS, "outputPreview", "Nullable(String)"),
|
|
3671
|
+
addColumn(TABLE_SPAN_EVENTS, "errorPreview", "Nullable(String)"),
|
|
3637
3672
|
addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
|
|
3638
3673
|
addColumn(TABLE_SPAN_EVENTS, "parentEntityVersionId", "Nullable(String)"),
|
|
3639
3674
|
addColumn(TABLE_SPAN_EVENTS, "rootEntityVersionId", "Nullable(String)"),
|
|
@@ -3670,17 +3705,7 @@ const ALL_MIGRATIONS = [
|
|
|
3670
3705
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_runId", "runId"),
|
|
3671
3706
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
|
|
3672
3707
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId"),
|
|
3673
|
-
...
|
|
3674
|
-
"traceId",
|
|
3675
|
-
"threadId",
|
|
3676
|
-
"resourceId",
|
|
3677
|
-
"userId",
|
|
3678
|
-
"organizationId",
|
|
3679
|
-
"experimentId",
|
|
3680
|
-
"runId",
|
|
3681
|
-
"sessionId",
|
|
3682
|
-
"requestId"
|
|
3683
|
-
].map((column) => addBloomIndex(TABLE_LOG_EVENTS, `idx_${column}`, column)),
|
|
3708
|
+
...LOG_SKIP_INDEX_COLUMNS.map((column) => addBloomIndex(TABLE_LOG_EVENTS, `idx_${column}`, column)),
|
|
3684
3709
|
addBloomIndex(TABLE_DELETION_REQUESTS, "idx_predicateValues", "predicateValues")
|
|
3685
3710
|
];
|
|
3686
3711
|
const ALL_TABLE_NAMES = [
|
|
@@ -3790,6 +3815,78 @@ function parseTtlExpression(expr) {
|
|
|
3790
3815
|
};
|
|
3791
3816
|
}
|
|
3792
3817
|
//#endregion
|
|
3818
|
+
//#region src/storage/domains/observability/v-next/preview-compat.ts
|
|
3819
|
+
const generator = _mastra_core_storage.buildStoredPreview;
|
|
3820
|
+
const canGeneratePreviews = typeof generator === "function";
|
|
3821
|
+
function needsPreview(value) {
|
|
3822
|
+
return canGeneratePreviews && value == null;
|
|
3823
|
+
}
|
|
3824
|
+
function storedPreview(preview, value, serialized) {
|
|
3825
|
+
return canGeneratePreviews ? generator(preview, value, serialized) : preview ?? null;
|
|
3826
|
+
}
|
|
3827
|
+
function readPreview(preview, payload, jsonEncoded = false) {
|
|
3828
|
+
const supplied = preview;
|
|
3829
|
+
if (!canGeneratePreviews) return supplied ?? null;
|
|
3830
|
+
if (supplied != null) return supplied.trim() || null;
|
|
3831
|
+
let value = payload;
|
|
3832
|
+
if (needsPreview(preview) && jsonEncoded && typeof payload === "string") try {
|
|
3833
|
+
value = JSON.parse(payload);
|
|
3834
|
+
} catch {
|
|
3835
|
+
value = null;
|
|
3836
|
+
}
|
|
3837
|
+
return storedPreview(supplied, value, jsonEncoded && typeof payload === "string" ? payload : void 0) || null;
|
|
3838
|
+
}
|
|
3839
|
+
/** Older response builders may shorten previews or turn empty strings into null. */
|
|
3840
|
+
function buildSpanQueryResponse(...args) {
|
|
3841
|
+
if (canGeneratePreviews) return _mastra_core_storage.buildSpanQueryResponse(...args);
|
|
3842
|
+
const [plan, selected, payloads, metrics] = args;
|
|
3843
|
+
const response = _mastra_core_storage.buildSpanQueryResponse(plan, selected, [], metrics);
|
|
3844
|
+
const byIdentity = new Map(payloads.map((payload) => [_mastra_core_storage.spanQueryIdentityKey(payload), payload]));
|
|
3845
|
+
for (const span of response.spans) {
|
|
3846
|
+
const payload = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(span));
|
|
3847
|
+
span.inputPreview = payload?.inputPreview ?? null;
|
|
3848
|
+
span.outputPreview = payload?.outputPreview ?? null;
|
|
3849
|
+
span.errorPreview = payload?.errorPreview ?? null;
|
|
3850
|
+
}
|
|
3851
|
+
return response;
|
|
3852
|
+
}
|
|
3853
|
+
//#endregion
|
|
3854
|
+
//#region src/storage/domains/observability/v-next/previews.ts
|
|
3855
|
+
/**
|
|
3856
|
+
* Stored span previews (`inputPreview` / `outputPreview` / `errorPreview`).
|
|
3857
|
+
*
|
|
3858
|
+
* The writer stores each preview in core's `buildPreview` format, `''` when there is nothing to
|
|
3859
|
+
* preview. A NULL preview means the row was written before the columns existed; readers fetch the
|
|
3860
|
+
* payload columns for just those rows and build the same preview from them.
|
|
3861
|
+
*/
|
|
3862
|
+
const PREVIEW_COLUMNS = [
|
|
3863
|
+
"inputPreview",
|
|
3864
|
+
"outputPreview",
|
|
3865
|
+
"errorPreview"
|
|
3866
|
+
];
|
|
3867
|
+
/** Payload column each preview is built from. */
|
|
3868
|
+
const PREVIEW_PAYLOAD_COLUMNS = {
|
|
3869
|
+
inputPreview: "input",
|
|
3870
|
+
outputPreview: "output",
|
|
3871
|
+
errorPreview: "error"
|
|
3872
|
+
};
|
|
3873
|
+
/** Whether a row was written before previews were stored, so it needs its payload columns. */
|
|
3874
|
+
function needsPreviewPayloads(row) {
|
|
3875
|
+
return PREVIEW_COLUMNS.some((column) => needsPreview(row[column]));
|
|
3876
|
+
}
|
|
3877
|
+
/**
|
|
3878
|
+
* The previews a list returns for a row: the stored ones (`''` becomes `null`), or for a row written
|
|
3879
|
+
* before they were stored, built from the payload columns on `payloads` (default: the row itself).
|
|
3880
|
+
*/
|
|
3881
|
+
function resolveStoredPreviews(row, payloads = row) {
|
|
3882
|
+
const previews = {};
|
|
3883
|
+
for (const column of PREVIEW_COLUMNS) {
|
|
3884
|
+
const stored = row[column];
|
|
3885
|
+
previews[column] = readPreview(stored, payloads?.[PREVIEW_PAYLOAD_COLUMNS[column]], true);
|
|
3886
|
+
}
|
|
3887
|
+
return previews;
|
|
3888
|
+
}
|
|
3889
|
+
//#endregion
|
|
3793
3890
|
//#region src/storage/domains/observability/v-next/review-status.ts
|
|
3794
3891
|
const FEEDBACK_REVIEW_STATUSES = ["needs-review", "reviewed"];
|
|
3795
3892
|
function isFeedbackReviewStatus(value) {
|
|
@@ -3996,7 +4093,7 @@ function rowToLightSpanRecord(row) {
|
|
|
3996
4093
|
endedAt
|
|
3997
4094
|
}),
|
|
3998
4095
|
metadata: parseJson(row.metadataRaw) ?? void 0,
|
|
3999
|
-
|
|
4096
|
+
...resolveStoredPreviews(row),
|
|
4000
4097
|
createdAt: startedAt,
|
|
4001
4098
|
updatedAt: null
|
|
4002
4099
|
};
|
|
@@ -4004,6 +4101,9 @@ function rowToLightSpanRecord(row) {
|
|
|
4004
4101
|
function spanRecordToRow(span) {
|
|
4005
4102
|
const endedAt = span.isEvent ? span.startedAt : span.endedAt ?? span.startedAt;
|
|
4006
4103
|
const metadata = span.metadata ?? null;
|
|
4104
|
+
const input = jsonEncode(span.input);
|
|
4105
|
+
const output = jsonEncode(span.output);
|
|
4106
|
+
const error = jsonEncode(span.error);
|
|
4007
4107
|
return {
|
|
4008
4108
|
dedupeKey: buildDedupeKey(span.traceId, span.spanId),
|
|
4009
4109
|
traceId: span.traceId,
|
|
@@ -4043,10 +4143,13 @@ function spanRecordToRow(span) {
|
|
|
4043
4143
|
scope: jsonEncode(span.scope),
|
|
4044
4144
|
attributes: jsonEncode(span.attributes),
|
|
4045
4145
|
links: jsonEncode(span.links),
|
|
4046
|
-
input
|
|
4047
|
-
output
|
|
4048
|
-
error
|
|
4049
|
-
requestContext: jsonEncode(span.requestContext)
|
|
4146
|
+
input,
|
|
4147
|
+
output,
|
|
4148
|
+
error,
|
|
4149
|
+
requestContext: jsonEncode(span.requestContext),
|
|
4150
|
+
inputPreview: storedPreview(span.inputPreview, span.input, input),
|
|
4151
|
+
outputPreview: storedPreview(span.outputPreview, span.output, output),
|
|
4152
|
+
errorPreview: storedPreview(span.errorPreview, span.error, error)
|
|
4050
4153
|
};
|
|
4051
4154
|
}
|
|
4052
4155
|
function rowToLogRecord(row) {
|
|
@@ -6961,11 +7064,15 @@ const FEEDBACK_FIELDS = {
|
|
|
6961
7064
|
const TRACE_SELECT = `
|
|
6962
7065
|
r.traceId AS traceId,
|
|
6963
7066
|
r.spanId AS rootSpanId,
|
|
7067
|
+
r.dedupeKey AS dedupeKey,
|
|
7068
|
+
r.organizationId AS organizationId,
|
|
6964
7069
|
r.name AS name,
|
|
6965
7070
|
r.entityId AS entityId,
|
|
6966
7071
|
r.parentSpanId AS parentSpanId,
|
|
6967
7072
|
r.metadataRaw AS metadata,
|
|
6968
|
-
r.
|
|
7073
|
+
r.inputPreview AS inputPreview,
|
|
7074
|
+
r.outputPreview AS outputPreview,
|
|
7075
|
+
r.errorPreview AS errorPreview,
|
|
6969
7076
|
r.threadId AS threadId,
|
|
6970
7077
|
r.resourceId AS resourceId,
|
|
6971
7078
|
r.startedAt AS startedAt,
|
|
@@ -6974,22 +7081,6 @@ const TRACE_SELECT = `
|
|
|
6974
7081
|
r.entityType AS entityType,
|
|
6975
7082
|
r.environment AS environment,
|
|
6976
7083
|
${TRACE_STATUS_SQL} AS status`;
|
|
6977
|
-
/** Candidate columns carried through the page-mode window sort (no payload blobs). */
|
|
6978
|
-
const TRACE_PAGE_COLUMNS = [
|
|
6979
|
-
"traceId",
|
|
6980
|
-
"rootSpanId",
|
|
6981
|
-
"name",
|
|
6982
|
-
"entityId",
|
|
6983
|
-
"parentSpanId",
|
|
6984
|
-
"threadId",
|
|
6985
|
-
"resourceId",
|
|
6986
|
-
"startedAt",
|
|
6987
|
-
"endedAt",
|
|
6988
|
-
"entityName",
|
|
6989
|
-
"entityType",
|
|
6990
|
-
"environment",
|
|
6991
|
-
"status"
|
|
6992
|
-
];
|
|
6993
7084
|
var ParameterBuilder = class {
|
|
6994
7085
|
params = {};
|
|
6995
7086
|
#next = 1;
|
|
@@ -7113,18 +7204,38 @@ function compileTenantScope(scope, parameters) {
|
|
|
7113
7204
|
if (scope.resourceId !== void 0) sql += `\n AND resourceId = ${parameters.add(scope.resourceId, "String")}`;
|
|
7114
7205
|
return sql;
|
|
7115
7206
|
}
|
|
7116
|
-
function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = []) {
|
|
7207
|
+
function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = [], options = {}) {
|
|
7117
7208
|
const from = parameters.add(selection.timeRange.from, "DateTime64(3, 'UTC')");
|
|
7118
7209
|
const to = parameters.add(selection.timeRange.to, "DateTime64(3, 'UTC')");
|
|
7119
7210
|
const tenant = compileTenantScope(scope, parameters);
|
|
7120
|
-
const
|
|
7211
|
+
const seedFilter = seedConjuncts.map((conjunct) => `\n AND (${compilePredicate(conjunct, parameters)})`).join("");
|
|
7212
|
+
const bound = options.delta ? `\n AND (traceId IN (SELECT traceId FROM delta_candidates))` : "";
|
|
7213
|
+
const windowTraces = (more) => `
|
|
7214
|
+
SELECT traceId
|
|
7215
|
+
FROM ${TABLE_TRACE_ROOTS} w
|
|
7216
|
+
WHERE startedAt >= ${from}
|
|
7217
|
+
AND startedAt < ${to}${tenant}${more}
|
|
7218
|
+
`;
|
|
7219
|
+
const delta = options.delta ? [`delta_candidates AS (
|
|
7220
|
+
SELECT traceId, max(cursorId) AS latestCursorId
|
|
7221
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA}
|
|
7222
|
+
WHERE ${options.delta}
|
|
7223
|
+
AND traceId IN (${windowTraces("")})
|
|
7224
|
+
GROUP BY traceId
|
|
7225
|
+
)`] : [];
|
|
7226
|
+
const ctes = options.oneRootPerTrace ? [...delta, `root_scope AS (
|
|
7227
|
+
SELECT *
|
|
7228
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
7229
|
+
WHERE startedAt >= ${from}
|
|
7230
|
+
AND startedAt < ${to}${tenant}${seedFilter}${bound}
|
|
7231
|
+
)`] : [`current_roots AS (
|
|
7121
7232
|
SELECT *
|
|
7122
7233
|
FROM ${TABLE_TRACE_ROOTS}
|
|
7123
7234
|
WHERE traceId IN (
|
|
7124
7235
|
SELECT traceId
|
|
7125
7236
|
FROM ${TABLE_TRACE_ROOTS} r
|
|
7126
7237
|
WHERE startedAt >= ${from}
|
|
7127
|
-
AND startedAt < ${to}${tenant}${
|
|
7238
|
+
AND startedAt < ${to}${tenant}${seedFilter}
|
|
7128
7239
|
)${tenant}
|
|
7129
7240
|
ORDER BY traceId, dedupeKey
|
|
7130
7241
|
LIMIT 1 BY traceId
|
|
@@ -7158,9 +7269,9 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7158
7269
|
organizationId
|
|
7159
7270
|
FROM ${TABLE_SPAN_EVENTS}
|
|
7160
7271
|
WHERE isNotNull(traceId)
|
|
7161
|
-
AND traceId IN (SELECT traceId FROM root_scope)${tenant}
|
|
7272
|
+
AND traceId IN (${options.spanExistence && options.oneRootPerTrace ? windowTraces(bound) : "SELECT traceId FROM root_scope"})${tenant}${options.spanExistence ? "" : `
|
|
7162
7273
|
ORDER BY dedupeKey
|
|
7163
|
-
LIMIT 1 BY dedupeKey
|
|
7274
|
+
LIMIT 1 BY dedupeKey`}
|
|
7164
7275
|
)`);
|
|
7165
7276
|
if (relationCollections.has("scores")) ctes.push(`current_scores AS (
|
|
7166
7277
|
SELECT
|
|
@@ -7228,9 +7339,9 @@ function parseDeltaWatermark(value) {
|
|
|
7228
7339
|
* selection's time range and tenant scope that match its `where` predicate. Trace queries and
|
|
7229
7340
|
* trace aggregates both select from this CTE so they always see the same population.
|
|
7230
7341
|
*/
|
|
7231
|
-
function compileClickHouseTraceCandidates(selection, columns, parameters) {
|
|
7342
|
+
function compileClickHouseTraceCandidates(selection, columns, parameters, options = {}) {
|
|
7232
7343
|
const rootConjuncts = selection.where ? (selection.where.type === "boolean" && selection.where.operator === "and" ? selection.where.args : [selection.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : [];
|
|
7233
|
-
const ctes = compileClickHouseTraceScope(selection, collectRelationCollections(selection.where), parameters, selection.scope, rootConjuncts);
|
|
7344
|
+
const ctes = compileClickHouseTraceScope(selection, collectRelationCollections(selection.where), parameters, selection.scope, rootConjuncts, options);
|
|
7234
7345
|
const predicate = selection.where ? compilePredicate(selection.where, parameters) : "1";
|
|
7235
7346
|
ctes.push(`candidates AS (
|
|
7236
7347
|
SELECT ${columns}
|
|
@@ -7239,94 +7350,191 @@ function compileClickHouseTraceCandidates(selection, columns, parameters) {
|
|
|
7239
7350
|
)`);
|
|
7240
7351
|
return ctes;
|
|
7241
7352
|
}
|
|
7353
|
+
/**
|
|
7354
|
+
* `queryTraces()` statements read one root per trace and only test spans for existence
|
|
7355
|
+
* (TraceScopeOptions). The list statements carry narrow columns; `metadata` and the previews are fetched
|
|
7356
|
+
* for the returned rows afterwards (compileClickHouseTraceRootPayloads).
|
|
7357
|
+
*/
|
|
7358
|
+
const QUERY_TRACES_SCOPE = {
|
|
7359
|
+
oneRootPerTrace: true,
|
|
7360
|
+
spanExistence: true
|
|
7361
|
+
};
|
|
7362
|
+
const TRACE_LIST_SELECT = TRACE_SELECT.split("\n").filter((line) => !/ AS (metadata|inputPreview|outputPreview|errorPreview),$/.test(line)).join("\n");
|
|
7363
|
+
function traceOrder(plan) {
|
|
7364
|
+
return {
|
|
7365
|
+
field: resolveOrderField(plan.orderBy.field),
|
|
7366
|
+
direction: plan.orderBy.direction === "asc" ? "ASC" : "DESC"
|
|
7367
|
+
};
|
|
7368
|
+
}
|
|
7369
|
+
/**
|
|
7370
|
+
* Keyset cursor on a trace's sort value. Applied after each trace is reduced to one sort value:
|
|
7371
|
+
* filtering root rows first would let a trace with several roots (or unmerged copies with
|
|
7372
|
+
* different `endedAt`) come back on a later page under a root that sorts after the cursor.
|
|
7373
|
+
*/
|
|
7374
|
+
function keysetCursorCondition(plan, order, parameters) {
|
|
7375
|
+
if (plan.paginationMode !== "keyset" || !plan.cursor) return "";
|
|
7376
|
+
const comparison = order.direction === "ASC" ? ">" : "<";
|
|
7377
|
+
const sortValue = parameters.add(plan.cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
7378
|
+
return `(sortValue ${comparison} ${sortValue} OR (sortValue = ${sortValue} AND traceId > ${parameters.add(plan.cursor.traceId, "String")}))`;
|
|
7379
|
+
}
|
|
7380
|
+
/**
|
|
7381
|
+
* First statement of keyset and page mode: the page's traces (after the keyset cursor, `limit + 1`
|
|
7382
|
+
* for keyset mode, `perPage` at the page offset for page mode) as root keys, and the
|
|
7383
|
+
* number of matching traces on every row. Per trace, the root row that sorts first gives the sort
|
|
7384
|
+
* value and the key, so the key always names a row with that sort value. No blob columns are read.
|
|
7385
|
+
*/
|
|
7386
|
+
function compileClickHouseTraceQueryKeys(plan) {
|
|
7387
|
+
if (plan.paginationMode === "delta") throw new Error("Delta trace queries have no keys statement");
|
|
7388
|
+
const parameters = new ParameterBuilder();
|
|
7389
|
+
const order = traceOrder(plan);
|
|
7390
|
+
const ctes = compileClickHouseTraceCandidates(plan, "r.traceId AS traceId, r.dedupeKey AS dedupeKey, r.startedAt AS startedAt, r.endedAt AS endedAt", parameters, QUERY_TRACES_SCOPE);
|
|
7391
|
+
const cursor = keysetCursorCondition(plan, order, parameters);
|
|
7392
|
+
const pick = order.direction === "DESC" ? "max" : "min";
|
|
7393
|
+
const limit = plan.paginationMode === "page" ? `LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}` : `LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`;
|
|
7394
|
+
return {
|
|
7395
|
+
query: `WITH ${ctes.join(",\n")},
|
|
7396
|
+
per_trace AS (
|
|
7397
|
+
SELECT traceId, ${pick}(${order.field}) AS sortValue,
|
|
7398
|
+
${pick === "max" ? "argMax" : "argMin"}((dedupeKey, startedAt), ${order.field}) AS root
|
|
7399
|
+
FROM candidates
|
|
7400
|
+
GROUP BY traceId${cursor ? `\n HAVING ${cursor}` : ""}
|
|
7401
|
+
)
|
|
7402
|
+
SELECT traceId, root.1 AS dedupeKey, sortValue, root.2 AS rootStartedAt, count() OVER () AS __total
|
|
7403
|
+
FROM per_trace
|
|
7404
|
+
ORDER BY sortValue ${order.direction}, traceId ASC
|
|
7405
|
+
${limit}`,
|
|
7406
|
+
query_params: parameters.params,
|
|
7407
|
+
sharedSnapshot: true
|
|
7408
|
+
};
|
|
7409
|
+
}
|
|
7410
|
+
/**
|
|
7411
|
+
* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata`
|
|
7412
|
+
* and the stored previews, read by sort-key tuple. Unmerged copies of a root key are redeliveries of one
|
|
7413
|
+
* span; the relation-free `where` conjuncts are re-applied so a copy that doesn't match isn't
|
|
7414
|
+
* returned, and the copy that sorts first is kept, as in the keys statement.
|
|
7415
|
+
*/
|
|
7416
|
+
function compileClickHouseTraceRowsByKey(plan, keys) {
|
|
7417
|
+
const parameters = new ParameterBuilder();
|
|
7418
|
+
const order = traceOrder(plan);
|
|
7419
|
+
const ts = (value) => parameters.add(value, "DateTime64(3, 'UTC')");
|
|
7420
|
+
const byEnd = order.field === "endedAt";
|
|
7421
|
+
const tuples = keys.map((key) => {
|
|
7422
|
+
const started = ts(byEnd ? key.rootStartedAt : key.sortValue);
|
|
7423
|
+
const id = `${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}`;
|
|
7424
|
+
return byEnd ? `(${started}, ${id}, ${ts(key.sortValue)})` : `(${started}, ${id})`;
|
|
7425
|
+
});
|
|
7426
|
+
const predicate = (plan.where ? (plan.where.type === "boolean" && plan.where.operator === "and" ? plan.where.args : [plan.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : []).map((conjunct) => `\n AND (${compilePredicate(conjunct, parameters)})`).join("");
|
|
7427
|
+
return {
|
|
7428
|
+
query: `SELECT ${TRACE_SELECT}
|
|
7429
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
7430
|
+
WHERE (r.startedAt, r.traceId, r.dedupeKey${byEnd ? ", r.endedAt" : ""}) IN (${tuples.join(", ")})${compileTenantScope(plan.scope, parameters)}${predicate}
|
|
7431
|
+
ORDER BY ${order.field} ${order.direction}, traceId ASC
|
|
7432
|
+
LIMIT 1 BY traceId`,
|
|
7433
|
+
query_params: parameters.params
|
|
7434
|
+
};
|
|
7435
|
+
}
|
|
7436
|
+
/** Number of matching traces, for numbered pages past the last one (the keys statement returns no rows). */
|
|
7437
|
+
function compileClickHouseTraceQueryTotal(plan) {
|
|
7438
|
+
const parameters = new ParameterBuilder();
|
|
7439
|
+
return {
|
|
7440
|
+
query: `WITH ${compileClickHouseTraceCandidates(plan, "r.traceId AS traceId", parameters, QUERY_TRACES_SCOPE).join(",\n")}
|
|
7441
|
+
SELECT uniqExact(traceId) AS traces
|
|
7442
|
+
FROM candidates`,
|
|
7443
|
+
query_params: parameters.params,
|
|
7444
|
+
sharedSnapshot: true
|
|
7445
|
+
};
|
|
7446
|
+
}
|
|
7447
|
+
/**
|
|
7448
|
+
* The list statement of groups and delta mode (rows without `metadata` and the previews). Keyset
|
|
7449
|
+
* and page mode list with compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey.
|
|
7450
|
+
* A trace with several candidate rows (several roots, or unmerged copies of one) is listed in delta
|
|
7451
|
+
* mode under the row that sorts first in the list order, as in the keys statement; ties go to the
|
|
7452
|
+
* lowest dedupeKey, then the latest endedAt, so every poll picks the same row.
|
|
7453
|
+
*/
|
|
7242
7454
|
function compileClickHouseTraceQuery(plan, deltaHead) {
|
|
7243
7455
|
const parameters = new ParameterBuilder();
|
|
7244
|
-
|
|
7245
|
-
|
|
7246
|
-
|
|
7456
|
+
if (plan.result === "groups") {
|
|
7457
|
+
const ctes = compileClickHouseTraceCandidates(plan, TRACE_LIST_SELECT, parameters, QUERY_TRACES_SCOPE);
|
|
7458
|
+
const pageCondition = plan.cursor ? `AND threadId > ${parameters.add(plan.cursor.threadId, "String")}` : "";
|
|
7459
|
+
const limit = parameters.add(plan.limit + 1, "UInt64");
|
|
7460
|
+
return {
|
|
7461
|
+
query: `WITH ${ctes.join(",\n")}
|
|
7247
7462
|
SELECT threadId
|
|
7248
7463
|
FROM candidates
|
|
7249
|
-
WHERE isNotNull(threadId) ${
|
|
7464
|
+
WHERE isNotNull(threadId) ${pageCondition}
|
|
7250
7465
|
GROUP BY threadId
|
|
7251
7466
|
ORDER BY threadId ASC
|
|
7252
|
-
LIMIT ${
|
|
7253
|
-
query_params: parameters.params
|
|
7254
|
-
};
|
|
7255
|
-
if (plan.paginationMode === "delta") {
|
|
7256
|
-
const watermark = _mastra_core_storage.getTraceQueryDeltaWatermark(plan, "clickhouse");
|
|
7257
|
-
const after = watermark ? parseDeltaWatermark(watermark) : {
|
|
7258
|
-
cursorId: "0",
|
|
7259
|
-
traceId: ""
|
|
7260
|
-
};
|
|
7261
|
-
const lowerCursor = parameters.add(after.cursorId, "UInt64");
|
|
7262
|
-
return {
|
|
7263
|
-
query: `${candidates}, delta_candidates AS (
|
|
7264
|
-
SELECT traceId, max(cursorId) AS latestCursorId
|
|
7265
|
-
FROM ${TABLE_TRACE_ROOTS_DELTA}
|
|
7266
|
-
WHERE cursorId >= ${lowerCursor} AND tuple(cursorId, traceId) > ${`tuple(${lowerCursor}, ${parameters.add(after.traceId, "String")})`} ${deltaHead ? `AND tuple(cursorId, traceId) <= tuple(${parameters.add(deltaHead.cursorId, "UInt64")}, ${parameters.add(deltaHead.traceId, "String")})` : ""}
|
|
7267
|
-
GROUP BY traceId
|
|
7268
|
-
)
|
|
7269
|
-
SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
|
|
7270
|
-
FROM candidates c
|
|
7271
|
-
INNER JOIN delta_candidates d ON c.traceId = d.traceId
|
|
7272
|
-
ORDER BY d.latestCursorId ASC, c.traceId ASC
|
|
7273
|
-
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
7274
|
-
query_params: parameters.params,
|
|
7275
|
-
sharedSnapshot: true
|
|
7276
|
-
};
|
|
7277
|
-
}
|
|
7278
|
-
const orderField = resolveOrderField(plan.orderBy.field);
|
|
7279
|
-
const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
|
|
7280
|
-
if (plan.paginationMode === "page") {
|
|
7281
|
-
const onPage = `__row_position > ${parameters.add(plan.page * plan.perPage, "UInt64")} AND __row_position <= ${parameters.add((plan.page + 1) * plan.perPage, "UInt64")}`;
|
|
7282
|
-
return {
|
|
7283
|
-
query: `${candidates},
|
|
7284
|
-
page_rows AS (
|
|
7285
|
-
SELECT
|
|
7286
|
-
${TRACE_PAGE_COLUMNS.join(",\n ")},
|
|
7287
|
-
row_number() OVER (ORDER BY ${orderField} ${direction}, traceId ASC) AS __row_position,
|
|
7288
|
-
count() OVER () AS total
|
|
7289
|
-
FROM candidates
|
|
7290
|
-
)
|
|
7291
|
-
SELECT *, if(${onPage}, 0, 1) AS __metadata
|
|
7292
|
-
FROM page_rows
|
|
7293
|
-
WHERE (${onPage}) OR __row_position = 1
|
|
7294
|
-
ORDER BY __metadata ASC, __row_position ASC`,
|
|
7467
|
+
LIMIT ${limit}`,
|
|
7295
7468
|
query_params: parameters.params,
|
|
7296
7469
|
sharedSnapshot: true
|
|
7297
7470
|
};
|
|
7298
7471
|
}
|
|
7299
|
-
|
|
7300
|
-
|
|
7301
|
-
|
|
7302
|
-
|
|
7303
|
-
|
|
7304
|
-
|
|
7472
|
+
if (plan.paginationMode !== "delta") throw new Error("Keyset and page trace queries use compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey");
|
|
7473
|
+
const order = traceOrder(plan);
|
|
7474
|
+
const watermark = _mastra_core_storage.getTraceQueryDeltaWatermark(plan, "clickhouse");
|
|
7475
|
+
const after = watermark ? parseDeltaWatermark(watermark) : {
|
|
7476
|
+
cursorId: "0",
|
|
7477
|
+
traceId: ""
|
|
7478
|
+
};
|
|
7479
|
+
const lowerCursor = parameters.add(after.cursorId, "UInt64");
|
|
7480
|
+
const lower = `tuple(${lowerCursor}, ${parameters.add(after.traceId, "String")})`;
|
|
7481
|
+
const upper = deltaHead ? `AND tuple(cursorId, traceId) <= tuple(${parameters.add(deltaHead.cursorId, "UInt64")}, ${parameters.add(deltaHead.traceId, "String")})` : "";
|
|
7482
|
+
const ctes = compileClickHouseTraceCandidates(plan, TRACE_LIST_SELECT, parameters, {
|
|
7483
|
+
...QUERY_TRACES_SCOPE,
|
|
7484
|
+
delta: `cursorId >= ${lowerCursor} AND tuple(cursorId, traceId) > ${lower} ${upper}`
|
|
7485
|
+
});
|
|
7305
7486
|
const limit = parameters.add(plan.limit + 1, "UInt64");
|
|
7306
7487
|
return {
|
|
7307
|
-
query:
|
|
7308
|
-
SELECT
|
|
7309
|
-
FROM candidates
|
|
7310
|
-
|
|
7311
|
-
ORDER BY
|
|
7488
|
+
query: `WITH ${ctes.join(",\n")}
|
|
7489
|
+
SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
|
|
7490
|
+
FROM candidates c
|
|
7491
|
+
INNER JOIN delta_candidates d ON c.traceId = d.traceId
|
|
7492
|
+
ORDER BY d.latestCursorId ASC, c.traceId ASC, c.${order.field} ${order.direction}, c.dedupeKey ASC, c.endedAt DESC
|
|
7493
|
+
LIMIT 1 BY c.traceId
|
|
7312
7494
|
LIMIT ${limit}`,
|
|
7313
|
-
query_params: parameters.params
|
|
7495
|
+
query_params: parameters.params,
|
|
7496
|
+
sharedSnapshot: true
|
|
7497
|
+
};
|
|
7498
|
+
}
|
|
7499
|
+
/** Map key matching a payload row to the listed row it was read for. */
|
|
7500
|
+
function traceRootRowKey(row) {
|
|
7501
|
+
return JSON.stringify([
|
|
7502
|
+
String(row.traceId),
|
|
7503
|
+
String(row.dedupeKey),
|
|
7504
|
+
row.organizationId == null ? null : String(row.organizationId),
|
|
7505
|
+
row.resourceId == null ? null : String(row.resourceId)
|
|
7506
|
+
]);
|
|
7507
|
+
}
|
|
7508
|
+
/** A listed row's {@link ClickHouseTraceRootRowKey}. */
|
|
7509
|
+
function toTraceRootRowKey(row) {
|
|
7510
|
+
return {
|
|
7511
|
+
traceId: String(row.traceId),
|
|
7512
|
+
dedupeKey: String(row.dedupeKey),
|
|
7513
|
+
startedAt: asIsoTimestamp(row.startedAt),
|
|
7514
|
+
endedAt: asIsoTimestamp(row.endedAt),
|
|
7515
|
+
organizationId: row.organizationId == null ? null : String(row.organizationId),
|
|
7516
|
+
resourceId: row.resourceId == null ? null : String(row.resourceId)
|
|
7314
7517
|
};
|
|
7315
7518
|
}
|
|
7316
7519
|
/**
|
|
7317
|
-
* Fetches
|
|
7318
|
-
*
|
|
7319
|
-
*
|
|
7320
|
-
*
|
|
7321
|
-
*
|
|
7520
|
+
* Fetches payloads for listed rows: `metadata` and the stored previews, or `input` / `output` /
|
|
7521
|
+
* `error` alone for rows written before the previews were stored. Each row is looked up by its
|
|
7522
|
+
* own identity: the trace_roots sort key `(startedAt, traceId, dedupeKey)`, so only the page's
|
|
7523
|
+
* granules are read; `endedAt`, so the payload comes from the same version as the listed row (a
|
|
7524
|
+
* root can have unmerged versions in different `endedAt` partitions); and its own
|
|
7525
|
+
* `organizationId` / `resourceId`, so a page holding several tenants' rows that share a trace and
|
|
7526
|
+
* span id never takes another tenant's payload. NULL tenant columns match only NULL. Callers
|
|
7527
|
+
* match the returned rows back with {@link traceRootRowKey}.
|
|
7322
7528
|
*/
|
|
7323
|
-
function compileClickHouseTraceRootPayloads(keys) {
|
|
7529
|
+
function compileClickHouseTraceRootPayloads(keys, scope, columns = "payload") {
|
|
7324
7530
|
const parameters = new ParameterBuilder();
|
|
7531
|
+
const tenant = (value) => `${parameters.add(value ?? "", "String")}, ${value === null ? 1 : 0}`;
|
|
7532
|
+
const tuples = keys.map((key) => `(${parameters.add(key.startedAt, "DateTime64(3, 'UTC')")}, ${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}, ${parameters.add(key.endedAt, "DateTime64(3, 'UTC')")}, ${tenant(key.organizationId)}, ${tenant(key.resourceId)})`);
|
|
7325
7533
|
return {
|
|
7326
|
-
query: `SELECT traceId, spanId AS rootSpanId, metadataRaw AS metadata, input
|
|
7534
|
+
query: `SELECT traceId, spanId AS rootSpanId, dedupeKey, organizationId, resourceId, ${columns === "payload" ? `metadataRaw AS metadata, ${PREVIEW_COLUMNS.join(", ")}` : "input, output, error"}
|
|
7327
7535
|
FROM ${TABLE_TRACE_ROOTS}
|
|
7328
|
-
WHERE (startedAt, traceId,
|
|
7329
|
-
LIMIT 1 BY traceId,
|
|
7536
|
+
WHERE (startedAt, traceId, dedupeKey, endedAt, ifNull(organizationId, ''), isNull(organizationId), ifNull(resourceId, ''), isNull(resourceId)) IN (${tuples.join(", ")})${compileTenantScope(scope, parameters)}
|
|
7537
|
+
LIMIT 1 BY traceId, dedupeKey, organizationId, resourceId`,
|
|
7330
7538
|
query_params: parameters.params
|
|
7331
7539
|
};
|
|
7332
7540
|
}
|
|
@@ -7520,56 +7728,15 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7520
7728
|
deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead))
|
|
7521
7729
|
});
|
|
7522
7730
|
}
|
|
7523
|
-
|
|
7524
|
-
|
|
7525
|
-
const
|
|
7526
|
-
|
|
7527
|
-
|
|
7528
|
-
|
|
7529
|
-
const payloadRows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: remaining() }, compileClickHouseTraceRootPayloads(pageRows.map((row) => ({
|
|
7530
|
-
traceId: String(row.traceId),
|
|
7531
|
-
rootSpanId: String(row.rootSpanId),
|
|
7532
|
-
startedAt: asIsoTimestamp(row.startedAt),
|
|
7533
|
-
endedAt: asIsoTimestamp(row.endedAt)
|
|
7534
|
-
}))));
|
|
7535
|
-
for (const payload of payloadRows) payloads.set(`${payload.traceId}\u0000${payload.rootSpanId}`, payload);
|
|
7536
|
-
}
|
|
7537
|
-
const traces = pageRows.map((row) => ({
|
|
7538
|
-
...row,
|
|
7539
|
-
...payloads.get(`${row.traceId}\u0000${row.rootSpanId}`)
|
|
7540
|
-
})).map((row) => ({
|
|
7541
|
-
traceId: String(row.traceId),
|
|
7542
|
-
rootSpanId: String(row.rootSpanId),
|
|
7543
|
-
name: row.name,
|
|
7544
|
-
entityId: row.entityId ?? null,
|
|
7545
|
-
parentSpanId: row.parentSpanId ?? null,
|
|
7546
|
-
createdAt: asIsoTimestamp(row.startedAt),
|
|
7547
|
-
metadata: parseJson(row.metadata) ?? null,
|
|
7548
|
-
inputPreview: _mastra_core_storage.buildInputPreview(row.input) ?? null,
|
|
7549
|
-
threadId: row.threadId == null ? null : String(row.threadId),
|
|
7550
|
-
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
7551
|
-
startedAt: asIsoTimestamp(row.startedAt),
|
|
7552
|
-
endedAt: asIsoTimestamp(row.endedAt),
|
|
7553
|
-
entityName: row.entityName == null ? null : String(row.entityName),
|
|
7554
|
-
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7555
|
-
environment: row.environment == null ? null : String(row.environment),
|
|
7556
|
-
status: row.status
|
|
7557
|
-
}));
|
|
7558
|
-
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7559
|
-
traces,
|
|
7560
|
-
...deltaHead ? { deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead)) } : {},
|
|
7561
|
-
pagination: {
|
|
7562
|
-
total,
|
|
7563
|
-
page: plan.page,
|
|
7564
|
-
perPage: plan.perPage,
|
|
7565
|
-
hasMore: (plan.page + 1) * plan.perPage < total
|
|
7566
|
-
}
|
|
7567
|
-
});
|
|
7568
|
-
}
|
|
7569
|
-
const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: deltaHead ? remaining() : timeoutMs }, compileClickHouseTraceQuery(plan, deltaHead));
|
|
7570
|
-
const visibleRows = rows.slice(0, plan.limit);
|
|
7731
|
+
let first = !deltaHead;
|
|
7732
|
+
const run = (compiled) => {
|
|
7733
|
+
const budget = first ? timeoutMs : remaining();
|
|
7734
|
+
first = false;
|
|
7735
|
+
return runWithClickHouseTraceQueryTimeout(client, { timeoutMs: budget }, compiled);
|
|
7736
|
+
};
|
|
7571
7737
|
if (plan.result === "groups") {
|
|
7572
|
-
const
|
|
7738
|
+
const rows = await run(compileClickHouseTraceQuery(plan));
|
|
7739
|
+
const groups = rows.slice(0, plan.limit).map((row) => ({ threadId: String(row.threadId) }));
|
|
7573
7740
|
const last = groups.at(-1);
|
|
7574
7741
|
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7575
7742
|
groups,
|
|
@@ -7579,7 +7746,32 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7579
7746
|
}) : null }
|
|
7580
7747
|
});
|
|
7581
7748
|
}
|
|
7582
|
-
|
|
7749
|
+
/** Rows written before the previews were stored have none; theirs are built from the payloads. */
|
|
7750
|
+
const toTraceRecords = async (rows) => {
|
|
7751
|
+
const missing = rows.filter(needsPreviewPayloads);
|
|
7752
|
+
const payloads = /* @__PURE__ */ new Map();
|
|
7753
|
+
if (missing.length > 0) {
|
|
7754
|
+
const payloadRows = await run(compileClickHouseTraceRootPayloads(missing.map(toTraceRootRowKey), plan.scope, "previewPayloads"));
|
|
7755
|
+
for (const row of payloadRows) payloads.set(traceRootRowKey(row), row);
|
|
7756
|
+
}
|
|
7757
|
+
return rows.map((row) => toTraceRecord({
|
|
7758
|
+
...row,
|
|
7759
|
+
...resolveStoredPreviews(row, payloads.get(traceRootRowKey(row)))
|
|
7760
|
+
}));
|
|
7761
|
+
};
|
|
7762
|
+
/** Lists rows, then fetches `metadata` and the previews for the rows that are returned. */
|
|
7763
|
+
const listTraces = async (rows) => {
|
|
7764
|
+
const payloads = /* @__PURE__ */ new Map();
|
|
7765
|
+
if (rows.length > 0) {
|
|
7766
|
+
const payloadRows = await run(compileClickHouseTraceRootPayloads(rows.map(toTraceRootRowKey), plan.scope));
|
|
7767
|
+
for (const payload of payloadRows) payloads.set(traceRootRowKey(payload), payload);
|
|
7768
|
+
}
|
|
7769
|
+
return toTraceRecords(rows.map((row) => ({
|
|
7770
|
+
...row,
|
|
7771
|
+
...payloads.get(traceRootRowKey(row))
|
|
7772
|
+
})));
|
|
7773
|
+
};
|
|
7774
|
+
const toTraceRecord = (row) => ({
|
|
7583
7775
|
traceId: String(row.traceId),
|
|
7584
7776
|
rootSpanId: String(row.rootSpanId),
|
|
7585
7777
|
name: row.name,
|
|
@@ -7587,7 +7779,9 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7587
7779
|
parentSpanId: row.parentSpanId ?? null,
|
|
7588
7780
|
createdAt: asIsoTimestamp(row.startedAt),
|
|
7589
7781
|
metadata: parseJson(row.metadata) ?? null,
|
|
7590
|
-
inputPreview:
|
|
7782
|
+
inputPreview: row.inputPreview ?? null,
|
|
7783
|
+
outputPreview: row.outputPreview ?? null,
|
|
7784
|
+
errorPreview: row.errorPreview ?? null,
|
|
7591
7785
|
threadId: row.threadId == null ? null : String(row.threadId),
|
|
7592
7786
|
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
7593
7787
|
startedAt: asIsoTimestamp(row.startedAt),
|
|
@@ -7596,9 +7790,11 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7596
7790
|
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7597
7791
|
environment: row.environment == null ? null : String(row.environment),
|
|
7598
7792
|
status: row.status
|
|
7599
|
-
})
|
|
7600
|
-
const last = traces.at(-1);
|
|
7793
|
+
});
|
|
7601
7794
|
if (plan.paginationMode === "delta") {
|
|
7795
|
+
const rows = await run(compileClickHouseTraceQuery(plan, deltaHead));
|
|
7796
|
+
const visibleRows = rows.slice(0, plan.limit);
|
|
7797
|
+
const traces = await listTraces(visibleRows);
|
|
7602
7798
|
const lastRow = visibleRows.at(-1);
|
|
7603
7799
|
let watermark = lastRow ? {
|
|
7604
7800
|
cursorId: String(lastRow.__delta_cursor),
|
|
@@ -7615,12 +7811,36 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7615
7811
|
deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(watermark))
|
|
7616
7812
|
});
|
|
7617
7813
|
}
|
|
7814
|
+
const keys = await run(compileClickHouseTraceQueryKeys(plan));
|
|
7815
|
+
let total = Number(keys[0]?.__total ?? 0);
|
|
7816
|
+
if (plan.paginationMode === "page" && keys.length === 0 && plan.page > 0) {
|
|
7817
|
+
const [stats] = await run(compileClickHouseTraceQueryTotal(plan));
|
|
7818
|
+
total = Number(stats?.traces ?? 0);
|
|
7819
|
+
}
|
|
7820
|
+
const visible = plan.paginationMode === "page" ? keys : keys.slice(0, plan.limit);
|
|
7821
|
+
const rows = visible.length > 0 ? await run(compileClickHouseTraceRowsByKey(plan, visible.map((key) => ({
|
|
7822
|
+
traceId: String(key.traceId),
|
|
7823
|
+
dedupeKey: String(key.dedupeKey),
|
|
7824
|
+
sortValue: asIsoTimestamp(key.sortValue),
|
|
7825
|
+
rootStartedAt: asIsoTimestamp(key.rootStartedAt)
|
|
7826
|
+
})))) : [];
|
|
7827
|
+
if (plan.paginationMode === "page") return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7828
|
+
traces: await toTraceRecords(rows),
|
|
7829
|
+
...deltaHead ? { deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead)) } : {},
|
|
7830
|
+
pagination: {
|
|
7831
|
+
total,
|
|
7832
|
+
page: plan.page,
|
|
7833
|
+
perPage: plan.perPage,
|
|
7834
|
+
hasMore: (plan.page + 1) * plan.perPage < total
|
|
7835
|
+
}
|
|
7836
|
+
});
|
|
7837
|
+
const lastKey = visible.at(-1);
|
|
7618
7838
|
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7619
|
-
traces,
|
|
7620
|
-
page: { next:
|
|
7839
|
+
traces: await toTraceRecords(rows),
|
|
7840
|
+
page: { next: keys.length > plan.limit && lastKey ? _mastra_core_storage.encodeTraceQueryCursor(plan, {
|
|
7621
7841
|
result: "traces",
|
|
7622
|
-
sortValue:
|
|
7623
|
-
traceId:
|
|
7842
|
+
sortValue: asIsoTimestamp(lastKey.sortValue),
|
|
7843
|
+
traceId: String(lastKey.traceId)
|
|
7624
7844
|
}) : null }
|
|
7625
7845
|
});
|
|
7626
7846
|
}
|
|
@@ -7790,13 +8010,29 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
7790
8010
|
page: { next: null }
|
|
7791
8011
|
};
|
|
7792
8012
|
const payloadWhere = `tuple(${identitySetColumns}, endedAt) IN {page:Array(Tuple(UInt8, String, UInt8, String, String, String, DateTime64(3, 'UTC')))}`;
|
|
7793
|
-
const
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
FROM (SELECT ${identities}, any(tuple(${displayFields.map(fieldExpression).join(", ")}
|
|
8013
|
+
const pageParameter = (rows) => rows.map((row) => new _clickhouse_client.TupleParam([...identityValues(row), row.endedAt.replace("T", " ").replace(/Z$/, "")]));
|
|
8014
|
+
const rowsWithPreviews = await run({
|
|
8015
|
+
query: `SELECT ${identities}, ${[...displayFields, ...PREVIEW_COLUMNS].map((field, index) => `data.${index + 1} AS ${field}`).join(", ")}
|
|
8016
|
+
FROM (SELECT ${identities}, any(tuple(${[...displayFields.map(fieldExpression), ...PREVIEW_COLUMNS].join(", ")})) AS data
|
|
7797
8017
|
FROM ${TABLE_SPAN_EVENTS} e WHERE ${payloadWhere} GROUP BY ${identities})`,
|
|
7798
|
-
query_params: { page: page
|
|
8018
|
+
query_params: { page: pageParameter(page) }
|
|
7799
8019
|
});
|
|
8020
|
+
const missingKeys = new Set(rowsWithPreviews.filter(needsPreviewPayloads).map(_mastra_core_storage.spanQueryIdentityKey));
|
|
8021
|
+
const missing = page.filter((row) => missingKeys.has(_mastra_core_storage.spanQueryIdentityKey(row)));
|
|
8022
|
+
const previewPayloads = /* @__PURE__ */ new Map();
|
|
8023
|
+
if (missing.length) {
|
|
8024
|
+
const rows = await run({
|
|
8025
|
+
query: `SELECT ${identities}, data.1 AS input, data.2 AS output, data.3 AS error
|
|
8026
|
+
FROM (SELECT ${identities}, any(tuple(input, output, error)) AS data
|
|
8027
|
+
FROM ${TABLE_SPAN_EVENTS} e WHERE ${payloadWhere} GROUP BY ${identities})`,
|
|
8028
|
+
query_params: { page: pageParameter(missing) }
|
|
8029
|
+
});
|
|
8030
|
+
for (const row of rows) previewPayloads.set(_mastra_core_storage.spanQueryIdentityKey(row), row);
|
|
8031
|
+
}
|
|
8032
|
+
const payloads = rowsWithPreviews.map((row) => ({
|
|
8033
|
+
...row,
|
|
8034
|
+
...resolveStoredPreviews(row, previewPayloads.get(_mastra_core_storage.spanQueryIdentityKey(row)))
|
|
8035
|
+
}));
|
|
7800
8036
|
const metricWhere = `${identitySetKey} IN {page:Array(Tuple(UInt8, String, UInt8, String, String, String))}`;
|
|
7801
8037
|
const metrics = await run({
|
|
7802
8038
|
query: `SELECT ${identities}, data.1 AS name, data.2 AS estimatedCost,
|
|
@@ -7810,17 +8046,16 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
7810
8046
|
});
|
|
7811
8047
|
if (metrics.length > _mastra_core_storage.SPAN_QUERY_MAX_COST_METRICS) throw new _mastra_core_storage.TraceQueryResourceLimitError();
|
|
7812
8048
|
const byIdentity = new Map(payloads.map((row) => [_mastra_core_storage.spanQueryIdentityKey(row), row]));
|
|
7813
|
-
const
|
|
8049
|
+
const result = buildSpanQueryResponse(plan, page.flatMap((identity) => {
|
|
7814
8050
|
const row = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(identity));
|
|
7815
8051
|
if (!row) return [];
|
|
7816
|
-
const { inputPreview: _input, outputPreview: _output, ...metadata } = row;
|
|
8052
|
+
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, ...metadata } = row;
|
|
7817
8053
|
return [{
|
|
7818
8054
|
...metadata,
|
|
7819
8055
|
...identity,
|
|
7820
8056
|
durationMs: Number(row.durationMs)
|
|
7821
8057
|
}];
|
|
7822
|
-
});
|
|
7823
|
-
const result = _mastra_core_storage.buildSpanQueryResponse(plan, hydrated, payloads, metrics);
|
|
8058
|
+
}), payloads, metrics);
|
|
7824
8059
|
const last = page.at(-1);
|
|
7825
8060
|
result.page.next = selected.length > plan.limit ? _mastra_core_storage.encodeSpanQueryCursor(plan, {
|
|
7826
8061
|
organizationId: last.organizationId,
|
|
@@ -7874,7 +8109,13 @@ const COMPARISON_SQL = {
|
|
|
7874
8109
|
*/
|
|
7875
8110
|
function compileClickHouseTraceAggregate(plan) {
|
|
7876
8111
|
const parameters = new ParameterBuilder();
|
|
7877
|
-
const ctes = compileClickHouseTraceCandidates(plan, "*", parameters);
|
|
8112
|
+
const ctes = compileClickHouseTraceCandidates(plan, "*", parameters, { oneRootPerTrace: true });
|
|
8113
|
+
ctes.push(`trace_roots AS (
|
|
8114
|
+
SELECT *
|
|
8115
|
+
FROM candidates
|
|
8116
|
+
ORDER BY traceId, dedupeKey
|
|
8117
|
+
LIMIT 1 BY traceId
|
|
8118
|
+
)`);
|
|
7878
8119
|
const dimensionSql = (field) => {
|
|
7879
8120
|
if (field.startsWith("metadata.")) return traceMetadataValueSql(parameters.add(field.slice(9), "String"));
|
|
7880
8121
|
const sql = Object.hasOwn(DIMENSION_SQL, field) ? DIMENSION_SQL[field] : void 0;
|
|
@@ -7908,7 +8149,7 @@ function compileClickHouseTraceAggregate(plan) {
|
|
|
7908
8149
|
}
|
|
7909
8150
|
ctes.push(`facts AS (
|
|
7910
8151
|
SELECT ${factColumns.join(",\n ")}
|
|
7911
|
-
FROM
|
|
8152
|
+
FROM trace_roots r${usageJoin}
|
|
7912
8153
|
)`);
|
|
7913
8154
|
const mixedUnitsSql = "minOrNull(unitMin) != maxOrNull(unitMax)";
|
|
7914
8155
|
const measureSql = (name) => {
|
|
@@ -8124,9 +8365,9 @@ async function getRootSpan(client, args) {
|
|
|
8124
8365
|
return { span: rowToSpanRecord(rows[0]) };
|
|
8125
8366
|
}
|
|
8126
8367
|
/**
|
|
8127
|
-
* Columns a trace list renders.
|
|
8128
|
-
*
|
|
8129
|
-
*
|
|
8368
|
+
* Columns a trace list renders. The previews are the stored ones; rows written
|
|
8369
|
+
* before they were stored get theirs from the payload columns, fetched for just
|
|
8370
|
+
* those rows (see `withPreviewPayloads`).
|
|
8130
8371
|
*/
|
|
8131
8372
|
const LIGHT_TRACE_ROOT_FIELDS = [
|
|
8132
8373
|
"traceId",
|
|
@@ -8141,10 +8382,13 @@ const LIGHT_TRACE_ROOT_FIELDS = [
|
|
|
8141
8382
|
"entityId",
|
|
8142
8383
|
"entityName",
|
|
8143
8384
|
"threadId",
|
|
8385
|
+
"organizationId",
|
|
8144
8386
|
"resourceId",
|
|
8145
8387
|
"error",
|
|
8146
8388
|
"metadataRaw",
|
|
8147
|
-
"
|
|
8389
|
+
"inputPreview",
|
|
8390
|
+
"outputPreview",
|
|
8391
|
+
"errorPreview"
|
|
8148
8392
|
];
|
|
8149
8393
|
const FULL_PROJECTION = {
|
|
8150
8394
|
innerSelect: "*",
|
|
@@ -8153,7 +8397,7 @@ const FULL_PROJECTION = {
|
|
|
8153
8397
|
};
|
|
8154
8398
|
const LIGHT_PROJECTION = {
|
|
8155
8399
|
innerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
|
|
8156
|
-
outerSelect: LIGHT_TRACE_ROOT_FIELDS.join(", "),
|
|
8400
|
+
outerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
|
|
8157
8401
|
deltaSelect: LIGHT_TRACE_ROOT_FIELDS.filter((field) => field !== "startedAt" && field !== "traceId").map((field) => `r.${field}`).join(", ")
|
|
8158
8402
|
};
|
|
8159
8403
|
/**
|
|
@@ -8199,7 +8443,7 @@ async function listTraceRows(client, args, strategy, projection, mapRows) {
|
|
|
8199
8443
|
const rows = await queryTracesAfterCursor(client, projection.deltaSelect, whereClause, params, limit, afterCursor);
|
|
8200
8444
|
const visibleRows = rows.slice(0, limit);
|
|
8201
8445
|
return {
|
|
8202
|
-
spans: mapRows(visibleRows),
|
|
8446
|
+
spans: await mapRows(visibleRows),
|
|
8203
8447
|
delta: {
|
|
8204
8448
|
limit,
|
|
8205
8449
|
hasMore: rows.length > limit
|
|
@@ -8274,7 +8518,7 @@ async function listTraceRows(client, args, strategy, projection, mapRows) {
|
|
|
8274
8518
|
perPage,
|
|
8275
8519
|
hasMore: (page + 1) * perPage < total
|
|
8276
8520
|
},
|
|
8277
|
-
spans: mapRows(rows),
|
|
8521
|
+
spans: await mapRows(rows),
|
|
8278
8522
|
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
8279
8523
|
};
|
|
8280
8524
|
}
|
|
@@ -8283,12 +8527,32 @@ async function listTraces(client, args, strategy) {
|
|
|
8283
8527
|
return listTraceRows(client, args, strategy, FULL_PROJECTION, (rows) => (0, _mastra_core_storage.toTraceSpans)(rows.map(rowToSpanRecord)));
|
|
8284
8528
|
}
|
|
8285
8529
|
/**
|
|
8530
|
+
* Adds `input` / `output` / `error` to the rows written before previews were stored, read from
|
|
8531
|
+
* the same root row version (same `endedAt`) the list kept. A page can hold several tenants' rows,
|
|
8532
|
+
* so each row is looked up by its own identity, tenant included, not by one shared scope.
|
|
8533
|
+
*/
|
|
8534
|
+
async function withPreviewPayloads(client, rows) {
|
|
8535
|
+
const missing = rows.filter(needsPreviewPayloads);
|
|
8536
|
+
if (missing.length === 0) return rows;
|
|
8537
|
+
const compiled = compileClickHouseTraceRootPayloads(missing.map(toTraceRootRowKey), void 0, "previewPayloads");
|
|
8538
|
+
const result = await client.query({
|
|
8539
|
+
...compiled,
|
|
8540
|
+
format: "JSONEachRow",
|
|
8541
|
+
clickhouse_settings: CH_SETTINGS
|
|
8542
|
+
});
|
|
8543
|
+
const payloads = new Map((await result.json()).map((row) => [traceRootRowKey(row), row]));
|
|
8544
|
+
return rows.map((row) => needsPreviewPayloads(row) ? {
|
|
8545
|
+
...row,
|
|
8546
|
+
...payloads.get(traceRootRowKey(row))
|
|
8547
|
+
} : row);
|
|
8548
|
+
}
|
|
8549
|
+
/**
|
|
8286
8550
|
* List traces projecting only the columns a trace list renders.
|
|
8287
|
-
* Skips the attributes/output blobs and
|
|
8288
|
-
*
|
|
8551
|
+
* Skips the attributes/input/output blobs and reads the stored previews instead,
|
|
8552
|
+
* so the response payload stays flat as traces grow.
|
|
8289
8553
|
*/
|
|
8290
8554
|
async function listTracesLight(client, args, strategy) {
|
|
8291
|
-
return listTraceRows(client, args, strategy, LIGHT_PROJECTION, (rows) => rows.map(rowToLightSpanRecord));
|
|
8555
|
+
return listTraceRows(client, args, strategy, LIGHT_PROJECTION, async (rows) => (await withPreviewPayloads(client, rows)).map(rowToLightSpanRecord));
|
|
8292
8556
|
}
|
|
8293
8557
|
async function queryTracesAfterCursor(client, deltaSelect, whereClause, params, limit, cursorId) {
|
|
8294
8558
|
const deltaKeys = `SELECT startedAt, traceId, dedupeKey FROM ${TABLE_TRACE_ROOTS_DELTA} WHERE cursorId > {afterCursor:UInt64}`;
|