@mastra/clickhouse 1.25.0 → 1.25.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/integrations-databases-clickhouse.md +1 -1
- package/dist/index.cjs +460 -191
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +461 -192
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +3 -3
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/helpers.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts +45 -0
- package/dist/storage/domains/observability/v-next/preview-compat.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/previews.d.ts +16 -0
- package/dist/storage/domains/observability/v-next/previews.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts +2 -0
- package/dist/storage/domains/observability/v-next/span-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +66 -11
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts +2 -2
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/index.cjs
CHANGED
|
@@ -965,12 +965,23 @@ var MemoryStorageClickhouse = class extends _mastra_core_storage.MemoryStorage {
|
|
|
965
965
|
query_params: { messageIds }
|
|
966
966
|
});
|
|
967
967
|
if (threadIds.length > 0) {
|
|
968
|
-
const now =
|
|
969
|
-
await this.
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
968
|
+
const now = Date.now();
|
|
969
|
+
const existingThreads = await this.getLatestThreads(threadIds);
|
|
970
|
+
if (existingThreads.size > 0) await this.client.insert({
|
|
971
|
+
table: _mastra_core_storage.TABLE_THREADS,
|
|
972
|
+
format: "JSONEachRow",
|
|
973
|
+
values: Array.from(existingThreads.values()).map((existingThread) => ({
|
|
974
|
+
id: existingThread.id,
|
|
975
|
+
resourceId: existingThread.resourceId,
|
|
976
|
+
title: existingThread.title,
|
|
977
|
+
metadata: serializeMetadata(existingThread.metadata),
|
|
978
|
+
createdAt: existingThread.createdAt,
|
|
979
|
+
updatedAt: new Date(Math.max(now, new Date(existingThread.updatedAt).getTime() + 1)).toISOString().replace("Z", "")
|
|
980
|
+
})),
|
|
981
|
+
clickhouse_settings: {
|
|
982
|
+
date_time_input_format: "best_effort",
|
|
983
|
+
use_client_time_zone: 1,
|
|
984
|
+
output_format_json_quote_64bit_integers: 0
|
|
974
985
|
}
|
|
975
986
|
});
|
|
976
987
|
}
|
|
@@ -2745,7 +2756,14 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SPAN_EVENTS} (
|
|
|
2745
2756
|
output Nullable(String),
|
|
2746
2757
|
error Nullable(String),
|
|
2747
2758
|
metadataRaw Nullable(String),
|
|
2748
|
-
requestContext Nullable(String)
|
|
2759
|
+
requestContext Nullable(String),
|
|
2760
|
+
|
|
2761
|
+
-- Previews of \`input\` / \`output\` / \`error\` in core's \`buildPreview\` format, written
|
|
2762
|
+
-- on every span ('' when there is nothing to preview). NULL on rows written before
|
|
2763
|
+
-- the columns existed; readers build those from the payload columns.
|
|
2764
|
+
inputPreview Nullable(String),
|
|
2765
|
+
outputPreview Nullable(String),
|
|
2766
|
+
errorPreview Nullable(String)
|
|
2749
2767
|
)
|
|
2750
2768
|
ENGINE = ReplacingMergeTree
|
|
2751
2769
|
PARTITION BY toDate(endedAt)
|
|
@@ -2813,6 +2831,12 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_ROOTS} (
|
|
|
2813
2831
|
metadataRaw Nullable(String),
|
|
2814
2832
|
requestContext Nullable(String),
|
|
2815
2833
|
|
|
2834
|
+
-- Previews of \`input\` / \`output\` / \`error\` ('' when there is nothing to preview).
|
|
2835
|
+
-- NULL on rows written before the columns existed.
|
|
2836
|
+
inputPreview Nullable(String),
|
|
2837
|
+
outputPreview Nullable(String),
|
|
2838
|
+
errorPreview Nullable(String),
|
|
2839
|
+
|
|
2816
2840
|
-- traceId is second in the sort key, behind startedAt, so point lookups by
|
|
2817
2841
|
-- traceId (getRootSpan, trace deletes, trace-query root dedupe) need a skip
|
|
2818
2842
|
-- index to avoid scanning every granule.
|
|
@@ -2891,6 +2915,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_BRANCHES} (
|
|
|
2891
2915
|
error Nullable(String),
|
|
2892
2916
|
metadataRaw Nullable(String),
|
|
2893
2917
|
requestContext Nullable(String),
|
|
2918
|
+
inputPreview Nullable(String),
|
|
2919
|
+
outputPreview Nullable(String),
|
|
2920
|
+
errorPreview Nullable(String),
|
|
2894
2921
|
|
|
2895
2922
|
INDEX idx_traceId traceId TYPE bloom_filter(0.01) GRANULARITY 1
|
|
2896
2923
|
)
|
|
@@ -3622,7 +3649,26 @@ const addBloomIndex = (table, name, column, granularity = 2) => ({
|
|
|
3622
3649
|
name,
|
|
3623
3650
|
sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY ${granularity}`
|
|
3624
3651
|
});
|
|
3652
|
+
const LOG_SKIP_INDEX_COLUMNS = [
|
|
3653
|
+
"traceId",
|
|
3654
|
+
"threadId",
|
|
3655
|
+
"resourceId",
|
|
3656
|
+
"userId",
|
|
3657
|
+
"organizationId",
|
|
3658
|
+
"experimentId",
|
|
3659
|
+
"runId",
|
|
3660
|
+
"sessionId",
|
|
3661
|
+
"requestId"
|
|
3662
|
+
];
|
|
3625
3663
|
const ALL_MIGRATIONS = [
|
|
3664
|
+
...[
|
|
3665
|
+
"inputPreview",
|
|
3666
|
+
"outputPreview",
|
|
3667
|
+
"errorPreview"
|
|
3668
|
+
].flatMap((column) => [addColumn(TABLE_TRACE_ROOTS, column, "Nullable(String)"), addColumn(TABLE_TRACE_BRANCHES, column, "Nullable(String)")]),
|
|
3669
|
+
addColumn(TABLE_SPAN_EVENTS, "inputPreview", "Nullable(String)"),
|
|
3670
|
+
addColumn(TABLE_SPAN_EVENTS, "outputPreview", "Nullable(String)"),
|
|
3671
|
+
addColumn(TABLE_SPAN_EVENTS, "errorPreview", "Nullable(String)"),
|
|
3626
3672
|
addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
|
|
3627
3673
|
addColumn(TABLE_SPAN_EVENTS, "parentEntityVersionId", "Nullable(String)"),
|
|
3628
3674
|
addColumn(TABLE_SPAN_EVENTS, "rootEntityVersionId", "Nullable(String)"),
|
|
@@ -3659,17 +3705,7 @@ const ALL_MIGRATIONS = [
|
|
|
3659
3705
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_runId", "runId"),
|
|
3660
3706
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
|
|
3661
3707
|
addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId"),
|
|
3662
|
-
...
|
|
3663
|
-
"traceId",
|
|
3664
|
-
"threadId",
|
|
3665
|
-
"resourceId",
|
|
3666
|
-
"userId",
|
|
3667
|
-
"organizationId",
|
|
3668
|
-
"experimentId",
|
|
3669
|
-
"runId",
|
|
3670
|
-
"sessionId",
|
|
3671
|
-
"requestId"
|
|
3672
|
-
].map((column) => addBloomIndex(TABLE_LOG_EVENTS, `idx_${column}`, column)),
|
|
3708
|
+
...LOG_SKIP_INDEX_COLUMNS.map((column) => addBloomIndex(TABLE_LOG_EVENTS, `idx_${column}`, column)),
|
|
3673
3709
|
addBloomIndex(TABLE_DELETION_REQUESTS, "idx_predicateValues", "predicateValues")
|
|
3674
3710
|
];
|
|
3675
3711
|
const ALL_TABLE_NAMES = [
|
|
@@ -3779,6 +3815,78 @@ function parseTtlExpression(expr) {
|
|
|
3779
3815
|
};
|
|
3780
3816
|
}
|
|
3781
3817
|
//#endregion
|
|
3818
|
+
//#region src/storage/domains/observability/v-next/preview-compat.ts
|
|
3819
|
+
const generator = _mastra_core_storage.buildStoredPreview;
|
|
3820
|
+
const canGeneratePreviews = typeof generator === "function";
|
|
3821
|
+
function needsPreview(value) {
|
|
3822
|
+
return canGeneratePreviews && value == null;
|
|
3823
|
+
}
|
|
3824
|
+
function storedPreview(preview, value, serialized) {
|
|
3825
|
+
return canGeneratePreviews ? generator(preview, value, serialized) : preview ?? null;
|
|
3826
|
+
}
|
|
3827
|
+
function readPreview(preview, payload, jsonEncoded = false) {
|
|
3828
|
+
const supplied = preview;
|
|
3829
|
+
if (!canGeneratePreviews) return supplied ?? null;
|
|
3830
|
+
if (supplied != null) return supplied.trim() || null;
|
|
3831
|
+
let value = payload;
|
|
3832
|
+
if (needsPreview(preview) && jsonEncoded && typeof payload === "string") try {
|
|
3833
|
+
value = JSON.parse(payload);
|
|
3834
|
+
} catch {
|
|
3835
|
+
value = null;
|
|
3836
|
+
}
|
|
3837
|
+
return storedPreview(supplied, value, jsonEncoded && typeof payload === "string" ? payload : void 0) || null;
|
|
3838
|
+
}
|
|
3839
|
+
/** Older response builders may shorten previews or turn empty strings into null. */
|
|
3840
|
+
function buildSpanQueryResponse(...args) {
|
|
3841
|
+
if (canGeneratePreviews) return _mastra_core_storage.buildSpanQueryResponse(...args);
|
|
3842
|
+
const [plan, selected, payloads, metrics] = args;
|
|
3843
|
+
const response = _mastra_core_storage.buildSpanQueryResponse(plan, selected, [], metrics);
|
|
3844
|
+
const byIdentity = new Map(payloads.map((payload) => [_mastra_core_storage.spanQueryIdentityKey(payload), payload]));
|
|
3845
|
+
for (const span of response.spans) {
|
|
3846
|
+
const payload = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(span));
|
|
3847
|
+
span.inputPreview = payload?.inputPreview ?? null;
|
|
3848
|
+
span.outputPreview = payload?.outputPreview ?? null;
|
|
3849
|
+
span.errorPreview = payload?.errorPreview ?? null;
|
|
3850
|
+
}
|
|
3851
|
+
return response;
|
|
3852
|
+
}
|
|
3853
|
+
//#endregion
|
|
3854
|
+
//#region src/storage/domains/observability/v-next/previews.ts
|
|
3855
|
+
/**
|
|
3856
|
+
* Stored span previews (`inputPreview` / `outputPreview` / `errorPreview`).
|
|
3857
|
+
*
|
|
3858
|
+
* The writer stores each preview in core's `buildPreview` format, `''` when there is nothing to
|
|
3859
|
+
* preview. A NULL preview means the row was written before the columns existed; readers fetch the
|
|
3860
|
+
* payload columns for just those rows and build the same preview from them.
|
|
3861
|
+
*/
|
|
3862
|
+
const PREVIEW_COLUMNS = [
|
|
3863
|
+
"inputPreview",
|
|
3864
|
+
"outputPreview",
|
|
3865
|
+
"errorPreview"
|
|
3866
|
+
];
|
|
3867
|
+
/** Payload column each preview is built from. */
|
|
3868
|
+
const PREVIEW_PAYLOAD_COLUMNS = {
|
|
3869
|
+
inputPreview: "input",
|
|
3870
|
+
outputPreview: "output",
|
|
3871
|
+
errorPreview: "error"
|
|
3872
|
+
};
|
|
3873
|
+
/** Whether a row was written before previews were stored, so it needs its payload columns. */
|
|
3874
|
+
function needsPreviewPayloads(row) {
|
|
3875
|
+
return PREVIEW_COLUMNS.some((column) => needsPreview(row[column]));
|
|
3876
|
+
}
|
|
3877
|
+
/**
|
|
3878
|
+
* The previews a list returns for a row: the stored ones (`''` becomes `null`), or for a row written
|
|
3879
|
+
* before they were stored, built from the payload columns on `payloads` (default: the row itself).
|
|
3880
|
+
*/
|
|
3881
|
+
function resolveStoredPreviews(row, payloads = row) {
|
|
3882
|
+
const previews = {};
|
|
3883
|
+
for (const column of PREVIEW_COLUMNS) {
|
|
3884
|
+
const stored = row[column];
|
|
3885
|
+
previews[column] = readPreview(stored, payloads?.[PREVIEW_PAYLOAD_COLUMNS[column]], true);
|
|
3886
|
+
}
|
|
3887
|
+
return previews;
|
|
3888
|
+
}
|
|
3889
|
+
//#endregion
|
|
3782
3890
|
//#region src/storage/domains/observability/v-next/review-status.ts
|
|
3783
3891
|
const FEEDBACK_REVIEW_STATUSES = ["needs-review", "reviewed"];
|
|
3784
3892
|
function isFeedbackReviewStatus(value) {
|
|
@@ -3985,7 +4093,7 @@ function rowToLightSpanRecord(row) {
|
|
|
3985
4093
|
endedAt
|
|
3986
4094
|
}),
|
|
3987
4095
|
metadata: parseJson(row.metadataRaw) ?? void 0,
|
|
3988
|
-
|
|
4096
|
+
...resolveStoredPreviews(row),
|
|
3989
4097
|
createdAt: startedAt,
|
|
3990
4098
|
updatedAt: null
|
|
3991
4099
|
};
|
|
@@ -3993,6 +4101,9 @@ function rowToLightSpanRecord(row) {
|
|
|
3993
4101
|
function spanRecordToRow(span) {
|
|
3994
4102
|
const endedAt = span.isEvent ? span.startedAt : span.endedAt ?? span.startedAt;
|
|
3995
4103
|
const metadata = span.metadata ?? null;
|
|
4104
|
+
const input = jsonEncode(span.input);
|
|
4105
|
+
const output = jsonEncode(span.output);
|
|
4106
|
+
const error = jsonEncode(span.error);
|
|
3996
4107
|
return {
|
|
3997
4108
|
dedupeKey: buildDedupeKey(span.traceId, span.spanId),
|
|
3998
4109
|
traceId: span.traceId,
|
|
@@ -4032,10 +4143,13 @@ function spanRecordToRow(span) {
|
|
|
4032
4143
|
scope: jsonEncode(span.scope),
|
|
4033
4144
|
attributes: jsonEncode(span.attributes),
|
|
4034
4145
|
links: jsonEncode(span.links),
|
|
4035
|
-
input
|
|
4036
|
-
output
|
|
4037
|
-
error
|
|
4038
|
-
requestContext: jsonEncode(span.requestContext)
|
|
4146
|
+
input,
|
|
4147
|
+
output,
|
|
4148
|
+
error,
|
|
4149
|
+
requestContext: jsonEncode(span.requestContext),
|
|
4150
|
+
inputPreview: storedPreview(span.inputPreview, span.input, input),
|
|
4151
|
+
outputPreview: storedPreview(span.outputPreview, span.output, output),
|
|
4152
|
+
errorPreview: storedPreview(span.errorPreview, span.error, error)
|
|
4039
4153
|
};
|
|
4040
4154
|
}
|
|
4041
4155
|
function rowToLogRecord(row) {
|
|
@@ -6950,11 +7064,15 @@ const FEEDBACK_FIELDS = {
|
|
|
6950
7064
|
const TRACE_SELECT = `
|
|
6951
7065
|
r.traceId AS traceId,
|
|
6952
7066
|
r.spanId AS rootSpanId,
|
|
7067
|
+
r.dedupeKey AS dedupeKey,
|
|
7068
|
+
r.organizationId AS organizationId,
|
|
6953
7069
|
r.name AS name,
|
|
6954
7070
|
r.entityId AS entityId,
|
|
6955
7071
|
r.parentSpanId AS parentSpanId,
|
|
6956
7072
|
r.metadataRaw AS metadata,
|
|
6957
|
-
r.
|
|
7073
|
+
r.inputPreview AS inputPreview,
|
|
7074
|
+
r.outputPreview AS outputPreview,
|
|
7075
|
+
r.errorPreview AS errorPreview,
|
|
6958
7076
|
r.threadId AS threadId,
|
|
6959
7077
|
r.resourceId AS resourceId,
|
|
6960
7078
|
r.startedAt AS startedAt,
|
|
@@ -6963,22 +7081,6 @@ const TRACE_SELECT = `
|
|
|
6963
7081
|
r.entityType AS entityType,
|
|
6964
7082
|
r.environment AS environment,
|
|
6965
7083
|
${TRACE_STATUS_SQL} AS status`;
|
|
6966
|
-
/** Candidate columns carried through the page-mode window sort (no payload blobs). */
|
|
6967
|
-
const TRACE_PAGE_COLUMNS = [
|
|
6968
|
-
"traceId",
|
|
6969
|
-
"rootSpanId",
|
|
6970
|
-
"name",
|
|
6971
|
-
"entityId",
|
|
6972
|
-
"parentSpanId",
|
|
6973
|
-
"threadId",
|
|
6974
|
-
"resourceId",
|
|
6975
|
-
"startedAt",
|
|
6976
|
-
"endedAt",
|
|
6977
|
-
"entityName",
|
|
6978
|
-
"entityType",
|
|
6979
|
-
"environment",
|
|
6980
|
-
"status"
|
|
6981
|
-
];
|
|
6982
7084
|
var ParameterBuilder = class {
|
|
6983
7085
|
params = {};
|
|
6984
7086
|
#next = 1;
|
|
@@ -7102,18 +7204,38 @@ function compileTenantScope(scope, parameters) {
|
|
|
7102
7204
|
if (scope.resourceId !== void 0) sql += `\n AND resourceId = ${parameters.add(scope.resourceId, "String")}`;
|
|
7103
7205
|
return sql;
|
|
7104
7206
|
}
|
|
7105
|
-
function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = []) {
|
|
7207
|
+
function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = [], options = {}) {
|
|
7106
7208
|
const from = parameters.add(selection.timeRange.from, "DateTime64(3, 'UTC')");
|
|
7107
7209
|
const to = parameters.add(selection.timeRange.to, "DateTime64(3, 'UTC')");
|
|
7108
7210
|
const tenant = compileTenantScope(scope, parameters);
|
|
7109
|
-
const
|
|
7211
|
+
const seedFilter = seedConjuncts.map((conjunct) => `\n AND (${compilePredicate(conjunct, parameters)})`).join("");
|
|
7212
|
+
const bound = options.delta ? `\n AND (traceId IN (SELECT traceId FROM delta_candidates))` : "";
|
|
7213
|
+
const windowTraces = (more) => `
|
|
7214
|
+
SELECT traceId
|
|
7215
|
+
FROM ${TABLE_TRACE_ROOTS} w
|
|
7216
|
+
WHERE startedAt >= ${from}
|
|
7217
|
+
AND startedAt < ${to}${tenant}${more}
|
|
7218
|
+
`;
|
|
7219
|
+
const delta = options.delta ? [`delta_candidates AS (
|
|
7220
|
+
SELECT traceId, max(cursorId) AS latestCursorId
|
|
7221
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA}
|
|
7222
|
+
WHERE ${options.delta}
|
|
7223
|
+
AND traceId IN (${windowTraces("")})
|
|
7224
|
+
GROUP BY traceId
|
|
7225
|
+
)`] : [];
|
|
7226
|
+
const ctes = options.oneRootPerTrace ? [...delta, `root_scope AS (
|
|
7227
|
+
SELECT *
|
|
7228
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
7229
|
+
WHERE startedAt >= ${from}
|
|
7230
|
+
AND startedAt < ${to}${tenant}${seedFilter}${bound}
|
|
7231
|
+
)`] : [`current_roots AS (
|
|
7110
7232
|
SELECT *
|
|
7111
7233
|
FROM ${TABLE_TRACE_ROOTS}
|
|
7112
7234
|
WHERE traceId IN (
|
|
7113
7235
|
SELECT traceId
|
|
7114
7236
|
FROM ${TABLE_TRACE_ROOTS} r
|
|
7115
7237
|
WHERE startedAt >= ${from}
|
|
7116
|
-
AND startedAt < ${to}${tenant}${
|
|
7238
|
+
AND startedAt < ${to}${tenant}${seedFilter}
|
|
7117
7239
|
)${tenant}
|
|
7118
7240
|
ORDER BY traceId, dedupeKey
|
|
7119
7241
|
LIMIT 1 BY traceId
|
|
@@ -7147,9 +7269,9 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7147
7269
|
organizationId
|
|
7148
7270
|
FROM ${TABLE_SPAN_EVENTS}
|
|
7149
7271
|
WHERE isNotNull(traceId)
|
|
7150
|
-
AND traceId IN (SELECT traceId FROM root_scope)${tenant}
|
|
7272
|
+
AND traceId IN (${options.spanExistence && options.oneRootPerTrace ? windowTraces(bound) : "SELECT traceId FROM root_scope"})${tenant}${options.spanExistence ? "" : `
|
|
7151
7273
|
ORDER BY dedupeKey
|
|
7152
|
-
LIMIT 1 BY dedupeKey
|
|
7274
|
+
LIMIT 1 BY dedupeKey`}
|
|
7153
7275
|
)`);
|
|
7154
7276
|
if (relationCollections.has("scores")) ctes.push(`current_scores AS (
|
|
7155
7277
|
SELECT
|
|
@@ -7217,9 +7339,9 @@ function parseDeltaWatermark(value) {
|
|
|
7217
7339
|
* selection's time range and tenant scope that match its `where` predicate. Trace queries and
|
|
7218
7340
|
* trace aggregates both select from this CTE so they always see the same population.
|
|
7219
7341
|
*/
|
|
7220
|
-
function compileClickHouseTraceCandidates(selection, columns, parameters) {
|
|
7342
|
+
function compileClickHouseTraceCandidates(selection, columns, parameters, options = {}) {
|
|
7221
7343
|
const rootConjuncts = selection.where ? (selection.where.type === "boolean" && selection.where.operator === "and" ? selection.where.args : [selection.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : [];
|
|
7222
|
-
const ctes = compileClickHouseTraceScope(selection, collectRelationCollections(selection.where), parameters, selection.scope, rootConjuncts);
|
|
7344
|
+
const ctes = compileClickHouseTraceScope(selection, collectRelationCollections(selection.where), parameters, selection.scope, rootConjuncts, options);
|
|
7223
7345
|
const predicate = selection.where ? compilePredicate(selection.where, parameters) : "1";
|
|
7224
7346
|
ctes.push(`candidates AS (
|
|
7225
7347
|
SELECT ${columns}
|
|
@@ -7228,94 +7350,191 @@ function compileClickHouseTraceCandidates(selection, columns, parameters) {
|
|
|
7228
7350
|
)`);
|
|
7229
7351
|
return ctes;
|
|
7230
7352
|
}
|
|
7353
|
+
/**
|
|
7354
|
+
* `queryTraces()` statements read one root per trace and only test spans for existence
|
|
7355
|
+
* (TraceScopeOptions). The list statements carry narrow columns; `metadata` and the previews are fetched
|
|
7356
|
+
* for the returned rows afterwards (compileClickHouseTraceRootPayloads).
|
|
7357
|
+
*/
|
|
7358
|
+
const QUERY_TRACES_SCOPE = {
|
|
7359
|
+
oneRootPerTrace: true,
|
|
7360
|
+
spanExistence: true
|
|
7361
|
+
};
|
|
7362
|
+
const TRACE_LIST_SELECT = TRACE_SELECT.split("\n").filter((line) => !/ AS (metadata|inputPreview|outputPreview|errorPreview),$/.test(line)).join("\n");
|
|
7363
|
+
function traceOrder(plan) {
|
|
7364
|
+
return {
|
|
7365
|
+
field: resolveOrderField(plan.orderBy.field),
|
|
7366
|
+
direction: plan.orderBy.direction === "asc" ? "ASC" : "DESC"
|
|
7367
|
+
};
|
|
7368
|
+
}
|
|
7369
|
+
/**
|
|
7370
|
+
* Keyset cursor on a trace's sort value. Applied after each trace is reduced to one sort value:
|
|
7371
|
+
* filtering root rows first would let a trace with several roots (or unmerged copies with
|
|
7372
|
+
* different `endedAt`) come back on a later page under a root that sorts after the cursor.
|
|
7373
|
+
*/
|
|
7374
|
+
function keysetCursorCondition(plan, order, parameters) {
|
|
7375
|
+
if (plan.paginationMode !== "keyset" || !plan.cursor) return "";
|
|
7376
|
+
const comparison = order.direction === "ASC" ? ">" : "<";
|
|
7377
|
+
const sortValue = parameters.add(plan.cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
7378
|
+
return `(sortValue ${comparison} ${sortValue} OR (sortValue = ${sortValue} AND traceId > ${parameters.add(plan.cursor.traceId, "String")}))`;
|
|
7379
|
+
}
|
|
7380
|
+
/**
|
|
7381
|
+
* First statement of keyset and page mode: the page's traces (after the keyset cursor, `limit + 1`
|
|
7382
|
+
* for keyset mode, `perPage` at the page offset for page mode) as root keys, and the
|
|
7383
|
+
* number of matching traces on every row. Per trace, the root row that sorts first gives the sort
|
|
7384
|
+
* value and the key, so the key always names a row with that sort value. No blob columns are read.
|
|
7385
|
+
*/
|
|
7386
|
+
function compileClickHouseTraceQueryKeys(plan) {
|
|
7387
|
+
if (plan.paginationMode === "delta") throw new Error("Delta trace queries have no keys statement");
|
|
7388
|
+
const parameters = new ParameterBuilder();
|
|
7389
|
+
const order = traceOrder(plan);
|
|
7390
|
+
const ctes = compileClickHouseTraceCandidates(plan, "r.traceId AS traceId, r.dedupeKey AS dedupeKey, r.startedAt AS startedAt, r.endedAt AS endedAt", parameters, QUERY_TRACES_SCOPE);
|
|
7391
|
+
const cursor = keysetCursorCondition(plan, order, parameters);
|
|
7392
|
+
const pick = order.direction === "DESC" ? "max" : "min";
|
|
7393
|
+
const limit = plan.paginationMode === "page" ? `LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}` : `LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`;
|
|
7394
|
+
return {
|
|
7395
|
+
query: `WITH ${ctes.join(",\n")},
|
|
7396
|
+
per_trace AS (
|
|
7397
|
+
SELECT traceId, ${pick}(${order.field}) AS sortValue,
|
|
7398
|
+
${pick === "max" ? "argMax" : "argMin"}((dedupeKey, startedAt), ${order.field}) AS root
|
|
7399
|
+
FROM candidates
|
|
7400
|
+
GROUP BY traceId${cursor ? `\n HAVING ${cursor}` : ""}
|
|
7401
|
+
)
|
|
7402
|
+
SELECT traceId, root.1 AS dedupeKey, sortValue, root.2 AS rootStartedAt, count() OVER () AS __total
|
|
7403
|
+
FROM per_trace
|
|
7404
|
+
ORDER BY sortValue ${order.direction}, traceId ASC
|
|
7405
|
+
${limit}`,
|
|
7406
|
+
query_params: parameters.params,
|
|
7407
|
+
sharedSnapshot: true
|
|
7408
|
+
};
|
|
7409
|
+
}
|
|
7410
|
+
/**
|
|
7411
|
+
* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata`
|
|
7412
|
+
* and the stored previews, read by sort-key tuple. Unmerged copies of a root key are redeliveries of one
|
|
7413
|
+
* span; the relation-free `where` conjuncts are re-applied so a copy that doesn't match isn't
|
|
7414
|
+
* returned, and the copy that sorts first is kept, as in the keys statement.
|
|
7415
|
+
*/
|
|
7416
|
+
function compileClickHouseTraceRowsByKey(plan, keys) {
|
|
7417
|
+
const parameters = new ParameterBuilder();
|
|
7418
|
+
const order = traceOrder(plan);
|
|
7419
|
+
const ts = (value) => parameters.add(value, "DateTime64(3, 'UTC')");
|
|
7420
|
+
const byEnd = order.field === "endedAt";
|
|
7421
|
+
const tuples = keys.map((key) => {
|
|
7422
|
+
const started = ts(byEnd ? key.rootStartedAt : key.sortValue);
|
|
7423
|
+
const id = `${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}`;
|
|
7424
|
+
return byEnd ? `(${started}, ${id}, ${ts(key.sortValue)})` : `(${started}, ${id})`;
|
|
7425
|
+
});
|
|
7426
|
+
const predicate = (plan.where ? (plan.where.type === "boolean" && plan.where.operator === "and" ? plan.where.args : [plan.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : []).map((conjunct) => `\n AND (${compilePredicate(conjunct, parameters)})`).join("");
|
|
7427
|
+
return {
|
|
7428
|
+
query: `SELECT ${TRACE_SELECT}
|
|
7429
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
7430
|
+
WHERE (r.startedAt, r.traceId, r.dedupeKey${byEnd ? ", r.endedAt" : ""}) IN (${tuples.join(", ")})${compileTenantScope(plan.scope, parameters)}${predicate}
|
|
7431
|
+
ORDER BY ${order.field} ${order.direction}, traceId ASC
|
|
7432
|
+
LIMIT 1 BY traceId`,
|
|
7433
|
+
query_params: parameters.params
|
|
7434
|
+
};
|
|
7435
|
+
}
|
|
7436
|
+
/** Number of matching traces, for numbered pages past the last one (the keys statement returns no rows). */
|
|
7437
|
+
function compileClickHouseTraceQueryTotal(plan) {
|
|
7438
|
+
const parameters = new ParameterBuilder();
|
|
7439
|
+
return {
|
|
7440
|
+
query: `WITH ${compileClickHouseTraceCandidates(plan, "r.traceId AS traceId", parameters, QUERY_TRACES_SCOPE).join(",\n")}
|
|
7441
|
+
SELECT uniqExact(traceId) AS traces
|
|
7442
|
+
FROM candidates`,
|
|
7443
|
+
query_params: parameters.params,
|
|
7444
|
+
sharedSnapshot: true
|
|
7445
|
+
};
|
|
7446
|
+
}
|
|
7447
|
+
/**
|
|
7448
|
+
* The list statement of groups and delta mode (rows without `metadata` and the previews). Keyset
|
|
7449
|
+
* and page mode list with compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey.
|
|
7450
|
+
* A trace with several candidate rows (several roots, or unmerged copies of one) is listed in delta
|
|
7451
|
+
* mode under the row that sorts first in the list order, as in the keys statement; ties go to the
|
|
7452
|
+
* lowest dedupeKey, then the latest endedAt, so every poll picks the same row.
|
|
7453
|
+
*/
|
|
7231
7454
|
function compileClickHouseTraceQuery(plan, deltaHead) {
|
|
7232
7455
|
const parameters = new ParameterBuilder();
|
|
7233
|
-
|
|
7234
|
-
|
|
7235
|
-
|
|
7456
|
+
if (plan.result === "groups") {
|
|
7457
|
+
const ctes = compileClickHouseTraceCandidates(plan, TRACE_LIST_SELECT, parameters, QUERY_TRACES_SCOPE);
|
|
7458
|
+
const pageCondition = plan.cursor ? `AND threadId > ${parameters.add(plan.cursor.threadId, "String")}` : "";
|
|
7459
|
+
const limit = parameters.add(plan.limit + 1, "UInt64");
|
|
7460
|
+
return {
|
|
7461
|
+
query: `WITH ${ctes.join(",\n")}
|
|
7236
7462
|
SELECT threadId
|
|
7237
7463
|
FROM candidates
|
|
7238
|
-
WHERE isNotNull(threadId) ${
|
|
7464
|
+
WHERE isNotNull(threadId) ${pageCondition}
|
|
7239
7465
|
GROUP BY threadId
|
|
7240
7466
|
ORDER BY threadId ASC
|
|
7241
|
-
LIMIT ${
|
|
7242
|
-
query_params: parameters.params
|
|
7243
|
-
};
|
|
7244
|
-
if (plan.paginationMode === "delta") {
|
|
7245
|
-
const watermark = _mastra_core_storage.getTraceQueryDeltaWatermark(plan, "clickhouse");
|
|
7246
|
-
const after = watermark ? parseDeltaWatermark(watermark) : {
|
|
7247
|
-
cursorId: "0",
|
|
7248
|
-
traceId: ""
|
|
7249
|
-
};
|
|
7250
|
-
const lowerCursor = parameters.add(after.cursorId, "UInt64");
|
|
7251
|
-
return {
|
|
7252
|
-
query: `${candidates}, delta_candidates AS (
|
|
7253
|
-
SELECT traceId, max(cursorId) AS latestCursorId
|
|
7254
|
-
FROM ${TABLE_TRACE_ROOTS_DELTA}
|
|
7255
|
-
WHERE cursorId >= ${lowerCursor} AND tuple(cursorId, traceId) > ${`tuple(${lowerCursor}, ${parameters.add(after.traceId, "String")})`} ${deltaHead ? `AND tuple(cursorId, traceId) <= tuple(${parameters.add(deltaHead.cursorId, "UInt64")}, ${parameters.add(deltaHead.traceId, "String")})` : ""}
|
|
7256
|
-
GROUP BY traceId
|
|
7257
|
-
)
|
|
7258
|
-
SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
|
|
7259
|
-
FROM candidates c
|
|
7260
|
-
INNER JOIN delta_candidates d ON c.traceId = d.traceId
|
|
7261
|
-
ORDER BY d.latestCursorId ASC, c.traceId ASC
|
|
7262
|
-
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
7263
|
-
query_params: parameters.params,
|
|
7264
|
-
sharedSnapshot: true
|
|
7265
|
-
};
|
|
7266
|
-
}
|
|
7267
|
-
const orderField = resolveOrderField(plan.orderBy.field);
|
|
7268
|
-
const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
|
|
7269
|
-
if (plan.paginationMode === "page") {
|
|
7270
|
-
const onPage = `__row_position > ${parameters.add(plan.page * plan.perPage, "UInt64")} AND __row_position <= ${parameters.add((plan.page + 1) * plan.perPage, "UInt64")}`;
|
|
7271
|
-
return {
|
|
7272
|
-
query: `${candidates},
|
|
7273
|
-
page_rows AS (
|
|
7274
|
-
SELECT
|
|
7275
|
-
${TRACE_PAGE_COLUMNS.join(",\n ")},
|
|
7276
|
-
row_number() OVER (ORDER BY ${orderField} ${direction}, traceId ASC) AS __row_position,
|
|
7277
|
-
count() OVER () AS total
|
|
7278
|
-
FROM candidates
|
|
7279
|
-
)
|
|
7280
|
-
SELECT *, if(${onPage}, 0, 1) AS __metadata
|
|
7281
|
-
FROM page_rows
|
|
7282
|
-
WHERE (${onPage}) OR __row_position = 1
|
|
7283
|
-
ORDER BY __metadata ASC, __row_position ASC`,
|
|
7467
|
+
LIMIT ${limit}`,
|
|
7284
7468
|
query_params: parameters.params,
|
|
7285
7469
|
sharedSnapshot: true
|
|
7286
7470
|
};
|
|
7287
7471
|
}
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
|
|
7291
|
-
|
|
7292
|
-
|
|
7293
|
-
|
|
7472
|
+
if (plan.paginationMode !== "delta") throw new Error("Keyset and page trace queries use compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey");
|
|
7473
|
+
const order = traceOrder(plan);
|
|
7474
|
+
const watermark = _mastra_core_storage.getTraceQueryDeltaWatermark(plan, "clickhouse");
|
|
7475
|
+
const after = watermark ? parseDeltaWatermark(watermark) : {
|
|
7476
|
+
cursorId: "0",
|
|
7477
|
+
traceId: ""
|
|
7478
|
+
};
|
|
7479
|
+
const lowerCursor = parameters.add(after.cursorId, "UInt64");
|
|
7480
|
+
const lower = `tuple(${lowerCursor}, ${parameters.add(after.traceId, "String")})`;
|
|
7481
|
+
const upper = deltaHead ? `AND tuple(cursorId, traceId) <= tuple(${parameters.add(deltaHead.cursorId, "UInt64")}, ${parameters.add(deltaHead.traceId, "String")})` : "";
|
|
7482
|
+
const ctes = compileClickHouseTraceCandidates(plan, TRACE_LIST_SELECT, parameters, {
|
|
7483
|
+
...QUERY_TRACES_SCOPE,
|
|
7484
|
+
delta: `cursorId >= ${lowerCursor} AND tuple(cursorId, traceId) > ${lower} ${upper}`
|
|
7485
|
+
});
|
|
7294
7486
|
const limit = parameters.add(plan.limit + 1, "UInt64");
|
|
7295
7487
|
return {
|
|
7296
|
-
query:
|
|
7297
|
-
SELECT
|
|
7298
|
-
FROM candidates
|
|
7299
|
-
|
|
7300
|
-
ORDER BY
|
|
7488
|
+
query: `WITH ${ctes.join(",\n")}
|
|
7489
|
+
SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
|
|
7490
|
+
FROM candidates c
|
|
7491
|
+
INNER JOIN delta_candidates d ON c.traceId = d.traceId
|
|
7492
|
+
ORDER BY d.latestCursorId ASC, c.traceId ASC, c.${order.field} ${order.direction}, c.dedupeKey ASC, c.endedAt DESC
|
|
7493
|
+
LIMIT 1 BY c.traceId
|
|
7301
7494
|
LIMIT ${limit}`,
|
|
7302
|
-
query_params: parameters.params
|
|
7495
|
+
query_params: parameters.params,
|
|
7496
|
+
sharedSnapshot: true
|
|
7497
|
+
};
|
|
7498
|
+
}
|
|
7499
|
+
/** Map key matching a payload row to the listed row it was read for. */
|
|
7500
|
+
function traceRootRowKey(row) {
|
|
7501
|
+
return JSON.stringify([
|
|
7502
|
+
String(row.traceId),
|
|
7503
|
+
String(row.dedupeKey),
|
|
7504
|
+
row.organizationId == null ? null : String(row.organizationId),
|
|
7505
|
+
row.resourceId == null ? null : String(row.resourceId)
|
|
7506
|
+
]);
|
|
7507
|
+
}
|
|
7508
|
+
/** A listed row's {@link ClickHouseTraceRootRowKey}. */
|
|
7509
|
+
function toTraceRootRowKey(row) {
|
|
7510
|
+
return {
|
|
7511
|
+
traceId: String(row.traceId),
|
|
7512
|
+
dedupeKey: String(row.dedupeKey),
|
|
7513
|
+
startedAt: asIsoTimestamp(row.startedAt),
|
|
7514
|
+
endedAt: asIsoTimestamp(row.endedAt),
|
|
7515
|
+
organizationId: row.organizationId == null ? null : String(row.organizationId),
|
|
7516
|
+
resourceId: row.resourceId == null ? null : String(row.resourceId)
|
|
7303
7517
|
};
|
|
7304
7518
|
}
|
|
7305
7519
|
/**
|
|
7306
|
-
* Fetches
|
|
7307
|
-
*
|
|
7308
|
-
*
|
|
7309
|
-
*
|
|
7310
|
-
*
|
|
7520
|
+
* Fetches payloads for listed rows: `metadata` and the stored previews, or `input` / `output` /
|
|
7521
|
+
* `error` alone for rows written before the previews were stored. Each row is looked up by its
|
|
7522
|
+
* own identity: the trace_roots sort key `(startedAt, traceId, dedupeKey)`, so only the page's
|
|
7523
|
+
* granules are read; `endedAt`, so the payload comes from the same version as the listed row (a
|
|
7524
|
+
* root can have unmerged versions in different `endedAt` partitions); and its own
|
|
7525
|
+
* `organizationId` / `resourceId`, so a page holding several tenants' rows that share a trace and
|
|
7526
|
+
* span id never takes another tenant's payload. NULL tenant columns match only NULL. Callers
|
|
7527
|
+
* match the returned rows back with {@link traceRootRowKey}.
|
|
7311
7528
|
*/
|
|
7312
|
-
function compileClickHouseTraceRootPayloads(keys) {
|
|
7529
|
+
function compileClickHouseTraceRootPayloads(keys, scope, columns = "payload") {
|
|
7313
7530
|
const parameters = new ParameterBuilder();
|
|
7531
|
+
const tenant = (value) => `${parameters.add(value ?? "", "String")}, ${value === null ? 1 : 0}`;
|
|
7532
|
+
const tuples = keys.map((key) => `(${parameters.add(key.startedAt, "DateTime64(3, 'UTC')")}, ${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}, ${parameters.add(key.endedAt, "DateTime64(3, 'UTC')")}, ${tenant(key.organizationId)}, ${tenant(key.resourceId)})`);
|
|
7314
7533
|
return {
|
|
7315
|
-
query: `SELECT traceId, spanId AS rootSpanId, metadataRaw AS metadata, input
|
|
7534
|
+
query: `SELECT traceId, spanId AS rootSpanId, dedupeKey, organizationId, resourceId, ${columns === "payload" ? `metadataRaw AS metadata, ${PREVIEW_COLUMNS.join(", ")}` : "input, output, error"}
|
|
7316
7535
|
FROM ${TABLE_TRACE_ROOTS}
|
|
7317
|
-
WHERE (startedAt, traceId,
|
|
7318
|
-
LIMIT 1 BY traceId,
|
|
7536
|
+
WHERE (startedAt, traceId, dedupeKey, endedAt, ifNull(organizationId, ''), isNull(organizationId), ifNull(resourceId, ''), isNull(resourceId)) IN (${tuples.join(", ")})${compileTenantScope(scope, parameters)}
|
|
7537
|
+
LIMIT 1 BY traceId, dedupeKey, organizationId, resourceId`,
|
|
7319
7538
|
query_params: parameters.params
|
|
7320
7539
|
};
|
|
7321
7540
|
}
|
|
@@ -7509,56 +7728,15 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7509
7728
|
deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead))
|
|
7510
7729
|
});
|
|
7511
7730
|
}
|
|
7512
|
-
|
|
7513
|
-
|
|
7514
|
-
const
|
|
7515
|
-
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
const payloadRows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: remaining() }, compileClickHouseTraceRootPayloads(pageRows.map((row) => ({
|
|
7519
|
-
traceId: String(row.traceId),
|
|
7520
|
-
rootSpanId: String(row.rootSpanId),
|
|
7521
|
-
startedAt: asIsoTimestamp(row.startedAt),
|
|
7522
|
-
endedAt: asIsoTimestamp(row.endedAt)
|
|
7523
|
-
}))));
|
|
7524
|
-
for (const payload of payloadRows) payloads.set(`${payload.traceId}\u0000${payload.rootSpanId}`, payload);
|
|
7525
|
-
}
|
|
7526
|
-
const traces = pageRows.map((row) => ({
|
|
7527
|
-
...row,
|
|
7528
|
-
...payloads.get(`${row.traceId}\u0000${row.rootSpanId}`)
|
|
7529
|
-
})).map((row) => ({
|
|
7530
|
-
traceId: String(row.traceId),
|
|
7531
|
-
rootSpanId: String(row.rootSpanId),
|
|
7532
|
-
name: row.name,
|
|
7533
|
-
entityId: row.entityId ?? null,
|
|
7534
|
-
parentSpanId: row.parentSpanId ?? null,
|
|
7535
|
-
createdAt: asIsoTimestamp(row.startedAt),
|
|
7536
|
-
metadata: parseJson(row.metadata) ?? null,
|
|
7537
|
-
inputPreview: _mastra_core_storage.buildInputPreview(row.input) ?? null,
|
|
7538
|
-
threadId: row.threadId == null ? null : String(row.threadId),
|
|
7539
|
-
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
7540
|
-
startedAt: asIsoTimestamp(row.startedAt),
|
|
7541
|
-
endedAt: asIsoTimestamp(row.endedAt),
|
|
7542
|
-
entityName: row.entityName == null ? null : String(row.entityName),
|
|
7543
|
-
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7544
|
-
environment: row.environment == null ? null : String(row.environment),
|
|
7545
|
-
status: row.status
|
|
7546
|
-
}));
|
|
7547
|
-
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7548
|
-
traces,
|
|
7549
|
-
...deltaHead ? { deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead)) } : {},
|
|
7550
|
-
pagination: {
|
|
7551
|
-
total,
|
|
7552
|
-
page: plan.page,
|
|
7553
|
-
perPage: plan.perPage,
|
|
7554
|
-
hasMore: (plan.page + 1) * plan.perPage < total
|
|
7555
|
-
}
|
|
7556
|
-
});
|
|
7557
|
-
}
|
|
7558
|
-
const rows = await runWithClickHouseTraceQueryTimeout(client, { timeoutMs: deltaHead ? remaining() : timeoutMs }, compileClickHouseTraceQuery(plan, deltaHead));
|
|
7559
|
-
const visibleRows = rows.slice(0, plan.limit);
|
|
7731
|
+
let first = !deltaHead;
|
|
7732
|
+
const run = (compiled) => {
|
|
7733
|
+
const budget = first ? timeoutMs : remaining();
|
|
7734
|
+
first = false;
|
|
7735
|
+
return runWithClickHouseTraceQueryTimeout(client, { timeoutMs: budget }, compiled);
|
|
7736
|
+
};
|
|
7560
7737
|
if (plan.result === "groups") {
|
|
7561
|
-
const
|
|
7738
|
+
const rows = await run(compileClickHouseTraceQuery(plan));
|
|
7739
|
+
const groups = rows.slice(0, plan.limit).map((row) => ({ threadId: String(row.threadId) }));
|
|
7562
7740
|
const last = groups.at(-1);
|
|
7563
7741
|
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7564
7742
|
groups,
|
|
@@ -7568,7 +7746,32 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7568
7746
|
}) : null }
|
|
7569
7747
|
});
|
|
7570
7748
|
}
|
|
7571
|
-
|
|
7749
|
+
/** Rows written before the previews were stored have none; theirs are built from the payloads. */
|
|
7750
|
+
const toTraceRecords = async (rows) => {
|
|
7751
|
+
const missing = rows.filter(needsPreviewPayloads);
|
|
7752
|
+
const payloads = /* @__PURE__ */ new Map();
|
|
7753
|
+
if (missing.length > 0) {
|
|
7754
|
+
const payloadRows = await run(compileClickHouseTraceRootPayloads(missing.map(toTraceRootRowKey), plan.scope, "previewPayloads"));
|
|
7755
|
+
for (const row of payloadRows) payloads.set(traceRootRowKey(row), row);
|
|
7756
|
+
}
|
|
7757
|
+
return rows.map((row) => toTraceRecord({
|
|
7758
|
+
...row,
|
|
7759
|
+
...resolveStoredPreviews(row, payloads.get(traceRootRowKey(row)))
|
|
7760
|
+
}));
|
|
7761
|
+
};
|
|
7762
|
+
/** Lists rows, then fetches `metadata` and the previews for the rows that are returned. */
|
|
7763
|
+
const listTraces = async (rows) => {
|
|
7764
|
+
const payloads = /* @__PURE__ */ new Map();
|
|
7765
|
+
if (rows.length > 0) {
|
|
7766
|
+
const payloadRows = await run(compileClickHouseTraceRootPayloads(rows.map(toTraceRootRowKey), plan.scope));
|
|
7767
|
+
for (const payload of payloadRows) payloads.set(traceRootRowKey(payload), payload);
|
|
7768
|
+
}
|
|
7769
|
+
return toTraceRecords(rows.map((row) => ({
|
|
7770
|
+
...row,
|
|
7771
|
+
...payloads.get(traceRootRowKey(row))
|
|
7772
|
+
})));
|
|
7773
|
+
};
|
|
7774
|
+
const toTraceRecord = (row) => ({
|
|
7572
7775
|
traceId: String(row.traceId),
|
|
7573
7776
|
rootSpanId: String(row.rootSpanId),
|
|
7574
7777
|
name: row.name,
|
|
@@ -7576,7 +7779,9 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7576
7779
|
parentSpanId: row.parentSpanId ?? null,
|
|
7577
7780
|
createdAt: asIsoTimestamp(row.startedAt),
|
|
7578
7781
|
metadata: parseJson(row.metadata) ?? null,
|
|
7579
|
-
inputPreview:
|
|
7782
|
+
inputPreview: row.inputPreview ?? null,
|
|
7783
|
+
outputPreview: row.outputPreview ?? null,
|
|
7784
|
+
errorPreview: row.errorPreview ?? null,
|
|
7580
7785
|
threadId: row.threadId == null ? null : String(row.threadId),
|
|
7581
7786
|
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
7582
7787
|
startedAt: asIsoTimestamp(row.startedAt),
|
|
@@ -7585,9 +7790,11 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7585
7790
|
entityType: row.entityType == null ? null : String(row.entityType),
|
|
7586
7791
|
environment: row.environment == null ? null : String(row.environment),
|
|
7587
7792
|
status: row.status
|
|
7588
|
-
})
|
|
7589
|
-
const last = traces.at(-1);
|
|
7793
|
+
});
|
|
7590
7794
|
if (plan.paginationMode === "delta") {
|
|
7795
|
+
const rows = await run(compileClickHouseTraceQuery(plan, deltaHead));
|
|
7796
|
+
const visibleRows = rows.slice(0, plan.limit);
|
|
7797
|
+
const traces = await listTraces(visibleRows);
|
|
7591
7798
|
const lastRow = visibleRows.at(-1);
|
|
7592
7799
|
let watermark = lastRow ? {
|
|
7593
7800
|
cursorId: String(lastRow.__delta_cursor),
|
|
@@ -7604,12 +7811,36 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7604
7811
|
deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(watermark))
|
|
7605
7812
|
});
|
|
7606
7813
|
}
|
|
7814
|
+
const keys = await run(compileClickHouseTraceQueryKeys(plan));
|
|
7815
|
+
let total = Number(keys[0]?.__total ?? 0);
|
|
7816
|
+
if (plan.paginationMode === "page" && keys.length === 0 && plan.page > 0) {
|
|
7817
|
+
const [stats] = await run(compileClickHouseTraceQueryTotal(plan));
|
|
7818
|
+
total = Number(stats?.traces ?? 0);
|
|
7819
|
+
}
|
|
7820
|
+
const visible = plan.paginationMode === "page" ? keys : keys.slice(0, plan.limit);
|
|
7821
|
+
const rows = visible.length > 0 ? await run(compileClickHouseTraceRowsByKey(plan, visible.map((key) => ({
|
|
7822
|
+
traceId: String(key.traceId),
|
|
7823
|
+
dedupeKey: String(key.dedupeKey),
|
|
7824
|
+
sortValue: asIsoTimestamp(key.sortValue),
|
|
7825
|
+
rootStartedAt: asIsoTimestamp(key.rootStartedAt)
|
|
7826
|
+
})))) : [];
|
|
7827
|
+
if (plan.paginationMode === "page") return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7828
|
+
traces: await toTraceRecords(rows),
|
|
7829
|
+
...deltaHead ? { deltaCursor: _mastra_core_storage.encodeTraceQueryDeltaCursor(plan, "clickhouse", JSON.stringify(deltaHead)) } : {},
|
|
7830
|
+
pagination: {
|
|
7831
|
+
total,
|
|
7832
|
+
page: plan.page,
|
|
7833
|
+
perPage: plan.perPage,
|
|
7834
|
+
hasMore: (plan.page + 1) * plan.perPage < total
|
|
7835
|
+
}
|
|
7836
|
+
});
|
|
7837
|
+
const lastKey = visible.at(-1);
|
|
7607
7838
|
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
7608
|
-
traces,
|
|
7609
|
-
page: { next:
|
|
7839
|
+
traces: await toTraceRecords(rows),
|
|
7840
|
+
page: { next: keys.length > plan.limit && lastKey ? _mastra_core_storage.encodeTraceQueryCursor(plan, {
|
|
7610
7841
|
result: "traces",
|
|
7611
|
-
sortValue:
|
|
7612
|
-
traceId:
|
|
7842
|
+
sortValue: asIsoTimestamp(lastKey.sortValue),
|
|
7843
|
+
traceId: String(lastKey.traceId)
|
|
7613
7844
|
}) : null }
|
|
7614
7845
|
});
|
|
7615
7846
|
}
|
|
@@ -7779,13 +8010,29 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
7779
8010
|
page: { next: null }
|
|
7780
8011
|
};
|
|
7781
8012
|
const payloadWhere = `tuple(${identitySetColumns}, endedAt) IN {page:Array(Tuple(UInt8, String, UInt8, String, String, String, DateTime64(3, 'UTC')))}`;
|
|
7782
|
-
const
|
|
7783
|
-
|
|
7784
|
-
|
|
7785
|
-
FROM (SELECT ${identities}, any(tuple(${displayFields.map(fieldExpression).join(", ")}
|
|
8013
|
+
const pageParameter = (rows) => rows.map((row) => new _clickhouse_client.TupleParam([...identityValues(row), row.endedAt.replace("T", " ").replace(/Z$/, "")]));
|
|
8014
|
+
const rowsWithPreviews = await run({
|
|
8015
|
+
query: `SELECT ${identities}, ${[...displayFields, ...PREVIEW_COLUMNS].map((field, index) => `data.${index + 1} AS ${field}`).join(", ")}
|
|
8016
|
+
FROM (SELECT ${identities}, any(tuple(${[...displayFields.map(fieldExpression), ...PREVIEW_COLUMNS].join(", ")})) AS data
|
|
7786
8017
|
FROM ${TABLE_SPAN_EVENTS} e WHERE ${payloadWhere} GROUP BY ${identities})`,
|
|
7787
|
-
query_params: { page: page
|
|
8018
|
+
query_params: { page: pageParameter(page) }
|
|
7788
8019
|
});
|
|
8020
|
+
const missingKeys = new Set(rowsWithPreviews.filter(needsPreviewPayloads).map(_mastra_core_storage.spanQueryIdentityKey));
|
|
8021
|
+
const missing = page.filter((row) => missingKeys.has(_mastra_core_storage.spanQueryIdentityKey(row)));
|
|
8022
|
+
const previewPayloads = /* @__PURE__ */ new Map();
|
|
8023
|
+
if (missing.length) {
|
|
8024
|
+
const rows = await run({
|
|
8025
|
+
query: `SELECT ${identities}, data.1 AS input, data.2 AS output, data.3 AS error
|
|
8026
|
+
FROM (SELECT ${identities}, any(tuple(input, output, error)) AS data
|
|
8027
|
+
FROM ${TABLE_SPAN_EVENTS} e WHERE ${payloadWhere} GROUP BY ${identities})`,
|
|
8028
|
+
query_params: { page: pageParameter(missing) }
|
|
8029
|
+
});
|
|
8030
|
+
for (const row of rows) previewPayloads.set(_mastra_core_storage.spanQueryIdentityKey(row), row);
|
|
8031
|
+
}
|
|
8032
|
+
const payloads = rowsWithPreviews.map((row) => ({
|
|
8033
|
+
...row,
|
|
8034
|
+
...resolveStoredPreviews(row, previewPayloads.get(_mastra_core_storage.spanQueryIdentityKey(row)))
|
|
8035
|
+
}));
|
|
7789
8036
|
const metricWhere = `${identitySetKey} IN {page:Array(Tuple(UInt8, String, UInt8, String, String, String))}`;
|
|
7790
8037
|
const metrics = await run({
|
|
7791
8038
|
query: `SELECT ${identities}, data.1 AS name, data.2 AS estimatedCost,
|
|
@@ -7799,17 +8046,16 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
7799
8046
|
});
|
|
7800
8047
|
if (metrics.length > _mastra_core_storage.SPAN_QUERY_MAX_COST_METRICS) throw new _mastra_core_storage.TraceQueryResourceLimitError();
|
|
7801
8048
|
const byIdentity = new Map(payloads.map((row) => [_mastra_core_storage.spanQueryIdentityKey(row), row]));
|
|
7802
|
-
const
|
|
8049
|
+
const result = buildSpanQueryResponse(plan, page.flatMap((identity) => {
|
|
7803
8050
|
const row = byIdentity.get(_mastra_core_storage.spanQueryIdentityKey(identity));
|
|
7804
8051
|
if (!row) return [];
|
|
7805
|
-
const { inputPreview: _input, outputPreview: _output, ...metadata } = row;
|
|
8052
|
+
const { inputPreview: _input, outputPreview: _output, errorPreview: _error, ...metadata } = row;
|
|
7806
8053
|
return [{
|
|
7807
8054
|
...metadata,
|
|
7808
8055
|
...identity,
|
|
7809
8056
|
durationMs: Number(row.durationMs)
|
|
7810
8057
|
}];
|
|
7811
|
-
});
|
|
7812
|
-
const result = _mastra_core_storage.buildSpanQueryResponse(plan, hydrated, payloads, metrics);
|
|
8058
|
+
}), payloads, metrics);
|
|
7813
8059
|
const last = page.at(-1);
|
|
7814
8060
|
result.page.next = selected.length > plan.limit ? _mastra_core_storage.encodeSpanQueryCursor(plan, {
|
|
7815
8061
|
organizationId: last.organizationId,
|
|
@@ -8113,9 +8359,9 @@ async function getRootSpan(client, args) {
|
|
|
8113
8359
|
return { span: rowToSpanRecord(rows[0]) };
|
|
8114
8360
|
}
|
|
8115
8361
|
/**
|
|
8116
|
-
* Columns a trace list renders.
|
|
8117
|
-
*
|
|
8118
|
-
*
|
|
8362
|
+
* Columns a trace list renders. The previews are the stored ones; rows written
|
|
8363
|
+
* before they were stored get theirs from the payload columns, fetched for just
|
|
8364
|
+
* those rows (see `withPreviewPayloads`).
|
|
8119
8365
|
*/
|
|
8120
8366
|
const LIGHT_TRACE_ROOT_FIELDS = [
|
|
8121
8367
|
"traceId",
|
|
@@ -8130,10 +8376,13 @@ const LIGHT_TRACE_ROOT_FIELDS = [
|
|
|
8130
8376
|
"entityId",
|
|
8131
8377
|
"entityName",
|
|
8132
8378
|
"threadId",
|
|
8379
|
+
"organizationId",
|
|
8133
8380
|
"resourceId",
|
|
8134
8381
|
"error",
|
|
8135
8382
|
"metadataRaw",
|
|
8136
|
-
"
|
|
8383
|
+
"inputPreview",
|
|
8384
|
+
"outputPreview",
|
|
8385
|
+
"errorPreview"
|
|
8137
8386
|
];
|
|
8138
8387
|
const FULL_PROJECTION = {
|
|
8139
8388
|
innerSelect: "*",
|
|
@@ -8142,7 +8391,7 @@ const FULL_PROJECTION = {
|
|
|
8142
8391
|
};
|
|
8143
8392
|
const LIGHT_PROJECTION = {
|
|
8144
8393
|
innerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
|
|
8145
|
-
outerSelect: LIGHT_TRACE_ROOT_FIELDS.join(", "),
|
|
8394
|
+
outerSelect: ["dedupeKey", ...LIGHT_TRACE_ROOT_FIELDS].join(", "),
|
|
8146
8395
|
deltaSelect: LIGHT_TRACE_ROOT_FIELDS.filter((field) => field !== "startedAt" && field !== "traceId").map((field) => `r.${field}`).join(", ")
|
|
8147
8396
|
};
|
|
8148
8397
|
/**
|
|
@@ -8188,7 +8437,7 @@ async function listTraceRows(client, args, strategy, projection, mapRows) {
|
|
|
8188
8437
|
const rows = await queryTracesAfterCursor(client, projection.deltaSelect, whereClause, params, limit, afterCursor);
|
|
8189
8438
|
const visibleRows = rows.slice(0, limit);
|
|
8190
8439
|
return {
|
|
8191
|
-
spans: mapRows(visibleRows),
|
|
8440
|
+
spans: await mapRows(visibleRows),
|
|
8192
8441
|
delta: {
|
|
8193
8442
|
limit,
|
|
8194
8443
|
hasMore: rows.length > limit
|
|
@@ -8263,7 +8512,7 @@ async function listTraceRows(client, args, strategy, projection, mapRows) {
|
|
|
8263
8512
|
perPage,
|
|
8264
8513
|
hasMore: (page + 1) * perPage < total
|
|
8265
8514
|
},
|
|
8266
|
-
spans: mapRows(rows),
|
|
8515
|
+
spans: await mapRows(rows),
|
|
8267
8516
|
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
8268
8517
|
};
|
|
8269
8518
|
}
|
|
@@ -8272,12 +8521,32 @@ async function listTraces(client, args, strategy) {
|
|
|
8272
8521
|
return listTraceRows(client, args, strategy, FULL_PROJECTION, (rows) => (0, _mastra_core_storage.toTraceSpans)(rows.map(rowToSpanRecord)));
|
|
8273
8522
|
}
|
|
8274
8523
|
/**
|
|
8524
|
+
* Adds `input` / `output` / `error` to the rows written before previews were stored, read from
|
|
8525
|
+
* the same root row version (same `endedAt`) the list kept. A page can hold several tenants' rows,
|
|
8526
|
+
* so each row is looked up by its own identity, tenant included, not by one shared scope.
|
|
8527
|
+
*/
|
|
8528
|
+
async function withPreviewPayloads(client, rows) {
|
|
8529
|
+
const missing = rows.filter(needsPreviewPayloads);
|
|
8530
|
+
if (missing.length === 0) return rows;
|
|
8531
|
+
const compiled = compileClickHouseTraceRootPayloads(missing.map(toTraceRootRowKey), void 0, "previewPayloads");
|
|
8532
|
+
const result = await client.query({
|
|
8533
|
+
...compiled,
|
|
8534
|
+
format: "JSONEachRow",
|
|
8535
|
+
clickhouse_settings: CH_SETTINGS
|
|
8536
|
+
});
|
|
8537
|
+
const payloads = new Map((await result.json()).map((row) => [traceRootRowKey(row), row]));
|
|
8538
|
+
return rows.map((row) => needsPreviewPayloads(row) ? {
|
|
8539
|
+
...row,
|
|
8540
|
+
...payloads.get(traceRootRowKey(row))
|
|
8541
|
+
} : row);
|
|
8542
|
+
}
|
|
8543
|
+
/**
|
|
8275
8544
|
* List traces projecting only the columns a trace list renders.
|
|
8276
|
-
* Skips the attributes/output blobs and
|
|
8277
|
-
*
|
|
8545
|
+
* Skips the attributes/input/output blobs and reads the stored previews instead,
|
|
8546
|
+
* so the response payload stays flat as traces grow.
|
|
8278
8547
|
*/
|
|
8279
8548
|
async function listTracesLight(client, args, strategy) {
|
|
8280
|
-
return listTraceRows(client, args, strategy, LIGHT_PROJECTION, (rows) => rows.map(rowToLightSpanRecord));
|
|
8549
|
+
return listTraceRows(client, args, strategy, LIGHT_PROJECTION, async (rows) => (await withPreviewPayloads(client, rows)).map(rowToLightSpanRecord));
|
|
8281
8550
|
}
|
|
8282
8551
|
async function queryTracesAfterCursor(client, deltaSelect, whereClause, params, limit, cursorId) {
|
|
8283
8552
|
const deltaKeys = `SELECT startedAt, traceId, dedupeKey FROM ${TABLE_TRACE_ROOTS_DELTA} WHERE cursorId > {afterCursor:UInt64}`;
|