@mastra/clickhouse 1.8.0 → 1.9.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +1020 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1020 -41
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +34 -3
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/feedback.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/feedback.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +4 -0
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/logs.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/logs.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/metrics.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/metrics.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/polling.d.ts +7 -0
- package/dist/storage/domains/observability/v-next/polling.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/scores.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/scores.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts +12 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/tracing.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/tracing.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/index.cjs
CHANGED
|
@@ -6,6 +6,7 @@ var storage = require('@mastra/core/storage');
|
|
|
6
6
|
var base = require('@mastra/core/base');
|
|
7
7
|
var agent = require('@mastra/core/agent');
|
|
8
8
|
var utils = require('@mastra/core/utils');
|
|
9
|
+
var features = require('@mastra/core/features');
|
|
9
10
|
var evals = require('@mastra/core/evals');
|
|
10
11
|
|
|
11
12
|
// src/storage/index.ts
|
|
@@ -2860,16 +2861,36 @@ time for large tables. Please ensure you have a backup before proceeding.
|
|
|
2860
2861
|
var TABLE_SPAN_EVENTS = "mastra_span_events";
|
|
2861
2862
|
var TABLE_TRACE_ROOTS = "mastra_trace_roots";
|
|
2862
2863
|
var TABLE_TRACE_BRANCHES = "mastra_trace_branches";
|
|
2864
|
+
var TABLE_TRACE_ROOTS_DELTA = "mastra_trace_roots_delta";
|
|
2865
|
+
var TABLE_TRACE_BRANCHES_DELTA = "mastra_trace_branches_delta";
|
|
2863
2866
|
var TABLE_METRIC_EVENTS = "mastra_metric_events";
|
|
2864
2867
|
var TABLE_LOG_EVENTS = "mastra_log_events";
|
|
2865
2868
|
var TABLE_SCORE_EVENTS = "mastra_score_events";
|
|
2866
2869
|
var TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
|
|
2870
|
+
var TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
|
|
2871
|
+
var TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
|
|
2872
|
+
var TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
|
|
2873
|
+
var TABLE_FEEDBACK_EVENTS_DELTA = "mastra_feedback_events_delta";
|
|
2867
2874
|
var TABLE_DISCOVERY_VALUES = "mastra_discovery_values";
|
|
2868
2875
|
var TABLE_DISCOVERY_PAIRS = "mastra_discovery_pairs";
|
|
2869
2876
|
var MV_TRACE_ROOTS = "mastra_mv_trace_roots";
|
|
2870
2877
|
var MV_TRACE_BRANCHES = "mastra_mv_trace_branches";
|
|
2878
|
+
var MV_TRACE_ROOTS_DELTA = "mastra_mv_trace_roots_delta";
|
|
2879
|
+
var MV_TRACE_BRANCHES_DELTA = "mastra_mv_trace_branches_delta";
|
|
2880
|
+
var MV_METRIC_EVENTS_DELTA = "mastra_mv_metric_events_delta";
|
|
2881
|
+
var MV_LOG_EVENTS_DELTA = "mastra_mv_log_events_delta";
|
|
2882
|
+
var MV_SCORE_EVENTS_DELTA = "mastra_mv_score_events_delta";
|
|
2883
|
+
var MV_FEEDBACK_EVENTS_DELTA = "mastra_mv_feedback_events_delta";
|
|
2871
2884
|
var MV_DISCOVERY_VALUES = "mastra_mv_discovery_values";
|
|
2872
2885
|
var MV_DISCOVERY_PAIRS = "mastra_mv_discovery_pairs";
|
|
2886
|
+
var DELTA_MV_NAMES = [
|
|
2887
|
+
MV_TRACE_ROOTS_DELTA,
|
|
2888
|
+
MV_TRACE_BRANCHES_DELTA,
|
|
2889
|
+
MV_METRIC_EVENTS_DELTA,
|
|
2890
|
+
MV_LOG_EVENTS_DELTA,
|
|
2891
|
+
MV_SCORE_EVENTS_DELTA,
|
|
2892
|
+
MV_FEEDBACK_EVENTS_DELTA
|
|
2893
|
+
];
|
|
2873
2894
|
var BRANCH_SPAN_TYPE_VALUES = [
|
|
2874
2895
|
"agent_run",
|
|
2875
2896
|
"workflow_run",
|
|
@@ -3093,6 +3114,103 @@ SELECT *
|
|
|
3093
3114
|
FROM ${TABLE_SPAN_EVENTS}
|
|
3094
3115
|
WHERE spanType IN (${BRANCH_SPAN_TYPE_VALUES.map((v) => `'${v}'`).join(", ")})
|
|
3095
3116
|
`;
|
|
3117
|
+
var DELTA_INGESTED_AT_TYPE = `DateTime64(9, 'UTC')`;
|
|
3118
|
+
var DELTA_CURSOR_EPOCH_MS = 17778528e5;
|
|
3119
|
+
var DELTA_CURSOR_SUFFIX_BITS = 26;
|
|
3120
|
+
var DELTA_CURSOR_SUFFIX_MASK = 67108863;
|
|
3121
|
+
function buildFallbackCursorExpr(stableKeyExpr) {
|
|
3122
|
+
return `bitOr(
|
|
3123
|
+
bitShiftLeft(
|
|
3124
|
+
toUInt64(toUnixTimestamp64Milli(ingestedAt) - ${DELTA_CURSOR_EPOCH_MS}),
|
|
3125
|
+
${DELTA_CURSOR_SUFFIX_BITS}
|
|
3126
|
+
),
|
|
3127
|
+
bitAnd(farmFingerprint64(toString(${stableKeyExpr})), toUInt64(${DELTA_CURSOR_SUFFIX_MASK}))
|
|
3128
|
+
)`;
|
|
3129
|
+
}
|
|
3130
|
+
function buildDeltaCursorExpr(strategy, counterName, stableKeyExpr) {
|
|
3131
|
+
if (strategy === "serial") {
|
|
3132
|
+
return `generateSerialID('${counterName}')`;
|
|
3133
|
+
}
|
|
3134
|
+
return buildFallbackCursorExpr(stableKeyExpr);
|
|
3135
|
+
}
|
|
3136
|
+
function buildTraceRootsDeltaDDL() {
|
|
3137
|
+
return `
|
|
3138
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_ROOTS_DELTA} (
|
|
3139
|
+
cursorId UInt64,
|
|
3140
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3141
|
+
startedAt DateTime64(3, 'UTC'),
|
|
3142
|
+
traceId String,
|
|
3143
|
+
dedupeKey String
|
|
3144
|
+
)
|
|
3145
|
+
ENGINE = MergeTree
|
|
3146
|
+
PARTITION BY toDate(ingestedAt)
|
|
3147
|
+
ORDER BY (cursorId)
|
|
3148
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3149
|
+
`;
|
|
3150
|
+
}
|
|
3151
|
+
function buildTraceRootsDeltaMvDDL(strategy) {
|
|
3152
|
+
return `
|
|
3153
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_TRACE_ROOTS_DELTA}
|
|
3154
|
+
TO ${TABLE_TRACE_ROOTS_DELTA}
|
|
3155
|
+
AS
|
|
3156
|
+
SELECT
|
|
3157
|
+
${buildDeltaCursorExpr(strategy, "mastra_trace_roots_delta_cursor", "dedupeKey")} AS cursorId,
|
|
3158
|
+
ingestedAt,
|
|
3159
|
+
startedAt,
|
|
3160
|
+
traceId,
|
|
3161
|
+
dedupeKey
|
|
3162
|
+
FROM (
|
|
3163
|
+
SELECT
|
|
3164
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3165
|
+
startedAt,
|
|
3166
|
+
traceId,
|
|
3167
|
+
dedupeKey
|
|
3168
|
+
FROM ${TABLE_TRACE_ROOTS}
|
|
3169
|
+
)
|
|
3170
|
+
`;
|
|
3171
|
+
}
|
|
3172
|
+
function buildTraceBranchesDeltaDDL() {
|
|
3173
|
+
return `
|
|
3174
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_BRANCHES_DELTA} (
|
|
3175
|
+
cursorId UInt64,
|
|
3176
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3177
|
+
spanType LowCardinality(String),
|
|
3178
|
+
startedAt DateTime64(3, 'UTC'),
|
|
3179
|
+
traceId String,
|
|
3180
|
+
spanId String,
|
|
3181
|
+
dedupeKey String
|
|
3182
|
+
)
|
|
3183
|
+
ENGINE = MergeTree
|
|
3184
|
+
PARTITION BY toDate(ingestedAt)
|
|
3185
|
+
ORDER BY (cursorId)
|
|
3186
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3187
|
+
`;
|
|
3188
|
+
}
|
|
3189
|
+
function buildTraceBranchesDeltaMvDDL(strategy) {
|
|
3190
|
+
return `
|
|
3191
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_TRACE_BRANCHES_DELTA}
|
|
3192
|
+
TO ${TABLE_TRACE_BRANCHES_DELTA}
|
|
3193
|
+
AS
|
|
3194
|
+
SELECT
|
|
3195
|
+
${buildDeltaCursorExpr(strategy, "mastra_trace_branches_delta_cursor", "dedupeKey")} AS cursorId,
|
|
3196
|
+
ingestedAt,
|
|
3197
|
+
spanType,
|
|
3198
|
+
startedAt,
|
|
3199
|
+
traceId,
|
|
3200
|
+
spanId,
|
|
3201
|
+
dedupeKey
|
|
3202
|
+
FROM (
|
|
3203
|
+
SELECT
|
|
3204
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3205
|
+
spanType,
|
|
3206
|
+
startedAt,
|
|
3207
|
+
traceId,
|
|
3208
|
+
spanId,
|
|
3209
|
+
dedupeKey
|
|
3210
|
+
FROM ${TABLE_TRACE_BRANCHES}
|
|
3211
|
+
)
|
|
3212
|
+
`;
|
|
3213
|
+
}
|
|
3096
3214
|
var METRIC_EVENTS_DDL = `
|
|
3097
3215
|
CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS} (
|
|
3098
3216
|
-- Timestamp
|
|
@@ -3219,6 +3337,39 @@ ENGINE = ReplacingMergeTree
|
|
|
3219
3337
|
PARTITION BY toDate(timestamp)
|
|
3220
3338
|
ORDER BY (timestamp, logId)
|
|
3221
3339
|
`;
|
|
3340
|
+
function buildLogEventsDeltaDDL() {
|
|
3341
|
+
return `
|
|
3342
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_LOG_EVENTS_DELTA} (
|
|
3343
|
+
cursorId UInt64,
|
|
3344
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3345
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3346
|
+
logId String
|
|
3347
|
+
)
|
|
3348
|
+
ENGINE = MergeTree
|
|
3349
|
+
PARTITION BY toDate(ingestedAt)
|
|
3350
|
+
ORDER BY (cursorId)
|
|
3351
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3352
|
+
`;
|
|
3353
|
+
}
|
|
3354
|
+
function buildLogEventsDeltaMvDDL(strategy) {
|
|
3355
|
+
return `
|
|
3356
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_LOG_EVENTS_DELTA}
|
|
3357
|
+
TO ${TABLE_LOG_EVENTS_DELTA}
|
|
3358
|
+
AS
|
|
3359
|
+
SELECT
|
|
3360
|
+
${buildDeltaCursorExpr(strategy, "mastra_log_events_delta_cursor", "logId")} AS cursorId,
|
|
3361
|
+
ingestedAt,
|
|
3362
|
+
timestamp,
|
|
3363
|
+
logId
|
|
3364
|
+
FROM (
|
|
3365
|
+
SELECT
|
|
3366
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3367
|
+
timestamp,
|
|
3368
|
+
logId
|
|
3369
|
+
FROM ${TABLE_LOG_EVENTS}
|
|
3370
|
+
)
|
|
3371
|
+
`;
|
|
3372
|
+
}
|
|
3222
3373
|
var SCORE_EVENTS_DDL = `
|
|
3223
3374
|
CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
|
|
3224
3375
|
-- Timestamp
|
|
@@ -3280,6 +3431,43 @@ PARTITION BY toDate(timestamp)
|
|
|
3280
3431
|
ORDER BY (traceId, timestamp, scoreId)
|
|
3281
3432
|
SETTINGS allow_nullable_key = 1
|
|
3282
3433
|
`;
|
|
3434
|
+
function buildScoreEventsDeltaDDL() {
|
|
3435
|
+
return `
|
|
3436
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
|
|
3437
|
+
cursorId UInt64,
|
|
3438
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3439
|
+
traceId Nullable(String),
|
|
3440
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3441
|
+
scoreId String
|
|
3442
|
+
)
|
|
3443
|
+
ENGINE = MergeTree
|
|
3444
|
+
PARTITION BY toDate(ingestedAt)
|
|
3445
|
+
ORDER BY (cursorId)
|
|
3446
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3447
|
+
SETTINGS allow_nullable_key = 1
|
|
3448
|
+
`;
|
|
3449
|
+
}
|
|
3450
|
+
function buildScoreEventsDeltaMvDDL(strategy) {
|
|
3451
|
+
return `
|
|
3452
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
|
|
3453
|
+
TO ${TABLE_SCORE_EVENTS_DELTA}
|
|
3454
|
+
AS
|
|
3455
|
+
SELECT
|
|
3456
|
+
${buildDeltaCursorExpr(strategy, "mastra_score_events_delta_cursor", "scoreId")} AS cursorId,
|
|
3457
|
+
ingestedAt,
|
|
3458
|
+
traceId,
|
|
3459
|
+
timestamp,
|
|
3460
|
+
scoreId
|
|
3461
|
+
FROM (
|
|
3462
|
+
SELECT
|
|
3463
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3464
|
+
traceId,
|
|
3465
|
+
timestamp,
|
|
3466
|
+
scoreId
|
|
3467
|
+
FROM ${TABLE_SCORE_EVENTS}
|
|
3468
|
+
)
|
|
3469
|
+
`;
|
|
3470
|
+
}
|
|
3283
3471
|
var FEEDBACK_EVENTS_DDL = `
|
|
3284
3472
|
CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS} (
|
|
3285
3473
|
-- Timestamp
|
|
@@ -3344,6 +3532,79 @@ PARTITION BY toDate(timestamp)
|
|
|
3344
3532
|
ORDER BY (traceId, timestamp, feedbackId)
|
|
3345
3533
|
SETTINGS allow_nullable_key = 1
|
|
3346
3534
|
`;
|
|
3535
|
+
function buildFeedbackEventsDeltaDDL() {
|
|
3536
|
+
return `
|
|
3537
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
|
|
3538
|
+
cursorId UInt64,
|
|
3539
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3540
|
+
traceId Nullable(String),
|
|
3541
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3542
|
+
feedbackId String
|
|
3543
|
+
)
|
|
3544
|
+
ENGINE = MergeTree
|
|
3545
|
+
PARTITION BY toDate(ingestedAt)
|
|
3546
|
+
ORDER BY (cursorId)
|
|
3547
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3548
|
+
SETTINGS allow_nullable_key = 1
|
|
3549
|
+
`;
|
|
3550
|
+
}
|
|
3551
|
+
function buildFeedbackEventsDeltaMvDDL(strategy) {
|
|
3552
|
+
return `
|
|
3553
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_FEEDBACK_EVENTS_DELTA}
|
|
3554
|
+
TO ${TABLE_FEEDBACK_EVENTS_DELTA}
|
|
3555
|
+
AS
|
|
3556
|
+
SELECT
|
|
3557
|
+
${buildDeltaCursorExpr(strategy, "mastra_feedback_events_delta_cursor", "feedbackId")} AS cursorId,
|
|
3558
|
+
ingestedAt,
|
|
3559
|
+
traceId,
|
|
3560
|
+
timestamp,
|
|
3561
|
+
feedbackId
|
|
3562
|
+
FROM (
|
|
3563
|
+
SELECT
|
|
3564
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3565
|
+
traceId,
|
|
3566
|
+
timestamp,
|
|
3567
|
+
feedbackId
|
|
3568
|
+
FROM ${TABLE_FEEDBACK_EVENTS}
|
|
3569
|
+
)
|
|
3570
|
+
`;
|
|
3571
|
+
}
|
|
3572
|
+
function buildMetricEventsDeltaDDL() {
|
|
3573
|
+
return `
|
|
3574
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS_DELTA} (
|
|
3575
|
+
cursorId UInt64,
|
|
3576
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3577
|
+
name LowCardinality(String),
|
|
3578
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3579
|
+
metricId String
|
|
3580
|
+
)
|
|
3581
|
+
ENGINE = MergeTree
|
|
3582
|
+
PARTITION BY toDate(ingestedAt)
|
|
3583
|
+
ORDER BY (cursorId)
|
|
3584
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3585
|
+
`;
|
|
3586
|
+
}
|
|
3587
|
+
function buildMetricEventsDeltaMvDDL(strategy) {
|
|
3588
|
+
return `
|
|
3589
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_METRIC_EVENTS_DELTA}
|
|
3590
|
+
TO ${TABLE_METRIC_EVENTS_DELTA}
|
|
3591
|
+
AS
|
|
3592
|
+
SELECT
|
|
3593
|
+
${buildDeltaCursorExpr(strategy, "mastra_metric_events_delta_cursor", "metricId")} AS cursorId,
|
|
3594
|
+
ingestedAt,
|
|
3595
|
+
name,
|
|
3596
|
+
timestamp,
|
|
3597
|
+
metricId
|
|
3598
|
+
FROM (
|
|
3599
|
+
SELECT
|
|
3600
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3601
|
+
name,
|
|
3602
|
+
timestamp,
|
|
3603
|
+
metricId
|
|
3604
|
+
FROM ${TABLE_METRIC_EVENTS}
|
|
3605
|
+
)
|
|
3606
|
+
`;
|
|
3607
|
+
}
|
|
3347
3608
|
var DISCOVERY_VALUES_DDL = `
|
|
3348
3609
|
CREATE TABLE IF NOT EXISTS ${TABLE_DISCOVERY_VALUES} (
|
|
3349
3610
|
kind LowCardinality(String),
|
|
@@ -3412,7 +3673,7 @@ SELECT DISTINCT kind, key1, key2, value FROM (
|
|
|
3412
3673
|
WHERE name != '' AND labelKey != '' AND labels[labelKey] != ''
|
|
3413
3674
|
)
|
|
3414
3675
|
`;
|
|
3415
|
-
var
|
|
3676
|
+
var BASE_TABLE_DDL = [
|
|
3416
3677
|
SPAN_EVENTS_DDL,
|
|
3417
3678
|
TRACE_ROOTS_DDL,
|
|
3418
3679
|
TRACE_BRANCHES_DDL,
|
|
@@ -3423,7 +3684,33 @@ var ALL_TABLE_DDL = [
|
|
|
3423
3684
|
DISCOVERY_VALUES_DDL,
|
|
3424
3685
|
DISCOVERY_PAIRS_DDL
|
|
3425
3686
|
];
|
|
3426
|
-
|
|
3687
|
+
function buildDeltaTableDDL() {
|
|
3688
|
+
return [
|
|
3689
|
+
buildTraceRootsDeltaDDL(),
|
|
3690
|
+
buildTraceBranchesDeltaDDL(),
|
|
3691
|
+
buildMetricEventsDeltaDDL(),
|
|
3692
|
+
buildLogEventsDeltaDDL(),
|
|
3693
|
+
buildScoreEventsDeltaDDL(),
|
|
3694
|
+
buildFeedbackEventsDeltaDDL()
|
|
3695
|
+
];
|
|
3696
|
+
}
|
|
3697
|
+
function buildAllTableDDL() {
|
|
3698
|
+
return [...BASE_TABLE_DDL, ...buildDeltaTableDDL()];
|
|
3699
|
+
}
|
|
3700
|
+
var BASE_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
|
|
3701
|
+
function buildDeltaMvDDL(strategy) {
|
|
3702
|
+
return [
|
|
3703
|
+
buildTraceRootsDeltaMvDDL(strategy),
|
|
3704
|
+
buildTraceBranchesDeltaMvDDL(strategy),
|
|
3705
|
+
buildMetricEventsDeltaMvDDL(strategy),
|
|
3706
|
+
buildLogEventsDeltaMvDDL(strategy),
|
|
3707
|
+
buildScoreEventsDeltaMvDDL(strategy),
|
|
3708
|
+
buildFeedbackEventsDeltaMvDDL(strategy)
|
|
3709
|
+
];
|
|
3710
|
+
}
|
|
3711
|
+
function buildAllMvDDL(strategy) {
|
|
3712
|
+
return [...BASE_MV_DDL, ...buildDeltaMvDDL(strategy)];
|
|
3713
|
+
}
|
|
3427
3714
|
var DISCOVERY_MV_DDL = [DISCOVERY_VALUES_MV_DDL, DISCOVERY_PAIRS_MV_DDL];
|
|
3428
3715
|
var addColumn = (table, name, type) => ({
|
|
3429
3716
|
kind: "column",
|
|
@@ -3480,10 +3767,16 @@ var ALL_TABLE_NAMES = [
|
|
|
3480
3767
|
TABLE_SPAN_EVENTS,
|
|
3481
3768
|
TABLE_TRACE_ROOTS,
|
|
3482
3769
|
TABLE_TRACE_BRANCHES,
|
|
3770
|
+
TABLE_TRACE_ROOTS_DELTA,
|
|
3771
|
+
TABLE_TRACE_BRANCHES_DELTA,
|
|
3483
3772
|
TABLE_METRIC_EVENTS,
|
|
3484
3773
|
TABLE_LOG_EVENTS,
|
|
3485
3774
|
TABLE_SCORE_EVENTS,
|
|
3486
3775
|
TABLE_FEEDBACK_EVENTS,
|
|
3776
|
+
TABLE_METRIC_EVENTS_DELTA,
|
|
3777
|
+
TABLE_LOG_EVENTS_DELTA,
|
|
3778
|
+
TABLE_SCORE_EVENTS_DELTA,
|
|
3779
|
+
TABLE_FEEDBACK_EVENTS_DELTA,
|
|
3487
3780
|
TABLE_DISCOVERY_VALUES,
|
|
3488
3781
|
TABLE_DISCOVERY_PAIRS
|
|
3489
3782
|
];
|
|
@@ -4354,6 +4647,35 @@ function buildPaginationClause(pagination) {
|
|
|
4354
4647
|
offset: page * perPage
|
|
4355
4648
|
};
|
|
4356
4649
|
}
|
|
4650
|
+
var OBSERVABILITY_DELTA_POLLING_FEATURE = "observability-delta-polling";
|
|
4651
|
+
function deltaPollingFeatureEnabled() {
|
|
4652
|
+
return features.coreFeatures.has(OBSERVABILITY_DELTA_POLLING_FEATURE);
|
|
4653
|
+
}
|
|
4654
|
+
function deltaPollingSupported(strategy) {
|
|
4655
|
+
return deltaPollingFeatureEnabled() && strategy !== null;
|
|
4656
|
+
}
|
|
4657
|
+
function assertDeltaPollingSupported(strategy) {
|
|
4658
|
+
if (deltaPollingSupported(strategy)) {
|
|
4659
|
+
return;
|
|
4660
|
+
}
|
|
4661
|
+
throw new error.MastraError({
|
|
4662
|
+
id: "OBSERVABILITY_DELTA_POLLING_NOT_SUPPORTED",
|
|
4663
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4664
|
+
category: error.ErrorCategory.SYSTEM,
|
|
4665
|
+
text: "This storage provider does not support observability delta polling"
|
|
4666
|
+
});
|
|
4667
|
+
}
|
|
4668
|
+
function validateCursorId(cursor) {
|
|
4669
|
+
if (/^\d+$/.test(cursor)) {
|
|
4670
|
+
return cursor;
|
|
4671
|
+
}
|
|
4672
|
+
throw new error.MastraError({
|
|
4673
|
+
id: "OBSERVABILITY_INVALID_DELTA_CURSOR",
|
|
4674
|
+
domain: error.ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4675
|
+
category: error.ErrorCategory.USER,
|
|
4676
|
+
text: "Invalid observability delta cursor"
|
|
4677
|
+
});
|
|
4678
|
+
}
|
|
4357
4679
|
|
|
4358
4680
|
// src/storage/domains/observability/v-next/feedback.ts
|
|
4359
4681
|
var FEEDBACK_TYPED_COLUMNS = /* @__PURE__ */ new Set([
|
|
@@ -4480,12 +4802,33 @@ async function batchCreateFeedback(client, args) {
|
|
|
4480
4802
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4481
4803
|
});
|
|
4482
4804
|
}
|
|
4483
|
-
async function listFeedback(client, args) {
|
|
4805
|
+
async function listFeedback(client, args, strategy) {
|
|
4484
4806
|
const parsed = storage.listFeedbackArgsSchema.parse(args);
|
|
4807
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4485
4808
|
const filter = buildFeedbackFilterConditions(parsed.filters, "f");
|
|
4486
4809
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4487
4810
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "f");
|
|
4488
4811
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
4812
|
+
if (parsed.mode === "delta") {
|
|
4813
|
+
assertDeltaPollingSupported(strategy);
|
|
4814
|
+
const streamHeadCursor = await getStreamHeadCursor(client);
|
|
4815
|
+
if (parsed.after === void 0) {
|
|
4816
|
+
return {
|
|
4817
|
+
feedback: [],
|
|
4818
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
4819
|
+
deltaCursor: streamHeadCursor
|
|
4820
|
+
};
|
|
4821
|
+
}
|
|
4822
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
4823
|
+
const rows2 = await queryFeedbackAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
4824
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
4825
|
+
return {
|
|
4826
|
+
feedback: visibleRows.map(rowToFeedbackRecord),
|
|
4827
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
4828
|
+
deltaCursor: visibleRows.length > 0 ? buildFeedbackCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
4829
|
+
};
|
|
4830
|
+
}
|
|
4831
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor(client, whereClause, filter.params) : void 0;
|
|
4489
4832
|
const countResult = await queryJson2(
|
|
4490
4833
|
client,
|
|
4491
4834
|
`SELECT count() AS total FROM ${TABLE_FEEDBACK_EVENTS} AS f ${whereClause}`,
|
|
@@ -4504,9 +4847,68 @@ async function listFeedback(client, args) {
|
|
|
4504
4847
|
perPage: pagination.perPage,
|
|
4505
4848
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4506
4849
|
},
|
|
4507
|
-
feedback: rows.map(rowToFeedbackRecord)
|
|
4850
|
+
feedback: rows.map(rowToFeedbackRecord),
|
|
4851
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4508
4852
|
};
|
|
4509
4853
|
}
|
|
4854
|
+
async function queryFeedbackAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
4855
|
+
return await queryJson2(
|
|
4856
|
+
client,
|
|
4857
|
+
`
|
|
4858
|
+
SELECT
|
|
4859
|
+
f.* EXCEPT(traceId, timestamp, feedbackId),
|
|
4860
|
+
f.traceId AS traceId,
|
|
4861
|
+
f.timestamp AS timestamp,
|
|
4862
|
+
f.feedbackId AS feedbackId,
|
|
4863
|
+
toString(d.cursorId) AS cursorId
|
|
4864
|
+
FROM ${TABLE_FEEDBACK_EVENTS_DELTA} d
|
|
4865
|
+
INNER JOIN ${TABLE_FEEDBACK_EVENTS} f
|
|
4866
|
+
ON ((f.traceId = d.traceId) OR (f.traceId IS NULL AND d.traceId IS NULL))
|
|
4867
|
+
AND f.timestamp = d.timestamp
|
|
4868
|
+
AND f.feedbackId = d.feedbackId
|
|
4869
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
4870
|
+
ORDER BY d.cursorId ASC
|
|
4871
|
+
LIMIT {fetchLimit:UInt32}
|
|
4872
|
+
`,
|
|
4873
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
4874
|
+
);
|
|
4875
|
+
}
|
|
4876
|
+
async function getDeltaCursor(client, whereClause, params) {
|
|
4877
|
+
const rows = await queryJson2(
|
|
4878
|
+
client,
|
|
4879
|
+
`
|
|
4880
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
4881
|
+
FROM ${TABLE_FEEDBACK_EVENTS_DELTA} d
|
|
4882
|
+
INNER JOIN ${TABLE_FEEDBACK_EVENTS} f
|
|
4883
|
+
ON ((f.traceId = d.traceId) OR (f.traceId IS NULL AND d.traceId IS NULL))
|
|
4884
|
+
AND f.timestamp = d.timestamp
|
|
4885
|
+
AND f.feedbackId = d.feedbackId
|
|
4886
|
+
${whereClause}
|
|
4887
|
+
`,
|
|
4888
|
+
params
|
|
4889
|
+
);
|
|
4890
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
4891
|
+
if (cursorId) {
|
|
4892
|
+
return cursorId;
|
|
4893
|
+
}
|
|
4894
|
+
const streamRows = await queryJson2(
|
|
4895
|
+
client,
|
|
4896
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_FEEDBACK_EVENTS_DELTA}`,
|
|
4897
|
+
{}
|
|
4898
|
+
);
|
|
4899
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
4900
|
+
}
|
|
4901
|
+
async function getStreamHeadCursor(client) {
|
|
4902
|
+
const streamRows = await queryJson2(
|
|
4903
|
+
client,
|
|
4904
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_FEEDBACK_EVENTS_DELTA}`,
|
|
4905
|
+
{}
|
|
4906
|
+
);
|
|
4907
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
4908
|
+
}
|
|
4909
|
+
function buildFeedbackCursor(row) {
|
|
4910
|
+
return row.cursorId ?? "0";
|
|
4911
|
+
}
|
|
4510
4912
|
async function getFeedbackAggregate(client, args) {
|
|
4511
4913
|
const aggSql = getAggregationSql(args.aggregation);
|
|
4512
4914
|
const identity = buildFeedbackIdentityFilter(args);
|
|
@@ -4676,12 +5078,33 @@ async function batchCreateLogs(client, args) {
|
|
|
4676
5078
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4677
5079
|
});
|
|
4678
5080
|
}
|
|
4679
|
-
async function listLogs(client, args) {
|
|
5081
|
+
async function listLogs(client, args, strategy) {
|
|
4680
5082
|
const parsed = storage.listLogsArgsSchema.parse(args);
|
|
5083
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4681
5084
|
const filter = buildLogsFilterConditions(parsed.filters, "l");
|
|
4682
5085
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4683
5086
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "l");
|
|
4684
5087
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5088
|
+
if (parsed.mode === "delta") {
|
|
5089
|
+
assertDeltaPollingSupported(strategy);
|
|
5090
|
+
const streamHeadCursor = await getStreamHeadCursor2(client);
|
|
5091
|
+
if (parsed.after === void 0) {
|
|
5092
|
+
return {
|
|
5093
|
+
logs: [],
|
|
5094
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5095
|
+
deltaCursor: streamHeadCursor
|
|
5096
|
+
};
|
|
5097
|
+
}
|
|
5098
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5099
|
+
const rows2 = await queryLogsAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5100
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5101
|
+
return {
|
|
5102
|
+
logs: visibleRows.map(rowToLogRecord),
|
|
5103
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5104
|
+
deltaCursor: visibleRows.length > 0 ? buildLogsCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5105
|
+
};
|
|
5106
|
+
}
|
|
5107
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor2(client, whereClause, filter.params) : void 0;
|
|
4685
5108
|
const countResult = await (await client.query({
|
|
4686
5109
|
query: `SELECT count() AS total FROM ${TABLE_LOG_EVENTS} AS l ${whereClause}`,
|
|
4687
5110
|
query_params: filter.params,
|
|
@@ -4712,9 +5135,71 @@ async function listLogs(client, args) {
|
|
|
4712
5135
|
perPage: pagination.perPage,
|
|
4713
5136
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4714
5137
|
},
|
|
4715
|
-
logs: rows.map(rowToLogRecord)
|
|
5138
|
+
logs: rows.map(rowToLogRecord),
|
|
5139
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4716
5140
|
};
|
|
4717
5141
|
}
|
|
5142
|
+
async function queryLogsAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
5143
|
+
return await (await client.query({
|
|
5144
|
+
query: `
|
|
5145
|
+
SELECT
|
|
5146
|
+
l.* EXCEPT(timestamp, logId),
|
|
5147
|
+
l.timestamp AS timestamp,
|
|
5148
|
+
l.logId AS logId,
|
|
5149
|
+
toString(d.cursorId) AS cursorId
|
|
5150
|
+
FROM ${TABLE_LOG_EVENTS_DELTA} d
|
|
5151
|
+
INNER JOIN ${TABLE_LOG_EVENTS} l
|
|
5152
|
+
ON l.timestamp = d.timestamp
|
|
5153
|
+
AND l.logId = d.logId
|
|
5154
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
5155
|
+
ORDER BY d.cursorId ASC
|
|
5156
|
+
LIMIT {fetchLimit:UInt32}
|
|
5157
|
+
`,
|
|
5158
|
+
query_params: {
|
|
5159
|
+
...params,
|
|
5160
|
+
afterCursor: cursorId,
|
|
5161
|
+
fetchLimit: limit + 1
|
|
5162
|
+
},
|
|
5163
|
+
format: "JSONEachRow",
|
|
5164
|
+
clickhouse_settings: CH_SETTINGS
|
|
5165
|
+
})).json();
|
|
5166
|
+
}
|
|
5167
|
+
async function getDeltaCursor2(client, whereClause, params) {
|
|
5168
|
+
const rows = await (await client.query({
|
|
5169
|
+
query: `
|
|
5170
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
5171
|
+
FROM ${TABLE_LOG_EVENTS_DELTA} d
|
|
5172
|
+
INNER JOIN ${TABLE_LOG_EVENTS} l
|
|
5173
|
+
ON l.timestamp = d.timestamp
|
|
5174
|
+
AND l.logId = d.logId
|
|
5175
|
+
${whereClause}
|
|
5176
|
+
`,
|
|
5177
|
+
query_params: params,
|
|
5178
|
+
format: "JSONEachRow",
|
|
5179
|
+
clickhouse_settings: CH_SETTINGS
|
|
5180
|
+
})).json();
|
|
5181
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
5182
|
+
if (cursorId) {
|
|
5183
|
+
return cursorId;
|
|
5184
|
+
}
|
|
5185
|
+
const streamRows = await (await client.query({
|
|
5186
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_LOG_EVENTS_DELTA}`,
|
|
5187
|
+
format: "JSONEachRow",
|
|
5188
|
+
clickhouse_settings: CH_SETTINGS
|
|
5189
|
+
})).json();
|
|
5190
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5191
|
+
}
|
|
5192
|
+
async function getStreamHeadCursor2(client) {
|
|
5193
|
+
const streamRows = await (await client.query({
|
|
5194
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_LOG_EVENTS_DELTA}`,
|
|
5195
|
+
format: "JSONEachRow",
|
|
5196
|
+
clickhouse_settings: CH_SETTINGS
|
|
5197
|
+
})).json();
|
|
5198
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5199
|
+
}
|
|
5200
|
+
function buildLogsCursor(row) {
|
|
5201
|
+
return row.cursorId ?? "0";
|
|
5202
|
+
}
|
|
4718
5203
|
var METRIC_TYPED_COLUMNS = /* @__PURE__ */ new Set([
|
|
4719
5204
|
"timestamp",
|
|
4720
5205
|
"name",
|
|
@@ -4884,12 +5369,33 @@ async function batchCreateMetrics(client, args) {
|
|
|
4884
5369
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4885
5370
|
});
|
|
4886
5371
|
}
|
|
4887
|
-
async function listMetrics(client, args) {
|
|
5372
|
+
async function listMetrics(client, args, strategy) {
|
|
4888
5373
|
const parsed = storage.listMetricsArgsSchema.parse(args);
|
|
5374
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4889
5375
|
const filter = buildMetricsFilterConditions(parsed.filters, "m");
|
|
4890
5376
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4891
5377
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "m");
|
|
4892
5378
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5379
|
+
if (parsed.mode === "delta") {
|
|
5380
|
+
assertDeltaPollingSupported(strategy);
|
|
5381
|
+
const streamHeadCursor = await getStreamHeadCursor3(client);
|
|
5382
|
+
if (parsed.after === void 0) {
|
|
5383
|
+
return {
|
|
5384
|
+
metrics: [],
|
|
5385
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5386
|
+
deltaCursor: streamHeadCursor
|
|
5387
|
+
};
|
|
5388
|
+
}
|
|
5389
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5390
|
+
const rows2 = await queryMetricsAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5391
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5392
|
+
return {
|
|
5393
|
+
metrics: visibleRows.map(rowToMetricRecord),
|
|
5394
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5395
|
+
deltaCursor: visibleRows.length > 0 ? buildMetricsCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5396
|
+
};
|
|
5397
|
+
}
|
|
5398
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor3(client, whereClause, filter.params) : void 0;
|
|
4893
5399
|
const countResult = await queryJson3(
|
|
4894
5400
|
client,
|
|
4895
5401
|
`SELECT count() AS total FROM ${TABLE_METRIC_EVENTS} AS m ${whereClause}`,
|
|
@@ -4908,9 +5414,68 @@ async function listMetrics(client, args) {
|
|
|
4908
5414
|
perPage: pagination.perPage,
|
|
4909
5415
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4910
5416
|
},
|
|
4911
|
-
metrics: rows.map(rowToMetricRecord)
|
|
5417
|
+
metrics: rows.map(rowToMetricRecord),
|
|
5418
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4912
5419
|
};
|
|
4913
5420
|
}
|
|
5421
|
+
async function queryMetricsAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
5422
|
+
return await queryJson3(
|
|
5423
|
+
client,
|
|
5424
|
+
`
|
|
5425
|
+
SELECT
|
|
5426
|
+
m.* EXCEPT(name, timestamp, metricId),
|
|
5427
|
+
m.name AS name,
|
|
5428
|
+
m.timestamp AS timestamp,
|
|
5429
|
+
m.metricId AS metricId,
|
|
5430
|
+
toString(d.cursorId) AS cursorId
|
|
5431
|
+
FROM ${TABLE_METRIC_EVENTS_DELTA} d
|
|
5432
|
+
INNER JOIN ${TABLE_METRIC_EVENTS} m
|
|
5433
|
+
ON m.name = d.name
|
|
5434
|
+
AND m.timestamp = d.timestamp
|
|
5435
|
+
AND m.metricId = d.metricId
|
|
5436
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
5437
|
+
ORDER BY d.cursorId ASC
|
|
5438
|
+
LIMIT {fetchLimit:UInt32}
|
|
5439
|
+
`,
|
|
5440
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
5441
|
+
);
|
|
5442
|
+
}
|
|
5443
|
+
async function getDeltaCursor3(client, whereClause, params) {
|
|
5444
|
+
const rows = await queryJson3(
|
|
5445
|
+
client,
|
|
5446
|
+
`
|
|
5447
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
5448
|
+
FROM ${TABLE_METRIC_EVENTS_DELTA} d
|
|
5449
|
+
INNER JOIN ${TABLE_METRIC_EVENTS} m
|
|
5450
|
+
ON m.name = d.name
|
|
5451
|
+
AND m.timestamp = d.timestamp
|
|
5452
|
+
AND m.metricId = d.metricId
|
|
5453
|
+
${whereClause}
|
|
5454
|
+
`,
|
|
5455
|
+
params
|
|
5456
|
+
);
|
|
5457
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
5458
|
+
if (cursorId) {
|
|
5459
|
+
return cursorId;
|
|
5460
|
+
}
|
|
5461
|
+
const streamRows = await queryJson3(
|
|
5462
|
+
client,
|
|
5463
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_METRIC_EVENTS_DELTA}`,
|
|
5464
|
+
{}
|
|
5465
|
+
);
|
|
5466
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5467
|
+
}
|
|
5468
|
+
async function getStreamHeadCursor3(client) {
|
|
5469
|
+
const streamRows = await queryJson3(
|
|
5470
|
+
client,
|
|
5471
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_METRIC_EVENTS_DELTA}`,
|
|
5472
|
+
{}
|
|
5473
|
+
);
|
|
5474
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5475
|
+
}
|
|
5476
|
+
function buildMetricsCursor(row) {
|
|
5477
|
+
return row.cursorId ?? "0";
|
|
5478
|
+
}
|
|
4914
5479
|
async function getMetricAggregate(client, args) {
|
|
4915
5480
|
const aggSql = getAggregationSql2(args.aggregation, "value", args.distinctColumn);
|
|
4916
5481
|
const nameFilter = buildMetricNameFilter(args.name);
|
|
@@ -5399,12 +5964,33 @@ async function batchCreateScores(client, args) {
|
|
|
5399
5964
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
5400
5965
|
});
|
|
5401
5966
|
}
|
|
5402
|
-
async function listScores(client, args) {
|
|
5967
|
+
async function listScores(client, args, strategy) {
|
|
5403
5968
|
const parsed = storage.listScoresArgsSchema.parse(args);
|
|
5969
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
5404
5970
|
const filter = buildScoresFilterConditions(parsed.filters, "s");
|
|
5405
5971
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
5406
5972
|
const orderBy = buildSignalOrderByClause(["timestamp", "score"], parsed.orderBy, "s");
|
|
5407
5973
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5974
|
+
if (parsed.mode === "delta") {
|
|
5975
|
+
assertDeltaPollingSupported(strategy);
|
|
5976
|
+
const streamHeadCursor = await getStreamHeadCursor4(client);
|
|
5977
|
+
if (parsed.after === void 0) {
|
|
5978
|
+
return {
|
|
5979
|
+
scores: [],
|
|
5980
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5981
|
+
deltaCursor: streamHeadCursor
|
|
5982
|
+
};
|
|
5983
|
+
}
|
|
5984
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5985
|
+
const rows2 = await queryScoresAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5986
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5987
|
+
return {
|
|
5988
|
+
scores: visibleRows.map(rowToScoreRecord),
|
|
5989
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5990
|
+
deltaCursor: visibleRows.length > 0 ? buildScoresCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5991
|
+
};
|
|
5992
|
+
}
|
|
5993
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor4(client, whereClause, filter.params) : void 0;
|
|
5408
5994
|
const countResult = await queryJson4(
|
|
5409
5995
|
client,
|
|
5410
5996
|
`SELECT count() AS total FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause}`,
|
|
@@ -5423,9 +6009,68 @@ async function listScores(client, args) {
|
|
|
5423
6009
|
perPage: pagination.perPage,
|
|
5424
6010
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
5425
6011
|
},
|
|
5426
|
-
scores: rows.map(rowToScoreRecord)
|
|
6012
|
+
scores: rows.map(rowToScoreRecord),
|
|
6013
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
5427
6014
|
};
|
|
5428
6015
|
}
|
|
6016
|
+
async function queryScoresAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6017
|
+
return await queryJson4(
|
|
6018
|
+
client,
|
|
6019
|
+
`
|
|
6020
|
+
SELECT
|
|
6021
|
+
s.* EXCEPT(traceId, timestamp, scoreId),
|
|
6022
|
+
s.traceId AS traceId,
|
|
6023
|
+
s.timestamp AS timestamp,
|
|
6024
|
+
s.scoreId AS scoreId,
|
|
6025
|
+
toString(d.cursorId) AS cursorId
|
|
6026
|
+
FROM ${TABLE_SCORE_EVENTS_DELTA} d
|
|
6027
|
+
INNER JOIN ${TABLE_SCORE_EVENTS} s
|
|
6028
|
+
ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
|
|
6029
|
+
AND s.timestamp = d.timestamp
|
|
6030
|
+
AND s.scoreId = d.scoreId
|
|
6031
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6032
|
+
ORDER BY d.cursorId ASC
|
|
6033
|
+
LIMIT {fetchLimit:UInt32}
|
|
6034
|
+
`,
|
|
6035
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
6036
|
+
);
|
|
6037
|
+
}
|
|
6038
|
+
async function getDeltaCursor4(client, whereClause, params) {
|
|
6039
|
+
const rows = await queryJson4(
|
|
6040
|
+
client,
|
|
6041
|
+
`
|
|
6042
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6043
|
+
FROM ${TABLE_SCORE_EVENTS_DELTA} d
|
|
6044
|
+
INNER JOIN ${TABLE_SCORE_EVENTS} s
|
|
6045
|
+
ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
|
|
6046
|
+
AND s.timestamp = d.timestamp
|
|
6047
|
+
AND s.scoreId = d.scoreId
|
|
6048
|
+
${whereClause}
|
|
6049
|
+
`,
|
|
6050
|
+
params
|
|
6051
|
+
);
|
|
6052
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6053
|
+
if (cursorId) {
|
|
6054
|
+
return cursorId;
|
|
6055
|
+
}
|
|
6056
|
+
const streamRows = await queryJson4(
|
|
6057
|
+
client,
|
|
6058
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_SCORE_EVENTS_DELTA}`,
|
|
6059
|
+
{}
|
|
6060
|
+
);
|
|
6061
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6062
|
+
}
|
|
6063
|
+
async function getStreamHeadCursor4(client) {
|
|
6064
|
+
const streamRows = await queryJson4(
|
|
6065
|
+
client,
|
|
6066
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_SCORE_EVENTS_DELTA}`,
|
|
6067
|
+
{}
|
|
6068
|
+
);
|
|
6069
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6070
|
+
}
|
|
6071
|
+
function buildScoresCursor(row) {
|
|
6072
|
+
return row.cursorId ?? "0";
|
|
6073
|
+
}
|
|
5429
6074
|
async function getScoreById(client, scoreId) {
|
|
5430
6075
|
const rows = await queryJson4(
|
|
5431
6076
|
client,
|
|
@@ -5688,8 +6333,107 @@ async function listTraceRows(client, args, selectClause, mapRows) {
|
|
|
5688
6333
|
spans: mapRows(rows)
|
|
5689
6334
|
};
|
|
5690
6335
|
}
|
|
5691
|
-
async function listTraces(client, args) {
|
|
5692
|
-
|
|
6336
|
+
async function listTraces(client, args, strategy) {
|
|
6337
|
+
const { mode, filters, pagination, orderBy, after, limit } = storage.listTracesArgsSchema.parse(args);
|
|
6338
|
+
const page = pagination?.page ?? 0;
|
|
6339
|
+
const perPage = pagination?.perPage ?? 10;
|
|
6340
|
+
const { conditions, params } = buildTraceFilterConditions(filters, "r");
|
|
6341
|
+
if (filters?.hasChildError != null) {
|
|
6342
|
+
if (filters.hasChildError) {
|
|
6343
|
+
conditions.push(`EXISTS (
|
|
6344
|
+
SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
|
|
6345
|
+
WHERE c.traceId = r.traceId
|
|
6346
|
+
AND c.parentSpanId IS NOT NULL
|
|
6347
|
+
AND c.error IS NOT NULL
|
|
6348
|
+
)`);
|
|
6349
|
+
} else {
|
|
6350
|
+
conditions.push(`NOT EXISTS (
|
|
6351
|
+
SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
|
|
6352
|
+
WHERE c.traceId = r.traceId
|
|
6353
|
+
AND c.parentSpanId IS NOT NULL
|
|
6354
|
+
AND c.error IS NOT NULL
|
|
6355
|
+
)`);
|
|
6356
|
+
}
|
|
6357
|
+
}
|
|
6358
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6359
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
6360
|
+
if (mode === "delta") {
|
|
6361
|
+
assertDeltaPollingSupported(strategy);
|
|
6362
|
+
const streamHeadCursor = await getStreamHeadCursor5(client);
|
|
6363
|
+
if (after === void 0) {
|
|
6364
|
+
return {
|
|
6365
|
+
spans: [],
|
|
6366
|
+
delta: { limit, hasMore: false },
|
|
6367
|
+
deltaCursor: streamHeadCursor
|
|
6368
|
+
};
|
|
6369
|
+
}
|
|
6370
|
+
const afterCursor = validateCursorId(after);
|
|
6371
|
+
const rows2 = await queryTracesAfterCursor(client, whereClause, params, limit, afterCursor);
|
|
6372
|
+
const visibleRows = rows2.slice(0, limit);
|
|
6373
|
+
return {
|
|
6374
|
+
spans: storage.toTraceSpans(visibleRows.map(rowToSpanRecord)),
|
|
6375
|
+
delta: { limit, hasMore: rows2.length > limit },
|
|
6376
|
+
deltaCursor: visibleRows.length > 0 ? buildTraceCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
6377
|
+
};
|
|
6378
|
+
}
|
|
6379
|
+
const orderClause = buildTraceOrderByClause(orderBy);
|
|
6380
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor5(client, whereClause, params) : void 0;
|
|
6381
|
+
const countResult = await client.query({
|
|
6382
|
+
query: `
|
|
6383
|
+
SELECT count() as cnt FROM (
|
|
6384
|
+
SELECT dedupeKey
|
|
6385
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
6386
|
+
${whereClause}
|
|
6387
|
+
ORDER BY dedupeKey
|
|
6388
|
+
LIMIT 1 BY dedupeKey
|
|
6389
|
+
)
|
|
6390
|
+
`,
|
|
6391
|
+
query_params: params,
|
|
6392
|
+
format: "JSONEachRow",
|
|
6393
|
+
clickhouse_settings: CH_SETTINGS
|
|
6394
|
+
});
|
|
6395
|
+
const countRows = await countResult.json();
|
|
6396
|
+
const total = Number(countRows[0]?.cnt ?? 0);
|
|
6397
|
+
if (total === 0) {
|
|
6398
|
+
return {
|
|
6399
|
+
pagination: { total: 0, page, perPage, hasMore: false },
|
|
6400
|
+
spans: [],
|
|
6401
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6402
|
+
};
|
|
6403
|
+
}
|
|
6404
|
+
const dataResult = await client.query({
|
|
6405
|
+
query: `
|
|
6406
|
+
SELECT * FROM (
|
|
6407
|
+
SELECT *
|
|
6408
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
6409
|
+
${whereClause}
|
|
6410
|
+
ORDER BY dedupeKey
|
|
6411
|
+
LIMIT 1 BY dedupeKey
|
|
6412
|
+
)
|
|
6413
|
+
ORDER BY ${orderClause}
|
|
6414
|
+
LIMIT {limit:UInt32}
|
|
6415
|
+
OFFSET {offset:UInt32}
|
|
6416
|
+
`,
|
|
6417
|
+
query_params: {
|
|
6418
|
+
...params,
|
|
6419
|
+
limit: perPage,
|
|
6420
|
+
offset: page * perPage
|
|
6421
|
+
},
|
|
6422
|
+
format: "JSONEachRow",
|
|
6423
|
+
clickhouse_settings: CH_SETTINGS
|
|
6424
|
+
});
|
|
6425
|
+
const rows = await dataResult.json();
|
|
6426
|
+
const spans = rows.map(rowToSpanRecord);
|
|
6427
|
+
return {
|
|
6428
|
+
pagination: {
|
|
6429
|
+
total,
|
|
6430
|
+
page,
|
|
6431
|
+
perPage,
|
|
6432
|
+
hasMore: (page + 1) * perPage < total
|
|
6433
|
+
},
|
|
6434
|
+
spans: storage.toTraceSpans(spans),
|
|
6435
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6436
|
+
};
|
|
5693
6437
|
}
|
|
5694
6438
|
var LIGHT_TRACE_ROOT_COLUMNS = [
|
|
5695
6439
|
"traceId",
|
|
@@ -5708,6 +6452,70 @@ var LIGHT_TRACE_ROOT_COLUMNS = [
|
|
|
5708
6452
|
async function listTracesLight(client, args) {
|
|
5709
6453
|
return listTraceRows(client, args, LIGHT_TRACE_ROOT_COLUMNS, (rows) => rows.map(rowToLightSpanRecord));
|
|
5710
6454
|
}
|
|
6455
|
+
async function queryTracesAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6456
|
+
return await (await client.query({
|
|
6457
|
+
query: `
|
|
6458
|
+
SELECT
|
|
6459
|
+
r.* EXCEPT(startedAt, traceId, dedupeKey),
|
|
6460
|
+
r.startedAt AS startedAt,
|
|
6461
|
+
r.traceId AS traceId,
|
|
6462
|
+
r.dedupeKey AS dedupeKey,
|
|
6463
|
+
toString(d.cursorId) AS cursorId
|
|
6464
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA} d
|
|
6465
|
+
INNER JOIN ${TABLE_TRACE_ROOTS} r
|
|
6466
|
+
ON r.startedAt = d.startedAt
|
|
6467
|
+
AND r.traceId = d.traceId
|
|
6468
|
+
AND r.dedupeKey = d.dedupeKey
|
|
6469
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6470
|
+
ORDER BY d.cursorId ASC
|
|
6471
|
+
LIMIT {fetchLimit:UInt32}
|
|
6472
|
+
`,
|
|
6473
|
+
query_params: {
|
|
6474
|
+
...params,
|
|
6475
|
+
afterCursor: cursorId,
|
|
6476
|
+
fetchLimit: limit + 1
|
|
6477
|
+
},
|
|
6478
|
+
format: "JSONEachRow",
|
|
6479
|
+
clickhouse_settings: CH_SETTINGS
|
|
6480
|
+
})).json();
|
|
6481
|
+
}
|
|
6482
|
+
async function getDeltaCursor5(client, whereClause, params) {
|
|
6483
|
+
const rows = await (await client.query({
|
|
6484
|
+
query: `
|
|
6485
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6486
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA} d
|
|
6487
|
+
INNER JOIN ${TABLE_TRACE_ROOTS} r
|
|
6488
|
+
ON r.startedAt = d.startedAt
|
|
6489
|
+
AND r.traceId = d.traceId
|
|
6490
|
+
AND r.dedupeKey = d.dedupeKey
|
|
6491
|
+
${whereClause}
|
|
6492
|
+
`,
|
|
6493
|
+
query_params: params,
|
|
6494
|
+
format: "JSONEachRow",
|
|
6495
|
+
clickhouse_settings: CH_SETTINGS
|
|
6496
|
+
})).json();
|
|
6497
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6498
|
+
if (cursorId) {
|
|
6499
|
+
return cursorId;
|
|
6500
|
+
}
|
|
6501
|
+
const streamRows = await (await client.query({
|
|
6502
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_ROOTS_DELTA}`,
|
|
6503
|
+
format: "JSONEachRow",
|
|
6504
|
+
clickhouse_settings: CH_SETTINGS
|
|
6505
|
+
})).json();
|
|
6506
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6507
|
+
}
|
|
6508
|
+
async function getStreamHeadCursor5(client) {
|
|
6509
|
+
const streamRows = await (await client.query({
|
|
6510
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_ROOTS_DELTA}`,
|
|
6511
|
+
format: "JSONEachRow",
|
|
6512
|
+
clickhouse_settings: CH_SETTINGS
|
|
6513
|
+
})).json();
|
|
6514
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6515
|
+
}
|
|
6516
|
+
function buildTraceCursor(row) {
|
|
6517
|
+
return row.cursorId ?? "0";
|
|
6518
|
+
}
|
|
5711
6519
|
var BRANCH_SPAN_TYPE_SQL_LIST = storage.BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
|
|
5712
6520
|
async function createSpan(client, args) {
|
|
5713
6521
|
const row = spanRecordToRow(args.span);
|
|
@@ -5839,36 +6647,36 @@ async function batchDeleteTraces(client, args) {
|
|
|
5839
6647
|
})
|
|
5840
6648
|
]);
|
|
5841
6649
|
}
|
|
5842
|
-
async function listBranches(client, args) {
|
|
5843
|
-
const { filters, pagination, orderBy } = storage.listBranchesArgsSchema.parse(args);
|
|
6650
|
+
async function listBranches(client, args, strategy) {
|
|
6651
|
+
const { mode, filters, pagination, orderBy, after, limit } = storage.listBranchesArgsSchema.parse(args);
|
|
5844
6652
|
const page = pagination?.page ?? 0;
|
|
5845
6653
|
const perPage = pagination?.perPage ?? 10;
|
|
5846
6654
|
const conditions = [];
|
|
5847
6655
|
const params = {};
|
|
5848
6656
|
if (filters?.spanType) {
|
|
5849
|
-
conditions.push(`spanType = {spanType:String}`);
|
|
6657
|
+
conditions.push(`b.spanType = {spanType:String}`);
|
|
5850
6658
|
params.spanType = filters.spanType;
|
|
5851
6659
|
} else {
|
|
5852
|
-
conditions.push(`spanType IN (${BRANCH_SPAN_TYPE_SQL_LIST})`);
|
|
6660
|
+
conditions.push(`b.spanType IN (${BRANCH_SPAN_TYPE_SQL_LIST})`);
|
|
5853
6661
|
}
|
|
5854
6662
|
if (filters?.startedAt?.start) {
|
|
5855
6663
|
const op = filters.startedAt.startExclusive ? ">" : ">=";
|
|
5856
|
-
conditions.push(`startedAt ${op} {startedAtStart:DateTime64(3)}`);
|
|
6664
|
+
conditions.push(`b.startedAt ${op} {startedAtStart:DateTime64(3)}`);
|
|
5857
6665
|
params.startedAtStart = filters.startedAt.start.getTime();
|
|
5858
6666
|
}
|
|
5859
6667
|
if (filters?.startedAt?.end) {
|
|
5860
6668
|
const op = filters.startedAt.endExclusive ? "<" : "<=";
|
|
5861
|
-
conditions.push(`startedAt ${op} {startedAtEnd:DateTime64(3)}`);
|
|
6669
|
+
conditions.push(`b.startedAt ${op} {startedAtEnd:DateTime64(3)}`);
|
|
5862
6670
|
params.startedAtEnd = filters.startedAt.end.getTime();
|
|
5863
6671
|
}
|
|
5864
6672
|
if (filters?.endedAt?.start) {
|
|
5865
6673
|
const op = filters.endedAt.startExclusive ? ">" : ">=";
|
|
5866
|
-
conditions.push(`endedAt ${op} {endedAtStart:DateTime64(3)}`);
|
|
6674
|
+
conditions.push(`b.endedAt ${op} {endedAtStart:DateTime64(3)}`);
|
|
5867
6675
|
params.endedAtStart = filters.endedAt.start.getTime();
|
|
5868
6676
|
}
|
|
5869
6677
|
if (filters?.endedAt?.end) {
|
|
5870
6678
|
const op = filters.endedAt.endExclusive ? "<" : "<=";
|
|
5871
|
-
conditions.push(`endedAt ${op} {endedAtEnd:DateTime64(3)}`);
|
|
6679
|
+
conditions.push(`b.endedAt ${op} {endedAtEnd:DateTime64(3)}`);
|
|
5872
6680
|
params.endedAtEnd = filters.endedAt.end.getTime();
|
|
5873
6681
|
}
|
|
5874
6682
|
const eq = [
|
|
@@ -5899,7 +6707,7 @@ async function listBranches(client, args) {
|
|
|
5899
6707
|
];
|
|
5900
6708
|
for (const { col, value, param } of eq) {
|
|
5901
6709
|
if (value == null) continue;
|
|
5902
|
-
conditions.push(
|
|
6710
|
+
conditions.push(`b.${col} = {${param}:String}`);
|
|
5903
6711
|
params[param] = value;
|
|
5904
6712
|
}
|
|
5905
6713
|
if (filters?.tags && filters.tags.length > 0) {
|
|
@@ -5907,7 +6715,7 @@ async function listBranches(client, args) {
|
|
|
5907
6715
|
const tag = filters.tags[i];
|
|
5908
6716
|
if (typeof tag !== "string" || tag.trim() === "") continue;
|
|
5909
6717
|
const param = `tag_${i}`;
|
|
5910
|
-
conditions.push(`has(tags, {${param}:String})`);
|
|
6718
|
+
conditions.push(`has(b.tags, {${param}:String})`);
|
|
5911
6719
|
params[param] = tag;
|
|
5912
6720
|
}
|
|
5913
6721
|
}
|
|
@@ -5917,7 +6725,7 @@ async function listBranches(client, args) {
|
|
|
5917
6725
|
if (typeof value !== "string") continue;
|
|
5918
6726
|
const keyParam = `meta_k_${i}`;
|
|
5919
6727
|
const valParam = `meta_v_${i}`;
|
|
5920
|
-
conditions.push(`metadataSearch[{${keyParam}:String}] = {${valParam}:String}`);
|
|
6728
|
+
conditions.push(`b.metadataSearch[{${keyParam}:String}] = {${valParam}:String}`);
|
|
5921
6729
|
params[keyParam] = key;
|
|
5922
6730
|
params[valParam] = value;
|
|
5923
6731
|
i++;
|
|
@@ -5931,30 +6739,51 @@ async function listBranches(client, args) {
|
|
|
5931
6739
|
if (normalized == null) continue;
|
|
5932
6740
|
const keyParam = `scope_k_${i}`;
|
|
5933
6741
|
const valParam = `scope_v_${i}`;
|
|
5934
|
-
conditions.push(`JSONExtractString(scope, {${keyParam}:String}) = {${valParam}:String}`);
|
|
6742
|
+
conditions.push(`JSONExtractString(b.scope, {${keyParam}:String}) = {${valParam}:String}`);
|
|
5935
6743
|
params[keyParam] = key;
|
|
5936
6744
|
params[valParam] = normalized;
|
|
5937
6745
|
i++;
|
|
5938
6746
|
}
|
|
5939
6747
|
}
|
|
5940
6748
|
if (filters?.status === storage.TraceStatus.ERROR) {
|
|
5941
|
-
conditions.push(`error IS NOT NULL`);
|
|
6749
|
+
conditions.push(`b.error IS NOT NULL`);
|
|
5942
6750
|
} else if (filters?.status === storage.TraceStatus.SUCCESS) {
|
|
5943
|
-
conditions.push(`error IS NULL`);
|
|
6751
|
+
conditions.push(`b.error IS NULL`);
|
|
5944
6752
|
} else if (filters?.status === storage.TraceStatus.RUNNING) {
|
|
5945
6753
|
conditions.push("1 = 0");
|
|
5946
6754
|
}
|
|
5947
6755
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6756
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
6757
|
+
if (mode === "delta") {
|
|
6758
|
+
assertDeltaPollingSupported(strategy);
|
|
6759
|
+
const streamHeadCursor = await getStreamHeadCursor6(client);
|
|
6760
|
+
if (after === void 0) {
|
|
6761
|
+
return {
|
|
6762
|
+
branches: [],
|
|
6763
|
+
delta: { limit, hasMore: false },
|
|
6764
|
+
deltaCursor: streamHeadCursor
|
|
6765
|
+
};
|
|
6766
|
+
}
|
|
6767
|
+
const afterCursor = validateCursorId(after);
|
|
6768
|
+
const rows2 = await queryBranchesAfterCursor(client, whereClause, params, limit, afterCursor);
|
|
6769
|
+
const visibleRows = rows2.slice(0, limit);
|
|
6770
|
+
return {
|
|
6771
|
+
branches: storage.toTraceSpans(visibleRows.map(rowToSpanRecord)),
|
|
6772
|
+
delta: { limit, hasMore: rows2.length > limit },
|
|
6773
|
+
deltaCursor: visibleRows.length > 0 ? buildBranchCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
6774
|
+
};
|
|
6775
|
+
}
|
|
5948
6776
|
const sortField = orderBy?.field === "endedAt" ? "endedAt" : "startedAt";
|
|
5949
6777
|
const sortDirection = orderBy?.direction === "ASC" ? "ASC" : "DESC";
|
|
6778
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor6(client, whereClause, params) : void 0;
|
|
5950
6779
|
const countResult = await client.query({
|
|
5951
6780
|
query: `
|
|
5952
6781
|
SELECT count() as cnt FROM (
|
|
5953
6782
|
SELECT dedupeKey
|
|
5954
|
-
FROM ${TABLE_TRACE_BRANCHES}
|
|
6783
|
+
FROM ${TABLE_TRACE_BRANCHES} b
|
|
5955
6784
|
${whereClause}
|
|
5956
|
-
ORDER BY dedupeKey
|
|
5957
|
-
LIMIT 1 BY dedupeKey
|
|
6785
|
+
ORDER BY b.dedupeKey
|
|
6786
|
+
LIMIT 1 BY b.dedupeKey
|
|
5958
6787
|
)
|
|
5959
6788
|
`,
|
|
5960
6789
|
query_params: params,
|
|
@@ -5966,17 +6795,18 @@ async function listBranches(client, args) {
|
|
|
5966
6795
|
if (total === 0) {
|
|
5967
6796
|
return {
|
|
5968
6797
|
pagination: { total: 0, page, perPage, hasMore: false },
|
|
5969
|
-
branches: []
|
|
6798
|
+
branches: [],
|
|
6799
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
5970
6800
|
};
|
|
5971
6801
|
}
|
|
5972
6802
|
const dataResult = await client.query({
|
|
5973
6803
|
query: `
|
|
5974
6804
|
SELECT * FROM (
|
|
5975
6805
|
SELECT *
|
|
5976
|
-
FROM ${TABLE_TRACE_BRANCHES}
|
|
6806
|
+
FROM ${TABLE_TRACE_BRANCHES} b
|
|
5977
6807
|
${whereClause}
|
|
5978
|
-
ORDER BY dedupeKey
|
|
5979
|
-
LIMIT 1 BY dedupeKey
|
|
6808
|
+
ORDER BY b.dedupeKey
|
|
6809
|
+
LIMIT 1 BY b.dedupeKey
|
|
5980
6810
|
)
|
|
5981
6811
|
ORDER BY ${sortField} ${sortDirection}, dedupeKey ASC
|
|
5982
6812
|
LIMIT {limit:UInt32}
|
|
@@ -5999,9 +6829,80 @@ async function listBranches(client, args) {
|
|
|
5999
6829
|
perPage,
|
|
6000
6830
|
hasMore: (page + 1) * perPage < total
|
|
6001
6831
|
},
|
|
6002
|
-
branches: storage.toTraceSpans(spans)
|
|
6832
|
+
branches: storage.toTraceSpans(spans),
|
|
6833
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6003
6834
|
};
|
|
6004
6835
|
}
|
|
6836
|
+
async function queryBranchesAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6837
|
+
return await (await client.query({
|
|
6838
|
+
query: `
|
|
6839
|
+
SELECT
|
|
6840
|
+
b.* EXCEPT(spanType, startedAt, traceId, spanId, dedupeKey),
|
|
6841
|
+
b.spanType AS spanType,
|
|
6842
|
+
b.startedAt AS startedAt,
|
|
6843
|
+
b.traceId AS traceId,
|
|
6844
|
+
b.spanId AS spanId,
|
|
6845
|
+
b.dedupeKey AS dedupeKey,
|
|
6846
|
+
toString(d.cursorId) AS cursorId
|
|
6847
|
+
FROM ${TABLE_TRACE_BRANCHES_DELTA} d
|
|
6848
|
+
INNER JOIN ${TABLE_TRACE_BRANCHES} b
|
|
6849
|
+
ON b.spanType = d.spanType
|
|
6850
|
+
AND b.startedAt = d.startedAt
|
|
6851
|
+
AND b.traceId = d.traceId
|
|
6852
|
+
AND b.spanId = d.spanId
|
|
6853
|
+
AND b.dedupeKey = d.dedupeKey
|
|
6854
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6855
|
+
ORDER BY d.cursorId ASC
|
|
6856
|
+
LIMIT {fetchLimit:UInt32}
|
|
6857
|
+
`,
|
|
6858
|
+
query_params: {
|
|
6859
|
+
...params,
|
|
6860
|
+
afterCursor: cursorId,
|
|
6861
|
+
fetchLimit: limit + 1
|
|
6862
|
+
},
|
|
6863
|
+
format: "JSONEachRow",
|
|
6864
|
+
clickhouse_settings: CH_SETTINGS
|
|
6865
|
+
})).json();
|
|
6866
|
+
}
|
|
6867
|
+
async function getDeltaCursor6(client, whereClause, params) {
|
|
6868
|
+
const rows = await (await client.query({
|
|
6869
|
+
query: `
|
|
6870
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6871
|
+
FROM ${TABLE_TRACE_BRANCHES_DELTA} d
|
|
6872
|
+
INNER JOIN ${TABLE_TRACE_BRANCHES} b
|
|
6873
|
+
ON b.spanType = d.spanType
|
|
6874
|
+
AND b.startedAt = d.startedAt
|
|
6875
|
+
AND b.traceId = d.traceId
|
|
6876
|
+
AND b.spanId = d.spanId
|
|
6877
|
+
AND b.dedupeKey = d.dedupeKey
|
|
6878
|
+
${whereClause}
|
|
6879
|
+
`,
|
|
6880
|
+
query_params: params,
|
|
6881
|
+
format: "JSONEachRow",
|
|
6882
|
+
clickhouse_settings: CH_SETTINGS
|
|
6883
|
+
})).json();
|
|
6884
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6885
|
+
if (cursorId) {
|
|
6886
|
+
return cursorId;
|
|
6887
|
+
}
|
|
6888
|
+
const streamRows = await (await client.query({
|
|
6889
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_BRANCHES_DELTA}`,
|
|
6890
|
+
format: "JSONEachRow",
|
|
6891
|
+
clickhouse_settings: CH_SETTINGS
|
|
6892
|
+
})).json();
|
|
6893
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6894
|
+
}
|
|
6895
|
+
async function getStreamHeadCursor6(client) {
|
|
6896
|
+
const streamRows = await (await client.query({
|
|
6897
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_BRANCHES_DELTA}`,
|
|
6898
|
+
format: "JSONEachRow",
|
|
6899
|
+
clickhouse_settings: CH_SETTINGS
|
|
6900
|
+
})).json();
|
|
6901
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6902
|
+
}
|
|
6903
|
+
function buildBranchCursor(row) {
|
|
6904
|
+
return row.cursorId ?? "0";
|
|
6905
|
+
}
|
|
6005
6906
|
|
|
6006
6907
|
// src/storage/domains/observability/v-next/index.ts
|
|
6007
6908
|
function buildSignalMigrationRequiredMessage(args) {
|
|
@@ -6098,14 +6999,72 @@ async function queryNamesByTable(client, query, tables) {
|
|
|
6098
6999
|
}
|
|
6099
7000
|
return out;
|
|
6100
7001
|
}
|
|
7002
|
+
async function detectDeltaCursorStrategy(client, override, existingStrategy) {
|
|
7003
|
+
if (existingStrategy && existingStrategy !== "mixed") {
|
|
7004
|
+
return existingStrategy;
|
|
7005
|
+
}
|
|
7006
|
+
try {
|
|
7007
|
+
await client.query({
|
|
7008
|
+
query: `SELECT generateSerialID({counterName:String}) AS cursorId`,
|
|
7009
|
+
query_params: { counterName: "mastra_observability_delta_cursor_probe" },
|
|
7010
|
+
format: "JSONEachRow"
|
|
7011
|
+
});
|
|
7012
|
+
return "serial";
|
|
7013
|
+
} catch {
|
|
7014
|
+
return "fallback";
|
|
7015
|
+
}
|
|
7016
|
+
}
|
|
7017
|
+
async function detectExistingDeltaCursorStrategy(client) {
|
|
7018
|
+
try {
|
|
7019
|
+
const mvResult = await client.query({
|
|
7020
|
+
query: `
|
|
7021
|
+
SELECT name, create_table_query
|
|
7022
|
+
FROM system.tables
|
|
7023
|
+
WHERE database = currentDatabase()
|
|
7024
|
+
AND name IN ({tables:Array(String)})
|
|
7025
|
+
`,
|
|
7026
|
+
query_params: { tables: [...DELTA_MV_NAMES] },
|
|
7027
|
+
format: "JSONEachRow"
|
|
7028
|
+
});
|
|
7029
|
+
const mvRows = await mvResult.json();
|
|
7030
|
+
if (mvRows.length === 0) {
|
|
7031
|
+
return null;
|
|
7032
|
+
}
|
|
7033
|
+
let sawSerialMv = false;
|
|
7034
|
+
let sawFallbackMv = false;
|
|
7035
|
+
for (const row of mvRows) {
|
|
7036
|
+
const ddl = row.create_table_query ?? "";
|
|
7037
|
+
if (ddl.includes("generateSerialID(")) {
|
|
7038
|
+
sawSerialMv = true;
|
|
7039
|
+
} else if (ddl.includes("farmFingerprint64(")) {
|
|
7040
|
+
sawFallbackMv = true;
|
|
7041
|
+
}
|
|
7042
|
+
}
|
|
7043
|
+
if (sawSerialMv && sawFallbackMv) {
|
|
7044
|
+
return "mixed";
|
|
7045
|
+
}
|
|
7046
|
+
if (sawSerialMv) {
|
|
7047
|
+
return "serial";
|
|
7048
|
+
}
|
|
7049
|
+
if (sawFallbackMv) {
|
|
7050
|
+
return "fallback";
|
|
7051
|
+
}
|
|
7052
|
+
return null;
|
|
7053
|
+
} catch {
|
|
7054
|
+
return null;
|
|
7055
|
+
}
|
|
7056
|
+
}
|
|
6101
7057
|
var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilityStorage {
|
|
6102
7058
|
#client;
|
|
6103
7059
|
#retention;
|
|
7060
|
+
#deltaCursorStrategyOverride;
|
|
7061
|
+
#deltaCursorStrategy = "fallback";
|
|
6104
7062
|
constructor(config) {
|
|
6105
7063
|
super();
|
|
6106
7064
|
const { client } = resolveClickhouseConfig(config);
|
|
6107
7065
|
this.#client = client;
|
|
6108
7066
|
this.#retention = config.retention;
|
|
7067
|
+
this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
|
|
6109
7068
|
}
|
|
6110
7069
|
// -------------------------------------------------------------------------
|
|
6111
7070
|
// Initialization
|
|
@@ -6124,7 +7083,21 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6124
7083
|
});
|
|
6125
7084
|
}
|
|
6126
7085
|
try {
|
|
6127
|
-
|
|
7086
|
+
const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
|
|
7087
|
+
if (existingStrategy === "mixed") {
|
|
7088
|
+
this.#deltaCursorStrategy = null;
|
|
7089
|
+
this.logger.error(
|
|
7090
|
+
"ClickHouse observability delta tables use mixed cursor schemas; delta polling has been disabled for this store instance."
|
|
7091
|
+
);
|
|
7092
|
+
} else if (this.#deltaCursorStrategyOverride) {
|
|
7093
|
+
this.#deltaCursorStrategy = this.#deltaCursorStrategyOverride;
|
|
7094
|
+
} else if (existingStrategy) {
|
|
7095
|
+
this.#deltaCursorStrategy = existingStrategy;
|
|
7096
|
+
} else {
|
|
7097
|
+
this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
|
|
7098
|
+
}
|
|
7099
|
+
const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
|
|
7100
|
+
for (const ddl of coreDdl) {
|
|
6128
7101
|
await this.#client.command({ query: ddl });
|
|
6129
7102
|
}
|
|
6130
7103
|
const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
|
|
@@ -6195,6 +7168,12 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6195
7168
|
supported: ["insert-only"]
|
|
6196
7169
|
};
|
|
6197
7170
|
}
|
|
7171
|
+
getFeatures() {
|
|
7172
|
+
if (!deltaPollingSupported(this.#deltaCursorStrategy)) {
|
|
7173
|
+
return void 0;
|
|
7174
|
+
}
|
|
7175
|
+
return ["delta-polling"];
|
|
7176
|
+
}
|
|
6198
7177
|
// -------------------------------------------------------------------------
|
|
6199
7178
|
// Tracing — writes
|
|
6200
7179
|
// -------------------------------------------------------------------------
|
|
@@ -6315,7 +7294,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6315
7294
|
}
|
|
6316
7295
|
async listTraces(args) {
|
|
6317
7296
|
try {
|
|
6318
|
-
return await listTraces(this.#client, args);
|
|
7297
|
+
return await listTraces(this.#client, args, this.#deltaCursorStrategy);
|
|
6319
7298
|
} catch (error$1) {
|
|
6320
7299
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6321
7300
|
throw new error.MastraError(
|
|
@@ -6345,7 +7324,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6345
7324
|
}
|
|
6346
7325
|
async listBranches(args) {
|
|
6347
7326
|
try {
|
|
6348
|
-
return await listBranches(this.#client, args);
|
|
7327
|
+
return await listBranches(this.#client, args, this.#deltaCursorStrategy);
|
|
6349
7328
|
} catch (error$1) {
|
|
6350
7329
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6351
7330
|
throw new error.MastraError(
|
|
@@ -6376,7 +7355,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6376
7355
|
}
|
|
6377
7356
|
async listLogs(args) {
|
|
6378
7357
|
try {
|
|
6379
|
-
return await listLogs(this.#client, args);
|
|
7358
|
+
return await listLogs(this.#client, args, this.#deltaCursorStrategy);
|
|
6380
7359
|
} catch (error$1) {
|
|
6381
7360
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6382
7361
|
throw new error.MastraError(
|
|
@@ -6407,7 +7386,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6407
7386
|
}
|
|
6408
7387
|
async listMetrics(args) {
|
|
6409
7388
|
try {
|
|
6410
|
-
return await listMetrics(this.#client, args);
|
|
7389
|
+
return await listMetrics(this.#client, args, this.#deltaCursorStrategy);
|
|
6411
7390
|
} catch (error$1) {
|
|
6412
7391
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6413
7392
|
throw new error.MastraError(
|
|
@@ -6453,7 +7432,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6453
7432
|
}
|
|
6454
7433
|
async listScores(args) {
|
|
6455
7434
|
try {
|
|
6456
|
-
return await listScores(this.#client, args);
|
|
7435
|
+
return await listScores(this.#client, args, this.#deltaCursorStrategy);
|
|
6457
7436
|
} catch (error$1) {
|
|
6458
7437
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6459
7438
|
throw new error.MastraError(
|
|
@@ -6515,7 +7494,7 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
6515
7494
|
}
|
|
6516
7495
|
async listFeedback(args) {
|
|
6517
7496
|
try {
|
|
6518
|
-
return await listFeedback(this.#client, args);
|
|
7497
|
+
return await listFeedback(this.#client, args, this.#deltaCursorStrategy);
|
|
6519
7498
|
} catch (error$1) {
|
|
6520
7499
|
if (error$1 instanceof error.MastraError) throw error$1;
|
|
6521
7500
|
throw new error.MastraError(
|