@mastra/clickhouse 1.8.0 → 1.9.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +1020 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1020 -41
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +34 -3
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/feedback.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/feedback.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +4 -0
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/logs.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/logs.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/metrics.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/metrics.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/polling.d.ts +7 -0
- package/dist/storage/domains/observability/v-next/polling.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/scores.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/scores.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts +12 -1
- package/dist/storage/domains/observability/v-next/trace-roots.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/tracing.d.ts +2 -1
- package/dist/storage/domains/observability/v-next/tracing.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import { BRANCH_SPAN_TYPES, TABLE_SCHEDULE_TRIGGERS, TABLE_SCHEDULES, TABLE_FAVO
|
|
|
4
4
|
import { MastraBase } from '@mastra/core/base';
|
|
5
5
|
import { MessageList } from '@mastra/core/agent';
|
|
6
6
|
import { parseFieldKey } from '@mastra/core/utils';
|
|
7
|
+
import { coreFeatures } from '@mastra/core/features';
|
|
7
8
|
import { saveScorePayloadSchema } from '@mastra/core/evals';
|
|
8
9
|
|
|
9
10
|
// src/storage/index.ts
|
|
@@ -2858,16 +2859,36 @@ time for large tables. Please ensure you have a backup before proceeding.
|
|
|
2858
2859
|
var TABLE_SPAN_EVENTS = "mastra_span_events";
|
|
2859
2860
|
var TABLE_TRACE_ROOTS = "mastra_trace_roots";
|
|
2860
2861
|
var TABLE_TRACE_BRANCHES = "mastra_trace_branches";
|
|
2862
|
+
var TABLE_TRACE_ROOTS_DELTA = "mastra_trace_roots_delta";
|
|
2863
|
+
var TABLE_TRACE_BRANCHES_DELTA = "mastra_trace_branches_delta";
|
|
2861
2864
|
var TABLE_METRIC_EVENTS = "mastra_metric_events";
|
|
2862
2865
|
var TABLE_LOG_EVENTS = "mastra_log_events";
|
|
2863
2866
|
var TABLE_SCORE_EVENTS = "mastra_score_events";
|
|
2864
2867
|
var TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
|
|
2868
|
+
var TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
|
|
2869
|
+
var TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
|
|
2870
|
+
var TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
|
|
2871
|
+
var TABLE_FEEDBACK_EVENTS_DELTA = "mastra_feedback_events_delta";
|
|
2865
2872
|
var TABLE_DISCOVERY_VALUES = "mastra_discovery_values";
|
|
2866
2873
|
var TABLE_DISCOVERY_PAIRS = "mastra_discovery_pairs";
|
|
2867
2874
|
var MV_TRACE_ROOTS = "mastra_mv_trace_roots";
|
|
2868
2875
|
var MV_TRACE_BRANCHES = "mastra_mv_trace_branches";
|
|
2876
|
+
var MV_TRACE_ROOTS_DELTA = "mastra_mv_trace_roots_delta";
|
|
2877
|
+
var MV_TRACE_BRANCHES_DELTA = "mastra_mv_trace_branches_delta";
|
|
2878
|
+
var MV_METRIC_EVENTS_DELTA = "mastra_mv_metric_events_delta";
|
|
2879
|
+
var MV_LOG_EVENTS_DELTA = "mastra_mv_log_events_delta";
|
|
2880
|
+
var MV_SCORE_EVENTS_DELTA = "mastra_mv_score_events_delta";
|
|
2881
|
+
var MV_FEEDBACK_EVENTS_DELTA = "mastra_mv_feedback_events_delta";
|
|
2869
2882
|
var MV_DISCOVERY_VALUES = "mastra_mv_discovery_values";
|
|
2870
2883
|
var MV_DISCOVERY_PAIRS = "mastra_mv_discovery_pairs";
|
|
2884
|
+
var DELTA_MV_NAMES = [
|
|
2885
|
+
MV_TRACE_ROOTS_DELTA,
|
|
2886
|
+
MV_TRACE_BRANCHES_DELTA,
|
|
2887
|
+
MV_METRIC_EVENTS_DELTA,
|
|
2888
|
+
MV_LOG_EVENTS_DELTA,
|
|
2889
|
+
MV_SCORE_EVENTS_DELTA,
|
|
2890
|
+
MV_FEEDBACK_EVENTS_DELTA
|
|
2891
|
+
];
|
|
2871
2892
|
var BRANCH_SPAN_TYPE_VALUES = [
|
|
2872
2893
|
"agent_run",
|
|
2873
2894
|
"workflow_run",
|
|
@@ -3091,6 +3112,103 @@ SELECT *
|
|
|
3091
3112
|
FROM ${TABLE_SPAN_EVENTS}
|
|
3092
3113
|
WHERE spanType IN (${BRANCH_SPAN_TYPE_VALUES.map((v) => `'${v}'`).join(", ")})
|
|
3093
3114
|
`;
|
|
3115
|
+
var DELTA_INGESTED_AT_TYPE = `DateTime64(9, 'UTC')`;
|
|
3116
|
+
var DELTA_CURSOR_EPOCH_MS = 17778528e5;
|
|
3117
|
+
var DELTA_CURSOR_SUFFIX_BITS = 26;
|
|
3118
|
+
var DELTA_CURSOR_SUFFIX_MASK = 67108863;
|
|
3119
|
+
function buildFallbackCursorExpr(stableKeyExpr) {
|
|
3120
|
+
return `bitOr(
|
|
3121
|
+
bitShiftLeft(
|
|
3122
|
+
toUInt64(toUnixTimestamp64Milli(ingestedAt) - ${DELTA_CURSOR_EPOCH_MS}),
|
|
3123
|
+
${DELTA_CURSOR_SUFFIX_BITS}
|
|
3124
|
+
),
|
|
3125
|
+
bitAnd(farmFingerprint64(toString(${stableKeyExpr})), toUInt64(${DELTA_CURSOR_SUFFIX_MASK}))
|
|
3126
|
+
)`;
|
|
3127
|
+
}
|
|
3128
|
+
function buildDeltaCursorExpr(strategy, counterName, stableKeyExpr) {
|
|
3129
|
+
if (strategy === "serial") {
|
|
3130
|
+
return `generateSerialID('${counterName}')`;
|
|
3131
|
+
}
|
|
3132
|
+
return buildFallbackCursorExpr(stableKeyExpr);
|
|
3133
|
+
}
|
|
3134
|
+
function buildTraceRootsDeltaDDL() {
|
|
3135
|
+
return `
|
|
3136
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_ROOTS_DELTA} (
|
|
3137
|
+
cursorId UInt64,
|
|
3138
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3139
|
+
startedAt DateTime64(3, 'UTC'),
|
|
3140
|
+
traceId String,
|
|
3141
|
+
dedupeKey String
|
|
3142
|
+
)
|
|
3143
|
+
ENGINE = MergeTree
|
|
3144
|
+
PARTITION BY toDate(ingestedAt)
|
|
3145
|
+
ORDER BY (cursorId)
|
|
3146
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3147
|
+
`;
|
|
3148
|
+
}
|
|
3149
|
+
function buildTraceRootsDeltaMvDDL(strategy) {
|
|
3150
|
+
return `
|
|
3151
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_TRACE_ROOTS_DELTA}
|
|
3152
|
+
TO ${TABLE_TRACE_ROOTS_DELTA}
|
|
3153
|
+
AS
|
|
3154
|
+
SELECT
|
|
3155
|
+
${buildDeltaCursorExpr(strategy, "mastra_trace_roots_delta_cursor", "dedupeKey")} AS cursorId,
|
|
3156
|
+
ingestedAt,
|
|
3157
|
+
startedAt,
|
|
3158
|
+
traceId,
|
|
3159
|
+
dedupeKey
|
|
3160
|
+
FROM (
|
|
3161
|
+
SELECT
|
|
3162
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3163
|
+
startedAt,
|
|
3164
|
+
traceId,
|
|
3165
|
+
dedupeKey
|
|
3166
|
+
FROM ${TABLE_TRACE_ROOTS}
|
|
3167
|
+
)
|
|
3168
|
+
`;
|
|
3169
|
+
}
|
|
3170
|
+
function buildTraceBranchesDeltaDDL() {
|
|
3171
|
+
return `
|
|
3172
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_TRACE_BRANCHES_DELTA} (
|
|
3173
|
+
cursorId UInt64,
|
|
3174
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3175
|
+
spanType LowCardinality(String),
|
|
3176
|
+
startedAt DateTime64(3, 'UTC'),
|
|
3177
|
+
traceId String,
|
|
3178
|
+
spanId String,
|
|
3179
|
+
dedupeKey String
|
|
3180
|
+
)
|
|
3181
|
+
ENGINE = MergeTree
|
|
3182
|
+
PARTITION BY toDate(ingestedAt)
|
|
3183
|
+
ORDER BY (cursorId)
|
|
3184
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3185
|
+
`;
|
|
3186
|
+
}
|
|
3187
|
+
function buildTraceBranchesDeltaMvDDL(strategy) {
|
|
3188
|
+
return `
|
|
3189
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_TRACE_BRANCHES_DELTA}
|
|
3190
|
+
TO ${TABLE_TRACE_BRANCHES_DELTA}
|
|
3191
|
+
AS
|
|
3192
|
+
SELECT
|
|
3193
|
+
${buildDeltaCursorExpr(strategy, "mastra_trace_branches_delta_cursor", "dedupeKey")} AS cursorId,
|
|
3194
|
+
ingestedAt,
|
|
3195
|
+
spanType,
|
|
3196
|
+
startedAt,
|
|
3197
|
+
traceId,
|
|
3198
|
+
spanId,
|
|
3199
|
+
dedupeKey
|
|
3200
|
+
FROM (
|
|
3201
|
+
SELECT
|
|
3202
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3203
|
+
spanType,
|
|
3204
|
+
startedAt,
|
|
3205
|
+
traceId,
|
|
3206
|
+
spanId,
|
|
3207
|
+
dedupeKey
|
|
3208
|
+
FROM ${TABLE_TRACE_BRANCHES}
|
|
3209
|
+
)
|
|
3210
|
+
`;
|
|
3211
|
+
}
|
|
3094
3212
|
var METRIC_EVENTS_DDL = `
|
|
3095
3213
|
CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS} (
|
|
3096
3214
|
-- Timestamp
|
|
@@ -3217,6 +3335,39 @@ ENGINE = ReplacingMergeTree
|
|
|
3217
3335
|
PARTITION BY toDate(timestamp)
|
|
3218
3336
|
ORDER BY (timestamp, logId)
|
|
3219
3337
|
`;
|
|
3338
|
+
function buildLogEventsDeltaDDL() {
|
|
3339
|
+
return `
|
|
3340
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_LOG_EVENTS_DELTA} (
|
|
3341
|
+
cursorId UInt64,
|
|
3342
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3343
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3344
|
+
logId String
|
|
3345
|
+
)
|
|
3346
|
+
ENGINE = MergeTree
|
|
3347
|
+
PARTITION BY toDate(ingestedAt)
|
|
3348
|
+
ORDER BY (cursorId)
|
|
3349
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3350
|
+
`;
|
|
3351
|
+
}
|
|
3352
|
+
function buildLogEventsDeltaMvDDL(strategy) {
|
|
3353
|
+
return `
|
|
3354
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_LOG_EVENTS_DELTA}
|
|
3355
|
+
TO ${TABLE_LOG_EVENTS_DELTA}
|
|
3356
|
+
AS
|
|
3357
|
+
SELECT
|
|
3358
|
+
${buildDeltaCursorExpr(strategy, "mastra_log_events_delta_cursor", "logId")} AS cursorId,
|
|
3359
|
+
ingestedAt,
|
|
3360
|
+
timestamp,
|
|
3361
|
+
logId
|
|
3362
|
+
FROM (
|
|
3363
|
+
SELECT
|
|
3364
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3365
|
+
timestamp,
|
|
3366
|
+
logId
|
|
3367
|
+
FROM ${TABLE_LOG_EVENTS}
|
|
3368
|
+
)
|
|
3369
|
+
`;
|
|
3370
|
+
}
|
|
3220
3371
|
var SCORE_EVENTS_DDL = `
|
|
3221
3372
|
CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
|
|
3222
3373
|
-- Timestamp
|
|
@@ -3278,6 +3429,43 @@ PARTITION BY toDate(timestamp)
|
|
|
3278
3429
|
ORDER BY (traceId, timestamp, scoreId)
|
|
3279
3430
|
SETTINGS allow_nullable_key = 1
|
|
3280
3431
|
`;
|
|
3432
|
+
function buildScoreEventsDeltaDDL() {
|
|
3433
|
+
return `
|
|
3434
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS_DELTA} (
|
|
3435
|
+
cursorId UInt64,
|
|
3436
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3437
|
+
traceId Nullable(String),
|
|
3438
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3439
|
+
scoreId String
|
|
3440
|
+
)
|
|
3441
|
+
ENGINE = MergeTree
|
|
3442
|
+
PARTITION BY toDate(ingestedAt)
|
|
3443
|
+
ORDER BY (cursorId)
|
|
3444
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3445
|
+
SETTINGS allow_nullable_key = 1
|
|
3446
|
+
`;
|
|
3447
|
+
}
|
|
3448
|
+
function buildScoreEventsDeltaMvDDL(strategy) {
|
|
3449
|
+
return `
|
|
3450
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_SCORE_EVENTS_DELTA}
|
|
3451
|
+
TO ${TABLE_SCORE_EVENTS_DELTA}
|
|
3452
|
+
AS
|
|
3453
|
+
SELECT
|
|
3454
|
+
${buildDeltaCursorExpr(strategy, "mastra_score_events_delta_cursor", "scoreId")} AS cursorId,
|
|
3455
|
+
ingestedAt,
|
|
3456
|
+
traceId,
|
|
3457
|
+
timestamp,
|
|
3458
|
+
scoreId
|
|
3459
|
+
FROM (
|
|
3460
|
+
SELECT
|
|
3461
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3462
|
+
traceId,
|
|
3463
|
+
timestamp,
|
|
3464
|
+
scoreId
|
|
3465
|
+
FROM ${TABLE_SCORE_EVENTS}
|
|
3466
|
+
)
|
|
3467
|
+
`;
|
|
3468
|
+
}
|
|
3281
3469
|
var FEEDBACK_EVENTS_DDL = `
|
|
3282
3470
|
CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS} (
|
|
3283
3471
|
-- Timestamp
|
|
@@ -3342,6 +3530,79 @@ PARTITION BY toDate(timestamp)
|
|
|
3342
3530
|
ORDER BY (traceId, timestamp, feedbackId)
|
|
3343
3531
|
SETTINGS allow_nullable_key = 1
|
|
3344
3532
|
`;
|
|
3533
|
+
function buildFeedbackEventsDeltaDDL() {
|
|
3534
|
+
return `
|
|
3535
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
|
|
3536
|
+
cursorId UInt64,
|
|
3537
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3538
|
+
traceId Nullable(String),
|
|
3539
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3540
|
+
feedbackId String
|
|
3541
|
+
)
|
|
3542
|
+
ENGINE = MergeTree
|
|
3543
|
+
PARTITION BY toDate(ingestedAt)
|
|
3544
|
+
ORDER BY (cursorId)
|
|
3545
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3546
|
+
SETTINGS allow_nullable_key = 1
|
|
3547
|
+
`;
|
|
3548
|
+
}
|
|
3549
|
+
function buildFeedbackEventsDeltaMvDDL(strategy) {
|
|
3550
|
+
return `
|
|
3551
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_FEEDBACK_EVENTS_DELTA}
|
|
3552
|
+
TO ${TABLE_FEEDBACK_EVENTS_DELTA}
|
|
3553
|
+
AS
|
|
3554
|
+
SELECT
|
|
3555
|
+
${buildDeltaCursorExpr(strategy, "mastra_feedback_events_delta_cursor", "feedbackId")} AS cursorId,
|
|
3556
|
+
ingestedAt,
|
|
3557
|
+
traceId,
|
|
3558
|
+
timestamp,
|
|
3559
|
+
feedbackId
|
|
3560
|
+
FROM (
|
|
3561
|
+
SELECT
|
|
3562
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3563
|
+
traceId,
|
|
3564
|
+
timestamp,
|
|
3565
|
+
feedbackId
|
|
3566
|
+
FROM ${TABLE_FEEDBACK_EVENTS}
|
|
3567
|
+
)
|
|
3568
|
+
`;
|
|
3569
|
+
}
|
|
3570
|
+
function buildMetricEventsDeltaDDL() {
|
|
3571
|
+
return `
|
|
3572
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS_DELTA} (
|
|
3573
|
+
cursorId UInt64,
|
|
3574
|
+
ingestedAt ${DELTA_INGESTED_AT_TYPE},
|
|
3575
|
+
name LowCardinality(String),
|
|
3576
|
+
timestamp DateTime64(3, 'UTC'),
|
|
3577
|
+
metricId String
|
|
3578
|
+
)
|
|
3579
|
+
ENGINE = MergeTree
|
|
3580
|
+
PARTITION BY toDate(ingestedAt)
|
|
3581
|
+
ORDER BY (cursorId)
|
|
3582
|
+
TTL ingestedAt + toIntervalDay(2)
|
|
3583
|
+
`;
|
|
3584
|
+
}
|
|
3585
|
+
function buildMetricEventsDeltaMvDDL(strategy) {
|
|
3586
|
+
return `
|
|
3587
|
+
CREATE MATERIALIZED VIEW IF NOT EXISTS ${MV_METRIC_EVENTS_DELTA}
|
|
3588
|
+
TO ${TABLE_METRIC_EVENTS_DELTA}
|
|
3589
|
+
AS
|
|
3590
|
+
SELECT
|
|
3591
|
+
${buildDeltaCursorExpr(strategy, "mastra_metric_events_delta_cursor", "metricId")} AS cursorId,
|
|
3592
|
+
ingestedAt,
|
|
3593
|
+
name,
|
|
3594
|
+
timestamp,
|
|
3595
|
+
metricId
|
|
3596
|
+
FROM (
|
|
3597
|
+
SELECT
|
|
3598
|
+
now64(9, 'UTC') AS ingestedAt,
|
|
3599
|
+
name,
|
|
3600
|
+
timestamp,
|
|
3601
|
+
metricId
|
|
3602
|
+
FROM ${TABLE_METRIC_EVENTS}
|
|
3603
|
+
)
|
|
3604
|
+
`;
|
|
3605
|
+
}
|
|
3345
3606
|
var DISCOVERY_VALUES_DDL = `
|
|
3346
3607
|
CREATE TABLE IF NOT EXISTS ${TABLE_DISCOVERY_VALUES} (
|
|
3347
3608
|
kind LowCardinality(String),
|
|
@@ -3410,7 +3671,7 @@ SELECT DISTINCT kind, key1, key2, value FROM (
|
|
|
3410
3671
|
WHERE name != '' AND labelKey != '' AND labels[labelKey] != ''
|
|
3411
3672
|
)
|
|
3412
3673
|
`;
|
|
3413
|
-
var
|
|
3674
|
+
var BASE_TABLE_DDL = [
|
|
3414
3675
|
SPAN_EVENTS_DDL,
|
|
3415
3676
|
TRACE_ROOTS_DDL,
|
|
3416
3677
|
TRACE_BRANCHES_DDL,
|
|
@@ -3421,7 +3682,33 @@ var ALL_TABLE_DDL = [
|
|
|
3421
3682
|
DISCOVERY_VALUES_DDL,
|
|
3422
3683
|
DISCOVERY_PAIRS_DDL
|
|
3423
3684
|
];
|
|
3424
|
-
|
|
3685
|
+
function buildDeltaTableDDL() {
|
|
3686
|
+
return [
|
|
3687
|
+
buildTraceRootsDeltaDDL(),
|
|
3688
|
+
buildTraceBranchesDeltaDDL(),
|
|
3689
|
+
buildMetricEventsDeltaDDL(),
|
|
3690
|
+
buildLogEventsDeltaDDL(),
|
|
3691
|
+
buildScoreEventsDeltaDDL(),
|
|
3692
|
+
buildFeedbackEventsDeltaDDL()
|
|
3693
|
+
];
|
|
3694
|
+
}
|
|
3695
|
+
function buildAllTableDDL() {
|
|
3696
|
+
return [...BASE_TABLE_DDL, ...buildDeltaTableDDL()];
|
|
3697
|
+
}
|
|
3698
|
+
var BASE_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
|
|
3699
|
+
function buildDeltaMvDDL(strategy) {
|
|
3700
|
+
return [
|
|
3701
|
+
buildTraceRootsDeltaMvDDL(strategy),
|
|
3702
|
+
buildTraceBranchesDeltaMvDDL(strategy),
|
|
3703
|
+
buildMetricEventsDeltaMvDDL(strategy),
|
|
3704
|
+
buildLogEventsDeltaMvDDL(strategy),
|
|
3705
|
+
buildScoreEventsDeltaMvDDL(strategy),
|
|
3706
|
+
buildFeedbackEventsDeltaMvDDL(strategy)
|
|
3707
|
+
];
|
|
3708
|
+
}
|
|
3709
|
+
function buildAllMvDDL(strategy) {
|
|
3710
|
+
return [...BASE_MV_DDL, ...buildDeltaMvDDL(strategy)];
|
|
3711
|
+
}
|
|
3425
3712
|
var DISCOVERY_MV_DDL = [DISCOVERY_VALUES_MV_DDL, DISCOVERY_PAIRS_MV_DDL];
|
|
3426
3713
|
var addColumn = (table, name, type) => ({
|
|
3427
3714
|
kind: "column",
|
|
@@ -3478,10 +3765,16 @@ var ALL_TABLE_NAMES = [
|
|
|
3478
3765
|
TABLE_SPAN_EVENTS,
|
|
3479
3766
|
TABLE_TRACE_ROOTS,
|
|
3480
3767
|
TABLE_TRACE_BRANCHES,
|
|
3768
|
+
TABLE_TRACE_ROOTS_DELTA,
|
|
3769
|
+
TABLE_TRACE_BRANCHES_DELTA,
|
|
3481
3770
|
TABLE_METRIC_EVENTS,
|
|
3482
3771
|
TABLE_LOG_EVENTS,
|
|
3483
3772
|
TABLE_SCORE_EVENTS,
|
|
3484
3773
|
TABLE_FEEDBACK_EVENTS,
|
|
3774
|
+
TABLE_METRIC_EVENTS_DELTA,
|
|
3775
|
+
TABLE_LOG_EVENTS_DELTA,
|
|
3776
|
+
TABLE_SCORE_EVENTS_DELTA,
|
|
3777
|
+
TABLE_FEEDBACK_EVENTS_DELTA,
|
|
3485
3778
|
TABLE_DISCOVERY_VALUES,
|
|
3486
3779
|
TABLE_DISCOVERY_PAIRS
|
|
3487
3780
|
];
|
|
@@ -4352,6 +4645,35 @@ function buildPaginationClause(pagination) {
|
|
|
4352
4645
|
offset: page * perPage
|
|
4353
4646
|
};
|
|
4354
4647
|
}
|
|
4648
|
+
var OBSERVABILITY_DELTA_POLLING_FEATURE = "observability-delta-polling";
|
|
4649
|
+
function deltaPollingFeatureEnabled() {
|
|
4650
|
+
return coreFeatures.has(OBSERVABILITY_DELTA_POLLING_FEATURE);
|
|
4651
|
+
}
|
|
4652
|
+
function deltaPollingSupported(strategy) {
|
|
4653
|
+
return deltaPollingFeatureEnabled() && strategy !== null;
|
|
4654
|
+
}
|
|
4655
|
+
function assertDeltaPollingSupported(strategy) {
|
|
4656
|
+
if (deltaPollingSupported(strategy)) {
|
|
4657
|
+
return;
|
|
4658
|
+
}
|
|
4659
|
+
throw new MastraError({
|
|
4660
|
+
id: "OBSERVABILITY_DELTA_POLLING_NOT_SUPPORTED",
|
|
4661
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4662
|
+
category: ErrorCategory.SYSTEM,
|
|
4663
|
+
text: "This storage provider does not support observability delta polling"
|
|
4664
|
+
});
|
|
4665
|
+
}
|
|
4666
|
+
function validateCursorId(cursor) {
|
|
4667
|
+
if (/^\d+$/.test(cursor)) {
|
|
4668
|
+
return cursor;
|
|
4669
|
+
}
|
|
4670
|
+
throw new MastraError({
|
|
4671
|
+
id: "OBSERVABILITY_INVALID_DELTA_CURSOR",
|
|
4672
|
+
domain: ErrorDomain.MASTRA_OBSERVABILITY,
|
|
4673
|
+
category: ErrorCategory.USER,
|
|
4674
|
+
text: "Invalid observability delta cursor"
|
|
4675
|
+
});
|
|
4676
|
+
}
|
|
4355
4677
|
|
|
4356
4678
|
// src/storage/domains/observability/v-next/feedback.ts
|
|
4357
4679
|
var FEEDBACK_TYPED_COLUMNS = /* @__PURE__ */ new Set([
|
|
@@ -4478,12 +4800,33 @@ async function batchCreateFeedback(client, args) {
|
|
|
4478
4800
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4479
4801
|
});
|
|
4480
4802
|
}
|
|
4481
|
-
async function listFeedback(client, args) {
|
|
4803
|
+
async function listFeedback(client, args, strategy) {
|
|
4482
4804
|
const parsed = listFeedbackArgsSchema.parse(args);
|
|
4805
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4483
4806
|
const filter = buildFeedbackFilterConditions(parsed.filters, "f");
|
|
4484
4807
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4485
4808
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "f");
|
|
4486
4809
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
4810
|
+
if (parsed.mode === "delta") {
|
|
4811
|
+
assertDeltaPollingSupported(strategy);
|
|
4812
|
+
const streamHeadCursor = await getStreamHeadCursor(client);
|
|
4813
|
+
if (parsed.after === void 0) {
|
|
4814
|
+
return {
|
|
4815
|
+
feedback: [],
|
|
4816
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
4817
|
+
deltaCursor: streamHeadCursor
|
|
4818
|
+
};
|
|
4819
|
+
}
|
|
4820
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
4821
|
+
const rows2 = await queryFeedbackAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
4822
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
4823
|
+
return {
|
|
4824
|
+
feedback: visibleRows.map(rowToFeedbackRecord),
|
|
4825
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
4826
|
+
deltaCursor: visibleRows.length > 0 ? buildFeedbackCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
4827
|
+
};
|
|
4828
|
+
}
|
|
4829
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor(client, whereClause, filter.params) : void 0;
|
|
4487
4830
|
const countResult = await queryJson2(
|
|
4488
4831
|
client,
|
|
4489
4832
|
`SELECT count() AS total FROM ${TABLE_FEEDBACK_EVENTS} AS f ${whereClause}`,
|
|
@@ -4502,9 +4845,68 @@ async function listFeedback(client, args) {
|
|
|
4502
4845
|
perPage: pagination.perPage,
|
|
4503
4846
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4504
4847
|
},
|
|
4505
|
-
feedback: rows.map(rowToFeedbackRecord)
|
|
4848
|
+
feedback: rows.map(rowToFeedbackRecord),
|
|
4849
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4506
4850
|
};
|
|
4507
4851
|
}
|
|
4852
|
+
async function queryFeedbackAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
4853
|
+
return await queryJson2(
|
|
4854
|
+
client,
|
|
4855
|
+
`
|
|
4856
|
+
SELECT
|
|
4857
|
+
f.* EXCEPT(traceId, timestamp, feedbackId),
|
|
4858
|
+
f.traceId AS traceId,
|
|
4859
|
+
f.timestamp AS timestamp,
|
|
4860
|
+
f.feedbackId AS feedbackId,
|
|
4861
|
+
toString(d.cursorId) AS cursorId
|
|
4862
|
+
FROM ${TABLE_FEEDBACK_EVENTS_DELTA} d
|
|
4863
|
+
INNER JOIN ${TABLE_FEEDBACK_EVENTS} f
|
|
4864
|
+
ON ((f.traceId = d.traceId) OR (f.traceId IS NULL AND d.traceId IS NULL))
|
|
4865
|
+
AND f.timestamp = d.timestamp
|
|
4866
|
+
AND f.feedbackId = d.feedbackId
|
|
4867
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
4868
|
+
ORDER BY d.cursorId ASC
|
|
4869
|
+
LIMIT {fetchLimit:UInt32}
|
|
4870
|
+
`,
|
|
4871
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
4872
|
+
);
|
|
4873
|
+
}
|
|
4874
|
+
async function getDeltaCursor(client, whereClause, params) {
|
|
4875
|
+
const rows = await queryJson2(
|
|
4876
|
+
client,
|
|
4877
|
+
`
|
|
4878
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
4879
|
+
FROM ${TABLE_FEEDBACK_EVENTS_DELTA} d
|
|
4880
|
+
INNER JOIN ${TABLE_FEEDBACK_EVENTS} f
|
|
4881
|
+
ON ((f.traceId = d.traceId) OR (f.traceId IS NULL AND d.traceId IS NULL))
|
|
4882
|
+
AND f.timestamp = d.timestamp
|
|
4883
|
+
AND f.feedbackId = d.feedbackId
|
|
4884
|
+
${whereClause}
|
|
4885
|
+
`,
|
|
4886
|
+
params
|
|
4887
|
+
);
|
|
4888
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
4889
|
+
if (cursorId) {
|
|
4890
|
+
return cursorId;
|
|
4891
|
+
}
|
|
4892
|
+
const streamRows = await queryJson2(
|
|
4893
|
+
client,
|
|
4894
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_FEEDBACK_EVENTS_DELTA}`,
|
|
4895
|
+
{}
|
|
4896
|
+
);
|
|
4897
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
4898
|
+
}
|
|
4899
|
+
async function getStreamHeadCursor(client) {
|
|
4900
|
+
const streamRows = await queryJson2(
|
|
4901
|
+
client,
|
|
4902
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_FEEDBACK_EVENTS_DELTA}`,
|
|
4903
|
+
{}
|
|
4904
|
+
);
|
|
4905
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
4906
|
+
}
|
|
4907
|
+
function buildFeedbackCursor(row) {
|
|
4908
|
+
return row.cursorId ?? "0";
|
|
4909
|
+
}
|
|
4508
4910
|
async function getFeedbackAggregate(client, args) {
|
|
4509
4911
|
const aggSql = getAggregationSql(args.aggregation);
|
|
4510
4912
|
const identity = buildFeedbackIdentityFilter(args);
|
|
@@ -4674,12 +5076,33 @@ async function batchCreateLogs(client, args) {
|
|
|
4674
5076
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4675
5077
|
});
|
|
4676
5078
|
}
|
|
4677
|
-
async function listLogs(client, args) {
|
|
5079
|
+
async function listLogs(client, args, strategy) {
|
|
4678
5080
|
const parsed = listLogsArgsSchema.parse(args);
|
|
5081
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4679
5082
|
const filter = buildLogsFilterConditions(parsed.filters, "l");
|
|
4680
5083
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4681
5084
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "l");
|
|
4682
5085
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5086
|
+
if (parsed.mode === "delta") {
|
|
5087
|
+
assertDeltaPollingSupported(strategy);
|
|
5088
|
+
const streamHeadCursor = await getStreamHeadCursor2(client);
|
|
5089
|
+
if (parsed.after === void 0) {
|
|
5090
|
+
return {
|
|
5091
|
+
logs: [],
|
|
5092
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5093
|
+
deltaCursor: streamHeadCursor
|
|
5094
|
+
};
|
|
5095
|
+
}
|
|
5096
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5097
|
+
const rows2 = await queryLogsAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5098
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5099
|
+
return {
|
|
5100
|
+
logs: visibleRows.map(rowToLogRecord),
|
|
5101
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5102
|
+
deltaCursor: visibleRows.length > 0 ? buildLogsCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5103
|
+
};
|
|
5104
|
+
}
|
|
5105
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor2(client, whereClause, filter.params) : void 0;
|
|
4683
5106
|
const countResult = await (await client.query({
|
|
4684
5107
|
query: `SELECT count() AS total FROM ${TABLE_LOG_EVENTS} AS l ${whereClause}`,
|
|
4685
5108
|
query_params: filter.params,
|
|
@@ -4710,9 +5133,71 @@ async function listLogs(client, args) {
|
|
|
4710
5133
|
perPage: pagination.perPage,
|
|
4711
5134
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4712
5135
|
},
|
|
4713
|
-
logs: rows.map(rowToLogRecord)
|
|
5136
|
+
logs: rows.map(rowToLogRecord),
|
|
5137
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4714
5138
|
};
|
|
4715
5139
|
}
|
|
5140
|
+
async function queryLogsAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
5141
|
+
return await (await client.query({
|
|
5142
|
+
query: `
|
|
5143
|
+
SELECT
|
|
5144
|
+
l.* EXCEPT(timestamp, logId),
|
|
5145
|
+
l.timestamp AS timestamp,
|
|
5146
|
+
l.logId AS logId,
|
|
5147
|
+
toString(d.cursorId) AS cursorId
|
|
5148
|
+
FROM ${TABLE_LOG_EVENTS_DELTA} d
|
|
5149
|
+
INNER JOIN ${TABLE_LOG_EVENTS} l
|
|
5150
|
+
ON l.timestamp = d.timestamp
|
|
5151
|
+
AND l.logId = d.logId
|
|
5152
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
5153
|
+
ORDER BY d.cursorId ASC
|
|
5154
|
+
LIMIT {fetchLimit:UInt32}
|
|
5155
|
+
`,
|
|
5156
|
+
query_params: {
|
|
5157
|
+
...params,
|
|
5158
|
+
afterCursor: cursorId,
|
|
5159
|
+
fetchLimit: limit + 1
|
|
5160
|
+
},
|
|
5161
|
+
format: "JSONEachRow",
|
|
5162
|
+
clickhouse_settings: CH_SETTINGS
|
|
5163
|
+
})).json();
|
|
5164
|
+
}
|
|
5165
|
+
async function getDeltaCursor2(client, whereClause, params) {
|
|
5166
|
+
const rows = await (await client.query({
|
|
5167
|
+
query: `
|
|
5168
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
5169
|
+
FROM ${TABLE_LOG_EVENTS_DELTA} d
|
|
5170
|
+
INNER JOIN ${TABLE_LOG_EVENTS} l
|
|
5171
|
+
ON l.timestamp = d.timestamp
|
|
5172
|
+
AND l.logId = d.logId
|
|
5173
|
+
${whereClause}
|
|
5174
|
+
`,
|
|
5175
|
+
query_params: params,
|
|
5176
|
+
format: "JSONEachRow",
|
|
5177
|
+
clickhouse_settings: CH_SETTINGS
|
|
5178
|
+
})).json();
|
|
5179
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
5180
|
+
if (cursorId) {
|
|
5181
|
+
return cursorId;
|
|
5182
|
+
}
|
|
5183
|
+
const streamRows = await (await client.query({
|
|
5184
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_LOG_EVENTS_DELTA}`,
|
|
5185
|
+
format: "JSONEachRow",
|
|
5186
|
+
clickhouse_settings: CH_SETTINGS
|
|
5187
|
+
})).json();
|
|
5188
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5189
|
+
}
|
|
5190
|
+
async function getStreamHeadCursor2(client) {
|
|
5191
|
+
const streamRows = await (await client.query({
|
|
5192
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_LOG_EVENTS_DELTA}`,
|
|
5193
|
+
format: "JSONEachRow",
|
|
5194
|
+
clickhouse_settings: CH_SETTINGS
|
|
5195
|
+
})).json();
|
|
5196
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5197
|
+
}
|
|
5198
|
+
function buildLogsCursor(row) {
|
|
5199
|
+
return row.cursorId ?? "0";
|
|
5200
|
+
}
|
|
4716
5201
|
var METRIC_TYPED_COLUMNS = /* @__PURE__ */ new Set([
|
|
4717
5202
|
"timestamp",
|
|
4718
5203
|
"name",
|
|
@@ -4882,12 +5367,33 @@ async function batchCreateMetrics(client, args) {
|
|
|
4882
5367
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
4883
5368
|
});
|
|
4884
5369
|
}
|
|
4885
|
-
async function listMetrics(client, args) {
|
|
5370
|
+
async function listMetrics(client, args, strategy) {
|
|
4886
5371
|
const parsed = listMetricsArgsSchema.parse(args);
|
|
5372
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
4887
5373
|
const filter = buildMetricsFilterConditions(parsed.filters, "m");
|
|
4888
5374
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
4889
5375
|
const orderBy = buildSignalOrderByClause(["timestamp"], parsed.orderBy, "m");
|
|
4890
5376
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5377
|
+
if (parsed.mode === "delta") {
|
|
5378
|
+
assertDeltaPollingSupported(strategy);
|
|
5379
|
+
const streamHeadCursor = await getStreamHeadCursor3(client);
|
|
5380
|
+
if (parsed.after === void 0) {
|
|
5381
|
+
return {
|
|
5382
|
+
metrics: [],
|
|
5383
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5384
|
+
deltaCursor: streamHeadCursor
|
|
5385
|
+
};
|
|
5386
|
+
}
|
|
5387
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5388
|
+
const rows2 = await queryMetricsAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5389
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5390
|
+
return {
|
|
5391
|
+
metrics: visibleRows.map(rowToMetricRecord),
|
|
5392
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5393
|
+
deltaCursor: visibleRows.length > 0 ? buildMetricsCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5394
|
+
};
|
|
5395
|
+
}
|
|
5396
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor3(client, whereClause, filter.params) : void 0;
|
|
4891
5397
|
const countResult = await queryJson3(
|
|
4892
5398
|
client,
|
|
4893
5399
|
`SELECT count() AS total FROM ${TABLE_METRIC_EVENTS} AS m ${whereClause}`,
|
|
@@ -4906,9 +5412,68 @@ async function listMetrics(client, args) {
|
|
|
4906
5412
|
perPage: pagination.perPage,
|
|
4907
5413
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
4908
5414
|
},
|
|
4909
|
-
metrics: rows.map(rowToMetricRecord)
|
|
5415
|
+
metrics: rows.map(rowToMetricRecord),
|
|
5416
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
4910
5417
|
};
|
|
4911
5418
|
}
|
|
5419
|
+
async function queryMetricsAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
5420
|
+
return await queryJson3(
|
|
5421
|
+
client,
|
|
5422
|
+
`
|
|
5423
|
+
SELECT
|
|
5424
|
+
m.* EXCEPT(name, timestamp, metricId),
|
|
5425
|
+
m.name AS name,
|
|
5426
|
+
m.timestamp AS timestamp,
|
|
5427
|
+
m.metricId AS metricId,
|
|
5428
|
+
toString(d.cursorId) AS cursorId
|
|
5429
|
+
FROM ${TABLE_METRIC_EVENTS_DELTA} d
|
|
5430
|
+
INNER JOIN ${TABLE_METRIC_EVENTS} m
|
|
5431
|
+
ON m.name = d.name
|
|
5432
|
+
AND m.timestamp = d.timestamp
|
|
5433
|
+
AND m.metricId = d.metricId
|
|
5434
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
5435
|
+
ORDER BY d.cursorId ASC
|
|
5436
|
+
LIMIT {fetchLimit:UInt32}
|
|
5437
|
+
`,
|
|
5438
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
5439
|
+
);
|
|
5440
|
+
}
|
|
5441
|
+
async function getDeltaCursor3(client, whereClause, params) {
|
|
5442
|
+
const rows = await queryJson3(
|
|
5443
|
+
client,
|
|
5444
|
+
`
|
|
5445
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
5446
|
+
FROM ${TABLE_METRIC_EVENTS_DELTA} d
|
|
5447
|
+
INNER JOIN ${TABLE_METRIC_EVENTS} m
|
|
5448
|
+
ON m.name = d.name
|
|
5449
|
+
AND m.timestamp = d.timestamp
|
|
5450
|
+
AND m.metricId = d.metricId
|
|
5451
|
+
${whereClause}
|
|
5452
|
+
`,
|
|
5453
|
+
params
|
|
5454
|
+
);
|
|
5455
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
5456
|
+
if (cursorId) {
|
|
5457
|
+
return cursorId;
|
|
5458
|
+
}
|
|
5459
|
+
const streamRows = await queryJson3(
|
|
5460
|
+
client,
|
|
5461
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_METRIC_EVENTS_DELTA}`,
|
|
5462
|
+
{}
|
|
5463
|
+
);
|
|
5464
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5465
|
+
}
|
|
5466
|
+
async function getStreamHeadCursor3(client) {
|
|
5467
|
+
const streamRows = await queryJson3(
|
|
5468
|
+
client,
|
|
5469
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_METRIC_EVENTS_DELTA}`,
|
|
5470
|
+
{}
|
|
5471
|
+
);
|
|
5472
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
5473
|
+
}
|
|
5474
|
+
function buildMetricsCursor(row) {
|
|
5475
|
+
return row.cursorId ?? "0";
|
|
5476
|
+
}
|
|
4912
5477
|
async function getMetricAggregate(client, args) {
|
|
4913
5478
|
const aggSql = getAggregationSql2(args.aggregation, "value", args.distinctColumn);
|
|
4914
5479
|
const nameFilter = buildMetricNameFilter(args.name);
|
|
@@ -5397,12 +5962,33 @@ async function batchCreateScores(client, args) {
|
|
|
5397
5962
|
clickhouse_settings: CH_INSERT_SETTINGS
|
|
5398
5963
|
});
|
|
5399
5964
|
}
|
|
5400
|
-
async function listScores(client, args) {
|
|
5965
|
+
async function listScores(client, args, strategy) {
|
|
5401
5966
|
const parsed = listScoresArgsSchema.parse(args);
|
|
5967
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
5402
5968
|
const filter = buildScoresFilterConditions(parsed.filters, "s");
|
|
5403
5969
|
const pagination = buildPaginationClause(parsed.pagination);
|
|
5404
5970
|
const orderBy = buildSignalOrderByClause(["timestamp", "score"], parsed.orderBy, "s");
|
|
5405
5971
|
const whereClause = filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
5972
|
+
if (parsed.mode === "delta") {
|
|
5973
|
+
assertDeltaPollingSupported(strategy);
|
|
5974
|
+
const streamHeadCursor = await getStreamHeadCursor4(client);
|
|
5975
|
+
if (parsed.after === void 0) {
|
|
5976
|
+
return {
|
|
5977
|
+
scores: [],
|
|
5978
|
+
delta: { limit: parsed.limit, hasMore: false },
|
|
5979
|
+
deltaCursor: streamHeadCursor
|
|
5980
|
+
};
|
|
5981
|
+
}
|
|
5982
|
+
const afterCursor = validateCursorId(parsed.after);
|
|
5983
|
+
const rows2 = await queryScoresAfterCursor(client, whereClause, filter.params, parsed.limit, afterCursor);
|
|
5984
|
+
const visibleRows = rows2.slice(0, parsed.limit);
|
|
5985
|
+
return {
|
|
5986
|
+
scores: visibleRows.map(rowToScoreRecord),
|
|
5987
|
+
delta: { limit: parsed.limit, hasMore: rows2.length > parsed.limit },
|
|
5988
|
+
deltaCursor: visibleRows.length > 0 ? buildScoresCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
5989
|
+
};
|
|
5990
|
+
}
|
|
5991
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor4(client, whereClause, filter.params) : void 0;
|
|
5406
5992
|
const countResult = await queryJson4(
|
|
5407
5993
|
client,
|
|
5408
5994
|
`SELECT count() AS total FROM ${TABLE_SCORE_EVENTS} AS s ${whereClause}`,
|
|
@@ -5421,9 +6007,68 @@ async function listScores(client, args) {
|
|
|
5421
6007
|
perPage: pagination.perPage,
|
|
5422
6008
|
hasMore: (pagination.page + 1) * pagination.perPage < total
|
|
5423
6009
|
},
|
|
5424
|
-
scores: rows.map(rowToScoreRecord)
|
|
6010
|
+
scores: rows.map(rowToScoreRecord),
|
|
6011
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
5425
6012
|
};
|
|
5426
6013
|
}
|
|
6014
|
+
async function queryScoresAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6015
|
+
return await queryJson4(
|
|
6016
|
+
client,
|
|
6017
|
+
`
|
|
6018
|
+
SELECT
|
|
6019
|
+
s.* EXCEPT(traceId, timestamp, scoreId),
|
|
6020
|
+
s.traceId AS traceId,
|
|
6021
|
+
s.timestamp AS timestamp,
|
|
6022
|
+
s.scoreId AS scoreId,
|
|
6023
|
+
toString(d.cursorId) AS cursorId
|
|
6024
|
+
FROM ${TABLE_SCORE_EVENTS_DELTA} d
|
|
6025
|
+
INNER JOIN ${TABLE_SCORE_EVENTS} s
|
|
6026
|
+
ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
|
|
6027
|
+
AND s.timestamp = d.timestamp
|
|
6028
|
+
AND s.scoreId = d.scoreId
|
|
6029
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6030
|
+
ORDER BY d.cursorId ASC
|
|
6031
|
+
LIMIT {fetchLimit:UInt32}
|
|
6032
|
+
`,
|
|
6033
|
+
{ ...params, afterCursor: cursorId, fetchLimit: limit + 1 }
|
|
6034
|
+
);
|
|
6035
|
+
}
|
|
6036
|
+
async function getDeltaCursor4(client, whereClause, params) {
|
|
6037
|
+
const rows = await queryJson4(
|
|
6038
|
+
client,
|
|
6039
|
+
`
|
|
6040
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6041
|
+
FROM ${TABLE_SCORE_EVENTS_DELTA} d
|
|
6042
|
+
INNER JOIN ${TABLE_SCORE_EVENTS} s
|
|
6043
|
+
ON ((s.traceId = d.traceId) OR (s.traceId IS NULL AND d.traceId IS NULL))
|
|
6044
|
+
AND s.timestamp = d.timestamp
|
|
6045
|
+
AND s.scoreId = d.scoreId
|
|
6046
|
+
${whereClause}
|
|
6047
|
+
`,
|
|
6048
|
+
params
|
|
6049
|
+
);
|
|
6050
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6051
|
+
if (cursorId) {
|
|
6052
|
+
return cursorId;
|
|
6053
|
+
}
|
|
6054
|
+
const streamRows = await queryJson4(
|
|
6055
|
+
client,
|
|
6056
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_SCORE_EVENTS_DELTA}`,
|
|
6057
|
+
{}
|
|
6058
|
+
);
|
|
6059
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6060
|
+
}
|
|
6061
|
+
async function getStreamHeadCursor4(client) {
|
|
6062
|
+
const streamRows = await queryJson4(
|
|
6063
|
+
client,
|
|
6064
|
+
`SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_SCORE_EVENTS_DELTA}`,
|
|
6065
|
+
{}
|
|
6066
|
+
);
|
|
6067
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6068
|
+
}
|
|
6069
|
+
function buildScoresCursor(row) {
|
|
6070
|
+
return row.cursorId ?? "0";
|
|
6071
|
+
}
|
|
5427
6072
|
async function getScoreById(client, scoreId) {
|
|
5428
6073
|
const rows = await queryJson4(
|
|
5429
6074
|
client,
|
|
@@ -5686,8 +6331,107 @@ async function listTraceRows(client, args, selectClause, mapRows) {
|
|
|
5686
6331
|
spans: mapRows(rows)
|
|
5687
6332
|
};
|
|
5688
6333
|
}
|
|
5689
|
-
async function listTraces(client, args) {
|
|
5690
|
-
|
|
6334
|
+
async function listTraces(client, args, strategy) {
|
|
6335
|
+
const { mode, filters, pagination, orderBy, after, limit } = listTracesArgsSchema.parse(args);
|
|
6336
|
+
const page = pagination?.page ?? 0;
|
|
6337
|
+
const perPage = pagination?.perPage ?? 10;
|
|
6338
|
+
const { conditions, params } = buildTraceFilterConditions(filters, "r");
|
|
6339
|
+
if (filters?.hasChildError != null) {
|
|
6340
|
+
if (filters.hasChildError) {
|
|
6341
|
+
conditions.push(`EXISTS (
|
|
6342
|
+
SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
|
|
6343
|
+
WHERE c.traceId = r.traceId
|
|
6344
|
+
AND c.parentSpanId IS NOT NULL
|
|
6345
|
+
AND c.error IS NOT NULL
|
|
6346
|
+
)`);
|
|
6347
|
+
} else {
|
|
6348
|
+
conditions.push(`NOT EXISTS (
|
|
6349
|
+
SELECT 1 FROM ${TABLE_SPAN_EVENTS} c
|
|
6350
|
+
WHERE c.traceId = r.traceId
|
|
6351
|
+
AND c.parentSpanId IS NOT NULL
|
|
6352
|
+
AND c.error IS NOT NULL
|
|
6353
|
+
)`);
|
|
6354
|
+
}
|
|
6355
|
+
}
|
|
6356
|
+
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6357
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
6358
|
+
if (mode === "delta") {
|
|
6359
|
+
assertDeltaPollingSupported(strategy);
|
|
6360
|
+
const streamHeadCursor = await getStreamHeadCursor5(client);
|
|
6361
|
+
if (after === void 0) {
|
|
6362
|
+
return {
|
|
6363
|
+
spans: [],
|
|
6364
|
+
delta: { limit, hasMore: false },
|
|
6365
|
+
deltaCursor: streamHeadCursor
|
|
6366
|
+
};
|
|
6367
|
+
}
|
|
6368
|
+
const afterCursor = validateCursorId(after);
|
|
6369
|
+
const rows2 = await queryTracesAfterCursor(client, whereClause, params, limit, afterCursor);
|
|
6370
|
+
const visibleRows = rows2.slice(0, limit);
|
|
6371
|
+
return {
|
|
6372
|
+
spans: toTraceSpans(visibleRows.map(rowToSpanRecord)),
|
|
6373
|
+
delta: { limit, hasMore: rows2.length > limit },
|
|
6374
|
+
deltaCursor: visibleRows.length > 0 ? buildTraceCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
6375
|
+
};
|
|
6376
|
+
}
|
|
6377
|
+
const orderClause = buildTraceOrderByClause(orderBy);
|
|
6378
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor5(client, whereClause, params) : void 0;
|
|
6379
|
+
const countResult = await client.query({
|
|
6380
|
+
query: `
|
|
6381
|
+
SELECT count() as cnt FROM (
|
|
6382
|
+
SELECT dedupeKey
|
|
6383
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
6384
|
+
${whereClause}
|
|
6385
|
+
ORDER BY dedupeKey
|
|
6386
|
+
LIMIT 1 BY dedupeKey
|
|
6387
|
+
)
|
|
6388
|
+
`,
|
|
6389
|
+
query_params: params,
|
|
6390
|
+
format: "JSONEachRow",
|
|
6391
|
+
clickhouse_settings: CH_SETTINGS
|
|
6392
|
+
});
|
|
6393
|
+
const countRows = await countResult.json();
|
|
6394
|
+
const total = Number(countRows[0]?.cnt ?? 0);
|
|
6395
|
+
if (total === 0) {
|
|
6396
|
+
return {
|
|
6397
|
+
pagination: { total: 0, page, perPage, hasMore: false },
|
|
6398
|
+
spans: [],
|
|
6399
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6400
|
+
};
|
|
6401
|
+
}
|
|
6402
|
+
const dataResult = await client.query({
|
|
6403
|
+
query: `
|
|
6404
|
+
SELECT * FROM (
|
|
6405
|
+
SELECT *
|
|
6406
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
6407
|
+
${whereClause}
|
|
6408
|
+
ORDER BY dedupeKey
|
|
6409
|
+
LIMIT 1 BY dedupeKey
|
|
6410
|
+
)
|
|
6411
|
+
ORDER BY ${orderClause}
|
|
6412
|
+
LIMIT {limit:UInt32}
|
|
6413
|
+
OFFSET {offset:UInt32}
|
|
6414
|
+
`,
|
|
6415
|
+
query_params: {
|
|
6416
|
+
...params,
|
|
6417
|
+
limit: perPage,
|
|
6418
|
+
offset: page * perPage
|
|
6419
|
+
},
|
|
6420
|
+
format: "JSONEachRow",
|
|
6421
|
+
clickhouse_settings: CH_SETTINGS
|
|
6422
|
+
});
|
|
6423
|
+
const rows = await dataResult.json();
|
|
6424
|
+
const spans = rows.map(rowToSpanRecord);
|
|
6425
|
+
return {
|
|
6426
|
+
pagination: {
|
|
6427
|
+
total,
|
|
6428
|
+
page,
|
|
6429
|
+
perPage,
|
|
6430
|
+
hasMore: (page + 1) * perPage < total
|
|
6431
|
+
},
|
|
6432
|
+
spans: toTraceSpans(spans),
|
|
6433
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6434
|
+
};
|
|
5691
6435
|
}
|
|
5692
6436
|
var LIGHT_TRACE_ROOT_COLUMNS = [
|
|
5693
6437
|
"traceId",
|
|
@@ -5706,6 +6450,70 @@ var LIGHT_TRACE_ROOT_COLUMNS = [
|
|
|
5706
6450
|
async function listTracesLight(client, args) {
|
|
5707
6451
|
return listTraceRows(client, args, LIGHT_TRACE_ROOT_COLUMNS, (rows) => rows.map(rowToLightSpanRecord));
|
|
5708
6452
|
}
|
|
6453
|
+
async function queryTracesAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6454
|
+
return await (await client.query({
|
|
6455
|
+
query: `
|
|
6456
|
+
SELECT
|
|
6457
|
+
r.* EXCEPT(startedAt, traceId, dedupeKey),
|
|
6458
|
+
r.startedAt AS startedAt,
|
|
6459
|
+
r.traceId AS traceId,
|
|
6460
|
+
r.dedupeKey AS dedupeKey,
|
|
6461
|
+
toString(d.cursorId) AS cursorId
|
|
6462
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA} d
|
|
6463
|
+
INNER JOIN ${TABLE_TRACE_ROOTS} r
|
|
6464
|
+
ON r.startedAt = d.startedAt
|
|
6465
|
+
AND r.traceId = d.traceId
|
|
6466
|
+
AND r.dedupeKey = d.dedupeKey
|
|
6467
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6468
|
+
ORDER BY d.cursorId ASC
|
|
6469
|
+
LIMIT {fetchLimit:UInt32}
|
|
6470
|
+
`,
|
|
6471
|
+
query_params: {
|
|
6472
|
+
...params,
|
|
6473
|
+
afterCursor: cursorId,
|
|
6474
|
+
fetchLimit: limit + 1
|
|
6475
|
+
},
|
|
6476
|
+
format: "JSONEachRow",
|
|
6477
|
+
clickhouse_settings: CH_SETTINGS
|
|
6478
|
+
})).json();
|
|
6479
|
+
}
|
|
6480
|
+
async function getDeltaCursor5(client, whereClause, params) {
|
|
6481
|
+
const rows = await (await client.query({
|
|
6482
|
+
query: `
|
|
6483
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6484
|
+
FROM ${TABLE_TRACE_ROOTS_DELTA} d
|
|
6485
|
+
INNER JOIN ${TABLE_TRACE_ROOTS} r
|
|
6486
|
+
ON r.startedAt = d.startedAt
|
|
6487
|
+
AND r.traceId = d.traceId
|
|
6488
|
+
AND r.dedupeKey = d.dedupeKey
|
|
6489
|
+
${whereClause}
|
|
6490
|
+
`,
|
|
6491
|
+
query_params: params,
|
|
6492
|
+
format: "JSONEachRow",
|
|
6493
|
+
clickhouse_settings: CH_SETTINGS
|
|
6494
|
+
})).json();
|
|
6495
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6496
|
+
if (cursorId) {
|
|
6497
|
+
return cursorId;
|
|
6498
|
+
}
|
|
6499
|
+
const streamRows = await (await client.query({
|
|
6500
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_ROOTS_DELTA}`,
|
|
6501
|
+
format: "JSONEachRow",
|
|
6502
|
+
clickhouse_settings: CH_SETTINGS
|
|
6503
|
+
})).json();
|
|
6504
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6505
|
+
}
|
|
6506
|
+
async function getStreamHeadCursor5(client) {
|
|
6507
|
+
const streamRows = await (await client.query({
|
|
6508
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_ROOTS_DELTA}`,
|
|
6509
|
+
format: "JSONEachRow",
|
|
6510
|
+
clickhouse_settings: CH_SETTINGS
|
|
6511
|
+
})).json();
|
|
6512
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6513
|
+
}
|
|
6514
|
+
function buildTraceCursor(row) {
|
|
6515
|
+
return row.cursorId ?? "0";
|
|
6516
|
+
}
|
|
5709
6517
|
var BRANCH_SPAN_TYPE_SQL_LIST = BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
|
|
5710
6518
|
async function createSpan(client, args) {
|
|
5711
6519
|
const row = spanRecordToRow(args.span);
|
|
@@ -5837,36 +6645,36 @@ async function batchDeleteTraces(client, args) {
|
|
|
5837
6645
|
})
|
|
5838
6646
|
]);
|
|
5839
6647
|
}
|
|
5840
|
-
async function listBranches(client, args) {
|
|
5841
|
-
const { filters, pagination, orderBy } = listBranchesArgsSchema.parse(args);
|
|
6648
|
+
async function listBranches(client, args, strategy) {
|
|
6649
|
+
const { mode, filters, pagination, orderBy, after, limit } = listBranchesArgsSchema.parse(args);
|
|
5842
6650
|
const page = pagination?.page ?? 0;
|
|
5843
6651
|
const perPage = pagination?.perPage ?? 10;
|
|
5844
6652
|
const conditions = [];
|
|
5845
6653
|
const params = {};
|
|
5846
6654
|
if (filters?.spanType) {
|
|
5847
|
-
conditions.push(`spanType = {spanType:String}`);
|
|
6655
|
+
conditions.push(`b.spanType = {spanType:String}`);
|
|
5848
6656
|
params.spanType = filters.spanType;
|
|
5849
6657
|
} else {
|
|
5850
|
-
conditions.push(`spanType IN (${BRANCH_SPAN_TYPE_SQL_LIST})`);
|
|
6658
|
+
conditions.push(`b.spanType IN (${BRANCH_SPAN_TYPE_SQL_LIST})`);
|
|
5851
6659
|
}
|
|
5852
6660
|
if (filters?.startedAt?.start) {
|
|
5853
6661
|
const op = filters.startedAt.startExclusive ? ">" : ">=";
|
|
5854
|
-
conditions.push(`startedAt ${op} {startedAtStart:DateTime64(3)}`);
|
|
6662
|
+
conditions.push(`b.startedAt ${op} {startedAtStart:DateTime64(3)}`);
|
|
5855
6663
|
params.startedAtStart = filters.startedAt.start.getTime();
|
|
5856
6664
|
}
|
|
5857
6665
|
if (filters?.startedAt?.end) {
|
|
5858
6666
|
const op = filters.startedAt.endExclusive ? "<" : "<=";
|
|
5859
|
-
conditions.push(`startedAt ${op} {startedAtEnd:DateTime64(3)}`);
|
|
6667
|
+
conditions.push(`b.startedAt ${op} {startedAtEnd:DateTime64(3)}`);
|
|
5860
6668
|
params.startedAtEnd = filters.startedAt.end.getTime();
|
|
5861
6669
|
}
|
|
5862
6670
|
if (filters?.endedAt?.start) {
|
|
5863
6671
|
const op = filters.endedAt.startExclusive ? ">" : ">=";
|
|
5864
|
-
conditions.push(`endedAt ${op} {endedAtStart:DateTime64(3)}`);
|
|
6672
|
+
conditions.push(`b.endedAt ${op} {endedAtStart:DateTime64(3)}`);
|
|
5865
6673
|
params.endedAtStart = filters.endedAt.start.getTime();
|
|
5866
6674
|
}
|
|
5867
6675
|
if (filters?.endedAt?.end) {
|
|
5868
6676
|
const op = filters.endedAt.endExclusive ? "<" : "<=";
|
|
5869
|
-
conditions.push(`endedAt ${op} {endedAtEnd:DateTime64(3)}`);
|
|
6677
|
+
conditions.push(`b.endedAt ${op} {endedAtEnd:DateTime64(3)}`);
|
|
5870
6678
|
params.endedAtEnd = filters.endedAt.end.getTime();
|
|
5871
6679
|
}
|
|
5872
6680
|
const eq = [
|
|
@@ -5897,7 +6705,7 @@ async function listBranches(client, args) {
|
|
|
5897
6705
|
];
|
|
5898
6706
|
for (const { col, value, param } of eq) {
|
|
5899
6707
|
if (value == null) continue;
|
|
5900
|
-
conditions.push(
|
|
6708
|
+
conditions.push(`b.${col} = {${param}:String}`);
|
|
5901
6709
|
params[param] = value;
|
|
5902
6710
|
}
|
|
5903
6711
|
if (filters?.tags && filters.tags.length > 0) {
|
|
@@ -5905,7 +6713,7 @@ async function listBranches(client, args) {
|
|
|
5905
6713
|
const tag = filters.tags[i];
|
|
5906
6714
|
if (typeof tag !== "string" || tag.trim() === "") continue;
|
|
5907
6715
|
const param = `tag_${i}`;
|
|
5908
|
-
conditions.push(`has(tags, {${param}:String})`);
|
|
6716
|
+
conditions.push(`has(b.tags, {${param}:String})`);
|
|
5909
6717
|
params[param] = tag;
|
|
5910
6718
|
}
|
|
5911
6719
|
}
|
|
@@ -5915,7 +6723,7 @@ async function listBranches(client, args) {
|
|
|
5915
6723
|
if (typeof value !== "string") continue;
|
|
5916
6724
|
const keyParam = `meta_k_${i}`;
|
|
5917
6725
|
const valParam = `meta_v_${i}`;
|
|
5918
|
-
conditions.push(`metadataSearch[{${keyParam}:String}] = {${valParam}:String}`);
|
|
6726
|
+
conditions.push(`b.metadataSearch[{${keyParam}:String}] = {${valParam}:String}`);
|
|
5919
6727
|
params[keyParam] = key;
|
|
5920
6728
|
params[valParam] = value;
|
|
5921
6729
|
i++;
|
|
@@ -5929,30 +6737,51 @@ async function listBranches(client, args) {
|
|
|
5929
6737
|
if (normalized == null) continue;
|
|
5930
6738
|
const keyParam = `scope_k_${i}`;
|
|
5931
6739
|
const valParam = `scope_v_${i}`;
|
|
5932
|
-
conditions.push(`JSONExtractString(scope, {${keyParam}:String}) = {${valParam}:String}`);
|
|
6740
|
+
conditions.push(`JSONExtractString(b.scope, {${keyParam}:String}) = {${valParam}:String}`);
|
|
5933
6741
|
params[keyParam] = key;
|
|
5934
6742
|
params[valParam] = normalized;
|
|
5935
6743
|
i++;
|
|
5936
6744
|
}
|
|
5937
6745
|
}
|
|
5938
6746
|
if (filters?.status === TraceStatus.ERROR) {
|
|
5939
|
-
conditions.push(`error IS NOT NULL`);
|
|
6747
|
+
conditions.push(`b.error IS NOT NULL`);
|
|
5940
6748
|
} else if (filters?.status === TraceStatus.SUCCESS) {
|
|
5941
|
-
conditions.push(`error IS NULL`);
|
|
6749
|
+
conditions.push(`b.error IS NULL`);
|
|
5942
6750
|
} else if (filters?.status === TraceStatus.RUNNING) {
|
|
5943
6751
|
conditions.push("1 = 0");
|
|
5944
6752
|
}
|
|
5945
6753
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
6754
|
+
const deltaCursorEnabled = deltaPollingSupported(strategy);
|
|
6755
|
+
if (mode === "delta") {
|
|
6756
|
+
assertDeltaPollingSupported(strategy);
|
|
6757
|
+
const streamHeadCursor = await getStreamHeadCursor6(client);
|
|
6758
|
+
if (after === void 0) {
|
|
6759
|
+
return {
|
|
6760
|
+
branches: [],
|
|
6761
|
+
delta: { limit, hasMore: false },
|
|
6762
|
+
deltaCursor: streamHeadCursor
|
|
6763
|
+
};
|
|
6764
|
+
}
|
|
6765
|
+
const afterCursor = validateCursorId(after);
|
|
6766
|
+
const rows2 = await queryBranchesAfterCursor(client, whereClause, params, limit, afterCursor);
|
|
6767
|
+
const visibleRows = rows2.slice(0, limit);
|
|
6768
|
+
return {
|
|
6769
|
+
branches: toTraceSpans(visibleRows.map(rowToSpanRecord)),
|
|
6770
|
+
delta: { limit, hasMore: rows2.length > limit },
|
|
6771
|
+
deltaCursor: visibleRows.length > 0 ? buildBranchCursor(visibleRows[visibleRows.length - 1]) : streamHeadCursor
|
|
6772
|
+
};
|
|
6773
|
+
}
|
|
5946
6774
|
const sortField = orderBy?.field === "endedAt" ? "endedAt" : "startedAt";
|
|
5947
6775
|
const sortDirection = orderBy?.direction === "ASC" ? "ASC" : "DESC";
|
|
6776
|
+
const currentDeltaCursor = deltaCursorEnabled ? await getDeltaCursor6(client, whereClause, params) : void 0;
|
|
5948
6777
|
const countResult = await client.query({
|
|
5949
6778
|
query: `
|
|
5950
6779
|
SELECT count() as cnt FROM (
|
|
5951
6780
|
SELECT dedupeKey
|
|
5952
|
-
FROM ${TABLE_TRACE_BRANCHES}
|
|
6781
|
+
FROM ${TABLE_TRACE_BRANCHES} b
|
|
5953
6782
|
${whereClause}
|
|
5954
|
-
ORDER BY dedupeKey
|
|
5955
|
-
LIMIT 1 BY dedupeKey
|
|
6783
|
+
ORDER BY b.dedupeKey
|
|
6784
|
+
LIMIT 1 BY b.dedupeKey
|
|
5956
6785
|
)
|
|
5957
6786
|
`,
|
|
5958
6787
|
query_params: params,
|
|
@@ -5964,17 +6793,18 @@ async function listBranches(client, args) {
|
|
|
5964
6793
|
if (total === 0) {
|
|
5965
6794
|
return {
|
|
5966
6795
|
pagination: { total: 0, page, perPage, hasMore: false },
|
|
5967
|
-
branches: []
|
|
6796
|
+
branches: [],
|
|
6797
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
5968
6798
|
};
|
|
5969
6799
|
}
|
|
5970
6800
|
const dataResult = await client.query({
|
|
5971
6801
|
query: `
|
|
5972
6802
|
SELECT * FROM (
|
|
5973
6803
|
SELECT *
|
|
5974
|
-
FROM ${TABLE_TRACE_BRANCHES}
|
|
6804
|
+
FROM ${TABLE_TRACE_BRANCHES} b
|
|
5975
6805
|
${whereClause}
|
|
5976
|
-
ORDER BY dedupeKey
|
|
5977
|
-
LIMIT 1 BY dedupeKey
|
|
6806
|
+
ORDER BY b.dedupeKey
|
|
6807
|
+
LIMIT 1 BY b.dedupeKey
|
|
5978
6808
|
)
|
|
5979
6809
|
ORDER BY ${sortField} ${sortDirection}, dedupeKey ASC
|
|
5980
6810
|
LIMIT {limit:UInt32}
|
|
@@ -5997,9 +6827,80 @@ async function listBranches(client, args) {
|
|
|
5997
6827
|
perPage,
|
|
5998
6828
|
hasMore: (page + 1) * perPage < total
|
|
5999
6829
|
},
|
|
6000
|
-
branches: toTraceSpans(spans)
|
|
6830
|
+
branches: toTraceSpans(spans),
|
|
6831
|
+
...deltaCursorEnabled ? { deltaCursor: currentDeltaCursor } : {}
|
|
6001
6832
|
};
|
|
6002
6833
|
}
|
|
6834
|
+
async function queryBranchesAfterCursor(client, whereClause, params, limit, cursorId) {
|
|
6835
|
+
return await (await client.query({
|
|
6836
|
+
query: `
|
|
6837
|
+
SELECT
|
|
6838
|
+
b.* EXCEPT(spanType, startedAt, traceId, spanId, dedupeKey),
|
|
6839
|
+
b.spanType AS spanType,
|
|
6840
|
+
b.startedAt AS startedAt,
|
|
6841
|
+
b.traceId AS traceId,
|
|
6842
|
+
b.spanId AS spanId,
|
|
6843
|
+
b.dedupeKey AS dedupeKey,
|
|
6844
|
+
toString(d.cursorId) AS cursorId
|
|
6845
|
+
FROM ${TABLE_TRACE_BRANCHES_DELTA} d
|
|
6846
|
+
INNER JOIN ${TABLE_TRACE_BRANCHES} b
|
|
6847
|
+
ON b.spanType = d.spanType
|
|
6848
|
+
AND b.startedAt = d.startedAt
|
|
6849
|
+
AND b.traceId = d.traceId
|
|
6850
|
+
AND b.spanId = d.spanId
|
|
6851
|
+
AND b.dedupeKey = d.dedupeKey
|
|
6852
|
+
${whereClause ? `${whereClause} AND d.cursorId > {afterCursor:UInt64}` : "WHERE d.cursorId > {afterCursor:UInt64}"}
|
|
6853
|
+
ORDER BY d.cursorId ASC
|
|
6854
|
+
LIMIT {fetchLimit:UInt32}
|
|
6855
|
+
`,
|
|
6856
|
+
query_params: {
|
|
6857
|
+
...params,
|
|
6858
|
+
afterCursor: cursorId,
|
|
6859
|
+
fetchLimit: limit + 1
|
|
6860
|
+
},
|
|
6861
|
+
format: "JSONEachRow",
|
|
6862
|
+
clickhouse_settings: CH_SETTINGS
|
|
6863
|
+
})).json();
|
|
6864
|
+
}
|
|
6865
|
+
async function getDeltaCursor6(client, whereClause, params) {
|
|
6866
|
+
const rows = await (await client.query({
|
|
6867
|
+
query: `
|
|
6868
|
+
SELECT toString(max(d.cursorId)) AS cursorId
|
|
6869
|
+
FROM ${TABLE_TRACE_BRANCHES_DELTA} d
|
|
6870
|
+
INNER JOIN ${TABLE_TRACE_BRANCHES} b
|
|
6871
|
+
ON b.spanType = d.spanType
|
|
6872
|
+
AND b.startedAt = d.startedAt
|
|
6873
|
+
AND b.traceId = d.traceId
|
|
6874
|
+
AND b.spanId = d.spanId
|
|
6875
|
+
AND b.dedupeKey = d.dedupeKey
|
|
6876
|
+
${whereClause}
|
|
6877
|
+
`,
|
|
6878
|
+
query_params: params,
|
|
6879
|
+
format: "JSONEachRow",
|
|
6880
|
+
clickhouse_settings: CH_SETTINGS
|
|
6881
|
+
})).json();
|
|
6882
|
+
const cursorId = rows[0]?.cursorId ?? null;
|
|
6883
|
+
if (cursorId) {
|
|
6884
|
+
return cursorId;
|
|
6885
|
+
}
|
|
6886
|
+
const streamRows = await (await client.query({
|
|
6887
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_BRANCHES_DELTA}`,
|
|
6888
|
+
format: "JSONEachRow",
|
|
6889
|
+
clickhouse_settings: CH_SETTINGS
|
|
6890
|
+
})).json();
|
|
6891
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6892
|
+
}
|
|
6893
|
+
async function getStreamHeadCursor6(client) {
|
|
6894
|
+
const streamRows = await (await client.query({
|
|
6895
|
+
query: `SELECT toString(max(cursorId)) AS cursorId FROM ${TABLE_TRACE_BRANCHES_DELTA}`,
|
|
6896
|
+
format: "JSONEachRow",
|
|
6897
|
+
clickhouse_settings: CH_SETTINGS
|
|
6898
|
+
})).json();
|
|
6899
|
+
return streamRows[0]?.cursorId ?? "0";
|
|
6900
|
+
}
|
|
6901
|
+
function buildBranchCursor(row) {
|
|
6902
|
+
return row.cursorId ?? "0";
|
|
6903
|
+
}
|
|
6003
6904
|
|
|
6004
6905
|
// src/storage/domains/observability/v-next/index.ts
|
|
6005
6906
|
function buildSignalMigrationRequiredMessage(args) {
|
|
@@ -6096,14 +6997,72 @@ async function queryNamesByTable(client, query, tables) {
|
|
|
6096
6997
|
}
|
|
6097
6998
|
return out;
|
|
6098
6999
|
}
|
|
7000
|
+
async function detectDeltaCursorStrategy(client, override, existingStrategy) {
|
|
7001
|
+
if (existingStrategy && existingStrategy !== "mixed") {
|
|
7002
|
+
return existingStrategy;
|
|
7003
|
+
}
|
|
7004
|
+
try {
|
|
7005
|
+
await client.query({
|
|
7006
|
+
query: `SELECT generateSerialID({counterName:String}) AS cursorId`,
|
|
7007
|
+
query_params: { counterName: "mastra_observability_delta_cursor_probe" },
|
|
7008
|
+
format: "JSONEachRow"
|
|
7009
|
+
});
|
|
7010
|
+
return "serial";
|
|
7011
|
+
} catch {
|
|
7012
|
+
return "fallback";
|
|
7013
|
+
}
|
|
7014
|
+
}
|
|
7015
|
+
async function detectExistingDeltaCursorStrategy(client) {
|
|
7016
|
+
try {
|
|
7017
|
+
const mvResult = await client.query({
|
|
7018
|
+
query: `
|
|
7019
|
+
SELECT name, create_table_query
|
|
7020
|
+
FROM system.tables
|
|
7021
|
+
WHERE database = currentDatabase()
|
|
7022
|
+
AND name IN ({tables:Array(String)})
|
|
7023
|
+
`,
|
|
7024
|
+
query_params: { tables: [...DELTA_MV_NAMES] },
|
|
7025
|
+
format: "JSONEachRow"
|
|
7026
|
+
});
|
|
7027
|
+
const mvRows = await mvResult.json();
|
|
7028
|
+
if (mvRows.length === 0) {
|
|
7029
|
+
return null;
|
|
7030
|
+
}
|
|
7031
|
+
let sawSerialMv = false;
|
|
7032
|
+
let sawFallbackMv = false;
|
|
7033
|
+
for (const row of mvRows) {
|
|
7034
|
+
const ddl = row.create_table_query ?? "";
|
|
7035
|
+
if (ddl.includes("generateSerialID(")) {
|
|
7036
|
+
sawSerialMv = true;
|
|
7037
|
+
} else if (ddl.includes("farmFingerprint64(")) {
|
|
7038
|
+
sawFallbackMv = true;
|
|
7039
|
+
}
|
|
7040
|
+
}
|
|
7041
|
+
if (sawSerialMv && sawFallbackMv) {
|
|
7042
|
+
return "mixed";
|
|
7043
|
+
}
|
|
7044
|
+
if (sawSerialMv) {
|
|
7045
|
+
return "serial";
|
|
7046
|
+
}
|
|
7047
|
+
if (sawFallbackMv) {
|
|
7048
|
+
return "fallback";
|
|
7049
|
+
}
|
|
7050
|
+
return null;
|
|
7051
|
+
} catch {
|
|
7052
|
+
return null;
|
|
7053
|
+
}
|
|
7054
|
+
}
|
|
6099
7055
|
var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
6100
7056
|
#client;
|
|
6101
7057
|
#retention;
|
|
7058
|
+
#deltaCursorStrategyOverride;
|
|
7059
|
+
#deltaCursorStrategy = "fallback";
|
|
6102
7060
|
constructor(config) {
|
|
6103
7061
|
super();
|
|
6104
7062
|
const { client } = resolveClickhouseConfig(config);
|
|
6105
7063
|
this.#client = client;
|
|
6106
7064
|
this.#retention = config.retention;
|
|
7065
|
+
this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
|
|
6107
7066
|
}
|
|
6108
7067
|
// -------------------------------------------------------------------------
|
|
6109
7068
|
// Initialization
|
|
@@ -6122,7 +7081,21 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6122
7081
|
});
|
|
6123
7082
|
}
|
|
6124
7083
|
try {
|
|
6125
|
-
|
|
7084
|
+
const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
|
|
7085
|
+
if (existingStrategy === "mixed") {
|
|
7086
|
+
this.#deltaCursorStrategy = null;
|
|
7087
|
+
this.logger.error(
|
|
7088
|
+
"ClickHouse observability delta tables use mixed cursor schemas; delta polling has been disabled for this store instance."
|
|
7089
|
+
);
|
|
7090
|
+
} else if (this.#deltaCursorStrategyOverride) {
|
|
7091
|
+
this.#deltaCursorStrategy = this.#deltaCursorStrategyOverride;
|
|
7092
|
+
} else if (existingStrategy) {
|
|
7093
|
+
this.#deltaCursorStrategy = existingStrategy;
|
|
7094
|
+
} else {
|
|
7095
|
+
this.#deltaCursorStrategy = await detectDeltaCursorStrategy(this.#client, void 0, existingStrategy);
|
|
7096
|
+
}
|
|
7097
|
+
const coreDdl = this.#deltaCursorStrategy === null ? [...BASE_TABLE_DDL, ...BASE_MV_DDL] : [...buildAllTableDDL(), ...buildAllMvDDL(this.#deltaCursorStrategy)];
|
|
7098
|
+
for (const ddl of coreDdl) {
|
|
6126
7099
|
await this.#client.command({ query: ddl });
|
|
6127
7100
|
}
|
|
6128
7101
|
const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
|
|
@@ -6193,6 +7166,12 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6193
7166
|
supported: ["insert-only"]
|
|
6194
7167
|
};
|
|
6195
7168
|
}
|
|
7169
|
+
getFeatures() {
|
|
7170
|
+
if (!deltaPollingSupported(this.#deltaCursorStrategy)) {
|
|
7171
|
+
return void 0;
|
|
7172
|
+
}
|
|
7173
|
+
return ["delta-polling"];
|
|
7174
|
+
}
|
|
6196
7175
|
// -------------------------------------------------------------------------
|
|
6197
7176
|
// Tracing — writes
|
|
6198
7177
|
// -------------------------------------------------------------------------
|
|
@@ -6313,7 +7292,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6313
7292
|
}
|
|
6314
7293
|
async listTraces(args) {
|
|
6315
7294
|
try {
|
|
6316
|
-
return await listTraces(this.#client, args);
|
|
7295
|
+
return await listTraces(this.#client, args, this.#deltaCursorStrategy);
|
|
6317
7296
|
} catch (error) {
|
|
6318
7297
|
if (error instanceof MastraError) throw error;
|
|
6319
7298
|
throw new MastraError(
|
|
@@ -6343,7 +7322,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6343
7322
|
}
|
|
6344
7323
|
async listBranches(args) {
|
|
6345
7324
|
try {
|
|
6346
|
-
return await listBranches(this.#client, args);
|
|
7325
|
+
return await listBranches(this.#client, args, this.#deltaCursorStrategy);
|
|
6347
7326
|
} catch (error) {
|
|
6348
7327
|
if (error instanceof MastraError) throw error;
|
|
6349
7328
|
throw new MastraError(
|
|
@@ -6374,7 +7353,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6374
7353
|
}
|
|
6375
7354
|
async listLogs(args) {
|
|
6376
7355
|
try {
|
|
6377
|
-
return await listLogs(this.#client, args);
|
|
7356
|
+
return await listLogs(this.#client, args, this.#deltaCursorStrategy);
|
|
6378
7357
|
} catch (error) {
|
|
6379
7358
|
if (error instanceof MastraError) throw error;
|
|
6380
7359
|
throw new MastraError(
|
|
@@ -6405,7 +7384,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6405
7384
|
}
|
|
6406
7385
|
async listMetrics(args) {
|
|
6407
7386
|
try {
|
|
6408
|
-
return await listMetrics(this.#client, args);
|
|
7387
|
+
return await listMetrics(this.#client, args, this.#deltaCursorStrategy);
|
|
6409
7388
|
} catch (error) {
|
|
6410
7389
|
if (error instanceof MastraError) throw error;
|
|
6411
7390
|
throw new MastraError(
|
|
@@ -6451,7 +7430,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6451
7430
|
}
|
|
6452
7431
|
async listScores(args) {
|
|
6453
7432
|
try {
|
|
6454
|
-
return await listScores(this.#client, args);
|
|
7433
|
+
return await listScores(this.#client, args, this.#deltaCursorStrategy);
|
|
6455
7434
|
} catch (error) {
|
|
6456
7435
|
if (error instanceof MastraError) throw error;
|
|
6457
7436
|
throw new MastraError(
|
|
@@ -6513,7 +7492,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
6513
7492
|
}
|
|
6514
7493
|
async listFeedback(args) {
|
|
6515
7494
|
try {
|
|
6516
|
-
return await listFeedback(this.#client, args);
|
|
7495
|
+
return await listFeedback(this.#client, args, this.#deltaCursorStrategy);
|
|
6517
7496
|
} catch (error) {
|
|
6518
7497
|
if (error instanceof MastraError) throw error;
|
|
6519
7498
|
throw new MastraError(
|