@mastra/duckdb 1.13.0-alpha.0 → 1.13.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/{db-zp2htaur.cjs → features-BzhPmFg2.cjs} +63 -2
- package/dist/features-BzhPmFg2.cjs.map +1 -0
- package/dist/{db-BWgrbB_u.js → features-D3sUOsCL.js} +23 -3
- package/dist/features-D3sUOsCL.js.map +1 -0
- package/dist/index.cjs +11 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/{observability-BCEHa8GU.cjs → observability-D4eFjsKJ.cjs} +279 -77
- package/dist/observability-D4eFjsKJ.cjs.map +1 -0
- package/dist/{observability-TpgAucZM.js → observability-DINJlR_a.js} +276 -52
- package/dist/observability-DINJlR_a.js.map +1 -0
- package/dist/storage/db/index.d.ts +5 -1
- package/dist/storage/db/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/helpers.d.ts +17 -0
- package/dist/storage/domains/observability/helpers.d.ts.map +1 -1
- package/dist/storage/domains/observability/index.d.ts +3 -2
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/span-query.d.ts +43 -0
- package/dist/storage/domains/observability/span-query.d.ts.map +1 -0
- package/dist/storage/domains/observability/trace-query.d.ts +8 -1
- package/dist/storage/domains/observability/trace-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/tracing.d.ts.map +1 -1
- package/dist/storage/features.d.ts +2 -0
- package/dist/storage/features.d.ts.map +1 -0
- package/dist/storage/index.d.ts +1 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/package.json +3 -3
- package/dist/db-BWgrbB_u.js.map +0 -1
- package/dist/db-zp2htaur.cjs.map +0 -1
- package/dist/observability-BCEHa8GU.cjs.map +0 -1
- package/dist/observability-TpgAucZM.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { n as DuckDBConnection, r as DuckDBQueryTimeoutError, t as spanQueryFeatures } from "./features-D3sUOsCL.js";
|
|
2
2
|
import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
|
|
3
3
|
import * as coreStorage from "@mastra/core/storage";
|
|
4
4
|
import { BRANCH_SPAN_TYPES, METRIC_DISTINCT_COLUMNS, ObservabilityStorage, buildInputPreview, computeTraceStatus, createStorageErrorId, executeRetentionPrune, feedbackRecordSchema, listBranchesArgsSchema, listFeedbackArgsSchema, listLogsArgsSchema, listMetricsArgsSchema, listScoresArgsSchema, listTracesArgsSchema, resolveRetentionTargets, retentionCutoffMs, toTraceSpans } from "@mastra/core/storage";
|
|
@@ -688,6 +688,33 @@ function jsonV(val) {
|
|
|
688
688
|
return DuckDBConnection.sqlValue(JSON.stringify(val));
|
|
689
689
|
}
|
|
690
690
|
/**
|
|
691
|
+
* `span_events` JSON columns that can hold large payloads. Missing values are
|
|
692
|
+
* stored as the JSON literal `null` instead of SQL NULL.
|
|
693
|
+
*
|
|
694
|
+
* DuckDB only reads the selected rows of a large-string column when the
|
|
695
|
+
* column's validity mask is constant (no NULLs in the row group). Any SQL NULL
|
|
696
|
+
* makes it fall back to reading the whole 2,048-row vector, which loads every
|
|
697
|
+
* large string in it even when the query needs one row. Keeping these columns
|
|
698
|
+
* NULL-free lets point lookups and page reads load only what they return.
|
|
699
|
+
*/
|
|
700
|
+
const SPAN_PAYLOAD_COLUMNS = [
|
|
701
|
+
"attributes",
|
|
702
|
+
"input",
|
|
703
|
+
"output",
|
|
704
|
+
"requestContext"
|
|
705
|
+
];
|
|
706
|
+
/** Stored in {@link SPAN_PAYLOAD_COLUMNS} in place of SQL NULL; read back as missing. */
|
|
707
|
+
const JSON_NULL_PLACEHOLDER = "null";
|
|
708
|
+
/** Like {@link jsonV}, but writes the JSON `null` placeholder instead of SQL NULL. */
|
|
709
|
+
function payloadJsonV(val) {
|
|
710
|
+
if (val === null || val === void 0) return `'${JSON_NULL_PLACEHOLDER}'`;
|
|
711
|
+
return jsonV(val);
|
|
712
|
+
}
|
|
713
|
+
/** SQL for a payload column that maps the JSON `null` placeholder back to SQL NULL. */
|
|
714
|
+
function payloadColumnSql(column) {
|
|
715
|
+
return `NULLIF(${column}, '${JSON_NULL_PLACEHOLDER}')`;
|
|
716
|
+
}
|
|
717
|
+
/**
|
|
691
718
|
* Trim, drop blank, and dedupe tags before they are written, matching the
|
|
692
719
|
* PostgreSQL and ClickHouse stores so tag predicates and discovery agree.
|
|
693
720
|
*/
|
|
@@ -2935,7 +2962,7 @@ const TRACE_SELECT = `
|
|
|
2935
2962
|
r.entityId AS entityId,
|
|
2936
2963
|
r.parentSpanId AS parentSpanId,
|
|
2937
2964
|
r.metadata AS metadata,
|
|
2938
|
-
r.input AS input,
|
|
2965
|
+
${payloadColumnSql("r.input")} AS input,
|
|
2939
2966
|
r.threadId AS threadId,
|
|
2940
2967
|
r.resourceId AS resourceId,
|
|
2941
2968
|
r.startedAt AS startedAt,
|
|
@@ -3218,6 +3245,11 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
|
|
|
3218
3245
|
GROUP BY r.traceId
|
|
3219
3246
|
HAVING bool_or(r.timestamp >= CAST(? AS TIMESTAMP) AND r.timestamp < CAST(? AS TIMESTAMP))
|
|
3220
3247
|
AND bool_or(${rowConditions.join("\n AND ")})
|
|
3248
|
+
)`,
|
|
3249
|
+
`root_bounds AS (
|
|
3250
|
+
SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
|
|
3251
|
+
FROM span_events
|
|
3252
|
+
WHERE parentSpanId IS NULL AND traceId IN (SELECT traceId FROM root_trace_ids)
|
|
3221
3253
|
)`,
|
|
3222
3254
|
`root_events AS (
|
|
3223
3255
|
SELECT
|
|
@@ -3228,6 +3260,8 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
|
|
|
3228
3260
|
END AS startedAt
|
|
3229
3261
|
FROM span_events
|
|
3230
3262
|
WHERE parentSpanId IS NULL
|
|
3263
|
+
AND timestamp >= (SELECT minTs FROM root_bounds)
|
|
3264
|
+
AND timestamp <= (SELECT maxTs FROM root_bounds)
|
|
3231
3265
|
AND traceId IN (SELECT traceId FROM root_trace_ids)
|
|
3232
3266
|
)`,
|
|
3233
3267
|
`current_roots AS (
|
|
@@ -3250,7 +3284,15 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
|
|
|
3250
3284
|
)`
|
|
3251
3285
|
];
|
|
3252
3286
|
if (relatedCollections.has("spans")) {
|
|
3253
|
-
ctes.push(`
|
|
3287
|
+
ctes.push(`span_bounds AS (
|
|
3288
|
+
-- Exact event-time range of the in-scope traces, from a narrow
|
|
3289
|
+
-- (traceId, timestamp) pass. The join below cannot be pushed into the
|
|
3290
|
+
-- scan, so the range keeps it from decompressing every span row.
|
|
3291
|
+
SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
|
|
3292
|
+
FROM span_events
|
|
3293
|
+
WHERE traceId IN (SELECT traceId FROM root_scope)
|
|
3294
|
+
),
|
|
3295
|
+
current_span_rows AS (
|
|
3254
3296
|
SELECT
|
|
3255
3297
|
e.*,
|
|
3256
3298
|
coalesce(
|
|
@@ -3262,7 +3304,9 @@ function compileDuckDBTraceScope(relatedCollections, scope, timeRange, rootPredi
|
|
|
3262
3304
|
ORDER BY CASE WHEN e.endedAt IS NULL THEN 1 ELSE 0 END ASC, e.cursorId DESC
|
|
3263
3305
|
) AS currentRank
|
|
3264
3306
|
FROM span_events e
|
|
3265
|
-
INNER JOIN root_scope roots ON roots.traceId = e.traceId
|
|
3307
|
+
INNER JOIN root_scope roots ON roots.traceId = e.traceId
|
|
3308
|
+
WHERE e.timestamp >= (SELECT minTs FROM span_bounds)
|
|
3309
|
+
AND e.timestamp <= (SELECT maxTs FROM span_bounds)${tenantConditions("e").map((condition) => `\n AND ${condition}`).join("")}
|
|
3266
3310
|
),
|
|
3267
3311
|
current_spans AS (
|
|
3268
3312
|
SELECT
|
|
@@ -3666,6 +3710,164 @@ async function queryThreads(db, plan) {
|
|
|
3666
3710
|
}) : null }
|
|
3667
3711
|
});
|
|
3668
3712
|
}
|
|
3713
|
+
/** Compile a span-row filter with exactly the same rules as trace span predicates. */
|
|
3714
|
+
function compileSpanQueryPredicate(predicate) {
|
|
3715
|
+
return compileScalarPredicate(predicate, SPAN_FIELDS);
|
|
3716
|
+
}
|
|
3717
|
+
//#endregion
|
|
3718
|
+
//#region src/storage/domains/observability/span-query.ts
|
|
3719
|
+
const identityFields = [
|
|
3720
|
+
"organizationId",
|
|
3721
|
+
"resourceId",
|
|
3722
|
+
"traceId",
|
|
3723
|
+
"spanId"
|
|
3724
|
+
];
|
|
3725
|
+
const identities = identityFields.join(", ");
|
|
3726
|
+
const fields = [
|
|
3727
|
+
...identityFields,
|
|
3728
|
+
"parentSpanId",
|
|
3729
|
+
"name",
|
|
3730
|
+
"spanType",
|
|
3731
|
+
"entityType",
|
|
3732
|
+
"entityId",
|
|
3733
|
+
"entityName",
|
|
3734
|
+
"entityVersionId",
|
|
3735
|
+
"parentEntityVersionId",
|
|
3736
|
+
"rootEntityVersionId",
|
|
3737
|
+
"runId",
|
|
3738
|
+
"sessionId",
|
|
3739
|
+
"userId"
|
|
3740
|
+
];
|
|
3741
|
+
function keys(rows, values, alias = "") {
|
|
3742
|
+
return rows.map((row) => `(${identityFields.map((field) => {
|
|
3743
|
+
values.push(row[field]);
|
|
3744
|
+
return `${alias}${field} IS NOT DISTINCT FROM ?`;
|
|
3745
|
+
}).join(" AND ")})`).join(" OR ");
|
|
3746
|
+
}
|
|
3747
|
+
function afterCursor(plan) {
|
|
3748
|
+
if (!plan.cursor) return {
|
|
3749
|
+
sql: "TRUE",
|
|
3750
|
+
values: []
|
|
3751
|
+
};
|
|
3752
|
+
const cursor = plan.cursor;
|
|
3753
|
+
const comparisons = [`s.${plan.orderBy.field} ${plan.orderBy.direction === "asc" ? ">" : "<"} CAST(? AS TIMESTAMP)`];
|
|
3754
|
+
const values = [cursor.sortValue];
|
|
3755
|
+
for (let i = 0; i < identityFields.length; i++) {
|
|
3756
|
+
const field = identityFields[i];
|
|
3757
|
+
if (cursor[field] === null) continue;
|
|
3758
|
+
const equal = [`s.${plan.orderBy.field} = CAST(? AS TIMESTAMP)`];
|
|
3759
|
+
values.push(cursor.sortValue);
|
|
3760
|
+
for (const prior of identityFields.slice(0, i)) {
|
|
3761
|
+
equal.push(`s.${prior} IS NOT DISTINCT FROM ?`);
|
|
3762
|
+
values.push(cursor[prior]);
|
|
3763
|
+
}
|
|
3764
|
+
values.push(cursor[field]);
|
|
3765
|
+
comparisons.push(`(${equal.join(" AND ")} AND (s.${field} > ? OR s.${field} IS NULL))`);
|
|
3766
|
+
}
|
|
3767
|
+
return {
|
|
3768
|
+
sql: `(${comparisons.join(" OR ")})`,
|
|
3769
|
+
values
|
|
3770
|
+
};
|
|
3771
|
+
}
|
|
3772
|
+
function compileDuckDBSpanQuery(plan) {
|
|
3773
|
+
const values = [plan.timeRange.from, plan.timeRange.to];
|
|
3774
|
+
const scope = [];
|
|
3775
|
+
if (plan.scope) {
|
|
3776
|
+
scope.push("organizationId = ?");
|
|
3777
|
+
values.push(plan.scope.organizationId);
|
|
3778
|
+
if (plan.scope.resourceId !== void 0) {
|
|
3779
|
+
scope.push("resourceId = ?");
|
|
3780
|
+
values.push(plan.scope.resourceId);
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
const predicate = plan.where ? compileSpanQueryPredicate(plan.where) : {
|
|
3784
|
+
sql: "TRUE",
|
|
3785
|
+
values: []
|
|
3786
|
+
};
|
|
3787
|
+
const after = afterCursor(plan);
|
|
3788
|
+
values.push(plan.timeRange.from, plan.timeRange.to, ...predicate.values, ...after.values, plan.limit + 1);
|
|
3789
|
+
const join = identityFields.map((field) => `e.${field} IS NOT DISTINCT FROM c.${field}`).join(" AND ");
|
|
3790
|
+
return {
|
|
3791
|
+
sql: `WITH candidates AS MATERIALIZED (
|
|
3792
|
+
SELECT ${identities} FROM span_events
|
|
3793
|
+
WHERE eventType = 'start' AND timestamp >= CAST(? AS TIMESTAMP) AND timestamp < CAST(? AS TIMESTAMP)
|
|
3794
|
+
${scope.length ? `AND ${scope.join(" AND ")}` : ""}
|
|
3795
|
+
GROUP BY ${identities}
|
|
3796
|
+
), ranked AS (
|
|
3797
|
+
SELECT ${fields.map((field) => `e.${field}`).join(", ")},
|
|
3798
|
+
min(e.timestamp) FILTER (WHERE e.eventType = 'start') OVER (PARTITION BY ${identityFields.map((field) => `e.${field}`).join(", ")}) AS startedAt, e.endedAt, e.cursorId,
|
|
3799
|
+
CASE WHEN json_type(e.attributes, '$.model') = 'VARCHAR' THEN json_extract_string(e.attributes, '$.model') END AS model,
|
|
3800
|
+
CASE WHEN json_type(e.attributes, '$.provider') = 'VARCHAR' THEN json_extract_string(e.attributes, '$.provider') END AS provider,
|
|
3801
|
+
date_diff('millisecond', min(e.timestamp) FILTER (WHERE e.eventType = 'start') OVER (PARTITION BY ${identityFields.map((field) => `e.${field}`).join(", ")}), e.endedAt) AS durationMs,
|
|
3802
|
+
CASE WHEN e.error IS NULL THEN 'success' ELSE 'error' END AS status,
|
|
3803
|
+
CASE WHEN e.error IS NULL THEN NULL ELSE 'error' END AS error,
|
|
3804
|
+
row_number() OVER (PARTITION BY ${identityFields.map((field) => `e.${field}`).join(", ")}
|
|
3805
|
+
ORDER BY CASE WHEN e.endedAt IS NULL THEN 1 ELSE 0 END, e.cursorId DESC NULLS LAST, e.endedAt DESC) AS rank
|
|
3806
|
+
FROM span_events e INNER JOIN candidates c ON ${join}
|
|
3807
|
+
)
|
|
3808
|
+
SELECT ${[
|
|
3809
|
+
...identityFields,
|
|
3810
|
+
"parentSpanId",
|
|
3811
|
+
"name",
|
|
3812
|
+
"spanType",
|
|
3813
|
+
"entityType",
|
|
3814
|
+
"entityId",
|
|
3815
|
+
"entityName",
|
|
3816
|
+
"model",
|
|
3817
|
+
"provider",
|
|
3818
|
+
"startedAt",
|
|
3819
|
+
"endedAt",
|
|
3820
|
+
"durationMs",
|
|
3821
|
+
"status",
|
|
3822
|
+
"cursorId"
|
|
3823
|
+
].map((field) => `s.${field}`).join(", ")}
|
|
3824
|
+
FROM ranked s WHERE rank = 1 AND endedAt IS NOT NULL AND startedAt >= CAST(? AS TIMESTAMP) AND startedAt < CAST(? AS TIMESTAMP) AND (${predicate.sql}) AND ${after.sql}
|
|
3825
|
+
ORDER BY s.${plan.orderBy.field} ${plan.orderBy.direction === "asc" ? "ASC" : "DESC"}, ${identityFields.map((field) => `s.${field} ASC NULLS LAST`).join(", ")}
|
|
3826
|
+
LIMIT ?`,
|
|
3827
|
+
values
|
|
3828
|
+
};
|
|
3829
|
+
}
|
|
3830
|
+
async function querySpans(db, plan) {
|
|
3831
|
+
const deadline = performance.now() + coreStorage.TRACE_QUERY_DEFAULT_TIMEOUT_MS;
|
|
3832
|
+
const query = async (sql, values) => {
|
|
3833
|
+
const remaining = Math.floor(deadline - performance.now());
|
|
3834
|
+
if (remaining <= 0) throw new coreStorage.TraceQueryExecutionError();
|
|
3835
|
+
try {
|
|
3836
|
+
return await db.query(sql, values, remaining);
|
|
3837
|
+
} catch (error) {
|
|
3838
|
+
if (error instanceof DuckDBQueryTimeoutError) throw new coreStorage.TraceQueryExecutionError();
|
|
3839
|
+
if (error instanceof Error && /out of memory/i.test(error.message)) throw new coreStorage.TraceQueryResourceLimitError();
|
|
3840
|
+
throw error;
|
|
3841
|
+
}
|
|
3842
|
+
};
|
|
3843
|
+
const compiled = compileDuckDBSpanQuery(plan);
|
|
3844
|
+
const rows = await query(compiled.sql, compiled.values);
|
|
3845
|
+
const selected = rows.map(({ cursorId: _cursorId, startedAt, endedAt, ...row }) => ({
|
|
3846
|
+
...row,
|
|
3847
|
+
startedAt: startedAt.toISOString(),
|
|
3848
|
+
endedAt: endedAt.toISOString()
|
|
3849
|
+
}));
|
|
3850
|
+
const page = rows.slice(0, plan.limit);
|
|
3851
|
+
if (!page.length) return coreStorage.buildSpanQueryResponse(plan, selected, [], []);
|
|
3852
|
+
const payloadValues = [];
|
|
3853
|
+
const payloadWhere = page.map((row) => {
|
|
3854
|
+
const key = keys([row], payloadValues);
|
|
3855
|
+
payloadValues.push(row.cursorId, row.endedAt);
|
|
3856
|
+
return `(${key} AND cursorId IS NOT DISTINCT FROM ? AND endedAt = ?)`;
|
|
3857
|
+
}).join(" OR ");
|
|
3858
|
+
const payloads = await query(`SELECT DISTINCT ${identities},
|
|
3859
|
+
substring(CAST(${payloadColumnSql("input")} AS VARCHAR), 1, ${coreStorage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS inputPreview, substring(CAST(${payloadColumnSql("output")} AS VARCHAR), 1, ${coreStorage.SPAN_QUERY_MAX_PREVIEW_CHARACTERS + 1}) AS outputPreview
|
|
3860
|
+
FROM span_events WHERE ${payloadWhere}`, payloadValues);
|
|
3861
|
+
const metricValues = [];
|
|
3862
|
+
const metricWhere = keys(page, metricValues);
|
|
3863
|
+
const metrics = await query(`SELECT ${identities}, name, estimatedCost, costUnit,
|
|
3864
|
+
json_extract_string(costMetadata, '$.allocation') AS allocation,
|
|
3865
|
+
COALESCE(json_exists(costMetadata, '$.error'), false) AS costError
|
|
3866
|
+
FROM metric_events WHERE (${metricWhere}) AND name LIKE 'mastra_model_%'
|
|
3867
|
+
LIMIT ${coreStorage.SPAN_QUERY_MAX_COST_METRICS + 1}`, metricValues);
|
|
3868
|
+
if (metrics.length > coreStorage.SPAN_QUERY_MAX_COST_METRICS) throw new coreStorage.TraceQueryResourceLimitError();
|
|
3869
|
+
return coreStorage.buildSpanQueryResponse(plan, selected, payloads, metrics);
|
|
3870
|
+
}
|
|
3669
3871
|
//#endregion
|
|
3670
3872
|
//#region src/storage/domains/observability/trace-aggregate.ts
|
|
3671
3873
|
/** Trace-root column expressions (alias `r`) for each groupable dimension; same values `where` sees. */
|
|
@@ -3909,13 +4111,15 @@ const COLUMNS_SQL = [
|
|
|
3909
4111
|
"error",
|
|
3910
4112
|
"requestContext"
|
|
3911
4113
|
].join(", ");
|
|
4114
|
+
const PAYLOAD_COLUMN_SET = new Set(SPAN_PAYLOAD_COLUMNS);
|
|
3912
4115
|
/**
|
|
3913
4116
|
* Reconstruction query uses `arg_max(field, timestamp) FILTER (WHERE field IS NOT NULL)`
|
|
3914
4117
|
* so that the final end event supplies the terminal span fields without wiping
|
|
3915
|
-
* stable values emitted on the start event.
|
|
4118
|
+
* stable values emitted on the start event. Payload columns also skip the JSON
|
|
4119
|
+
* `null` placeholder, which stands in for a missing value.
|
|
3916
4120
|
*/
|
|
3917
4121
|
function argMaxNonNull(col) {
|
|
3918
|
-
return `arg_max(${col}, timestamp) FILTER (WHERE ${col} IS NOT NULL) as ${col}`;
|
|
4122
|
+
return `arg_max(${col}, timestamp) FILTER (WHERE ${col} IS NOT NULL${PAYLOAD_COLUMN_SET.has(col) ? ` AND ${col} <> '${JSON_NULL_PLACEHOLDER}'` : ""}) as ${col}`;
|
|
3919
4123
|
}
|
|
3920
4124
|
const SPAN_RECONSTRUCT_SELECT = `
|
|
3921
4125
|
SELECT
|
|
@@ -4026,28 +4230,55 @@ function buildPostAggReconstructSelect(postAgg, orderByField) {
|
|
|
4026
4230
|
`;
|
|
4027
4231
|
}
|
|
4028
4232
|
/**
|
|
4029
|
-
* Reconstruct spans
|
|
4030
|
-
*
|
|
4233
|
+
* Reconstruct the spans of one list page, reading payload columns only for
|
|
4234
|
+
* the page's own events.
|
|
4031
4235
|
*
|
|
4032
|
-
*
|
|
4033
|
-
*
|
|
4034
|
-
*
|
|
4035
|
-
*
|
|
4236
|
+
* Two queries:
|
|
4237
|
+
* 1. `anchorSql` (a `WITH ...` prefix defining `anchorCte`) picks the page's
|
|
4238
|
+
* `(traceId, spanId)` anchors, and a narrow pass over `traceId`, `spanId`
|
|
4239
|
+
* and `timestamp` finds the exact time range of their events.
|
|
4240
|
+
* 2. Reconstruction scans only that range for exactly those anchors.
|
|
4036
4241
|
*
|
|
4037
|
-
*
|
|
4038
|
-
*
|
|
4039
|
-
*
|
|
4040
|
-
*
|
|
4242
|
+
* The `(traceId, spanId) IN (...)` semi-join cannot be pushed into the table
|
|
4243
|
+
* scan, so without a range DuckDB decompresses every payload column of every
|
|
4244
|
+
* row it reads. A lower bound alone (the earliest anchor) prunes nothing for
|
|
4245
|
+
* the oldest pages, which then read the whole table and can run out of memory.
|
|
4246
|
+
* Both bounds are passed as literal parameters so they always reach the scan
|
|
4247
|
+
* as plain range filters that zone maps can prune on.
|
|
4248
|
+
*
|
|
4249
|
+
* Anchors and bounds are frozen after the first query, so spans written in
|
|
4250
|
+
* between cannot change which rows are on the page. An event a page span
|
|
4251
|
+
* receives in between is left for the next read, as if the page had been
|
|
4252
|
+
* read a moment earlier.
|
|
4041
4253
|
*/
|
|
4042
|
-
function
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
|
|
4046
|
-
|
|
4047
|
-
|
|
4254
|
+
async function reconstructAnchoredPage(db, anchorSql, anchorCte, params, reconstructSelect, orderBySql) {
|
|
4255
|
+
const anchors = await db.query(`
|
|
4256
|
+
${anchorSql}
|
|
4257
|
+
SELECT anchors.traceId, anchors.spanId, bounds.minTs, bounds.maxTs
|
|
4258
|
+
FROM ${anchorCte} AS anchors,
|
|
4259
|
+
(
|
|
4260
|
+
SELECT min(timestamp) AS minTs, max(timestamp) AS maxTs
|
|
4261
|
+
FROM span_events
|
|
4262
|
+
WHERE (traceId, spanId) IN (SELECT traceId, spanId FROM ${anchorCte})
|
|
4263
|
+
) AS bounds
|
|
4264
|
+
`, params);
|
|
4265
|
+
if (anchors.length === 0) return [];
|
|
4266
|
+
const { minTs, maxTs } = anchors[0];
|
|
4267
|
+
const anchorValues = anchors.map(() => "(?, ?)").join(", ");
|
|
4268
|
+
return db.query(`
|
|
4269
|
+
${reconstructSelect}
|
|
4270
|
+
WHERE timestamp >= ? AND timestamp <= ?
|
|
4271
|
+
AND (traceId, spanId) IN (SELECT * FROM (VALUES ${anchorValues}))
|
|
4272
|
+
GROUP BY traceId, spanId
|
|
4273
|
+
${orderBySql}
|
|
4274
|
+
`, [
|
|
4275
|
+
minTs,
|
|
4276
|
+
maxTs,
|
|
4277
|
+
...anchors.flatMap((anchor) => [anchor.traceId, anchor.spanId])
|
|
4278
|
+
]);
|
|
4048
4279
|
}
|
|
4049
4280
|
/**
|
|
4050
|
-
* Same time-bound trick as {@link
|
|
4281
|
+
* Same time-bound trick as {@link reconstructAnchoredPage}, but with the bound
|
|
4051
4282
|
* precomputed in JS and passed as a `?` parameter. The delta-poll query shape
|
|
4052
4283
|
* references its candidate CTE multiple times, which makes DuckDB materialize
|
|
4053
4284
|
* it and lose the dynamic-filter pushdown a scalar subquery bound relies on —
|
|
@@ -4281,15 +4512,15 @@ function toValuesTuple(row) {
|
|
|
4281
4512
|
v(row.environment),
|
|
4282
4513
|
v(row.source),
|
|
4283
4514
|
v(row.serviceName),
|
|
4284
|
-
|
|
4515
|
+
payloadJsonV(row.attributes),
|
|
4285
4516
|
jsonV(row.metadata),
|
|
4286
4517
|
jsonV(row.tags),
|
|
4287
4518
|
jsonV(row.scope),
|
|
4288
4519
|
jsonV(row.links),
|
|
4289
|
-
|
|
4290
|
-
|
|
4520
|
+
payloadJsonV(row.input),
|
|
4521
|
+
payloadJsonV(row.output),
|
|
4291
4522
|
jsonV(row.error),
|
|
4292
|
-
|
|
4523
|
+
payloadJsonV(row.requestContext)
|
|
4293
4524
|
].join(", ");
|
|
4294
4525
|
}
|
|
4295
4526
|
async function insertSpanEvents(db, rows) {
|
|
@@ -4503,7 +4734,7 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
|
|
|
4503
4734
|
},
|
|
4504
4735
|
spans: []
|
|
4505
4736
|
};
|
|
4506
|
-
const
|
|
4737
|
+
const spans = (await reconstructAnchoredPage(db, `
|
|
4507
4738
|
WITH page_roots AS (
|
|
4508
4739
|
SELECT traceId, spanId, min(timestamp) AS anchorStartedAt
|
|
4509
4740
|
FROM span_events AS ${outerAlias}
|
|
@@ -4512,14 +4743,11 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
|
|
|
4512
4743
|
${prefilterOrderBy}
|
|
4513
4744
|
LIMIT ? OFFSET ?
|
|
4514
4745
|
)
|
|
4515
|
-
|
|
4516
|
-
${buildOrderByClause(orderBy)}, traceId, spanId
|
|
4517
|
-
`;
|
|
4518
|
-
const spans = (await db.query(pageSql, [
|
|
4746
|
+
`, "page_roots", [
|
|
4519
4747
|
...prefilterParams,
|
|
4520
4748
|
perPage,
|
|
4521
4749
|
offset
|
|
4522
|
-
])).map((row) => mapRow(row));
|
|
4750
|
+
], reconstructSelect, `${buildOrderByClause(orderBy)}, traceId, spanId`)).map((row) => mapRow(row));
|
|
4523
4751
|
return {
|
|
4524
4752
|
pagination: {
|
|
4525
4753
|
total,
|
|
@@ -4568,20 +4796,17 @@ async function listTraceRows(db, args, reconstructSelect, mapRow, toSpans) {
|
|
|
4568
4796
|
},
|
|
4569
4797
|
spans: []
|
|
4570
4798
|
};
|
|
4571
|
-
const
|
|
4799
|
+
const spans = (await reconstructAnchoredPage(db, `
|
|
4572
4800
|
${cteSql},
|
|
4573
4801
|
page_roots AS (
|
|
4574
4802
|
SELECT traceId, spanId, startedAt AS anchorStartedAt
|
|
4575
4803
|
FROM root_spans ${postAggWhere} ${orderByClause} ${paginationClause}
|
|
4576
4804
|
)
|
|
4577
|
-
|
|
4578
|
-
${orderByClause}
|
|
4579
|
-
`;
|
|
4580
|
-
const spans = (await db.query(dataSql, [
|
|
4805
|
+
`, "page_roots", [
|
|
4581
4806
|
...prefilterParams,
|
|
4582
4807
|
...postAggParams,
|
|
4583
4808
|
...paginationParams
|
|
4584
|
-
])).map((row) => mapRow(row));
|
|
4809
|
+
], reconstructSelect, orderByClause)).map((row) => mapRow(row));
|
|
4585
4810
|
return {
|
|
4586
4811
|
pagination: {
|
|
4587
4812
|
total,
|
|
@@ -4885,7 +5110,7 @@ async function listBranches(db, args) {
|
|
|
4885
5110
|
branches: [],
|
|
4886
5111
|
...deltaPollingFeatureEnabled() ? { deltaCursor: currentDeltaCursor } : {}
|
|
4887
5112
|
};
|
|
4888
|
-
const
|
|
5113
|
+
const spans = (await reconstructAnchoredPage(db, `
|
|
4889
5114
|
WITH page_anchors AS (
|
|
4890
5115
|
SELECT traceId, spanId, min(timestamp) AS anchorStartedAt
|
|
4891
5116
|
FROM span_events AS ${outerAlias}
|
|
@@ -4894,14 +5119,11 @@ async function listBranches(db, args) {
|
|
|
4894
5119
|
${prefilterOrderBy}
|
|
4895
5120
|
LIMIT ? OFFSET ?
|
|
4896
5121
|
)
|
|
4897
|
-
|
|
4898
|
-
${buildOrderByClause(orderBy)}, traceId, spanId
|
|
4899
|
-
`;
|
|
4900
|
-
const spans = (await db.query(pageSql, [
|
|
5122
|
+
`, "page_anchors", [
|
|
4901
5123
|
...prefilterParams,
|
|
4902
5124
|
perPage,
|
|
4903
5125
|
offset
|
|
4904
|
-
])).map((row) => rowToSpanRecord(row));
|
|
5126
|
+
], SPAN_RECONSTRUCT_SELECT, `${buildOrderByClause(orderBy)}, traceId, spanId`)).map((row) => rowToSpanRecord(row));
|
|
4905
5127
|
return {
|
|
4906
5128
|
pagination: {
|
|
4907
5129
|
total,
|
|
@@ -4948,20 +5170,17 @@ async function listBranches(db, args) {
|
|
|
4948
5170
|
branches: [],
|
|
4949
5171
|
...deltaPollingFeatureEnabled() ? { deltaCursor: currentDeltaCursor } : {}
|
|
4950
5172
|
};
|
|
4951
|
-
const
|
|
5173
|
+
const spans = (await reconstructAnchoredPage(db, `
|
|
4952
5174
|
${cteSql},
|
|
4953
5175
|
page_anchors AS (
|
|
4954
5176
|
SELECT traceId, spanId, startedAt AS anchorStartedAt
|
|
4955
5177
|
FROM branch_anchors ${postAggWhere} ${orderByClause} ${paginationClause}
|
|
4956
5178
|
)
|
|
4957
|
-
|
|
4958
|
-
${orderByClause}
|
|
4959
|
-
`;
|
|
4960
|
-
const spans = (await db.query(dataSql, [
|
|
5179
|
+
`, "page_anchors", [
|
|
4961
5180
|
...prefilterParams,
|
|
4962
5181
|
...postAggParams,
|
|
4963
5182
|
...paginationParams
|
|
4964
|
-
])).map((row) => rowToSpanRecord(row));
|
|
5183
|
+
], SPAN_RECONSTRUCT_SELECT, orderByClause)).map((row) => rowToSpanRecord(row));
|
|
4965
5184
|
return {
|
|
4966
5185
|
pagination: {
|
|
4967
5186
|
total,
|
|
@@ -5198,6 +5417,7 @@ var ObservabilityStorageDuckDB = class ObservabilityStorageDuckDB extends Observ
|
|
|
5198
5417
|
"metric-discovery",
|
|
5199
5418
|
"trace-query",
|
|
5200
5419
|
"trace-aggregate",
|
|
5420
|
+
...spanQueryFeatures,
|
|
5201
5421
|
"trace-query-root-duration",
|
|
5202
5422
|
"trace-query-discovery",
|
|
5203
5423
|
"thread-query",
|
|
@@ -5217,6 +5437,7 @@ var ObservabilityStorageDuckDB = class ObservabilityStorageDuckDB extends Observ
|
|
|
5217
5437
|
"delta-polling",
|
|
5218
5438
|
"trace-query",
|
|
5219
5439
|
"trace-aggregate",
|
|
5440
|
+
...spanQueryFeatures,
|
|
5220
5441
|
"trace-query-root-duration",
|
|
5221
5442
|
"trace-query-discovery",
|
|
5222
5443
|
"thread-query",
|
|
@@ -5252,6 +5473,9 @@ var ObservabilityStorageDuckDB = class ObservabilityStorageDuckDB extends Observ
|
|
|
5252
5473
|
async listTraces(args) {
|
|
5253
5474
|
return listTraces(this.db, args);
|
|
5254
5475
|
}
|
|
5476
|
+
async querySpans(plan) {
|
|
5477
|
+
return querySpans(this.db, plan);
|
|
5478
|
+
}
|
|
5255
5479
|
async queryTraces(plan) {
|
|
5256
5480
|
return queryTraces(this.db, plan);
|
|
5257
5481
|
}
|
|
@@ -5380,4 +5604,4 @@ var ObservabilityStorageDuckDB = class ObservabilityStorageDuckDB extends Observ
|
|
|
5380
5604
|
//#endregion
|
|
5381
5605
|
export { ObservabilityStorageDuckDB };
|
|
5382
5606
|
|
|
5383
|
-
//# sourceMappingURL=observability-
|
|
5607
|
+
//# sourceMappingURL=observability-DINJlR_a.js.map
|