@mastra/clickhouse 1.26.0-alpha.4 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +94 -70
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +93 -69
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/thread-query.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-aggregate.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +41 -15
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -1
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -1587,6 +1587,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
|
1587
1587
|
metadataIndex++;
|
|
1588
1588
|
}
|
|
1589
1589
|
}
|
|
1590
|
+
const idScope = filter?.resourceId ? `WHERE id IN (SELECT id FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String})` : "";
|
|
1590
1591
|
const latestThreads = `
|
|
1591
1592
|
latest_threads AS (
|
|
1592
1593
|
SELECT
|
|
@@ -1597,7 +1598,7 @@ var MemoryStorageClickhouse = class extends MemoryStorage {
|
|
|
1597
1598
|
toDateTime64(createdAt, 3) as createdAt,
|
|
1598
1599
|
toDateTime64(updatedAt, 3) as updatedAt
|
|
1599
1600
|
FROM ${TABLE_THREADS}
|
|
1600
|
-
${
|
|
1601
|
+
${idScope}
|
|
1601
1602
|
ORDER BY updatedAt DESC
|
|
1602
1603
|
LIMIT 1 BY id
|
|
1603
1604
|
)`;
|
|
@@ -2418,9 +2419,7 @@ var ObservabilityStorageClickhouse = class extends ObservabilityStorage {
|
|
|
2418
2419
|
case TraceStatus.RUNNING:
|
|
2419
2420
|
conditions.push(`endedAt IS NULL AND (error IS NULL OR error = '')`);
|
|
2420
2421
|
break;
|
|
2421
|
-
case TraceStatus.SUCCESS:
|
|
2422
|
-
conditions.push(`endedAt IS NOT NULL AND (error IS NULL OR error = '')`);
|
|
2423
|
-
break;
|
|
2422
|
+
case TraceStatus.SUCCESS: conditions.push(`endedAt IS NOT NULL AND (error IS NULL OR error = '')`);
|
|
2424
2423
|
}
|
|
2425
2424
|
if (filters.hasChildError !== void 0) {
|
|
2426
2425
|
const finalClause = (TABLE_ENGINES[TABLE_SPANS] ?? "MergeTree()").startsWith("ReplacingMergeTree") ? "FINAL" : "";
|
|
@@ -5261,7 +5260,8 @@ async function getFeedbackAggregate(client, args) {
|
|
|
5261
5260
|
const aggSql = getAggregationSql$2(args.aggregation);
|
|
5262
5261
|
const identity = buildFeedbackIdentityFilter(args);
|
|
5263
5262
|
const combined = mergeFilters$2(identity, buildFeedbackFilterConditions(args.filters));
|
|
5264
|
-
const
|
|
5263
|
+
const whereClause = toWhereClause$2(combined);
|
|
5264
|
+
const result = await queryJson$2(client, `SELECT ${aggSql} AS value FROM ${TABLE_FEEDBACK_EVENTS} FINAL ${whereClause}`, combined.params);
|
|
5265
5265
|
const value = result[0]?.value == null ? null : Number(result[0]?.value);
|
|
5266
5266
|
if (args.comparePeriod && args.filters?.timestamp) {
|
|
5267
5267
|
const ts = args.filters.timestamp;
|
|
@@ -5295,7 +5295,8 @@ async function getFeedbackAggregate(client, args) {
|
|
|
5295
5295
|
endExclusive: ts.endExclusive
|
|
5296
5296
|
}
|
|
5297
5297
|
}));
|
|
5298
|
-
const
|
|
5298
|
+
const prevWhereClause = toWhereClause$2(prevCombined);
|
|
5299
|
+
const prevResult = await queryJson$2(client, `SELECT ${aggSql} AS value FROM ${TABLE_FEEDBACK_EVENTS} FINAL ${prevWhereClause}`, prevCombined.params);
|
|
5299
5300
|
const previousValue = prevResult[0]?.value == null ? null : Number(prevResult[0]?.value);
|
|
5300
5301
|
let changePercent = null;
|
|
5301
5302
|
if (previousValue !== null && previousValue !== 0 && value !== null) changePercent = (value - previousValue) / Math.abs(previousValue) * 100;
|
|
@@ -6202,8 +6203,10 @@ function buildLegacySpanSelectExprs(legacyColumnTypes) {
|
|
|
6202
6203
|
const promotedIn = PROMOTED_METADATA_KEYS.map((k) => `'${k}'`).join(", ");
|
|
6203
6204
|
const metadataSearchExpr = has("metadata") ? `CAST(arrayFilter(x -> x.1 NOT IN (${promotedIn}) AND trim(BOTH ' ' FROM x.2) != '', arrayMap(x -> (x.1, trim(BOTH ' ' FROM x.2)), JSONExtractKeysAndValues(COALESCE("metadata", '{}'), 'String'))), 'Map(String, String)')` : `CAST(map(), 'Map(String, String)')`;
|
|
6204
6205
|
let tagsExpr = "[]";
|
|
6205
|
-
if (has("tags"))
|
|
6206
|
-
|
|
6206
|
+
if (has("tags")) {
|
|
6207
|
+
if (legacyColumnTypes.get("tags").includes("Array")) tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), "tags")))`;
|
|
6208
|
+
else tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), JSONExtract(COALESCE("tags", '[]'), 'Array(String)'))))`;
|
|
6209
|
+
}
|
|
6207
6210
|
const exprs = {
|
|
6208
6211
|
dedupeKey: `concat("traceId", ':', "spanId")`,
|
|
6209
6212
|
traceId: `"traceId"`,
|
|
@@ -6450,10 +6453,11 @@ function toWhereClause(filter) {
|
|
|
6450
6453
|
return filter.conditions.length ? `WHERE ${filter.conditions.join(" AND ")}` : "";
|
|
6451
6454
|
}
|
|
6452
6455
|
function currentScoresRelation(sourcePredicate) {
|
|
6456
|
+
const whereClause = sourcePredicate ? `WHERE ${sourcePredicate}` : "";
|
|
6453
6457
|
return `(
|
|
6454
6458
|
SELECT *
|
|
6455
6459
|
FROM ${TABLE_SCORE_EVENTS_CURRENT} FINAL
|
|
6456
|
-
${
|
|
6460
|
+
${whereClause}
|
|
6457
6461
|
)`;
|
|
6458
6462
|
}
|
|
6459
6463
|
/**
|
|
@@ -6901,6 +6905,10 @@ const TRACE_FIELDS = {
|
|
|
6901
6905
|
sql: "r.timeToFirstChunkMs",
|
|
6902
6906
|
parameterType: "Float64"
|
|
6903
6907
|
},
|
|
6908
|
+
entityId: {
|
|
6909
|
+
sql: "r.entityId",
|
|
6910
|
+
parameterType: "String"
|
|
6911
|
+
},
|
|
6904
6912
|
entityName: {
|
|
6905
6913
|
sql: "r.entityName",
|
|
6906
6914
|
parameterType: "String"
|
|
@@ -7299,6 +7307,28 @@ function compileTenantScope(scope, parameters) {
|
|
|
7299
7307
|
if (scope.resourceId !== void 0) sql += `\n AND resourceId = ${parameters.add(scope.resourceId, "String")}`;
|
|
7300
7308
|
return sql;
|
|
7301
7309
|
}
|
|
7310
|
+
/**
|
|
7311
|
+
* Order of the versions of one root span, current version first. `mastra_trace_roots` has no
|
|
7312
|
+
* version column, so until a background merge runs, every export of a root is its own row with the
|
|
7313
|
+
* same sort key, and merges never meet versions that ended on different days (it is partitioned by
|
|
7314
|
+
* `endedAt`). The version that ended last is current, as a resumed or completed root is exported
|
|
7315
|
+
* again with a later `endedAt`; a pending copy stores `endedAt = startedAt` and can lack `threadId`,
|
|
7316
|
+
* so a copy with a thread wins an equal-time tie. Remaining ties are broken by insertion order, last
|
|
7317
|
+
* insert first: the part's first block number, then the row's offset in the part. Tied versions
|
|
7318
|
+
* share a partition and sort key, so a merge meets them and keeps the last insert too, and reads
|
|
7319
|
+
* pick the same version before and after it. Both are virtual columns, so the order reads nothing
|
|
7320
|
+
* beyond `endedAt` and `threadId`.
|
|
7321
|
+
*
|
|
7322
|
+
* Merges don't follow this order across `endedAt`: within one day's partition a merge keeps the last
|
|
7323
|
+
* insert, not the version that ended last. If an outdated export of a root is redelivered after its
|
|
7324
|
+
* current version, on the same day, reads pick the current version until a merge, and the outdated
|
|
7325
|
+
* one afterwards (last write wins, as in the PG and DuckDB stores). Making merges keep the version
|
|
7326
|
+
* that ended last needs `ReplacingMergeTree(endedAt)`, a schema change.
|
|
7327
|
+
*
|
|
7328
|
+
* Every root collapse uses this order: trace rows, aggregates, thread rows and the discovery reads.
|
|
7329
|
+
*/
|
|
7330
|
+
const ROOT_VERSION_ORDER = `endedAt DESC, isNotNull(threadId) DESC, toUInt64(splitByChar('_', _part)[2]) DESC,
|
|
7331
|
+
_part_offset DESC`;
|
|
7302
7332
|
function compileClickHouseTraceScope(selection, relationCollections, parameters, scope, seedConjuncts = [], options = {}) {
|
|
7303
7333
|
const from = parameters.add(selection.timeRange.from, "DateTime64(3, 'UTC')");
|
|
7304
7334
|
const to = parameters.add(selection.timeRange.to, "DateTime64(3, 'UTC')");
|
|
@@ -7322,7 +7352,15 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7322
7352
|
SELECT *
|
|
7323
7353
|
FROM ${TABLE_TRACE_ROOTS} r
|
|
7324
7354
|
WHERE startedAt >= ${from}
|
|
7325
|
-
AND startedAt < ${to}${tenant}${
|
|
7355
|
+
AND startedAt < ${to}${tenant}${bound}${seedFilter ? `
|
|
7356
|
+
AND traceId IN (
|
|
7357
|
+
SELECT traceId
|
|
7358
|
+
FROM ${TABLE_TRACE_ROOTS} r
|
|
7359
|
+
WHERE startedAt >= ${from}
|
|
7360
|
+
AND startedAt < ${to}${tenant}${seedFilter}
|
|
7361
|
+
)` : ""}
|
|
7362
|
+
ORDER BY startedAt, traceId, spanId, ${ROOT_VERSION_ORDER}
|
|
7363
|
+
LIMIT 1 BY traceId
|
|
7326
7364
|
)`] : [`current_roots AS (
|
|
7327
7365
|
SELECT *
|
|
7328
7366
|
FROM ${TABLE_TRACE_ROOTS}
|
|
@@ -7332,7 +7370,7 @@ function compileClickHouseTraceScope(selection, relationCollections, parameters,
|
|
|
7332
7370
|
WHERE startedAt >= ${from}
|
|
7333
7371
|
AND startedAt < ${to}${tenant}${seedFilter}
|
|
7334
7372
|
)${tenant}
|
|
7335
|
-
ORDER BY traceId,
|
|
7373
|
+
ORDER BY traceId, startedAt, spanId, ${ROOT_VERSION_ORDER}
|
|
7336
7374
|
LIMIT 1 BY traceId
|
|
7337
7375
|
)`, `root_scope AS (
|
|
7338
7376
|
SELECT *
|
|
@@ -7462,11 +7500,7 @@ function traceOrder(plan) {
|
|
|
7462
7500
|
direction: plan.orderBy.direction === "asc" ? "ASC" : "DESC"
|
|
7463
7501
|
};
|
|
7464
7502
|
}
|
|
7465
|
-
/**
|
|
7466
|
-
* Keyset cursor on a trace's sort value. Applied after each trace is reduced to one sort value:
|
|
7467
|
-
* filtering root rows first would let a trace with several roots (or unmerged copies with
|
|
7468
|
-
* different `endedAt`) come back on a later page under a root that sorts after the cursor.
|
|
7469
|
-
*/
|
|
7503
|
+
/** Keyset cursor on a trace's sort value (the current root's), with traceId breaking ties. */
|
|
7470
7504
|
function keysetCursorCondition(plan, order, parameters) {
|
|
7471
7505
|
if (plan.paginationMode !== "keyset" || !plan.cursor) return "";
|
|
7472
7506
|
if (plan.cursor.sortValue === null) return `(isNull(sortValue) AND traceId > ${parameters.add(plan.cursor.traceId, "String")})`;
|
|
@@ -7477,9 +7511,8 @@ function keysetCursorCondition(plan, order, parameters) {
|
|
|
7477
7511
|
}
|
|
7478
7512
|
/**
|
|
7479
7513
|
* First statement of keyset and page mode: the page's traces (after the keyset cursor, `limit + 1`
|
|
7480
|
-
* for keyset mode, `perPage` at the page offset for page mode) as
|
|
7481
|
-
* number of matching traces on every row.
|
|
7482
|
-
* value and the key, so the key always names a row with that sort value. No blob columns are read.
|
|
7514
|
+
* for keyset mode, `perPage` at the page offset for page mode) as keys of their current roots, and
|
|
7515
|
+
* the number of matching traces on every row. No blob columns are read.
|
|
7483
7516
|
*/
|
|
7484
7517
|
function compileClickHouseTraceQueryKeys(plan) {
|
|
7485
7518
|
if (plan.paginationMode === "delta") throw new Error("Delta trace queries have no keys statement");
|
|
@@ -7487,19 +7520,12 @@ function compileClickHouseTraceQueryKeys(plan) {
|
|
|
7487
7520
|
const order = traceOrder(plan);
|
|
7488
7521
|
const ctes = compileClickHouseTraceCandidates(plan, "r.traceId AS traceId, r.dedupeKey AS dedupeKey, r.startedAt AS startedAt, r.endedAt AS endedAt, r.timeToFirstChunkMs AS timeToFirstChunkMs", parameters, QUERY_TRACES_SCOPE);
|
|
7489
7522
|
const cursor = keysetCursorCondition(plan, order, parameters);
|
|
7490
|
-
const pick = order.direction === "DESC" ? "max" : "min";
|
|
7491
|
-
const rootOrder = order.field === "timeToFirstChunkMs" ? `ifNull(${order.field}, ${pick === "max" ? "-inf" : "inf"})` : order.field;
|
|
7492
7523
|
const limit = plan.paginationMode === "page" ? `LIMIT ${parameters.add(plan.perPage, "UInt64")} OFFSET ${parameters.add(plan.page * plan.perPage, "UInt64")}` : `LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`;
|
|
7493
7524
|
return {
|
|
7494
|
-
query: `WITH ${ctes.join(",\n")}
|
|
7495
|
-
|
|
7496
|
-
|
|
7497
|
-
|
|
7498
|
-
FROM candidates
|
|
7499
|
-
GROUP BY traceId${cursor ? `\n HAVING ${cursor}` : ""}
|
|
7500
|
-
)
|
|
7501
|
-
SELECT traceId, root.1 AS dedupeKey, sortValue, root.2 AS rootStartedAt, count() OVER () AS __total
|
|
7502
|
-
FROM per_trace
|
|
7525
|
+
query: `WITH ${ctes.join(",\n")}
|
|
7526
|
+
SELECT traceId, dedupeKey, ${order.field} AS sortValue, startedAt AS rootStartedAt, endedAt AS rootEndedAt,
|
|
7527
|
+
count() OVER () AS __total
|
|
7528
|
+
FROM candidates${cursor ? `\nWHERE ${cursor}` : ""}
|
|
7503
7529
|
ORDER BY sortValue ${order.direction} NULLS LAST, traceId ASC
|
|
7504
7530
|
${limit}`,
|
|
7505
7531
|
query_params: parameters.params,
|
|
@@ -7507,27 +7533,28 @@ ${limit}`,
|
|
|
7507
7533
|
};
|
|
7508
7534
|
}
|
|
7509
7535
|
/**
|
|
7510
|
-
* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata`
|
|
7511
|
-
*
|
|
7512
|
-
*
|
|
7513
|
-
*
|
|
7536
|
+
* Second statement of keyset and page mode: the rows of the keyed roots, with `metadata` and the
|
|
7537
|
+
* stored previews, read by sort key and `endedAt`, so only the keyed version's rows match. Versions
|
|
7538
|
+
* that end at the same instant are ordered as in the keys statement (ROOT_VERSION_ORDER). The
|
|
7539
|
+
* relation-free `where` conjuncts are re-applied, so a version written between the two statements
|
|
7540
|
+
* that doesn't match isn't returned.
|
|
7541
|
+
*
|
|
7542
|
+
* Trade-off of matching on `endedAt`: if a newer version is written between the two statements and
|
|
7543
|
+
* a same-day merge then drops the keyed version, the trace has no row and drops off the page. Without
|
|
7544
|
+
* `endedAt` the lookup would find the replacement, but could also return a version other than the
|
|
7545
|
+
* keyed one whenever both are unmerged. The window is the gap between the two statements.
|
|
7514
7546
|
*/
|
|
7515
7547
|
function compileClickHouseTraceRowsByKey(plan, keys) {
|
|
7516
7548
|
const parameters = new ParameterBuilder();
|
|
7517
7549
|
const order = traceOrder(plan);
|
|
7518
7550
|
const ts = (value) => parameters.add(value, "DateTime64(3, 'UTC')");
|
|
7519
|
-
const
|
|
7520
|
-
const tuples = keys.map((key) => {
|
|
7521
|
-
const started = ts(key.rootStartedAt);
|
|
7522
|
-
const id = `${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}`;
|
|
7523
|
-
return byEnd ? `(${started}, ${id}, ${ts(String(key.sortValue))})` : `(${started}, ${id})`;
|
|
7524
|
-
});
|
|
7551
|
+
const tuples = keys.map((key) => `(${ts(key.rootStartedAt)}, ${parameters.add(key.traceId, "String")}, ${parameters.add(key.dedupeKey, "String")}, ${ts(key.rootEndedAt)})`);
|
|
7525
7552
|
const predicate = (plan.where ? (plan.where.type === "boolean" && plan.where.operator === "and" ? plan.where.args : [plan.where]).filter((conjunct) => collectRelationCollections(conjunct).size === 0) : []).map((conjunct) => `\n AND (${compilePredicate(conjunct, parameters)})`).join("");
|
|
7526
7553
|
return {
|
|
7527
7554
|
query: `SELECT ${TRACE_SELECT}
|
|
7528
7555
|
FROM ${TABLE_TRACE_ROOTS} r
|
|
7529
|
-
WHERE (r.startedAt, r.traceId, r.dedupeKey
|
|
7530
|
-
ORDER BY ${order.field} ${order.direction} NULLS LAST, traceId ASC
|
|
7556
|
+
WHERE (r.startedAt, r.traceId, r.dedupeKey, r.endedAt) IN (${tuples.join(", ")})${compileTenantScope(plan.scope, parameters)}${predicate}
|
|
7557
|
+
ORDER BY ${order.field} ${order.direction} NULLS LAST, traceId ASC, ${ROOT_VERSION_ORDER}
|
|
7531
7558
|
LIMIT 1 BY traceId`,
|
|
7532
7559
|
query_params: parameters.params
|
|
7533
7560
|
};
|
|
@@ -7546,9 +7573,7 @@ FROM candidates`,
|
|
|
7546
7573
|
/**
|
|
7547
7574
|
* The list statement of groups and delta mode (rows without `metadata` and the previews). Keyset
|
|
7548
7575
|
* and page mode list with compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey.
|
|
7549
|
-
*
|
|
7550
|
-
* mode under the row that sorts first in the list order, as in the keys statement; ties go to the
|
|
7551
|
-
* lowest dedupeKey, then the latest endedAt, so every poll picks the same row.
|
|
7576
|
+
* Candidates hold one current root per trace, as in the keys statement.
|
|
7552
7577
|
*/
|
|
7553
7578
|
function compileClickHouseTraceQuery(plan, deltaHead) {
|
|
7554
7579
|
const parameters = new ParameterBuilder();
|
|
@@ -7569,7 +7594,6 @@ LIMIT ${limit}`,
|
|
|
7569
7594
|
};
|
|
7570
7595
|
}
|
|
7571
7596
|
if (plan.paginationMode !== "delta") throw new Error("Keyset and page trace queries use compileClickHouseTraceQueryKeys and compileClickHouseTraceRowsByKey");
|
|
7572
|
-
const order = traceOrder(plan);
|
|
7573
7597
|
const watermark = coreStorage.getTraceQueryDeltaWatermark(plan, "clickhouse");
|
|
7574
7598
|
const after = watermark ? parseDeltaWatermark(watermark) : {
|
|
7575
7599
|
cursorId: "0",
|
|
@@ -7588,8 +7612,7 @@ LIMIT ${limit}`,
|
|
|
7588
7612
|
SELECT c.*, toString(d.latestCursorId) AS __delta_cursor
|
|
7589
7613
|
FROM candidates c
|
|
7590
7614
|
INNER JOIN delta_candidates d ON c.traceId = d.traceId
|
|
7591
|
-
ORDER BY d.latestCursorId ASC, c.traceId ASC
|
|
7592
|
-
LIMIT 1 BY c.traceId
|
|
7615
|
+
ORDER BY d.latestCursorId ASC, c.traceId ASC
|
|
7593
7616
|
LIMIT ${limit}`,
|
|
7594
7617
|
query_params: parameters.params,
|
|
7595
7618
|
sharedSnapshot: true
|
|
@@ -7633,6 +7656,7 @@ function compileClickHouseTraceRootPayloads(keys, scope, columns = "payload") {
|
|
|
7633
7656
|
query: `SELECT traceId, spanId AS rootSpanId, dedupeKey, organizationId, resourceId, ${columns === "payload" ? `metadataRaw AS metadata, ${PREVIEW_COLUMNS.join(", ")}` : "input, output, error"}
|
|
7634
7657
|
FROM ${TABLE_TRACE_ROOTS}
|
|
7635
7658
|
WHERE (startedAt, traceId, dedupeKey, endedAt, ifNull(organizationId, ''), isNull(organizationId), ifNull(resourceId, ''), isNull(resourceId)) IN (${tuples.join(", ")})${compileTenantScope(scope, parameters)}
|
|
7659
|
+
ORDER BY ${ROOT_VERSION_ORDER}
|
|
7636
7660
|
LIMIT 1 BY traceId, dedupeKey, organizationId, resourceId`,
|
|
7637
7661
|
query_params: parameters.params
|
|
7638
7662
|
};
|
|
@@ -7929,7 +7953,8 @@ async function queryTraces(client, plan, timeoutMs, strategy = null) {
|
|
|
7929
7953
|
traceId: String(key.traceId),
|
|
7930
7954
|
dedupeKey: String(key.dedupeKey),
|
|
7931
7955
|
sortValue: sortValue(key.sortValue),
|
|
7932
|
-
rootStartedAt: asIsoTimestamp(key.rootStartedAt)
|
|
7956
|
+
rootStartedAt: asIsoTimestamp(key.rootStartedAt),
|
|
7957
|
+
rootEndedAt: asIsoTimestamp(key.rootEndedAt)
|
|
7933
7958
|
})))) : [];
|
|
7934
7959
|
if (plan.paginationMode === "page") return coreStorage.traceQueryResponseSchema.parse({
|
|
7935
7960
|
traces: await toTraceRecords(rows),
|
|
@@ -8137,7 +8162,7 @@ async function querySpans(client, plan, timeoutMs) {
|
|
|
8137
8162
|
if (remaining <= 0) throw new coreStorage.TraceQueryExecutionError();
|
|
8138
8163
|
return await runWithClickHouseTraceQueryTimeout(client, {
|
|
8139
8164
|
timeoutMs: remaining,
|
|
8140
|
-
memoryLimitBytes:
|
|
8165
|
+
memoryLimitBytes: 536870912
|
|
8141
8166
|
}, compiled);
|
|
8142
8167
|
};
|
|
8143
8168
|
const selected = (await run(compileClickHouseSpanQuery(plan))).map((row) => ({
|
|
@@ -8254,14 +8279,14 @@ function afterCursor(plan, parameters) {
|
|
|
8254
8279
|
return `(${terms.join(" OR ")})`;
|
|
8255
8280
|
}
|
|
8256
8281
|
const THREAD_QUERY_EXECUTION_LIMITS = {
|
|
8257
|
-
memoryLimitBytes:
|
|
8282
|
+
memoryLimitBytes: 536870912,
|
|
8258
8283
|
maxThreads: 2,
|
|
8259
8284
|
maxBlockSize: 4096,
|
|
8260
|
-
spillThresholdBytes:
|
|
8285
|
+
spillThresholdBytes: 33554432
|
|
8261
8286
|
};
|
|
8262
8287
|
const MAX_CANDIDATE_THREADS = 16384;
|
|
8263
8288
|
const MAX_CANDIDATE_TRACES = 1e5;
|
|
8264
|
-
const MAX_CANDIDATE_ID_BYTES =
|
|
8289
|
+
const MAX_CANDIDATE_ID_BYTES = 4194304;
|
|
8265
8290
|
const MAX_FULL_HISTORY_ROOTS = 25e4;
|
|
8266
8291
|
function canUseThreadCandidates(probe) {
|
|
8267
8292
|
return Number(probe.threadCount) <= MAX_CANDIDATE_THREADS && Number(probe.traceCount) <= MAX_CANDIDATE_TRACES && Number(probe.threadBytes) + Number(probe.traceBytes) <= MAX_CANDIDATE_ID_BYTES;
|
|
@@ -8338,7 +8363,7 @@ function compileClickHouseThreadRowsQuery(plan, strategy = "full") {
|
|
|
8338
8363
|
const sortedRoots = `roots AS (
|
|
8339
8364
|
SELECT organizationId, traceId, ${rootDataFields.map((field) => `${rootExpression(field)} AS ${field}`).join(", ")}
|
|
8340
8365
|
FROM ${TABLE_TRACE_ROOTS} ${scope.length || strategy === "candidates" ? `WHERE ${[...scope, ...strategy === "candidates" ? [candidateTracesFilter] : []].join(" AND ")}` : ""}
|
|
8341
|
-
ORDER BY organizationId, traceId, startedAt ASC, spanId,
|
|
8366
|
+
ORDER BY organizationId, traceId, startedAt ASC, spanId, ${ROOT_VERSION_ORDER}
|
|
8342
8367
|
LIMIT 1 BY organizationId, traceId
|
|
8343
8368
|
)`;
|
|
8344
8369
|
return {
|
|
@@ -8371,7 +8396,7 @@ function* batches(rows, maxRows = 256) {
|
|
|
8371
8396
|
let bytes = 0;
|
|
8372
8397
|
for (const row of rows) {
|
|
8373
8398
|
const size = Buffer.byteLength(JSON.stringify(row), "utf8") * 2;
|
|
8374
|
-
if (batch.length && (batch.length >= maxRows || bytes + size >
|
|
8399
|
+
if (batch.length && (batch.length >= maxRows || bytes + size > 49152)) {
|
|
8375
8400
|
yield batch;
|
|
8376
8401
|
batch = [];
|
|
8377
8402
|
bytes = 0;
|
|
@@ -8440,7 +8465,7 @@ async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
|
|
|
8440
8465
|
${contextFields.join(", ")}, if(spanType = 'agent_run', timeToFirstChunkMs, NULL) AS timeToFirstChunkMs,
|
|
8441
8466
|
if(isNull(error), 'success', 'error') AS status, ${PREVIEW_COLUMNS.join(", ")}
|
|
8442
8467
|
FROM ${TABLE_TRACE_ROOTS} WHERE ${rootWhere}
|
|
8443
|
-
ORDER BY
|
|
8468
|
+
ORDER BY ${ROOT_VERSION_ORDER} LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8444
8469
|
query_params: { roots: keys(batch) }
|
|
8445
8470
|
}));
|
|
8446
8471
|
const byRoot = new Map(stored.map((row) => [rootKey(row), row]));
|
|
@@ -8448,7 +8473,7 @@ async function queryThreadRows(client, plan, timeoutMs, strategy = "auto") {
|
|
|
8448
8473
|
for (const batch of batches(missing, 32)) {
|
|
8449
8474
|
const payloads = await run({
|
|
8450
8475
|
query: `SELECT organizationId, traceId, spanId, input, output, error FROM ${TABLE_TRACE_ROOTS}
|
|
8451
|
-
WHERE ${rootWhere} ORDER BY
|
|
8476
|
+
WHERE ${rootWhere} ORDER BY ${ROOT_VERSION_ORDER} LIMIT 1 BY organizationId, traceId, spanId`,
|
|
8452
8477
|
query_params: { roots: keys(batch) }
|
|
8453
8478
|
});
|
|
8454
8479
|
for (const payload of payloads) {
|
|
@@ -8517,6 +8542,7 @@ const DIMENSION_SQL = {
|
|
|
8517
8542
|
entityName: "r.entityName",
|
|
8518
8543
|
name: "r.name",
|
|
8519
8544
|
spanType: "r.spanType",
|
|
8545
|
+
entityId: "r.entityId",
|
|
8520
8546
|
environment: "r.environment",
|
|
8521
8547
|
status: TRACE_STATUS_SQL,
|
|
8522
8548
|
serviceName: "r.serviceName",
|
|
@@ -8555,12 +8581,6 @@ const COMPARISON_SQL = {
|
|
|
8555
8581
|
function compileClickHouseTraceAggregate(plan) {
|
|
8556
8582
|
const parameters = new ParameterBuilder();
|
|
8557
8583
|
const ctes = compileClickHouseTraceCandidates(plan, "*", parameters, { oneRootPerTrace: true });
|
|
8558
|
-
ctes.push(`trace_roots AS (
|
|
8559
|
-
SELECT *
|
|
8560
|
-
FROM candidates
|
|
8561
|
-
ORDER BY traceId, dedupeKey
|
|
8562
|
-
LIMIT 1 BY traceId
|
|
8563
|
-
)`);
|
|
8564
8584
|
const dimensionSql = (field) => {
|
|
8565
8585
|
if (field.startsWith("metadata.")) return traceMetadataValueSql(parameters.add(field.slice(9), "String"));
|
|
8566
8586
|
const sql = Object.hasOwn(DIMENSION_SQL, field) ? DIMENSION_SQL[field] : void 0;
|
|
@@ -8594,7 +8614,7 @@ function compileClickHouseTraceAggregate(plan) {
|
|
|
8594
8614
|
}
|
|
8595
8615
|
ctes.push(`facts AS (
|
|
8596
8616
|
SELECT ${factColumns.join(",\n ")}
|
|
8597
|
-
FROM
|
|
8617
|
+
FROM candidates r${usageJoin}
|
|
8598
8618
|
)`);
|
|
8599
8619
|
const mixedUnitsSql = "minOrNull(unitMin) != maxOrNull(unitMax)";
|
|
8600
8620
|
const measureSql = (name) => {
|
|
@@ -9946,7 +9966,7 @@ async function detectExistingDeltaCursorStrategy(client) {
|
|
|
9946
9966
|
}
|
|
9947
9967
|
}
|
|
9948
9968
|
const TRACE_QUERY_DISCOVERY_DEFAULT_TIMEOUT_MS = 5e3;
|
|
9949
|
-
const TRACE_QUERY_DISCOVERY_DEFAULT_MEMORY_LIMIT_BYTES =
|
|
9969
|
+
const TRACE_QUERY_DISCOVERY_DEFAULT_MEMORY_LIMIT_BYTES = 268435456;
|
|
9950
9970
|
function resolveTraceQueryDiscoveryMemoryLimitBytes(memoryLimitBytes = TRACE_QUERY_DISCOVERY_DEFAULT_MEMORY_LIMIT_BYTES) {
|
|
9951
9971
|
if (!Number.isSafeInteger(memoryLimitBytes) || memoryLimitBytes <= 0) throw new RangeError("traceQuery.discovery.memoryLimitBytes must be a positive safe integer");
|
|
9952
9972
|
return memoryLimitBytes;
|
|
@@ -10100,6 +10120,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
10100
10120
|
"trace-query-context-ids",
|
|
10101
10121
|
...timeToFirstChunkFeatures,
|
|
10102
10122
|
"trace-query-root-span-fields",
|
|
10123
|
+
"trace-query-root-entity-id",
|
|
10103
10124
|
"trace-aggregate",
|
|
10104
10125
|
...spanQueryFeatures,
|
|
10105
10126
|
...threadQueryFeatures
|
|
@@ -10123,6 +10144,7 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
|
|
|
10123
10144
|
"trace-query-context-ids",
|
|
10124
10145
|
...timeToFirstChunkFeatures,
|
|
10125
10146
|
"trace-query-root-span-fields",
|
|
10147
|
+
"trace-query-root-entity-id",
|
|
10126
10148
|
"trace-aggregate",
|
|
10127
10149
|
...spanQueryFeatures,
|
|
10128
10150
|
...threadQueryFeatures
|
|
@@ -11337,10 +11359,12 @@ var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
|
|
|
11337
11359
|
conditions.push(`JSONExtractString(snapshot, 'status') = {var_status:String}`);
|
|
11338
11360
|
values.var_status = status;
|
|
11339
11361
|
}
|
|
11340
|
-
if (resourceId)
|
|
11341
|
-
|
|
11342
|
-
|
|
11343
|
-
|
|
11362
|
+
if (resourceId) {
|
|
11363
|
+
if (await this.#db.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId")) {
|
|
11364
|
+
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
11365
|
+
values.var_resourceId = resourceId;
|
|
11366
|
+
} else this.logger.warn(`[${TABLE_WORKFLOW_SNAPSHOT}] resourceId column not found. Skipping resourceId filter.`);
|
|
11367
|
+
}
|
|
11344
11368
|
if (fromDate) {
|
|
11345
11369
|
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
11346
11370
|
values.var_from_date = fromDate.getTime() / 1e3;
|