@mastra/clickhouse 1.16.1 → 1.17.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/integrations-databases-clickhouse.md +6 -0
- package/dist/index.cjs +492 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +469 -36
- package/dist/index.js.map +1 -1
- package/dist/storage/db/replication.d.ts.map +1 -1
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +3 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/deletion-requests.d.ts +28 -0
- package/dist/storage/domains/observability/v-next/deletion-requests.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/index.d.ts +8 -2
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/trace-query.d.ts +12 -0
- package/dist/storage/domains/observability/v-next/trace-query.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/tracing.d.ts +22 -7
- package/dist/storage/domains/observability/v-next/tracing.d.ts.map +1 -1
- package/dist/storage/index.d.ts +4 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,11 +1,35 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
//#region \0rolldown/runtime.js
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
11
|
+
key = keys[i];
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
13
|
+
get: ((k) => from[k]).bind(null, key),
|
|
14
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
return to;
|
|
18
|
+
};
|
|
19
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
20
|
+
value: mod,
|
|
21
|
+
enumerable: true
|
|
22
|
+
}) : target, mod));
|
|
23
|
+
//#endregion
|
|
2
24
|
let _clickhouse_client = require("@clickhouse/client");
|
|
3
25
|
let _mastra_core_error = require("@mastra/core/error");
|
|
4
26
|
let _mastra_core_storage = require("@mastra/core/storage");
|
|
27
|
+
_mastra_core_storage = __toESM(_mastra_core_storage, 1);
|
|
5
28
|
let _mastra_core_agent = require("@mastra/core/agent");
|
|
6
29
|
let _mastra_core_base = require("@mastra/core/base");
|
|
7
30
|
let _mastra_core_utils = require("@mastra/core/utils");
|
|
8
31
|
let _mastra_core_features = require("@mastra/core/features");
|
|
32
|
+
let crypto$1 = require("crypto");
|
|
9
33
|
let _mastra_core_evals = require("@mastra/core/evals");
|
|
10
34
|
//#region src/storage/db/replication.ts
|
|
11
35
|
const DEFAULT_ZOOKEEPER_PATH = "/clickhouse/tables/{shard}/{database}/{table}";
|
|
@@ -71,12 +95,22 @@ function getEngineNameAndArgs(engine) {
|
|
|
71
95
|
}
|
|
72
96
|
function hasBalancedParens(s) {
|
|
73
97
|
let depth = 0;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (
|
|
98
|
+
let quote;
|
|
99
|
+
for (let i = 0; i < s.length; i++) {
|
|
100
|
+
const c = s[i];
|
|
101
|
+
if (quote) {
|
|
102
|
+
if (c === "\\") i++;
|
|
103
|
+
else if (c === quote) quote = void 0;
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (c === "`" || c === "\"" || c === "'") quote = c;
|
|
107
|
+
else if (c === "(") depth++;
|
|
108
|
+
else if (c === ")") {
|
|
109
|
+
depth--;
|
|
110
|
+
if (depth < 0) return false;
|
|
111
|
+
}
|
|
78
112
|
}
|
|
79
|
-
return depth === 0;
|
|
113
|
+
return depth === 0 && quote === void 0;
|
|
80
114
|
}
|
|
81
115
|
function buildReplicatedTableEngine(engine, replication) {
|
|
82
116
|
if (!replication) return engine;
|
|
@@ -148,20 +182,51 @@ function addOnClusterToDDL(sql, replication) {
|
|
|
148
182
|
return out;
|
|
149
183
|
}
|
|
150
184
|
function rewriteEngineClauses(sql, replication) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
185
|
+
const enginePattern = /ENGINE\s*=\s*(\w+)/gi;
|
|
186
|
+
let result = "";
|
|
187
|
+
let consumedUntil = 0;
|
|
188
|
+
for (let match = enginePattern.exec(sql); match; match = enginePattern.exec(sql)) {
|
|
189
|
+
const engineName = match[1];
|
|
190
|
+
if (!engineName) continue;
|
|
191
|
+
let engineEnd = enginePattern.lastIndex;
|
|
192
|
+
let argsStart = engineEnd;
|
|
193
|
+
while (/\s/.test(sql[argsStart] ?? "")) argsStart++;
|
|
194
|
+
let engine = engineName;
|
|
195
|
+
if (sql[argsStart] === "(") {
|
|
196
|
+
let depth = 0;
|
|
197
|
+
let closingParenEnd;
|
|
198
|
+
let quote;
|
|
199
|
+
for (let i = argsStart; i < sql.length; i++) {
|
|
200
|
+
const char = sql[i];
|
|
201
|
+
if (quote) {
|
|
202
|
+
if (char === "\\") i++;
|
|
203
|
+
else if (char === quote) quote = void 0;
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
if (char === "`" || char === "\"" || char === "'") quote = char;
|
|
207
|
+
else if (char === "(") depth++;
|
|
208
|
+
else if (char === ")") {
|
|
209
|
+
depth--;
|
|
210
|
+
if (depth === 0) {
|
|
211
|
+
closingParenEnd = i + 1;
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
161
215
|
}
|
|
216
|
+
if (closingParenEnd === void 0) {
|
|
217
|
+
result += sql.slice(consumedUntil, enginePattern.lastIndex);
|
|
218
|
+
consumedUntil = enginePattern.lastIndex;
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
engineEnd = closingParenEnd;
|
|
222
|
+
engine = `${engineName}${sql.slice(argsStart, engineEnd)}`;
|
|
162
223
|
}
|
|
163
|
-
|
|
164
|
-
|
|
224
|
+
result += sql.slice(consumedUntil, match.index);
|
|
225
|
+
result += `ENGINE = ${buildReplicatedTableEngine(engine, replication)}`;
|
|
226
|
+
consumedUntil = engineEnd;
|
|
227
|
+
enginePattern.lastIndex = engineEnd;
|
|
228
|
+
}
|
|
229
|
+
return result + sql.slice(consumedUntil);
|
|
165
230
|
}
|
|
166
231
|
function applyReplicationToDDL(sql, replication) {
|
|
167
232
|
return addOnClusterToDDL(replication ? rewriteEngineClauses(sql, replication) : sql, replication);
|
|
@@ -2478,6 +2543,7 @@ var ObservabilityStorageClickhouse = class extends _mastra_core_storage.Observab
|
|
|
2478
2543
|
}
|
|
2479
2544
|
}
|
|
2480
2545
|
async batchDeleteTraces(args) {
|
|
2546
|
+
this.assertUnscopedBatchDeleteTraces(args);
|
|
2481
2547
|
try {
|
|
2482
2548
|
if (args.traceIds.length === 0) return;
|
|
2483
2549
|
await this.client.command({
|
|
@@ -2504,6 +2570,7 @@ const TABLE_METRIC_EVENTS = "mastra_metric_events";
|
|
|
2504
2570
|
const TABLE_LOG_EVENTS = "mastra_log_events";
|
|
2505
2571
|
const TABLE_SCORE_EVENTS = "mastra_score_events";
|
|
2506
2572
|
const TABLE_FEEDBACK_EVENTS = "mastra_feedback_events";
|
|
2573
|
+
const TABLE_DELETION_REQUESTS = "mastra_deletion_requests";
|
|
2507
2574
|
const TABLE_METRIC_EVENTS_DELTA = "mastra_metric_events_delta";
|
|
2508
2575
|
const TABLE_LOG_EVENTS_DELTA = "mastra_log_events_delta";
|
|
2509
2576
|
const TABLE_SCORE_EVENTS_DELTA = "mastra_score_events_delta";
|
|
@@ -3199,6 +3266,23 @@ PARTITION BY toDate(timestamp)
|
|
|
3199
3266
|
ORDER BY (traceId, timestamp, feedbackId)
|
|
3200
3267
|
SETTINGS allow_nullable_key = 1
|
|
3201
3268
|
`;
|
|
3269
|
+
const DELETION_REQUESTS_DDL = `
|
|
3270
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_DELETION_REQUESTS} (
|
|
3271
|
+
requestId String,
|
|
3272
|
+
organizationId String DEFAULT '',
|
|
3273
|
+
resourceId String DEFAULT '',
|
|
3274
|
+
signal LowCardinality(String),
|
|
3275
|
+
predicateType LowCardinality(String),
|
|
3276
|
+
predicateValues Array(String),
|
|
3277
|
+
requestedAt DateTime64(3),
|
|
3278
|
+
requestedBy String DEFAULT '',
|
|
3279
|
+
lastAppliedAt DateTime64(3) DEFAULT 0,
|
|
3280
|
+
purgeVerifiedAt DateTime64(3) DEFAULT 0,
|
|
3281
|
+
updatedAt DateTime64(3)
|
|
3282
|
+
)
|
|
3283
|
+
ENGINE = ReplacingMergeTree(updatedAt)
|
|
3284
|
+
ORDER BY (organizationId, resourceId, requestId)
|
|
3285
|
+
`;
|
|
3202
3286
|
function buildFeedbackEventsDeltaDDL() {
|
|
3203
3287
|
return `
|
|
3204
3288
|
CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS_DELTA} (
|
|
@@ -3348,6 +3432,7 @@ const BASE_TABLE_DDL = [
|
|
|
3348
3432
|
LOG_EVENTS_DDL,
|
|
3349
3433
|
SCORE_EVENTS_DDL,
|
|
3350
3434
|
FEEDBACK_EVENTS_DDL,
|
|
3435
|
+
DELETION_REQUESTS_DDL,
|
|
3351
3436
|
DISCOVERY_VALUES_DDL,
|
|
3352
3437
|
DISCOVERY_PAIRS_DDL
|
|
3353
3438
|
];
|
|
@@ -3432,6 +3517,7 @@ const ALL_TABLE_NAMES = [
|
|
|
3432
3517
|
TABLE_LOG_EVENTS,
|
|
3433
3518
|
TABLE_SCORE_EVENTS,
|
|
3434
3519
|
TABLE_FEEDBACK_EVENTS,
|
|
3520
|
+
TABLE_DELETION_REQUESTS,
|
|
3435
3521
|
TABLE_METRIC_EVENTS_DELTA,
|
|
3436
3522
|
TABLE_LOG_EVENTS_DELTA,
|
|
3437
3523
|
TABLE_SCORE_EVENTS_DELTA,
|
|
@@ -4089,6 +4175,35 @@ function feedbackRecordToRow(feedback) {
|
|
|
4089
4175
|
};
|
|
4090
4176
|
}
|
|
4091
4177
|
//#endregion
|
|
4178
|
+
//#region src/storage/domains/observability/v-next/deletion-requests.ts
|
|
4179
|
+
const EPOCH = "1970-01-01T00:00:00.000Z";
|
|
4180
|
+
async function recordDeletionRequest(client, args) {
|
|
4181
|
+
const row = {
|
|
4182
|
+
requestId: args.requestId,
|
|
4183
|
+
organizationId: args.organizationId ?? "",
|
|
4184
|
+
resourceId: args.resourceId ?? "",
|
|
4185
|
+
signal: args.signal,
|
|
4186
|
+
predicateType: args.predicateType,
|
|
4187
|
+
predicateValues: args.predicateValues,
|
|
4188
|
+
requestedAt: args.requestedAt,
|
|
4189
|
+
requestedBy: args.requestedBy ?? "",
|
|
4190
|
+
lastAppliedAt: EPOCH,
|
|
4191
|
+
purgeVerifiedAt: EPOCH,
|
|
4192
|
+
updatedAt: args.requestedAt
|
|
4193
|
+
};
|
|
4194
|
+
await client.insert({
|
|
4195
|
+
table: TABLE_DELETION_REQUESTS,
|
|
4196
|
+
values: [row],
|
|
4197
|
+
format: "JSONEachRow",
|
|
4198
|
+
clickhouse_settings: isReplicationConfigured(args.replication) ? {
|
|
4199
|
+
...CH_INSERT_SETTINGS,
|
|
4200
|
+
insert_quorum: "auto",
|
|
4201
|
+
insert_quorum_parallel: 1
|
|
4202
|
+
} : CH_INSERT_SETTINGS
|
|
4203
|
+
});
|
|
4204
|
+
return row;
|
|
4205
|
+
}
|
|
4206
|
+
//#endregion
|
|
4092
4207
|
//#region src/storage/domains/observability/v-next/discovery.ts
|
|
4093
4208
|
async function queryJson$3(client, query, params = {}) {
|
|
4094
4209
|
return await (await client.query({
|
|
@@ -6092,6 +6207,269 @@ async function getScorePercentiles(client, args) {
|
|
|
6092
6207
|
return { series };
|
|
6093
6208
|
}
|
|
6094
6209
|
//#endregion
|
|
6210
|
+
//#region src/storage/domains/observability/v-next/trace-query.ts
|
|
6211
|
+
const TRACE_STATUS_SQL = `if(isNotNull(r.error), 'error', 'success')`;
|
|
6212
|
+
const TRACE_FIELDS = {
|
|
6213
|
+
traceId: {
|
|
6214
|
+
sql: "r.traceId",
|
|
6215
|
+
parameterType: "String"
|
|
6216
|
+
},
|
|
6217
|
+
threadId: {
|
|
6218
|
+
sql: "r.threadId",
|
|
6219
|
+
parameterType: "String"
|
|
6220
|
+
},
|
|
6221
|
+
resourceId: {
|
|
6222
|
+
sql: "r.resourceId",
|
|
6223
|
+
parameterType: "String"
|
|
6224
|
+
},
|
|
6225
|
+
startedAt: {
|
|
6226
|
+
sql: "r.startedAt",
|
|
6227
|
+
parameterType: "DateTime64(3, 'UTC')"
|
|
6228
|
+
},
|
|
6229
|
+
endedAt: {
|
|
6230
|
+
sql: "r.endedAt",
|
|
6231
|
+
parameterType: "DateTime64(3, 'UTC')"
|
|
6232
|
+
},
|
|
6233
|
+
entityName: {
|
|
6234
|
+
sql: "r.entityName",
|
|
6235
|
+
parameterType: "String"
|
|
6236
|
+
},
|
|
6237
|
+
entityType: {
|
|
6238
|
+
sql: "r.entityType",
|
|
6239
|
+
parameterType: "String"
|
|
6240
|
+
},
|
|
6241
|
+
environment: {
|
|
6242
|
+
sql: "r.environment",
|
|
6243
|
+
parameterType: "String"
|
|
6244
|
+
},
|
|
6245
|
+
status: {
|
|
6246
|
+
sql: TRACE_STATUS_SQL,
|
|
6247
|
+
parameterType: "String"
|
|
6248
|
+
}
|
|
6249
|
+
};
|
|
6250
|
+
const SPAN_FIELDS = {
|
|
6251
|
+
spanType: {
|
|
6252
|
+
sql: "s.spanType",
|
|
6253
|
+
parameterType: "String"
|
|
6254
|
+
},
|
|
6255
|
+
error: {
|
|
6256
|
+
sql: "s.error",
|
|
6257
|
+
parameterType: "String"
|
|
6258
|
+
}
|
|
6259
|
+
};
|
|
6260
|
+
const SCORE_FIELDS = {
|
|
6261
|
+
scorerId: {
|
|
6262
|
+
sql: "s.scorerId",
|
|
6263
|
+
parameterType: "String"
|
|
6264
|
+
},
|
|
6265
|
+
score: {
|
|
6266
|
+
sql: "s.score",
|
|
6267
|
+
parameterType: "Float64"
|
|
6268
|
+
}
|
|
6269
|
+
};
|
|
6270
|
+
const TRACE_SELECT = `
|
|
6271
|
+
r.traceId AS traceId,
|
|
6272
|
+
r.spanId AS rootSpanId,
|
|
6273
|
+
r.threadId AS threadId,
|
|
6274
|
+
r.resourceId AS resourceId,
|
|
6275
|
+
r.startedAt AS startedAt,
|
|
6276
|
+
r.endedAt AS endedAt,
|
|
6277
|
+
r.entityName AS entityName,
|
|
6278
|
+
r.entityType AS entityType,
|
|
6279
|
+
r.environment AS environment,
|
|
6280
|
+
${TRACE_STATUS_SQL} AS status`;
|
|
6281
|
+
var ParameterBuilder = class {
|
|
6282
|
+
params = {};
|
|
6283
|
+
#next = 1;
|
|
6284
|
+
add(value, type) {
|
|
6285
|
+
const name = `trace_query_${this.#next++}`;
|
|
6286
|
+
this.params[name] = type === "DateTime64(3, 'UTC')" ? new Date(value).toISOString().replace("T", " ").replace(/Z$/, "") : value;
|
|
6287
|
+
return `{${name}:${type}}`;
|
|
6288
|
+
}
|
|
6289
|
+
};
|
|
6290
|
+
function fieldDefinition(registry, field) {
|
|
6291
|
+
const definition = registry[field];
|
|
6292
|
+
if (definition === void 0) throw new Error(`Unsupported trusted trace-query field: ${field}`);
|
|
6293
|
+
return definition;
|
|
6294
|
+
}
|
|
6295
|
+
function resolveOrderField(field) {
|
|
6296
|
+
if (field === "startedAt" || field === "endedAt") return field;
|
|
6297
|
+
throw new Error(`Unsupported trusted trace-query field: ${field}`);
|
|
6298
|
+
}
|
|
6299
|
+
function compileScalarPredicate(predicate, registry, parameters) {
|
|
6300
|
+
if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compileScalarPredicate(arg, registry, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
|
|
6301
|
+
if (predicate.type === "not") return `NOT (${compileScalarPredicate(predicate.arg, registry, parameters)})`;
|
|
6302
|
+
const field = fieldDefinition(registry, predicate.field);
|
|
6303
|
+
if (predicate.type === "presence") return `${predicate.operator === "exists" ? "isNotNull" : "isNull"}(${field.sql})`;
|
|
6304
|
+
if (predicate.type === "membership") {
|
|
6305
|
+
const values = predicate.values.map((value) => parameters.add(value, field.parameterType)).join(", ");
|
|
6306
|
+
return `ifNull(${`${field.sql} ${predicate.operator === "in" ? "IN" : "NOT IN"} (${values})`}, ${predicate.operator === "in" ? "0" : "1"})`;
|
|
6307
|
+
}
|
|
6308
|
+
const parameter = parameters.add(predicate.value, field.parameterType);
|
|
6309
|
+
const operator = {
|
|
6310
|
+
eq: "=",
|
|
6311
|
+
ne: "!=",
|
|
6312
|
+
lt: "<",
|
|
6313
|
+
lte: "<=",
|
|
6314
|
+
gt: ">",
|
|
6315
|
+
gte: ">="
|
|
6316
|
+
}[predicate.operator];
|
|
6317
|
+
if (operator === void 0) throw new Error(`Unsupported trusted trace-query operator: ${predicate.operator}`);
|
|
6318
|
+
return `ifNull(${field.sql} ${operator} ${parameter}, ${predicate.operator === "ne" ? "1" : "0"})`;
|
|
6319
|
+
}
|
|
6320
|
+
function collectRelationCollections(predicate, collections = /* @__PURE__ */ new Set()) {
|
|
6321
|
+
if (!predicate) return collections;
|
|
6322
|
+
if (predicate.type === "relation") collections.add(predicate.collection);
|
|
6323
|
+
else if (predicate.type === "boolean") for (const arg of predicate.args) collectRelationCollections(arg, collections);
|
|
6324
|
+
else if (predicate.type === "not") collectRelationCollections(predicate.arg, collections);
|
|
6325
|
+
return collections;
|
|
6326
|
+
}
|
|
6327
|
+
function compilePredicate(predicate, parameters) {
|
|
6328
|
+
if (predicate.type === "relation") {
|
|
6329
|
+
const registry = predicate.collection === "spans" ? SPAN_FIELDS : SCORE_FIELDS;
|
|
6330
|
+
const existence = `EXISTS (
|
|
6331
|
+
SELECT 1 FROM ${predicate.collection === "spans" ? "current_spans" : "current_scores"} s
|
|
6332
|
+
WHERE s.traceId = r.traceId
|
|
6333
|
+
AND (${compileScalarPredicate(predicate.predicate, registry, parameters)})
|
|
6334
|
+
)`;
|
|
6335
|
+
return predicate.quantifier === "some" ? existence : `NOT ${existence}`;
|
|
6336
|
+
}
|
|
6337
|
+
if (predicate.type === "boolean") return predicate.args.map((arg) => `(${compilePredicate(arg, parameters)})`).join(predicate.operator === "and" ? " AND " : " OR ");
|
|
6338
|
+
if (predicate.type === "not") return `NOT (${compilePredicate(predicate.arg, parameters)})`;
|
|
6339
|
+
return compileScalarPredicate(predicate, TRACE_FIELDS, parameters);
|
|
6340
|
+
}
|
|
6341
|
+
function compileClickHouseTraceQuery(plan) {
|
|
6342
|
+
const parameters = new ParameterBuilder();
|
|
6343
|
+
const from = parameters.add(plan.timeRange.from, "DateTime64(3, 'UTC')");
|
|
6344
|
+
const to = parameters.add(plan.timeRange.to, "DateTime64(3, 'UTC')");
|
|
6345
|
+
const relationCollections = collectRelationCollections(plan.where);
|
|
6346
|
+
const ctes = [`current_roots AS (
|
|
6347
|
+
SELECT * FROM (
|
|
6348
|
+
SELECT *
|
|
6349
|
+
FROM ${TABLE_TRACE_ROOTS}
|
|
6350
|
+
ORDER BY dedupeKey
|
|
6351
|
+
LIMIT 1 BY dedupeKey
|
|
6352
|
+
)
|
|
6353
|
+
ORDER BY traceId, dedupeKey
|
|
6354
|
+
LIMIT 1 BY traceId
|
|
6355
|
+
)`, `root_scope AS (
|
|
6356
|
+
SELECT *
|
|
6357
|
+
FROM current_roots
|
|
6358
|
+
WHERE startedAt >= ${from}
|
|
6359
|
+
AND startedAt < ${to}
|
|
6360
|
+
)`];
|
|
6361
|
+
if (relationCollections.has("spans")) ctes.push(`current_spans AS (
|
|
6362
|
+
SELECT traceId, spanType, error
|
|
6363
|
+
FROM ${TABLE_SPAN_EVENTS}
|
|
6364
|
+
WHERE traceId IN (SELECT traceId FROM root_scope)
|
|
6365
|
+
ORDER BY dedupeKey
|
|
6366
|
+
LIMIT 1 BY dedupeKey
|
|
6367
|
+
)`);
|
|
6368
|
+
if (relationCollections.has("scores")) ctes.push(`current_scores AS (
|
|
6369
|
+
SELECT traceId, scorerId, score
|
|
6370
|
+
FROM ${TABLE_SCORE_EVENTS}
|
|
6371
|
+
WHERE isNotNull(traceId)
|
|
6372
|
+
AND traceId IN (SELECT traceId FROM root_scope)
|
|
6373
|
+
)`);
|
|
6374
|
+
const predicate = plan.where ? compilePredicate(plan.where, parameters) : "1";
|
|
6375
|
+
ctes.push(`candidates AS (
|
|
6376
|
+
SELECT ${TRACE_SELECT}
|
|
6377
|
+
FROM root_scope r
|
|
6378
|
+
WHERE ${predicate}
|
|
6379
|
+
)`);
|
|
6380
|
+
const candidates = `WITH ${ctes.join(",\n")}`;
|
|
6381
|
+
if (plan.result === "groups") return {
|
|
6382
|
+
query: `${candidates}
|
|
6383
|
+
SELECT threadId
|
|
6384
|
+
FROM candidates
|
|
6385
|
+
WHERE isNotNull(threadId) ${plan.cursor ? `AND threadId > ${parameters.add(plan.cursor.threadId, "String")}` : ""}
|
|
6386
|
+
GROUP BY threadId
|
|
6387
|
+
ORDER BY threadId ASC
|
|
6388
|
+
LIMIT ${parameters.add(plan.limit + 1, "UInt64")}`,
|
|
6389
|
+
query_params: parameters.params
|
|
6390
|
+
};
|
|
6391
|
+
const orderField = resolveOrderField(plan.orderBy.field);
|
|
6392
|
+
const direction = plan.orderBy.direction === "asc" ? "ASC" : "DESC";
|
|
6393
|
+
let pageCondition = "";
|
|
6394
|
+
if (plan.cursor) {
|
|
6395
|
+
const comparison = plan.orderBy.direction === "asc" ? ">" : "<";
|
|
6396
|
+
const sortValue = parameters.add(plan.cursor.sortValue, "DateTime64(3, 'UTC')");
|
|
6397
|
+
pageCondition = `WHERE (${orderField} ${comparison} ${sortValue} OR (${orderField} = ${sortValue} AND traceId > ${parameters.add(plan.cursor.traceId, "String")}))`;
|
|
6398
|
+
}
|
|
6399
|
+
const limit = parameters.add(plan.limit + 1, "UInt64");
|
|
6400
|
+
return {
|
|
6401
|
+
query: `${candidates}
|
|
6402
|
+
SELECT *
|
|
6403
|
+
FROM candidates
|
|
6404
|
+
${pageCondition}
|
|
6405
|
+
ORDER BY ${orderField} ${direction}, traceId ASC
|
|
6406
|
+
LIMIT ${limit}`,
|
|
6407
|
+
query_params: parameters.params
|
|
6408
|
+
};
|
|
6409
|
+
}
|
|
6410
|
+
function asIsoTimestamp(value) {
|
|
6411
|
+
return new Date(value).toISOString();
|
|
6412
|
+
}
|
|
6413
|
+
function isClickHouseExecutionTimeout(error) {
|
|
6414
|
+
if (!error || typeof error !== "object") return false;
|
|
6415
|
+
const candidate = error;
|
|
6416
|
+
return String(candidate.code ?? "") === "159" || candidate.type === "TIMEOUT_EXCEEDED";
|
|
6417
|
+
}
|
|
6418
|
+
async function runWithClickHouseTraceQueryTimeout(client, timeoutMs, compiled, queryId) {
|
|
6419
|
+
const resolvedTimeoutMs = _mastra_core_storage.resolveTraceQueryTimeoutMs(timeoutMs);
|
|
6420
|
+
try {
|
|
6421
|
+
return await (await client.query({
|
|
6422
|
+
query: compiled.query,
|
|
6423
|
+
query_params: compiled.query_params,
|
|
6424
|
+
query_id: queryId,
|
|
6425
|
+
format: "JSONEachRow",
|
|
6426
|
+
clickhouse_settings: {
|
|
6427
|
+
...CH_SETTINGS,
|
|
6428
|
+
max_execution_time: resolvedTimeoutMs / 1e3
|
|
6429
|
+
}
|
|
6430
|
+
})).json();
|
|
6431
|
+
} catch (error) {
|
|
6432
|
+
if (isClickHouseExecutionTimeout(error)) throw new _mastra_core_storage.TraceQueryExecutionError();
|
|
6433
|
+
throw error;
|
|
6434
|
+
}
|
|
6435
|
+
}
|
|
6436
|
+
async function queryTraces(client, plan, timeoutMs) {
|
|
6437
|
+
const rows = await runWithClickHouseTraceQueryTimeout(client, timeoutMs, compileClickHouseTraceQuery(plan));
|
|
6438
|
+
const visibleRows = rows.slice(0, plan.limit);
|
|
6439
|
+
if (plan.result === "groups") {
|
|
6440
|
+
const groups = visibleRows.map((row) => ({ threadId: String(row.threadId) }));
|
|
6441
|
+
const last = groups.at(-1);
|
|
6442
|
+
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
6443
|
+
groups,
|
|
6444
|
+
page: { next: rows.length > plan.limit && last ? _mastra_core_storage.encodeTraceQueryCursor(plan, {
|
|
6445
|
+
result: "groups",
|
|
6446
|
+
threadId: last.threadId
|
|
6447
|
+
}) : null }
|
|
6448
|
+
});
|
|
6449
|
+
}
|
|
6450
|
+
const traces = visibleRows.map((row) => ({
|
|
6451
|
+
traceId: String(row.traceId),
|
|
6452
|
+
rootSpanId: String(row.rootSpanId),
|
|
6453
|
+
threadId: row.threadId == null ? null : String(row.threadId),
|
|
6454
|
+
resourceId: row.resourceId == null ? null : String(row.resourceId),
|
|
6455
|
+
startedAt: asIsoTimestamp(row.startedAt),
|
|
6456
|
+
endedAt: asIsoTimestamp(row.endedAt),
|
|
6457
|
+
entityName: row.entityName == null ? null : String(row.entityName),
|
|
6458
|
+
entityType: row.entityType == null ? null : String(row.entityType),
|
|
6459
|
+
environment: row.environment == null ? null : String(row.environment),
|
|
6460
|
+
status: row.status
|
|
6461
|
+
}));
|
|
6462
|
+
const last = traces.at(-1);
|
|
6463
|
+
return _mastra_core_storage.traceQueryResponseSchema.parse({
|
|
6464
|
+
traces,
|
|
6465
|
+
page: { next: rows.length > plan.limit && last ? _mastra_core_storage.encodeTraceQueryCursor(plan, {
|
|
6466
|
+
result: "traces",
|
|
6467
|
+
sortValue: last[plan.orderBy.field],
|
|
6468
|
+
traceId: last.traceId
|
|
6469
|
+
}) : null }
|
|
6470
|
+
});
|
|
6471
|
+
}
|
|
6472
|
+
//#endregion
|
|
6095
6473
|
//#region src/storage/domains/observability/v-next/trace-roots.ts
|
|
6096
6474
|
/**
|
|
6097
6475
|
* Get the root span for a trace, reading from trace_roots as compatibility path.
|
|
@@ -6328,6 +6706,17 @@ function buildTraceCursor(row) {
|
|
|
6328
6706
|
}
|
|
6329
6707
|
//#endregion
|
|
6330
6708
|
//#region src/storage/domains/observability/v-next/tracing.ts
|
|
6709
|
+
/**
|
|
6710
|
+
* Tracing operations for ClickHouse v-next observability.
|
|
6711
|
+
*
|
|
6712
|
+
* Owns: batchCreateSpans, getSpan, getSpans, getTrace, getTraceLight,
|
|
6713
|
+
* listBranches, batchDeleteTraces, dangerouslyClearSpanEvents.
|
|
6714
|
+
* Delegates to trace-roots.ts: listTraces, getRootSpan.
|
|
6715
|
+
*
|
|
6716
|
+
* `listBranches` reads from the MV-fed `mastra_trace_branches` table (one row
|
|
6717
|
+
* per branch anchor span). It lives here -- alongside the other read paths
|
|
6718
|
+
* over the trace data -- since branches are conceptually a subset of traces.
|
|
6719
|
+
*/
|
|
6331
6720
|
const BRANCH_SPAN_TYPE_SQL_LIST = _mastra_core_storage.BRANCH_SPAN_TYPES.map((t) => `'${t}'`).join(", ");
|
|
6332
6721
|
/** Insert a single completed span. */
|
|
6333
6722
|
async function createSpan(client, args) {
|
|
@@ -6467,16 +6856,40 @@ async function getTraceLight(client, args) {
|
|
|
6467
6856
|
};
|
|
6468
6857
|
}
|
|
6469
6858
|
/**
|
|
6470
|
-
* Delete traces by traceId
|
|
6471
|
-
*
|
|
6859
|
+
* Delete traces by traceId, cascading to trace-derived tables and trace-linked
|
|
6860
|
+
* signal events (metrics, logs, scores, feedback). Signal rows with a NULL
|
|
6861
|
+
* traceId are never affected.
|
|
6862
|
+
*
|
|
6863
|
+
* On the tracing tables, rows are targeted by tracing identity: traceId +
|
|
6864
|
+
* dedupeKey (which starts with traceId). The dedupeKey condition is redundant
|
|
6865
|
+
* for correctness (dedupeKey = traceId:spanId) but satisfies the design-doc
|
|
6866
|
+
* requirement that trace deletes reference dedupeKey and helps the engine
|
|
6867
|
+
* narrow within the sorted ORDER BY key. Signal tables key on their own event
|
|
6868
|
+
* ids, so they are targeted by traceId alone.
|
|
6869
|
+
*
|
|
6870
|
+
* `trace_branches` must be deleted explicitly: its MV fires on insert only,
|
|
6871
|
+
* so span deletes never propagate to it. Delta tables self-expire via TTL and
|
|
6872
|
+
* discovery tables self-heal, so neither needs explicit deletes.
|
|
6873
|
+
*
|
|
6874
|
+
* Records the predicate before using lightweight DELETE FROM on every table.
|
|
6875
|
+
* Lightweight deletes hide rows through ClickHouse's delete mask; physical
|
|
6876
|
+
* removal depends on the deployment's configured retention and merge policy.
|
|
6472
6877
|
*
|
|
6473
|
-
*
|
|
6474
|
-
*
|
|
6475
|
-
* but satisfies the design-doc requirement that trace deletes reference dedupeKey
|
|
6476
|
-
* and helps the engine narrow within the sorted ORDER BY key.
|
|
6878
|
+
* When the optional tenant scope (`organizationId` / `resourceId`) is set,
|
|
6879
|
+
* every DELETE additionally requires the row's tenant columns to match.
|
|
6477
6880
|
*/
|
|
6478
|
-
async function batchDeleteTraces(client, args) {
|
|
6881
|
+
async function batchDeleteTraces(client, args, replication) {
|
|
6479
6882
|
if (args.traceIds.length === 0) return;
|
|
6883
|
+
await recordDeletionRequest(client, {
|
|
6884
|
+
requestId: (0, crypto$1.randomUUID)(),
|
|
6885
|
+
organizationId: args.organizationId,
|
|
6886
|
+
resourceId: args.resourceId,
|
|
6887
|
+
signal: "traces",
|
|
6888
|
+
predicateType: "traceIds",
|
|
6889
|
+
predicateValues: [...args.traceIds],
|
|
6890
|
+
requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6891
|
+
replication
|
|
6892
|
+
});
|
|
6480
6893
|
const params = {};
|
|
6481
6894
|
const traceInPlaceholders = [];
|
|
6482
6895
|
const dedupeOrParts = [];
|
|
@@ -6490,13 +6903,35 @@ async function batchDeleteTraces(client, args) {
|
|
|
6490
6903
|
}
|
|
6491
6904
|
const traceInList = traceInPlaceholders.join(", ");
|
|
6492
6905
|
const dedupeCondition = dedupeOrParts.length === 1 ? dedupeOrParts[0] : `(${dedupeOrParts.join(" OR ")})`;
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6906
|
+
let scopeCondition = "";
|
|
6907
|
+
if (args.organizationId !== void 0) {
|
|
6908
|
+
params.scope_org = args.organizationId;
|
|
6909
|
+
scopeCondition += ` AND organizationId = {scope_org:String}`;
|
|
6910
|
+
}
|
|
6911
|
+
if (args.resourceId !== void 0) {
|
|
6912
|
+
params.scope_res = args.resourceId;
|
|
6913
|
+
scopeCondition += ` AND resourceId = {scope_res:String}`;
|
|
6914
|
+
}
|
|
6915
|
+
const tracingTables = [
|
|
6916
|
+
TABLE_SPAN_EVENTS,
|
|
6917
|
+
TABLE_TRACE_ROOTS,
|
|
6918
|
+
TABLE_TRACE_BRANCHES
|
|
6919
|
+
];
|
|
6920
|
+
const signalTables = [
|
|
6921
|
+
TABLE_METRIC_EVENTS,
|
|
6922
|
+
TABLE_LOG_EVENTS,
|
|
6923
|
+
TABLE_SCORE_EVENTS,
|
|
6924
|
+
TABLE_FEEDBACK_EVENTS
|
|
6925
|
+
];
|
|
6926
|
+
await Promise.all([...tracingTables.map((table) => client.command({
|
|
6927
|
+
query: `DELETE FROM ${table} WHERE traceId IN (${traceInList}) AND ${dedupeCondition}${scopeCondition}`,
|
|
6928
|
+
query_params: params,
|
|
6929
|
+
clickhouse_settings: { lightweight_deletes_sync: "2" }
|
|
6930
|
+
})), ...signalTables.map((table) => client.command({
|
|
6931
|
+
query: `DELETE FROM ${table} WHERE traceId IN (${traceInList})${scopeCondition}`,
|
|
6932
|
+
query_params: params,
|
|
6933
|
+
clickhouse_settings: { lightweight_deletes_sync: "2" }
|
|
6934
|
+
}))]);
|
|
6500
6935
|
}
|
|
6501
6936
|
/**
|
|
6502
6937
|
* List trace branches with optional filtering, pagination, and ordering.
|
|
@@ -7034,6 +7469,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
7034
7469
|
#retention;
|
|
7035
7470
|
#replication;
|
|
7036
7471
|
#deltaCursorStrategyOverride;
|
|
7472
|
+
#traceQueryTimeoutMs;
|
|
7037
7473
|
#deltaCursorStrategy = "fallback";
|
|
7038
7474
|
constructor(config) {
|
|
7039
7475
|
super();
|
|
@@ -7042,6 +7478,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
7042
7478
|
this.#replication = replication;
|
|
7043
7479
|
this.#retention = config.retention;
|
|
7044
7480
|
this.#deltaCursorStrategyOverride = config.deltaCursorStrategy;
|
|
7481
|
+
this.#traceQueryTimeoutMs = _mastra_core_storage.resolveTraceQueryTimeoutMs(config.traceQueryTimeoutMs);
|
|
7045
7482
|
}
|
|
7046
7483
|
async init() {
|
|
7047
7484
|
const migrationStatus = await checkSignalTablesMigrationStatus(this.#client);
|
|
@@ -7145,11 +7582,16 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
7145
7582
|
};
|
|
7146
7583
|
}
|
|
7147
7584
|
getFeatures() {
|
|
7148
|
-
if (!deltaPollingSupported(this.#deltaCursorStrategy)) return [
|
|
7585
|
+
if (!deltaPollingSupported(this.#deltaCursorStrategy)) return [
|
|
7586
|
+
"metrics",
|
|
7587
|
+
"logs",
|
|
7588
|
+
"trace-query"
|
|
7589
|
+
];
|
|
7149
7590
|
return [
|
|
7150
7591
|
"metrics",
|
|
7151
7592
|
"logs",
|
|
7152
|
-
"delta-polling"
|
|
7593
|
+
"delta-polling",
|
|
7594
|
+
"trace-query"
|
|
7153
7595
|
];
|
|
7154
7596
|
}
|
|
7155
7597
|
async createSpan(args) {
|
|
@@ -7264,6 +7706,18 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
7264
7706
|
}, error);
|
|
7265
7707
|
}
|
|
7266
7708
|
}
|
|
7709
|
+
async queryTraces(plan) {
|
|
7710
|
+
try {
|
|
7711
|
+
return await queryTraces(this.#client, plan, this.#traceQueryTimeoutMs);
|
|
7712
|
+
} catch (error) {
|
|
7713
|
+
if (error instanceof _mastra_core_error.MastraError || error instanceof _mastra_core_storage.TraceQueryExecutionError) throw error;
|
|
7714
|
+
throw new _mastra_core_error.MastraError({
|
|
7715
|
+
id: (0, _mastra_core_storage.createStorageErrorId)("CLICKHOUSE", "QUERY_TRACES", "FAILED"),
|
|
7716
|
+
domain: _mastra_core_error.ErrorDomain.STORAGE,
|
|
7717
|
+
category: _mastra_core_error.ErrorCategory.THIRD_PARTY
|
|
7718
|
+
}, error);
|
|
7719
|
+
}
|
|
7720
|
+
}
|
|
7267
7721
|
async listTracesLight(args) {
|
|
7268
7722
|
try {
|
|
7269
7723
|
return await listTracesLight(this.#client, args, this.#deltaCursorStrategy);
|
|
@@ -7680,7 +8134,7 @@ var ObservabilityStorageClickhouseVNext = class extends _mastra_core_storage.Obs
|
|
|
7680
8134
|
}
|
|
7681
8135
|
async batchDeleteTraces(args) {
|
|
7682
8136
|
try {
|
|
7683
|
-
await batchDeleteTraces(this.#client, args);
|
|
8137
|
+
await batchDeleteTraces(this.#client, args, this.#replication);
|
|
7684
8138
|
} catch (error) {
|
|
7685
8139
|
if (error instanceof _mastra_core_error.MastraError) throw error;
|
|
7686
8140
|
throw new _mastra_core_error.MastraError({
|
|
@@ -8540,7 +8994,8 @@ var ClickhouseStoreVNext = class extends ClickhouseStore {
|
|
|
8540
8994
|
this.name = "ClickhouseStoreVNext";
|
|
8541
8995
|
const observability = new ObservabilityStorageClickhouseVNext({
|
|
8542
8996
|
client: this.db,
|
|
8543
|
-
replication: config.replication
|
|
8997
|
+
replication: config.replication,
|
|
8998
|
+
traceQueryTimeoutMs: config.traceQueryTimeoutMs
|
|
8544
8999
|
});
|
|
8545
9000
|
this.stores = {
|
|
8546
9001
|
...this.stores,
|
|
@@ -8556,7 +9011,9 @@ exports.MemoryStorageClickhouse = MemoryStorageClickhouse;
|
|
|
8556
9011
|
exports.ObservabilityStorageClickhouse = ObservabilityStorageClickhouse;
|
|
8557
9012
|
exports.ObservabilityStorageClickhouseVNext = ObservabilityStorageClickhouseVNext;
|
|
8558
9013
|
exports.ScoresStorageClickhouse = ScoresStorageClickhouse;
|
|
9014
|
+
exports.TABLE_DELETION_REQUESTS = TABLE_DELETION_REQUESTS;
|
|
8559
9015
|
exports.TABLE_ENGINES = TABLE_ENGINES;
|
|
8560
9016
|
exports.WorkflowsStorageClickhouse = WorkflowsStorageClickhouse;
|
|
9017
|
+
exports.recordDeletionRequest = recordDeletionRequest;
|
|
8561
9018
|
|
|
8562
9019
|
//# sourceMappingURL=index.cjs.map
|