@mastra/clickhouse 1.4.1 → 1.5.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/index.cjs +458 -13
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +459 -15
- package/dist/index.js.map +1 -1
- package/dist/storage/db/utils.d.ts.map +1 -1
- package/dist/storage/domains/background-tasks/index.d.ts +20 -0
- package/dist/storage/domains/background-tasks/index.d.ts.map +1 -0
- package/dist/storage/domains/observability/index.d.ts +2 -1
- package/dist/storage/domains/observability/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/ddl.d.ts +4 -4
- package/dist/storage/domains/observability/v-next/ddl.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/helpers.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +15 -4
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/migration.d.ts +20 -0
- package/dist/storage/domains/observability/v-next/migration.d.ts.map +1 -0
- package/dist/storage/domains/observability/v-next/tracing.d.ts +5 -1
- package/dist/storage/domains/observability/v-next/tracing.d.ts.map +1 -1
- package/dist/storage/index.d.ts +2 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/package.json +8 -8
package/dist/index.cjs
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
var client = require('@clickhouse/client');
|
|
4
4
|
var error = require('@mastra/core/error');
|
|
5
5
|
var storage = require('@mastra/core/storage');
|
|
6
|
-
var agent = require('@mastra/core/agent');
|
|
7
6
|
var base = require('@mastra/core/base');
|
|
7
|
+
var agent = require('@mastra/core/agent');
|
|
8
8
|
var utils = require('@mastra/core/utils');
|
|
9
9
|
var evals = require('@mastra/core/evals');
|
|
10
10
|
|
|
@@ -39,7 +39,8 @@ var TABLE_ENGINES = {
|
|
|
39
39
|
[storage.TABLE_WORKSPACE_VERSIONS]: `MergeTree()`,
|
|
40
40
|
[storage.TABLE_SKILLS]: `ReplacingMergeTree()`,
|
|
41
41
|
[storage.TABLE_SKILL_VERSIONS]: `MergeTree()`,
|
|
42
|
-
[storage.TABLE_SKILL_BLOBS]: `ReplacingMergeTree()
|
|
42
|
+
[storage.TABLE_SKILL_BLOBS]: `ReplacingMergeTree()`,
|
|
43
|
+
mastra_background_tasks: `ReplacingMergeTree()`
|
|
43
44
|
};
|
|
44
45
|
var COLUMN_TYPES = {
|
|
45
46
|
text: "String",
|
|
@@ -684,7 +685,203 @@ var ClickhouseDB = class extends base.MastraBase {
|
|
|
684
685
|
}
|
|
685
686
|
};
|
|
686
687
|
|
|
687
|
-
// src/storage/domains/
|
|
688
|
+
// src/storage/domains/background-tasks/index.ts
|
|
689
|
+
function serializeJson(v) {
|
|
690
|
+
if (typeof v === "object" && v != null) return JSON.stringify(v);
|
|
691
|
+
return v ?? "";
|
|
692
|
+
}
|
|
693
|
+
function rowToTask(row) {
|
|
694
|
+
const parseJson2 = (val) => {
|
|
695
|
+
if (val == null || val === "") return void 0;
|
|
696
|
+
if (typeof val === "string") {
|
|
697
|
+
try {
|
|
698
|
+
return JSON.parse(val);
|
|
699
|
+
} catch {
|
|
700
|
+
return val;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return val;
|
|
704
|
+
};
|
|
705
|
+
return {
|
|
706
|
+
id: row.id,
|
|
707
|
+
status: row.status,
|
|
708
|
+
toolName: row.tool_name,
|
|
709
|
+
toolCallId: row.tool_call_id,
|
|
710
|
+
args: parseJson2(row.args) ?? {},
|
|
711
|
+
agentId: row.agent_id,
|
|
712
|
+
threadId: row.thread_id || void 0,
|
|
713
|
+
resourceId: row.resource_id || void 0,
|
|
714
|
+
runId: row.run_id ?? "",
|
|
715
|
+
result: parseJson2(row.result),
|
|
716
|
+
error: parseJson2(row.error),
|
|
717
|
+
retryCount: Number(row.retry_count ?? 0),
|
|
718
|
+
maxRetries: Number(row.max_retries ?? 0),
|
|
719
|
+
timeoutMs: Number(row.timeout_ms ?? 3e5),
|
|
720
|
+
createdAt: new Date(row.createdAt),
|
|
721
|
+
startedAt: row.startedAt ? new Date(row.startedAt) : void 0,
|
|
722
|
+
completedAt: row.completedAt ? new Date(row.completedAt) : void 0
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
var BackgroundTasksStorageClickhouse = class extends storage.BackgroundTasksStorage {
|
|
726
|
+
client;
|
|
727
|
+
#db;
|
|
728
|
+
constructor(config) {
|
|
729
|
+
super();
|
|
730
|
+
const { client, ttl } = resolveClickhouseConfig(config);
|
|
731
|
+
this.client = client;
|
|
732
|
+
this.#db = new ClickhouseDB({ client, ttl });
|
|
733
|
+
}
|
|
734
|
+
async init() {
|
|
735
|
+
await this.#db.createTable({ tableName: storage.TABLE_BACKGROUND_TASKS, schema: storage.TABLE_SCHEMAS[storage.TABLE_BACKGROUND_TASKS] });
|
|
736
|
+
}
|
|
737
|
+
async dangerouslyClearAll() {
|
|
738
|
+
await this.#db.clearTable({ tableName: storage.TABLE_BACKGROUND_TASKS });
|
|
739
|
+
}
|
|
740
|
+
async createTask(task) {
|
|
741
|
+
await this.client.insert({
|
|
742
|
+
table: storage.TABLE_BACKGROUND_TASKS,
|
|
743
|
+
values: [
|
|
744
|
+
{
|
|
745
|
+
id: task.id,
|
|
746
|
+
tool_call_id: task.toolCallId,
|
|
747
|
+
tool_name: task.toolName,
|
|
748
|
+
agent_id: task.agentId,
|
|
749
|
+
thread_id: task.threadId ?? "",
|
|
750
|
+
resource_id: task.resourceId ?? "",
|
|
751
|
+
run_id: task.runId,
|
|
752
|
+
status: task.status,
|
|
753
|
+
args: serializeJson(task.args),
|
|
754
|
+
result: serializeJson(task.result),
|
|
755
|
+
error: serializeJson(task.error),
|
|
756
|
+
retry_count: task.retryCount,
|
|
757
|
+
max_retries: task.maxRetries,
|
|
758
|
+
timeout_ms: task.timeoutMs,
|
|
759
|
+
createdAt: task.createdAt.toISOString(),
|
|
760
|
+
startedAt: task.startedAt?.toISOString() ?? "1970-01-01T00:00:00.000Z",
|
|
761
|
+
completedAt: task.completedAt?.toISOString() ?? "1970-01-01T00:00:00.000Z"
|
|
762
|
+
}
|
|
763
|
+
],
|
|
764
|
+
format: "JSONEachRow",
|
|
765
|
+
clickhouse_settings: { date_time_input_format: "best_effort" }
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
async updateTask(taskId, update) {
|
|
769
|
+
const existing = await this.getTask(taskId);
|
|
770
|
+
if (!existing) return;
|
|
771
|
+
const merged = { ...existing };
|
|
772
|
+
if ("status" in update) merged.status = update.status;
|
|
773
|
+
if ("result" in update) merged.result = update.result;
|
|
774
|
+
if ("error" in update) merged.error = update.error;
|
|
775
|
+
if ("retryCount" in update) merged.retryCount = update.retryCount;
|
|
776
|
+
if ("startedAt" in update) merged.startedAt = update.startedAt;
|
|
777
|
+
if ("completedAt" in update) merged.completedAt = update.completedAt;
|
|
778
|
+
await this.createTask(merged);
|
|
779
|
+
}
|
|
780
|
+
async getTask(taskId) {
|
|
781
|
+
const result = await this.client.query({
|
|
782
|
+
query: `SELECT * FROM ${storage.TABLE_BACKGROUND_TASKS} FINAL WHERE id = {var_id:String} LIMIT 1`,
|
|
783
|
+
query_params: { var_id: taskId },
|
|
784
|
+
format: "JSONEachRow",
|
|
785
|
+
clickhouse_settings: { date_time_input_format: "best_effort" }
|
|
786
|
+
});
|
|
787
|
+
const rows = await result.json();
|
|
788
|
+
return rows.length > 0 ? rowToTask(rows[0]) : null;
|
|
789
|
+
}
|
|
790
|
+
async listTasks(filter) {
|
|
791
|
+
const conditions = [];
|
|
792
|
+
const params = {};
|
|
793
|
+
if (filter.status) {
|
|
794
|
+
const statuses = Array.isArray(filter.status) ? filter.status : [filter.status];
|
|
795
|
+
conditions.push(`status IN ({var_statuses:Array(String)})`);
|
|
796
|
+
params.var_statuses = statuses;
|
|
797
|
+
}
|
|
798
|
+
if (filter.agentId) {
|
|
799
|
+
conditions.push(`agent_id = {var_agent:String}`);
|
|
800
|
+
params.var_agent = filter.agentId;
|
|
801
|
+
}
|
|
802
|
+
if (filter.threadId) {
|
|
803
|
+
conditions.push(`thread_id = {var_thread:String}`);
|
|
804
|
+
params.var_thread = filter.threadId;
|
|
805
|
+
}
|
|
806
|
+
if (filter.runId) {
|
|
807
|
+
conditions.push(`run_id = {var_run:String}`);
|
|
808
|
+
params.var_run = filter.runId;
|
|
809
|
+
}
|
|
810
|
+
if (filter.toolName) {
|
|
811
|
+
conditions.push(`tool_name = {var_tool:String}`);
|
|
812
|
+
params.var_tool = filter.toolName;
|
|
813
|
+
}
|
|
814
|
+
const dateCol = filter.dateFilterBy === "startedAt" ? "startedAt" : filter.dateFilterBy === "completedAt" ? "completedAt" : "createdAt";
|
|
815
|
+
if (filter.fromDate) {
|
|
816
|
+
conditions.push(`${dateCol} >= parseDateTimeBestEffort({var_from_date:String})`);
|
|
817
|
+
params.var_from_date = filter.fromDate.toISOString();
|
|
818
|
+
}
|
|
819
|
+
if (filter.toDate) {
|
|
820
|
+
conditions.push(`${dateCol} < parseDateTimeBestEffort({var_to_date:String})`);
|
|
821
|
+
params.var_to_date = filter.toDate.toISOString();
|
|
822
|
+
}
|
|
823
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
824
|
+
const countResult = await this.client.query({
|
|
825
|
+
query: `SELECT count() as count FROM ${storage.TABLE_BACKGROUND_TASKS} FINAL ${where}`,
|
|
826
|
+
query_params: params,
|
|
827
|
+
format: "JSONEachRow",
|
|
828
|
+
clickhouse_settings: { date_time_input_format: "best_effort" }
|
|
829
|
+
});
|
|
830
|
+
const countRows = await countResult.json();
|
|
831
|
+
const total = Number(countRows[0]?.count ?? 0);
|
|
832
|
+
const orderCol = filter.orderBy === "startedAt" ? "startedAt" : filter.orderBy === "completedAt" ? "completedAt" : "createdAt";
|
|
833
|
+
const direction = filter.orderDirection === "desc" ? "DESC" : "ASC";
|
|
834
|
+
let sql = `SELECT * FROM ${storage.TABLE_BACKGROUND_TASKS} FINAL ${where} ORDER BY ${orderCol} ${direction}`;
|
|
835
|
+
if (filter.perPage != null) {
|
|
836
|
+
sql += ` LIMIT {var_limit:UInt32}`;
|
|
837
|
+
params.var_limit = filter.perPage;
|
|
838
|
+
if (filter.page != null) {
|
|
839
|
+
sql += ` OFFSET {var_offset:UInt32}`;
|
|
840
|
+
params.var_offset = filter.page * filter.perPage;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
const result = await this.client.query({
|
|
844
|
+
query: sql,
|
|
845
|
+
query_params: params,
|
|
846
|
+
format: "JSONEachRow",
|
|
847
|
+
clickhouse_settings: { date_time_input_format: "best_effort" }
|
|
848
|
+
});
|
|
849
|
+
const tasks = (await result.json()).map(rowToTask);
|
|
850
|
+
return { tasks, total };
|
|
851
|
+
}
|
|
852
|
+
async deleteTask(taskId) {
|
|
853
|
+
await this.client.query({
|
|
854
|
+
query: `ALTER TABLE ${storage.TABLE_BACKGROUND_TASKS} DELETE WHERE id = {var_id:String}`,
|
|
855
|
+
query_params: { var_id: taskId }
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
async deleteTasks(filter) {
|
|
859
|
+
const { tasks } = await this.listTasks(filter);
|
|
860
|
+
for (const task of tasks) {
|
|
861
|
+
await this.client.query({
|
|
862
|
+
query: `ALTER TABLE ${storage.TABLE_BACKGROUND_TASKS} DELETE WHERE id = {var_id:String}`,
|
|
863
|
+
query_params: { var_id: task.id }
|
|
864
|
+
});
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
async getRunningCount() {
|
|
868
|
+
const result = await this.client.query({
|
|
869
|
+
query: `SELECT count() as count FROM ${storage.TABLE_BACKGROUND_TASKS} FINAL WHERE status = 'running'`,
|
|
870
|
+
format: "JSONEachRow"
|
|
871
|
+
});
|
|
872
|
+
const rows = await result.json();
|
|
873
|
+
return Number(rows[0]?.count ?? 0);
|
|
874
|
+
}
|
|
875
|
+
async getRunningCountByAgent(agentId) {
|
|
876
|
+
const result = await this.client.query({
|
|
877
|
+
query: `SELECT count() as count FROM ${storage.TABLE_BACKGROUND_TASKS} FINAL WHERE status = 'running' AND agent_id = {var_agent:String}`,
|
|
878
|
+
query_params: { var_agent: agentId },
|
|
879
|
+
format: "JSONEachRow"
|
|
880
|
+
});
|
|
881
|
+
const rows = await result.json();
|
|
882
|
+
return Number(rows[0]?.count ?? 0);
|
|
883
|
+
}
|
|
884
|
+
};
|
|
688
885
|
function serializeMetadata(metadata) {
|
|
689
886
|
if (!metadata || Object.keys(metadata).length === 0) {
|
|
690
887
|
return "{}";
|
|
@@ -2233,6 +2430,49 @@ time for large tables. Please ensure you have a backup before proceeding.
|
|
|
2233
2430
|
);
|
|
2234
2431
|
}
|
|
2235
2432
|
}
|
|
2433
|
+
async getTraceLight(args) {
|
|
2434
|
+
const { traceId } = args;
|
|
2435
|
+
try {
|
|
2436
|
+
const engine = TABLE_ENGINES[storage.TABLE_SPANS] ?? "MergeTree()";
|
|
2437
|
+
const result = await this.client.query({
|
|
2438
|
+
query: `
|
|
2439
|
+
SELECT traceId, spanId, parentSpanId, name,
|
|
2440
|
+
entityType, entityId, entityName,
|
|
2441
|
+
spanType, error, isEvent,
|
|
2442
|
+
startedAt, endedAt, createdAt, updatedAt
|
|
2443
|
+
FROM ${storage.TABLE_SPANS} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""}
|
|
2444
|
+
WHERE traceId = {traceId:String}
|
|
2445
|
+
ORDER BY startedAt ASC
|
|
2446
|
+
`,
|
|
2447
|
+
query_params: { traceId },
|
|
2448
|
+
format: "JSONEachRow",
|
|
2449
|
+
clickhouse_settings: {
|
|
2450
|
+
date_time_input_format: "best_effort",
|
|
2451
|
+
date_time_output_format: "iso",
|
|
2452
|
+
use_client_time_zone: 1,
|
|
2453
|
+
output_format_json_quote_64bit_integers: 0
|
|
2454
|
+
}
|
|
2455
|
+
});
|
|
2456
|
+
const rows = await result.json();
|
|
2457
|
+
if (!rows || rows.length === 0) {
|
|
2458
|
+
return null;
|
|
2459
|
+
}
|
|
2460
|
+
return {
|
|
2461
|
+
traceId,
|
|
2462
|
+
spans: transformRows(rows)
|
|
2463
|
+
};
|
|
2464
|
+
} catch (error$1) {
|
|
2465
|
+
throw new error.MastraError(
|
|
2466
|
+
{
|
|
2467
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "GET_TRACE_LIGHT", "FAILED"),
|
|
2468
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2469
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
2470
|
+
details: { traceId }
|
|
2471
|
+
},
|
|
2472
|
+
error$1
|
|
2473
|
+
);
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2236
2476
|
async updateSpan(args) {
|
|
2237
2477
|
const { traceId, spanId, updates } = args;
|
|
2238
2478
|
try {
|
|
@@ -2747,6 +2987,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS} (
|
|
|
2747
2987
|
timestamp DateTime64(3, 'UTC'),
|
|
2748
2988
|
|
|
2749
2989
|
-- IDs
|
|
2990
|
+
metricId String,
|
|
2750
2991
|
traceId Nullable(String),
|
|
2751
2992
|
spanId Nullable(String),
|
|
2752
2993
|
experimentId Nullable(String),
|
|
@@ -2794,9 +3035,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_METRIC_EVENTS} (
|
|
|
2794
3035
|
metadata Nullable(String),
|
|
2795
3036
|
scope Nullable(String)
|
|
2796
3037
|
)
|
|
2797
|
-
ENGINE =
|
|
3038
|
+
ENGINE = ReplacingMergeTree
|
|
2798
3039
|
PARTITION BY toDate(timestamp)
|
|
2799
|
-
ORDER BY (name, timestamp)
|
|
3040
|
+
ORDER BY (name, timestamp, metricId)
|
|
2800
3041
|
`;
|
|
2801
3042
|
var LOG_EVENTS_DDL = `
|
|
2802
3043
|
CREATE TABLE IF NOT EXISTS ${TABLE_LOG_EVENTS} (
|
|
@@ -2804,6 +3045,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_LOG_EVENTS} (
|
|
|
2804
3045
|
timestamp DateTime64(3, 'UTC'),
|
|
2805
3046
|
|
|
2806
3047
|
-- IDs
|
|
3048
|
+
logId String,
|
|
2807
3049
|
traceId Nullable(String),
|
|
2808
3050
|
spanId Nullable(String),
|
|
2809
3051
|
experimentId Nullable(String),
|
|
@@ -2846,10 +3088,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_LOG_EVENTS} (
|
|
|
2846
3088
|
metadata Nullable(String),
|
|
2847
3089
|
scope Nullable(String)
|
|
2848
3090
|
)
|
|
2849
|
-
ENGINE =
|
|
3091
|
+
ENGINE = ReplacingMergeTree
|
|
2850
3092
|
PARTITION BY toDate(timestamp)
|
|
2851
|
-
ORDER BY (timestamp,
|
|
2852
|
-
SETTINGS allow_nullable_key = 1
|
|
3093
|
+
ORDER BY (timestamp, logId)
|
|
2853
3094
|
`;
|
|
2854
3095
|
var SCORE_EVENTS_DDL = `
|
|
2855
3096
|
CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
|
|
@@ -2857,6 +3098,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
|
|
|
2857
3098
|
timestamp DateTime64(3, 'UTC'),
|
|
2858
3099
|
|
|
2859
3100
|
-- IDs
|
|
3101
|
+
scoreId String,
|
|
2860
3102
|
traceId Nullable(String),
|
|
2861
3103
|
spanId Nullable(String),
|
|
2862
3104
|
experimentId Nullable(String),
|
|
@@ -2906,9 +3148,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_SCORE_EVENTS} (
|
|
|
2906
3148
|
metadata Nullable(String),
|
|
2907
3149
|
scope Nullable(String)
|
|
2908
3150
|
)
|
|
2909
|
-
ENGINE =
|
|
3151
|
+
ENGINE = ReplacingMergeTree
|
|
2910
3152
|
PARTITION BY toDate(timestamp)
|
|
2911
|
-
ORDER BY (traceId, timestamp)
|
|
3153
|
+
ORDER BY (traceId, timestamp, scoreId)
|
|
2912
3154
|
SETTINGS allow_nullable_key = 1
|
|
2913
3155
|
`;
|
|
2914
3156
|
var FEEDBACK_EVENTS_DDL = `
|
|
@@ -2917,6 +3159,7 @@ CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS} (
|
|
|
2917
3159
|
timestamp DateTime64(3, 'UTC'),
|
|
2918
3160
|
|
|
2919
3161
|
-- IDs
|
|
3162
|
+
feedbackId String,
|
|
2920
3163
|
traceId Nullable(String),
|
|
2921
3164
|
spanId Nullable(String),
|
|
2922
3165
|
experimentId Nullable(String),
|
|
@@ -2969,9 +3212,9 @@ CREATE TABLE IF NOT EXISTS ${TABLE_FEEDBACK_EVENTS} (
|
|
|
2969
3212
|
metadata Nullable(String),
|
|
2970
3213
|
scope Nullable(String)
|
|
2971
3214
|
)
|
|
2972
|
-
ENGINE =
|
|
3215
|
+
ENGINE = ReplacingMergeTree
|
|
2973
3216
|
PARTITION BY toDate(timestamp)
|
|
2974
|
-
ORDER BY (traceId, timestamp)
|
|
3217
|
+
ORDER BY (traceId, timestamp, feedbackId)
|
|
2975
3218
|
SETTINGS allow_nullable_key = 1
|
|
2976
3219
|
`;
|
|
2977
3220
|
var DISCOVERY_VALUES_DDL = `
|
|
@@ -3326,6 +3569,7 @@ function spanRecordToRow(span) {
|
|
|
3326
3569
|
}
|
|
3327
3570
|
function rowToLogRecord(row) {
|
|
3328
3571
|
return {
|
|
3572
|
+
logId: row.logId,
|
|
3329
3573
|
timestamp: toDate(row.timestamp),
|
|
3330
3574
|
level: row.level,
|
|
3331
3575
|
message: row.message,
|
|
@@ -3362,6 +3606,7 @@ function rowToLogRecord(row) {
|
|
|
3362
3606
|
}
|
|
3363
3607
|
function logRecordToRow(log) {
|
|
3364
3608
|
return {
|
|
3609
|
+
logId: log.logId,
|
|
3365
3610
|
timestamp: toISOString(log.timestamp),
|
|
3366
3611
|
level: log.level,
|
|
3367
3612
|
message: log.message,
|
|
@@ -3398,6 +3643,7 @@ function logRecordToRow(log) {
|
|
|
3398
3643
|
}
|
|
3399
3644
|
function rowToMetricRecord(row) {
|
|
3400
3645
|
return {
|
|
3646
|
+
metricId: row.metricId,
|
|
3401
3647
|
timestamp: toDate(row.timestamp),
|
|
3402
3648
|
name: row.name,
|
|
3403
3649
|
value: Number(row.value),
|
|
@@ -3439,6 +3685,7 @@ function rowToMetricRecord(row) {
|
|
|
3439
3685
|
}
|
|
3440
3686
|
function metricRecordToRow(metric) {
|
|
3441
3687
|
return {
|
|
3688
|
+
metricId: metric.metricId,
|
|
3442
3689
|
timestamp: toISOString(metric.timestamp),
|
|
3443
3690
|
name: metric.name,
|
|
3444
3691
|
value: metric.value,
|
|
@@ -3480,6 +3727,7 @@ function metricRecordToRow(metric) {
|
|
|
3480
3727
|
}
|
|
3481
3728
|
function rowToScoreRecord(row) {
|
|
3482
3729
|
return {
|
|
3730
|
+
scoreId: row.scoreId,
|
|
3483
3731
|
timestamp: toDate(row.timestamp),
|
|
3484
3732
|
// Core score/feedback shapes still type traceId as required for now.
|
|
3485
3733
|
traceId: nullableString(row.traceId),
|
|
@@ -3522,6 +3770,7 @@ function scoreRecordToRow(score) {
|
|
|
3522
3770
|
const metadata = score.metadata ?? null;
|
|
3523
3771
|
const scoreSource = score.scoreSource ?? score.source ?? null;
|
|
3524
3772
|
return {
|
|
3773
|
+
scoreId: score.scoreId,
|
|
3525
3774
|
timestamp: toISOString(score.timestamp),
|
|
3526
3775
|
traceId: score.traceId ?? null,
|
|
3527
3776
|
spanId: score.spanId ?? null,
|
|
@@ -3564,6 +3813,7 @@ function rowToFeedbackRecord(row) {
|
|
|
3564
3813
|
const feedbackSource = nullableString(row.feedbackSource);
|
|
3565
3814
|
const feedbackUserId = nullableString(row.feedbackUserId) ?? nullableString(row.userId);
|
|
3566
3815
|
return {
|
|
3816
|
+
feedbackId: row.feedbackId,
|
|
3567
3817
|
timestamp: toDate(row.timestamp),
|
|
3568
3818
|
// Core score/feedback shapes still type traceId as required for now.
|
|
3569
3819
|
traceId: nullableString(row.traceId),
|
|
@@ -3607,6 +3857,7 @@ function feedbackRecordToRow(feedback) {
|
|
|
3607
3857
|
const feedbackSource = feedback.feedbackSource ?? feedback.source ?? "";
|
|
3608
3858
|
const feedbackUserId = feedback.feedbackUserId ?? feedback.userId ?? null;
|
|
3609
3859
|
return {
|
|
3860
|
+
feedbackId: feedback.feedbackId,
|
|
3610
3861
|
timestamp: toISOString(feedback.timestamp),
|
|
3611
3862
|
traceId: feedback.traceId ?? null,
|
|
3612
3863
|
spanId: feedback.spanId ?? null,
|
|
@@ -4726,6 +4977,93 @@ async function getMetricLabelValues(client, args) {
|
|
|
4726
4977
|
);
|
|
4727
4978
|
return { values: rows.map((r) => r.value) };
|
|
4728
4979
|
}
|
|
4980
|
+
var SIGNAL_MIGRATIONS = [
|
|
4981
|
+
{ table: TABLE_METRIC_EVENTS, createDDL: METRIC_EVENTS_DDL, idColumn: "metricId" },
|
|
4982
|
+
{ table: TABLE_LOG_EVENTS, createDDL: LOG_EVENTS_DDL, idColumn: "logId" },
|
|
4983
|
+
{ table: TABLE_SCORE_EVENTS, createDDL: SCORE_EVENTS_DDL, idColumn: "scoreId" },
|
|
4984
|
+
{ table: TABLE_FEEDBACK_EVENTS, createDDL: FEEDBACK_EVENTS_DDL, idColumn: "feedbackId" }
|
|
4985
|
+
];
|
|
4986
|
+
async function getTableEngine(client, table) {
|
|
4987
|
+
const result = await client.query({
|
|
4988
|
+
query: `SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = {table:String}`,
|
|
4989
|
+
query_params: { table },
|
|
4990
|
+
format: "JSONEachRow"
|
|
4991
|
+
});
|
|
4992
|
+
const rows = await result.json();
|
|
4993
|
+
return rows[0]?.engine ?? null;
|
|
4994
|
+
}
|
|
4995
|
+
async function getTableColumns(client, table) {
|
|
4996
|
+
const result = await client.query({ query: `DESCRIBE TABLE ${table}`, format: "JSONEachRow" });
|
|
4997
|
+
const rows = await result.json();
|
|
4998
|
+
return rows.map((r) => r.name);
|
|
4999
|
+
}
|
|
5000
|
+
function buildTemporaryTableDDL(createDDL, table, tempTable) {
|
|
5001
|
+
return createDDL.replace(`CREATE TABLE IF NOT EXISTS ${table}`, `CREATE TABLE ${tempTable}`);
|
|
5002
|
+
}
|
|
5003
|
+
async function dropTableIfExists(client, table) {
|
|
5004
|
+
if (await getTableEngine(client, table) !== null) {
|
|
5005
|
+
await client.command({ query: `DROP TABLE ${table}` });
|
|
5006
|
+
}
|
|
5007
|
+
}
|
|
5008
|
+
function createMigrationError(args, error$1) {
|
|
5009
|
+
return new error.MastraError(
|
|
5010
|
+
{
|
|
5011
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "MIGRATE_SIGNAL_TABLES", "FAILED"),
|
|
5012
|
+
domain: error.ErrorDomain.STORAGE,
|
|
5013
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
5014
|
+
details: args
|
|
5015
|
+
},
|
|
5016
|
+
error$1
|
|
5017
|
+
);
|
|
5018
|
+
}
|
|
5019
|
+
async function checkSignalTablesMigrationStatus(client) {
|
|
5020
|
+
const tables = [];
|
|
5021
|
+
for (const { table, idColumn } of SIGNAL_MIGRATIONS) {
|
|
5022
|
+
const engine = await getTableEngine(client, table);
|
|
5023
|
+
if (!engine || engine === "ReplacingMergeTree") {
|
|
5024
|
+
continue;
|
|
5025
|
+
}
|
|
5026
|
+
tables.push({ table, engine, idColumn });
|
|
5027
|
+
}
|
|
5028
|
+
return {
|
|
5029
|
+
needsMigration: tables.length > 0,
|
|
5030
|
+
tables
|
|
5031
|
+
};
|
|
5032
|
+
}
|
|
5033
|
+
async function migrateSignalTables(client, logger) {
|
|
5034
|
+
for (const { table, createDDL, idColumn } of SIGNAL_MIGRATIONS) {
|
|
5035
|
+
const engine = await getTableEngine(client, table);
|
|
5036
|
+
if (!engine || engine === "ReplacingMergeTree") continue;
|
|
5037
|
+
logger?.info?.(`Migrating ${table} from ${engine} to ReplacingMergeTree with ${idColumn} column`);
|
|
5038
|
+
const temp = `${table}_migrating_${Date.now()}`;
|
|
5039
|
+
try {
|
|
5040
|
+
await client.command({ query: buildTemporaryTableDDL(createDDL, table, temp) });
|
|
5041
|
+
const newColumns = await getTableColumns(client, temp);
|
|
5042
|
+
const currentColumns = new Set(await getTableColumns(client, table));
|
|
5043
|
+
const columnList = newColumns.map((c) => `"${c}"`).join(", ");
|
|
5044
|
+
const selectExprs = newColumns.map((c) => {
|
|
5045
|
+
if (c === idColumn) {
|
|
5046
|
+
return currentColumns.has(c) ? `COALESCE(nullIf("${c}", ''), toString(generateUUIDv4())) AS "${c}"` : `toString(generateUUIDv4()) AS "${c}"`;
|
|
5047
|
+
}
|
|
5048
|
+
return currentColumns.has(c) ? `"${c}"` : `NULL AS "${c}"`;
|
|
5049
|
+
}).join(", ");
|
|
5050
|
+
await client.command({
|
|
5051
|
+
query: `INSERT INTO ${temp} (${columnList}) SELECT ${selectExprs} FROM ${table}`
|
|
5052
|
+
});
|
|
5053
|
+
await client.command({ query: `EXCHANGE TABLES ${temp} AND ${table}` });
|
|
5054
|
+
await client.command({ query: `DROP TABLE ${temp}` });
|
|
5055
|
+
logger?.info?.(`Successfully migrated ${table}`);
|
|
5056
|
+
} catch (error) {
|
|
5057
|
+
logger?.error?.(`Migration of ${table} failed: ${error.message}`);
|
|
5058
|
+
try {
|
|
5059
|
+
await dropTableIfExists(client, temp);
|
|
5060
|
+
} catch (restoreError) {
|
|
5061
|
+
logger?.error?.(`Failed to clean up temporary table ${temp}: ${restoreError.message}`);
|
|
5062
|
+
}
|
|
5063
|
+
throw createMigrationError({ table, idColumn }, error);
|
|
5064
|
+
}
|
|
5065
|
+
}
|
|
5066
|
+
}
|
|
4729
5067
|
var SCORE_TYPED_COLUMNS = /* @__PURE__ */ new Set([
|
|
4730
5068
|
"timestamp",
|
|
4731
5069
|
"traceId",
|
|
@@ -5189,6 +5527,31 @@ async function getTrace(client, args) {
|
|
|
5189
5527
|
const spans = rows.map(rowToSpanRecord);
|
|
5190
5528
|
return { traceId: args.traceId, spans };
|
|
5191
5529
|
}
|
|
5530
|
+
async function getTraceLight(client, args) {
|
|
5531
|
+
const result = await client.query({
|
|
5532
|
+
query: `
|
|
5533
|
+
SELECT traceId, spanId, parentSpanId, name,
|
|
5534
|
+
entityType, entityId, entityName,
|
|
5535
|
+
spanType, error, isEvent,
|
|
5536
|
+
startedAt, endedAt
|
|
5537
|
+
FROM (
|
|
5538
|
+
SELECT *
|
|
5539
|
+
FROM ${TABLE_SPAN_EVENTS}
|
|
5540
|
+
WHERE traceId = {traceId:String}
|
|
5541
|
+
ORDER BY dedupeKey, endedAt DESC
|
|
5542
|
+
LIMIT 1 BY dedupeKey
|
|
5543
|
+
)
|
|
5544
|
+
ORDER BY startedAt ASC
|
|
5545
|
+
`,
|
|
5546
|
+
query_params: { traceId: args.traceId },
|
|
5547
|
+
format: "JSONEachRow",
|
|
5548
|
+
clickhouse_settings: CH_SETTINGS
|
|
5549
|
+
});
|
|
5550
|
+
const rows = await result.json();
|
|
5551
|
+
if (!rows || rows.length === 0) return null;
|
|
5552
|
+
const spans = rows.map(rowToSpanRecord);
|
|
5553
|
+
return { traceId: args.traceId, spans };
|
|
5554
|
+
}
|
|
5192
5555
|
async function batchDeleteTraces(client, args) {
|
|
5193
5556
|
if (args.traceIds.length === 0) return;
|
|
5194
5557
|
const params = {};
|
|
@@ -5217,6 +5580,32 @@ async function batchDeleteTraces(client, args) {
|
|
|
5217
5580
|
}
|
|
5218
5581
|
|
|
5219
5582
|
// src/storage/domains/observability/v-next/index.ts
|
|
5583
|
+
function buildSignalMigrationRequiredMessage(args) {
|
|
5584
|
+
const tableList = args.tables.map((table) => ` - ${table.table} (${table.engine})`).join("\n");
|
|
5585
|
+
return `
|
|
5586
|
+
===========================================================================
|
|
5587
|
+
MIGRATION REQUIRED: ${args.store} observability signal tables need signal IDs
|
|
5588
|
+
===========================================================================
|
|
5589
|
+
|
|
5590
|
+
The following signal tables still use the legacy schema and must be migrated
|
|
5591
|
+
before observability storage can initialize:
|
|
5592
|
+
|
|
5593
|
+
${tableList}
|
|
5594
|
+
|
|
5595
|
+
To fix this, run the manual migration command:
|
|
5596
|
+
|
|
5597
|
+
npx mastra migrate
|
|
5598
|
+
|
|
5599
|
+
This command will:
|
|
5600
|
+
1. Create replacement signal tables with signal-ID dedupe keys
|
|
5601
|
+
2. Backfill missing signal IDs for legacy rows
|
|
5602
|
+
3. Swap the migrated tables into place
|
|
5603
|
+
|
|
5604
|
+
WARNING: This migration recreates the signal tables and may take significant
|
|
5605
|
+
time for large databases. Please ensure you have a backup before proceeding.
|
|
5606
|
+
===========================================================================
|
|
5607
|
+
`;
|
|
5608
|
+
}
|
|
5220
5609
|
var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilityStorage {
|
|
5221
5610
|
#client;
|
|
5222
5611
|
#retention;
|
|
@@ -5230,6 +5619,18 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
5230
5619
|
// Initialization
|
|
5231
5620
|
// -------------------------------------------------------------------------
|
|
5232
5621
|
async init() {
|
|
5622
|
+
const migrationStatus = await checkSignalTablesMigrationStatus(this.#client);
|
|
5623
|
+
if (migrationStatus.needsMigration) {
|
|
5624
|
+
throw new error.MastraError({
|
|
5625
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "MIGRATION_REQUIRED", "SIGNAL_TABLES"),
|
|
5626
|
+
domain: error.ErrorDomain.STORAGE,
|
|
5627
|
+
category: error.ErrorCategory.USER,
|
|
5628
|
+
text: buildSignalMigrationRequiredMessage({
|
|
5629
|
+
store: "ClickHouse",
|
|
5630
|
+
tables: migrationStatus.tables.map(({ table, engine }) => ({ table, engine }))
|
|
5631
|
+
})
|
|
5632
|
+
});
|
|
5633
|
+
}
|
|
5233
5634
|
try {
|
|
5234
5635
|
for (const ddl of [...ALL_TABLE_DDL, ...ALL_MV_DDL]) {
|
|
5235
5636
|
await this.#client.command({ query: ddl });
|
|
@@ -5244,6 +5645,9 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
5244
5645
|
}
|
|
5245
5646
|
}
|
|
5246
5647
|
} catch (error$1) {
|
|
5648
|
+
if (error$1 instanceof error.MastraError) {
|
|
5649
|
+
throw error$1;
|
|
5650
|
+
}
|
|
5247
5651
|
throw new error.MastraError(
|
|
5248
5652
|
{
|
|
5249
5653
|
id: storage.createStorageErrorId("CLICKHOUSE", "VNEXT_INIT", "FAILED"),
|
|
@@ -5265,6 +5669,29 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
5265
5669
|
} catch {
|
|
5266
5670
|
}
|
|
5267
5671
|
}
|
|
5672
|
+
/**
|
|
5673
|
+
* Manually migrate legacy signal tables to the signal-ID ReplacingMergeTree schema.
|
|
5674
|
+
* The public method name is historical; the CLI still calls `migrateSpans()`
|
|
5675
|
+
* for observability migrations even though this now also migrates signal tables.
|
|
5676
|
+
*/
|
|
5677
|
+
async migrateSpans() {
|
|
5678
|
+
const migrationStatus = await checkSignalTablesMigrationStatus(this.#client);
|
|
5679
|
+
if (!migrationStatus.needsMigration) {
|
|
5680
|
+
return {
|
|
5681
|
+
success: true,
|
|
5682
|
+
alreadyMigrated: true,
|
|
5683
|
+
duplicatesRemoved: 0,
|
|
5684
|
+
message: "Migration already complete. Signal tables already use signal-ID dedupe keys."
|
|
5685
|
+
};
|
|
5686
|
+
}
|
|
5687
|
+
await migrateSignalTables(this.#client, this.logger);
|
|
5688
|
+
return {
|
|
5689
|
+
success: true,
|
|
5690
|
+
alreadyMigrated: false,
|
|
5691
|
+
duplicatesRemoved: 0,
|
|
5692
|
+
message: `Migration complete. Migrated signal tables: ${migrationStatus.tables.map((t) => t.table).join(", ")}.`
|
|
5693
|
+
};
|
|
5694
|
+
}
|
|
5268
5695
|
// -------------------------------------------------------------------------
|
|
5269
5696
|
// Strategy
|
|
5270
5697
|
// -------------------------------------------------------------------------
|
|
@@ -5360,6 +5787,22 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
5360
5787
|
);
|
|
5361
5788
|
}
|
|
5362
5789
|
}
|
|
5790
|
+
async getTraceLight(args) {
|
|
5791
|
+
try {
|
|
5792
|
+
return await getTraceLight(this.#client, args);
|
|
5793
|
+
} catch (error$1) {
|
|
5794
|
+
if (error$1 instanceof error.MastraError) throw error$1;
|
|
5795
|
+
throw new error.MastraError(
|
|
5796
|
+
{
|
|
5797
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "GET_TRACE_LIGHT", "FAILED"),
|
|
5798
|
+
domain: error.ErrorDomain.STORAGE,
|
|
5799
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
5800
|
+
details: { traceId: args.traceId }
|
|
5801
|
+
},
|
|
5802
|
+
error$1
|
|
5803
|
+
);
|
|
5804
|
+
}
|
|
5805
|
+
}
|
|
5363
5806
|
async listTraces(args) {
|
|
5364
5807
|
try {
|
|
5365
5808
|
return await listTraces(this.#client, args);
|
|
@@ -6648,7 +7091,8 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
6648
7091
|
workflows,
|
|
6649
7092
|
scores,
|
|
6650
7093
|
memory,
|
|
6651
|
-
observability
|
|
7094
|
+
observability,
|
|
7095
|
+
backgroundTasks: new BackgroundTasksStorageClickhouse(domainConfig)
|
|
6652
7096
|
};
|
|
6653
7097
|
}
|
|
6654
7098
|
async optimizeTable({ tableName }) {
|
|
@@ -6707,6 +7151,7 @@ var ClickhouseStore = class extends storage.MastraCompositeStore {
|
|
|
6707
7151
|
}
|
|
6708
7152
|
};
|
|
6709
7153
|
|
|
7154
|
+
exports.BackgroundTasksStorageClickhouse = BackgroundTasksStorageClickhouse;
|
|
6710
7155
|
exports.COLUMN_TYPES = COLUMN_TYPES;
|
|
6711
7156
|
exports.ClickhouseStore = ClickhouseStore;
|
|
6712
7157
|
exports.MemoryStorageClickhouse = MemoryStorageClickhouse;
|