@mastra/clickhouse 1.11.1-alpha.0 → 1.12.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,36 @@
1
1
  # @mastra/clickhouse
2
2
 
3
+ ## 1.12.0-alpha.0
4
+
5
+ ### Minor Changes
6
+
7
+ - Added legacy-to-VNext span migration for ClickHouse observability storage. Customers using `ObservabilityStorageClickhouse` (legacy) can now run `npx mastra migrate` to copy historical spans from `mastra_ai_spans` to the VNext `mastra_span_events` schema. The migration handles column mapping, batches by day for memory safety, deduplicates legacy rows, converts empty-string parentSpanId to NULL for correct root span detection, and supports both old (`Nullable(String)` JSON) and new (`Array(String)`) tags schemas. The legacy table is preserved as a backup after migration. ([#19050](https://github.com/mastra-ai/mastra/pull/19050))
8
+
9
+ ### Patch Changes
10
+
11
+ - Updated dependencies [[`e955965`](https://github.com/mastra-ai/mastra/commit/e955965dce575a903e37cf054d28ea99aa48785e), [`860ef7e`](https://github.com/mastra-ai/mastra/commit/860ef7e77d92b63469cbe5857aa1e626197e43e9), [`17e818c`](https://github.com/mastra-ai/mastra/commit/17e818c51a958ba90641b1a959dc38faf8c034e9), [`4451dfe`](https://github.com/mastra-ai/mastra/commit/4451dfe857428e7abcc0261a507a2e186dae6d47), [`1d39058`](https://github.com/mastra-ai/mastra/commit/1d39058e548efd691799985d5c8af2737f1c3bd2)]:
12
+ - @mastra/core@1.51.0-alpha.2
13
+
14
+ ## 1.11.1
15
+
16
+ ### Patch Changes
17
+
18
+ - Added optional `organizationId` and `projectId` fields to scores for multi-tenant isolation. Scores can now be saved with tenancy metadata and the `listScoresBy*` methods accept a `filters` option to scope results by organization and project. ([#18331](https://github.com/mastra-ai/mastra/pull/18331))
19
+
20
+ ```ts
21
+ await storage.saveScore({ ...score, organizationId: 'org-a', projectId: 'proj-1' });
22
+
23
+ const result = await storage.listScoresByScorerId({
24
+ scorerId,
25
+ filters: { organizationId: 'org-a', projectId: 'proj-1' },
26
+ });
27
+ ```
28
+
29
+ `projectId` identifies the project scope, separate from `resourceId` which continues to mean the agent memory resource.
30
+
31
+ - Updated dependencies [[`700619b`](https://github.com/mastra-ai/mastra/commit/700619b61d572e592cbaaf758121d168844ca4d2), [`0f69865`](https://github.com/mastra-ai/mastra/commit/0f69865aced225d98eac812e22699dc445ee18cb), [`9250acd`](https://github.com/mastra-ai/mastra/commit/9250acd1357f0f1f33d0dcca16f9655084c58eca), [`0c3d4bc`](https://github.com/mastra-ai/mastra/commit/0c3d4bcae13ea3699d379403e6f350d5cf4efe9f), [`cc440a3`](https://github.com/mastra-ai/mastra/commit/cc440a39400d8ce06655462b26c1666a1b3d4320), [`6a61846`](https://github.com/mastra-ai/mastra/commit/6a61846eeda29fb714549b70f1bee2bf6b141c44), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`17369b2`](https://github.com/mastra-ai/mastra/commit/17369b25250561e9ed994ae509be1d15bfb33bcb), [`c64c2a8`](https://github.com/mastra-ai/mastra/commit/c64c2a8503a50252f9ca6b8e8c54cadee31b92a2), [`bcae929`](https://github.com/mastra-ai/mastra/commit/bcae929945cbf265bd9f327cc715ecafa072b5b9), [`ea6327b`](https://github.com/mastra-ai/mastra/commit/ea6327ba2d63ca647804bc97b347e03a58617162), [`3439fa8`](https://github.com/mastra-ai/mastra/commit/3439fa836ecfcaa257b40c20b30ac2a8be22e9ea), [`85107f2`](https://github.com/mastra-ai/mastra/commit/85107f2758b527147fccbedff962961927c2d3b8), [`b33822e`](https://github.com/mastra-ai/mastra/commit/b33822e8d470884954b02f7b0745407ee4ef74b1), [`06e2680`](https://github.com/mastra-ai/mastra/commit/06e26806b51d2cbd858afdc66daa2b86ff3ba64a), [`06ff9e0`](https://github.com/mastra-ai/mastra/commit/06ff9e0befd1d642ab87ff749285ee4091205c7e), [`d5c11e3`](https://github.com/mastra-ai/mastra/commit/d5c11e3ba5045969caa7272a7bd1fd141c93ab6c), [`7f5e1ff`](https://github.com/mastra-ai/mastra/commit/7f5e1ff695a92f672bb3976363925d1e9136b54a), [`ff80671`](https://github.com/mastra-ai/mastra/commit/ff8067185e208b27198b4e5b71803013175c3643), [`b8375c1`](https://github.com/mastra-ai/mastra/commit/b8375c1f8fe905df8ae2ae9a893bb365f17aec4e), [`dab1257`](https://github.com/mastra-ai/mastra/commit/dab1257b64e4ed576dc5038bb7a3f7072338bc9f), [`1240f05`](https://github.com/mastra-ai/mastra/commit/1240f051c8e5371f1c014448bf37b1a1b9a05e47), [`705ff39`](https://github.com/mastra-ai/mastra/commit/705ff3969e57214ff2fdaf3815d751dd558886ed), [`e6fbd5b`](https://github.com/mastra-ai/mastra/commit/e6fbd5bfdc28e92c0c0433f29aa1bc152d3430f6), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`6f2026c`](https://github.com/mastra-ai/mastra/commit/6f2026cdf114ff1e21e49133ca774ec7d5085059), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`003f35d`](https://github.com/mastra-ai/mastra/commit/003f35d19e07b23b4bacc591c8bc0c59b42124ae), [`f890eda`](https://github.com/mastra-ai/mastra/commit/f890eda2c8a2ae83d9b30bc6d85842f93b6c266b), [`1340fb7`](https://github.com/mastra-ai/mastra/commit/1340fb76262a3ca062130aa71859f07257a0a5a4)]:
32
+ - @mastra/core@1.49.0
33
+
3
34
  ## 1.11.1-alpha.0
4
35
 
5
36
  ### Patch Changes
@@ -3,7 +3,7 @@ name: mastra-clickhouse
3
3
  description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
4
4
  metadata:
5
5
  package: "@mastra/clickhouse"
6
- version: "1.11.1-alpha.0"
6
+ version: "1.12.0-alpha.0"
7
7
  ---
8
8
 
9
9
  ## When to use
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.11.1-alpha.0",
2
+ "version": "1.12.0-alpha.0",
3
3
  "package": "@mastra/clickhouse",
4
4
  "exports": {},
5
5
  "modules": {}
@@ -101,6 +101,38 @@ const observabilityStore = new ObservabilityStorageClickhouse({
101
101
 
102
102
  New projects should use `ObservabilityStorageClickhouseVNext` instead.
103
103
 
104
+ ### Migrating from legacy to vNext
105
+
106
+ To migrate historical spans from the legacy `mastra_ai_spans` table to the vNext schema, run:
107
+
108
+ **npm**:
109
+
110
+ ```bash
111
+ npx mastra migrate
112
+ ```
113
+
114
+ **pnpm**:
115
+
116
+ ```bash
117
+ pnpm dlx mastra migrate
118
+ ```
119
+
120
+ **Yarn**:
121
+
122
+ ```bash
123
+ yarn dlx mastra migrate
124
+ ```
125
+
126
+ **Bun**:
127
+
128
+ ```bash
129
+ bun x mastra migrate
130
+ ```
131
+
132
+ The migration copies span data from `mastra_ai_spans` into `mastra_span_events` in day-sized batches. It handles column mapping, deduplicates legacy rows, and preserves the original table as a backup. After migration, traces appear in Studio through the vNext adapter.
133
+
134
+ > **Note:** The legacy table is not deleted. Drop it manually after verifying the migration.
135
+
104
136
  ### ClickHouse for every domain
105
137
 
106
138
  `ClickhouseStoreVNext` backs the `memory`, `workflows`, and `observability` domains with ClickHouse and uses the vNext observability adapter automatically. Use it when you want ClickHouse to back the entire application without wiring a composite store manually.
@@ -174,19 +206,19 @@ The same `client` form is accepted by `ObservabilityStorageClickhouse` and `Obse
174
206
 
175
207
  **id** (`string`): Unique identifier for this storage instance.
176
208
 
177
- **url** (`string`): ClickHouse server URL (for example, \`https\://your-instance.clickhouse.cloud:8443\` or \`http\://localhost:8123\`). Required when not passing a pre-configured \`client\`.
209
+ **url** (`string`): ClickHouse server URL (for example, https\://your-instance.clickhouse.cloud:8443 or http\://localhost:8123). Required when not passing a pre-configured client.
178
210
 
179
- **username** (`string`): ClickHouse username. Required when not passing a pre-configured \`client\`.
211
+ **username** (`string`): ClickHouse username. Required when not passing a pre-configured client.
180
212
 
181
- **password** (`string`): ClickHouse password. Required when not passing a pre-configured \`client\`. It can be an empty string for the default user on a local instance.
213
+ **password** (`string`): ClickHouse password. Required when not passing a pre-configured client. It can be an empty string for the default user on a local instance.
182
214
 
183
- **client** (`ClickHouseClient`): Pre-configured ClickHouse client from \`@clickhouse/client\`. Use this when you need custom request settings. Mutually exclusive with the credential fields above.
215
+ **client** (`ClickHouseClient`): Pre-configured ClickHouse client from @clickhouse/client. Use this when you need custom request settings. Mutually exclusive with the credential fields above.
184
216
 
185
- **ttl** (`object`): Per-table TTL configuration applied at table creation time. Accepts row-level and column-level TTLs in interval units from \`NANOSECOND\` through \`YEAR\`.
217
+ **ttl** (`object`): Per-table TTL configuration applied at table creation time. Accepts row-level and column-level TTLs in interval units from NANOSECOND through YEAR.
186
218
 
187
- **replication** (`{ cluster?: string; zookeeperPath?: string; replicaName?: string }`): Opt-in replicated table configuration for multi-replica ClickHouse clusters. When set, Mastra creates replicated MergeTree tables. When \`cluster\` is set, Mastra also adds \`ON CLUSTER\` to Mastra-owned data definition language (DDL).
219
+ **replication** (`{ cluster?: string; zookeeperPath?: string; replicaName?: string }`): Opt-in replicated table configuration for multi-replica ClickHouse clusters. When set, Mastra creates replicated MergeTree tables. When cluster is set, Mastra also adds ON CLUSTER to Mastra-owned data definition language (DDL).
188
220
 
189
- **disableInit** (`boolean`): When \`true\`, the store does not run table creation or migrations on first use. Call \`storage.init()\` explicitly from your deployment scripts. (Default: `false`)
221
+ **disableInit** (`boolean`): When true, the store does not run table creation or migrations on first use. Call storage.init() explicitly from your deployment scripts. (Default: `false`)
190
222
 
191
223
  `ClickhouseStore` also accepts every option from `ClickHouseClientConfigOptions` (such as `database`, `request_timeout`, `compression`, `keep_alive`, and `max_open_connections`).
192
224
 
@@ -211,7 +243,7 @@ When `replication` is set, Mastra rewrites its `MergeTree` and `ReplacingMergeTr
211
243
  - `zookeeperPath`: `'/clickhouse/tables/{shard}/{database}/{table}'`
212
244
  - `replicaName`: `'{replica}'`
213
245
 
214
- The defaults match the most common self-managed convention. If your cluster's existing tables use a different layout (for example `/clickhouse/tables/{shard}/{table}` without the `{database}` segment), set `zookeeperPath` explicitly to match. Mastra does not read your cluster's convention from Keeper, so a mismatched default writes Mastra's metadata to a separate branch from the rest of the cluster.
246
+ The defaults match the most common self-managed convention. If your cluster's existing tables use a different layout (for example `/clickhouse/tables/{shard}/{table}` without the `{database}` segment), set `zookeeperPath` explicitly to match. Mastra doesn't read your cluster's convention from Keeper, so a mismatched default writes Mastra's metadata to a separate branch from the rest of the cluster.
215
247
 
216
248
  ```typescript
217
249
  new ClickhouseStoreVNext({
@@ -231,7 +263,7 @@ Manual maintenance such as `optimizeTable()` and `materializeTtl()` runs on ever
231
263
 
232
264
  If existing Mastra tables use local `MergeTree` or `ReplacingMergeTree` engines, initialization fails while `replication` is enabled. Mastra refuses to silently convert local tables because copy-and-swap is unsafe across replicas. To migrate, recreate the affected tables as `Replicated*` before enabling replication. The typical sequence is: rename the local table, run `CREATE TABLE ... ENGINE = ReplicatedMergeTree(...) ON CLUSTER ...`, run `INSERT INTO ... SELECT * FROM <renamed_local>`, then drop the renamed local table.
233
265
 
234
- Do not set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side, and explicit `ReplicatedMergeTree` engines produce incorrect DDL. `replication` is only for self-managed multi-replica clusters.
266
+ Don't set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side, and explicit `ReplicatedMergeTree` engines produce incorrect DDL. `replication` is only for self-managed multi-replica clusters.
235
267
 
236
268
  ### Observability domain options
237
269
 
@@ -37,25 +37,25 @@ You'll also need to install the storage providers you want to compose:
37
37
  **npm**:
38
38
 
39
39
  ```bash
40
- npm install @mastra/pg@latest @mastra/libsql@latest
40
+ npm install @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
41
41
  ```
42
42
 
43
43
  **pnpm**:
44
44
 
45
45
  ```bash
46
- pnpm add @mastra/pg@latest @mastra/libsql@latest
46
+ pnpm add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
47
47
  ```
48
48
 
49
49
  **Yarn**:
50
50
 
51
51
  ```bash
52
- yarn add @mastra/pg@latest @mastra/libsql@latest
52
+ yarn add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
53
53
  ```
54
54
 
55
55
  **Bun**:
56
56
 
57
57
  ```bash
58
- bun add @mastra/pg@latest @mastra/libsql@latest
58
+ bun add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
59
59
  ```
60
60
 
61
61
  ## Storage domains
@@ -124,15 +124,73 @@ export const mastra = new Mastra({
124
124
  })
125
125
  ```
126
126
 
127
+ ### Mixed backends
128
+
129
+ Use domain classes from each storage package to route different domains to different backends. The following example stores memory and workflow state in MongoDB, then routes observability to ClickHouse:
130
+
131
+ ```typescript
132
+ import { Mastra } from '@mastra/core'
133
+ import { MastraCompositeStore } from '@mastra/core/storage'
134
+ import { ObservabilityStorageClickhouse } from '@mastra/clickhouse'
135
+ import { MemoryStorageMongoDB, WorkflowsStorageMongoDB } from '@mastra/mongodb'
136
+
137
+ export const mastra = new Mastra({
138
+ storage: new MastraCompositeStore({
139
+ id: 'composite',
140
+ domains: {
141
+ memory: new MemoryStorageMongoDB({
142
+ uri: process.env.MONGODB_URI,
143
+ dbName: 'mastra_memory',
144
+ }),
145
+ workflows: new WorkflowsStorageMongoDB({
146
+ uri: process.env.MONGODB_URI,
147
+ dbName: 'mastra_workflows',
148
+ }),
149
+ observability: new ObservabilityStorageClickhouse({
150
+ url: process.env.CLICKHOUSE_URL,
151
+ username: process.env.CLICKHOUSE_USERNAME,
152
+ password: process.env.CLICKHOUSE_PASSWORD,
153
+ }),
154
+ },
155
+ }),
156
+ })
157
+ ```
158
+
159
+ ### Disabling a domain
160
+
161
+ Set a domain to `false` to disable it. A disabled domain doesn't fall back to `default`, so data for that domain isn't persisted:
162
+
163
+ ```typescript
164
+ import { MastraCompositeStore } from '@mastra/core/storage'
165
+ import { PostgresStore } from '@mastra/pg'
166
+ import { Mastra } from '@mastra/core'
167
+
168
+ const pgStore = new PostgresStore({
169
+ id: 'pg',
170
+ connectionString: process.env.DATABASE_URL,
171
+ })
172
+
173
+ export const mastra = new Mastra({
174
+ storage: new MastraCompositeStore({
175
+ id: 'composite',
176
+ default: pgStore,
177
+ domains: {
178
+ // don't persist traces and spans
179
+ observability: false,
180
+ },
181
+ }),
182
+ })
183
+ ```
184
+
127
185
  ## Options
128
186
 
129
187
  **id** (`string`): Unique identifier for this storage instance.
130
188
 
131
- **default** (`MastraCompositeStore`): Default storage adapter. Domains not explicitly specified in \`domains\` will use this storage's domains as fallbacks.
189
+ **default** (`MastraCompositeStore`): Default storage adapter. Domains not explicitly specified in domains will use this storage's domains as fallbacks.
132
190
 
133
191
  **disableInit** (`boolean`): When true, automatic initialization is disabled. You must call init() explicitly.
134
192
 
135
- **domains** (`object`): Individual domain overrides. Each domain can come from a different storage adapter. These take precedence over both \`editor\` and \`default\` storage.
193
+ **domains** (`object`): Individual domain overrides. Each domain can come from a different storage adapter. These take precedence over both editor and default storage. Set a domain to false to disable it entirely; a disabled domain does not fall back to editor or default.
136
194
 
137
195
  **domains.memory** (`MemoryStorage`): Storage for threads, messages, and resources.
138
196
 
@@ -274,6 +332,6 @@ const storage = new MastraCompositeStore({
274
332
  })
275
333
  ```
276
334
 
277
- Do not set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for the full config shape and operator notes.
335
+ Don't set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for the full config shape and operator notes.
278
336
 
279
337
  > **Info:** This approach is also required when using storage providers that don't support observability (like Convex, DynamoDB, or Cloudflare). See the [MastraStorageExporter documentation](https://mastra.ai/docs/observability/integrations/exporters/mastra-storage) for the full list of supported providers.
package/dist/index.cjs CHANGED
@@ -5949,6 +5949,11 @@ async function getTableColumns(client, table) {
5949
5949
  const rows = await result.json();
5950
5950
  return rows.map((r) => r.name);
5951
5951
  }
5952
+ async function getTableColumnTypes(client, table) {
5953
+ const result = await client.query({ query: `DESCRIBE TABLE ${table}`, format: "JSONEachRow" });
5954
+ const rows = await result.json();
5955
+ return new Map(rows.map((r) => [r.name, r.type]));
5956
+ }
5952
5957
  function buildTemporaryTableDDL(createDDL, table, tempTable) {
5953
5958
  return createDDL.replace(`CREATE TABLE IF NOT EXISTS ${table}`, `CREATE TABLE ${tempTable}`);
5954
5959
  }
@@ -5982,6 +5987,218 @@ async function checkSignalTablesMigrationStatus(client) {
5982
5987
  tables
5983
5988
  };
5984
5989
  }
5990
+ var TABLE_LEGACY_SPANS = "mastra_ai_spans";
5991
+ var PROMOTED_METADATA_KEYS = [
5992
+ "experimentId",
5993
+ "entityType",
5994
+ "entityId",
5995
+ "entityName",
5996
+ "entityVersionId",
5997
+ "parentEntityVersionId",
5998
+ "rootEntityVersionId",
5999
+ "userId",
6000
+ "organizationId",
6001
+ "resourceId",
6002
+ "runId",
6003
+ "sessionId",
6004
+ "threadId",
6005
+ "requestId",
6006
+ "environment",
6007
+ "executionSource",
6008
+ "serviceName"
6009
+ ];
6010
+ var TABLE_LEGACY_MIGRATION_DONE = "mastra_legacy_span_migration_done";
6011
+ async function hasMarkerRow(client) {
6012
+ const engine = await getTableEngine(client, TABLE_LEGACY_MIGRATION_DONE);
6013
+ if (!engine) return false;
6014
+ const result = await client.query({
6015
+ query: `SELECT count() as cnt FROM ${TABLE_LEGACY_MIGRATION_DONE}`,
6016
+ format: "JSONEachRow"
6017
+ });
6018
+ const rows = await result.json();
6019
+ return Number(rows[0]?.cnt ?? 0) > 0;
6020
+ }
6021
+ async function checkLegacySpanMigrationStatus(client) {
6022
+ if (await hasMarkerRow(client)) {
6023
+ return { needsMigration: false };
6024
+ }
6025
+ const engine = await getTableEngine(client, TABLE_LEGACY_SPANS);
6026
+ if (!engine) {
6027
+ return { needsMigration: false };
6028
+ }
6029
+ const countResult = await client.query({
6030
+ query: `SELECT count() as cnt FROM ${TABLE_LEGACY_SPANS}`,
6031
+ format: "JSONEachRow"
6032
+ });
6033
+ const countRows = await countResult.json();
6034
+ const legacyRowCount = Number(countRows[0]?.cnt ?? 0);
6035
+ if (legacyRowCount === 0) {
6036
+ return { needsMigration: false };
6037
+ }
6038
+ return { needsMigration: true };
6039
+ }
6040
+ var VNEXT_SPAN_COLUMNS = [
6041
+ "dedupeKey",
6042
+ "traceId",
6043
+ "spanId",
6044
+ "parentSpanId",
6045
+ "experimentId",
6046
+ "entityType",
6047
+ "entityId",
6048
+ "entityName",
6049
+ "entityVersionId",
6050
+ "parentEntityVersionId",
6051
+ "parentEntityType",
6052
+ "parentEntityId",
6053
+ "parentEntityName",
6054
+ "rootEntityVersionId",
6055
+ "rootEntityType",
6056
+ "rootEntityId",
6057
+ "rootEntityName",
6058
+ "userId",
6059
+ "organizationId",
6060
+ "resourceId",
6061
+ "runId",
6062
+ "sessionId",
6063
+ "threadId",
6064
+ "requestId",
6065
+ "environment",
6066
+ "executionSource",
6067
+ "serviceName",
6068
+ "name",
6069
+ "spanType",
6070
+ "isEvent",
6071
+ "startedAt",
6072
+ "endedAt",
6073
+ "tags",
6074
+ "metadataSearch",
6075
+ "attributes",
6076
+ "scope",
6077
+ "links",
6078
+ "input",
6079
+ "output",
6080
+ "error",
6081
+ "metadataRaw",
6082
+ "requestContext"
6083
+ ];
6084
+ function buildLegacySpanSelectExprs(legacyColumnTypes) {
6085
+ const has = (col) => legacyColumnTypes.has(col);
6086
+ const colOrNull = (legacy) => has(legacy) ? `"${legacy}"` : "NULL";
6087
+ const promotedIn = PROMOTED_METADATA_KEYS.map((k) => `'${k}'`).join(", ");
6088
+ const metadataSearchExpr = has("metadata") ? `CAST(arrayFilter(x -> x.1 NOT IN (${promotedIn}) AND trim(BOTH ' ' FROM x.2) != '', arrayMap(x -> (x.1, trim(BOTH ' ' FROM x.2)), JSONExtractKeysAndValues(COALESCE("metadata", '{}'), 'String'))), 'Map(String, String)')` : `CAST(map(), 'Map(String, String)')`;
6089
+ let tagsExpr = "[]";
6090
+ if (has("tags")) {
6091
+ const tagsType = legacyColumnTypes.get("tags");
6092
+ if (tagsType.includes("Array")) {
6093
+ tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), "tags")))`;
6094
+ } else {
6095
+ tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), JSONExtract(COALESCE("tags", '[]'), 'Array(String)'))))`;
6096
+ }
6097
+ }
6098
+ const exprs = {
6099
+ dedupeKey: `concat("traceId", ':', "spanId")`,
6100
+ traceId: `"traceId"`,
6101
+ spanId: `"spanId"`,
6102
+ // Legacy stores '' for root spans; VNext MV filters on IS NULL.
6103
+ parentSpanId: has("parentSpanId") ? `nullIf("parentSpanId", '')` : "NULL",
6104
+ experimentId: colOrNull("experimentId"),
6105
+ entityType: colOrNull("entityType"),
6106
+ entityId: colOrNull("entityId"),
6107
+ entityName: colOrNull("entityName"),
6108
+ entityVersionId: colOrNull("entityVersionId"),
6109
+ parentEntityVersionId: colOrNull("parentEntityVersionId"),
6110
+ parentEntityType: colOrNull("parentEntityType"),
6111
+ parentEntityId: colOrNull("parentEntityId"),
6112
+ parentEntityName: colOrNull("parentEntityName"),
6113
+ rootEntityVersionId: colOrNull("rootEntityVersionId"),
6114
+ rootEntityType: colOrNull("rootEntityType"),
6115
+ rootEntityId: colOrNull("rootEntityId"),
6116
+ rootEntityName: colOrNull("rootEntityName"),
6117
+ userId: colOrNull("userId"),
6118
+ organizationId: colOrNull("organizationId"),
6119
+ resourceId: colOrNull("resourceId"),
6120
+ runId: colOrNull("runId"),
6121
+ sessionId: colOrNull("sessionId"),
6122
+ threadId: colOrNull("threadId"),
6123
+ requestId: colOrNull("requestId"),
6124
+ environment: colOrNull("environment"),
6125
+ executionSource: has("source") ? `"source"` : "NULL",
6126
+ serviceName: colOrNull("serviceName"),
6127
+ name: `"name"`,
6128
+ spanType: `"spanType"`,
6129
+ isEvent: `"isEvent"`,
6130
+ startedAt: `"startedAt"`,
6131
+ // VNext requires non-null endedAt; events always use startedAt.
6132
+ endedAt: `if("isEvent", "startedAt", COALESCE("endedAt", "startedAt"))`,
6133
+ tags: tagsExpr,
6134
+ metadataSearch: metadataSearchExpr,
6135
+ attributes: colOrNull("attributes"),
6136
+ scope: colOrNull("scope"),
6137
+ links: colOrNull("links"),
6138
+ input: colOrNull("input"),
6139
+ output: colOrNull("output"),
6140
+ error: colOrNull("error"),
6141
+ metadataRaw: has("metadata") ? `"metadata"` : "NULL",
6142
+ requestContext: colOrNull("requestContext")
6143
+ };
6144
+ return VNEXT_SPAN_COLUMNS.map((c) => `${exprs[c]} AS "${c}"`).join(",\n ");
6145
+ }
6146
+ async function migrateLegacySpans(client, logger) {
6147
+ if (await hasMarkerRow(client)) {
6148
+ return { migratedRows: 0, batches: 0 };
6149
+ }
6150
+ const engine = await getTableEngine(client, TABLE_LEGACY_SPANS);
6151
+ if (!engine) {
6152
+ return { migratedRows: 0, batches: 0 };
6153
+ }
6154
+ const legacyColumnTypes = await getTableColumnTypes(client, TABLE_LEGACY_SPANS);
6155
+ const hasUpdatedAt = legacyColumnTypes.has("updatedAt");
6156
+ await client.command({ query: SPAN_EVENTS_DDL });
6157
+ await client.command({ query: TRACE_ROOTS_DDL });
6158
+ await client.command({ query: TRACE_BRANCHES_DDL });
6159
+ await client.command({ query: TRACE_ROOTS_MV_DDL });
6160
+ await client.command({ query: TRACE_BRANCHES_MV_DDL });
6161
+ const selectExprs = buildLegacySpanSelectExprs(legacyColumnTypes);
6162
+ const columnList = VNEXT_SPAN_COLUMNS.map((c) => `"${c}"`).join(", ");
6163
+ const dedupOrder = hasUpdatedAt ? `ORDER BY "traceId", "spanId", COALESCE("updatedAt", "createdAt") DESC` : `ORDER BY "traceId", "spanId", "createdAt" DESC`;
6164
+ const daysResult = await client.query({
6165
+ query: `SELECT
6166
+ toString(toDate(COALESCE(endedAt, startedAt))) as day,
6167
+ count() as cnt
6168
+ FROM ${TABLE_LEGACY_SPANS}
6169
+ GROUP BY day
6170
+ ORDER BY day`,
6171
+ format: "JSONEachRow"
6172
+ });
6173
+ const days = await daysResult.json();
6174
+ let migratedRows = 0;
6175
+ let batches = 0;
6176
+ for (const { day, cnt } of days) {
6177
+ const dayCount = Number(cnt);
6178
+ await client.command({
6179
+ query: `INSERT INTO ${TABLE_SPAN_EVENTS} (${columnList})
6180
+ SELECT ${selectExprs}
6181
+ FROM ${TABLE_LEGACY_SPANS}
6182
+ WHERE toDate(COALESCE(endedAt, startedAt)) = {batchDate:Date}
6183
+ ${dedupOrder}
6184
+ LIMIT 1 BY "traceId", "spanId"`,
6185
+ query_params: { batchDate: day }
6186
+ });
6187
+ migratedRows += dayCount;
6188
+ batches++;
6189
+ logger?.info?.(`Migrated batch ${day}: ${dayCount} rows`);
6190
+ }
6191
+ await client.command({
6192
+ query: `CREATE TABLE IF NOT EXISTS ${TABLE_LEGACY_MIGRATION_DONE} (completedAt DateTime64(3, 'UTC')) ENGINE = MergeTree ORDER BY completedAt`
6193
+ });
6194
+ await client.command({
6195
+ query: `INSERT INTO ${TABLE_LEGACY_MIGRATION_DONE} VALUES (now64(3))`
6196
+ });
6197
+ logger?.info?.(
6198
+ `Legacy span migration complete: ${migratedRows} rows in ${batches} batches. The legacy table '${TABLE_LEGACY_SPANS}' has been preserved. Drop it manually after verifying the migration.`
6199
+ );
6200
+ return { migratedRows, batches };
6201
+ }
5985
6202
  async function migrateSignalTables(client, logger) {
5986
6203
  for (const { table, createDDL, idColumn } of SIGNAL_MIGRATIONS) {
5987
6204
  const engine = await getTableEngine(client, table);
@@ -7296,6 +7513,15 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
7296
7513
  })
7297
7514
  });
7298
7515
  }
7516
+ try {
7517
+ const legacyStatus = await checkLegacySpanMigrationStatus(this.#client);
7518
+ if (legacyStatus.needsMigration) {
7519
+ this.logger?.warn?.(
7520
+ `Legacy span table 'mastra_ai_spans' detected. Run 'npx mastra migrate' to migrate historical spans to the v-next schema.`
7521
+ );
7522
+ }
7523
+ } catch {
7524
+ }
7299
7525
  try {
7300
7526
  await assertExistingTablesCompatibleWithReplication(this.#client, this.#replication);
7301
7527
  const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
@@ -7370,34 +7596,44 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
7370
7596
  }
7371
7597
  }
7372
7598
  /**
7373
- * Manually migrate legacy signal tables to the signal-ID ReplacingMergeTree schema.
7374
- * The public method name is historical; the CLI still calls `migrateSpans()`
7375
- * for observability migrations even though this now also migrates signal tables.
7599
+ * Manually migrate legacy tables to the v-next schema.
7600
+ * Handles both signal table migrations (MergeTree ReplacingMergeTree)
7601
+ * and legacy span migration (mastra_ai_spans mastra_span_events).
7376
7602
  */
7377
7603
  async migrateSpans() {
7378
- const migrationStatus = await checkSignalTablesMigrationStatus(this.#client);
7379
- if (!migrationStatus.needsMigration) {
7380
- return {
7381
- success: true,
7382
- alreadyMigrated: true,
7383
- duplicatesRemoved: 0,
7384
- message: "Migration already complete. Signal tables already use signal-ID dedupe keys."
7385
- };
7604
+ const messages = [];
7605
+ const signalStatus = await checkSignalTablesMigrationStatus(this.#client);
7606
+ if (signalStatus.needsMigration) {
7607
+ if (isReplicationConfigured(this.#replication)) {
7608
+ throw new error.MastraError({
7609
+ id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "SIGNAL_TABLES_MIGRATION_UNSUPPORTED"),
7610
+ domain: error.ErrorDomain.STORAGE,
7611
+ category: error.ErrorCategory.USER,
7612
+ text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap signal table migrations automatically. Migrate existing local signal tables manually before enabling replication."
7613
+ });
7614
+ }
7615
+ await migrateSignalTables(this.#client, this.logger);
7616
+ messages.push(`Migrated signal tables: ${signalStatus.tables.map((t) => t.table).join(", ")}.`);
7386
7617
  }
7387
- if (isReplicationConfigured(this.#replication)) {
7388
- throw new error.MastraError({
7389
- id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "SIGNAL_TABLES_MIGRATION_UNSUPPORTED"),
7390
- domain: error.ErrorDomain.STORAGE,
7391
- category: error.ErrorCategory.USER,
7392
- text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap signal table migrations automatically. Migrate existing local signal tables manually before enabling replication."
7393
- });
7618
+ const legacyStatus = await checkLegacySpanMigrationStatus(this.#client);
7619
+ if (legacyStatus.needsMigration) {
7620
+ if (isReplicationConfigured(this.#replication)) {
7621
+ throw new error.MastraError({
7622
+ id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "LEGACY_SPAN_MIGRATION_UNSUPPORTED"),
7623
+ domain: error.ErrorDomain.STORAGE,
7624
+ category: error.ErrorCategory.USER,
7625
+ text: "ClickHouse replication is enabled. Migrate legacy mastra_ai_spans manually before enabling replication."
7626
+ });
7627
+ }
7628
+ const result = await migrateLegacySpans(this.#client, this.logger);
7629
+ messages.push(`Migrated ${result.migratedRows} legacy spans in ${result.batches} batches.`);
7394
7630
  }
7395
- await migrateSignalTables(this.#client, this.logger);
7631
+ const alreadyMigrated = !signalStatus.needsMigration && !legacyStatus.needsMigration;
7396
7632
  return {
7397
7633
  success: true,
7398
- alreadyMigrated: false,
7634
+ alreadyMigrated,
7399
7635
  duplicatesRemoved: 0,
7400
- message: `Migration complete. Migrated signal tables: ${migrationStatus.tables.map((t) => t.table).join(", ")}.`
7636
+ message: alreadyMigrated ? "Migration already complete." : `Migration complete. ${messages.join(" ")}`
7401
7637
  };
7402
7638
  }
7403
7639
  // -------------------------------------------------------------------------