@mastra/clickhouse 1.11.1-alpha.0 → 1.12.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/reference-storage-clickhouse.md +41 -9
- package/dist/docs/references/reference-storage-composite.md +65 -7
- package/dist/index.cjs +257 -21
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +257 -21
- package/dist/index.js.map +1 -1
- package/dist/storage/domains/observability/v-next/index.d.ts +3 -3
- package/dist/storage/domains/observability/v-next/index.d.ts.map +1 -1
- package/dist/storage/domains/observability/v-next/migration.d.ts +12 -4
- package/dist/storage/domains/observability/v-next/migration.d.ts.map +1 -1
- package/package.json +8 -8
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# @mastra/clickhouse
|
|
2
2
|
|
|
3
|
+
## 1.12.0-alpha.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Added legacy-to-VNext span migration for ClickHouse observability storage. Customers using `ObservabilityStorageClickhouse` (legacy) can now run `npx mastra migrate` to copy historical spans from `mastra_ai_spans` to the VNext `mastra_span_events` schema. The migration handles column mapping, batches by day for memory safety, deduplicates legacy rows, converts empty-string parentSpanId to NULL for correct root span detection, and supports both old (`Nullable(String)` JSON) and new (`Array(String)`) tags schemas. The legacy table is preserved as a backup after migration. ([#19050](https://github.com/mastra-ai/mastra/pull/19050))
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies [[`e955965`](https://github.com/mastra-ai/mastra/commit/e955965dce575a903e37cf054d28ea99aa48785e), [`860ef7e`](https://github.com/mastra-ai/mastra/commit/860ef7e77d92b63469cbe5857aa1e626197e43e9), [`17e818c`](https://github.com/mastra-ai/mastra/commit/17e818c51a958ba90641b1a959dc38faf8c034e9), [`4451dfe`](https://github.com/mastra-ai/mastra/commit/4451dfe857428e7abcc0261a507a2e186dae6d47), [`1d39058`](https://github.com/mastra-ai/mastra/commit/1d39058e548efd691799985d5c8af2737f1c3bd2)]:
|
|
12
|
+
- @mastra/core@1.51.0-alpha.2
|
|
13
|
+
|
|
14
|
+
## 1.11.1
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- Added optional `organizationId` and `projectId` fields to scores for multi-tenant isolation. Scores can now be saved with tenancy metadata and the `listScoresBy*` methods accept a `filters` option to scope results by organization and project. ([#18331](https://github.com/mastra-ai/mastra/pull/18331))
|
|
19
|
+
|
|
20
|
+
```ts
|
|
21
|
+
await storage.saveScore({ ...score, organizationId: 'org-a', projectId: 'proj-1' });
|
|
22
|
+
|
|
23
|
+
const result = await storage.listScoresByScorerId({
|
|
24
|
+
scorerId,
|
|
25
|
+
filters: { organizationId: 'org-a', projectId: 'proj-1' },
|
|
26
|
+
});
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`projectId` identifies the project scope, separate from `resourceId` which continues to mean the agent memory resource.
|
|
30
|
+
|
|
31
|
+
- Updated dependencies [[`700619b`](https://github.com/mastra-ai/mastra/commit/700619b61d572e592cbaaf758121d168844ca4d2), [`0f69865`](https://github.com/mastra-ai/mastra/commit/0f69865aced225d98eac812e22699dc445ee18cb), [`9250acd`](https://github.com/mastra-ai/mastra/commit/9250acd1357f0f1f33d0dcca16f9655084c58eca), [`0c3d4bc`](https://github.com/mastra-ai/mastra/commit/0c3d4bcae13ea3699d379403e6f350d5cf4efe9f), [`cc440a3`](https://github.com/mastra-ai/mastra/commit/cc440a39400d8ce06655462b26c1666a1b3d4320), [`6a61846`](https://github.com/mastra-ai/mastra/commit/6a61846eeda29fb714549b70f1bee2bf6b141c44), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`17369b2`](https://github.com/mastra-ai/mastra/commit/17369b25250561e9ed994ae509be1d15bfb33bcb), [`c64c2a8`](https://github.com/mastra-ai/mastra/commit/c64c2a8503a50252f9ca6b8e8c54cadee31b92a2), [`bcae929`](https://github.com/mastra-ai/mastra/commit/bcae929945cbf265bd9f327cc715ecafa072b5b9), [`ea6327b`](https://github.com/mastra-ai/mastra/commit/ea6327ba2d63ca647804bc97b347e03a58617162), [`3439fa8`](https://github.com/mastra-ai/mastra/commit/3439fa836ecfcaa257b40c20b30ac2a8be22e9ea), [`85107f2`](https://github.com/mastra-ai/mastra/commit/85107f2758b527147fccbedff962961927c2d3b8), [`b33822e`](https://github.com/mastra-ai/mastra/commit/b33822e8d470884954b02f7b0745407ee4ef74b1), [`06e2680`](https://github.com/mastra-ai/mastra/commit/06e26806b51d2cbd858afdc66daa2b86ff3ba64a), [`06ff9e0`](https://github.com/mastra-ai/mastra/commit/06ff9e0befd1d642ab87ff749285ee4091205c7e), [`d5c11e3`](https://github.com/mastra-ai/mastra/commit/d5c11e3ba5045969caa7272a7bd1fd141c93ab6c), [`7f5e1ff`](https://github.com/mastra-ai/mastra/commit/7f5e1ff695a92f672bb3976363925d1e9136b54a), [`ff80671`](https://github.com/mastra-ai/mastra/commit/ff8067185e208b27198b4e5b71803013175c3643), [`b8375c1`](https://github.com/mastra-ai/mastra/commit/b8375c1f8fe905df8ae2ae9a893bb365f17aec4e), [`dab1257`](https://github.com/mastra-ai/mastra/commit/dab1257b64e4ed576dc5038bb7a3f7072338bc9f), [`1240f05`](https://github.com/mastra-ai/mastra/commit/1240f051c8e5371f1c014448bf37b1a1b9a05e47), [`705ff39`](https://github.com/mastra-ai/mastra/commit/705ff3969e57214ff2fdaf3815d751dd558886ed), [`e6fbd5b`](https://github.com/mastra-ai/mastra/commit/e6fbd5bfdc28e92c0c0433f29aa1bc152d3430f6), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`6f2026c`](https://github.com/mastra-ai/mastra/commit/6f2026cdf114ff1e21e49133ca774ec7d5085059), [`24c10d3`](https://github.com/mastra-ai/mastra/commit/24c10d333e6649ac06075903aeeee13a933db3b3), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`215f9b0`](https://github.com/mastra-ai/mastra/commit/215f9b0f3f3f6fc165edad360582dd4d3d7ea748), [`003f35d`](https://github.com/mastra-ai/mastra/commit/003f35d19e07b23b4bacc591c8bc0c59b42124ae), [`f890eda`](https://github.com/mastra-ai/mastra/commit/f890eda2c8a2ae83d9b30bc6d85842f93b6c266b), [`1340fb7`](https://github.com/mastra-ai/mastra/commit/1340fb76262a3ca062130aa71859f07257a0a5a4)]:
|
|
32
|
+
- @mastra/core@1.49.0
|
|
33
|
+
|
|
3
34
|
## 1.11.1-alpha.0
|
|
4
35
|
|
|
5
36
|
### Patch Changes
|
package/dist/docs/SKILL.md
CHANGED
|
@@ -3,7 +3,7 @@ name: mastra-clickhouse
|
|
|
3
3
|
description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
|
|
4
4
|
metadata:
|
|
5
5
|
package: "@mastra/clickhouse"
|
|
6
|
-
version: "1.
|
|
6
|
+
version: "1.12.0-alpha.0"
|
|
7
7
|
---
|
|
8
8
|
|
|
9
9
|
## When to use
|
|
@@ -101,6 +101,38 @@ const observabilityStore = new ObservabilityStorageClickhouse({
|
|
|
101
101
|
|
|
102
102
|
New projects should use `ObservabilityStorageClickhouseVNext` instead.
|
|
103
103
|
|
|
104
|
+
### Migrating from legacy to vNext
|
|
105
|
+
|
|
106
|
+
To migrate historical spans from the legacy `mastra_ai_spans` table to the vNext schema, run:
|
|
107
|
+
|
|
108
|
+
**npm**:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
npx mastra migrate
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**pnpm**:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pnpm dlx mastra migrate
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Yarn**:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
yarn dlx mastra migrate
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**Bun**:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
bun x mastra migrate
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The migration copies span data from `mastra_ai_spans` into `mastra_span_events` in day-sized batches. It handles column mapping, deduplicates legacy rows, and preserves the original table as a backup. After migration, traces appear in Studio through the vNext adapter.
|
|
133
|
+
|
|
134
|
+
> **Note:** The legacy table is not deleted. Drop it manually after verifying the migration.
|
|
135
|
+
|
|
104
136
|
### ClickHouse for every domain
|
|
105
137
|
|
|
106
138
|
`ClickhouseStoreVNext` backs the `memory`, `workflows`, and `observability` domains with ClickHouse and uses the vNext observability adapter automatically. Use it when you want ClickHouse to back the entire application without wiring a composite store manually.
|
|
@@ -174,19 +206,19 @@ The same `client` form is accepted by `ObservabilityStorageClickhouse` and `Obse
|
|
|
174
206
|
|
|
175
207
|
**id** (`string`): Unique identifier for this storage instance.
|
|
176
208
|
|
|
177
|
-
**url** (`string`): ClickHouse server URL (for example,
|
|
209
|
+
**url** (`string`): ClickHouse server URL (for example, https\://your-instance.clickhouse.cloud:8443 or http\://localhost:8123). Required when not passing a pre-configured client.
|
|
178
210
|
|
|
179
|
-
**username** (`string`): ClickHouse username. Required when not passing a pre-configured
|
|
211
|
+
**username** (`string`): ClickHouse username. Required when not passing a pre-configured client.
|
|
180
212
|
|
|
181
|
-
**password** (`string`): ClickHouse password. Required when not passing a pre-configured
|
|
213
|
+
**password** (`string`): ClickHouse password. Required when not passing a pre-configured client. It can be an empty string for the default user on a local instance.
|
|
182
214
|
|
|
183
|
-
**client** (`ClickHouseClient`): Pre-configured ClickHouse client from
|
|
215
|
+
**client** (`ClickHouseClient`): Pre-configured ClickHouse client from @clickhouse/client. Use this when you need custom request settings. Mutually exclusive with the credential fields above.
|
|
184
216
|
|
|
185
|
-
**ttl** (`object`): Per-table TTL configuration applied at table creation time. Accepts row-level and column-level TTLs in interval units from
|
|
217
|
+
**ttl** (`object`): Per-table TTL configuration applied at table creation time. Accepts row-level and column-level TTLs in interval units from NANOSECOND through YEAR.
|
|
186
218
|
|
|
187
|
-
**replication** (`{ cluster?: string; zookeeperPath?: string; replicaName?: string }`): Opt-in replicated table configuration for multi-replica ClickHouse clusters. When set, Mastra creates replicated MergeTree tables. When
|
|
219
|
+
**replication** (`{ cluster?: string; zookeeperPath?: string; replicaName?: string }`): Opt-in replicated table configuration for multi-replica ClickHouse clusters. When set, Mastra creates replicated MergeTree tables. When cluster is set, Mastra also adds ON CLUSTER to Mastra-owned data definition language (DDL).
|
|
188
220
|
|
|
189
|
-
**disableInit** (`boolean`): When
|
|
221
|
+
**disableInit** (`boolean`): When true, the store does not run table creation or migrations on first use. Call storage.init() explicitly from your deployment scripts. (Default: `false`)
|
|
190
222
|
|
|
191
223
|
`ClickhouseStore` also accepts every option from `ClickHouseClientConfigOptions` (such as `database`, `request_timeout`, `compression`, `keep_alive`, and `max_open_connections`).
|
|
192
224
|
|
|
@@ -211,7 +243,7 @@ When `replication` is set, Mastra rewrites its `MergeTree` and `ReplacingMergeTr
|
|
|
211
243
|
- `zookeeperPath`: `'/clickhouse/tables/{shard}/{database}/{table}'`
|
|
212
244
|
- `replicaName`: `'{replica}'`
|
|
213
245
|
|
|
214
|
-
The defaults match the most common self-managed convention. If your cluster's existing tables use a different layout (for example `/clickhouse/tables/{shard}/{table}` without the `{database}` segment), set `zookeeperPath` explicitly to match. Mastra
|
|
246
|
+
The defaults match the most common self-managed convention. If your cluster's existing tables use a different layout (for example `/clickhouse/tables/{shard}/{table}` without the `{database}` segment), set `zookeeperPath` explicitly to match. Mastra doesn't read your cluster's convention from Keeper, so a mismatched default writes Mastra's metadata to a separate branch from the rest of the cluster.
|
|
215
247
|
|
|
216
248
|
```typescript
|
|
217
249
|
new ClickhouseStoreVNext({
|
|
@@ -231,7 +263,7 @@ Manual maintenance such as `optimizeTable()` and `materializeTtl()` runs on ever
|
|
|
231
263
|
|
|
232
264
|
If existing Mastra tables use local `MergeTree` or `ReplacingMergeTree` engines, initialization fails while `replication` is enabled. Mastra refuses to silently convert local tables because copy-and-swap is unsafe across replicas. To migrate, recreate the affected tables as `Replicated*` before enabling replication. The typical sequence is: rename the local table, run `CREATE TABLE ... ENGINE = ReplicatedMergeTree(...) ON CLUSTER ...`, run `INSERT INTO ... SELECT * FROM <renamed_local>`, then drop the renamed local table.
|
|
233
265
|
|
|
234
|
-
|
|
266
|
+
Don't set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side, and explicit `ReplicatedMergeTree` engines produce incorrect DDL. `replication` is only for self-managed multi-replica clusters.
|
|
235
267
|
|
|
236
268
|
### Observability domain options
|
|
237
269
|
|
|
@@ -37,25 +37,25 @@ You'll also need to install the storage providers you want to compose:
|
|
|
37
37
|
**npm**:
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
|
-
npm install @mastra/pg@latest @mastra/libsql@latest
|
|
40
|
+
npm install @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
41
41
|
```
|
|
42
42
|
|
|
43
43
|
**pnpm**:
|
|
44
44
|
|
|
45
45
|
```bash
|
|
46
|
-
pnpm add @mastra/pg@latest @mastra/libsql@latest
|
|
46
|
+
pnpm add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
**Yarn**:
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
yarn add @mastra/pg@latest @mastra/libsql@latest
|
|
52
|
+
yarn add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
**Bun**:
|
|
56
56
|
|
|
57
57
|
```bash
|
|
58
|
-
bun add @mastra/pg@latest @mastra/libsql@latest
|
|
58
|
+
bun add @mastra/pg@latest @mastra/libsql@latest @mastra/mongodb@latest
|
|
59
59
|
```
|
|
60
60
|
|
|
61
61
|
## Storage domains
|
|
@@ -124,15 +124,73 @@ export const mastra = new Mastra({
|
|
|
124
124
|
})
|
|
125
125
|
```
|
|
126
126
|
|
|
127
|
+
### Mixed backends
|
|
128
|
+
|
|
129
|
+
Use domain classes from each storage package to route different domains to different backends. The following example stores memory and workflow state in MongoDB, then routes observability to ClickHouse:
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
import { Mastra } from '@mastra/core'
|
|
133
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
134
|
+
import { ObservabilityStorageClickhouse } from '@mastra/clickhouse'
|
|
135
|
+
import { MemoryStorageMongoDB, WorkflowsStorageMongoDB } from '@mastra/mongodb'
|
|
136
|
+
|
|
137
|
+
export const mastra = new Mastra({
|
|
138
|
+
storage: new MastraCompositeStore({
|
|
139
|
+
id: 'composite',
|
|
140
|
+
domains: {
|
|
141
|
+
memory: new MemoryStorageMongoDB({
|
|
142
|
+
uri: process.env.MONGODB_URI,
|
|
143
|
+
dbName: 'mastra_memory',
|
|
144
|
+
}),
|
|
145
|
+
workflows: new WorkflowsStorageMongoDB({
|
|
146
|
+
uri: process.env.MONGODB_URI,
|
|
147
|
+
dbName: 'mastra_workflows',
|
|
148
|
+
}),
|
|
149
|
+
observability: new ObservabilityStorageClickhouse({
|
|
150
|
+
url: process.env.CLICKHOUSE_URL,
|
|
151
|
+
username: process.env.CLICKHOUSE_USERNAME,
|
|
152
|
+
password: process.env.CLICKHOUSE_PASSWORD,
|
|
153
|
+
}),
|
|
154
|
+
},
|
|
155
|
+
}),
|
|
156
|
+
})
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Disabling a domain
|
|
160
|
+
|
|
161
|
+
Set a domain to `false` to disable it. A disabled domain doesn't fall back to `default`, so data for that domain isn't persisted:
|
|
162
|
+
|
|
163
|
+
```typescript
|
|
164
|
+
import { MastraCompositeStore } from '@mastra/core/storage'
|
|
165
|
+
import { PostgresStore } from '@mastra/pg'
|
|
166
|
+
import { Mastra } from '@mastra/core'
|
|
167
|
+
|
|
168
|
+
const pgStore = new PostgresStore({
|
|
169
|
+
id: 'pg',
|
|
170
|
+
connectionString: process.env.DATABASE_URL,
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
export const mastra = new Mastra({
|
|
174
|
+
storage: new MastraCompositeStore({
|
|
175
|
+
id: 'composite',
|
|
176
|
+
default: pgStore,
|
|
177
|
+
domains: {
|
|
178
|
+
// don't persist traces and spans
|
|
179
|
+
observability: false,
|
|
180
|
+
},
|
|
181
|
+
}),
|
|
182
|
+
})
|
|
183
|
+
```
|
|
184
|
+
|
|
127
185
|
## Options
|
|
128
186
|
|
|
129
187
|
**id** (`string`): Unique identifier for this storage instance.
|
|
130
188
|
|
|
131
|
-
**default** (`MastraCompositeStore`): Default storage adapter. Domains not explicitly specified in
|
|
189
|
+
**default** (`MastraCompositeStore`): Default storage adapter. Domains not explicitly specified in domains will use this storage's domains as fallbacks.
|
|
132
190
|
|
|
133
191
|
**disableInit** (`boolean`): When true, automatic initialization is disabled. You must call init() explicitly.
|
|
134
192
|
|
|
135
|
-
**domains** (`object`): Individual domain overrides. Each domain can come from a different storage adapter. These take precedence over both
|
|
193
|
+
**domains** (`object`): Individual domain overrides. Each domain can come from a different storage adapter. These take precedence over both editor and default storage. Set a domain to false to disable it entirely; a disabled domain does not fall back to editor or default.
|
|
136
194
|
|
|
137
195
|
**domains.memory** (`MemoryStorage`): Storage for threads, messages, and resources.
|
|
138
196
|
|
|
@@ -274,6 +332,6 @@ const storage = new MastraCompositeStore({
|
|
|
274
332
|
})
|
|
275
333
|
```
|
|
276
334
|
|
|
277
|
-
|
|
335
|
+
Don't set `replication` on ClickHouse Cloud. Cloud rewrites `MergeTree` to `SharedMergeTree` server-side. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for the full config shape and operator notes.
|
|
278
336
|
|
|
279
337
|
> **Info:** This approach is also required when using storage providers that don't support observability (like Convex, DynamoDB, or Cloudflare). See the [MastraStorageExporter documentation](https://mastra.ai/docs/observability/integrations/exporters/mastra-storage) for the full list of supported providers.
|
package/dist/index.cjs
CHANGED
|
@@ -5949,6 +5949,11 @@ async function getTableColumns(client, table) {
|
|
|
5949
5949
|
const rows = await result.json();
|
|
5950
5950
|
return rows.map((r) => r.name);
|
|
5951
5951
|
}
|
|
5952
|
+
async function getTableColumnTypes(client, table) {
|
|
5953
|
+
const result = await client.query({ query: `DESCRIBE TABLE ${table}`, format: "JSONEachRow" });
|
|
5954
|
+
const rows = await result.json();
|
|
5955
|
+
return new Map(rows.map((r) => [r.name, r.type]));
|
|
5956
|
+
}
|
|
5952
5957
|
function buildTemporaryTableDDL(createDDL, table, tempTable) {
|
|
5953
5958
|
return createDDL.replace(`CREATE TABLE IF NOT EXISTS ${table}`, `CREATE TABLE ${tempTable}`);
|
|
5954
5959
|
}
|
|
@@ -5982,6 +5987,218 @@ async function checkSignalTablesMigrationStatus(client) {
|
|
|
5982
5987
|
tables
|
|
5983
5988
|
};
|
|
5984
5989
|
}
|
|
5990
|
+
var TABLE_LEGACY_SPANS = "mastra_ai_spans";
|
|
5991
|
+
var PROMOTED_METADATA_KEYS = [
|
|
5992
|
+
"experimentId",
|
|
5993
|
+
"entityType",
|
|
5994
|
+
"entityId",
|
|
5995
|
+
"entityName",
|
|
5996
|
+
"entityVersionId",
|
|
5997
|
+
"parentEntityVersionId",
|
|
5998
|
+
"rootEntityVersionId",
|
|
5999
|
+
"userId",
|
|
6000
|
+
"organizationId",
|
|
6001
|
+
"resourceId",
|
|
6002
|
+
"runId",
|
|
6003
|
+
"sessionId",
|
|
6004
|
+
"threadId",
|
|
6005
|
+
"requestId",
|
|
6006
|
+
"environment",
|
|
6007
|
+
"executionSource",
|
|
6008
|
+
"serviceName"
|
|
6009
|
+
];
|
|
6010
|
+
var TABLE_LEGACY_MIGRATION_DONE = "mastra_legacy_span_migration_done";
|
|
6011
|
+
async function hasMarkerRow(client) {
|
|
6012
|
+
const engine = await getTableEngine(client, TABLE_LEGACY_MIGRATION_DONE);
|
|
6013
|
+
if (!engine) return false;
|
|
6014
|
+
const result = await client.query({
|
|
6015
|
+
query: `SELECT count() as cnt FROM ${TABLE_LEGACY_MIGRATION_DONE}`,
|
|
6016
|
+
format: "JSONEachRow"
|
|
6017
|
+
});
|
|
6018
|
+
const rows = await result.json();
|
|
6019
|
+
return Number(rows[0]?.cnt ?? 0) > 0;
|
|
6020
|
+
}
|
|
6021
|
+
async function checkLegacySpanMigrationStatus(client) {
|
|
6022
|
+
if (await hasMarkerRow(client)) {
|
|
6023
|
+
return { needsMigration: false };
|
|
6024
|
+
}
|
|
6025
|
+
const engine = await getTableEngine(client, TABLE_LEGACY_SPANS);
|
|
6026
|
+
if (!engine) {
|
|
6027
|
+
return { needsMigration: false };
|
|
6028
|
+
}
|
|
6029
|
+
const countResult = await client.query({
|
|
6030
|
+
query: `SELECT count() as cnt FROM ${TABLE_LEGACY_SPANS}`,
|
|
6031
|
+
format: "JSONEachRow"
|
|
6032
|
+
});
|
|
6033
|
+
const countRows = await countResult.json();
|
|
6034
|
+
const legacyRowCount = Number(countRows[0]?.cnt ?? 0);
|
|
6035
|
+
if (legacyRowCount === 0) {
|
|
6036
|
+
return { needsMigration: false };
|
|
6037
|
+
}
|
|
6038
|
+
return { needsMigration: true };
|
|
6039
|
+
}
|
|
6040
|
+
var VNEXT_SPAN_COLUMNS = [
|
|
6041
|
+
"dedupeKey",
|
|
6042
|
+
"traceId",
|
|
6043
|
+
"spanId",
|
|
6044
|
+
"parentSpanId",
|
|
6045
|
+
"experimentId",
|
|
6046
|
+
"entityType",
|
|
6047
|
+
"entityId",
|
|
6048
|
+
"entityName",
|
|
6049
|
+
"entityVersionId",
|
|
6050
|
+
"parentEntityVersionId",
|
|
6051
|
+
"parentEntityType",
|
|
6052
|
+
"parentEntityId",
|
|
6053
|
+
"parentEntityName",
|
|
6054
|
+
"rootEntityVersionId",
|
|
6055
|
+
"rootEntityType",
|
|
6056
|
+
"rootEntityId",
|
|
6057
|
+
"rootEntityName",
|
|
6058
|
+
"userId",
|
|
6059
|
+
"organizationId",
|
|
6060
|
+
"resourceId",
|
|
6061
|
+
"runId",
|
|
6062
|
+
"sessionId",
|
|
6063
|
+
"threadId",
|
|
6064
|
+
"requestId",
|
|
6065
|
+
"environment",
|
|
6066
|
+
"executionSource",
|
|
6067
|
+
"serviceName",
|
|
6068
|
+
"name",
|
|
6069
|
+
"spanType",
|
|
6070
|
+
"isEvent",
|
|
6071
|
+
"startedAt",
|
|
6072
|
+
"endedAt",
|
|
6073
|
+
"tags",
|
|
6074
|
+
"metadataSearch",
|
|
6075
|
+
"attributes",
|
|
6076
|
+
"scope",
|
|
6077
|
+
"links",
|
|
6078
|
+
"input",
|
|
6079
|
+
"output",
|
|
6080
|
+
"error",
|
|
6081
|
+
"metadataRaw",
|
|
6082
|
+
"requestContext"
|
|
6083
|
+
];
|
|
6084
|
+
function buildLegacySpanSelectExprs(legacyColumnTypes) {
|
|
6085
|
+
const has = (col) => legacyColumnTypes.has(col);
|
|
6086
|
+
const colOrNull = (legacy) => has(legacy) ? `"${legacy}"` : "NULL";
|
|
6087
|
+
const promotedIn = PROMOTED_METADATA_KEYS.map((k) => `'${k}'`).join(", ");
|
|
6088
|
+
const metadataSearchExpr = has("metadata") ? `CAST(arrayFilter(x -> x.1 NOT IN (${promotedIn}) AND trim(BOTH ' ' FROM x.2) != '', arrayMap(x -> (x.1, trim(BOTH ' ' FROM x.2)), JSONExtractKeysAndValues(COALESCE("metadata", '{}'), 'String'))), 'Map(String, String)')` : `CAST(map(), 'Map(String, String)')`;
|
|
6089
|
+
let tagsExpr = "[]";
|
|
6090
|
+
if (has("tags")) {
|
|
6091
|
+
const tagsType = legacyColumnTypes.get("tags");
|
|
6092
|
+
if (tagsType.includes("Array")) {
|
|
6093
|
+
tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), "tags")))`;
|
|
6094
|
+
} else {
|
|
6095
|
+
tagsExpr = `arrayFilter(x -> x != '', arrayDistinct(arrayMap(x -> trim(BOTH ' ' FROM x), JSONExtract(COALESCE("tags", '[]'), 'Array(String)'))))`;
|
|
6096
|
+
}
|
|
6097
|
+
}
|
|
6098
|
+
const exprs = {
|
|
6099
|
+
dedupeKey: `concat("traceId", ':', "spanId")`,
|
|
6100
|
+
traceId: `"traceId"`,
|
|
6101
|
+
spanId: `"spanId"`,
|
|
6102
|
+
// Legacy stores '' for root spans; VNext MV filters on IS NULL.
|
|
6103
|
+
parentSpanId: has("parentSpanId") ? `nullIf("parentSpanId", '')` : "NULL",
|
|
6104
|
+
experimentId: colOrNull("experimentId"),
|
|
6105
|
+
entityType: colOrNull("entityType"),
|
|
6106
|
+
entityId: colOrNull("entityId"),
|
|
6107
|
+
entityName: colOrNull("entityName"),
|
|
6108
|
+
entityVersionId: colOrNull("entityVersionId"),
|
|
6109
|
+
parentEntityVersionId: colOrNull("parentEntityVersionId"),
|
|
6110
|
+
parentEntityType: colOrNull("parentEntityType"),
|
|
6111
|
+
parentEntityId: colOrNull("parentEntityId"),
|
|
6112
|
+
parentEntityName: colOrNull("parentEntityName"),
|
|
6113
|
+
rootEntityVersionId: colOrNull("rootEntityVersionId"),
|
|
6114
|
+
rootEntityType: colOrNull("rootEntityType"),
|
|
6115
|
+
rootEntityId: colOrNull("rootEntityId"),
|
|
6116
|
+
rootEntityName: colOrNull("rootEntityName"),
|
|
6117
|
+
userId: colOrNull("userId"),
|
|
6118
|
+
organizationId: colOrNull("organizationId"),
|
|
6119
|
+
resourceId: colOrNull("resourceId"),
|
|
6120
|
+
runId: colOrNull("runId"),
|
|
6121
|
+
sessionId: colOrNull("sessionId"),
|
|
6122
|
+
threadId: colOrNull("threadId"),
|
|
6123
|
+
requestId: colOrNull("requestId"),
|
|
6124
|
+
environment: colOrNull("environment"),
|
|
6125
|
+
executionSource: has("source") ? `"source"` : "NULL",
|
|
6126
|
+
serviceName: colOrNull("serviceName"),
|
|
6127
|
+
name: `"name"`,
|
|
6128
|
+
spanType: `"spanType"`,
|
|
6129
|
+
isEvent: `"isEvent"`,
|
|
6130
|
+
startedAt: `"startedAt"`,
|
|
6131
|
+
// VNext requires non-null endedAt; events always use startedAt.
|
|
6132
|
+
endedAt: `if("isEvent", "startedAt", COALESCE("endedAt", "startedAt"))`,
|
|
6133
|
+
tags: tagsExpr,
|
|
6134
|
+
metadataSearch: metadataSearchExpr,
|
|
6135
|
+
attributes: colOrNull("attributes"),
|
|
6136
|
+
scope: colOrNull("scope"),
|
|
6137
|
+
links: colOrNull("links"),
|
|
6138
|
+
input: colOrNull("input"),
|
|
6139
|
+
output: colOrNull("output"),
|
|
6140
|
+
error: colOrNull("error"),
|
|
6141
|
+
metadataRaw: has("metadata") ? `"metadata"` : "NULL",
|
|
6142
|
+
requestContext: colOrNull("requestContext")
|
|
6143
|
+
};
|
|
6144
|
+
return VNEXT_SPAN_COLUMNS.map((c) => `${exprs[c]} AS "${c}"`).join(",\n ");
|
|
6145
|
+
}
|
|
6146
|
+
async function migrateLegacySpans(client, logger) {
|
|
6147
|
+
if (await hasMarkerRow(client)) {
|
|
6148
|
+
return { migratedRows: 0, batches: 0 };
|
|
6149
|
+
}
|
|
6150
|
+
const engine = await getTableEngine(client, TABLE_LEGACY_SPANS);
|
|
6151
|
+
if (!engine) {
|
|
6152
|
+
return { migratedRows: 0, batches: 0 };
|
|
6153
|
+
}
|
|
6154
|
+
const legacyColumnTypes = await getTableColumnTypes(client, TABLE_LEGACY_SPANS);
|
|
6155
|
+
const hasUpdatedAt = legacyColumnTypes.has("updatedAt");
|
|
6156
|
+
await client.command({ query: SPAN_EVENTS_DDL });
|
|
6157
|
+
await client.command({ query: TRACE_ROOTS_DDL });
|
|
6158
|
+
await client.command({ query: TRACE_BRANCHES_DDL });
|
|
6159
|
+
await client.command({ query: TRACE_ROOTS_MV_DDL });
|
|
6160
|
+
await client.command({ query: TRACE_BRANCHES_MV_DDL });
|
|
6161
|
+
const selectExprs = buildLegacySpanSelectExprs(legacyColumnTypes);
|
|
6162
|
+
const columnList = VNEXT_SPAN_COLUMNS.map((c) => `"${c}"`).join(", ");
|
|
6163
|
+
const dedupOrder = hasUpdatedAt ? `ORDER BY "traceId", "spanId", COALESCE("updatedAt", "createdAt") DESC` : `ORDER BY "traceId", "spanId", "createdAt" DESC`;
|
|
6164
|
+
const daysResult = await client.query({
|
|
6165
|
+
query: `SELECT
|
|
6166
|
+
toString(toDate(COALESCE(endedAt, startedAt))) as day,
|
|
6167
|
+
count() as cnt
|
|
6168
|
+
FROM ${TABLE_LEGACY_SPANS}
|
|
6169
|
+
GROUP BY day
|
|
6170
|
+
ORDER BY day`,
|
|
6171
|
+
format: "JSONEachRow"
|
|
6172
|
+
});
|
|
6173
|
+
const days = await daysResult.json();
|
|
6174
|
+
let migratedRows = 0;
|
|
6175
|
+
let batches = 0;
|
|
6176
|
+
for (const { day, cnt } of days) {
|
|
6177
|
+
const dayCount = Number(cnt);
|
|
6178
|
+
await client.command({
|
|
6179
|
+
query: `INSERT INTO ${TABLE_SPAN_EVENTS} (${columnList})
|
|
6180
|
+
SELECT ${selectExprs}
|
|
6181
|
+
FROM ${TABLE_LEGACY_SPANS}
|
|
6182
|
+
WHERE toDate(COALESCE(endedAt, startedAt)) = {batchDate:Date}
|
|
6183
|
+
${dedupOrder}
|
|
6184
|
+
LIMIT 1 BY "traceId", "spanId"`,
|
|
6185
|
+
query_params: { batchDate: day }
|
|
6186
|
+
});
|
|
6187
|
+
migratedRows += dayCount;
|
|
6188
|
+
batches++;
|
|
6189
|
+
logger?.info?.(`Migrated batch ${day}: ${dayCount} rows`);
|
|
6190
|
+
}
|
|
6191
|
+
await client.command({
|
|
6192
|
+
query: `CREATE TABLE IF NOT EXISTS ${TABLE_LEGACY_MIGRATION_DONE} (completedAt DateTime64(3, 'UTC')) ENGINE = MergeTree ORDER BY completedAt`
|
|
6193
|
+
});
|
|
6194
|
+
await client.command({
|
|
6195
|
+
query: `INSERT INTO ${TABLE_LEGACY_MIGRATION_DONE} VALUES (now64(3))`
|
|
6196
|
+
});
|
|
6197
|
+
logger?.info?.(
|
|
6198
|
+
`Legacy span migration complete: ${migratedRows} rows in ${batches} batches. The legacy table '${TABLE_LEGACY_SPANS}' has been preserved. Drop it manually after verifying the migration.`
|
|
6199
|
+
);
|
|
6200
|
+
return { migratedRows, batches };
|
|
6201
|
+
}
|
|
5985
6202
|
async function migrateSignalTables(client, logger) {
|
|
5986
6203
|
for (const { table, createDDL, idColumn } of SIGNAL_MIGRATIONS) {
|
|
5987
6204
|
const engine = await getTableEngine(client, table);
|
|
@@ -7296,6 +7513,15 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7296
7513
|
})
|
|
7297
7514
|
});
|
|
7298
7515
|
}
|
|
7516
|
+
try {
|
|
7517
|
+
const legacyStatus = await checkLegacySpanMigrationStatus(this.#client);
|
|
7518
|
+
if (legacyStatus.needsMigration) {
|
|
7519
|
+
this.logger?.warn?.(
|
|
7520
|
+
`Legacy span table 'mastra_ai_spans' detected. Run 'npx mastra migrate' to migrate historical spans to the v-next schema.`
|
|
7521
|
+
);
|
|
7522
|
+
}
|
|
7523
|
+
} catch {
|
|
7524
|
+
}
|
|
7299
7525
|
try {
|
|
7300
7526
|
await assertExistingTablesCompatibleWithReplication(this.#client, this.#replication);
|
|
7301
7527
|
const existingStrategy = await detectExistingDeltaCursorStrategy(this.#client);
|
|
@@ -7370,34 +7596,44 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
|
|
|
7370
7596
|
}
|
|
7371
7597
|
}
|
|
7372
7598
|
/**
|
|
7373
|
-
* Manually migrate legacy
|
|
7374
|
-
*
|
|
7375
|
-
*
|
|
7599
|
+
* Manually migrate legacy tables to the v-next schema.
|
|
7600
|
+
* Handles both signal table migrations (MergeTree → ReplacingMergeTree)
|
|
7601
|
+
* and legacy span migration (mastra_ai_spans → mastra_span_events).
|
|
7376
7602
|
*/
|
|
7377
7603
|
async migrateSpans() {
|
|
7378
|
-
const
|
|
7379
|
-
|
|
7380
|
-
|
|
7381
|
-
|
|
7382
|
-
|
|
7383
|
-
|
|
7384
|
-
|
|
7385
|
-
|
|
7604
|
+
const messages = [];
|
|
7605
|
+
const signalStatus = await checkSignalTablesMigrationStatus(this.#client);
|
|
7606
|
+
if (signalStatus.needsMigration) {
|
|
7607
|
+
if (isReplicationConfigured(this.#replication)) {
|
|
7608
|
+
throw new error.MastraError({
|
|
7609
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "SIGNAL_TABLES_MIGRATION_UNSUPPORTED"),
|
|
7610
|
+
domain: error.ErrorDomain.STORAGE,
|
|
7611
|
+
category: error.ErrorCategory.USER,
|
|
7612
|
+
text: "ClickHouse replication is enabled, so Mastra will not run copy-and-swap signal table migrations automatically. Migrate existing local signal tables manually before enabling replication."
|
|
7613
|
+
});
|
|
7614
|
+
}
|
|
7615
|
+
await migrateSignalTables(this.#client, this.logger);
|
|
7616
|
+
messages.push(`Migrated signal tables: ${signalStatus.tables.map((t) => t.table).join(", ")}.`);
|
|
7386
7617
|
}
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
7390
|
-
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
|
|
7618
|
+
const legacyStatus = await checkLegacySpanMigrationStatus(this.#client);
|
|
7619
|
+
if (legacyStatus.needsMigration) {
|
|
7620
|
+
if (isReplicationConfigured(this.#replication)) {
|
|
7621
|
+
throw new error.MastraError({
|
|
7622
|
+
id: storage.createStorageErrorId("CLICKHOUSE", "REPLICATION", "LEGACY_SPAN_MIGRATION_UNSUPPORTED"),
|
|
7623
|
+
domain: error.ErrorDomain.STORAGE,
|
|
7624
|
+
category: error.ErrorCategory.USER,
|
|
7625
|
+
text: "ClickHouse replication is enabled. Migrate legacy mastra_ai_spans manually before enabling replication."
|
|
7626
|
+
});
|
|
7627
|
+
}
|
|
7628
|
+
const result = await migrateLegacySpans(this.#client, this.logger);
|
|
7629
|
+
messages.push(`Migrated ${result.migratedRows} legacy spans in ${result.batches} batches.`);
|
|
7394
7630
|
}
|
|
7395
|
-
|
|
7631
|
+
const alreadyMigrated = !signalStatus.needsMigration && !legacyStatus.needsMigration;
|
|
7396
7632
|
return {
|
|
7397
7633
|
success: true,
|
|
7398
|
-
alreadyMigrated
|
|
7634
|
+
alreadyMigrated,
|
|
7399
7635
|
duplicatesRemoved: 0,
|
|
7400
|
-
message:
|
|
7636
|
+
message: alreadyMigrated ? "Migration already complete." : `Migration complete. ${messages.join(" ")}`
|
|
7401
7637
|
};
|
|
7402
7638
|
}
|
|
7403
7639
|
// -------------------------------------------------------------------------
|