@mastra/clickhouse 1.7.1-alpha.0 → 1.7.1-alpha.2

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @mastra/clickhouse
 
+## 1.7.1-alpha.2
+
+### Patch Changes
+
+- Fixed agent streams intermittently hanging when observability storage was backed by Replicated/Shared ClickHouse. Startup no longer re-applies no-op schema updates (e.g. `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT EXISTS`, `MODIFY TTL`), so it no longer triggers replica-lag retry errors that could leave storage in a stuck state. ([#16420](https://github.com/mastra-ai/mastra/pull/16420))
+
+- Updated dependencies [[`7ad5585`](https://github.com/mastra-ai/mastra/commit/7ad55856406f1de398dc713f6a9eaa78b2784bb6), [`210ea7a`](https://github.com/mastra-ai/mastra/commit/210ea7af559791b73a44fc9c12179908aaa3183f), [`83218c8`](https://github.com/mastra-ai/mastra/commit/83218c88b37773c9424fbe733b37be556e55e94d), [`265ec9f`](https://github.com/mastra-ai/mastra/commit/265ec9f887b5c81255c873a76ff7796f16e4f99b), [`6ce80bf`](https://github.com/mastra-ai/mastra/commit/6ce80bf4872a891e0bddf8b80561a80584efb14b), [`9268531`](https://github.com/mastra-ai/mastra/commit/9268531e7ec4be98beeba3b3ae8be0a7ea380662), [`13ead79`](https://github.com/mastra-ai/mastra/commit/13ead79149486b88144db7e11e6ff551caef5be1), [`bd36d8e`](https://github.com/mastra-ai/mastra/commit/bd36d8eb6de8c9a0310352649dbd4b06703c2299), [`8ac9141`](https://github.com/mastra-ai/mastra/commit/8ac9141439caa8fdd674944c4d84f29b3c730296)]:
+  - @mastra/core@1.33.0-alpha.10
+
+## 1.7.1-alpha.1
+
+### Patch Changes
+
+- Respect optional `resourceId` in `getThreadById` so scoped thread lookups return `null` when the thread belongs to a different resource. ([#14237](https://github.com/mastra-ai/mastra/pull/14237))
+
+  Example:
+
+  ```typescript
+  const thread = await memory.getThreadById({
+    threadId: 'my-thread-id',
+    resourceId: 'my-user-id',
+  });
+  // Returns null if the thread does not belong to 'my-user-id'.
+  ```
+
+- Updated dependencies [[`7c275a8`](https://github.com/mastra-ai/mastra/commit/7c275a810595e1a6c41ccc39720531ab65734700), [`890b24c`](https://github.com/mastra-ai/mastra/commit/890b24cc7d32ed6aa4dfe253e54dc6bf4099f690), [`0f48ebf`](https://github.com/mastra-ai/mastra/commit/0f48ebfc7ac7897b2092a189f45751924cf56d1c), [`f180e49`](https://github.com/mastra-ai/mastra/commit/f180e4990e71b04c9a475b523584071712f0048f), [`9260e01`](https://github.com/mastra-ai/mastra/commit/9260e015276fb1b500f7878ee452b47476bf1583), [`2f6c54e`](https://github.com/mastra-ai/mastra/commit/2f6c54e17c041cac1def54baaa6b771647836414), [`e06a159`](https://github.com/mastra-ai/mastra/commit/e06a1598ca07a6c3778aefc2a2d288363c6294ff), [`db34bc6`](https://github.com/mastra-ai/mastra/commit/db34bc6fb36cf125bda0c46be4d3fdc774b70cc4)]:
+  - @mastra/core@1.33.0-alpha.8
+
 ## 1.7.1-alpha.0
 
 ### Patch Changes

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-clickhouse
 description: Documentation for @mastra/clickhouse. Use when working with @mastra/clickhouse APIs, configuration, or implementation.
 metadata:
   package: "@mastra/clickhouse"
-  version: "1.7.1-alpha.0"
+  version: "1.7.1-alpha.2"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.7.1-alpha.0",
+  "version": "1.7.1-alpha.2",
   "package": "@mastra/clickhouse",
   "exports": {},
   "modules": {}

package/dist/docs/references/reference-storage-clickhouse.md

@@ -51,7 +51,7 @@ import { Mastra } from '@mastra/core'
 import { MastraCompositeStore } from '@mastra/core/storage'
 import { PostgresStore } from '@mastra/pg'
 import { ObservabilityStorageClickhouseVNext } from '@mastra/clickhouse'
-import { Observability, DefaultExporter } from '@mastra/observability'
+import { Observability, MastraStorageExporter } from '@mastra/observability'
 
 const observabilityStore = new ObservabilityStorageClickhouseVNext({
   url: process.env.CLICKHOUSE_URL!,
@@ -74,14 +74,14 @@ export const mastra = new Mastra({
     configs: {
       default: {
         serviceName: 'mastra',
-        exporters: [new DefaultExporter()],
+        exporters: [new MastraStorageExporter()],
       },
     },
   }),
 })
 ```
 
-`DefaultExporter` automatically selects the `insert-only` strategy when ClickHouse is the observability backend, which gives the highest write throughput. See [tracing strategies](https://mastra.ai/docs/observability/tracing/exporters/default) for details.
+`MastraStorageExporter` automatically selects the `insert-only` strategy when ClickHouse is the observability backend, which gives the highest write throughput. See [tracing strategies](https://mastra.ai/docs/observability/tracing/exporters/mastra-storage) for details.
 
 ### Observability with the legacy domain
 
@@ -226,7 +226,7 @@ import { Mastra } from '@mastra/core'
 import { MastraCompositeStore } from '@mastra/core/storage'
 import { PostgresStore } from '@mastra/pg'
 import { ObservabilityStorageClickhouseVNext } from '@mastra/clickhouse'
-import { Observability, DefaultExporter } from '@mastra/observability'
+import { Observability, MastraStorageExporter } from '@mastra/observability'
 
 export const mastra = new Mastra({
   storage: new MastraCompositeStore({
@@ -247,7 +247,7 @@ export const mastra = new Mastra({
     configs: {
       default: {
         serviceName: 'mastra',
-        exporters: [new DefaultExporter()],
+        exporters: [new MastraStorageExporter()],
       },
     },
   }),
@@ -287,14 +287,14 @@ In CI/CD pipelines, set `disableInit: true` on `ClickhouseStore` and run `init()
 
 ClickHouse is the recommended backend for production observability:
 
-- **Insert-only strategy**: `DefaultExporter` writes completed spans in batches without per-span updates, which is the highest-throughput strategy available.
+- **Insert-only strategy**: `MastraStorageExporter` writes completed spans in batches without per-span updates, which is the highest-throughput strategy available.
 - **Columnar compression**: Span attributes and log payloads compress well compared to the same data in row-oriented databases.
 
-For the full strategy matrix and production guidance, see the [`DefaultExporter` reference](https://mastra.ai/docs/observability/tracing/exporters/default).
+For the full strategy matrix and production guidance, see the [`MastraStorageExporter` reference](https://mastra.ai/docs/observability/tracing/exporters/mastra-storage).
 
 ## Related
 
 - [Storage overview](https://mastra.ai/reference/storage/overview)
 - [Composite storage](https://mastra.ai/reference/storage/composite)
-- [`DefaultExporter`](https://mastra.ai/docs/observability/tracing/exporters/default)
+- [`MastraStorageExporter`](https://mastra.ai/docs/observability/tracing/exporters/mastra-storage)
 - [Observability overview](https://mastra.ai/docs/observability/overview)

package/dist/docs/references/reference-storage-composite.md

@@ -242,4 +242,4 @@ const storage = new MastraCompositeStore({
 
 > **Note:** `ObservabilityStorageClickhouseVNext` is the current observability domain implementation. The legacy `ObservabilityStorageClickhouse` class is also exported and remains supported for projects that have not migrated. See the [ClickHouse storage reference](https://mastra.ai/reference/storage/clickhouse) for details.
 
-> **Info:** This approach is also required when using storage providers that don't support observability (like Convex, DynamoDB, or Cloudflare). See the [DefaultExporter documentation](https://mastra.ai/docs/observability/tracing/exporters/default) for the full list of supported providers.
+> **Info:** This approach is also required when using storage providers that don't support observability (like Convex, DynamoDB, or Cloudflare). See the [MastraStorageExporter documentation](https://mastra.ai/docs/observability/tracing/exporters/mastra-storage) for the full list of supported providers.

package/dist/index.cjs

@@ -1458,7 +1458,10 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
       );
     }
   }
-  async getThreadById({ threadId }) {
+  async getThreadById({
+    threadId,
+    resourceId
+  }) {
     try {
       const result = await this.client.query({
         query: `SELECT 
@@ -1483,7 +1486,7 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
       });
       const rows = await result.json();
       const thread = transformRow(rows.data[0]);
-      if (!thread) {
+      if (!thread || resourceId !== void 0 && thread.resourceId !== resourceId) {
         return null;
       }
       return {
@@ -3421,44 +3424,56 @@ var ALL_TABLE_DDL = [
 ];
 var ALL_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
 var DISCOVERY_MV_DDL = [DISCOVERY_VALUES_MV_DDL, DISCOVERY_PAIRS_MV_DDL];
+var addColumn = (table, name, type) => ({
+  kind: "column",
+  table,
+  name,
+  sql: `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${name} ${type}`
+});
+var addBloomIndex = (table, name, column) => ({
+  kind: "index",
+  table,
+  name,
+  sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY 2`
+});
 var ALL_MIGRATIONS = [
   // Span events
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SPAN_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SPAN_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Trace roots
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_TRACE_ROOTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_TRACE_ROOTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_TRACE_ROOTS, "rootEntityVersionId", "Nullable(String)"),
   // Metrics
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_METRIC_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_METRIC_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_METRIC_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Logs
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_LOG_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_LOG_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_LOG_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Scores
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_SCORE_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Feedback
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_FEEDBACK_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_FEEDBACK_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Metric skip indexes — additive, instant DDL. Existing parts keep no index
   // until merged or `MATERIALIZE INDEX` is run; new parts are bloom-filtered
   // immediately. With normal retention turning over the table, the index
   // converges to full coverage without an explicit backfill.
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_traceId traceId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_threadId threadId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_resourceId resourceId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_userId userId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_organizationId organizationId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_experimentId experimentId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_runId runId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_sessionId sessionId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_requestId requestId TYPE bloom_filter(0.01) GRANULARITY 2`
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_traceId", "traceId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_threadId", "threadId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_resourceId", "resourceId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_userId", "userId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_organizationId", "organizationId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_experimentId", "experimentId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_runId", "runId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId")
 ];
 var ALL_TABLE_NAMES = [
   TABLE_SPAN_EVENTS,
@@ -3487,8 +3502,8 @@ var SIGNAL_TO_TABLES = {
   scores: [TABLE_SCORE_EVENTS],
   feedback: [TABLE_FEEDBACK_EVENTS]
 };
-function buildRetentionDDL(retention) {
-  const statements = [];
+function buildRetentionEntries(retention) {
+  const entries = [];
   for (const [signal, days] of Object.entries(retention)) {
     const safeDays = Math.floor(Number(days));
     if (!Number.isFinite(safeDays) || safeDays <= 0) continue;
@@ -3497,10 +3512,24 @@ function buildRetentionDDL(retention) {
     for (const table of tables) {
       const col = SIGNAL_TTL_COLUMNS[table];
       if (!col) continue;
-      statements.push(`ALTER TABLE ${table} MODIFY TTL ${col} + INTERVAL ${safeDays} DAY`);
+      entries.push({
+        table,
+        column: col,
+        days: safeDays,
+        sql: `ALTER TABLE ${table} MODIFY TTL ${col} + INTERVAL ${safeDays} DAY`
+      });
     }
   }
-  return statements;
+  return entries;
+}
+function parseTtlExpression(expr) {
+  const match = expr.match(/TTL\s+(?:`([^`]+)`|(\w+))\s*\+\s*(?:toIntervalDay\((\d+)\)|INTERVAL\s+(\d+)\s+DAY)/i);
+  if (!match) return null;
+  const column = match[1] ?? match[2];
+  if (!column) return null;
+  const days = Number(match[3] ?? match[4]);
+  if (!Number.isFinite(days)) return null;
+  return { column, days };
 }
 var CH_SETTINGS = {
   date_time_input_format: "best_effort",
@@ -5961,6 +5990,74 @@ time for large databases. Please ensure you have a backup before proceeding.
 ===========================================================================
 `;
 }
+async function filterAppliedMigrations(client, migrations) {
+  if (migrations.length === 0) return migrations;
+  const tables = [...new Set(migrations.map((m) => m.table))];
+  let existingColumns;
+  let existingIndices;
+  try {
+    [existingColumns, existingIndices] = await Promise.all([
+      queryNamesByTable(
+        client,
+        `SELECT table, name FROM system.columns WHERE database = currentDatabase() AND table IN ({tables:Array(String)})`,
+        tables
+      ),
+      queryNamesByTable(
+        client,
+        `SELECT table, name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table IN ({tables:Array(String)})`,
+        tables
+      )
+    ]);
+  } catch {
+    return migrations;
+  }
+  return migrations.filter((m) => {
+    const present = m.kind === "column" ? existingColumns.get(m.table) : existingIndices.get(m.table);
+    if (!present) return true;
+    return !present.has(m.name);
+  });
+}
+async function filterAppliedRetention(client, entries) {
+  if (entries.length === 0) return entries;
+  const tables = [...new Set(entries.map((e) => e.table))];
+  let createQueries;
+  try {
+    const result = await client.query({
+      query: `SELECT name, create_table_query FROM system.tables WHERE database = currentDatabase() AND name IN ({tables:Array(String)})`,
+      query_params: { tables },
+      format: "JSONEachRow"
+    });
+    const rows = await result.json();
+    createQueries = new Map(rows.map((r) => [r.name, r.create_table_query ?? ""]));
+  } catch {
+    return entries;
+  }
+  return entries.filter((e) => {
+    const createQuery = createQueries.get(e.table);
+    if (!createQuery) return true;
+    const current = parseTtlExpression(createQuery);
+    if (!current) return true;
+    return current.column !== e.column || current.days !== e.days;
+  });
+}
+async function queryNamesByTable(client, query, tables) {
+  const result = await client.query({
+    query,
+    query_params: { tables },
+    format: "JSONEachRow"
+  });
+  const rows = await result.json();
+  const out = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    let set = out.get(row.table);
+    if (!set) {
+      set = /* @__PURE__ */ new Set();
+      out.set(row.table, set);
+    }
+    set.add(row.name);
+  }
+  return out;
+}
 var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilityStorage {
   #client;
   #retention;
@@ -5990,13 +6087,14 @@ var ObservabilityStorageClickhouseVNext = class extends storage.ObservabilitySto
       for (const ddl of [...ALL_TABLE_DDL, ...ALL_MV_DDL]) {
         await this.#client.command({ query: ddl });
       }
-      for (const migration of ALL_MIGRATIONS) {
-        await this.#client.command({ query: migration });
+      const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
+      for (const migration of pendingMigrations) {
+        await this.#client.command({ query: migration.sql });
       }
       if (this.#retention) {
-        const ttlStatements = buildRetentionDDL(this.#retention);
-        for (const stmt of ttlStatements) {
-          await this.#client.command({ query: stmt });
+        const pendingRetention = await filterAppliedRetention(this.#client, buildRetentionEntries(this.#retention));
+        for (const entry of pendingRetention) {
+          await this.#client.command({ query: entry.sql });
         }
       }
     } catch (error$1) {