@mastra/clickhouse 1.7.1-alpha.1 → 1.7.1

package/dist/index.js

@@ -3422,44 +3422,56 @@ var ALL_TABLE_DDL = [
 ];
 var ALL_MV_DDL = [TRACE_ROOTS_MV_DDL, TRACE_BRANCHES_MV_DDL];
 var DISCOVERY_MV_DDL = [DISCOVERY_VALUES_MV_DDL, DISCOVERY_PAIRS_MV_DDL];
+var addColumn = (table, name, type) => ({
+  kind: "column",
+  table,
+  name,
+  sql: `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${name} ${type}`
+});
+var addBloomIndex = (table, name, column) => ({
+  kind: "index",
+  table,
+  name,
+  sql: `ALTER TABLE ${table} ADD INDEX IF NOT EXISTS ${name} ${column} TYPE bloom_filter(0.01) GRANULARITY 2`
+});
 var ALL_MIGRATIONS = [
   // Span events
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SPAN_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_SPAN_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SPAN_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SPAN_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Trace roots
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_TRACE_ROOTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_TRACE_ROOTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_TRACE_ROOTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_TRACE_ROOTS, "rootEntityVersionId", "Nullable(String)"),
   // Metrics
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_METRIC_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_METRIC_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_METRIC_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Logs
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_LOG_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_LOG_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_LOG_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_LOG_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Scores
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_SCORE_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_SCORE_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SCORE_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_SCORE_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Feedback
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS entityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS parentEntityVersionId Nullable(String)`,
-  `ALTER TABLE ${TABLE_FEEDBACK_EVENTS} ADD COLUMN IF NOT EXISTS rootEntityVersionId Nullable(String)`,
+  addColumn(TABLE_FEEDBACK_EVENTS, "entityVersionId", "Nullable(String)"),
+  addColumn(TABLE_FEEDBACK_EVENTS, "parentEntityVersionId", "Nullable(String)"),
+  addColumn(TABLE_FEEDBACK_EVENTS, "rootEntityVersionId", "Nullable(String)"),
   // Metric skip indexes — additive, instant DDL. Existing parts keep no index
   // until merged or `MATERIALIZE INDEX` is run; new parts are bloom-filtered
   // immediately. With normal retention turning over the table, the index
   // converges to full coverage without an explicit backfill.
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_traceId traceId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_threadId threadId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_resourceId resourceId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_userId userId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_organizationId organizationId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_experimentId experimentId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_runId runId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_sessionId sessionId TYPE bloom_filter(0.01) GRANULARITY 2`,
-  `ALTER TABLE ${TABLE_METRIC_EVENTS} ADD INDEX IF NOT EXISTS idx_requestId requestId TYPE bloom_filter(0.01) GRANULARITY 2`
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_traceId", "traceId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_threadId", "threadId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_resourceId", "resourceId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_userId", "userId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_organizationId", "organizationId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_experimentId", "experimentId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_runId", "runId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_sessionId", "sessionId"),
+  addBloomIndex(TABLE_METRIC_EVENTS, "idx_requestId", "requestId")
 ];
 var ALL_TABLE_NAMES = [
   TABLE_SPAN_EVENTS,
@@ -3488,8 +3500,8 @@ var SIGNAL_TO_TABLES = {
   scores: [TABLE_SCORE_EVENTS],
   feedback: [TABLE_FEEDBACK_EVENTS]
 };
-function buildRetentionDDL(retention) {
-  const statements = [];
+function buildRetentionEntries(retention) {
+  const entries = [];
   for (const [signal, days] of Object.entries(retention)) {
     const safeDays = Math.floor(Number(days));
     if (!Number.isFinite(safeDays) || safeDays <= 0) continue;
@@ -3498,10 +3510,24 @@ function buildRetentionDDL(retention) {
     for (const table of tables) {
       const col = SIGNAL_TTL_COLUMNS[table];
       if (!col) continue;
-      statements.push(`ALTER TABLE ${table} MODIFY TTL ${col} + INTERVAL ${safeDays} DAY`);
+      entries.push({
+        table,
+        column: col,
+        days: safeDays,
+        sql: `ALTER TABLE ${table} MODIFY TTL ${col} + INTERVAL ${safeDays} DAY`
+      });
     }
   }
-  return statements;
+  return entries;
+}
+function parseTtlExpression(expr) {
+  const match = expr.match(/TTL\s+(?:`([^`]+)`|(\w+))\s*\+\s*(?:toIntervalDay\((\d+)\)|INTERVAL\s+(\d+)\s+DAY)/i);
+  if (!match) return null;
+  const column = match[1] ?? match[2];
+  if (!column) return null;
+  const days = Number(match[3] ?? match[4]);
+  if (!Number.isFinite(days)) return null;
+  return { column, days };
 }
 var CH_SETTINGS = {
   date_time_input_format: "best_effort",
@@ -5962,6 +5988,74 @@ time for large databases. Please ensure you have a backup before proceeding.
 ===========================================================================
 `;
 }
+async function filterAppliedMigrations(client, migrations) {
+  if (migrations.length === 0) return migrations;
+  const tables = [...new Set(migrations.map((m) => m.table))];
+  let existingColumns;
+  let existingIndices;
+  try {
+    [existingColumns, existingIndices] = await Promise.all([
+      queryNamesByTable(
+        client,
+        `SELECT table, name FROM system.columns WHERE database = currentDatabase() AND table IN ({tables:Array(String)})`,
+        tables
+      ),
+      queryNamesByTable(
+        client,
+        `SELECT table, name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table IN ({tables:Array(String)})`,
+        tables
+      )
+    ]);
+  } catch {
+    return migrations;
+  }
+  return migrations.filter((m) => {
+    const present = m.kind === "column" ? existingColumns.get(m.table) : existingIndices.get(m.table);
+    if (!present) return true;
+    return !present.has(m.name);
+  });
+}
+async function filterAppliedRetention(client, entries) {
+  if (entries.length === 0) return entries;
+  const tables = [...new Set(entries.map((e) => e.table))];
+  let createQueries;
+  try {
+    const result = await client.query({
+      query: `SELECT name, create_table_query FROM system.tables WHERE database = currentDatabase() AND name IN ({tables:Array(String)})`,
+      query_params: { tables },
+      format: "JSONEachRow"
+    });
+    const rows = await result.json();
+    createQueries = new Map(rows.map((r) => [r.name, r.create_table_query ?? ""]));
+  } catch {
+    return entries;
+  }
+  return entries.filter((e) => {
+    const createQuery = createQueries.get(e.table);
+    if (!createQuery) return true;
+    const current = parseTtlExpression(createQuery);
+    if (!current) return true;
+    return current.column !== e.column || current.days !== e.days;
+  });
+}
+async function queryNamesByTable(client, query, tables) {
+  const result = await client.query({
+    query,
+    query_params: { tables },
+    format: "JSONEachRow"
+  });
+  const rows = await result.json();
+  const out = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    let set = out.get(row.table);
+    if (!set) {
+      set = /* @__PURE__ */ new Set();
+      out.set(row.table, set);
+    }
+    set.add(row.name);
+  }
+  return out;
+}
 var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
   #client;
   #retention;
@@ -5991,13 +6085,14 @@ var ObservabilityStorageClickhouseVNext = class extends ObservabilityStorage {
       for (const ddl of [...ALL_TABLE_DDL, ...ALL_MV_DDL]) {
         await this.#client.command({ query: ddl });
       }
-      for (const migration of ALL_MIGRATIONS) {
-        await this.#client.command({ query: migration });
+      const pendingMigrations = await filterAppliedMigrations(this.#client, ALL_MIGRATIONS);
+      for (const migration of pendingMigrations) {
+        await this.#client.command({ query: migration.sql });
       }
       if (this.#retention) {
-        const ttlStatements = buildRetentionDDL(this.#retention);
-        for (const stmt of ttlStatements) {
-          await this.#client.command({ query: stmt });
+        const pendingRetention = await filterAppliedRetention(this.#client, buildRetentionEntries(this.#retention));
+        for (const entry of pendingRetention) {
+          await this.#client.command({ query: entry.sql });
         }
       }
     } catch (error) {