@mastra/clickhouse 1.0.0-beta.10 → 1.0.0-beta.12

package/dist/index.cjs

@@ -15,8 +15,10 @@ var TABLE_ENGINES = {
   [storage.TABLE_THREADS]: `ReplacingMergeTree()`,
   [storage.TABLE_SCORERS]: `MergeTree()`,
   [storage.TABLE_RESOURCES]: `ReplacingMergeTree()`,
-  // TODO: verify this is the correct engine for Spans when implementing clickhouse storage
-  [storage.TABLE_SPANS]: `ReplacingMergeTree()`,
+  // ReplacingMergeTree(updatedAt) deduplicates rows with the same (traceId, spanId) sorting key,
+  // keeping the row with the highest updatedAt value. Combined with ORDER BY (traceId, spanId),
+  // this provides eventual uniqueness for the (traceId, spanId) composite key.
+  [storage.TABLE_SPANS]: `ReplacingMergeTree(updatedAt)`,
   mastra_agents: `ReplacingMergeTree()`
 };
 var COLUMN_TYPES = {
@@ -99,6 +101,213 @@ var ClickhouseDB = class extends base.MastraBase {
     const columns = await result.json();
     return columns.some((c) => c.name === column);
   }
+  /**
+   * Checks if a table exists in the database.
+   */
+  async tableExists(tableName) {
+    try {
+      const result = await this.client.query({
+        query: `EXISTS TABLE ${tableName}`,
+        format: "JSONEachRow"
+      });
+      const rows = await result.json();
+      return rows[0]?.result === 1;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Gets the sorting key (ORDER BY columns) for a table.
+   * Returns null if the table doesn't exist.
+   */
+  async getTableSortingKey(tableName) {
+    try {
+      const result = await this.client.query({
+        query: `SELECT sorting_key FROM system.tables WHERE database = currentDatabase() AND name = {tableName:String}`,
+        query_params: { tableName },
+        format: "JSONEachRow"
+      });
+      const rows = await result.json();
+      return rows[0]?.sorting_key ?? null;
+    } catch {
+      return null;
+    }
+  }
+  /**
+   * Checks if migration is needed for the spans table.
+   * Returns information about the current state.
+   */
+  async checkSpansMigrationStatus(tableName) {
+    const exists = await this.tableExists(tableName);
+    if (!exists) {
+      return { needsMigration: false, currentSortingKey: null };
+    }
+    const currentSortingKey = await this.getTableSortingKey(tableName);
+    if (!currentSortingKey) {
+      return { needsMigration: false, currentSortingKey: null };
+    }
+    const needsMigration = currentSortingKey.toLowerCase().startsWith("createdat");
+    return { needsMigration, currentSortingKey };
+  }
+  /**
+   * Checks for duplicate (traceId, spanId) combinations in the spans table.
+   * Returns information about duplicates for logging/CLI purposes.
+   */
+  async checkForDuplicateSpans(tableName) {
+    try {
+      const result = await this.client.query({
+        query: `
+          SELECT count() as duplicate_count
+          FROM (
+            SELECT traceId, spanId
+            FROM ${tableName}
+            GROUP BY traceId, spanId
+            HAVING count() > 1
+          )
+        `,
+        format: "JSONEachRow"
+      });
+      const rows = await result.json();
+      const duplicateCount = parseInt(rows[0]?.duplicate_count ?? "0", 10);
+      return {
+        hasDuplicates: duplicateCount > 0,
+        duplicateCount
+      };
+    } catch (error) {
+      this.logger?.debug?.(`Could not check for duplicates: ${error}`);
+      return { hasDuplicates: false, duplicateCount: 0 };
+    }
+  }
+  /**
+   * Migrates the spans table from the old sorting key (createdAt, traceId, spanId)
+   * to the new sorting key (traceId, spanId) for proper uniqueness enforcement.
+   *
+   * This migration:
+   * 1. Renames the old table to a backup
+   * 2. Creates a new table with the correct sorting key
+   * 3. Copies all data from the backup to the new table, deduplicating by (traceId, spanId)
+   *    using priority-based selection:
+   *    - First, prefer completed spans (those with endedAt set)
+   *    - Then prefer the most recently updated span (highest updatedAt)
+   *    - Finally use creation time as tiebreaker (highest createdAt)
+   * 4. Drops the backup table
+   *
+   * The deduplication strategy matches the PostgreSQL migration (PR #12073) to ensure
+   * consistent behavior across storage backends.
+   *
+   * The migration is idempotent - it only runs if the old sorting key is detected.
+   *
+   * @returns true if migration was performed, false if not needed
+   */
+  async migrateSpansTableSortingKey({
+    tableName,
+    schema
+  }) {
+    if (tableName !== storage.TABLE_SPANS) {
+      return false;
+    }
+    const exists = await this.tableExists(tableName);
+    if (!exists) {
+      return false;
+    }
+    const currentSortingKey = await this.getTableSortingKey(tableName);
+    if (!currentSortingKey) {
+      return false;
+    }
+    const needsMigration = currentSortingKey.toLowerCase().startsWith("createdat");
+    if (!needsMigration) {
+      this.logger?.debug?.(`Spans table already has correct sorting key: ${currentSortingKey}`);
+      return false;
+    }
+    this.logger?.info?.(`Migrating spans table from sorting key "${currentSortingKey}" to "(traceId, spanId)"`);
+    const backupTableName = `${tableName}_backup_${Date.now()}`;
+    const rowTtl = this.ttl?.[tableName]?.row;
+    try {
+      await this.client.command({
+        query: `RENAME TABLE ${tableName} TO ${backupTableName}`
+      });
+      const columns = Object.entries(schema).map(([name, def]) => {
+        let sqlType = this.getSqlType(def.type);
+        let isNullable = def.nullable === true;
+        if (tableName === storage.TABLE_SPANS && name === "updatedAt") {
+          isNullable = false;
+        }
+        if (isNullable) {
+          sqlType = `Nullable(${sqlType})`;
+        }
+        const constraints = [];
+        if (name === "metadata" && (def.type === "text" || def.type === "jsonb") && isNullable) {
+          constraints.push("DEFAULT '{}'");
+        }
+        const columnTtl = this.ttl?.[tableName]?.columns?.[name];
+        return `"${name}" ${sqlType} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
+      }).join(",\n");
+      const createSql = `
+        CREATE TABLE ${tableName} (
+          ${columns}
+        )
+        ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
+        PRIMARY KEY (traceId, spanId)
+        ORDER BY (traceId, spanId)
+        ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
+        SETTINGS index_granularity = 8192
+      `;
+      await this.client.command({
+        query: createSql
+      });
+      const describeResult = await this.client.query({
+        query: `DESCRIBE TABLE ${backupTableName}`,
+        format: "JSONEachRow"
+      });
+      const backupColumns = await describeResult.json();
+      const backupColumnNames = new Set(backupColumns.map((c) => c.name));
+      const columnsToInsert = Object.keys(schema).filter((col) => backupColumnNames.has(col));
+      const columnList = columnsToInsert.map((c) => `"${c}"`).join(", ");
+      const selectExpressions = columnsToInsert.map((c) => c === "updatedAt" ? `COALESCE("updatedAt", "createdAt") as "updatedAt"` : `"${c}"`).join(", ");
+      await this.client.command({
+        query: `INSERT INTO ${tableName} (${columnList})
+                SELECT ${selectExpressions}
+                FROM ${backupTableName}
+                ORDER BY traceId, spanId,
+                         (endedAt IS NOT NULL AND endedAt != '') DESC,
+                         COALESCE(updatedAt, createdAt) DESC,
+                         createdAt DESC
+                LIMIT 1 BY traceId, spanId`
+      });
+      await this.client.command({
+        query: `DROP TABLE ${backupTableName}`
+      });
+      this.logger?.info?.(`Successfully migrated spans table to new sorting key`);
+      return true;
+    } catch (error$1) {
+      this.logger?.error?.(`Migration failed: ${error$1.message}`);
+      try {
+        const originalExists = await this.tableExists(tableName);
+        const backupExists = await this.tableExists(backupTableName);
+        if (!originalExists && backupExists) {
+          this.logger?.info?.(`Restoring spans table from backup`);
+          await this.client.command({
+            query: `RENAME TABLE ${backupTableName} TO ${tableName}`
+          });
+        } else if (originalExists && backupExists) {
+          await this.client.command({
+            query: `DROP TABLE IF EXISTS ${backupTableName}`
+          });
+        }
+      } catch (restoreError) {
+        this.logger?.error?.(`Failed to restore from backup: ${restoreError}`);
+      }
+      throw new error.MastraError(
+        {
+          id: storage.createStorageErrorId("CLICKHOUSE", "MIGRATE_SPANS_SORTING_KEY", "FAILED"),
+          domain: error.ErrorDomain.STORAGE,
+          category: error.ErrorCategory.THIRD_PARTY,
+          details: { tableName, currentSortingKey }
+        },
+        error$1
+      );
+    }
+  }
   getSqlType(type) {
     switch (type) {
       case "text":
@@ -125,7 +334,10 @@ var ClickhouseDB = class extends base.MastraBase {
     try {
       const columns = Object.entries(schema).map(([name, def]) => {
         let sqlType = this.getSqlType(def.type);
-        const isNullable = def.nullable === true;
+        let isNullable = def.nullable === true;
+        if (tableName === storage.TABLE_SPANS && name === "updatedAt") {
+          isNullable = false;
+        }
         if (isNullable) {
           sqlType = `Nullable(${sqlType})`;
         }
@@ -155,8 +367,8 @@ var ClickhouseDB = class extends base.MastraBase {
               ${columns}
             )
             ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
-            PRIMARY KEY (createdAt, traceId, spanId)
-            ORDER BY (createdAt, traceId, spanId)
+            PRIMARY KEY (traceId, spanId)
+            ORDER BY (traceId, spanId)
             ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
             SETTINGS index_granularity = 8192
           `;
@@ -302,7 +514,9 @@ var ClickhouseDB = class extends base.MastraBase {
       ...Object.fromEntries(
         Object.entries(record).map(([key, value]) => [
           key,
-          storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
+          // Only convert to Date if it's a timestamp column AND value is not null/undefined
+          // new Date(null) returns epoch date, not null, so we must check first
+          storage.TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" && value != null ? new Date(value).toISOString() : value
         ])
       )
     }));
@@ -1068,26 +1282,68 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
       );
     }
   }
-  async listThreadsByResourceId(args) {
-    const { resourceId, page = 0, perPage: perPageInput, orderBy } = args;
+  async listThreads(args) {
+    const { page = 0, perPage: perPageInput, orderBy, filter } = args;
+    try {
+      this.validatePaginationInput(page, perPageInput ?? 100);
+    } catch (error$1) {
+      throw new error.MastraError(
+        {
+          id: storage.createStorageErrorId("CLICKHOUSE", "LIST_THREADS", "INVALID_PAGE"),
+          domain: error.ErrorDomain.STORAGE,
+          category: error.ErrorCategory.USER,
+          details: { page, ...perPageInput !== void 0 && { perPage: perPageInput } }
+        },
+        error$1 instanceof Error ? error$1 : new Error("Invalid pagination parameters")
+      );
+    }
     const perPage = storage.normalizePerPage(perPageInput, 100);
-    if (page < 0) {
+    try {
+      this.validateMetadataKeys(filter?.metadata);
+    } catch (error$1) {
       throw new error.MastraError(
         {
-          id: storage.createStorageErrorId("CLICKHOUSE", "LIST_THREADS_BY_RESOURCE_ID", "INVALID_PAGE"),
+          id: storage.createStorageErrorId("CLICKHOUSE", "LIST_THREADS", "INVALID_METADATA_KEY"),
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.USER,
-          details: { page }
+          details: { metadataKeys: filter?.metadata ? Object.keys(filter.metadata).join(", ") : "" }
         },
-        new Error("page must be >= 0")
+        error$1 instanceof Error ? error$1 : new Error("Invalid metadata key")
       );
     }
     const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
     const { field, direction } = this.parseOrderBy(orderBy);
     try {
+      const whereClauses = [];
+      const queryParams = {};
+      if (filter?.resourceId) {
+        whereClauses.push("resourceId = {resourceId:String}");
+        queryParams.resourceId = filter.resourceId;
+      }
+      if (filter?.metadata && Object.keys(filter.metadata).length > 0) {
+        let metadataIndex = 0;
+        for (const [key, value] of Object.entries(filter.metadata)) {
+          const paramName = `metadata${metadataIndex}`;
+          whereClauses.push(`JSONExtractRaw(metadata, '${key}') = {${paramName}:String}`);
+          queryParams[paramName] = JSON.stringify(value);
+          metadataIndex++;
+        }
+      }
       const countResult = await this.client.query({
-        query: `SELECT count(DISTINCT id) as total FROM ${storage.TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
-        query_params: { resourceId },
+        query: `
+          WITH ranked_threads AS (
+            SELECT
+              id,
+              resourceId,
+              metadata,
+              ROW_NUMBER() OVER (PARTITION BY id ORDER BY updatedAt DESC) as row_num
+            FROM ${storage.TABLE_THREADS}
+          )
+          SELECT count(*) as total 
+          FROM ranked_threads 
+          WHERE row_num = 1 ${whereClauses.length > 0 ? `AND ${whereClauses.join(" AND ")}` : ""}
+        `,
+        query_params: queryParams,
         clickhouse_settings: {
           date_time_input_format: "best_effort",
           date_time_output_format: "iso",
@@ -1118,7 +1374,6 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
                   toDateTime64(updatedAt, 3) as updatedAt,
                   ROW_NUMBER() OVER (PARTITION BY id ORDER BY updatedAt DESC) as row_num
                 FROM ${storage.TABLE_THREADS}
-                WHERE resourceId = {resourceId:String}
               )
               SELECT
                 id,
@@ -1128,12 +1383,12 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
                 createdAt,
                 updatedAt
               FROM ranked_threads
-              WHERE row_num = 1
+              WHERE row_num = 1 ${whereClauses.length > 0 ? `AND ${whereClauses.join(" AND ")}` : ""}
               ORDER BY "${field}" ${direction === "DESC" ? "DESC" : "ASC"}
               LIMIT {perPage:Int64} OFFSET {offset:Int64}
             `,
         query_params: {
-          resourceId,
+          ...queryParams,
           perPage,
           offset
         },
@@ -1159,10 +1414,14 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
     } catch (error$1) {
       throw new error.MastraError(
         {
-          id: storage.createStorageErrorId("CLICKHOUSE", "LIST_THREADS_BY_RESOURCE_ID", "FAILED"),
+          id: storage.createStorageErrorId("CLICKHOUSE", "LIST_THREADS", "FAILED"),
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
-          details: { resourceId, page }
+          details: {
+            ...filter?.resourceId && { resourceId: filter.resourceId },
+            hasMetadataFilter: !!filter?.metadata,
+            page
+          }
         },
         error$1
       );
@@ -1599,11 +1858,101 @@ var ObservabilityStorageClickhouse = class extends storage.ObservabilityStorage
     this.#db = new ClickhouseDB({ client, ttl });
   }
   async init() {
+    const migrationStatus = await this.#db.checkSpansMigrationStatus(storage.TABLE_SPANS);
+    if (migrationStatus.needsMigration) {
+      const duplicateInfo = await this.#db.checkForDuplicateSpans(storage.TABLE_SPANS);
+      const duplicateMessage = duplicateInfo.hasDuplicates ? `
+Found ${duplicateInfo.duplicateCount} duplicate (traceId, spanId) combinations that will be removed.
+` : "";
+      const errorMessage = `
+===========================================================================
+MIGRATION REQUIRED: ClickHouse spans table needs sorting key update
+===========================================================================
+
+The spans table structure has changed. ClickHouse requires a table recreation
+to update the sorting key from (traceId) to (traceId, spanId).
+` + duplicateMessage + `
+To fix this, run the manual migration command:
+
+  npx mastra migrate
+
+This command will:
+  1. Create a new table with the correct sorting key
+  2. Copy data from the old table (deduplicating if needed)
+  3. Replace the old table with the new one
+
+WARNING: This migration involves table recreation and may take significant
+time for large tables. Please ensure you have a backup before proceeding.
+===========================================================================
+`;
+      throw new error.MastraError({
+        id: storage.createStorageErrorId("CLICKHOUSE", "MIGRATION_REQUIRED", "SORTING_KEY_CHANGE"),
+        domain: error.ErrorDomain.STORAGE,
+        category: error.ErrorCategory.USER,
+        text: errorMessage
+      });
+    }
     await this.#db.createTable({ tableName: storage.TABLE_SPANS, schema: storage.SPAN_SCHEMA });
   }
   async dangerouslyClearAll() {
     await this.#db.clearTable({ tableName: storage.TABLE_SPANS });
   }
+  /**
+   * Manually run the spans migration to deduplicate and update the sorting key.
+   * This is intended to be called from the CLI when duplicates are detected.
+   *
+   * @returns Migration result with status and details
+   */
+  async migrateSpans() {
+    const migrationStatus = await this.#db.checkSpansMigrationStatus(storage.TABLE_SPANS);
+    if (!migrationStatus.needsMigration) {
+      return {
+        success: true,
+        alreadyMigrated: true,
+        duplicatesRemoved: 0,
+        message: `Migration already complete. Spans table has correct sorting key.`
+      };
+    }
+    const duplicateInfo = await this.#db.checkForDuplicateSpans(storage.TABLE_SPANS);
+    if (duplicateInfo.hasDuplicates) {
+      this.logger?.info?.(
+        `Found ${duplicateInfo.duplicateCount} duplicate (traceId, spanId) combinations. Starting migration with deduplication...`
+      );
+    } else {
+      this.logger?.info?.(`No duplicate spans found. Starting sorting key migration...`);
+    }
+    await this.#db.migrateSpansTableSortingKey({ tableName: storage.TABLE_SPANS, schema: storage.SPAN_SCHEMA });
+    return {
+      success: true,
+      alreadyMigrated: false,
+      duplicatesRemoved: duplicateInfo.duplicateCount,
+      message: duplicateInfo.hasDuplicates ? `Migration complete. Removed duplicates and updated sorting key for ${storage.TABLE_SPANS}.` : `Migration complete. Updated sorting key for ${storage.TABLE_SPANS}.`
+    };
+  }
+  /**
+   * Check migration status for the spans table.
+   * Returns information about whether migration is needed.
+   */
+  async checkSpansMigrationStatus() {
+    const migrationStatus = await this.#db.checkSpansMigrationStatus(storage.TABLE_SPANS);
+    if (!migrationStatus.needsMigration) {
+      return {
+        needsMigration: false,
+        hasDuplicates: false,
+        duplicateCount: 0,
+        constraintExists: true,
+        tableName: storage.TABLE_SPANS
+      };
+    }
+    const duplicateInfo = await this.#db.checkForDuplicateSpans(storage.TABLE_SPANS);
+    return {
+      needsMigration: true,
+      hasDuplicates: duplicateInfo.hasDuplicates,
+      duplicateCount: duplicateInfo.duplicateCount,
+      constraintExists: false,
+      tableName: storage.TABLE_SPANS
+    };
+  }
   get tracingStrategy() {
     return {
       preferred: "insert-only",
@@ -1937,10 +2286,10 @@ var ObservabilityStorageClickhouse = class extends storage.ObservabilityStorage
               conditions.push(`(error IS NOT NULL AND error != '')`);
               break;
             case storage.TraceStatus.RUNNING:
-              conditions.push(`(endedAt IS NULL OR endedAt = '') AND (error IS NULL OR error = '')`);
+              conditions.push(`endedAt IS NULL AND (error IS NULL OR error = '')`);
               break;
             case storage.TraceStatus.SUCCESS:
-              conditions.push(`(endedAt IS NOT NULL AND endedAt != '') AND (error IS NULL OR error = '')`);
+              conditions.push(`endedAt IS NOT NULL AND (error IS NULL OR error = '')`);
               break;
           }
         }
@@ -1969,7 +2318,7 @@ var ObservabilityStorageClickhouse = class extends storage.ObservabilityStorage
       if (sortField === "endedAt") {
         const nullSortValue = sortDirection === "DESC" ? 0 : 1;
         const nonNullSortValue = sortDirection === "DESC" ? 1 : 0;
-        orderClause = `ORDER BY CASE WHEN ${sortField} IS NULL OR ${sortField} = '' THEN ${nullSortValue} ELSE ${nonNullSortValue} END, ${sortField} ${sortDirection}`;
+        orderClause = `ORDER BY CASE WHEN ${sortField} IS NULL THEN ${nullSortValue} ELSE ${nonNullSortValue} END, ${sortField} ${sortDirection}`;
       } else {
         orderClause = `ORDER BY ${sortField} ${sortDirection}`;
       }
@@ -2875,7 +3224,7 @@ var WorkflowsStorageClickhouse = class extends storage.WorkflowsStorage {
 var isClientConfig = (config) => {
   return "client" in config;
 };
-var ClickhouseStore = class extends storage.MastraStorage {
+var ClickhouseStore = class extends storage.MastraCompositeStore {
   db;
   ttl = {};
   stores;