@mastra/clickhouse 1.0.0-beta.1 → 1.0.0-beta.2

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @mastra/clickhouse
 
+## 1.0.0-beta.2
+
+### Patch Changes
+
+- feat(storage): support querying messages from multiple threads ([#10663](https://github.com/mastra-ai/mastra/pull/10663))
+  - Fixed TypeScript errors where `threadId: string | string[]` was being passed to places expecting `Scalar` type
+  - Added proper multi-thread support for `listMessages` across all adapters when `threadId` is an array
+  - Updated `_getIncludedMessages` to look up message threadId by ID (since message IDs are globally unique)
+  - **upstash**: Added `msg-idx:{messageId}` index for O(1) message lookups (backwards compatible with fallback to scan for old messages, with automatic backfill)
+
+- fix: ensure score responses match saved payloads for Mastra Stores. ([#10557](https://github.com/mastra-ai/mastra/pull/10557))
+
+- Unify transformScoreRow functions across storage adapters ([#10648](https://github.com/mastra-ai/mastra/pull/10648))
+
+  Added a unified `transformScoreRow` function in `@mastra/core/storage` that provides schema-driven row transformation for score data. This eliminates code duplication across 10 storage adapters while maintaining store-specific behavior through configurable options:
+  - `preferredTimestampFields`: Preferred source fields for timestamps (PostgreSQL, Cloudflare D1)
+  - `convertTimestamps`: Convert timestamp strings to Date objects (MSSQL, MongoDB, ClickHouse)
+  - `nullValuePattern`: Skip values matching pattern (ClickHouse's `'_null_'`)
+  - `fieldMappings`: Map source column names to schema fields (LibSQL's `additionalLLMContext`)
+
+  Each store adapter now uses the unified function with appropriate options, reducing ~200 lines of duplicate transformation logic while ensuring consistent behavior across all storage backends.
+
+- Updated dependencies [[`ac0d2f4`](https://github.com/mastra-ai/mastra/commit/ac0d2f4ff8831f72c1c66c2be809706d17f65789), [`1a0d3fc`](https://github.com/mastra-ai/mastra/commit/1a0d3fc811482c9c376cdf79ee615c23bae9b2d6), [`85a628b`](https://github.com/mastra-ai/mastra/commit/85a628b1224a8f64cd82ea7f033774bf22df7a7e), [`c237233`](https://github.com/mastra-ai/mastra/commit/c23723399ccedf7f5744b3f40997b79246bfbe64), [`15f9e21`](https://github.com/mastra-ai/mastra/commit/15f9e216177201ea6e3f6d0bfb063fcc0953444f), [`ff94dea`](https://github.com/mastra-ai/mastra/commit/ff94dea935f4e34545c63bcb6c29804732698809), [`5b2ff46`](https://github.com/mastra-ai/mastra/commit/5b2ff4651df70c146523a7fca773f8eb0a2272f8), [`db41688`](https://github.com/mastra-ai/mastra/commit/db4168806d007417e2e60b4f68656dca4e5f40c9), [`5ca599d`](https://github.com/mastra-ai/mastra/commit/5ca599d0bb59a1595f19f58473fcd67cc71cef58), [`bff1145`](https://github.com/mastra-ai/mastra/commit/bff114556b3cbadad9b2768488708f8ad0e91475), [`5c8ca24`](https://github.com/mastra-ai/mastra/commit/5c8ca247094e0cc2cdbd7137822fb47241f86e77), [`e191844`](https://github.com/mastra-ai/mastra/commit/e1918444ca3f80e82feef1dad506cd4ec6e2875f), [`22553f1`](https://github.com/mastra-ai/mastra/commit/22553f11c63ee5e966a9c034a349822249584691), [`7237163`](https://github.com/mastra-ai/mastra/commit/72371635dbf96a87df4b073cc48fc655afbdce3d), [`2500740`](https://github.com/mastra-ai/mastra/commit/2500740ea23da067d6e50ec71c625ab3ce275e64), [`873ecbb`](https://github.com/mastra-ai/mastra/commit/873ecbb517586aa17d2f1e99283755b3ebb2863f), [`4f9bbe5`](https://github.com/mastra-ai/mastra/commit/4f9bbe5968f42c86f4930b8193de3c3c17e5bd36), [`02e51fe`](https://github.com/mastra-ai/mastra/commit/02e51feddb3d4155cfbcc42624fd0d0970d032c0), [`8f3fa3a`](https://github.com/mastra-ai/mastra/commit/8f3fa3a652bb77da092f913ec51ae46e3a7e27dc), [`cd29ad2`](https://github.com/mastra-ai/mastra/commit/cd29ad23a255534e8191f249593849ed29160886), [`bdf4d8c`](https://github.com/mastra-ai/mastra/commit/bdf4d8cdc656d8a2c21d81834bfa3bfa70f56c16), [`854e3da`](https://github.com/mastra-ai/mastra/commit/854e3dad5daac17a91a20986399d3a51f54bf68b), [`ce18d38`](https://github.com/mastra-ai/mastra/commit/ce18d38678c65870350d123955014a8432075fd9), [`cccf9c8`](https://github.com/mastra-ai/mastra/commit/cccf9c8b2d2dfc1a5e63919395b83d78c89682a0), [`61a5705`](https://github.com/mastra-ai/mastra/commit/61a570551278b6743e64243b3ce7d73de915ca8a), [`db70a48`](https://github.com/mastra-ai/mastra/commit/db70a48aeeeeb8e5f92007e8ede52c364ce15287), [`f0fdc14`](https://github.com/mastra-ai/mastra/commit/f0fdc14ee233d619266b3d2bbdeea7d25cfc6d13), [`db18bc9`](https://github.com/mastra-ai/mastra/commit/db18bc9c3825e2c1a0ad9a183cc9935f6691bfa1), [`9b37b56`](https://github.com/mastra-ai/mastra/commit/9b37b565e1f2a76c24f728945cc740c2b09be9da), [`41a23c3`](https://github.com/mastra-ai/mastra/commit/41a23c32f9877d71810f37e24930515df2ff7a0f), [`5d171ad`](https://github.com/mastra-ai/mastra/commit/5d171ad9ef340387276b77c2bb3e83e83332d729), [`f03ae60`](https://github.com/mastra-ai/mastra/commit/f03ae60500fe350c9d828621006cdafe1975fdd8), [`d1e74a0`](https://github.com/mastra-ai/mastra/commit/d1e74a0a293866dece31022047f5dbab65a304d0), [`39e7869`](https://github.com/mastra-ai/mastra/commit/39e7869bc7d0ee391077ce291474d8a84eedccff), [`5761926`](https://github.com/mastra-ai/mastra/commit/57619260c4a2cdd598763abbacd90de594c6bc76), [`c900fdd`](https://github.com/mastra-ai/mastra/commit/c900fdd504c41348efdffb205cfe80d48c38fa33), [`604a79f`](https://github.com/mastra-ai/mastra/commit/604a79fecf276e26a54a3fe01bb94e65315d2e0e), [`887f0b4`](https://github.com/mastra-ai/mastra/commit/887f0b4746cdbd7cb7d6b17ac9f82aeb58037ea5), [`2562143`](https://github.com/mastra-ai/mastra/commit/256214336b4faa78646c9c1776612393790d8784), [`ef11a61`](https://github.com/mastra-ai/mastra/commit/ef11a61920fa0ed08a5b7ceedd192875af119749)]:
+  - @mastra/core@1.0.0-beta.6
+
 ## 1.0.0-beta.1
 
 ### Patch Changes

package/dist/index.cjs

@@ -127,6 +127,8 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
   }
   async listMessages(args) {
     const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
+    const rawThreadIds = Array.isArray(threadId) ? threadId : [threadId];
+    const threadIds = rawThreadIds.filter((id) => id !== void 0 && id !== null).map((id) => (typeof id === "string" ? id : String(id)).trim()).filter((id) => id.length > 0);
     if (page < 0) {
       throw new error.MastraError(
         {
@@ -138,20 +140,21 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
         new Error("page must be >= 0")
       );
     }
-    if (!threadId.trim()) {
+    if (threadIds.length === 0) {
       throw new error.MastraError(
         {
           id: "STORAGE_CLICKHOUSE_LIST_MESSAGES_INVALID_THREAD_ID",
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
-          details: { threadId }
+          details: { threadId: Array.isArray(threadId) ? JSON.stringify(threadId) : String(threadId) }
         },
-        new Error("threadId must be a non-empty string")
+        new Error("threadId must be a non-empty string or array of non-empty strings")
       );
     }
     const perPageForQuery = storage.normalizePerPage(perPageInput, 40);
     const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPageForQuery);
     try {
+      const threadCondition = threadIds.length === 1 ? `thread_id = {threadId0:String}` : `thread_id IN (${threadIds.map((_, i) => `{threadId${i}:String}`).join(", ")})`;
       let dataQuery = `
         SELECT 
           id,
@@ -162,9 +165,12 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
           thread_id AS "threadId",
           resourceId
         FROM ${storage.TABLE_MESSAGES}
-        WHERE thread_id = {threadId:String}
+        WHERE ${threadCondition}
       `;
-      const dataParams = { threadId };
+      const dataParams = {};
+      threadIds.forEach((tid, i) => {
+        dataParams[`threadId${i}`] = tid;
+      });
       if (resourceId) {
         dataQuery += ` AND resourceId = {resourceId:String}`;
         dataParams.resourceId = resourceId;
@@ -199,8 +205,11 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
       const rows = await result.json();
       const paginatedMessages = transformRows(rows.data);
       const paginatedCount = paginatedMessages.length;
-      let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
-      const countParams = { threadId };
+      let countQuery = `SELECT count() as total FROM ${storage.TABLE_MESSAGES} WHERE ${threadCondition}`;
+      const countParams = {};
+      threadIds.forEach((tid, i) => {
+        countParams[`threadId${i}`] = tid;
+      });
       if (resourceId) {
         countQuery += ` AND resourceId = {resourceId:String}`;
         countParams.resourceId = resourceId;
@@ -239,12 +248,25 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
       const messageIds = new Set(paginatedMessages.map((m) => m.id));
       let includeMessages = [];
       if (include && include.length > 0) {
+        const includesNeedingThread = include.filter((inc) => !inc.threadId);
+        const threadByMessageId = /* @__PURE__ */ new Map();
+        if (includesNeedingThread.length > 0) {
+          const { messages: includeLookup } = await this.listMessagesById({
+            messageIds: includesNeedingThread.map((inc) => inc.id)
+          });
+          for (const msg of includeLookup) {
+            if (msg.threadId) {
+              threadByMessageId.set(msg.id, msg.threadId);
+            }
+          }
+        }
         const unionQueries = [];
         const params = [];
         let paramIdx = 1;
         for (const inc of include) {
           const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
-          const searchId = inc.threadId || threadId;
+          const searchThreadId = inc.threadId ?? threadByMessageId.get(id);
+          if (!searchThreadId) continue;
           unionQueries.push(`
             SELECT * FROM (
               WITH numbered_messages AS (
@@ -266,31 +288,33 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
             ) AS query_${paramIdx}
           `);
           params.push(
-            { [`var_thread_id_${paramIdx}`]: searchId },
+            { [`var_thread_id_${paramIdx}`]: searchThreadId },
             { [`var_include_id_${paramIdx}`]: id },
             { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
             { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
           );
           paramIdx++;
         }
-        const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" ASC';
-        const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
-        const includeResult = await this.client.query({
-          query: finalQuery,
-          query_params: mergedParams,
-          clickhouse_settings: {
-            date_time_input_format: "best_effort",
-            date_time_output_format: "iso",
-            use_client_time_zone: 1,
-            output_format_json_quote_64bit_integers: 0
-          }
-        });
-        const includeRows = await includeResult.json();
-        includeMessages = transformRows(includeRows.data);
-        for (const includeMsg of includeMessages) {
-          if (!messageIds.has(includeMsg.id)) {
-            paginatedMessages.push(includeMsg);
-            messageIds.add(includeMsg.id);
+        if (unionQueries.length > 0) {
+          const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" ASC';
+          const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
+          const includeResult = await this.client.query({
+            query: finalQuery,
+            query_params: mergedParams,
+            clickhouse_settings: {
+              date_time_input_format: "best_effort",
+              date_time_output_format: "iso",
+              use_client_time_zone: 1,
+              output_format_json_quote_64bit_integers: 0
+            }
+          });
+          const includeRows = await includeResult.json();
+          includeMessages = transformRows(includeRows.data);
+          for (const includeMsg of includeMessages) {
+            if (!messageIds.has(includeMsg.id)) {
+              paginatedMessages.push(includeMsg);
+              messageIds.add(includeMsg.id);
+            }
           }
         }
       }
@@ -308,7 +332,10 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
         }
         return direction === "ASC" ? String(aValue).localeCompare(String(bValue)) : String(bValue).localeCompare(String(aValue));
       });
-      const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
+      const threadIdSet = new Set(threadIds);
+      const returnedThreadMessageIds = new Set(
+        finalMessages.filter((m) => m.threadId && threadIdSet.has(m.threadId)).map((m) => m.id)
+      );
       const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
       const hasMore = perPageForResponse === false ? false : allThreadMessagesReturned ? false : offset + paginatedCount < total;
       return {
@@ -325,7 +352,7 @@ var MemoryStorageClickhouse = class extends storage.MemoryStorage {
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
           details: {
-            threadId,
+            threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
             resourceId: resourceId ?? ""
           }
         },
@@ -1451,30 +1478,15 @@ var ScoresStorageClickhouse = class extends storage.ScoresStorage {
     this.client = client;
     this.operations = operations;
   }
+  /**
+   * ClickHouse-specific score row transformation.
+   * Converts timestamps to Date objects and filters out '_null_' values.
+   */
   transformScoreRow(row) {
-    const scorer = storage.safelyParseJSON(row.scorer);
-    const preprocessStepResult = storage.safelyParseJSON(row.preprocessStepResult);
-    const analyzeStepResult = storage.safelyParseJSON(row.analyzeStepResult);
-    const metadata = storage.safelyParseJSON(row.metadata);
-    const input = storage.safelyParseJSON(row.input);
-    const output = storage.safelyParseJSON(row.output);
-    const additionalContext = storage.safelyParseJSON(row.additionalContext);
-    const requestContext = storage.safelyParseJSON(row.requestContext);
-    const entity = storage.safelyParseJSON(row.entity);
-    return {
-      ...row,
-      scorer,
-      preprocessStepResult,
-      analyzeStepResult,
-      metadata,
-      input,
-      output,
-      additionalContext,
-      requestContext,
-      entity,
-      createdAt: new Date(row.createdAt),
-      updatedAt: new Date(row.updatedAt)
-    };
+    return storage.transformScoreRow(row, {
+      convertTimestamps: true,
+      nullValuePattern: "_null_"
+    });
   }
   async getScoreById({ id }) {
     try {
@@ -1523,9 +1535,15 @@ var ScoresStorageClickhouse = class extends storage.ScoresStorage {
       );
     }
     try {
-      const record = {
-        ...parsedScore
-      };
+      const record = {};
+      for (const key of Object.keys(storage.SCORERS_SCHEMA)) {
+        const value = parsedScore[key];
+        if (key === "createdAt" || key === "updatedAt") {
+          record[key] = (/* @__PURE__ */ new Date()).toISOString();
+          continue;
+        }
+        record[key] = value === void 0 || value === null ? "_null_" : value;
+      }
       await this.client.insert({
         table: storage.TABLE_SCORERS,
         values: [record],