@mastra/upstash 1.0.0-beta.2 → 1.0.0-beta.3

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @mastra/upstash
 
+## 1.0.0-beta.3
+
+### Minor Changes
+
+- feat(storage): support querying messages from multiple threads ([#10663](https://github.com/mastra-ai/mastra/pull/10663))
+  - Fixed TypeScript errors where `threadId: string | string[]` was being passed to places expecting `Scalar` type
+  - Added proper multi-thread support for `listMessages` across all adapters when `threadId` is an array
+  - Updated `_getIncludedMessages` to look up message threadId by ID (since message IDs are globally unique)
+  - **upstash**: Added `msg-idx:{messageId}` index for O(1) message lookups (backwards compatible with fallback to scan for old messages, with automatic backfill)
+
+### Patch Changes
+
+- Unify transformScoreRow functions across storage adapters ([#10648](https://github.com/mastra-ai/mastra/pull/10648))
+
+  Added a unified `transformScoreRow` function in `@mastra/core/storage` that provides schema-driven row transformation for score data. This eliminates code duplication across 10 storage adapters while maintaining store-specific behavior through configurable options:
+  - `preferredTimestampFields`: Preferred source fields for timestamps (PostgreSQL, Cloudflare D1)
+  - `convertTimestamps`: Convert timestamp strings to Date objects (MSSQL, MongoDB, ClickHouse)
+  - `nullValuePattern`: Skip values matching pattern (ClickHouse's `'_null_'`)
+  - `fieldMappings`: Map source column names to schema fields (LibSQL's `additionalLLMContext`)
+
+  Each store adapter now uses the unified function with appropriate options, reducing ~200 lines of duplicate transformation logic while ensuring consistent behavior across all storage backends.
+
+- Updated dependencies [[`ac0d2f4`](https://github.com/mastra-ai/mastra/commit/ac0d2f4ff8831f72c1c66c2be809706d17f65789), [`1a0d3fc`](https://github.com/mastra-ai/mastra/commit/1a0d3fc811482c9c376cdf79ee615c23bae9b2d6), [`85a628b`](https://github.com/mastra-ai/mastra/commit/85a628b1224a8f64cd82ea7f033774bf22df7a7e), [`c237233`](https://github.com/mastra-ai/mastra/commit/c23723399ccedf7f5744b3f40997b79246bfbe64), [`15f9e21`](https://github.com/mastra-ai/mastra/commit/15f9e216177201ea6e3f6d0bfb063fcc0953444f), [`ff94dea`](https://github.com/mastra-ai/mastra/commit/ff94dea935f4e34545c63bcb6c29804732698809), [`5b2ff46`](https://github.com/mastra-ai/mastra/commit/5b2ff4651df70c146523a7fca773f8eb0a2272f8), [`db41688`](https://github.com/mastra-ai/mastra/commit/db4168806d007417e2e60b4f68656dca4e5f40c9), [`5ca599d`](https://github.com/mastra-ai/mastra/commit/5ca599d0bb59a1595f19f58473fcd67cc71cef58), [`bff1145`](https://github.com/mastra-ai/mastra/commit/bff114556b3cbadad9b2768488708f8ad0e91475), [`5c8ca24`](https://github.com/mastra-ai/mastra/commit/5c8ca247094e0cc2cdbd7137822fb47241f86e77), [`e191844`](https://github.com/mastra-ai/mastra/commit/e1918444ca3f80e82feef1dad506cd4ec6e2875f), [`22553f1`](https://github.com/mastra-ai/mastra/commit/22553f11c63ee5e966a9c034a349822249584691), [`7237163`](https://github.com/mastra-ai/mastra/commit/72371635dbf96a87df4b073cc48fc655afbdce3d), [`2500740`](https://github.com/mastra-ai/mastra/commit/2500740ea23da067d6e50ec71c625ab3ce275e64), [`873ecbb`](https://github.com/mastra-ai/mastra/commit/873ecbb517586aa17d2f1e99283755b3ebb2863f), [`4f9bbe5`](https://github.com/mastra-ai/mastra/commit/4f9bbe5968f42c86f4930b8193de3c3c17e5bd36), [`02e51fe`](https://github.com/mastra-ai/mastra/commit/02e51feddb3d4155cfbcc42624fd0d0970d032c0), [`8f3fa3a`](https://github.com/mastra-ai/mastra/commit/8f3fa3a652bb77da092f913ec51ae46e3a7e27dc), [`cd29ad2`](https://github.com/mastra-ai/mastra/commit/cd29ad23a255534e8191f249593849ed29160886), [`bdf4d8c`](https://github.com/mastra-ai/mastra/commit/bdf4d8cdc656d8a2c21d81834bfa3bfa70f56c16), [`854e3da`](https://github.com/mastra-ai/mastra/commit/854e3dad5daac17a91a20986399d3a51f54bf68b), [`ce18d38`](https://github.com/mastra-ai/mastra/commit/ce18d38678c65870350d123955014a8432075fd9), [`cccf9c8`](https://github.com/mastra-ai/mastra/commit/cccf9c8b2d2dfc1a5e63919395b83d78c89682a0), [`61a5705`](https://github.com/mastra-ai/mastra/commit/61a570551278b6743e64243b3ce7d73de915ca8a), [`db70a48`](https://github.com/mastra-ai/mastra/commit/db70a48aeeeeb8e5f92007e8ede52c364ce15287), [`f0fdc14`](https://github.com/mastra-ai/mastra/commit/f0fdc14ee233d619266b3d2bbdeea7d25cfc6d13), [`db18bc9`](https://github.com/mastra-ai/mastra/commit/db18bc9c3825e2c1a0ad9a183cc9935f6691bfa1), [`9b37b56`](https://github.com/mastra-ai/mastra/commit/9b37b565e1f2a76c24f728945cc740c2b09be9da), [`41a23c3`](https://github.com/mastra-ai/mastra/commit/41a23c32f9877d71810f37e24930515df2ff7a0f), [`5d171ad`](https://github.com/mastra-ai/mastra/commit/5d171ad9ef340387276b77c2bb3e83e83332d729), [`f03ae60`](https://github.com/mastra-ai/mastra/commit/f03ae60500fe350c9d828621006cdafe1975fdd8), [`d1e74a0`](https://github.com/mastra-ai/mastra/commit/d1e74a0a293866dece31022047f5dbab65a304d0), [`39e7869`](https://github.com/mastra-ai/mastra/commit/39e7869bc7d0ee391077ce291474d8a84eedccff), [`5761926`](https://github.com/mastra-ai/mastra/commit/57619260c4a2cdd598763abbacd90de594c6bc76), [`c900fdd`](https://github.com/mastra-ai/mastra/commit/c900fdd504c41348efdffb205cfe80d48c38fa33), [`604a79f`](https://github.com/mastra-ai/mastra/commit/604a79fecf276e26a54a3fe01bb94e65315d2e0e), [`887f0b4`](https://github.com/mastra-ai/mastra/commit/887f0b4746cdbd7cb7d6b17ac9f82aeb58037ea5), [`2562143`](https://github.com/mastra-ai/mastra/commit/256214336b4faa78646c9c1776612393790d8784), [`ef11a61`](https://github.com/mastra-ai/mastra/commit/ef11a61920fa0ed08a5b7ceedd192875af119749)]:
+  - @mastra/core@1.0.0-beta.6
+
 ## 1.0.0-beta.2
 
 ### Patch Changes

package/dist/index.cjs

@@ -54,6 +54,9 @@ function getMessageKey(threadId, messageId) {
   const key = getKey(storage.TABLE_MESSAGES, { threadId, id: messageId });
   return key;
 }
+function getMessageIndexKey(messageId) {
+  return `msg-idx:${messageId}`;
+}
 var StoreMemoryUpstash = class extends storage.MemoryStorage {
   client;
   operations;
@@ -320,6 +323,7 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
             }
           }
           pipeline.set(key, message);
+          pipeline.set(getMessageIndexKey(message.id), message.threadId);
           pipeline.zadd(getThreadMessagesKey(message.threadId), {
             score,
             member: message.id
@@ -350,43 +354,60 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
       );
     }
   }
-  async _getIncludedMessages(threadId, include) {
-    if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
+  /**
+   * Lookup threadId for a message - tries index first (O(1)), falls back to scan (backwards compatible)
+   */
+  async _getThreadIdForMessage(messageId) {
+    const indexedThreadId = await this.client.get(getMessageIndexKey(messageId));
+    if (indexedThreadId) {
+      return indexedThreadId;
+    }
+    const existingKeyPattern = getMessageKey("*", messageId);
+    const keys = await this.operations.scanKeys(existingKeyPattern);
+    if (keys.length === 0) return null;
+    const messageData = await this.client.get(keys[0]);
+    if (!messageData) return null;
+    if (messageData.threadId) {
+      await this.client.set(getMessageIndexKey(messageId), messageData.threadId);
+    }
+    return messageData.threadId || null;
+  }
+  async _getIncludedMessages(include) {
+    if (!include?.length) return [];
     const messageIds = /* @__PURE__ */ new Set();
     const messageIdToThreadIds = {};
-    if (include?.length) {
-      for (const item of include) {
-        messageIds.add(item.id);
-        const itemThreadId = item.threadId || threadId;
-        messageIdToThreadIds[item.id] = itemThreadId;
-        const itemThreadMessagesKey = getThreadMessagesKey(itemThreadId);
-        const rank = await this.client.zrank(itemThreadMessagesKey, item.id);
-        if (rank === null) continue;
-        if (item.withPreviousMessages) {
-          const start = Math.max(0, rank - item.withPreviousMessages);
-          const prevIds = rank === 0 ? [] : await this.client.zrange(itemThreadMessagesKey, start, rank - 1);
-          prevIds.forEach((id) => {
-            messageIds.add(id);
-            messageIdToThreadIds[id] = itemThreadId;
-          });
-        }
-        if (item.withNextMessages) {
-          const nextIds = await this.client.zrange(itemThreadMessagesKey, rank + 1, rank + item.withNextMessages);
-          nextIds.forEach((id) => {
-            messageIds.add(id);
-            messageIdToThreadIds[id] = itemThreadId;
-          });
-        }
+    for (const item of include) {
+      const itemThreadId = await this._getThreadIdForMessage(item.id);
+      if (!itemThreadId) continue;
+      messageIds.add(item.id);
+      messageIdToThreadIds[item.id] = itemThreadId;
+      const itemThreadMessagesKey = getThreadMessagesKey(itemThreadId);
+      const rank = await this.client.zrank(itemThreadMessagesKey, item.id);
+      if (rank === null) continue;
+      if (item.withPreviousMessages) {
+        const start = Math.max(0, rank - item.withPreviousMessages);
+        const prevIds = rank === 0 ? [] : await this.client.zrange(itemThreadMessagesKey, start, rank - 1);
+        prevIds.forEach((id) => {
+          messageIds.add(id);
+          messageIdToThreadIds[id] = itemThreadId;
+        });
+      }
+      if (item.withNextMessages) {
+        const nextIds = await this.client.zrange(itemThreadMessagesKey, rank + 1, rank + item.withNextMessages);
+        nextIds.forEach((id) => {
+          messageIds.add(id);
+          messageIdToThreadIds[id] = itemThreadId;
+        });
       }
-      const pipeline = this.client.pipeline();
-      Array.from(messageIds).forEach((id) => {
-        const tId = messageIdToThreadIds[id] || threadId;
-        pipeline.get(getMessageKey(tId, id));
-      });
-      const results = await pipeline.exec();
-      return results.filter((result) => result !== null);
     }
-    return [];
+    if (messageIds.size === 0) return [];
+    const pipeline = this.client.pipeline();
+    Array.from(messageIds).forEach((id) => {
+      const tId = messageIdToThreadIds[id];
+      pipeline.get(getMessageKey(tId, id));
+    });
+    const results = await pipeline.exec();
+    return results.filter((result) => result !== null);
   }
   parseStoredMessage(storedMessage) {
     const defaultMessageContent = { format: 2, parts: [{ type: "text", text: "" }] };
@@ -400,17 +421,49 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
   async listMessagesById({ messageIds }) {
     if (messageIds.length === 0) return { messages: [] };
     try {
-      const threadKeys = await this.client.keys("thread:*");
-      const result = await Promise.all(
-        threadKeys.map((threadKey) => {
-          const threadId = threadKey.split(":")[1];
-          if (!threadId) throw new Error(`Failed to parse thread ID from thread key "${threadKey}"`);
-          return this.client.mget(
-            messageIds.map((id) => getMessageKey(threadId, id))
-          );
-        })
-      );
-      const rawMessages = result.flat(1).filter((msg) => !!msg);
+      const rawMessages = [];
+      const indexPipeline = this.client.pipeline();
+      messageIds.forEach((id) => indexPipeline.get(getMessageIndexKey(id)));
+      const indexResults = await indexPipeline.exec();
+      const indexedIds = [];
+      const unindexedIds = [];
+      messageIds.forEach((id, i) => {
+        const threadId = indexResults[i];
+        if (threadId) {
+          indexedIds.push({ messageId: id, threadId });
+        } else {
+          unindexedIds.push(id);
+        }
+      });
+      if (indexedIds.length > 0) {
+        const messagePipeline = this.client.pipeline();
+        indexedIds.forEach(({ messageId, threadId }) => messagePipeline.get(getMessageKey(threadId, messageId)));
+        const messageResults = await messagePipeline.exec();
+        rawMessages.push(...messageResults.filter((msg) => msg !== null));
+      }
+      if (unindexedIds.length > 0) {
+        const threadKeys = await this.client.keys("thread:*");
+        const result = await Promise.all(
+          threadKeys.map((threadKey) => {
+            const threadId = threadKey.split(":")[1];
+            if (!threadId) throw new Error(`Failed to parse thread ID from thread key "${threadKey}"`);
+            return this.client.mget(
+              unindexedIds.map((id) => getMessageKey(threadId, id))
+            );
+          })
+        );
+        const foundMessages = result.flat(1).filter((msg) => !!msg);
+        rawMessages.push(...foundMessages);
+        if (foundMessages.length > 0) {
+          const backfillPipeline = this.client.pipeline();
+          foundMessages.forEach((msg) => {
+            if (msg.threadId) {
+              backfillPipeline.set(getMessageIndexKey(msg.id), msg.threadId);
+            }
+          });
+          await backfillPipeline.exec();
+        }
+      }
       const list = new agent.MessageList().add(rawMessages.map(this.parseStoredMessage), "memory");
       return { messages: list.get.all.db() };
     } catch (error$1) {
@@ -429,18 +482,18 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
   }
   async listMessages(args) {
     const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
-    if (!threadId.trim()) {
+    const threadIds = Array.isArray(threadId) ? threadId : [threadId];
+    if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
       throw new error.MastraError(
         {
           id: "STORAGE_UPSTASH_LIST_MESSAGES_INVALID_THREAD_ID",
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
-          details: { threadId }
+          details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
         },
-        new Error("threadId must be a non-empty string")
+        new Error("threadId must be a non-empty string or array of non-empty strings")
       );
     }
-    const threadMessagesKey = getThreadMessagesKey(threadId);
     const perPage = storage.normalizePerPage(perPageInput, 40);
     const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
     try {
@@ -457,11 +510,18 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
       }
       let includedMessages = [];
       if (include && include.length > 0) {
-        const included = await this._getIncludedMessages(threadId, include);
+        const included = await this._getIncludedMessages(include);
         includedMessages = included.map(this.parseStoredMessage);
       }
-      const allMessageIds = await this.client.zrange(threadMessagesKey, 0, -1);
-      if (allMessageIds.length === 0) {
+      const allMessageIdsWithThreads = [];
+      for (const tid of threadIds) {
+        const threadMessagesKey = getThreadMessagesKey(tid);
+        const messageIds2 = await this.client.zrange(threadMessagesKey, 0, -1);
+        for (const mid of messageIds2) {
+          allMessageIdsWithThreads.push({ threadId: tid, messageId: mid });
+        }
+      }
+      if (allMessageIdsWithThreads.length === 0) {
         return {
           messages: [],
           total: 0,
@@ -471,7 +531,7 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
         };
       }
       const pipeline = this.client.pipeline();
-      allMessageIds.forEach((id) => pipeline.get(getMessageKey(threadId, id)));
+      allMessageIdsWithThreads.forEach(({ threadId: tid, messageId }) => pipeline.get(getMessageKey(tid, messageId)));
       const results = await pipeline.exec();
       let messagesData = results.filter((msg) => msg !== null).map(this.parseStoredMessage);
       if (resourceId) {
@@ -547,7 +607,7 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
           domain: error.ErrorDomain.STORAGE,
           category: error.ErrorCategory.THIRD_PARTY,
           details: {
-            threadId,
+            threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
             resourceId: resourceId ?? ""
           }
         },
@@ -752,13 +812,33 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
     try {
       const threadIds = /* @__PURE__ */ new Set();
       const messageKeys = [];
-      for (const messageId of messageIds) {
+      const foundMessageIds = [];
+      const indexPipeline = this.client.pipeline();
+      messageIds.forEach((id) => indexPipeline.get(getMessageIndexKey(id)));
+      const indexResults = await indexPipeline.exec();
+      const indexedMessages = [];
+      const unindexedMessageIds = [];
+      messageIds.forEach((id, i) => {
+        const threadId = indexResults[i];
+        if (threadId) {
+          indexedMessages.push({ messageId: id, threadId });
+        } else {
+          unindexedMessageIds.push(id);
+        }
+      });
+      for (const { messageId, threadId } of indexedMessages) {
+        messageKeys.push(getMessageKey(threadId, messageId));
+        foundMessageIds.push(messageId);
+        threadIds.add(threadId);
+      }
+      for (const messageId of unindexedMessageIds) {
         const pattern = getMessageKey("*", messageId);
         const keys = await this.operations.scanKeys(pattern);
         for (const key of keys) {
           const message = await this.client.get(key);
           if (message && message.id === messageId) {
             messageKeys.push(key);
+            foundMessageIds.push(messageId);
             if (message.threadId) {
               threadIds.add(message.threadId);
             }
@@ -773,6 +853,9 @@ var StoreMemoryUpstash = class extends storage.MemoryStorage {
       for (const key of messageKeys) {
         pipeline.del(key);
       }
+      for (const messageId of foundMessageIds) {
+        pipeline.del(getMessageIndexKey(messageId));
+      }
       if (threadIds.size > 0) {
         for (const threadId of threadIds) {
           const threadKey = getKey(storage.TABLE_THREADS, { id: threadId });
@@ -948,32 +1031,7 @@ var StoreOperationsUpstash = class extends storage.StoreOperations {
   }
 };
 function transformScoreRow(row) {
-  const parseField = (v) => {
-    if (typeof v === "string") {
-      try {
-        return JSON.parse(v);
-      } catch {
-        return v;
-      }
-    }
-    return v;
-  };
-  return {
-    ...row,
-    scorer: parseField(row.scorer),
-    preprocessStepResult: parseField(row.preprocessStepResult),
-    generateScorePrompt: row.generateScorePrompt,
-    generateReasonPrompt: row.generateReasonPrompt,
-    analyzeStepResult: parseField(row.analyzeStepResult),
-    metadata: parseField(row.metadata),
-    input: parseField(row.input),
-    output: parseField(row.output),
-    additionalContext: parseField(row.additionalContext),
-    requestContext: parseField(row.requestContext),
-    entity: parseField(row.entity),
-    createdAt: row.createdAt,
-    updatedAt: row.updatedAt
-  };
+  return storage.transformScoreRow(row);
 }
 var ScoresUpstash = class extends storage.ScoresStorage {
   client;