@mastra/upstash 1.0.0-beta.2 → 1.0.0-beta.3

package/dist/index.js

@@ -1,4 +1,4 @@
-import { MastraStorage, StoreOperations, ScoresStorage, TABLE_SCORERS, normalizePerPage, calculatePagination, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, MemoryStorage, TABLE_THREADS, TABLE_RESOURCES, TABLE_MESSAGES, serializeDate } from '@mastra/core/storage';
+import { MastraStorage, StoreOperations, ScoresStorage, TABLE_SCORERS, normalizePerPage, calculatePagination, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, MemoryStorage, TABLE_THREADS, TABLE_RESOURCES, TABLE_MESSAGES, serializeDate, transformScoreRow as transformScoreRow$1 } from '@mastra/core/storage';
 import { Redis } from '@upstash/redis';
 import { MessageList } from '@mastra/core/agent';
 import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
@@ -52,6 +52,9 @@ function getMessageKey(threadId, messageId) {
   const key = getKey(TABLE_MESSAGES, { threadId, id: messageId });
   return key;
 }
+function getMessageIndexKey(messageId) {
+  return `msg-idx:${messageId}`;
+}
 var StoreMemoryUpstash = class extends MemoryStorage {
   client;
   operations;
@@ -318,6 +321,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
             }
           }
           pipeline.set(key, message);
+          pipeline.set(getMessageIndexKey(message.id), message.threadId);
           pipeline.zadd(getThreadMessagesKey(message.threadId), {
             score,
             member: message.id
@@ -348,43 +352,60 @@ var StoreMemoryUpstash = class extends MemoryStorage {
       );
     }
   }
-  async _getIncludedMessages(threadId, include) {
-    if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
+  /**
+   * Lookup threadId for a message - tries index first (O(1)), falls back to scan (backwards compatible)
+   */
+  async _getThreadIdForMessage(messageId) {
+    const indexedThreadId = await this.client.get(getMessageIndexKey(messageId));
+    if (indexedThreadId) {
+      return indexedThreadId;
+    }
+    const existingKeyPattern = getMessageKey("*", messageId);
+    const keys = await this.operations.scanKeys(existingKeyPattern);
+    if (keys.length === 0) return null;
+    const messageData = await this.client.get(keys[0]);
+    if (!messageData) return null;
+    if (messageData.threadId) {
+      await this.client.set(getMessageIndexKey(messageId), messageData.threadId);
+    }
+    return messageData.threadId || null;
+  }
+  async _getIncludedMessages(include) {
+    if (!include?.length) return [];
     const messageIds = /* @__PURE__ */ new Set();
     const messageIdToThreadIds = {};
-    if (include?.length) {
-      for (const item of include) {
-        messageIds.add(item.id);
-        const itemThreadId = item.threadId || threadId;
-        messageIdToThreadIds[item.id] = itemThreadId;
-        const itemThreadMessagesKey = getThreadMessagesKey(itemThreadId);
-        const rank = await this.client.zrank(itemThreadMessagesKey, item.id);
-        if (rank === null) continue;
-        if (item.withPreviousMessages) {
-          const start = Math.max(0, rank - item.withPreviousMessages);
-          const prevIds = rank === 0 ? [] : await this.client.zrange(itemThreadMessagesKey, start, rank - 1);
-          prevIds.forEach((id) => {
-            messageIds.add(id);
-            messageIdToThreadIds[id] = itemThreadId;
-          });
-        }
-        if (item.withNextMessages) {
-          const nextIds = await this.client.zrange(itemThreadMessagesKey, rank + 1, rank + item.withNextMessages);
-          nextIds.forEach((id) => {
-            messageIds.add(id);
-            messageIdToThreadIds[id] = itemThreadId;
-          });
-        }
+    for (const item of include) {
+      const itemThreadId = await this._getThreadIdForMessage(item.id);
+      if (!itemThreadId) continue;
+      messageIds.add(item.id);
+      messageIdToThreadIds[item.id] = itemThreadId;
+      const itemThreadMessagesKey = getThreadMessagesKey(itemThreadId);
+      const rank = await this.client.zrank(itemThreadMessagesKey, item.id);
+      if (rank === null) continue;
+      if (item.withPreviousMessages) {
+        const start = Math.max(0, rank - item.withPreviousMessages);
+        const prevIds = rank === 0 ? [] : await this.client.zrange(itemThreadMessagesKey, start, rank - 1);
+        prevIds.forEach((id) => {
+          messageIds.add(id);
+          messageIdToThreadIds[id] = itemThreadId;
+        });
+      }
+      if (item.withNextMessages) {
+        const nextIds = await this.client.zrange(itemThreadMessagesKey, rank + 1, rank + item.withNextMessages);
+        nextIds.forEach((id) => {
+          messageIds.add(id);
+          messageIdToThreadIds[id] = itemThreadId;
+        });
       }
-      const pipeline = this.client.pipeline();
-      Array.from(messageIds).forEach((id) => {
-        const tId = messageIdToThreadIds[id] || threadId;
-        pipeline.get(getMessageKey(tId, id));
-      });
-      const results = await pipeline.exec();
-      return results.filter((result) => result !== null);
     }
-    return [];
+    if (messageIds.size === 0) return [];
+    const pipeline = this.client.pipeline();
+    Array.from(messageIds).forEach((id) => {
+      const tId = messageIdToThreadIds[id];
+      pipeline.get(getMessageKey(tId, id));
+    });
+    const results = await pipeline.exec();
+    return results.filter((result) => result !== null);
   }
   parseStoredMessage(storedMessage) {
     const defaultMessageContent = { format: 2, parts: [{ type: "text", text: "" }] };
@@ -398,17 +419,49 @@ var StoreMemoryUpstash = class extends MemoryStorage {
   async listMessagesById({ messageIds }) {
     if (messageIds.length === 0) return { messages: [] };
     try {
-      const threadKeys = await this.client.keys("thread:*");
-      const result = await Promise.all(
-        threadKeys.map((threadKey) => {
-          const threadId = threadKey.split(":")[1];
-          if (!threadId) throw new Error(`Failed to parse thread ID from thread key "${threadKey}"`);
-          return this.client.mget(
-            messageIds.map((id) => getMessageKey(threadId, id))
-          );
-        })
-      );
-      const rawMessages = result.flat(1).filter((msg) => !!msg);
+      const rawMessages = [];
+      const indexPipeline = this.client.pipeline();
+      messageIds.forEach((id) => indexPipeline.get(getMessageIndexKey(id)));
+      const indexResults = await indexPipeline.exec();
+      const indexedIds = [];
+      const unindexedIds = [];
+      messageIds.forEach((id, i) => {
+        const threadId = indexResults[i];
+        if (threadId) {
+          indexedIds.push({ messageId: id, threadId });
+        } else {
+          unindexedIds.push(id);
+        }
+      });
+      if (indexedIds.length > 0) {
+        const messagePipeline = this.client.pipeline();
+        indexedIds.forEach(({ messageId, threadId }) => messagePipeline.get(getMessageKey(threadId, messageId)));
+        const messageResults = await messagePipeline.exec();
+        rawMessages.push(...messageResults.filter((msg) => msg !== null));
+      }
+      if (unindexedIds.length > 0) {
+        const threadKeys = await this.client.keys("thread:*");
+        const result = await Promise.all(
+          threadKeys.map((threadKey) => {
+            const threadId = threadKey.split(":")[1];
+            if (!threadId) throw new Error(`Failed to parse thread ID from thread key "${threadKey}"`);
+            return this.client.mget(
+              unindexedIds.map((id) => getMessageKey(threadId, id))
+            );
+          })
+        );
+        const foundMessages = result.flat(1).filter((msg) => !!msg);
+        rawMessages.push(...foundMessages);
+        if (foundMessages.length > 0) {
+          const backfillPipeline = this.client.pipeline();
+          foundMessages.forEach((msg) => {
+            if (msg.threadId) {
+              backfillPipeline.set(getMessageIndexKey(msg.id), msg.threadId);
+            }
+          });
+          await backfillPipeline.exec();
+        }
+      }
       const list = new MessageList().add(rawMessages.map(this.parseStoredMessage), "memory");
       return { messages: list.get.all.db() };
     } catch (error) {
@@ -427,18 +480,18 @@ var StoreMemoryUpstash = class extends MemoryStorage {
   }
   async listMessages(args) {
     const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
-    if (!threadId.trim()) {
+    const threadIds = Array.isArray(threadId) ? threadId : [threadId];
+    if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
       throw new MastraError(
         {
           id: "STORAGE_UPSTASH_LIST_MESSAGES_INVALID_THREAD_ID",
           domain: ErrorDomain.STORAGE,
           category: ErrorCategory.THIRD_PARTY,
-          details: { threadId }
+          details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
         },
-        new Error("threadId must be a non-empty string")
+        new Error("threadId must be a non-empty string or array of non-empty strings")
       );
     }
-    const threadMessagesKey = getThreadMessagesKey(threadId);
     const perPage = normalizePerPage(perPageInput, 40);
     const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
     try {
@@ -455,11 +508,18 @@ var StoreMemoryUpstash = class extends MemoryStorage {
       }
       let includedMessages = [];
       if (include && include.length > 0) {
-        const included = await this._getIncludedMessages(threadId, include);
+        const included = await this._getIncludedMessages(include);
         includedMessages = included.map(this.parseStoredMessage);
       }
-      const allMessageIds = await this.client.zrange(threadMessagesKey, 0, -1);
-      if (allMessageIds.length === 0) {
+      const allMessageIdsWithThreads = [];
+      for (const tid of threadIds) {
+        const threadMessagesKey = getThreadMessagesKey(tid);
+        const messageIds2 = await this.client.zrange(threadMessagesKey, 0, -1);
+        for (const mid of messageIds2) {
+          allMessageIdsWithThreads.push({ threadId: tid, messageId: mid });
+        }
+      }
+      if (allMessageIdsWithThreads.length === 0) {
         return {
           messages: [],
           total: 0,
@@ -469,7 +529,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
         };
       }
       const pipeline = this.client.pipeline();
-      allMessageIds.forEach((id) => pipeline.get(getMessageKey(threadId, id)));
+      allMessageIdsWithThreads.forEach(({ threadId: tid, messageId }) => pipeline.get(getMessageKey(tid, messageId)));
       const results = await pipeline.exec();
       let messagesData = results.filter((msg) => msg !== null).map(this.parseStoredMessage);
       if (resourceId) {
@@ -545,7 +605,7 @@ var StoreMemoryUpstash = class extends MemoryStorage {
           domain: ErrorDomain.STORAGE,
           category: ErrorCategory.THIRD_PARTY,
           details: {
-            threadId,
+            threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
             resourceId: resourceId ?? ""
           }
         },
@@ -750,13 +810,33 @@ var StoreMemoryUpstash = class extends MemoryStorage {
     try {
       const threadIds = /* @__PURE__ */ new Set();
       const messageKeys = [];
-      for (const messageId of messageIds) {
+      const foundMessageIds = [];
+      const indexPipeline = this.client.pipeline();
+      messageIds.forEach((id) => indexPipeline.get(getMessageIndexKey(id)));
+      const indexResults = await indexPipeline.exec();
+      const indexedMessages = [];
+      const unindexedMessageIds = [];
+      messageIds.forEach((id, i) => {
+        const threadId = indexResults[i];
+        if (threadId) {
+          indexedMessages.push({ messageId: id, threadId });
+        } else {
+          unindexedMessageIds.push(id);
+        }
+      });
+      for (const { messageId, threadId } of indexedMessages) {
+        messageKeys.push(getMessageKey(threadId, messageId));
+        foundMessageIds.push(messageId);
+        threadIds.add(threadId);
+      }
+      for (const messageId of unindexedMessageIds) {
         const pattern = getMessageKey("*", messageId);
         const keys = await this.operations.scanKeys(pattern);
         for (const key of keys) {
           const message = await this.client.get(key);
           if (message && message.id === messageId) {
             messageKeys.push(key);
+            foundMessageIds.push(messageId);
             if (message.threadId) {
               threadIds.add(message.threadId);
             }
@@ -771,6 +851,9 @@ var StoreMemoryUpstash = class extends MemoryStorage {
       for (const key of messageKeys) {
         pipeline.del(key);
       }
+      for (const messageId of foundMessageIds) {
+        pipeline.del(getMessageIndexKey(messageId));
+      }
       if (threadIds.size > 0) {
         for (const threadId of threadIds) {
           const threadKey = getKey(TABLE_THREADS, { id: threadId });
@@ -946,32 +1029,7 @@ var StoreOperationsUpstash = class extends StoreOperations {
   }
 };
 function transformScoreRow(row) {
-  const parseField = (v) => {
-    if (typeof v === "string") {
-      try {
-        return JSON.parse(v);
-      } catch {
-        return v;
-      }
-    }
-    return v;
-  };
-  return {
-    ...row,
-    scorer: parseField(row.scorer),
-    preprocessStepResult: parseField(row.preprocessStepResult),
-    generateScorePrompt: row.generateScorePrompt,
-    generateReasonPrompt: row.generateReasonPrompt,
-    analyzeStepResult: parseField(row.analyzeStepResult),
-    metadata: parseField(row.metadata),
-    input: parseField(row.input),
-    output: parseField(row.output),
-    additionalContext: parseField(row.additionalContext),
-    requestContext: parseField(row.requestContext),
-    entity: parseField(row.entity),
-    createdAt: row.createdAt,
-    updatedAt: row.updatedAt
-  };
+  return transformScoreRow$1(row);
 }
 var ScoresUpstash = class extends ScoresStorage {
   client;