@mastra/upstash 0.10.0 → 0.10.1

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/upstash@0.10.0-alpha.1 build /home/runner/work/mastra/mastra/stores/upstash
+> @mastra/upstash@0.10.1-alpha.0 build /home/runner/work/mastra/mastra/stores/upstash
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 8981ms
+TSC ⚡️ Build success in 8661ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/stores/upstash/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.3
 Writing package typings: /home/runner/work/mastra/mastra/stores/upstash/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 10676ms
+DTS ⚡️ Build success in 12455ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js 30.09 KB
-ESM ⚡️ Build success in 1208ms
-CJS dist/index.cjs 30.23 KB
-CJS ⚡️ Build success in 1217ms
+ESM dist/index.js 31.90 KB
+ESM ⚡️ Build success in 970ms
+CJS dist/index.cjs 32.04 KB
+CJS ⚡️ Build success in 980ms

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # @mastra/upstash
 
+## 0.10.1
+
+### Patch Changes
+
+- e60402a: Use scan instead of keys for upstash redis and implement batchInsert
+- Updated dependencies [d70b807]
+- Updated dependencies [6d16390]
+- Updated dependencies [1e4a421]
+- Updated dependencies [200d0da]
+- Updated dependencies [bf5f17b]
+- Updated dependencies [5343f93]
+- Updated dependencies [38aee50]
+- Updated dependencies [5c41100]
+- Updated dependencies [d6a759b]
+- Updated dependencies [6015bdf]
+  - @mastra/core@0.10.1
+
+## 0.10.1-alpha.0
+
+### Patch Changes
+
+- e60402a: Use scan instead of keys for upstash redis and implement batchInsert
+- Updated dependencies [6d16390]
+- Updated dependencies [1e4a421]
+  - @mastra/core@0.10.1-alpha.0
+
 ## 0.10.0
 
 ### Minor Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -52,12 +52,30 @@ export declare class UpstashFilterTranslator extends BaseFilterTranslator {
 }
 
 declare class UpstashStore extends MastraStorage {
-    batchInsert(_input: {
-        tableName: TABLE_NAMES;
-        records: Record<string, any>[];
-    }): Promise<void>;
-    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
+    private redis;
+    constructor(config: UpstashConfig);
     private transformEvalRecord;
+    private parseJSON;
+    private getKey;
+    private ensureDate;
+    private serializeDate;
+    /**
+     * Scans for keys matching the given pattern using SCAN and returns them as an array.
+     * @param pattern Redis key pattern, e.g. "table:*"
+     * @param batchSize Number of keys to scan per batch (default: 1000)
+     */
+    private scanKeys;
+    /**
+     * Deletes all keys matching the given pattern using SCAN and DEL in batches.
+     * @param pattern Redis key pattern, e.g. "table:*"
+     * @param batchSize Number of keys to delete per batch (default: 1000)
+     */
+    private scanAndDelete;
+    private getMessageKey;
+    private getThreadMessagesKey;
+    private parseWorkflowRun;
+    private processRecord;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
     getTraces({ name, scope, page, perPage, attributes, filters, fromDate, toDate, }?: {
         name?: string;
         scope?: string;
@@ -68,12 +86,6 @@ declare class UpstashStore extends MastraStorage {
         fromDate?: Date;
         toDate?: Date;
     }): Promise<any[]>;
-    private parseJSON;
-    private redis;
-    constructor(config: UpstashConfig);
-    private getKey;
-    private ensureDate;
-    private serializeDate;
     createTable({ tableName, schema, }: {
         tableName: TABLE_NAMES;
         schema: Record<string, StorageColumn>;
@@ -85,6 +97,10 @@ declare class UpstashStore extends MastraStorage {
         tableName: TABLE_NAMES;
         record: Record<string, any>;
     }): Promise<void>;
+    batchInsert(input: {
+        tableName: TABLE_NAMES;
+        records: Record<string, any>[];
+    }): Promise<void>;
     load<R>({ tableName, keys }: {
         tableName: TABLE_NAMES;
         keys: Record<string, string>;
@@ -106,8 +122,6 @@ declare class UpstashStore extends MastraStorage {
     deleteThread({ threadId }: {
         threadId: string;
     }): Promise<void>;
-    private getMessageKey;
-    private getThreadMessagesKey;
     saveMessages({ messages }: {
         messages: MessageType[];
     }): Promise<MessageType[]>;
@@ -123,7 +137,6 @@ declare class UpstashStore extends MastraStorage {
         workflowName: string;
         runId: string;
     }): Promise<WorkflowRunState | null>;
-    private parseWorkflowRun;
     getWorkflowRuns({ namespace, workflowName, fromDate, toDate, limit, offset, resourceId, }?: {
         namespace: string;
         workflowName?: string;

package/dist/_tsup-dts-rollup.d.ts

@@ -52,12 +52,30 @@ export declare class UpstashFilterTranslator extends BaseFilterTranslator {
 }
 
 declare class UpstashStore extends MastraStorage {
-    batchInsert(_input: {
-        tableName: TABLE_NAMES;
-        records: Record<string, any>[];
-    }): Promise<void>;
-    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
+    private redis;
+    constructor(config: UpstashConfig);
     private transformEvalRecord;
+    private parseJSON;
+    private getKey;
+    private ensureDate;
+    private serializeDate;
+    /**
+     * Scans for keys matching the given pattern using SCAN and returns them as an array.
+     * @param pattern Redis key pattern, e.g. "table:*"
+     * @param batchSize Number of keys to scan per batch (default: 1000)
+     */
+    private scanKeys;
+    /**
+     * Deletes all keys matching the given pattern using SCAN and DEL in batches.
+     * @param pattern Redis key pattern, e.g. "table:*"
+     * @param batchSize Number of keys to delete per batch (default: 1000)
+     */
+    private scanAndDelete;
+    private getMessageKey;
+    private getThreadMessagesKey;
+    private parseWorkflowRun;
+    private processRecord;
+    getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
     getTraces({ name, scope, page, perPage, attributes, filters, fromDate, toDate, }?: {
         name?: string;
         scope?: string;
@@ -68,12 +86,6 @@ declare class UpstashStore extends MastraStorage {
         fromDate?: Date;
         toDate?: Date;
     }): Promise<any[]>;
-    private parseJSON;
-    private redis;
-    constructor(config: UpstashConfig);
-    private getKey;
-    private ensureDate;
-    private serializeDate;
     createTable({ tableName, schema, }: {
         tableName: TABLE_NAMES;
         schema: Record<string, StorageColumn>;
@@ -85,6 +97,10 @@ declare class UpstashStore extends MastraStorage {
         tableName: TABLE_NAMES;
         record: Record<string, any>;
     }): Promise<void>;
+    batchInsert(input: {
+        tableName: TABLE_NAMES;
+        records: Record<string, any>[];
+    }): Promise<void>;
     load<R>({ tableName, keys }: {
         tableName: TABLE_NAMES;
         keys: Record<string, string>;
@@ -106,8 +122,6 @@ declare class UpstashStore extends MastraStorage {
     deleteThread({ threadId }: {
         threadId: string;
     }): Promise<void>;
-    private getMessageKey;
-    private getThreadMessagesKey;
     saveMessages({ messages }: {
         messages: MessageType[];
     }): Promise<MessageType[]>;
@@ -123,7 +137,6 @@ declare class UpstashStore extends MastraStorage {
         workflowName: string;
         runId: string;
     }): Promise<WorkflowRunState | null>;
-    private parseWorkflowRun;
     getWorkflowRuns({ namespace, workflowName, fromDate, toDate, limit, offset, resourceId, }?: {
         namespace: string;
         workflowName?: string;

package/dist/index.cjs

@@ -8,13 +8,157 @@ var filter = require('@mastra/core/vector/filter');
 
 // src/storage/index.ts
 var UpstashStore = class extends storage.MastraStorage {
-  batchInsert(_input) {
-    throw new Error("Method not implemented.");
+  redis;
+  constructor(config) {
+    super({ name: "Upstash" });
+    this.redis = new redis.Redis({
+      url: config.url,
+      token: config.token
+    });
+  }
+  transformEvalRecord(record) {
+    let result = record.result;
+    if (typeof result === "string") {
+      try {
+        result = JSON.parse(result);
+      } catch {
+        console.warn("Failed to parse result JSON:");
+      }
+    }
+    let testInfo = record.test_info;
+    if (typeof testInfo === "string") {
+      try {
+        testInfo = JSON.parse(testInfo);
+      } catch {
+        console.warn("Failed to parse test_info JSON:");
+      }
+    }
+    return {
+      agentName: record.agent_name,
+      input: record.input,
+      output: record.output,
+      result,
+      metricName: record.metric_name,
+      instructions: record.instructions,
+      testInfo,
+      globalRunId: record.global_run_id,
+      runId: record.run_id,
+      createdAt: typeof record.created_at === "string" ? record.created_at : record.created_at instanceof Date ? record.created_at.toISOString() : (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  parseJSON(value) {
+    if (typeof value === "string") {
+      try {
+        return JSON.parse(value);
+      } catch {
+        return value;
+      }
+    }
+    return value;
+  }
+  getKey(tableName, keys) {
+    const keyParts = Object.entries(keys).filter(([_, value]) => value !== void 0).map(([key, value]) => `${key}:${value}`);
+    return `${tableName}:${keyParts.join(":")}`;
+  }
+  ensureDate(date) {
+    if (!date) return void 0;
+    return date instanceof Date ? date : new Date(date);
+  }
+  serializeDate(date) {
+    if (!date) return void 0;
+    const dateObj = this.ensureDate(date);
+    return dateObj?.toISOString();
+  }
+  /**
+   * Scans for keys matching the given pattern using SCAN and returns them as an array.
+   * @param pattern Redis key pattern, e.g. "table:*"
+   * @param batchSize Number of keys to scan per batch (default: 1000)
+   */
+  async scanKeys(pattern, batchSize = 1e4) {
+    let cursor = "0";
+    let keys = [];
+    do {
+      const [nextCursor, batch] = await this.redis.scan(cursor, {
+        match: pattern,
+        count: batchSize
+      });
+      keys.push(...batch);
+      cursor = nextCursor;
+    } while (cursor !== "0");
+    return keys;
+  }
+  /**
+   * Deletes all keys matching the given pattern using SCAN and DEL in batches.
+   * @param pattern Redis key pattern, e.g. "table:*"
+   * @param batchSize Number of keys to delete per batch (default: 1000)
+   */
+  async scanAndDelete(pattern, batchSize = 1e4) {
+    let cursor = "0";
+    let totalDeleted = 0;
+    do {
+      const [nextCursor, keys] = await this.redis.scan(cursor, {
+        match: pattern,
+        count: batchSize
+      });
+      if (keys.length > 0) {
+        await this.redis.del(...keys);
+        totalDeleted += keys.length;
+      }
+      cursor = nextCursor;
+    } while (cursor !== "0");
+    return totalDeleted;
+  }
+  getMessageKey(threadId, messageId) {
+    return this.getKey(storage.TABLE_MESSAGES, { threadId, id: messageId });
+  }
+  getThreadMessagesKey(threadId) {
+    return `thread:${threadId}:messages`;
+  }
+  parseWorkflowRun(row) {
+    let parsedSnapshot = row.snapshot;
+    if (typeof parsedSnapshot === "string") {
+      try {
+        parsedSnapshot = JSON.parse(row.snapshot);
+      } catch (e) {
+        console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
+      }
+    }
+    return {
+      workflowName: row.workflow_name,
+      runId: row.run_id,
+      snapshot: parsedSnapshot,
+      createdAt: this.ensureDate(row.createdAt),
+      updatedAt: this.ensureDate(row.updatedAt),
+      resourceId: row.resourceId
+    };
+  }
+  processRecord(tableName, record) {
+    let key;
+    if (tableName === storage.TABLE_MESSAGES) {
+      key = this.getKey(tableName, { threadId: record.threadId, id: record.id });
+    } else if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
+      key = this.getKey(tableName, {
+        namespace: record.namespace || "workflows",
+        workflow_name: record.workflow_name,
+        run_id: record.run_id,
+        ...record.resourceId ? { resourceId: record.resourceId } : {}
+      });
+    } else if (tableName === storage.TABLE_EVALS) {
+      key = this.getKey(tableName, { id: record.run_id });
+    } else {
+      key = this.getKey(tableName, { id: record.id });
+    }
+    const processedRecord = {
+      ...record,
+      createdAt: this.serializeDate(record.createdAt),
+      updatedAt: this.serializeDate(record.updatedAt)
+    };
+    return { key, processedRecord };
   }
   async getEvalsByAgentName(agentName, type) {
     try {
       const pattern = `${storage.TABLE_EVALS}:*`;
-      const keys = await this.redis.keys(pattern);
+      const keys = await this.scanKeys(pattern);
       const evalRecords = await Promise.all(
         keys.map(async (key) => {
           const data = await this.redis.get(key);
@@ -58,36 +202,6 @@ var UpstashStore = class extends storage.MastraStorage {
       return [];
     }
   }
-  transformEvalRecord(record) {
-    let result = record.result;
-    if (typeof result === "string") {
-      try {
-        result = JSON.parse(result);
-      } catch {
-        console.warn("Failed to parse result JSON:");
-      }
-    }
-    let testInfo = record.test_info;
-    if (typeof testInfo === "string") {
-      try {
-        testInfo = JSON.parse(testInfo);
-      } catch {
-        console.warn("Failed to parse test_info JSON:");
-      }
-    }
-    return {
-      agentName: record.agent_name,
-      input: record.input,
-      output: record.output,
-      result,
-      metricName: record.metric_name,
-      instructions: record.instructions,
-      testInfo,
-      globalRunId: record.global_run_id,
-      runId: record.run_id,
-      createdAt: typeof record.created_at === "string" ? record.created_at : record.created_at instanceof Date ? record.created_at.toISOString() : (/* @__PURE__ */ new Date()).toISOString()
-    };
-  }
   async getTraces({
     name,
     scope,
@@ -103,7 +217,7 @@ var UpstashStore = class extends storage.MastraStorage {
   }) {
     try {
       const pattern = `${storage.TABLE_TRACES}:*`;
-      const keys = await this.redis.keys(pattern);
+      const keys = await this.scanKeys(pattern);
       const traceRecords = await Promise.all(
         keys.map(async (key) => {
           const data = await this.redis.get(key);
@@ -167,37 +281,6 @@ var UpstashStore = class extends storage.MastraStorage {
       return [];
     }
   }
-  parseJSON(value) {
-    if (typeof value === "string") {
-      try {
-        return JSON.parse(value);
-      } catch {
-        return value;
-      }
-    }
-    return value;
-  }
-  redis;
-  constructor(config) {
-    super({ name: "Upstash" });
-    this.redis = new redis.Redis({
-      url: config.url,
-      token: config.token
-    });
-  }
-  getKey(tableName, keys) {
-    const keyParts = Object.entries(keys).filter(([_, value]) => value !== void 0).map(([key, value]) => `${key}:${value}`);
-    return `${tableName}:${keyParts.join(":")}`;
-  }
-  ensureDate(date) {
-    if (!date) return void 0;
-    return date instanceof Date ? date : new Date(date);
-  }
-  serializeDate(date) {
-    if (!date) return void 0;
-    const dateObj = this.ensureDate(date);
-    return dateObj?.toISOString();
-  }
   async createTable({
     tableName,
     schema
@@ -206,34 +289,26 @@ var UpstashStore = class extends storage.MastraStorage {
   }
   async clearTable({ tableName }) {
     const pattern = `${tableName}:*`;
-    const keys = await this.redis.keys(pattern);
-    if (keys.length > 0) {
-      await this.redis.del(...keys);
-    }
+    await this.scanAndDelete(pattern);
   }
   async insert({ tableName, record }) {
-    let key;
-    if (tableName === storage.TABLE_MESSAGES) {
-      key = this.getKey(tableName, { threadId: record.threadId, id: record.id });
-    } else if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
-      key = this.getKey(tableName, {
-        namespace: record.namespace || "workflows",
-        workflow_name: record.workflow_name,
-        run_id: record.run_id,
-        ...record.resourceId ? { resourceId: record.resourceId } : {}
-      });
-    } else if (tableName === storage.TABLE_EVALS) {
-      key = this.getKey(tableName, { id: record.run_id });
-    } else {
-      key = this.getKey(tableName, { id: record.id });
-    }
-    const processedRecord = {
-      ...record,
-      createdAt: this.serializeDate(record.createdAt),
-      updatedAt: this.serializeDate(record.updatedAt)
-    };
+    const { key, processedRecord } = this.processRecord(tableName, record);
     await this.redis.set(key, processedRecord);
   }
+  async batchInsert(input) {
+    const { tableName, records } = input;
+    if (!records.length) return;
+    const batchSize = 1e3;
+    for (let i = 0; i < records.length; i += batchSize) {
+      const batch = records.slice(i, i + batchSize);
+      const pipeline = this.redis.pipeline();
+      for (const record of batch) {
+        const { key, processedRecord } = this.processRecord(tableName, record);
+        pipeline.set(key, processedRecord);
+      }
+      await pipeline.exec();
+    }
+  }
   async load({ tableName, keys }) {
     const key = this.getKey(tableName, keys);
     const data = await this.redis.get(key);
@@ -254,7 +329,7 @@ var UpstashStore = class extends storage.MastraStorage {
   }
   async getThreadsByResourceId({ resourceId }) {
     const pattern = `${storage.TABLE_THREADS}:*`;
-    const keys = await this.redis.keys(pattern);
+    const keys = await this.scanKeys(pattern);
     const threads = await Promise.all(
       keys.map(async (key) => {
         const data = await this.redis.get(key);
@@ -299,29 +374,27 @@ var UpstashStore = class extends storage.MastraStorage {
     const key = this.getKey(storage.TABLE_THREADS, { id: threadId });
     await this.redis.del(key);
   }
-  getMessageKey(threadId, messageId) {
-    return this.getKey(storage.TABLE_MESSAGES, { threadId, id: messageId });
-  }
-  getThreadMessagesKey(threadId) {
-    return `thread:${threadId}:messages`;
-  }
   async saveMessages({ messages }) {
     if (messages.length === 0) return [];
-    const pipeline = this.redis.pipeline();
     const messagesWithIndex = messages.map((message, index) => ({
       ...message,
       _index: index
     }));
-    for (const message of messagesWithIndex) {
-      const key = this.getMessageKey(message.threadId, message.id);
-      const score = message._index !== void 0 ? message._index : new Date(message.createdAt).getTime();
-      pipeline.set(key, message);
-      pipeline.zadd(this.getThreadMessagesKey(message.threadId), {
-        score,
-        member: message.id
-      });
+    const batchSize = 1e3;
+    for (let i = 0; i < messagesWithIndex.length; i += batchSize) {
+      const batch = messagesWithIndex.slice(i, i + batchSize);
+      const pipeline = this.redis.pipeline();
+      for (const message of batch) {
+        const key = this.getMessageKey(message.threadId, message.id);
+        const score = message._index !== void 0 ? message._index : new Date(message.createdAt).getTime();
+        pipeline.set(key, message);
+        pipeline.zadd(this.getThreadMessagesKey(message.threadId), {
+          score,
+          member: message.id
+        });
+      }
+      await pipeline.exec();
     }
-    await pipeline.exec();
     return messages;
   }
   async getMessages({ threadId, selectBy }) {
@@ -385,24 +458,6 @@ var UpstashStore = class extends storage.MastraStorage {
     if (!data) return null;
     return data.snapshot;
   }
-  parseWorkflowRun(row) {
-    let parsedSnapshot = row.snapshot;
-    if (typeof parsedSnapshot === "string") {
-      try {
-        parsedSnapshot = JSON.parse(row.snapshot);
-      } catch (e) {
-        console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
-      }
-    }
-    return {
-      workflowName: row.workflow_name,
-      runId: row.run_id,
-      snapshot: parsedSnapshot,
-      createdAt: this.ensureDate(row.createdAt),
-      updatedAt: this.ensureDate(row.updatedAt),
-      resourceId: row.resourceId
-    };
-  }
   async getWorkflowRuns({
     namespace,
     workflowName,
@@ -426,7 +481,7 @@ var UpstashStore = class extends storage.MastraStorage {
       } else if (resourceId) {
         pattern = this.getKey(storage.TABLE_WORKFLOW_SNAPSHOT, { namespace, workflow_name: "*", run_id: "*", resourceId });
       }
-      const keys = await this.redis.keys(pattern);
+      const keys = await this.scanKeys(pattern);
       const workflows = await Promise.all(
         keys.map(async (key) => {
           const data = await this.redis.get(key);
@@ -455,7 +510,7 @@ var UpstashStore = class extends storage.MastraStorage {
   }) {
     try {
       const key = this.getKey(storage.TABLE_WORKFLOW_SNAPSHOT, { namespace, workflow_name: workflowName, run_id: runId }) + "*";
-      const keys = await this.redis.keys(key);
+      const keys = await this.scanKeys(key);
       const workflows = await Promise.all(
         keys.map(async (key2) => {
           const data2 = await this.redis.get(key2);