@mastra/memory 0.2.6 → 0.2.7-alpha.2

package/.turbo/turbo-build.log

@@ -1,27 +1,29 @@
 
-> @mastra/memory@0.2.6-alpha.4 build /home/runner/work/mastra/mastra/packages/memory
-> pnpm run check && tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
+> @mastra/memory@0.2.7-alpha.2 build /home/runner/work/mastra/mastra/packages/memory
+> pnpm run check && tsup src/index.ts src/processors/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 
-> @mastra/memory@0.2.6-alpha.4 check /home/runner/work/mastra/mastra/packages/memory
+> @mastra/memory@0.2.7-alpha.2 check /home/runner/work/mastra/mastra/packages/memory
 > tsc --noEmit
 
-CLI Building entry: src/index.ts
+CLI Building entry: src/index.ts, src/processors/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 5100ms
+TSC ⚡️ Build success in 11238ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/memory/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/memory/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 2321ms
+DTS ⚡️ Build success in 5847ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-CJS dist/index.cjs 13.56 KB
-CJS ⚡️ Build success in 194ms
-ESM dist/index.js 13.53 KB
-ESM ⚡️ Build success in 194ms
+ESM dist/index.js            13.60 KB
+ESM dist/processors/index.js 5.33 KB
+ESM ⚡️ Build success in 238ms
+CJS dist/index.cjs            13.62 KB
+CJS dist/processors/index.cjs 5.54 KB
+CJS ⚡️ Build success in 238ms

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # @mastra/memory
 
+## 0.2.7-alpha.2
+
+### Patch Changes
+
+- Updated dependencies [56c31b7]
+- Updated dependencies [dbbbf80]
+- Updated dependencies [99d43b9]
+  - @mastra/core@0.8.0-alpha.2
+  - @mastra/rag@0.1.15-alpha.2
+
+## 0.2.7-alpha.1
+
+### Patch Changes
+
+- a0967a0: Added new "Memory Processor" feature to @mastra/core and @mastra/memory, allowing devs to modify Mastra Memory before it's sent to the LLM
+- 0118361: Add resourceId to memory metadata
+- Updated dependencies [619c39d]
+- Updated dependencies [fe56be0]
+- Updated dependencies [a0967a0]
+- Updated dependencies [e47f529]
+- Updated dependencies [fca3b21]
+- Updated dependencies [0118361]
+- Updated dependencies [619c39d]
+  - @mastra/core@0.8.0-alpha.1
+  - @mastra/rag@0.1.15-alpha.1
+
+## 0.2.7-alpha.0
+
+### Patch Changes
+
+- 7599d77: fix(deps): update ai sdk to ^4.2.2
+- Updated dependencies [107bcfe]
+- Updated dependencies [5b4e19f]
+- Updated dependencies [7599d77]
+- Updated dependencies [cafae83]
+- Updated dependencies [8076ecf]
+- Updated dependencies [304397c]
+  - @mastra/core@0.7.1-alpha.0
+  - @mastra/rag@0.1.15-alpha.0
+
 ## 0.2.6
 
 ### Patch Changes

package/README.md

@@ -1,3 +1,7 @@
-# @mastra/memory
+# Mastra Memory
 
-Experimental
+Memory management for Mastra agents. Visit [the docs](https://mastra.ai/docs/agents/01-agent-memory) for more information.
+
+## License
+
+MIT

package/dist/_tsup-dts-rollup.d.cts

@@ -3,10 +3,14 @@ import type { CoreMessage } from '@mastra/core';
 import type { CoreTool } from '@mastra/core';
 import { MastraMemory } from '@mastra/core/memory';
 import type { MemoryConfig } from '@mastra/core/memory';
+import { MemoryProcessor } from '@mastra/core/memory';
+import { MemoryProcessor as MemoryProcessor_2 } from '@mastra/core';
+import type { MemoryProcessorOpts } from '@mastra/core';
 import type { MessageType } from '@mastra/core/memory';
 import type { SharedMemoryConfig } from '@mastra/core/memory';
 import type { StorageGetMessagesArg } from '@mastra/core/storage';
 import type { StorageThreadType } from '@mastra/core/memory';
+import type { TiktokenBPE } from 'js-tiktoken/lite';
 
 /**
  * Concrete implementation of MastraMemory that adds support for thread configuration
@@ -15,7 +19,9 @@ import type { StorageThreadType } from '@mastra/core/memory';
 export declare class Memory extends MastraMemory {
     constructor(config?: SharedMemoryConfig);
     private validateThreadIsOwnedByResource;
-    query({ threadId, resourceId, selectBy, threadConfig, }: StorageGetMessagesArg): Promise<{
+    query({ threadId, resourceId, selectBy, threadConfig, }: StorageGetMessagesArg & {
+        threadConfig?: MemoryConfig;
+    }): Promise<{
         messages: CoreMessage[];
         uiMessages: AiMessageType[];
     }>;
@@ -66,6 +72,57 @@ export declare class Memory extends MastraMemory {
     getTools(config?: MemoryConfig): Record<string, CoreTool>;
 }
 
+/**
+ * Limits the total number of tokens in the messages.
+ * Uses js-tiktoken with o200k_base encoding by default for accurate token counting with modern models.
+ */
+declare class TokenLimiter extends MemoryProcessor {
+    private encoder;
+    private maxTokens;
+    TOKENS_PER_MESSAGE: number;
+    TOKENS_PER_TOOL: number;
+    TOKENS_PER_CONVERSATION: number;
+    /**
+     * Create a token limiter for messages.
+     * @param options Either a number (token limit) or a configuration object
+     */
+    constructor(options: number | TokenLimiterOptions);
+    process(messages: CoreMessage[], { systemMessage, memorySystemMessage, newMessages }?: MemoryProcessorOpts): CoreMessage[];
+    countTokens(message: string | CoreMessage): number;
+}
+export { TokenLimiter }
+export { TokenLimiter as TokenLimiter_alias_1 }
+
+/**
+ * Configuration options for TokenLimiter
+ */
+declare interface TokenLimiterOptions {
+    /** Maximum number of tokens to allow */
+    limit: number;
+    /** Optional encoding to use (defaults to o200k_base which is used by gpt-4o) */
+    encoding?: TiktokenBPE;
+}
+
+/**
+ * Filters out tool calls and results from messages.
+ * By default (with no arguments), excludes all tool calls and their results.
+ * Can be configured to exclude only specific tools by name.
+ */
+declare class ToolCallFilter extends MemoryProcessor_2 {
+    private exclude;
+    /**
+     * Create a filter for tool calls and results.
+     * @param options Configuration options
+     * @param options.exclude List of specific tool names to exclude. If not provided, all tool calls are excluded.
+     */
+    constructor(options?: {
+        exclude?: string[];
+    });
+    process(messages: CoreMessage[]): CoreMessage[];
+}
+export { ToolCallFilter }
+export { ToolCallFilter as ToolCallFilter_alias_1 }
+
 export declare const updateWorkingMemoryTool: CoreTool;
 
 export { }

package/dist/_tsup-dts-rollup.d.ts

@@ -3,10 +3,14 @@ import type { CoreMessage } from '@mastra/core';
 import type { CoreTool } from '@mastra/core';
 import { MastraMemory } from '@mastra/core/memory';
 import type { MemoryConfig } from '@mastra/core/memory';
+import { MemoryProcessor } from '@mastra/core/memory';
+import { MemoryProcessor as MemoryProcessor_2 } from '@mastra/core';
+import type { MemoryProcessorOpts } from '@mastra/core';
 import type { MessageType } from '@mastra/core/memory';
 import type { SharedMemoryConfig } from '@mastra/core/memory';
 import type { StorageGetMessagesArg } from '@mastra/core/storage';
 import type { StorageThreadType } from '@mastra/core/memory';
+import type { TiktokenBPE } from 'js-tiktoken/lite';
 
 /**
  * Concrete implementation of MastraMemory that adds support for thread configuration
@@ -15,7 +19,9 @@ import type { StorageThreadType } from '@mastra/core/memory';
 export declare class Memory extends MastraMemory {
     constructor(config?: SharedMemoryConfig);
     private validateThreadIsOwnedByResource;
-    query({ threadId, resourceId, selectBy, threadConfig, }: StorageGetMessagesArg): Promise<{
+    query({ threadId, resourceId, selectBy, threadConfig, }: StorageGetMessagesArg & {
+        threadConfig?: MemoryConfig;
+    }): Promise<{
         messages: CoreMessage[];
         uiMessages: AiMessageType[];
     }>;
@@ -66,6 +72,57 @@ export declare class Memory extends MastraMemory {
     getTools(config?: MemoryConfig): Record<string, CoreTool>;
 }
 
+/**
+ * Limits the total number of tokens in the messages.
+ * Uses js-tiktoken with o200k_base encoding by default for accurate token counting with modern models.
+ */
+declare class TokenLimiter extends MemoryProcessor {
+    private encoder;
+    private maxTokens;
+    TOKENS_PER_MESSAGE: number;
+    TOKENS_PER_TOOL: number;
+    TOKENS_PER_CONVERSATION: number;
+    /**
+     * Create a token limiter for messages.
+     * @param options Either a number (token limit) or a configuration object
+     */
+    constructor(options: number | TokenLimiterOptions);
+    process(messages: CoreMessage[], { systemMessage, memorySystemMessage, newMessages }?: MemoryProcessorOpts): CoreMessage[];
+    countTokens(message: string | CoreMessage): number;
+}
+export { TokenLimiter }
+export { TokenLimiter as TokenLimiter_alias_1 }
+
+/**
+ * Configuration options for TokenLimiter
+ */
+declare interface TokenLimiterOptions {
+    /** Maximum number of tokens to allow */
+    limit: number;
+    /** Optional encoding to use (defaults to o200k_base which is used by gpt-4o) */
+    encoding?: TiktokenBPE;
+}
+
+/**
+ * Filters out tool calls and results from messages.
+ * By default (with no arguments), excludes all tool calls and their results.
+ * Can be configured to exclude only specific tools by name.
+ */
+declare class ToolCallFilter extends MemoryProcessor_2 {
+    private exclude;
+    /**
+     * Create a filter for tool calls and results.
+     * @param options Configuration options
+     * @param options.exclude List of specific tool names to exclude. If not provided, all tool calls are excluded.
+     */
+    constructor(options?: {
+        exclude?: string[];
+    });
+    process(messages: CoreMessage[]): CoreMessage[];
+}
+export { ToolCallFilter }
+export { ToolCallFilter as ToolCallFilter_alias_1 }
+
 export declare const updateWorkingMemoryTool: CoreTool;
 
 export { }

package/dist/index.cjs

@@ -129,7 +129,7 @@ var Memory = class extends memory.MastraMemory {
         threadId
       };
     }
-    const messages = await this.query({
+    const messagesResult = await this.query({
       threadId,
       selectBy: {
         last: threadConfig.lastMessages,
@@ -137,11 +137,11 @@ var Memory = class extends memory.MastraMemory {
       },
       threadConfig: config
     });
-    this.logger.debug(`Remembered message history includes ${messages.messages.length} messages.`);
+    this.logger.debug(`Remembered message history includes ${messagesResult.messages.length} messages.`);
     return {
       threadId,
-      messages: messages.messages,
-      uiMessages: messages.uiMessages
+      messages: messagesResult.messages,
+      uiMessages: messagesResult.uiMessages
     };
   }
   async getThreadById({ threadId }) {
@@ -214,7 +214,8 @@ var Memory = class extends memory.MastraMemory {
           vectors: embeddings,
           metadata: chunks.map(() => ({
             message_id: message.id,
-            thread_id: message.threadId
+            thread_id: message.threadId,
+            resource_id: message.resourceId
           }))
         });
       }

package/dist/index.js

@@ -127,7 +127,7 @@ var Memory = class extends MastraMemory {
         threadId
       };
     }
-    const messages = await this.query({
+    const messagesResult = await this.query({
       threadId,
       selectBy: {
         last: threadConfig.lastMessages,
@@ -135,11 +135,11 @@ var Memory = class extends MastraMemory {
       },
       threadConfig: config
     });
-    this.logger.debug(`Remembered message history includes ${messages.messages.length} messages.`);
+    this.logger.debug(`Remembered message history includes ${messagesResult.messages.length} messages.`);
     return {
       threadId,
-      messages: messages.messages,
-      uiMessages: messages.uiMessages
+      messages: messagesResult.messages,
+      uiMessages: messagesResult.uiMessages
     };
   }
   async getThreadById({ threadId }) {
@@ -212,7 +212,8 @@ var Memory = class extends MastraMemory {
           vectors: embeddings,
           metadata: chunks.map(() => ({
             message_id: message.id,
-            thread_id: message.threadId
+            thread_id: message.threadId,
+            resource_id: message.resourceId
           }))
         });
       }

package/dist/processors/index.cjs

@@ -0,0 +1,161 @@
+'use strict';
+
+var memory = require('@mastra/core/memory');
+var lite = require('js-tiktoken/lite');
+var o200k_base = require('js-tiktoken/ranks/o200k_base');
+var core = require('@mastra/core');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var o200k_base__default = /*#__PURE__*/_interopDefault(o200k_base);
+
+// src/processors/token-limiter.ts
+var TokenLimiter = class extends memory.MemoryProcessor {
+  encoder;
+  maxTokens;
+  // Token overheads per OpenAI's documentation
+  // See: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken#6-counting-tokens-for-chat-completions-api-calls
+  // Every message follows <|start|>{role/name}\n{content}<|end|>
+  TOKENS_PER_MESSAGE = 3;
+  // tokens added for each message (start & end tokens)
+  TOKENS_PER_TOOL = 2;
+  // empirical adjustment for tool calls
+  TOKENS_PER_CONVERSATION = 25;
+  // fixed overhead for the conversation
+  /**
+   * Create a token limiter for messages.
+   * @param options Either a number (token limit) or a configuration object
+   */
+  constructor(options) {
+    super({
+      name: "TokenLimiter"
+    });
+    if (typeof options === "number") {
+      this.maxTokens = options;
+      this.encoder = new lite.Tiktoken(o200k_base__default.default);
+    } else {
+      this.maxTokens = options.limit;
+      this.encoder = new lite.Tiktoken(options.encoding || o200k_base__default.default);
+    }
+  }
+  process(messages, { systemMessage, memorySystemMessage, newMessages } = {}) {
+    let totalTokens = 0;
+    totalTokens += this.TOKENS_PER_CONVERSATION;
+    if (systemMessage) {
+      totalTokens += this.countTokens(systemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE;
+    }
+    if (memorySystemMessage) {
+      totalTokens += this.countTokens(memorySystemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE;
+    }
+    const allMessages = [...messages, ...newMessages || []];
+    const result = [];
+    for (let i = allMessages.length - 1; i >= 0; i--) {
+      const message = allMessages[i];
+      if (!message) continue;
+      const messageTokens = this.countTokens(message);
+      if (totalTokens + messageTokens <= this.maxTokens) {
+        result.unshift(message);
+        totalTokens += messageTokens;
+      } else {
+        this.logger.info(
+          `filtering ${allMessages.length - result.length}/${allMessages.length} messages, token limit of ${this.maxTokens} exceeded`
+        );
+        break;
+      }
+    }
+    return result;
+  }
+  countTokens(message) {
+    if (typeof message === `string`) {
+      return this.encoder.encode(message).length;
+    }
+    let tokenString = message.role;
+    if (typeof message.content === "string") {
+      tokenString += message.content;
+    } else if (Array.isArray(message.content)) {
+      for (const part of message.content) {
+        tokenString += part.type;
+        if (part.type === "text") {
+          tokenString += part.text;
+        } else if (part.type === "tool-call") {
+          tokenString += part.toolName;
+          if (part.args) {
+            tokenString += typeof part.args === "string" ? part.args : JSON.stringify(part.args);
+          }
+        } else if (part.type === "tool-result") {
+          if (part.result !== void 0) {
+            tokenString += typeof part.result === "string" ? part.result : JSON.stringify(part.result);
+          }
+        } else {
+          tokenString += JSON.stringify(part);
+        }
+      }
+    }
+    const messageOverhead = this.TOKENS_PER_MESSAGE;
+    let toolOverhead = 0;
+    if (Array.isArray(message.content)) {
+      for (const part of message.content) {
+        if (part.type === "tool-call" || part.type === "tool-result") {
+          toolOverhead += this.TOKENS_PER_TOOL;
+        }
+      }
+    }
+    const totalMessageOverhead = messageOverhead + toolOverhead;
+    return this.encoder.encode(tokenString).length + totalMessageOverhead;
+  }
+};
+var ToolCallFilter = class extends core.MemoryProcessor {
+  exclude;
+  /**
+   * Create a filter for tool calls and results.
+   * @param options Configuration options
+   * @param options.exclude List of specific tool names to exclude. If not provided, all tool calls are excluded.
+   */
+  constructor(options = {}) {
+    super({ name: "ToolCallFilter" });
+    if (!options || !options.exclude) {
+      this.exclude = "all";
+    } else {
+      this.exclude = Array.isArray(options.exclude) ? options.exclude : [];
+    }
+  }
+  process(messages) {
+    if (this.exclude === "all") {
+      return messages.filter((message) => {
+        if (Array.isArray(message.content)) {
+          return !message.content.some((part) => part.type === "tool-call" || part.type === "tool-result");
+        }
+        return true;
+      });
+    }
+    if (this.exclude.length > 0) {
+      const excludedToolCallIds = /* @__PURE__ */ new Set();
+      return messages.filter((message) => {
+        if (!Array.isArray(message.content)) return true;
+        if (message.role === "assistant") {
+          let shouldExclude = false;
+          for (const part of message.content) {
+            if (part.type === "tool-call" && this.exclude.includes(part.toolName)) {
+              excludedToolCallIds.add(part.toolCallId);
+              shouldExclude = true;
+            }
+          }
+          return !shouldExclude;
+        }
+        if (message.role === "tool") {
+          const shouldExclude = message.content.some(
+            (part) => part.type === "tool-result" && excludedToolCallIds.has(part.toolCallId)
+          );
+          return !shouldExclude;
+        }
+        return true;
+      });
+    }
+    return messages;
+  }
+};
+
+exports.TokenLimiter = TokenLimiter;
+exports.ToolCallFilter = ToolCallFilter;

package/dist/processors/index.d.cts

@@ -0,0 +1,2 @@
+export { TokenLimiter } from '../_tsup-dts-rollup.cjs';
+export { ToolCallFilter } from '../_tsup-dts-rollup.cjs';

package/dist/processors/index.d.ts

@@ -0,0 +1,2 @@
+export { TokenLimiter } from '../_tsup-dts-rollup.js';
+export { ToolCallFilter } from '../_tsup-dts-rollup.js';

package/dist/processors/index.js

@@ -0,0 +1,154 @@
+import { MemoryProcessor } from '@mastra/core/memory';
+import { Tiktoken } from 'js-tiktoken/lite';
+import o200k_base from 'js-tiktoken/ranks/o200k_base';
+import { MemoryProcessor as MemoryProcessor$1 } from '@mastra/core';
+
+// src/processors/token-limiter.ts
+var TokenLimiter = class extends MemoryProcessor {
+  encoder;
+  maxTokens;
+  // Token overheads per OpenAI's documentation
+  // See: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken#6-counting-tokens-for-chat-completions-api-calls
+  // Every message follows <|start|>{role/name}\n{content}<|end|>
+  TOKENS_PER_MESSAGE = 3;
+  // tokens added for each message (start & end tokens)
+  TOKENS_PER_TOOL = 2;
+  // empirical adjustment for tool calls
+  TOKENS_PER_CONVERSATION = 25;
+  // fixed overhead for the conversation
+  /**
+   * Create a token limiter for messages.
+   * @param options Either a number (token limit) or a configuration object
+   */
+  constructor(options) {
+    super({
+      name: "TokenLimiter"
+    });
+    if (typeof options === "number") {
+      this.maxTokens = options;
+      this.encoder = new Tiktoken(o200k_base);
+    } else {
+      this.maxTokens = options.limit;
+      this.encoder = new Tiktoken(options.encoding || o200k_base);
+    }
+  }
+  process(messages, { systemMessage, memorySystemMessage, newMessages } = {}) {
+    let totalTokens = 0;
+    totalTokens += this.TOKENS_PER_CONVERSATION;
+    if (systemMessage) {
+      totalTokens += this.countTokens(systemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE;
+    }
+    if (memorySystemMessage) {
+      totalTokens += this.countTokens(memorySystemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE;
+    }
+    const allMessages = [...messages, ...newMessages || []];
+    const result = [];
+    for (let i = allMessages.length - 1; i >= 0; i--) {
+      const message = allMessages[i];
+      if (!message) continue;
+      const messageTokens = this.countTokens(message);
+      if (totalTokens + messageTokens <= this.maxTokens) {
+        result.unshift(message);
+        totalTokens += messageTokens;
+      } else {
+        this.logger.info(
+          `filtering ${allMessages.length - result.length}/${allMessages.length} messages, token limit of ${this.maxTokens} exceeded`
+        );
+        break;
+      }
+    }
+    return result;
+  }
+  countTokens(message) {
+    if (typeof message === `string`) {
+      return this.encoder.encode(message).length;
+    }
+    let tokenString = message.role;
+    if (typeof message.content === "string") {
+      tokenString += message.content;
+    } else if (Array.isArray(message.content)) {
+      for (const part of message.content) {
+        tokenString += part.type;
+        if (part.type === "text") {
+          tokenString += part.text;
+        } else if (part.type === "tool-call") {
+          tokenString += part.toolName;
+          if (part.args) {
+            tokenString += typeof part.args === "string" ? part.args : JSON.stringify(part.args);
+          }
+        } else if (part.type === "tool-result") {
+          if (part.result !== void 0) {
+            tokenString += typeof part.result === "string" ? part.result : JSON.stringify(part.result);
+          }
+        } else {
+          tokenString += JSON.stringify(part);
+        }
+      }
+    }
+    const messageOverhead = this.TOKENS_PER_MESSAGE;
+    let toolOverhead = 0;
+    if (Array.isArray(message.content)) {
+      for (const part of message.content) {
+        if (part.type === "tool-call" || part.type === "tool-result") {
+          toolOverhead += this.TOKENS_PER_TOOL;
+        }
+      }
+    }
+    const totalMessageOverhead = messageOverhead + toolOverhead;
+    return this.encoder.encode(tokenString).length + totalMessageOverhead;
+  }
+};
+var ToolCallFilter = class extends MemoryProcessor$1 {
+  exclude;
+  /**
+   * Create a filter for tool calls and results.
+   * @param options Configuration options
+   * @param options.exclude List of specific tool names to exclude. If not provided, all tool calls are excluded.
+   */
+  constructor(options = {}) {
+    super({ name: "ToolCallFilter" });
+    if (!options || !options.exclude) {
+      this.exclude = "all";
+    } else {
+      this.exclude = Array.isArray(options.exclude) ? options.exclude : [];
+    }
+  }
+  process(messages) {
+    if (this.exclude === "all") {
+      return messages.filter((message) => {
+        if (Array.isArray(message.content)) {
+          return !message.content.some((part) => part.type === "tool-call" || part.type === "tool-result");
+        }
+        return true;
+      });
+    }
+    if (this.exclude.length > 0) {
+      const excludedToolCallIds = /* @__PURE__ */ new Set();
+      return messages.filter((message) => {
+        if (!Array.isArray(message.content)) return true;
+        if (message.role === "assistant") {
+          let shouldExclude = false;
+          for (const part of message.content) {
+            if (part.type === "tool-call" && this.exclude.includes(part.toolName)) {
+              excludedToolCallIds.add(part.toolCallId);
+              shouldExclude = true;
+            }
+          }
+          return !shouldExclude;
+        }
+        if (message.role === "tool") {
+          const shouldExclude = message.content.some(
+            (part) => part.type === "tool-result" && excludedToolCallIds.has(part.toolCallId)
+          );
+          return !shouldExclude;
+        }
+        return true;
+      });
+    }
+    return messages;
+  }
+};
+
+export { TokenLimiter, ToolCallFilter };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/memory",
-  "version": "0.2.6",
+  "version": "0.2.7-alpha.2",
   "description": "",
   "type": "module",
   "main": "./dist/index.js",
@@ -16,24 +16,14 @@
         "default": "./dist/index.cjs"
       }
     },
-    "./kv-upstash": {
+    "./processors": {
       "import": {
-        "types": "./dist/kv/upstash.d.ts",
-        "default": "./dist/kv/upstash.js"
+        "types": "./dist/processors/index.d.ts",
+        "default": "./dist/processors/index.js"
       },
       "require": {
-        "types": "./dist/kv/upstash.d.cts",
-        "default": "./dist/kv/upstash.cjs"
-      }
-    },
-    "./postgres": {
-      "import": {
-        "types": "./dist/postgres/index.d.ts",
-        "default": "./dist/postgres/index.js"
-      },
-      "require": {
-        "types": "./dist/postgres/index.d.cts",
-        "default": "./dist/postgres/index.cjs"
+        "types": "./dist/processors/index.d.cts",
+        "default": "./dist/processors/index.cjs"
       }
     },
     "./package.json": "./package.json"
@@ -43,16 +33,18 @@
   "license": "ISC",
   "dependencies": {
     "@upstash/redis": "^1.34.5",
-    "ai": "^4.1.54",
+    "js-tiktoken": "^1.0.19",
+    "ai": "^4.2.2",
     "pg": "^8.13.3",
     "pg-pool": "^3.7.1",
     "postgres": "^3.4.5",
     "redis": "^4.7.0",
     "zod": "^3.24.2",
-    "@mastra/core": "^0.7.0",
-    "@mastra/rag": "^0.1.14"
+    "@mastra/core": "^0.8.0-alpha.2",
+    "@mastra/rag": "^0.1.15-alpha.2"
   },
   "devDependencies": {
+    "@ai-sdk/openai": "^1.3.3",
     "@microsoft/api-extractor": "^7.52.1",
     "@types/node": "^20.17.27",
     "@types/pg": "^8.11.11",
@@ -65,10 +57,11 @@
   },
   "scripts": {
     "check": "tsc --noEmit",
-    "build": "pnpm run check && tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting",
+    "build": "pnpm run check && tsup src/index.ts src/processors/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting",
     "build:watch": "pnpm build --watch",
     "test:integration": "cd integration-tests && pnpm run test",
-    "test": "pnpm test:integration",
+    "test:unit": "pnpm vitest run ./src/*",
+    "test": "pnpm test:integration && pnpm test:unit",
     "lint": "eslint ."
   }
 }

package/src/index.ts

@@ -41,7 +41,9 @@ export class Memory extends MastraMemory {
     resourceId,
     selectBy,
     threadConfig,
-  }: StorageGetMessagesArg): Promise<{ messages: CoreMessage[]; uiMessages: AiMessageType[] }> {
+  }: StorageGetMessagesArg & {
+    threadConfig?: MemoryConfig;
+  }): Promise<{ messages: CoreMessage[]; uiMessages: AiMessageType[] }> {
     if (resourceId) await this.validateThreadIsOwnedByResource(threadId, resourceId);
 
     const vectorResults: {
@@ -148,7 +150,7 @@ export class Memory extends MastraMemory {
       };
     }
 
-    const messages = await this.query({
+    const messagesResult = await this.query({
       threadId,
       selectBy: {
         last: threadConfig.lastMessages,
@@ -157,11 +159,11 @@ export class Memory extends MastraMemory {
       threadConfig: config,
     });
 
-    this.logger.debug(`Remembered message history includes ${messages.messages.length} messages.`);
+    this.logger.debug(`Remembered message history includes ${messagesResult.messages.length} messages.`);
     return {
       threadId,
-      messages: messages.messages,
-      uiMessages: messages.uiMessages,
+      messages: messagesResult.messages,
+      uiMessages: messagesResult.uiMessages,
     };
   }
 
@@ -271,6 +273,7 @@ export class Memory extends MastraMemory {
           metadata: chunks.map(() => ({
             message_id: message.id,
             thread_id: message.threadId,
+            resource_id: message.resourceId,
           })),
         });
       }

package/src/processors/index.test.ts

@@ -0,0 +1,236 @@
+import { openai } from '@ai-sdk/openai';
+import { createTool } from '@mastra/core';
+import type { CoreMessage, MessageType } from '@mastra/core';
+import { Agent } from '@mastra/core/agent';
+import cl100k_base from 'js-tiktoken/ranks/cl100k_base';
+import { describe, it, expect } from 'vitest';
+import { z } from 'zod';
+import { generateConversationHistory } from '../../integration-tests/src/test-utils';
+import { TokenLimiter, ToolCallFilter } from './index';
+
+describe('TokenLimiter', () => {
+  it('should limit messages to the specified token count', () => {
+    // Create messages with predictable token counts (approximately 25 tokens each)
+    const { messages } = generateConversationHistory({
+      threadId: '1',
+      messageCount: 5,
+      toolNames: [],
+      toolFrequency: 0,
+    });
+
+    const limiter = new TokenLimiter(200);
+    // @ts-ignore
+    const result = limiter.process(messages);
+
+    // Should prioritize newest messages (higher ids)
+    expect(result.length).toBe(2);
+    expect((result[0] as MessageType).id).toBe('message-8');
+    expect((result[1] as MessageType).id).toBe('message-9');
+  });
+
+  it('should handle empty messages array', () => {
+    const limiter = new TokenLimiter(1000);
+    const result = limiter.process([]);
+    expect(result).toEqual([]);
+  });
+
+  it('should use different encodings based on configuration', () => {
+    const { messages } = generateConversationHistory({
+      threadId: '6',
+      messageCount: 1,
+      toolNames: [],
+      toolFrequency: 0,
+    });
+
+    // Create limiters with different encoding settings
+    const defaultLimiter = new TokenLimiter(1000);
+    const customLimiter = new TokenLimiter({
+      limit: 1000,
+      encoding: cl100k_base,
+    });
+
+    // All should process messages successfully but potentially with different token counts
+    const defaultResult = defaultLimiter.process(messages as CoreMessage[]);
+    const customResult = customLimiter.process(messages as CoreMessage[]);
+
+    // Each should return the same messages but with potentially different token counts
+    expect(defaultResult.length).toBe(messages.length);
+    expect(customResult.length).toBe(messages.length);
+  });
+
+  function estimateTokens(messages: MessageType[]) {
+    // Create a TokenLimiter just for counting tokens
+    const testLimiter = new TokenLimiter(Infinity);
+
+    let estimatedTokens = testLimiter.TOKENS_PER_CONVERSATION;
+
+    // Count tokens for each message including all overheads
+    for (const message of messages) {
+      // Base token count from the countTokens method
+      estimatedTokens += testLimiter.countTokens(message as CoreMessage);
+    }
+
+    return estimatedTokens;
+  }
+
+  function percentDifference(a: number, b: number) {
+    const difference = Number(((Math.abs(a - b) / b) * 100).toFixed(2));
+    console.log(`${a} and ${b} are ${difference}% different`);
+    return difference;
+  }
+
+  async function expectTokenEstimate(config: Parameters<typeof generateConversationHistory>[0], agent: Agent) {
+    const { messages, counts } = generateConversationHistory(config);
+
+    const estimate = estimateTokens(messages);
+    const used = (await agent.generate(messages.slice(0, -1) as CoreMessage[])).usage.totalTokens;
+
+    console.log(`Estimated ${estimate} tokens, used ${used} tokens.\n`, counts);
+
+    // Check if within 2% margin
+    expect(percentDifference(estimate, used)).toBeLessThanOrEqual(2);
+  }
+
+  const calculatorTool = createTool({
+    id: 'calculator',
+    description: 'Perform a simple calculation',
+    inputSchema: z.object({
+      expression: z.string().describe('The mathematical expression to calculate'),
+    }),
+    execute: async ({ context: { expression } }) => {
+      return `The result of ${expression} is ${eval(expression)}`;
+    },
+  });
+
+  const agent = new Agent({
+    name: 'token estimate agent',
+    model: openai('gpt-4o-mini'),
+    instructions: ``,
+    tools: { calculatorTool },
+  });
+
+  describe.concurrent(`98% accuracy`, () => {
+    it(`20 messages, no tools`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 10,
+          toolFrequency: 0,
+          threadId: '2',
+        },
+        agent,
+      );
+    });
+
+    it(`60 messages, no tools`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 30,
+          toolFrequency: 0,
+          threadId: '3',
+        },
+        agent,
+      );
+    });
+
+    it(`4 messages, 0 tools`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 2,
+          toolFrequency: 0,
+          threadId: '3',
+        },
+        agent,
+      );
+    });
+
+    it(`20 messages, 2 tool messages`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 10,
+          toolFrequency: 5,
+          threadId: '3',
+        },
+        agent,
+      );
+    });
+
+    it(`40 messages, 6 tool messages`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 20,
+          toolFrequency: 5,
+          threadId: '4',
+        },
+        agent,
+      );
+    });
+
+    it(`100 messages, 24 tool messages`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 50,
+          toolFrequency: 4,
+          threadId: '5',
+        },
+        agent,
+      );
+    });
+
+    it(`101 messages, 49 tool calls`, async () => {
+      await expectTokenEstimate(
+        {
+          messageCount: 50,
+          toolFrequency: 1,
+          threadId: '5',
+        },
+        agent,
+      );
+    });
+  });
+});
+
+describe.concurrent('ToolCallFilter', () => {
+  it('should exclude all tool calls when created with no arguments', () => {
+    const { messages } = generateConversationHistory({
+      threadId: '3',
+      toolNames: ['weather', 'calculator', 'search'],
+      messageCount: 1,
+    });
+    const filter = new ToolCallFilter();
+    const result = filter.process(messages as CoreMessage[]) as MessageType[];
+
+    // Should only keep the text message and assistant res
+    expect(result.length).toBe(2);
+    expect(result[0].id).toBe('message-0');
+  });
+
+  it('should exclude specific tool calls by name', () => {
+    const { messages } = generateConversationHistory({
+      threadId: '4',
+      toolNames: ['weather', 'calculator'],
+      messageCount: 2,
+    });
+    const filter = new ToolCallFilter({ exclude: ['weather'] });
+    const result = filter.process(messages as CoreMessage[]) as MessageType[];
+
+    // Should keep text message, assistant reply, calculator tool call, and calculator result
+    expect(result.length).toBe(4);
+    expect(result[0].id).toBe('message-0');
+    expect(result[1].id).toBe('message-1');
+    expect(result[2].id).toBe('message-2');
+    expect(result[3].id).toBe('message-3');
+  });
+
+  it('should keep all messages when exclude list is empty', () => {
+    const { messages } = generateConversationHistory({
+      threadId: '5',
+      toolNames: ['weather', 'calculator'],
+    });
+
+    const filter = new ToolCallFilter({ exclude: [] });
+    const result = filter.process(messages as CoreMessage[]);
+
+    // Should keep all messages
+    expect(result.length).toBe(messages.length);
+  });
+});

package/src/processors/index.ts

@@ -0,0 +1,2 @@
+export * from './token-limiter';
+export * from './tool-call-filter';

package/src/processors/token-limiter.ts

@@ -0,0 +1,150 @@
+import type { CoreMessage, MemoryProcessorOpts } from '@mastra/core';
+import { MemoryProcessor } from '@mastra/core/memory';
+import { Tiktoken } from 'js-tiktoken/lite';
+import type { TiktokenBPE } from 'js-tiktoken/lite';
+import o200k_base from 'js-tiktoken/ranks/o200k_base';
+
+/**
+ * Configuration options for TokenLimiter
+ */
+interface TokenLimiterOptions {
+  /** Maximum number of tokens to allow */
+  limit: number;
+  /** Optional encoding to use (defaults to o200k_base which is used by gpt-4o) */
+  encoding?: TiktokenBPE;
+}
+
+/**
+ * Limits the total number of tokens in the messages.
+ * Uses js-tiktoken with o200k_base encoding by default for accurate token counting with modern models.
+ */
+export class TokenLimiter extends MemoryProcessor {
+  private encoder: Tiktoken;
+  private maxTokens: number;
+
+  // Token overheads per OpenAI's documentation
+  // See: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken#6-counting-tokens-for-chat-completions-api-calls
+  // Every message follows <|start|>{role/name}\n{content}<|end|>
+  public TOKENS_PER_MESSAGE = 3; // tokens added for each message (start & end tokens)
+  public TOKENS_PER_TOOL = 2; // empirical adjustment for tool calls
+  public TOKENS_PER_CONVERSATION = 25; // fixed overhead for the conversation
+
+  /**
+   * Create a token limiter for messages.
+   * @param options Either a number (token limit) or a configuration object
+   */
+  constructor(options: number | TokenLimiterOptions) {
+    super({
+      name: 'TokenLimiter',
+    });
+
+    if (typeof options === 'number') {
+      // Simple number format - just the token limit with default encoding
+      this.maxTokens = options;
+      this.encoder = new Tiktoken(o200k_base);
+    } else {
+      // Object format with limit and optional encoding
+      this.maxTokens = options.limit;
+      this.encoder = new Tiktoken(options.encoding || o200k_base);
+    }
+  }
+
+  process(
+    messages: CoreMessage[],
+    { systemMessage, memorySystemMessage, newMessages }: MemoryProcessorOpts = {},
+  ): CoreMessage[] {
+    // Messages are already chronologically ordered - take most recent ones up to the token limit
+    let totalTokens = 0;
+
+    // Start with the conversation overhead
+    totalTokens += this.TOKENS_PER_CONVERSATION;
+
+    if (systemMessage) {
+      totalTokens += this.countTokens(systemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE; // Add message overhead for system message
+    }
+
+    if (memorySystemMessage) {
+      totalTokens += this.countTokens(memorySystemMessage);
+      totalTokens += this.TOKENS_PER_MESSAGE; // Add message overhead for memory system message
+    }
+
+    const allMessages = [...messages, ...(newMessages || [])];
+
+    const result: CoreMessage[] = [];
+
+    // Process messages in reverse (newest first)
+    for (let i = allMessages.length - 1; i >= 0; i--) {
+      const message = allMessages[i];
+
+      // Skip undefined messages (shouldn't happen, but TypeScript is concerned)
+      if (!message) continue;
+
+      const messageTokens = this.countTokens(message);
+
+      if (totalTokens + messageTokens <= this.maxTokens) {
+        // Insert at the beginning to maintain chronological order
+        result.unshift(message);
+        totalTokens += messageTokens;
+      } else {
+        this.logger.info(
+          `filtering ${allMessages.length - result.length}/${allMessages.length} messages, token limit of ${this.maxTokens} exceeded`,
+        );
+        // If we can't fit the message, we stop
+        break;
+      }
+    }
+
+    return result;
+  }
+
+  public countTokens(message: string | CoreMessage): number {
+    if (typeof message === `string`) {
+      return this.encoder.encode(message).length;
+    }
+
+    let tokenString = message.role;
+
+    if (typeof message.content === 'string') {
+      tokenString += message.content;
+    } else if (Array.isArray(message.content)) {
+      // Calculate tokens for each content part
+      for (const part of message.content) {
+        tokenString += part.type;
+        if (part.type === 'text') {
+          tokenString += part.text;
+        } else if (part.type === 'tool-call') {
+          tokenString += part.toolName as any;
+          if (part.args) {
+            tokenString += typeof part.args === 'string' ? part.args : JSON.stringify(part.args);
+          }
+        } else if (part.type === 'tool-result') {
+          // Token cost for result if present
+          if (part.result !== undefined) {
+            tokenString += typeof part.result === 'string' ? part.result : JSON.stringify(part.result);
+          }
+        } else {
+          tokenString += JSON.stringify(part);
+        }
+      }
+    }
+
+    // Ensure we account for message formatting tokens
+    // See: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken#6-counting-tokens-for-chat-completions-api-calls
+    const messageOverhead = this.TOKENS_PER_MESSAGE;
+
+    // Count tool calls for additional overhead
+    let toolOverhead = 0;
+    if (Array.isArray(message.content)) {
+      for (const part of message.content) {
+        if (part.type === 'tool-call' || part.type === 'tool-result') {
+          toolOverhead += this.TOKENS_PER_TOOL;
+        }
+      }
+    }
+
+    const totalMessageOverhead = messageOverhead + toolOverhead;
+
+    return this.encoder.encode(tokenString).length + totalMessageOverhead;
+  }
+}

package/src/processors/tool-call-filter.ts

@@ -0,0 +1,77 @@
+import type { CoreMessage } from '@mastra/core';
+import { MemoryProcessor } from '@mastra/core';
+
+/**
+ * Filters out tool calls and results from messages.
+ * By default (with no arguments), excludes all tool calls and their results.
+ * Can be configured to exclude only specific tools by name.
+ */
+export class ToolCallFilter extends MemoryProcessor {
+  private exclude: string[] | 'all';
+
+  /**
+   * Create a filter for tool calls and results.
+   * @param options Configuration options
+   * @param options.exclude List of specific tool names to exclude. If not provided, all tool calls are excluded.
+   */
+  constructor(options: { exclude?: string[] } = {}) {
+    super({ name: 'ToolCallFilter' });
+    // If no options or exclude is provided, exclude all tools
+    if (!options || !options.exclude) {
+      this.exclude = 'all'; // Exclude all tools
+    } else {
+      // Exclude specific tools
+      this.exclude = Array.isArray(options.exclude) ? options.exclude : [];
+    }
+  }
+
+  process(messages: CoreMessage[]): CoreMessage[] {
+    // Case 1: Exclude all tool calls and tool results
+    if (this.exclude === 'all') {
+      return messages.filter(message => {
+        if (Array.isArray(message.content)) {
+          return !message.content.some(part => part.type === 'tool-call' || part.type === 'tool-result');
+        }
+        return true;
+      });
+    }
+
+    // Case 2: Exclude specific tools by name
+    if (this.exclude.length > 0) {
+      // Single pass approach - track excluded tool call IDs while filtering
+      const excludedToolCallIds = new Set<string>();
+
+      return messages.filter(message => {
+        if (!Array.isArray(message.content)) return true;
+
+        // For assistant messages, check for excluded tool calls and track their IDs
+        if (message.role === 'assistant') {
+          let shouldExclude = false;
+
+          for (const part of message.content) {
+            if (part.type === 'tool-call' && this.exclude.includes(part.toolName)) {
+              excludedToolCallIds.add(part.toolCallId);
+              shouldExclude = true;
+            }
+          }
+
+          return !shouldExclude;
+        }
+
+        // For tool messages, filter out results for excluded tool calls
+        if (message.role === 'tool') {
+          const shouldExclude = message.content.some(
+            part => part.type === 'tool-result' && excludedToolCallIds.has(part.toolCallId),
+          );
+
+          return !shouldExclude;
+        }
+
+        return true;
+      });
+    }
+
+    // Case 3: Empty exclude array, return original messages
+    return messages;
+  }
+}