@mastra/memory 0.2.10 → 0.2.11-alpha.2

package/.turbo/turbo-build.log

@@ -1,29 +1,29 @@
 
-> @mastra/memory@0.2.10-alpha.5 build /home/runner/work/mastra/mastra/packages/memory
+> @mastra/memory@0.2.11-alpha.2 build /home/runner/work/mastra/mastra/packages/memory
 > pnpm run check && tsup src/index.ts src/processors/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
 
 
-> @mastra/memory@0.2.10-alpha.5 check /home/runner/work/mastra/mastra/packages/memory
+> @mastra/memory@0.2.11-alpha.2 check /home/runner/work/mastra/mastra/packages/memory
 > tsc --noEmit
 
 CLI Building entry: src/index.ts, src/processors/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.4.0
 TSC Build start
-TSC ⚡️ Build success in 9816ms
+TSC ⚡️ Build success in 10363ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/memory/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.8.2
 Writing package typings: /home/runner/work/mastra/mastra/packages/memory/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 9724ms
+DTS ⚡️ Build success in 12416ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-ESM dist/index.js            15.44 KB
-ESM dist/processors/index.js 5.33 KB
-ESM ⚡️ Build success in 696ms
-CJS dist/index.cjs            15.71 KB
+CJS dist/index.cjs            17.58 KB
 CJS dist/processors/index.cjs 5.54 KB
-CJS ⚡️ Build success in 696ms
+CJS ⚡️ Build success in 946ms
+ESM dist/index.js            17.39 KB
+ESM dist/processors/index.js 5.33 KB
+ESM ⚡️ Build success in 950ms

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @mastra/memory
 
+## 0.2.11-alpha.2
+
+### Patch Changes
+
+- 5c6825c: [MASTRA-2782] removed tiktoken from memory chunktext
+
+## 0.2.11-alpha.1
+
+### Patch Changes
+
+- 6f92295: Fixed an issue where some user messages and llm messages would have the exact same createdAt date, leading to incorrect message ordering. Added a fix for new messages as well as any that were saved before the fix in the wrong order
+- Updated dependencies [8a8a73b]
+- Updated dependencies [6f92295]
+  - @mastra/core@0.8.4-alpha.1
+
+## 0.2.11-alpha.0
+
+### Patch Changes
+
+- Updated dependencies [03f3cd0]
+  - @mastra/core@0.8.4-alpha.0
+
 ## 0.2.10
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -76,6 +76,12 @@ export declare class Memory extends MastraMemory {
     getTools(config?: MemoryConfig): Record<string, CoreTool>;
 }
 
+/**
+ * Self-heals message ordering to ensure tool calls are directly before their matching tool results.
+ * This is needed due to a bug where messages were saved in the wrong order. That bug is fixed, but this code ensures any tool calls saved in the wrong order in the past will still be usable now.
+ */
+export declare function reorderToolCallsAndResults(messages: MessageType[]): MessageType[];
+
 /**
  * Limits the total number of tokens in the messages.
  * Uses js-tiktoken with o200k_base encoding by default for accurate token counting with modern models.

package/dist/_tsup-dts-rollup.d.ts

@@ -76,6 +76,12 @@ export declare class Memory extends MastraMemory {
     getTools(config?: MemoryConfig): Record<string, CoreTool>;
 }
 
+/**
+ * Self-heals message ordering to ensure tool calls are directly before their matching tool results.
+ * This is needed due to a bug where messages were saved in the wrong order. That bug is fixed, but this code ensures any tool calls saved in the wrong order in the past will still be usable now.
+ */
+export declare function reorderToolCallsAndResults(messages: MessageType[]): MessageType[];
+
 /**
  * Limits the total number of tokens in the messages.
  * Uses js-tiktoken with o200k_base encoding by default for accurate token counting with modern models.

package/dist/index.cjs

@@ -3,14 +3,11 @@
 var core = require('@mastra/core');
 var memory = require('@mastra/core/memory');
 var ai = require('ai');
-var lite = require('js-tiktoken/lite');
-var o200k_base = require('js-tiktoken/ranks/o200k_base');
 var xxhash = require('xxhash-wasm');
 var zod = require('zod');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
-var o200k_base__default = /*#__PURE__*/_interopDefault(o200k_base);
 var xxhash__default = /*#__PURE__*/_interopDefault(xxhash);
 
 // src/index.ts
@@ -41,8 +38,50 @@ var updateWorkingMemoryTool = {
   }
 };
 
+// src/utils/index.ts
+var isToolCallWithId = (message, targetToolCallId) => {
+  if (!message || !Array.isArray(message.content)) return false;
+  return message.content.some(
+    (part) => part && typeof part === "object" && "type" in part && part.type === "tool-call" && "toolCallId" in part && part.toolCallId === targetToolCallId
+  );
+};
+var getToolResultIndexById = (id, results) => results.findIndex((message) => {
+  if (!Array.isArray(message?.content)) return false;
+  return message.content.some(
+    (part) => part && typeof part === "object" && "type" in part && part.type === "tool-result" && "toolCallId" in part && part.toolCallId === id
+  );
+});
+function reorderToolCallsAndResults(messages) {
+  if (!messages.length) return messages;
+  const results = [...messages];
+  const toolCallIds = /* @__PURE__ */ new Set();
+  for (const message of results) {
+    if (!Array.isArray(message.content)) continue;
+    for (const part of message.content) {
+      if (part && typeof part === "object" && "type" in part && part.type === "tool-result" && "toolCallId" in part && part.toolCallId) {
+        toolCallIds.add(part.toolCallId);
+      }
+    }
+  }
+  for (const toolCallId of toolCallIds) {
+    const resultIndex = getToolResultIndexById(toolCallId, results);
+    const oneMessagePrev = results[resultIndex - 1];
+    if (isToolCallWithId(oneMessagePrev, toolCallId)) {
+      continue;
+    }
+    const toolCallIndex = results.findIndex((message) => isToolCallWithId(message, toolCallId));
+    if (toolCallIndex !== -1 && toolCallIndex !== resultIndex - 1) {
+      const toolCall = results[toolCallIndex];
+      if (!toolCall) continue;
+      results.splice(toolCallIndex, 1);
+      results.splice(getToolResultIndexById(toolCallId, results), 0, toolCall);
+    }
+  }
+  return results;
+}
+
 // src/index.ts
-var encoder = new lite.Tiktoken(o200k_base__default.default);
+var CHARS_PER_TOKEN = 4;
 var Memory = class extends memory.MastraMemory {
   constructor(config = {}) {
     super({ name: "Memory", ...config });
@@ -123,8 +162,10 @@ var Memory = class extends memory.MastraMemory {
       },
       threadConfig: config
     });
-    const messages = this.parseMessages(rawMessages);
-    const uiMessages = this.convertToUIMessages(rawMessages);
+    const orderedByDate = rawMessages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+    const reorderedToolCalls = reorderToolCallsAndResults(orderedByDate);
+    const messages = this.parseMessages(reorderedToolCalls);
+    const uiMessages = this.convertToUIMessages(reorderedToolCalls);
     return { messages, uiMessages };
   }
   async rememberMessages({
@@ -193,19 +234,22 @@ var Memory = class extends memory.MastraMemory {
   async deleteThread(threadId) {
     await this.storage.__deleteThread({ threadId });
   }
-  chunkText(text, size = 4096) {
-    const tokens = encoder.encode(text);
+  chunkText(text, tokenSize = 4096) {
+    const charSize = tokenSize * CHARS_PER_TOKEN;
     const chunks = [];
-    let currentChunk = [];
-    for (const token of tokens) {
-      currentChunk.push(token);
-      if (currentChunk.length >= size) {
-        chunks.push(encoder.decode(currentChunk));
-        currentChunk = [];
+    let currentChunk = "";
+    const words = text.split(/\s+/);
+    for (const word of words) {
+      const wordWithSpace = currentChunk ? " " + word : word;
+      if (currentChunk.length + wordWithSpace.length > charSize) {
+        chunks.push(currentChunk);
+        currentChunk = word;
+      } else {
+        currentChunk += wordWithSpace;
       }
     }
-    if (currentChunk.length > 0) {
-      chunks.push(encoder.decode(currentChunk));
+    if (currentChunk) {
+      chunks.push(currentChunk);
     }
     return chunks;
   }

package/dist/index.js

@@ -1,8 +1,6 @@
 import { deepMerge } from '@mastra/core';
 import { MastraMemory } from '@mastra/core/memory';
 import { embedMany } from 'ai';
-import { Tiktoken } from 'js-tiktoken/lite';
-import o200k_base from 'js-tiktoken/ranks/o200k_base';
 import xxhash from 'xxhash-wasm';
 import { z } from 'zod';
 
@@ -34,8 +32,50 @@ var updateWorkingMemoryTool = {
   }
 };
 
+// src/utils/index.ts
+var isToolCallWithId = (message, targetToolCallId) => {
+  if (!message || !Array.isArray(message.content)) return false;
+  return message.content.some(
+    (part) => part && typeof part === "object" && "type" in part && part.type === "tool-call" && "toolCallId" in part && part.toolCallId === targetToolCallId
+  );
+};
+var getToolResultIndexById = (id, results) => results.findIndex((message) => {
+  if (!Array.isArray(message?.content)) return false;
+  return message.content.some(
+    (part) => part && typeof part === "object" && "type" in part && part.type === "tool-result" && "toolCallId" in part && part.toolCallId === id
+  );
+});
+function reorderToolCallsAndResults(messages) {
+  if (!messages.length) return messages;
+  const results = [...messages];
+  const toolCallIds = /* @__PURE__ */ new Set();
+  for (const message of results) {
+    if (!Array.isArray(message.content)) continue;
+    for (const part of message.content) {
+      if (part && typeof part === "object" && "type" in part && part.type === "tool-result" && "toolCallId" in part && part.toolCallId) {
+        toolCallIds.add(part.toolCallId);
+      }
+    }
+  }
+  for (const toolCallId of toolCallIds) {
+    const resultIndex = getToolResultIndexById(toolCallId, results);
+    const oneMessagePrev = results[resultIndex - 1];
+    if (isToolCallWithId(oneMessagePrev, toolCallId)) {
+      continue;
+    }
+    const toolCallIndex = results.findIndex((message) => isToolCallWithId(message, toolCallId));
+    if (toolCallIndex !== -1 && toolCallIndex !== resultIndex - 1) {
+      const toolCall = results[toolCallIndex];
+      if (!toolCall) continue;
+      results.splice(toolCallIndex, 1);
+      results.splice(getToolResultIndexById(toolCallId, results), 0, toolCall);
+    }
+  }
+  return results;
+}
+
 // src/index.ts
-var encoder = new Tiktoken(o200k_base);
+var CHARS_PER_TOKEN = 4;
 var Memory = class extends MastraMemory {
   constructor(config = {}) {
     super({ name: "Memory", ...config });
@@ -116,8 +156,10 @@ var Memory = class extends MastraMemory {
       },
       threadConfig: config
     });
-    const messages = this.parseMessages(rawMessages);
-    const uiMessages = this.convertToUIMessages(rawMessages);
+    const orderedByDate = rawMessages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+    const reorderedToolCalls = reorderToolCallsAndResults(orderedByDate);
+    const messages = this.parseMessages(reorderedToolCalls);
+    const uiMessages = this.convertToUIMessages(reorderedToolCalls);
     return { messages, uiMessages };
   }
   async rememberMessages({
@@ -186,19 +228,22 @@ var Memory = class extends MastraMemory {
   async deleteThread(threadId) {
     await this.storage.__deleteThread({ threadId });
   }
-  chunkText(text, size = 4096) {
-    const tokens = encoder.encode(text);
+  chunkText(text, tokenSize = 4096) {
+    const charSize = tokenSize * CHARS_PER_TOKEN;
     const chunks = [];
-    let currentChunk = [];
-    for (const token of tokens) {
-      currentChunk.push(token);
-      if (currentChunk.length >= size) {
-        chunks.push(encoder.decode(currentChunk));
-        currentChunk = [];
+    let currentChunk = "";
+    const words = text.split(/\s+/);
+    for (const word of words) {
+      const wordWithSpace = currentChunk ? " " + word : word;
+      if (currentChunk.length + wordWithSpace.length > charSize) {
+        chunks.push(currentChunk);
+        currentChunk = word;
+      } else {
+        currentChunk += wordWithSpace;
       }
     }
-    if (currentChunk.length > 0) {
-      chunks.push(encoder.decode(currentChunk));
+    if (currentChunk) {
+      chunks.push(currentChunk);
     }
     return chunks;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mastra/memory",
-  "version": "0.2.10",
+  "version": "0.2.11-alpha.2",
   "description": "",
   "type": "module",
   "main": "./dist/index.js",
@@ -41,7 +41,7 @@
     "redis": "^4.7.0",
     "xxhash-wasm": "^1.1.0",
     "zod": "^3.24.2",
-    "@mastra/core": "^0.8.3"
+    "@mastra/core": "^0.8.4-alpha.1"
   },
   "devDependencies": {
     "@ai-sdk/openai": "^1.3.3",

package/src/index.ts

@@ -4,12 +4,13 @@ import { MastraMemory } from '@mastra/core/memory';
 import type { MessageType, MemoryConfig, SharedMemoryConfig, StorageThreadType } from '@mastra/core/memory';
 import type { StorageGetMessagesArg } from '@mastra/core/storage';
 import { embedMany } from 'ai';
-import { Tiktoken } from 'js-tiktoken/lite';
-import o200k_base from 'js-tiktoken/ranks/o200k_base';
+
 import xxhash from 'xxhash-wasm';
 import { updateWorkingMemoryTool } from './tools/working-memory';
+import { reorderToolCallsAndResults } from './utils';
 
-const encoder = new Tiktoken(o200k_base);
+// Average characters per token based on OpenAI's tokenization
+const CHARS_PER_TOKEN = 4;
 
 /**
  * Concrete implementation of MastraMemory that adds support for thread configuration
@@ -126,9 +127,14 @@ export class Memory extends MastraMemory {
       threadConfig: config,
     });
 
+    // First sort messages by date
+    const orderedByDate = rawMessages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+    // Then reorder tool calls to be directly before their results
+    const reorderedToolCalls = reorderToolCallsAndResults(orderedByDate);
+
     // Parse and convert messages
-    const messages = this.parseMessages(rawMessages);
-    const uiMessages = this.convertToUIMessages(rawMessages);
+    const messages = this.parseMessages(reorderedToolCalls);
+    const uiMessages = this.convertToUIMessages(reorderedToolCalls);
 
     return { messages, uiMessages };
   }
@@ -227,24 +233,31 @@ export class Memory extends MastraMemory {
     await this.storage.__deleteThread({ threadId });
   }
 
-  private chunkText(text: string, size = 4096) {
-    const tokens = encoder.encode(text);
+  private chunkText(text: string, tokenSize = 4096) {
+    // Convert token size to character size with some buffer
+    const charSize = tokenSize * CHARS_PER_TOKEN;
     const chunks: string[] = [];
-    let currentChunk: number[] = [];
+    let currentChunk = '';
+
+    // Split text into words to avoid breaking words
+    const words = text.split(/\s+/);
 
-    for (const token of tokens) {
-      currentChunk.push(token);
+    for (const word of words) {
+      // Add space before word unless it's the first word in the chunk
+      const wordWithSpace = currentChunk ? ' ' + word : word;
 
-      // If current chunk reaches size limit, add it to chunks and start a new one
-      if (currentChunk.length >= size) {
-        chunks.push(encoder.decode(currentChunk));
-        currentChunk = [];
+      // If adding this word would exceed the chunk size, start a new chunk
+      if (currentChunk.length + wordWithSpace.length > charSize) {
+        chunks.push(currentChunk);
+        currentChunk = word;
+      } else {
+        currentChunk += wordWithSpace;
       }
     }
 
-    // Add any remaining tokens as the final chunk
-    if (currentChunk.length > 0) {
-      chunks.push(encoder.decode(currentChunk));
+    // Add the final chunk if not empty
+    if (currentChunk) {
+      chunks.push(currentChunk);
     }
 
     return chunks;

package/src/utils/index.ts

@@ -0,0 +1,88 @@
+import type { MessageType } from '@mastra/core/memory';
+
+const isToolCallWithId = (message: MessageType | undefined, targetToolCallId: string): boolean => {
+  if (!message || !Array.isArray(message.content)) return false;
+  return message.content.some(
+    part =>
+      part &&
+      typeof part === 'object' &&
+      'type' in part &&
+      part.type === 'tool-call' &&
+      'toolCallId' in part &&
+      part.toolCallId === targetToolCallId,
+  );
+};
+
+const getToolResultIndexById = (id: string, results: MessageType[]) =>
+  results.findIndex(message => {
+    if (!Array.isArray(message?.content)) return false;
+    return message.content.some(
+      part =>
+        part &&
+        typeof part === 'object' &&
+        'type' in part &&
+        part.type === 'tool-result' &&
+        'toolCallId' in part &&
+        part.toolCallId === id,
+    );
+  });
+
+/**
+ * Self-heals message ordering to ensure tool calls are directly before their matching tool results.
+ * This is needed due to a bug where messages were saved in the wrong order. That bug is fixed, but this code ensures any tool calls saved in the wrong order in the past will still be usable now.
+ */
+export function reorderToolCallsAndResults(messages: MessageType[]): MessageType[] {
+  if (!messages.length) return messages;
+
+  // Create a copy of messages to avoid modifying the original
+  const results = [...messages];
+
+  const toolCallIds = new Set<string>();
+
+  // First loop: collect all tool result IDs in a set
+  for (const message of results) {
+    if (!Array.isArray(message.content)) continue;
+
+    for (const part of message.content) {
+      if (
+        part &&
+        typeof part === 'object' &&
+        'type' in part &&
+        part.type === 'tool-result' &&
+        'toolCallId' in part &&
+        part.toolCallId
+      ) {
+        toolCallIds.add(part.toolCallId);
+      }
+    }
+  }
+
+  // Second loop: for each tool ID, ensure tool calls come before tool results
+  for (const toolCallId of toolCallIds) {
+    // Find tool result index
+    const resultIndex = getToolResultIndexById(toolCallId, results);
+
+    // Check if tool call is at resultIndex - 1
+    const oneMessagePrev = results[resultIndex - 1];
+    if (isToolCallWithId(oneMessagePrev, toolCallId)) {
+      continue; // Tool call is already in the correct position
+    }
+
+    // Find the tool call anywhere in the array
+    const toolCallIndex = results.findIndex(message => isToolCallWithId(message, toolCallId));
+
+    if (toolCallIndex !== -1 && toolCallIndex !== resultIndex - 1) {
+      // Store the tool call message
+      const toolCall = results[toolCallIndex];
+      if (!toolCall) continue;
+
+      // Remove the tool call from its current position
+      results.splice(toolCallIndex, 1);
+
+      // Insert right before the tool result
+      results.splice(getToolResultIndexById(toolCallId, results), 0, toolCall);
+    }
+  }
+
+  return results;
+}