@mastra/memory 1.2.0 → 1.3.0-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,52 @@
 # @mastra/memory
 
+## 1.3.0-alpha.0
+
+### Minor Changes
+
+- @mastra/opencode: Add opencode plugin for Observational Memory integration ([#12925](https://github.com/mastra-ai/mastra/pull/12925))
+
+  Added standalone `observe()` API that accepts external messages directly, so integrations can trigger observation without duplicating messages into Mastra's storage.
+
+  **New exports:**
+  - `ObserveHooks` — lifecycle callbacks (`onObservationStart`, `onObservationEnd`, `onReflectionStart`, `onReflectionEnd`) for hooking into observation/reflection cycles
+  - `OBSERVATION_CONTEXT_PROMPT` — preamble that introduces the observations block
+  - `OBSERVATION_CONTEXT_INSTRUCTIONS` — rules for interpreting observations (placed after the `<observations>` block)
+  - `OBSERVATION_CONTINUATION_HINT` — behavioral guidance that prevents models from awkwardly acknowledging the memory system
+  - `getOrCreateRecord()` — now public, allows eager record initialization before the first observation cycle
+
+  ```ts
+  import { ObservationalMemory } from '@mastra/memory/processors';
+
+  const om = new ObservationalMemory({ storage, model: 'google/gemini-2.5-flash' });
+
+  // Eagerly initialize a record
+  await om.getOrCreateRecord(threadId);
+
+  // Pass messages directly with lifecycle hooks
+  await om.observe({
+    threadId,
+    messages: myMessages,
+    hooks: {
+      onObservationStart: () => console.log('Observing...'),
+      onObservationEnd: () => console.log('Done!'),
+      onReflectionStart: () => console.log('Reflecting...'),
+      onReflectionEnd: () => console.log('Reflected!'),
+    },
+  });
+  ```
+
+  **Breaking:** `observe()` now takes an object param instead of positional args. Update calls from `observe(threadId, resourceId)` to `observe({ threadId, resourceId })`.
+
+### Patch Changes
+
+- Fixed observational memory writing non-integer token counts to PostgreSQL, which caused `invalid input syntax for type integer` errors. Token counts are now correctly rounded to integers before all database writes. ([#12976](https://github.com/mastra-ai/mastra/pull/12976))
+
+- Fixed cloneThread not copying working memory to the cloned thread. Thread-scoped working memory is now properly carried over when cloning, and resource-scoped working memory is copied when the clone uses a different resourceId. ([#12833](https://github.com/mastra-ai/mastra/pull/12833))
+
+- Updated dependencies [[`7ef618f`](https://github.com/mastra-ai/mastra/commit/7ef618f3c49c27e2f6b27d7f564c557c0734325b), [`b373564`](https://github.com/mastra-ai/mastra/commit/b37356491d43b4d53067f10cb669abaf2502f218), [`927c2af`](https://github.com/mastra-ai/mastra/commit/927c2af9792286c122e04409efce0f3c804f777f), [`b896b41`](https://github.com/mastra-ai/mastra/commit/b896b41343de7fcc14442fb40fe82d189e65bbe2), [`6415277`](https://github.com/mastra-ai/mastra/commit/6415277a438faa00db2af850ead5dee25f40c428), [`0831bbb`](https://github.com/mastra-ai/mastra/commit/0831bbb5bc750c18e9b22b45f18687c964b70828), [`63f7eda`](https://github.com/mastra-ai/mastra/commit/63f7eda605eb3e0c8c35ee3912ffe7c999c69f69), [`a5b67a3`](https://github.com/mastra-ai/mastra/commit/a5b67a3589a74415feb663a55d1858324a2afde9), [`877b02c`](https://github.com/mastra-ai/mastra/commit/877b02cdbb15e199184c7f2b8f217be8d3ebada7), [`7567222`](https://github.com/mastra-ai/mastra/commit/7567222b1366f0d39980594792dd9d5060bfe2ab), [`af71458`](https://github.com/mastra-ai/mastra/commit/af71458e3b566f09c11d0e5a0a836dc818e7a24a), [`eb36bd8`](https://github.com/mastra-ai/mastra/commit/eb36bd8c52fcd6ec9674ac3b7a6412405b5983e1), [`3cbf121`](https://github.com/mastra-ai/mastra/commit/3cbf121f55418141924754a83102aade89835947)]:
+  - @mastra/core@1.4.0-alpha.0
+
 ## 1.2.0
 
 ### Minor Changes

package/dist/{chunk-5YW6JV6Y.js → chunk-F5P5HTMC.js}

@@ -1026,7 +1026,7 @@ var TokenCounter = class _TokenCounter {
     if (toolResultCount > 0) {
       overhead += toolResultCount * _TokenCounter.TOKENS_PER_MESSAGE;
     }
-    return this.encoder.encode(tokenString, "all").length + overhead;
+    return Math.round(this.encoder.encode(tokenString, "all").length + overhead);
   }
   /**
    * Count tokens in an array of messages
@@ -1280,6 +1280,17 @@ var OBSERVATIONAL_MEMORY_DEFAULTS = {
     // Start buffering at 50% of observationTokens
   }
 };
+var OBSERVATION_CONTINUATION_HINT = `This message is not from the user, the conversation history grew too long and wouldn't fit in context! Thankfully the entire conversation is stored in your memory observations. Please continue from where the observations left off. Do not refer to your "memory observations" directly, the user doesn't know about them, they are your memories! Just respond naturally as if you're remembering the conversation (you are!). Do not say "Hi there!" or "based on our previous conversation" as if the conversation is just starting, this is not a new conversation. This is an ongoing conversation, keep continuity by responding based on your memory. For example do not say "I understand. I've reviewed my memory observations", or "I remember [...]". Answer naturally following the suggestion from your memory. Note that your memory may contain a suggested first response, which you should follow.
+
+IMPORTANT: this system reminder is NOT from the user. The system placed it here as part of your memory system. This message is part of you remembering your conversation with the user.
+
+NOTE: Any messages following this system reminder are newer than your memories.`;
+var OBSERVATION_CONTEXT_PROMPT = `The following observations block contains your memory of past conversations with this user.`;
+var OBSERVATION_CONTEXT_INSTRUCTIONS = `IMPORTANT: When responding, reference specific details from these observations. Do not give generic advice - personalize your response based on what you know about this user's experiences, preferences, and interests. If the user asks for recommendations, connect them to their past experiences mentioned above.
+
+KNOWLEDGE UPDATES: When asked about current state (e.g., "where do I currently...", "what is my current..."), always prefer the MOST RECENT information. Observations include dates - if you see conflicting information, the newer observation supersedes the older one. Look for phrases like "will start", "is switching", "changed to", "moved to" as indicators that previous information has been updated.
+
+PLANNED ACTIONS: If the user stated they planned to do something (e.g., "I'm going to...", "I'm looking forward to...", "I will...") and the date they planned to do it is now in the past (check the relative time like "3 weeks ago"), assume they completed the action unless there's evidence they didn't. For example, if someone said "I'll start my new diet on Monday" and that was 2 weeks ago, assume they started the diet.`;
 var ObservationalMemory = class _ObservationalMemory {
   id = "observational-memory";
   name = "Observational Memory";
@@ -1867,6 +1878,16 @@ Async buffering is enabled by default \u2014 this opt-out is only needed when us
     const effectiveThreshold = Math.max(totalBudget - currentObservationTokens, baseThreshold);
     return Math.round(effectiveThreshold);
   }
+  /**
+   * Check whether the unobserved message tokens meet the observation threshold.
+   */
+  meetsObservationThreshold(opts) {
+    const { record, unobservedTokens, extraTokens = 0 } = opts;
+    const pendingTokens = (record.pendingMessageTokens ?? 0) + unobservedTokens + extraTokens;
+    const currentObservationTokens = record.observationTokenCount ?? 0;
+    const threshold = this.calculateDynamicThreshold(this.observationConfig.messageTokens, currentObservationTokens);
+    return pendingTokens >= threshold;
+  }
   /**
    * Get or create the Observer agent
    */
@@ -1913,7 +1934,8 @@ Async buffering is enabled by default \u2014 this opt-out is only needed when us
     };
   }
   /**
-   * Get or create the observational memory record
+   * Get or create the observational memory record.
+   * Returns the existing record if one exists, otherwise initializes a new one.
    */
   async getOrCreateRecord(threadId, resourceId) {
     const ids = this.getStorageIds(threadId, resourceId);
@@ -2582,17 +2604,13 @@ Async buffering is enabled by default \u2014 this opt-out is only needed when us
       optimized = addRelativeTimeToObservations(optimized, currentDate);
     }
     let content = `
-The following observations block contains your memory of past conversations with this user.
+${OBSERVATION_CONTEXT_PROMPT}
 
 <observations>
 ${optimized}
 </observations>
 
-IMPORTANT: When responding, reference specific details from these observations. Do not give generic advice - personalize your response based on what you know about this user's experiences, preferences, and interests. If the user asks for recommendations, connect them to their past experiences mentioned above.
-
-KNOWLEDGE UPDATES: When asked about current state (e.g., "where do I currently...", "what is my current..."), always prefer the MOST RECENT information. Observations include dates - if you see conflicting information, the newer observation supersedes the older one. Look for phrases like "will start", "is switching", "changed to", "moved to" as indicators that previous information has been updated.
-
-PLANNED ACTIONS: If the user stated they planned to do something (e.g., "I'm going to...", "I'm looking forward to...", "I will...") and the date they planned to do it is now in the past (check the relative time like "3 weeks ago"), assume they completed the action unless there's evidence they didn't. For example, if someone said "I'll start my new diet on Monday" and that was 2 weeks ago, assume they started the diet.`;
+${OBSERVATION_CONTEXT_INSTRUCTIONS}`;
     if (unobservedContextBlocks) {
       content += `
 
@@ -2856,16 +2874,22 @@ ${suggestedResponse}
       if (freshUnobservedMessages.length > 0) {
         try {
           if (this.scope === "resource" && resourceId) {
-            await this.doResourceScopedObservation(
-              freshRecord,
-              threadId,
+            await this.doResourceScopedObservation({
+              record: freshRecord,
+              currentThreadId: threadId,
               resourceId,
-              freshUnobservedMessages,
+              currentThreadMessages: freshUnobservedMessages,
               writer,
               abortSignal
-            );
+            });
           } else {
-            await this.doSynchronousObservation(freshRecord, threadId, freshUnobservedMessages, writer, abortSignal);
+            await this.doSynchronousObservation({
+              record: freshRecord,
+              threadId,
+              unobservedMessages: freshUnobservedMessages,
+              writer,
+              abortSignal
+            });
           }
           updatedRecord = await this.getOrCreateRecord(threadId, resourceId);
           const updatedTime = updatedRecord.lastObservedAt?.getTime() ?? 0;
@@ -3005,12 +3029,7 @@ ${suggestedResponse}
         parts: [
           {
             type: "text",
-            text: `<system-reminder>This message is not from the user, the conversation history grew too long and wouldn't fit in context! Thankfully the entire conversation is stored in your memory observations. Please continue from where the observations left off. Do not refer to your "memory observations" directly, the user doesn't know about them, they are your memories! Just respond naturally as if you're remembering the conversation (you are!). Do not say "Hi there!" or "based on our previous conversation" as if the conversation is just starting, this is not a new conversation. This is an ongoing conversation, keep continuity by responding based on your memory. For example do not say "I understand. I've reviewed my memory observations", or "I remember [...]". Answer naturally following the suggestion from your memory. Note that your memory may contain a suggested first response, which you should follow.
-
-IMPORTANT: this system reminder is NOT from the user. The system placed it here as part of your memory system. This message is part of you remembering your conversation with the user.
-
-NOTE: Any messages following this system reminder are newer than your memories.
-</system-reminder>`
+            text: `<system-reminder>${OBSERVATION_CONTINUATION_HINT}</system-reminder>`
           }
         ]
       },
@@ -3167,14 +3186,13 @@ NOTE: Any messages following this system reminder are newer than your memories.
             _ObservationalMemory.lastBufferedBoundary.set(bufKey, 0);
             this.storage.setBufferingObservationFlag(record.id, false, 0).catch(() => {
             });
-            await this.maybeReflect(
+            await this.maybeReflect({
               record,
-              record.observationTokenCount ?? 0,
+              observationTokens: record.observationTokenCount ?? 0,
               threadId,
               writer,
-              void 0,
               messageList
-            );
+            });
             record = await this.getOrCreateRecord(threadId, resourceId);
           }
         }
@@ -3184,7 +3202,7 @@ NOTE: Any messages following this system reminder are newer than your memories.
       const obsTokens = record.observationTokenCount ?? 0;
       if (this.shouldReflect(obsTokens)) {
         omDebug(`[OM:step0-reflect] obsTokens=${obsTokens} over reflectThreshold, triggering reflection`);
-        await this.maybeReflect(record, obsTokens, threadId, writer, void 0, messageList);
+        await this.maybeReflect({ record, observationTokens: obsTokens, threadId, writer, messageList });
         record = await this.getOrCreateRecord(threadId, resourceId);
       } else if (this.isAsyncReflectionEnabled()) {
         const lockKey = this.getLockKey(threadId, resourceId);
@@ -3524,21 +3542,35 @@ ${newThreadSection}`;
     }
     const newThreadId = threadIdMatch[1];
     const newDate = dateMatch[1];
-    const existingPattern = new RegExp(
-      `<thread id="${newThreadId}">\\s*Date:\\s*${newDate.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}([\\s\\S]*?)</thread>`
-    );
-    const existingMatch = existingObservations.match(existingPattern);
-    if (existingMatch) {
-      const newObsMatch = newThreadSection.match(/<thread id="[^"]+">[\s\S]*?Date:[^\n]*\n([\s\S]*?)\n<\/thread>/);
-      if (newObsMatch && newObsMatch[1]) {
-        const newObsContent = newObsMatch[1].trim();
-        const mergedSection = existingObservations.replace(existingPattern, (match) => {
-          const withoutClose = match.replace(/<\/thread>$/, "").trimEnd();
-          return `${withoutClose}
+    const threadOpen = `<thread id="${newThreadId}">`;
+    const threadClose = "</thread>";
+    const startIdx = existingObservations.indexOf(threadOpen);
+    let existingSection = null;
+    let existingSectionStart = -1;
+    let existingSectionEnd = -1;
+    if (startIdx !== -1) {
+      const closeIdx = existingObservations.indexOf(threadClose, startIdx);
+      if (closeIdx !== -1) {
+        existingSectionEnd = closeIdx + threadClose.length;
+        existingSectionStart = startIdx;
+        const section = existingObservations.slice(startIdx, existingSectionEnd);
+        if (section.includes(`Date: ${newDate}`) || section.includes(`Date:${newDate}`)) {
+          existingSection = section;
+        }
+      }
+    }
+    if (existingSection) {
+      const dateLineEnd = newThreadSection.indexOf("\n", newThreadSection.indexOf("Date:"));
+      const newCloseIdx = newThreadSection.lastIndexOf(threadClose);
+      if (dateLineEnd !== -1 && newCloseIdx !== -1) {
+        const newObsContent = newThreadSection.slice(dateLineEnd + 1, newCloseIdx).trim();
+        if (newObsContent) {
+          const withoutClose = existingSection.slice(0, existingSection.length - threadClose.length).trimEnd();
+          const merged = `${withoutClose}
 ${newObsContent}
-</thread>`;
-        });
-        return mergedSection;
+${threadClose}`;
+          return existingObservations.slice(0, existingSectionStart) + merged + existingObservations.slice(existingSectionEnd);
+        }
       }
     }
     return `${existingObservations}
@@ -3562,7 +3594,8 @@ ${newThreadSection}`;
   /**
    * Do synchronous observation (fallback when no buffering)
    */
-  async doSynchronousObservation(record, threadId, unobservedMessages, writer, abortSignal) {
+  async doSynchronousObservation(opts) {
+    const { record, threadId, unobservedMessages, writer, abortSignal, reflectionHooks } = opts;
     this.emitDebugEvent({
       type: "observation_triggered",
       timestamp: /* @__PURE__ */ new Date(),
@@ -3687,13 +3720,14 @@ ${result.observations}` : result.observations;
         })),
         usage: result.usage
       });
-      await this.maybeReflect(
-        { ...record, activeObservations: newObservations },
-        totalTokenCount,
+      await this.maybeReflect({
+        record: { ...record, activeObservations: newObservations },
+        observationTokens: totalTokenCount,
         threadId,
         writer,
-        abortSignal
-      );
+        abortSignal,
+        reflectionHooks
+      });
     } catch (error) {
       if (lastMessage?.id) {
         const failedMarker = this.createObservationFailedMarker({
@@ -4236,7 +4270,8 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
    * 3. Only updates lastObservedAt AFTER all threads are observed
    * 4. Only triggers reflection AFTER all threads are observed
    */
-  async doResourceScopedObservation(record, currentThreadId, resourceId, currentThreadMessages, writer, abortSignal) {
+  async doResourceScopedObservation(opts) {
+    const { record, currentThreadId, resourceId, currentThreadMessages, writer, abortSignal, reflectionHooks } = opts;
     const { threads: allThreads } = await this.storage.listThreads({ filter: { resourceId } });
     const threadMetadataMap = /* @__PURE__ */ new Map();
     for (const thread of allThreads) {
@@ -4498,13 +4533,14 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
           }
         }
       }
-      await this.maybeReflect(
-        { ...record, activeObservations: currentObservations },
-        totalTokenCount,
-        currentThreadId,
+      await this.maybeReflect({
+        record: { ...record, activeObservations: currentObservations },
+        observationTokens: totalTokenCount,
+        threadId: currentThreadId,
         writer,
-        abortSignal
-      );
+        abortSignal,
+        reflectionHooks
+      });
     } catch (error) {
       for (const [threadId, msgs] of threadsWithMessages) {
         const lastMessage = msgs[msgs.length - 1];
@@ -4575,7 +4611,8 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
    * When async buffering is enabled via `bufferTokens`, reflection is triggered
    * in the background at intervals, and activated when the threshold is reached.
    */
-  async maybeReflect(record, observationTokens, _threadId, writer, abortSignal, messageList) {
+  async maybeReflect(opts) {
+    const { record, observationTokens, writer, abortSignal, messageList, reflectionHooks } = opts;
     const lockKey = this.getLockKey(record.threadId, record.resourceId);
     const reflectThreshold = this.getMaxThreshold(this.reflectionConfig.observationTokens);
     if (this.isAsyncReflectionEnabled() && observationTokens < reflectThreshold) {
@@ -4611,11 +4648,12 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
         return;
       }
     }
+    reflectionHooks?.onReflectionStart?.();
     await this.storage.setReflectingFlag(record.id, true);
     registerOp(record.id, "reflecting");
     const cycleId = crypto.randomUUID();
     const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-    const threadId = _threadId ?? "unknown";
+    const threadId = opts.threadId ?? "unknown";
     if (writer) {
       const startMarker = this.createObservationStartMarker({
         cycleId,
@@ -4701,26 +4739,45 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
       omError("[OM] Reflection failed", error);
     } finally {
       await this.storage.setReflectingFlag(record.id, false);
+      reflectionHooks?.onReflectionEnd?.();
       unregisterOp(record.id, "reflecting");
     }
   }
   /**
    * Manually trigger observation.
+   *
+   * When `messages` is provided, those are used directly (filtered for unobserved)
+   * instead of reading from storage. This allows external systems (e.g., opencode)
+   * to pass conversation messages without duplicating them into Mastra's DB.
    */
-  async observe(threadId, resourceId, _prompt) {
+  async observe(opts) {
+    const { threadId, resourceId, messages, hooks } = opts;
     const lockKey = this.getLockKey(threadId, resourceId);
+    const reflectionHooks = hooks ? { onReflectionStart: hooks.onReflectionStart, onReflectionEnd: hooks.onReflectionEnd } : void 0;
     await this.withLock(lockKey, async () => {
       const freshRecord = await this.getOrCreateRecord(threadId, resourceId);
       if (this.scope === "resource" && resourceId) {
-        await this.doResourceScopedObservation(
-          freshRecord,
-          threadId,
-          resourceId,
-          []
-          // no in-flight messages — everything is already in the DB
-        );
+        const currentMessages = messages ?? [];
+        if (!this.meetsObservationThreshold({
+          record: freshRecord,
+          unobservedTokens: this.tokenCounter.countMessages(currentMessages)
+        })) {
+          return;
+        }
+        hooks?.onObservationStart?.();
+        try {
+          await this.doResourceScopedObservation({
+            record: freshRecord,
+            currentThreadId: threadId,
+            resourceId,
+            currentThreadMessages: currentMessages,
+            reflectionHooks
+          });
+        } finally {
+          hooks?.onObservationEnd?.();
+        }
       } else {
-        const unobservedMessages = await this.loadUnobservedMessages(
+        const unobservedMessages = messages ? this.getUnobservedMessages(messages, freshRecord) : await this.loadUnobservedMessages(
           threadId,
           resourceId,
           freshRecord.lastObservedAt ? new Date(freshRecord.lastObservedAt) : void 0
@@ -4728,7 +4785,18 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
         if (unobservedMessages.length === 0) {
           return;
         }
-        await this.doSynchronousObservation(freshRecord, threadId, unobservedMessages);
+        if (!this.meetsObservationThreshold({
+          record: freshRecord,
+          unobservedTokens: this.tokenCounter.countMessages(unobservedMessages)
+        })) {
+          return;
+        }
+        hooks?.onObservationStart?.();
+        try {
+          await this.doSynchronousObservation({ record: freshRecord, threadId, unobservedMessages, reflectionHooks });
+        } finally {
+          hooks?.onObservationEnd?.();
+        }
       }
     });
   }
@@ -4820,6 +4888,6 @@ ${unreflectedContent}` : freshRecord.bufferedReflection;
   }
 };
 
-export { OBSERVATIONAL_MEMORY_DEFAULTS, OBSERVER_SYSTEM_PROMPT, ObservationalMemory, TokenCounter, buildObserverPrompt, buildObserverSystemPrompt, extractCurrentTask, formatMessagesForObserver, hasCurrentTaskSection, optimizeObservationsForContext, parseObserverOutput };
-//# sourceMappingURL=chunk-5YW6JV6Y.js.map
-//# sourceMappingURL=chunk-5YW6JV6Y.js.map
+export { OBSERVATIONAL_MEMORY_DEFAULTS, OBSERVATION_CONTEXT_INSTRUCTIONS, OBSERVATION_CONTEXT_PROMPT, OBSERVATION_CONTINUATION_HINT, OBSERVER_SYSTEM_PROMPT, ObservationalMemory, TokenCounter, buildObserverPrompt, buildObserverSystemPrompt, extractCurrentTask, formatMessagesForObserver, hasCurrentTaskSection, optimizeObservationsForContext, parseObserverOutput };
+//# sourceMappingURL=chunk-F5P5HTMC.js.map
+//# sourceMappingURL=chunk-F5P5HTMC.js.map