@mastra/memory 1.38.0-alpha.0 → 1.38.0

package/dist/docs/SKILL.md

@@ -3,7 +3,7 @@ name: mastra-memory
 description: Documentation for @mastra/memory. Use when working with @mastra/memory APIs, configuration, or implementation.
 metadata:
   package: "@mastra/memory"
-  version: "1.38.0-alpha.0"
+  version: "1.38.0"
 ---
 
 ## When to use

package/dist/docs/assets/SOURCE_MAP.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.38.0-alpha.0",
+  "version": "1.38.0",
   "package": "@mastra/memory",
   "exports": {},
   "modules": {

package/dist/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_src = require("./src-BfhwVNr7.cjs");
+const require_src = require("./src-DZmiwOWq.cjs");
 let _mastra_core_processors = require("@mastra/core/processors");
 exports.AUTO_MODEL_BY_PROVIDER = require_src.AUTO_MODEL_BY_PROVIDER;
 exports.Extractor = require_src.Extractor;

package/dist/index.js

@@ -1,2 +1,2 @@
-import { C as WorkingMemoryExtractor, E as summarizeConversation, G as Subconscious, K as KnowledgeSemanticIndexCoordinator, T as SUMMARIZE_THREAD_DEFAULTS, Y as ModelByInputTokens, a as extractWorkingMemoryContent, c as getObservationsAsOf, d as AUTO_MODEL_BY_PROVIDER, et as Extractor, f as resolveAutoModelId, h as WorkingMemoryStateProcessor, i as WorkingMemory, m as WORKING_MEMORY_STATE_PROCESSOR_ID, n as MessageHistory, o as extractWorkingMemoryTags, p as WORKING_MEMORY_STATE_ID, q as StaleKnowledgeSemanticIndexError, r as SemanticRecall, s as removeWorkingMemoryTags, t as Memory, w as deepMergeWorkingMemory } from "./src-CnLuefUv.js";
+import { C as WorkingMemoryExtractor, E as summarizeConversation, G as Subconscious, K as KnowledgeSemanticIndexCoordinator, T as SUMMARIZE_THREAD_DEFAULTS, Y as ModelByInputTokens, a as extractWorkingMemoryContent, c as getObservationsAsOf, d as AUTO_MODEL_BY_PROVIDER, et as Extractor, f as resolveAutoModelId, h as WorkingMemoryStateProcessor, i as WorkingMemory, m as WORKING_MEMORY_STATE_PROCESSOR_ID, n as MessageHistory, o as extractWorkingMemoryTags, p as WORKING_MEMORY_STATE_ID, q as StaleKnowledgeSemanticIndexError, r as SemanticRecall, s as removeWorkingMemoryTags, t as Memory, w as deepMergeWorkingMemory } from "./src-CTyZJ6A-.js";
 export { AUTO_MODEL_BY_PROVIDER, Extractor, KnowledgeSemanticIndexCoordinator, Memory, MessageHistory, ModelByInputTokens, SUMMARIZE_THREAD_DEFAULTS, SemanticRecall, StaleKnowledgeSemanticIndexError, Subconscious, WORKING_MEMORY_STATE_ID, WORKING_MEMORY_STATE_PROCESSOR_ID, WorkingMemory, WorkingMemoryExtractor, WorkingMemoryStateProcessor, deepMergeWorkingMemory, extractWorkingMemoryContent, extractWorkingMemoryTags, getObservationsAsOf, removeWorkingMemoryTags, resolveAutoModelId, summarizeConversation };

package/dist/processors/index.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_src = require("../src-BfhwVNr7.cjs");
+const require_src = require("../src-DZmiwOWq.cjs");
 exports.AUTO_MODEL_BY_PROVIDER = require_src.AUTO_MODEL_BY_PROVIDER;
 exports.Extractor = require_src.Extractor;
 exports.KnowledgeSemanticIndexCoordinator = require_src.KnowledgeSemanticIndexCoordinator;

package/dist/processors/index.js

@@ -1,2 +1,2 @@
-import { $ as renderSubconsciousActivity, A as buildObserverSystemPrompt, B as OBSERVATION_CONTEXT_INSTRUCTIONS, C as WorkingMemoryExtractor, D as TokenCounter, E as summarizeConversation, F as parseObserverOutput, G as Subconscious, H as OBSERVATION_CONTEXT_PROMPT_THREAD, I as injectAnchorIds, J as SubconsciousRemindExtractor, K as KnowledgeSemanticIndexCoordinator, L as parseAnchorId, M as formatMessagesForObserver, N as hasCurrentTaskSection, O as OBSERVER_SYSTEM_PROMPT, P as optimizeObservationsForContext, Q as publishSubconsciousActivity, R as stripEphemeralAnchorIds, S as wrapInObservationGroup, U as OBSERVATION_CONTINUATION_HINT, V as OBSERVATION_CONTEXT_PROMPT, W as getObservationContextPrompt, X as SUBCONSCIOUS_ACTIVITY_STATE_ID, Y as ModelByInputTokens, Z as buildSubconsciousActivitySnapshot, _ as deriveObservationGroupProvenance, b as renderObservationGroupsForReflection, c as getObservationsAsOf, d as AUTO_MODEL_BY_PROVIDER, et as Extractor, f as resolveAutoModelId, g as combineObservationGroupRanges, j as extractCurrentTask, k as buildObserverPrompt, l as ObservationalMemoryProcessor, q as StaleKnowledgeSemanticIndexError, u as ObservationalMemory, v as parseObservationGroups, x as stripObservationGroups, y as reconcileObservationGroupsFromReflection, z as OBSERVATIONAL_MEMORY_DEFAULTS } from "../src-CnLuefUv.js";
+import { $ as renderSubconsciousActivity, A as buildObserverSystemPrompt, B as OBSERVATION_CONTEXT_INSTRUCTIONS, C as WorkingMemoryExtractor, D as TokenCounter, E as summarizeConversation, F as parseObserverOutput, G as Subconscious, H as OBSERVATION_CONTEXT_PROMPT_THREAD, I as injectAnchorIds, J as SubconsciousRemindExtractor, K as KnowledgeSemanticIndexCoordinator, L as parseAnchorId, M as formatMessagesForObserver, N as hasCurrentTaskSection, O as OBSERVER_SYSTEM_PROMPT, P as optimizeObservationsForContext, Q as publishSubconsciousActivity, R as stripEphemeralAnchorIds, S as wrapInObservationGroup, U as OBSERVATION_CONTINUATION_HINT, V as OBSERVATION_CONTEXT_PROMPT, W as getObservationContextPrompt, X as SUBCONSCIOUS_ACTIVITY_STATE_ID, Y as ModelByInputTokens, Z as buildSubconsciousActivitySnapshot, _ as deriveObservationGroupProvenance, b as renderObservationGroupsForReflection, c as getObservationsAsOf, d as AUTO_MODEL_BY_PROVIDER, et as Extractor, f as resolveAutoModelId, g as combineObservationGroupRanges, j as extractCurrentTask, k as buildObserverPrompt, l as ObservationalMemoryProcessor, q as StaleKnowledgeSemanticIndexError, u as ObservationalMemory, v as parseObservationGroups, x as stripObservationGroups, y as reconcileObservationGroupsFromReflection, z as OBSERVATIONAL_MEMORY_DEFAULTS } from "../src-CTyZJ6A-.js";
 export { AUTO_MODEL_BY_PROVIDER, Extractor, KnowledgeSemanticIndexCoordinator, ModelByInputTokens, OBSERVATIONAL_MEMORY_DEFAULTS, OBSERVATION_CONTEXT_INSTRUCTIONS, OBSERVATION_CONTEXT_PROMPT, OBSERVATION_CONTEXT_PROMPT_THREAD, OBSERVATION_CONTINUATION_HINT, OBSERVER_SYSTEM_PROMPT, ObservationalMemory, ObservationalMemoryProcessor, SUBCONSCIOUS_ACTIVITY_STATE_ID, StaleKnowledgeSemanticIndexError, Subconscious, SubconsciousRemindExtractor, TokenCounter, WorkingMemoryExtractor, buildObserverPrompt, buildObserverSystemPrompt, buildSubconsciousActivitySnapshot, combineObservationGroupRanges, deriveObservationGroupProvenance, extractCurrentTask, formatMessagesForObserver, getObservationContextPrompt, getObservationsAsOf, hasCurrentTaskSection, injectAnchorIds, optimizeObservationsForContext, parseAnchorId, parseObservationGroups, parseObserverOutput, publishSubconsciousActivity, reconcileObservationGroupsFromReflection, renderObservationGroupsForReflection, renderSubconsciousActivity, resolveAutoModelId, stripEphemeralAnchorIds, stripObservationGroups, summarizeConversation, wrapInObservationGroup };

package/dist/{src-CnLuefUv.js → src-CTyZJ6A-.js}

@@ -19804,21 +19804,22 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
-//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.6.0/394289cb1bed8c542941f77748eea1a31bd3419018d0fb2ac7029ece2458cdd1/node_modules/tokenx/dist/index.mjs
 const PATTERNS = {
 	whitespace: /^\s+$/,
-	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
-	numeric: /^\d+(?:[.,]\d+)*$/,
-	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
-	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+	structuredWhitespace: /\n\s/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/
 };
-const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
 const DEFAULT_CHARS_PER_TOKEN = 6;
 const SHORT_TOKEN_THRESHOLD = 3;
+const KANA_CHARS_PER_TOKEN = 1.35;
 const DEFAULT_LANGUAGE_CONFIGS = [
 	{
 		pattern: /[äöüßẞ]/i,
-		averageCharsPerToken: 3
+		averageCharsPerToken: 2.6
 	},
 	{
 		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
@@ -19826,39 +19827,75 @@ const DEFAULT_LANGUAGE_CONFIGS = [
 	},
 	{
 		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
-		averageCharsPerToken: 3.5
+		averageCharsPerToken: 2.5
+	},
+	{
+		pattern: /[\u0430-\u044F\u0451]/i,
+		averageCharsPerToken: 4
+	},
+	{
+		pattern: /[\u03AC-\u03CE]/i,
+		averageCharsPerToken: 2.75
+	},
+	{
+		pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
+		averageCharsPerToken: .75
 	}
 ];
 /**
-* Estimates the number of tokens in a text string using heuristic rules.
+* Walks a text as (segment, estimated token count) pairs. The segments
+* concatenate back to the original text; whitespace segments count zero
+* tokens unless they carry structure (indentation or blank lines).
 */
-function estimateTokenCount(text, options = {}) {
-	if (!text) return 0;
-	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
-	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
-	let tokenCount = 0;
-	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
-	return tokenCount;
+function* walkSegments(text, options = {}) {
+	if (!text) return;
+	const resolvedOptions = resolveOptions(options);
+	for (const segment of text.split(TOKEN_SPLIT_PATTERN)) if (segment) yield {
+		segment,
+		tokenCount: estimateSegmentTokens(segment, resolvedOptions)
+	};
 }
-function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
-	if (PATTERNS.whitespace.test(segment)) return 0;
-	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
-	if (PATTERNS.numeric.test(segment)) return 1;
+function resolveOptions(options) {
+	return {
+		defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
+		languageConfigs: options.languageConfigs ?? DEFAULT_LANGUAGE_CONFIGS
+	};
+}
+function estimateSegmentTokens(segment, { languageConfigs, defaultCharsPerToken }) {
+	if (PATTERNS.whitespace.test(segment)) return PATTERNS.structuredWhitespace.test(segment) ? 1 : 0;
+	const languageCharsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs);
+	if (languageCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / languageCharsPerToken);
+	if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
+	if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
 	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
-	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
-	if (PATTERNS.alphanumeric.test(segment)) {
-		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-		return Math.ceil(segment.length / charsPerToken$1);
-	}
-	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-	return Math.ceil(segment.length / charsPerToken);
+	if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / 2);
+	return Math.ceil(segment.length / defaultCharsPerToken);
 }
 function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
-	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+	for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
 }
 function getCharacterCount(text) {
 	return Array.from(text).length;
 }
+function estimateCjkTokens(segment) {
+	let kanaCount = 0;
+	let otherCount = 0;
+	for (const character of segment) {
+		const codePoint = character.codePointAt(0);
+		if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
+		else otherCount++;
+	}
+	return otherCount + Math.ceil(kanaCount / KANA_CHARS_PER_TOKEN);
+}
+/**
+* Estimates the number of tokens in a text string using heuristic rules
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	let tokenCount = 0;
+	for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
+	return tokenCount;
+}
 //#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
@@ -22379,7 +22416,7 @@ const IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
 	"heif",
 	"avif"
 ]);
-const TOKEN_ESTIMATE_CACHE_VERSION = 7;
+const TOKEN_ESTIMATE_CACHE_VERSION = 8;
 /**
 * Cache `source` marker for token estimates supplied by the caller via
 * `part.providerMetadata.mastra.tokenEstimate`. Pipelines that strip the real
@@ -36680,4 +36717,4 @@ Notes:
 //#endregion
 export { renderSubconsciousActivity as $, buildObserverSystemPrompt as A, OBSERVATION_CONTEXT_INSTRUCTIONS as B, WorkingMemoryExtractor as C, TokenCounter as D, summarizeConversation as E, parseObserverOutput as F, Subconscious as G, OBSERVATION_CONTEXT_PROMPT_THREAD as H, injectAnchorIds as I, SubconsciousRemindExtractor as J, KnowledgeSemanticIndexCoordinator as K, parseAnchorId as L, formatMessagesForObserver as M, hasCurrentTaskSection as N, OBSERVER_SYSTEM_PROMPT as O, optimizeObservationsForContext as P, publishSubconsciousActivity as Q, stripEphemeralAnchorIds as R, wrapInObservationGroup as S, SUMMARIZE_THREAD_DEFAULTS as T, OBSERVATION_CONTINUATION_HINT as U, OBSERVATION_CONTEXT_PROMPT as V, getObservationContextPrompt as W, SUBCONSCIOUS_ACTIVITY_STATE_ID as X, ModelByInputTokens as Y, buildSubconsciousActivitySnapshot as Z, deriveObservationGroupProvenance as _, extractWorkingMemoryContent as a, renderObservationGroupsForReflection as b, getObservationsAsOf as c, AUTO_MODEL_BY_PROVIDER as d, Extractor as et, resolveAutoModelId as f, combineObservationGroupRanges as g, WorkingMemoryStateProcessor as h, WorkingMemory as i, extractCurrentTask as j, buildObserverPrompt as k, ObservationalMemoryProcessor as l, WORKING_MEMORY_STATE_PROCESSOR_ID as m, MessageHistory$1 as n, extractWorkingMemoryTags as o, WORKING_MEMORY_STATE_ID as p, StaleKnowledgeSemanticIndexError as q, SemanticRecall as r, removeWorkingMemoryTags as s, Memory as t, ObservationalMemory as u, parseObservationGroups as v, deepMergeWorkingMemory as w, stripObservationGroups as x, reconcileObservationGroupsFromReflection as y, OBSERVATIONAL_MEMORY_DEFAULTS as z };
 
-//# sourceMappingURL=src-CnLuefUv.js.map
+//# sourceMappingURL=src-CTyZJ6A-.js.map