@mastra/memory 1.37.1 → 1.38.0

package/dist/{src-DWoBX5f9.js → src-CTyZJ6A-.js}

@@ -19804,21 +19804,22 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
-//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.6.0/394289cb1bed8c542941f77748eea1a31bd3419018d0fb2ac7029ece2458cdd1/node_modules/tokenx/dist/index.mjs
 const PATTERNS = {
 	whitespace: /^\s+$/,
-	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
-	numeric: /^\d+(?:[.,]\d+)*$/,
-	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
-	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+	structuredWhitespace: /\n\s/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/
 };
-const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
 const DEFAULT_CHARS_PER_TOKEN = 6;
 const SHORT_TOKEN_THRESHOLD = 3;
+const KANA_CHARS_PER_TOKEN = 1.35;
 const DEFAULT_LANGUAGE_CONFIGS = [
 	{
 		pattern: /[äöüßẞ]/i,
-		averageCharsPerToken: 3
+		averageCharsPerToken: 2.6
 	},
 	{
 		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
@@ -19826,39 +19827,75 @@ const DEFAULT_LANGUAGE_CONFIGS = [
 	},
 	{
 		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
-		averageCharsPerToken: 3.5
+		averageCharsPerToken: 2.5
+	},
+	{
+		pattern: /[\u0430-\u044F\u0451]/i,
+		averageCharsPerToken: 4
+	},
+	{
+		pattern: /[\u03AC-\u03CE]/i,
+		averageCharsPerToken: 2.75
+	},
+	{
+		pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
+		averageCharsPerToken: .75
 	}
 ];
 /**
-* Estimates the number of tokens in a text string using heuristic rules.
+* Walks a text as (segment, estimated token count) pairs. The segments
+* concatenate back to the original text; whitespace segments count zero
+* tokens unless they carry structure (indentation or blank lines).
 */
-function estimateTokenCount(text, options = {}) {
-	if (!text) return 0;
-	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
-	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
-	let tokenCount = 0;
-	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
-	return tokenCount;
+function* walkSegments(text, options = {}) {
+	if (!text) return;
+	const resolvedOptions = resolveOptions(options);
+	for (const segment of text.split(TOKEN_SPLIT_PATTERN)) if (segment) yield {
+		segment,
+		tokenCount: estimateSegmentTokens(segment, resolvedOptions)
+	};
 }
-function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
-	if (PATTERNS.whitespace.test(segment)) return 0;
-	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
-	if (PATTERNS.numeric.test(segment)) return 1;
+function resolveOptions(options) {
+	return {
+		defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
+		languageConfigs: options.languageConfigs ?? DEFAULT_LANGUAGE_CONFIGS
+	};
+}
+function estimateSegmentTokens(segment, { languageConfigs, defaultCharsPerToken }) {
+	if (PATTERNS.whitespace.test(segment)) return PATTERNS.structuredWhitespace.test(segment) ? 1 : 0;
+	const languageCharsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs);
+	if (languageCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / languageCharsPerToken);
+	if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
+	if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
 	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
-	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
-	if (PATTERNS.alphanumeric.test(segment)) {
-		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-		return Math.ceil(segment.length / charsPerToken$1);
-	}
-	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-	return Math.ceil(segment.length / charsPerToken);
+	if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / 2);
+	return Math.ceil(segment.length / defaultCharsPerToken);
 }
 function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
-	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+	for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
 }
 function getCharacterCount(text) {
 	return Array.from(text).length;
 }
+function estimateCjkTokens(segment) {
+	let kanaCount = 0;
+	let otherCount = 0;
+	for (const character of segment) {
+		const codePoint = character.codePointAt(0);
+		if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
+		else otherCount++;
+	}
+	return otherCount + Math.ceil(kanaCount / KANA_CHARS_PER_TOKEN);
+}
+/**
+* Estimates the number of tokens in a text string using heuristic rules
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	let tokenCount = 0;
+	for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
+	return tokenCount;
+}
 //#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
@@ -22379,7 +22416,7 @@ const IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
 	"heif",
 	"avif"
 ]);
-const TOKEN_ESTIMATE_CACHE_VERSION = 7;
+const TOKEN_ESTIMATE_CACHE_VERSION = 8;
 /**
 * Cache `source` marker for token estimates supplied by the caller via
 * `part.providerMetadata.mastra.tokenEstimate`. Pipelines that strip the real
@@ -23823,6 +23860,7 @@ async function summarizeConversation(opts) {
 			providerOptions: OBSERVATIONAL_MEMORY_DEFAULTS.observation.providerOptions,
 			maxTokensPerBatch: OBSERVATIONAL_MEMORY_DEFAULTS.observation.maxTokensPerBatch,
 			bufferOnIdle: false,
+			observeOnContextOverflow: false,
 			observeAttachments: "auto",
 			instruction: opts.instructions,
 			threadTitle: false,
@@ -31317,6 +31355,7 @@ var ObservationalMemory = class ObservationalMemory {
 			maxTokensPerBatch: config.observation?.maxTokensPerBatch ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.maxTokensPerBatch,
 			bufferTokens: asyncBufferingDisabled ? void 0 : resolveBufferTokens(config.observation?.bufferTokens ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.bufferTokens, config.observation?.messageTokens ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens),
 			bufferOnIdle: config.observation?.bufferOnIdle ?? false,
+			observeOnContextOverflow: config.observation?.observeOnContextOverflow ?? true,
 			bufferActivation: asyncBufferingDisabled ? void 0 : config.observation?.bufferActivation ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.bufferActivation,
 			activateAfterIdle: parseActivationTTLConfig(observationActivateAfterIdle, observationActivateAfterIdlePath),
 			activateOnProviderChange: config.observation?.activateOnProviderChange ?? config.activateOnProviderChange ?? false,
@@ -33337,7 +33376,9 @@ ${formattedMessages}
 				recordInsideLock = freshRecord;
 				generationBefore = freshRecord.generationCount;
 				const unobservedMessages = messages ? this.getUnobservedMessages(messages, freshRecord) : await this.loadMessagesFromStorage(threadId, resourceId, freshRecord.lastObservedAt ? new Date(freshRecord.lastObservedAt) : void 0);
-				if (!this.meetsObservationThreshold({
+				if (opts.force) {
+					if (unobservedMessages.length === 0) return;
+				} else if (!this.meetsObservationThreshold({
 					record: freshRecord,
 					unobservedTokens: await this.tokenCounter.countMessagesAsync(unobservedMessages)
 				})) return;
@@ -33645,6 +33686,55 @@ ${formattedMessages}
 	}
 };
 //#endregion
+//#region src/processors/observational-memory/context-overflow.ts
+/**
+* Detects provider errors that reject a request for exceeding the model's context window.
+*
+* Providers report this with different codes and wording, so this matches the known shapes
+* (OpenAI and OpenAI-compatible APIs, Anthropic, Google, Bedrock, xAI, Mistral, local servers)
+* against the error's message, code, response body and causes.
+*/
+const CONTEXT_OVERFLOW_PATTERNS = [
+	/context[_ ]length[_ ]exceeded/i,
+	/maximum context length/i,
+	/maximum prompt length/i,
+	/prompt is too long/i,
+	/input is too long/i,
+	/exceed\w*\b[^.]{0,40}\bcontext (?:window|length|limit|size)/i,
+	/input token count[^.]{0,40}exceeds the maximum/i,
+	/reduce the length of the (?:messages|prompt|input)/i
+];
+const MAX_CAUSE_DEPTH = 4;
+function collectErrorText(error, depth = 0) {
+	if (depth > MAX_CAUSE_DEPTH || error == null) return [];
+	if (typeof error === "string") return [error];
+	if (typeof error !== "object") return [];
+	const record = error;
+	const texts = [];
+	for (const key of [
+		"message",
+		"code",
+		"type",
+		"responseBody"
+	]) {
+		const value = record[key];
+		if (typeof value === "string") texts.push(value);
+	}
+	if (record.data && typeof record.data === "object") texts.push(...collectErrorText(record.data.error ?? record.data, depth + 1));
+	if (record.error && typeof record.error === "object") texts.push(...collectErrorText(record.error, depth + 1));
+	texts.push(...collectErrorText(record.cause, depth + 1));
+	return texts;
+}
+function isRateLimited(error, depth = 0) {
+	if (depth > MAX_CAUSE_DEPTH || error == null || typeof error !== "object") return false;
+	const record = error;
+	return record.statusCode === 429 || isRateLimited(record.cause, depth + 1);
+}
+function isContextOverflowError(error) {
+	if (isRateLimited(error)) return false;
+	return collectErrorText(error).some((text) => CONTEXT_OVERFLOW_PATTERNS.some((pattern) => pattern.test(text)));
+}
+//#endregion
 //#region src/processors/observational-memory/temporal-markers.ts
 const TEMPORAL_GAP_REMINDER_TYPE = "temporal-gap";
 function getTemporalGapReminderText(gapText, timestamp) {
@@ -33755,6 +33845,8 @@ function getOmObservabilityContext(args) {
 }
 /** Key used to store gateway detection result in per-processor state. */
 const GATEWAY_STATE_KEY = "__isGatewayModel";
+/** Step number OM last recovered from a context-overflow error, so each step retries at most once. */
+const CONTEXT_OVERFLOW_STEP_KEY = "__omContextOverflowStep";
 /** Check if the model is routed through a Mastra gateway (duck-type check to avoid cross-package instanceof issues). */
 function isMastraGatewayModel(model) {
 	return typeof model === "object" && model !== null && "gatewayId" in model && model.gatewayId === "mastra";
@@ -33994,6 +34086,42 @@ var ObservationalMemoryProcessor = class {
 			return messageList;
 		});
 	}
+	async processAPIError(args) {
+		const { error, messageList, requestContext, stepNumber, state } = args;
+		if (!this.engine.getObservationConfig().observeOnContextOverflow) return;
+		if (state[GATEWAY_STATE_KEY]) return;
+		if (state[CONTEXT_OVERFLOW_STEP_KEY] === stepNumber) return;
+		if (!isContextOverflowError(error)) return;
+		const context = this.engine.getThreadContext(requestContext, messageList);
+		if (!context) return;
+		if (parseMemoryRequestContext(requestContext)?.memoryConfig?.readOnly) return;
+		state[CONTEXT_OVERFLOW_STEP_KEY] = stepNumber;
+		omDebug(`[OM:processAPIError] context overflow at step ${stepNumber} — observing pending messages`);
+		const { threadId, resourceId } = context;
+		await this.engine.waitForBuffering(threadId, resourceId);
+		const messages = getObservableMessages(messageList);
+		const { activated } = await this.engine.activate({
+			threadId,
+			resourceId,
+			messages,
+			messageList,
+			writer: args.writer
+		});
+		const { observed } = await this.engine.getTokenCounter().runWithModelContext(state.__omActorModelContext, () => this.engine.observe({
+			threadId,
+			resourceId,
+			messages: getObservableMessages(messageList),
+			messageList,
+			force: true,
+			agent: args.agent,
+			sendSignal: args.sendSignal,
+			sendStateSignal: args.sendStateSignal,
+			requestContext,
+			writer: args.writer,
+			observabilityContext: getOmObservabilityContext(args)
+		}));
+		return { retry: activated || observed };
+	}
 	get config() {
 		return this.engine.config;
 	}
@@ -35447,6 +35575,7 @@ ${workingMemory}`;
 				providerOptions: omConfig.observation.providerOptions,
 				bufferTokens: omConfig.observation.bufferTokens,
 				bufferOnIdle: omConfig.observation.bufferOnIdle,
+				observeOnContextOverflow: omConfig.observation.observeOnContextOverflow,
 				bufferActivation: omConfig.observation.bufferActivation,
 				blockAfter: omConfig.observation.blockAfter,
 				previousObserverTokens: omConfig.observation.previousObserverTokens,
@@ -36525,6 +36654,17 @@ Notes:
 		return processors;
 	}
 	/**
+	* Extends the base implementation to add ObservationalMemory as an error processor.
+	* OM uses processAPIError to observe pending messages and retry when the provider
+	* rejects a request for exceeding the model's context window.
+	*/
+	async getErrorProcessors(configuredProcessors = [], context) {
+		const processors = await super.getErrorProcessors(configuredProcessors, context);
+		const om = await this.createOMProcessor(configuredProcessors, context);
+		if (om) processors.push(om);
+		return processors;
+	}
+	/**
 	* Creates an ObservationalMemory processor wrapping the shared engine.
 	* Returns null if OM is not configured, not supported, or already present
 	* in the user's configured processors.
@@ -36577,4 +36717,4 @@ Notes:
 //#endregion
 export { renderSubconsciousActivity as $, buildObserverSystemPrompt as A, OBSERVATION_CONTEXT_INSTRUCTIONS as B, WorkingMemoryExtractor as C, TokenCounter as D, summarizeConversation as E, parseObserverOutput as F, Subconscious as G, OBSERVATION_CONTEXT_PROMPT_THREAD as H, injectAnchorIds as I, SubconsciousRemindExtractor as J, KnowledgeSemanticIndexCoordinator as K, parseAnchorId as L, formatMessagesForObserver as M, hasCurrentTaskSection as N, OBSERVER_SYSTEM_PROMPT as O, optimizeObservationsForContext as P, publishSubconsciousActivity as Q, stripEphemeralAnchorIds as R, wrapInObservationGroup as S, SUMMARIZE_THREAD_DEFAULTS as T, OBSERVATION_CONTINUATION_HINT as U, OBSERVATION_CONTEXT_PROMPT as V, getObservationContextPrompt as W, SUBCONSCIOUS_ACTIVITY_STATE_ID as X, ModelByInputTokens as Y, buildSubconsciousActivitySnapshot as Z, deriveObservationGroupProvenance as _, extractWorkingMemoryContent as a, renderObservationGroupsForReflection as b, getObservationsAsOf as c, AUTO_MODEL_BY_PROVIDER as d, Extractor as et, resolveAutoModelId as f, combineObservationGroupRanges as g, WorkingMemoryStateProcessor as h, WorkingMemory as i, extractCurrentTask as j, buildObserverPrompt as k, ObservationalMemoryProcessor as l, WORKING_MEMORY_STATE_PROCESSOR_ID as m, MessageHistory$1 as n, extractWorkingMemoryTags as o, WORKING_MEMORY_STATE_ID as p, StaleKnowledgeSemanticIndexError as q, SemanticRecall as r, removeWorkingMemoryTags as s, Memory as t, ObservationalMemory as u, parseObservationGroups as v, deepMergeWorkingMemory as w, stripObservationGroups as x, reconcileObservationGroupsFromReflection as y, OBSERVATIONAL_MEMORY_DEFAULTS as z };
 
-//# sourceMappingURL=src-DWoBX5f9.js.map
+//# sourceMappingURL=src-CTyZJ6A-.js.map