@mastra/memory 1.37.1 → 1.38.0

package/dist/{src-BU8L6v1o.cjs → src-DZmiwOWq.cjs}

@@ -19825,21 +19825,22 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
-//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.6.0/394289cb1bed8c542941f77748eea1a31bd3419018d0fb2ac7029ece2458cdd1/node_modules/tokenx/dist/index.mjs
 const PATTERNS = {
 	whitespace: /^\s+$/,
-	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
-	numeric: /^\d+(?:[.,]\d+)*$/,
-	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
-	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+	structuredWhitespace: /\n\s/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/
 };
-const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
 const DEFAULT_CHARS_PER_TOKEN = 6;
 const SHORT_TOKEN_THRESHOLD = 3;
+const KANA_CHARS_PER_TOKEN = 1.35;
 const DEFAULT_LANGUAGE_CONFIGS = [
 	{
 		pattern: /[äöüßẞ]/i,
-		averageCharsPerToken: 3
+		averageCharsPerToken: 2.6
 	},
 	{
 		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
@@ -19847,39 +19848,75 @@ const DEFAULT_LANGUAGE_CONFIGS = [
 	},
 	{
 		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
-		averageCharsPerToken: 3.5
+		averageCharsPerToken: 2.5
+	},
+	{
+		pattern: /[\u0430-\u044F\u0451]/i,
+		averageCharsPerToken: 4
+	},
+	{
+		pattern: /[\u03AC-\u03CE]/i,
+		averageCharsPerToken: 2.75
+	},
+	{
+		pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
+		averageCharsPerToken: .75
 	}
 ];
 /**
-* Estimates the number of tokens in a text string using heuristic rules.
+* Walks a text as (segment, estimated token count) pairs. The segments
+* concatenate back to the original text; whitespace segments count zero
+* tokens unless they carry structure (indentation or blank lines).
 */
-function estimateTokenCount(text, options = {}) {
-	if (!text) return 0;
-	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
-	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
-	let tokenCount = 0;
-	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
-	return tokenCount;
+function* walkSegments(text, options = {}) {
+	if (!text) return;
+	const resolvedOptions = resolveOptions(options);
+	for (const segment of text.split(TOKEN_SPLIT_PATTERN)) if (segment) yield {
+		segment,
+		tokenCount: estimateSegmentTokens(segment, resolvedOptions)
+	};
 }
-function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
-	if (PATTERNS.whitespace.test(segment)) return 0;
-	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
-	if (PATTERNS.numeric.test(segment)) return 1;
+function resolveOptions(options) {
+	return {
+		defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
+		languageConfigs: options.languageConfigs ?? DEFAULT_LANGUAGE_CONFIGS
+	};
+}
+function estimateSegmentTokens(segment, { languageConfigs, defaultCharsPerToken }) {
+	if (PATTERNS.whitespace.test(segment)) return PATTERNS.structuredWhitespace.test(segment) ? 1 : 0;
+	const languageCharsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs);
+	if (languageCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / languageCharsPerToken);
+	if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
+	if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
 	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
-	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
-	if (PATTERNS.alphanumeric.test(segment)) {
-		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-		return Math.ceil(segment.length / charsPerToken$1);
-	}
-	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-	return Math.ceil(segment.length / charsPerToken);
+	if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / 2);
+	return Math.ceil(segment.length / defaultCharsPerToken);
 }
 function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
-	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+	for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
 }
 function getCharacterCount(text) {
 	return Array.from(text).length;
 }
+function estimateCjkTokens(segment) {
+	let kanaCount = 0;
+	let otherCount = 0;
+	for (const character of segment) {
+		const codePoint = character.codePointAt(0);
+		if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
+		else otherCount++;
+	}
+	return otherCount + Math.ceil(kanaCount / KANA_CHARS_PER_TOKEN);
+}
+/**
+* Estimates the number of tokens in a text string using heuristic rules
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	let tokenCount = 0;
+	for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
+	return tokenCount;
+}
 //#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
@@ -22400,7 +22437,7 @@ const IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
 	"heif",
 	"avif"
 ]);
-const TOKEN_ESTIMATE_CACHE_VERSION = 7;
+const TOKEN_ESTIMATE_CACHE_VERSION = 8;
 /**
 * Cache `source` marker for token estimates supplied by the caller via
 * `part.providerMetadata.mastra.tokenEstimate`. Pipelines that strip the real
@@ -23844,6 +23881,7 @@ async function summarizeConversation(opts) {
 			providerOptions: OBSERVATIONAL_MEMORY_DEFAULTS.observation.providerOptions,
 			maxTokensPerBatch: OBSERVATIONAL_MEMORY_DEFAULTS.observation.maxTokensPerBatch,
 			bufferOnIdle: false,
+			observeOnContextOverflow: false,
 			observeAttachments: "auto",
 			instruction: opts.instructions,
 			threadTitle: false,
@@ -31338,6 +31376,7 @@ var ObservationalMemory = class ObservationalMemory {
 			maxTokensPerBatch: config.observation?.maxTokensPerBatch ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.maxTokensPerBatch,
 			bufferTokens: asyncBufferingDisabled ? void 0 : resolveBufferTokens(config.observation?.bufferTokens ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.bufferTokens, config.observation?.messageTokens ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens),
 			bufferOnIdle: config.observation?.bufferOnIdle ?? false,
+			observeOnContextOverflow: config.observation?.observeOnContextOverflow ?? true,
 			bufferActivation: asyncBufferingDisabled ? void 0 : config.observation?.bufferActivation ?? OBSERVATIONAL_MEMORY_DEFAULTS.observation.bufferActivation,
 			activateAfterIdle: parseActivationTTLConfig(observationActivateAfterIdle, observationActivateAfterIdlePath),
 			activateOnProviderChange: config.observation?.activateOnProviderChange ?? config.activateOnProviderChange ?? false,
@@ -33358,7 +33397,9 @@ ${formattedMessages}
 				recordInsideLock = freshRecord;
 				generationBefore = freshRecord.generationCount;
 				const unobservedMessages = messages ? this.getUnobservedMessages(messages, freshRecord) : await this.loadMessagesFromStorage(threadId, resourceId, freshRecord.lastObservedAt ? new Date(freshRecord.lastObservedAt) : void 0);
-				if (!this.meetsObservationThreshold({
+				if (opts.force) {
+					if (unobservedMessages.length === 0) return;
+				} else if (!this.meetsObservationThreshold({
 					record: freshRecord,
 					unobservedTokens: await this.tokenCounter.countMessagesAsync(unobservedMessages)
 				})) return;
@@ -33666,6 +33707,55 @@ ${formattedMessages}
 	}
 };
 //#endregion
+//#region src/processors/observational-memory/context-overflow.ts
+/**
+* Detects provider errors that reject a request for exceeding the model's context window.
+*
+* Providers report this with different codes and wording, so this matches the known shapes
+* (OpenAI and OpenAI-compatible APIs, Anthropic, Google, Bedrock, xAI, Mistral, local servers)
+* against the error's message, code, response body and causes.
+*/
+const CONTEXT_OVERFLOW_PATTERNS = [
+	/context[_ ]length[_ ]exceeded/i,
+	/maximum context length/i,
+	/maximum prompt length/i,
+	/prompt is too long/i,
+	/input is too long/i,
+	/exceed\w*\b[^.]{0,40}\bcontext (?:window|length|limit|size)/i,
+	/input token count[^.]{0,40}exceeds the maximum/i,
+	/reduce the length of the (?:messages|prompt|input)/i
+];
+const MAX_CAUSE_DEPTH = 4;
+function collectErrorText(error, depth = 0) {
+	if (depth > MAX_CAUSE_DEPTH || error == null) return [];
+	if (typeof error === "string") return [error];
+	if (typeof error !== "object") return [];
+	const record = error;
+	const texts = [];
+	for (const key of [
+		"message",
+		"code",
+		"type",
+		"responseBody"
+	]) {
+		const value = record[key];
+		if (typeof value === "string") texts.push(value);
+	}
+	if (record.data && typeof record.data === "object") texts.push(...collectErrorText(record.data.error ?? record.data, depth + 1));
+	if (record.error && typeof record.error === "object") texts.push(...collectErrorText(record.error, depth + 1));
+	texts.push(...collectErrorText(record.cause, depth + 1));
+	return texts;
+}
+function isRateLimited(error, depth = 0) {
+	if (depth > MAX_CAUSE_DEPTH || error == null || typeof error !== "object") return false;
+	const record = error;
+	return record.statusCode === 429 || isRateLimited(record.cause, depth + 1);
+}
+function isContextOverflowError(error) {
+	if (isRateLimited(error)) return false;
+	return collectErrorText(error).some((text) => CONTEXT_OVERFLOW_PATTERNS.some((pattern) => pattern.test(text)));
+}
+//#endregion
 //#region src/processors/observational-memory/temporal-markers.ts
 const TEMPORAL_GAP_REMINDER_TYPE = "temporal-gap";
 function getTemporalGapReminderText(gapText, timestamp) {
@@ -33776,6 +33866,8 @@ function getOmObservabilityContext(args) {
 }
 /** Key used to store gateway detection result in per-processor state. */
 const GATEWAY_STATE_KEY = "__isGatewayModel";
+/** Step number OM last recovered from a context-overflow error, so each step retries at most once. */
+const CONTEXT_OVERFLOW_STEP_KEY = "__omContextOverflowStep";
 /** Check if the model is routed through a Mastra gateway (duck-type check to avoid cross-package instanceof issues). */
 function isMastraGatewayModel(model) {
 	return typeof model === "object" && model !== null && "gatewayId" in model && model.gatewayId === "mastra";
@@ -34015,6 +34107,42 @@ var ObservationalMemoryProcessor = class {
 			return messageList;
 		});
 	}
+	async processAPIError(args) {
+		const { error, messageList, requestContext, stepNumber, state } = args;
+		if (!this.engine.getObservationConfig().observeOnContextOverflow) return;
+		if (state[GATEWAY_STATE_KEY]) return;
+		if (state[CONTEXT_OVERFLOW_STEP_KEY] === stepNumber) return;
+		if (!isContextOverflowError(error)) return;
+		const context = this.engine.getThreadContext(requestContext, messageList);
+		if (!context) return;
+		if ((0, _mastra_core_memory.parseMemoryRequestContext)(requestContext)?.memoryConfig?.readOnly) return;
+		state[CONTEXT_OVERFLOW_STEP_KEY] = stepNumber;
+		omDebug(`[OM:processAPIError] context overflow at step ${stepNumber} — observing pending messages`);
+		const { threadId, resourceId } = context;
+		await this.engine.waitForBuffering(threadId, resourceId);
+		const messages = getObservableMessages(messageList);
+		const { activated } = await this.engine.activate({
+			threadId,
+			resourceId,
+			messages,
+			messageList,
+			writer: args.writer
+		});
+		const { observed } = await this.engine.getTokenCounter().runWithModelContext(state.__omActorModelContext, () => this.engine.observe({
+			threadId,
+			resourceId,
+			messages: getObservableMessages(messageList),
+			messageList,
+			force: true,
+			agent: args.agent,
+			sendSignal: args.sendSignal,
+			sendStateSignal: args.sendStateSignal,
+			requestContext,
+			writer: args.writer,
+			observabilityContext: getOmObservabilityContext(args)
+		}));
+		return { retry: activated || observed };
+	}
 	get config() {
 		return this.engine.config;
 	}
@@ -35468,6 +35596,7 @@ ${workingMemory}`;
 				providerOptions: omConfig.observation.providerOptions,
 				bufferTokens: omConfig.observation.bufferTokens,
 				bufferOnIdle: omConfig.observation.bufferOnIdle,
+				observeOnContextOverflow: omConfig.observation.observeOnContextOverflow,
 				bufferActivation: omConfig.observation.bufferActivation,
 				blockAfter: omConfig.observation.blockAfter,
 				previousObserverTokens: omConfig.observation.previousObserverTokens,
@@ -36546,6 +36675,17 @@ Notes:
 		return processors;
 	}
 	/**
+	* Extends the base implementation to add ObservationalMemory as an error processor.
+	* OM uses processAPIError to observe pending messages and retry when the provider
+	* rejects a request for exceeding the model's context window.
+	*/
+	async getErrorProcessors(configuredProcessors = [], context) {
+		const processors = await super.getErrorProcessors(configuredProcessors, context);
+		const om = await this.createOMProcessor(configuredProcessors, context);
+		if (om) processors.push(om);
+		return processors;
+	}
+	/**
 	* Creates an ObservationalMemory processor wrapping the shared engine.
 	* Returns null if OM is not configured, not supported, or already present
 	* in the user's configured processors.
@@ -36903,4 +37043,4 @@ Object.defineProperty(exports, "wrapInObservationGroup", {
 	}
 });
 
-//# sourceMappingURL=src-BU8L6v1o.cjs.map
+//# sourceMappingURL=src-DZmiwOWq.cjs.map