@mastra/memory 1.38.0-alpha.0 → 1.39.0-alpha.0

package/dist/{src-CnLuefUv.js → src-D9Uu93Xu.js}

@@ -19804,21 +19804,22 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
-//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.6.0/394289cb1bed8c542941f77748eea1a31bd3419018d0fb2ac7029ece2458cdd1/node_modules/tokenx/dist/index.mjs
 const PATTERNS = {
 	whitespace: /^\s+$/,
-	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
-	numeric: /^\d+(?:[.,]\d+)*$/,
-	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
-	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+	structuredWhitespace: /\n\s/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/
 };
-const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
 const DEFAULT_CHARS_PER_TOKEN = 6;
 const SHORT_TOKEN_THRESHOLD = 3;
+const KANA_CHARS_PER_TOKEN = 1.35;
 const DEFAULT_LANGUAGE_CONFIGS = [
 	{
 		pattern: /[äöüßẞ]/i,
-		averageCharsPerToken: 3
+		averageCharsPerToken: 2.6
 	},
 	{
 		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
@@ -19826,39 +19827,75 @@ const DEFAULT_LANGUAGE_CONFIGS = [
 	},
 	{
 		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
-		averageCharsPerToken: 3.5
+		averageCharsPerToken: 2.5
+	},
+	{
+		pattern: /[\u0430-\u044F\u0451]/i,
+		averageCharsPerToken: 4
+	},
+	{
+		pattern: /[\u03AC-\u03CE]/i,
+		averageCharsPerToken: 2.75
+	},
+	{
+		pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
+		averageCharsPerToken: .75
 	}
 ];
 /**
-* Estimates the number of tokens in a text string using heuristic rules.
+* Walks a text as (segment, estimated token count) pairs. The segments
+* concatenate back to the original text; whitespace segments count zero
+* tokens unless they carry structure (indentation or blank lines).
 */
-function estimateTokenCount(text, options = {}) {
-	if (!text) return 0;
-	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
-	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
-	let tokenCount = 0;
-	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
-	return tokenCount;
+function* walkSegments(text, options = {}) {
+	if (!text) return;
+	const resolvedOptions = resolveOptions(options);
+	for (const segment of text.split(TOKEN_SPLIT_PATTERN)) if (segment) yield {
+		segment,
+		tokenCount: estimateSegmentTokens(segment, resolvedOptions)
+	};
+}
+function resolveOptions(options) {
+	return {
+		defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
+		languageConfigs: options.languageConfigs ?? DEFAULT_LANGUAGE_CONFIGS
+	};
 }
-function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
-	if (PATTERNS.whitespace.test(segment)) return 0;
-	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
-	if (PATTERNS.numeric.test(segment)) return 1;
+function estimateSegmentTokens(segment, { languageConfigs, defaultCharsPerToken }) {
+	if (PATTERNS.whitespace.test(segment)) return PATTERNS.structuredWhitespace.test(segment) ? 1 : 0;
+	const languageCharsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs);
+	if (languageCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / languageCharsPerToken);
+	if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
+	if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
 	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
-	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
-	if (PATTERNS.alphanumeric.test(segment)) {
-		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-		return Math.ceil(segment.length / charsPerToken$1);
-	}
-	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-	return Math.ceil(segment.length / charsPerToken);
+	if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / 2);
+	return Math.ceil(segment.length / defaultCharsPerToken);
 }
 function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
-	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+	for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
 }
 function getCharacterCount(text) {
 	return Array.from(text).length;
 }
+function estimateCjkTokens(segment) {
+	let kanaCount = 0;
+	let otherCount = 0;
+	for (const character of segment) {
+		const codePoint = character.codePointAt(0);
+		if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
+		else otherCount++;
+	}
+	return otherCount + Math.ceil(kanaCount / KANA_CHARS_PER_TOKEN);
+}
+/**
+* Estimates the number of tokens in a text string using heuristic rules
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	let tokenCount = 0;
+	for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
+	return tokenCount;
+}
 //#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
@@ -22379,7 +22416,7 @@ const IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
 	"heif",
 	"avif"
 ]);
-const TOKEN_ESTIMATE_CACHE_VERSION = 7;
+const TOKEN_ESTIMATE_CACHE_VERSION = 8;
 /**
 * Cache `source` marker for token estimates supplied by the caller via
 * `part.providerMetadata.mastra.tokenEstimate`. Pipelines that strip the real
@@ -26480,7 +26517,8 @@ function hasGoogleGenerativeAIKey() {
 /**
 * Resolve the concrete model ID `'auto'` selects for a main model ID.
 *
-* Order: Gemini when `GOOGLE_GENERATIVE_AI_API_KEY` is set; else the main provider's low-cost
+* Order: Gemini when Google is usable (`canUseProvider('google')`, or `GOOGLE_GENERATIVE_AI_API_KEY`
+* when no check is given); else the main provider's low-cost
 * model (keeping a `mastra/` gateway prefix); else the main model ID itself. Returns `undefined`
 * only when there is no Google key and no main model ID.
 */
@@ -26489,7 +26527,7 @@ function resolveAutoModelId(mainModelId, options) {
 		...AUTO_MODEL_BY_PROVIDER,
 		...options?.autoModels
 	};
-	if (hasGoogleGenerativeAIKey() && models.google) return models.google;
+	if ((options?.canUseProvider ? options.canUseProvider("google") : hasGoogleGenerativeAIKey()) && models.google) return models.google;
 	if (!mainModelId) return;
 	const prefix = mainModelId.startsWith(ROUTE_PREFIX) ? ROUTE_PREFIX : "";
 	const provider = mainModelId.slice(prefix.length).split("/", 1)[0];
@@ -31170,6 +31208,7 @@ var ObservationalMemory = class ObservationalMemory {
 	mastra;
 	autoModels;
 	resolveAutoModelHook;
+	canUseProvider;
 	memory;
 	/**
 	* Track message IDs observed during this instance's lifetime.
@@ -31268,6 +31307,7 @@ var ObservationalMemory = class ObservationalMemory {
 		this.hooks = config.hooks;
 		this.autoModels = config.autoModels;
 		this.resolveAutoModelHook = config.resolveModel;
+		this.canUseProvider = config.canUseProvider;
 		this.hookExecution = config.hookExecution ?? "non-blocking";
 		this.mastra = config.mastra;
 		this.memory = config.memory;
@@ -31467,11 +31507,18 @@ var ObservationalMemory = class ObservationalMemory {
 			model: await this.resolveAutoModel(options)
 		};
 	}
+	autoModelOptions(requestContext) {
+		const canUseProvider = this.canUseProvider;
+		return {
+			autoModels: this.autoModels,
+			canUseProvider: canUseProvider ? (providerId) => canUseProvider(providerId, { requestContext }) : void 0
+		};
+	}
 	async resolveAutoModel(options) {
 		let mainModel = options?.currentModel?.model;
 		if (!mainModel && options?.mainAgent) mainModel = await options.mainAgent.getModel({ requestContext: options.requestContext });
 		const mainModelId = getModelId(mainModel);
-		const pick = resolveAutoModelId(mainModelId, { autoModels: this.autoModels }) ?? (mainModel ? void 0 : OBSERVATIONAL_MEMORY_DEFAULTS.observation.model);
+		const pick = resolveAutoModelId(mainModelId, this.autoModelOptions(options?.requestContext)) ?? (mainModel ? void 0 : OBSERVATIONAL_MEMORY_DEFAULTS.observation.model);
 		if (mainModel && (!pick || pick === mainModelId)) return mainModel;
 		const modelId = pick;
 		return this.resolveAutoModelHook ? await this.resolveAutoModelHook(modelId, { requestContext: options?.requestContext }) : modelId;
@@ -31538,7 +31585,7 @@ var ObservationalMemory = class ObservationalMemory {
 	async getCompressionStartLevel(requestContext) {
 		try {
 			const reflectionModel = await this.resolveModelFunction(this.reflectionConfig.model, requestContext);
-			if ((reflectionModel === "auto" ? resolveAutoModelId(void 0, { autoModels: this.autoModels }) ?? "" : (await this.resolveModelContext(reflectionModel, requestContext))?.modelId ?? "").includes("gemini-2.5-flash")) return 2;
+			if ((reflectionModel === "auto" ? resolveAutoModelId(void 0, this.autoModelOptions(requestContext)) ?? "" : (await this.resolveModelContext(reflectionModel, requestContext))?.modelId ?? "").includes("gemini-2.5-flash")) return 2;
 			return 1;
 		} catch {
 			return 1;
@@ -35525,6 +35572,7 @@ ${workingMemory}`;
 			shareTokenBudget: omConfig.shareTokenBudget,
 			model: omConfig.model,
 			autoModels: omConfig.autoModels,
+			canUseProvider: omConfig.canUseProvider,
 			resolveModel: omConfig.resolveModel,
 			mastra: this._mastraInstance,
 			onIndexObservations,
@@ -36680,4 +36728,4 @@ Notes:
 //#endregion
 export { renderSubconsciousActivity as $, buildObserverSystemPrompt as A, OBSERVATION_CONTEXT_INSTRUCTIONS as B, WorkingMemoryExtractor as C, TokenCounter as D, summarizeConversation as E, parseObserverOutput as F, Subconscious as G, OBSERVATION_CONTEXT_PROMPT_THREAD as H, injectAnchorIds as I, SubconsciousRemindExtractor as J, KnowledgeSemanticIndexCoordinator as K, parseAnchorId as L, formatMessagesForObserver as M, hasCurrentTaskSection as N, OBSERVER_SYSTEM_PROMPT as O, optimizeObservationsForContext as P, publishSubconsciousActivity as Q, stripEphemeralAnchorIds as R, wrapInObservationGroup as S, SUMMARIZE_THREAD_DEFAULTS as T, OBSERVATION_CONTINUATION_HINT as U, OBSERVATION_CONTEXT_PROMPT as V, getObservationContextPrompt as W, SUBCONSCIOUS_ACTIVITY_STATE_ID as X, ModelByInputTokens as Y, buildSubconsciousActivitySnapshot as Z, deriveObservationGroupProvenance as _, extractWorkingMemoryContent as a, renderObservationGroupsForReflection as b, getObservationsAsOf as c, AUTO_MODEL_BY_PROVIDER as d, Extractor as et, resolveAutoModelId as f, combineObservationGroupRanges as g, WorkingMemoryStateProcessor as h, WorkingMemory as i, extractCurrentTask as j, buildObserverPrompt as k, ObservationalMemoryProcessor as l, WORKING_MEMORY_STATE_PROCESSOR_ID as m, MessageHistory$1 as n, extractWorkingMemoryTags as o, WORKING_MEMORY_STATE_ID as p, StaleKnowledgeSemanticIndexError as q, SemanticRecall as r, removeWorkingMemoryTags as s, Memory as t, ObservationalMemory as u, parseObservationGroups as v, deepMergeWorkingMemory as w, stripObservationGroups as x, reconcileObservationGroupsFromReflection as y, OBSERVATIONAL_MEMORY_DEFAULTS as z };
 
-//# sourceMappingURL=src-CnLuefUv.js.map
+//# sourceMappingURL=src-D9Uu93Xu.js.map