@mastra/memory 1.38.0-alpha.0 → 1.38.0

package/dist/{src-BfhwVNr7.cjs → src-DZmiwOWq.cjs}

@@ -19825,21 +19825,22 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
-//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.6.0/394289cb1bed8c542941f77748eea1a31bd3419018d0fb2ac7029ece2458cdd1/node_modules/tokenx/dist/index.mjs
 const PATTERNS = {
 	whitespace: /^\s+$/,
-	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
-	numeric: /^\d+(?:[.,]\d+)*$/,
-	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
-	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+	structuredWhitespace: /\n\s/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/
 };
-const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
 const DEFAULT_CHARS_PER_TOKEN = 6;
 const SHORT_TOKEN_THRESHOLD = 3;
+const KANA_CHARS_PER_TOKEN = 1.35;
 const DEFAULT_LANGUAGE_CONFIGS = [
 	{
 		pattern: /[äöüßẞ]/i,
-		averageCharsPerToken: 3
+		averageCharsPerToken: 2.6
 	},
 	{
 		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
@@ -19847,39 +19848,75 @@ const DEFAULT_LANGUAGE_CONFIGS = [
 	},
 	{
 		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
-		averageCharsPerToken: 3.5
+		averageCharsPerToken: 2.5
+	},
+	{
+		pattern: /[\u0430-\u044F\u0451]/i,
+		averageCharsPerToken: 4
+	},
+	{
+		pattern: /[\u03AC-\u03CE]/i,
+		averageCharsPerToken: 2.75
+	},
+	{
+		pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
+		averageCharsPerToken: .75
 	}
 ];
 /**
-* Estimates the number of tokens in a text string using heuristic rules.
+* Walks a text as (segment, estimated token count) pairs. The segments
+* concatenate back to the original text; whitespace segments count zero
+* tokens unless they carry structure (indentation or blank lines).
 */
-function estimateTokenCount(text, options = {}) {
-	if (!text) return 0;
-	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
-	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
-	let tokenCount = 0;
-	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
-	return tokenCount;
+function* walkSegments(text, options = {}) {
+	if (!text) return;
+	const resolvedOptions = resolveOptions(options);
+	for (const segment of text.split(TOKEN_SPLIT_PATTERN)) if (segment) yield {
+		segment,
+		tokenCount: estimateSegmentTokens(segment, resolvedOptions)
+	};
 }
-function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
-	if (PATTERNS.whitespace.test(segment)) return 0;
-	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
-	if (PATTERNS.numeric.test(segment)) return 1;
+function resolveOptions(options) {
+	return {
+		defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
+		languageConfigs: options.languageConfigs ?? DEFAULT_LANGUAGE_CONFIGS
+	};
+}
+function estimateSegmentTokens(segment, { languageConfigs, defaultCharsPerToken }) {
+	if (PATTERNS.whitespace.test(segment)) return PATTERNS.structuredWhitespace.test(segment) ? 1 : 0;
+	const languageCharsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs);
+	if (languageCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / languageCharsPerToken);
+	if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
+	if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
 	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
-	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
-	if (PATTERNS.alphanumeric.test(segment)) {
-		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-		return Math.ceil(segment.length / charsPerToken$1);
-	}
-	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
-	return Math.ceil(segment.length / charsPerToken);
+	if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / 2);
+	return Math.ceil(segment.length / defaultCharsPerToken);
 }
 function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
-	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+	for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
 }
 function getCharacterCount(text) {
 	return Array.from(text).length;
 }
+function estimateCjkTokens(segment) {
+	let kanaCount = 0;
+	let otherCount = 0;
+	for (const character of segment) {
+		const codePoint = character.codePointAt(0);
+		if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
+		else otherCount++;
+	}
+	return otherCount + Math.ceil(kanaCount / KANA_CHARS_PER_TOKEN);
+}
+/**
+* Estimates the number of tokens in a text string using heuristic rules
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	let tokenCount = 0;
+	for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
+	return tokenCount;
+}
 //#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
@@ -22400,7 +22437,7 @@ const IMAGE_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
 	"heif",
 	"avif"
 ]);
-const TOKEN_ESTIMATE_CACHE_VERSION = 7;
+const TOKEN_ESTIMATE_CACHE_VERSION = 8;
 /**
 * Cache `source` marker for token estimates supplied by the caller via
 * `part.providerMetadata.mastra.tokenEstimate`. Pipelines that strip the real
@@ -37006,4 +37043,4 @@ Object.defineProperty(exports, "wrapInObservationGroup", {
 	}
 });
 
-//# sourceMappingURL=src-BfhwVNr7.cjs.map
+//# sourceMappingURL=src-DZmiwOWq.cjs.map