@mastra/memory 1.31.0-alpha.2 → 1.31.0-alpha.3

package/dist/{src-C4niho1P.cjs → src-QqTmOY5X.cjs}

@@ -52,7 +52,6 @@ let path = require("path");
 let _mastra_core_tools = require("@mastra/core/tools");
 let _mastra_core_request_context = require("@mastra/core/request-context");
 let _mastra_core_llm = require("@mastra/core/llm");
-let tokenx = require("tokenx");
 let async_hooks = require("async_hooks");
 let probe_image_size_sync_js = require("probe-image-size/sync.js");
 probe_image_size_sync_js = __toESM$1(probe_image_size_sync_js, 1);
@@ -18326,6 +18325,62 @@ function safeSlice(str, end) {
 	return str.slice(0, safeEnd);
 }
 //#endregion
+//#region ../../../../../setup-pnpm/node_modules/.bin/store/v11/links/@/tokenx/1.3.0/09093cdaeebfe25c39ba9052d6277b45fc99abf5e14e859f60f338f5c5160afd/node_modules/tokenx/dist/index.mjs
+const PATTERNS = {
+	whitespace: /^\s+$/,
+	cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
+	numeric: /^\d+(?:[.,]\d+)*$/,
+	punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]/,
+	alphanumeric: /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/
+};
+const TOKEN_SPLIT_PATTERN = /* @__PURE__ */ new RegExp(`(\\s+|${PATTERNS.punctuation.source}+)`);
+const DEFAULT_CHARS_PER_TOKEN = 6;
+const SHORT_TOKEN_THRESHOLD = 3;
+const DEFAULT_LANGUAGE_CONFIGS = [
+	{
+		pattern: /[äöüßẞ]/i,
+		averageCharsPerToken: 3
+	},
+	{
+		pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
+		averageCharsPerToken: 3
+	},
+	{
+		pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
+		averageCharsPerToken: 3.5
+	}
+];
+/**
+* Estimates the number of tokens in a text string using heuristic rules.
+*/
+function estimateTokenCount(text, options = {}) {
+	if (!text) return 0;
+	const { defaultCharsPerToken = DEFAULT_CHARS_PER_TOKEN, languageConfigs = DEFAULT_LANGUAGE_CONFIGS } = options;
+	const segments = text.split(TOKEN_SPLIT_PATTERN).filter(Boolean);
+	let tokenCount = 0;
+	for (const segment of segments) tokenCount += estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken);
+	return tokenCount;
+}
+function estimateSegmentTokens(segment, languageConfigs, defaultCharsPerToken) {
+	if (PATTERNS.whitespace.test(segment)) return 0;
+	if (PATTERNS.cjk.test(segment)) return getCharacterCount(segment);
+	if (PATTERNS.numeric.test(segment)) return 1;
+	if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
+	if (PATTERNS.punctuation.test(segment)) return segment.length > 1 ? Math.ceil(segment.length / 2) : 1;
+	if (PATTERNS.alphanumeric.test(segment)) {
+		const charsPerToken$1 = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
+		return Math.ceil(segment.length / charsPerToken$1);
+	}
+	const charsPerToken = getLanguageSpecificCharsPerToken(segment, languageConfigs) ?? defaultCharsPerToken;
+	return Math.ceil(segment.length / charsPerToken);
+}
+function getLanguageSpecificCharsPerToken(segment, languageConfigs) {
+	for (const config of languageConfigs) if (config.pattern.test(segment)) return config.averageCharsPerToken;
+}
+function getCharacterCount(text) {
+	return Array.from(text).length;
+}
+//#endregion
 //#region src/processors/observational-memory/tool-argument-helpers.ts
 const DEFAULT_OBSERVER_TOOL_ARGUMENT_MAX_TOKENS = 2e3;
 const OBSERVER_TOOL_ARGUMENT_INLINE_STRING_MAX_CHARS = 160;
@@ -18459,7 +18514,7 @@ function readEntries(value) {
 	}
 }
 function fits(text, limits) {
-	return text.length <= limits.maxCharacters && (0, tokenx.estimateTokenCount)(text) <= limits.maxTokens;
+	return text.length <= limits.maxCharacters && estimateTokenCount(text) <= limits.maxTokens;
 }
 function joinEntries(entries, marker) {
 	const lines = [...entries].sort((a, b) => a.order - b.order).map((entry) => entry.text);
@@ -18473,7 +18528,7 @@ function selectOutlineEntries(entries, limits) {
 	const selected = [];
 	const candidates = [...entries].sort((a, b) => {
 		if (a.depth !== b.depth) return a.depth - b.depth;
-		return (0, tokenx.estimateTokenCount)(a.text) - (0, tokenx.estimateTokenCount)(b.text) || a.text.length - b.text.length || a.order - b.order;
+		return estimateTokenCount(a.text) - estimateTokenCount(b.text) || a.text.length - b.text.length || a.order - b.order;
 	});
 	for (const candidate of candidates) if (fits(joinEntries([...selected, candidate], marker), limits)) selected.push(candidate);
 	const rendered = joinEntries(selected, marker);
@@ -18651,10 +18706,10 @@ function formatToolArgumentsForObserver(value, options) {
 	let omittedPreviews = 0;
 	for (let index = 0; index < previews.length; index++) {
 		const remainingCount = previews.length - index;
-		const remainingTokens = Math.max(0, limits.maxTokens - (0, tokenx.estimateTokenCount)(rendered));
+		const remainingTokens = Math.max(0, limits.maxTokens - estimateTokenCount(rendered));
 		const remainingCharacters = Math.max(0, limits.maxCharacters - rendered.length - 1);
 		const previewLimits = {
-			maxTokens: (0, tokenx.estimateTokenCount)(rendered) + Math.floor(remainingTokens / remainingCount),
+			maxTokens: estimateTokenCount(rendered) + Math.floor(remainingTokens / remainingCount),
 			maxCharacters: rendered.length + 1 + Math.floor(remainingCharacters / remainingCount)
 		};
 		const preview = formatPreview(previews[index].path, previews[index].value, previewLimits, rendered);
@@ -18736,11 +18791,11 @@ function resolveToolResultValue(part, invocationResult) {
 }
 function truncateStringByTokens(text, maxTokens) {
 	if (!text || maxTokens <= 0) return "";
-	const totalTokens = (0, tokenx.estimateTokenCount)(text);
+	const totalTokens = estimateTokenCount(text);
 	if (totalTokens <= maxTokens) return text;
 	const buildCandidate = (sliceEnd) => {
 		const visible = safeSlice(text, sliceEnd);
-		return `${visible}\n... [truncated ~${totalTokens - (0, tokenx.estimateTokenCount)(visible)} tokens]`;
+		return `${visible}\n... [truncated ~${totalTokens - estimateTokenCount(visible)} tokens]`;
 	};
 	let low = 0;
 	let high = text.length;
@@ -18748,7 +18803,7 @@ function truncateStringByTokens(text, maxTokens) {
 	while (low <= high) {
 		const mid = Math.floor((low + high) / 2);
 		const candidate = buildCandidate(mid);
-		if ((0, tokenx.estimateTokenCount)(candidate) <= maxTokens) {
+		if (estimateTokenCount(candidate) <= maxTokens) {
 			best = candidate;
 			low = mid + 1;
 		} else high = mid - 1;
@@ -21616,7 +21671,7 @@ var TokenCounter = class TokenCounter {
 	*/
 	countString(text) {
 		if (!text) return 0;
-		return (0, tokenx.estimateTokenCount)(text);
+		return estimateTokenCount(text);
 	}
 	readOrPersistPartEstimate(part, kind, payload) {
 		const key = buildEstimateKey(kind, payload);
@@ -22544,7 +22599,7 @@ function formatTimestamp(date) {
 	return date.toISOString().replace("T", " ").replace(/\.\d{3}Z$/, "Z");
 }
 function truncateByTokens(text, maxTokens, hint) {
-	if ((0, tokenx.estimateTokenCount)(text) <= maxTokens) return {
+	if (estimateTokenCount(text) <= maxTokens) return {
 		text,
 		wasTruncated: false
 	};
@@ -22558,7 +22613,7 @@ function chunkTextByTokens(text, maxTokens, charOffset = 0) {
 	const startCode = text.charCodeAt(startOffset);
 	if (startCode >= 56320 && startCode <= 57343) startOffset += 1;
 	const remaining = text.slice(startOffset);
-	if (!remaining || (0, tokenx.estimateTokenCount)(remaining) <= maxTokens) return {
+	if (!remaining || estimateTokenCount(remaining) <= maxTokens) return {
 		text: remaining,
 		charOffset: startOffset,
 		truncated: false
@@ -22569,7 +22624,7 @@ function chunkTextByTokens(text, maxTokens, charOffset = 0) {
 	while (low <= high) {
 		const mid = Math.floor((low + high) / 2);
 		const candidate = safeSlice(remaining, mid);
-		const candidateTokens = (0, tokenx.estimateTokenCount)(candidate);
+		const candidateTokens = estimateTokenCount(candidate);
 		if (candidate && candidateTokens <= maxTokens) {
 			best = candidate;
 			low = mid + 1;
@@ -22745,7 +22800,7 @@ function expandPriority(part) {
 }
 function renderFormattedParts(parts, timestamps, options) {
 	const text = buildRenderedText(parts, timestamps);
-	let totalTokens = (0, tokenx.estimateTokenCount)(text);
+	let totalTokens = estimateTokenCount(text);
 	if (totalTokens > options.maxTokens) return {
 		text: truncateStringByTokens(text, options.maxTokens),
 		truncated: true,
@@ -22764,8 +22819,8 @@ function renderFormattedParts(parts, timestamps, options) {
 	for (const { part, index } of truncatedIndices) {
 		if (remaining <= 0) break;
 		const maxTokens = expandLimit(part);
-		const fullTokens = (0, tokenx.estimateTokenCount)(part.fullText);
-		const currentTokens = (0, tokenx.estimateTokenCount)(part.text);
+		const fullTokens = estimateTokenCount(part.fullText);
+		const currentTokens = estimateTokenCount(part.text);
 		const targetTokens = Math.min(fullTokens, maxTokens);
 		const delta = targetTokens - currentTokens;
 		if (delta <= 0) continue;
@@ -22779,7 +22834,7 @@ function renderFormattedParts(parts, timestamps, options) {
 			const expandedLimit = Math.min(currentTokens + remaining, maxTokens);
 			const hint = `recall cursor="${part.messageId}" partIndex=${part.partIndex} detail="high"`;
 			const { text: expanded } = truncateByTokens(part.fullText, expandedLimit, hint);
-			const expandedDelta = (0, tokenx.estimateTokenCount)(expanded) - currentTokens;
+			const expandedDelta = estimateTokenCount(expanded) - currentTokens;
 			parts[index] = {
 				...part,
 				text: expanded
@@ -22788,7 +22843,7 @@ function renderFormattedParts(parts, timestamps, options) {
 		}
 	}
 	const expanded = buildRenderedText(parts, timestamps);
-	const expandedTokens = (0, tokenx.estimateTokenCount)(expanded);
+	const expandedTokens = estimateTokenCount(expanded);
 	if (expandedTokens <= options.maxTokens) return {
 		text: expanded,
 		truncated: false,
@@ -25182,6 +25237,10 @@ var SyncObservationStrategy = class extends ObservationStrategy {
 	async persist(processed) {
 		const { record, threadId, resourceId, messages } = this.opts;
 		const thread = await this.storage.getThreadById({ threadId });
+		if (!await this.storage.getObservationalMemory(record.threadId, record.resourceId)) {
+			omDebug(`[OM:sync-obs] skipping persist for thread ${threadId}: observational memory record is gone`);
+			return;
+		}
 		let threadUpdateMarker;
 		if (thread) {
 			const oldTitle = thread.title?.trim();
@@ -25363,6 +25422,10 @@ var AsyncBufferObservationStrategy = class extends ObservationStrategy {
 	async persist(processed) {
 		if (!processed.observations) return;
 		const { record, threadId, resourceId, messages } = this.opts;
+		if (!await this.storage.getObservationalMemory(record.threadId, record.resourceId)) {
+			omDebug(`[OM:asyncBuffer] skipping persist for thread ${threadId}: observational memory record is gone`);
+			return;
+		}
 		const messageTokens = await this.tokenCounter.countMessagesAsync(messages);
 		await withRetry(() => this.storage.updateBufferedObservations({
 			id: record.id,
@@ -26356,6 +26419,10 @@ var ObservationTurn = class {
 		if (!this._context) throw new Error("Turn not started — call start() first");
 		return this._context;
 	}
+	/** Whether the turn has been ended and can no longer accept steps. */
+	get ended() {
+		return this._ended;
+	}
 	/** The current step, if one exists. */
 	get currentStep() {
 		return this._currentStep;
@@ -30092,9 +30159,11 @@ ${formattedMessages}
 		let lifecycleError;
 		let observationStarted = false;
 		let generationBefore = -1;
+		let recordInsideLock;
 		try {
 			await this.withLock(lockKey, async () => {
 				const freshRecord = await this.getOrCreateRecord(threadId, resourceId);
+				recordInsideLock = freshRecord;
 				generationBefore = freshRecord.generationCount;
 				const unobservedMessages = messages ? this.getUnobservedMessages(messages, freshRecord) : await this.loadMessagesFromStorage(threadId, resourceId, freshRecord.lastObservedAt ? new Date(freshRecord.lastObservedAt) : void 0);
 				if (!this.meetsObservationThreshold({
@@ -30137,7 +30206,7 @@ ${formattedMessages}
 			omDebug(`[OM:hooks] onObservationEnd hook failed after cycle failure: ${endHookError instanceof Error ? endHookError.message : String(endHookError)}`);
 		}
 		if (lifecycleError !== void 0) throw lifecycleError;
-		const record = await this.getOrCreateRecord(threadId, resourceId);
+		const record = await this.getRecord(threadId, resourceId) ?? recordInsideLock;
 		const reflected = record.generationCount > generationBefore && generationBefore >= 0;
 		return {
 			observed,
@@ -30618,6 +30687,10 @@ var ObservationalMemoryProcessor = class {
 				if (this.turn === activeTurn) this.turn = void 0;
 				state.__omTurn = void 0;
 			}
+			if (activeTurn?.ended || this.turn?.ended) {
+				if (this.turn?.ended) this.turn = void 0;
+				state.__omTurn = void 0;
+			}
 			if (!this.turn || !state.__omTurn) {
 				if (this.turn && !state.__omTurn) await this.turn.end().catch(() => {});
 				this.turn = this.engine.beginTurn({
@@ -30718,11 +30791,11 @@ var ObservationalMemoryProcessor = class {
 		return this.engine.getTokenCounter().runWithModelContext(state.__omActorModelContext, async () => {
 			if ((0, _mastra_core_memory.parseMemoryRequestContext)(requestContext)?.memoryConfig?.readOnly) return messageList;
 			const turn = asLiveTurn(state.__omTurn) ?? this.turn;
-			if (turn) {
-				await turn.end();
-				this.turn = void 0;
-				state.__omTurn = void 0;
-			} else {
+			const liveTurn = turn && !turn.ended ? turn : void 0;
+			if (liveTurn) await liveTurn.end();
+			this.turn = void 0;
+			state.__omTurn = void 0;
+			if (!liveTurn) {
 				const newOutput = messageList.get.response.db();
 				const messagesToSave = [...messageList.get.input.db(), ...newOutput];
 				if (messagesToSave.length > 0 && context.threadId) await this.engine.persistMessages(messagesToSave, context.threadId, context.resourceId);
@@ -30932,6 +31005,7 @@ const DEFAULT_MESSAGE_RANGE = {
 };
 const DEFAULT_TOP_K = 4;
 const VECTOR_DELETE_BATCH_SIZE = 100;
+const OM_DELETE_DRAIN_TIMEOUT_MS = 1e4;
 const DEFAULT_EMBEDDING_CACHE_MAX_SIZE = 1e3;
 /**
 * Gives Mastra agents conversation history, with optional working memory,
@@ -31383,6 +31457,8 @@ var Memory = class Memory extends _mastra_core_memory.MastraMemory {
 		await this.deleteStoredThread(memoryStore, threadId, thread?.resourceId);
 	}
 	async deleteStoredThread(memoryStore, threadId, resourceId) {
+		const engine = this._omEngine ? await this._omEngine : this._omEngineInstance;
+		if (engine && resourceId) await engine.waitForBuffering(threadId, resourceId, OM_DELETE_DRAIN_TIMEOUT_MS);
 		await memoryStore.deleteThread({ threadId });
 		if (resourceId && memoryStore.supportsObservationalMemory) await memoryStore.clearObservationalMemory(threadId, resourceId);
 		if (this.vector) this.trackVectorCleanup(this.deleteThreadVectors(threadId));
@@ -33397,4 +33473,4 @@ Object.defineProperty(exports, "wrapInObservationGroup", {
 	}
 });
 
-//# sourceMappingURL=src-C4niho1P.cjs.map
+//# sourceMappingURL=src-QqTmOY5X.cjs.map