@mastra/memory 1.36.0-alpha.0 → 1.36.0-alpha.2

package/dist/{src-DhCg-LqR.cjs → src-OOB19zXe.cjs}

@@ -3733,7 +3733,7 @@ function trimStartOfStream() {
 	};
 }
 //#endregion
-//#region ../_vendored/ai_v5/dist/dist-DqVGDtnX.js
+//#region ../_vendored/ai_v5/dist/dist-CwCWl16R.js
 var marker$2$4 = "vercel.ai.error";
 var symbol$2$4 = Symbol.for(marker$2$4);
 var _a$2$4;
@@ -9020,7 +9020,7 @@ var object$1 = ({ schema: inputSchema }) => {
 	};
 };
 //#endregion
-//#region ../_vendored/ai_v6/dist/dist-BcG-LFmj.js
+//#region ../_vendored/ai_v6/dist/dist-B5Rt0qDx.js
 var marker$2 = "vercel.ai.error";
 var symbol$2 = Symbol.for(marker$2);
 var _a$2;
@@ -11565,7 +11565,7 @@ var _b14;
 	}
 });
 //#endregion
-//#region ../_vendored/ai_v6/dist/dist-t-eoX9xC.js
+//#region ../_vendored/ai_v6/dist/dist-BgUIUei9.js
 function getContext() {
 	return { headers: {} };
 }
@@ -22201,6 +22201,7 @@ function serializeNonImageFilePartForTokenCounting(part) {
 function isValidCacheEntry(entry, expectedKey, expectedSource) {
 	return Boolean(entry && entry.v === TOKEN_ESTIMATE_CACHE_VERSION && entry.source === expectedSource && entry.key === expectedKey && Number.isFinite(entry.tokens));
 }
+const modelContextStorage = new async_hooks.AsyncLocalStorage();
 function parseModelContext(model) {
 	if (!model) return void 0;
 	if (typeof model === "object") return model.provider || model.modelId ? {
@@ -22822,7 +22823,6 @@ async function fetchGoogleAttachmentTokenEstimate(modelId, part) {
 var TokenCounter = class TokenCounter {
 	cacheSource;
 	defaultModelContext;
-	modelContextStorage = new async_hooks.AsyncLocalStorage();
 	inFlightAttachmentCounts = /* @__PURE__ */ new Map();
 	multimodalToolResultCounts = /* @__PURE__ */ new WeakMap();
 	static TOKENS_PER_MESSAGE = 3.8;
@@ -22832,10 +22832,12 @@ var TokenCounter = class TokenCounter {
 		this.defaultModelContext = parseModelContext(options?.model);
 	}
 	runWithModelContext(model, fn) {
-		return this.modelContextStorage.run(parseModelContext(model), fn);
+		const contexts = new Map(modelContextStorage.getStore());
+		contexts.set(this, parseModelContext(model));
+		return modelContextStorage.run(contexts, fn);
 	}
 	getModelContext() {
-		return this.modelContextStorage.getStore() ?? this.defaultModelContext;
+		return modelContextStorage.getStore()?.get(this) ?? this.defaultModelContext;
 	}
 	/**
 	* Count tokens in a plain string
@@ -28039,7 +28041,7 @@ var ObservationStep = class {
 				if (obsResult.succeeded) {
 					observed = true;
 					didThresholdCleanup = true;
-					const observedIds = obsResult.activatedMessageIds ?? obsResult.record.observedMessageIds ?? [];
+					const observedIds = obsResult.cleanupMessageIds;
 					const minRemaining = resolveRetentionFloor(om.getObservationConfig().bufferActivation ?? 1, statusSnapshot.threshold);
 					await om.cleanupMessages({
 						threadId,
@@ -28106,8 +28108,8 @@ var ObservationStep = class {
 	}
 	/**
 	* Run the full threshold observation pipeline:
-	* waitForBuffering → re-check → activate → reflect → observe (sync fallback when
-	* buffered activation did not happen)
+	* waitForBuffering → re-check → activate buffered chunks → reflect → observe
+	* (sync fallback when pending tokens are still at or above the threshold)
 	*/
 	async runThresholdObservation() {
 		const { threadId, resourceId, messageList } = this.turn;
@@ -28115,59 +28117,76 @@ var ObservationStep = class {
 		await om.waitForBuffering(threadId, resourceId);
 		await this.turn.refreshRecord();
 		const observableMessages = this.seededResponseMessage ? getObservableMessages(messageList).filter((msg) => msg.id !== this.turn.responseMessageId) : getObservableMessages(messageList);
-		const freshStatus = await om.getStatus({
+		let status = await om.getStatus({
 			threadId,
 			resourceId,
 			record: this.turn.record,
 			messages: observableMessages
 		});
-		if (!freshStatus.shouldObserve) return {
+		if (!status.shouldObserve) return {
 			succeeded: false,
-			record: freshStatus.record
+			record: status.record,
+			cleanupMessageIds: []
 		};
-		if (freshStatus.canActivate) {
+		let pendingMessages = observableMessages;
+		const activatedMessageIds = [];
+		let activated = false;
+		while (status.shouldObserve && status.canActivate) {
 			const activation = await om.activate({
 				threadId,
 				resourceId,
 				record: this.turn.record,
-				messages: observableMessages,
+				messages: pendingMessages,
+				pendingTokens: status.pendingTokens,
 				currentModel: this.turn.actorModelContext,
 				writer: this.turn.writer,
 				messageList
 			});
 			this.turn.setRecord(activation.record);
-			if (activation.activated) {
-				const postActivationRecord = activation.record;
-				await om.reflector.maybeReflect({
-					record: postActivationRecord,
-					observationTokens: postActivationRecord.observationTokenCount ?? 0,
+			if (!activation.activated) break;
+			activated = true;
+			const ids = new Set(activation.activatedMessageIds ?? []);
+			activatedMessageIds.push(...ids);
+			pendingMessages = pendingMessages.filter((msg) => !ids.has(msg.id));
+			status = await om.getStatus({
+				threadId,
+				resourceId,
+				record: this.turn.record,
+				messages: pendingMessages
+			});
+		}
+		if (activated) {
+			const postActivationRecord = this.turn.record;
+			await om.reflector.maybeReflect({
+				record: postActivationRecord,
+				observationTokens: postActivationRecord.observationTokenCount ?? 0,
+				threadId,
+				writer: this.turn.writer,
+				messageList,
+				currentModel: this.turn.actorModelContext,
+				requestContext: this.turn.requestContext,
+				observabilityContext: this.turn.observabilityContext,
+				lastActivityAt: getLastActivityFromMessages(getObservableMessages(messageList)),
+				reflectionHooks: om.composeHooks(void 0, {
 					threadId,
-					writer: this.turn.writer,
-					messageList,
-					currentModel: this.turn.actorModelContext,
-					requestContext: this.turn.requestContext,
-					observabilityContext: this.turn.observabilityContext,
-					lastActivityAt: getLastActivityFromMessages(getObservableMessages(messageList)),
-					reflectionHooks: om.composeHooks(void 0, {
-						threadId,
-						resourceId,
-						trigger: "turn-sync"
-					}),
+					resourceId,
 					trigger: "turn-sync"
-				});
-				return {
-					succeeded: true,
-					record: activation.record,
-					activatedMessageIds: activation.activatedMessageIds
-				};
-			}
+				}),
+				trigger: "turn-sync"
+			});
+			if (!status.shouldObserve) return {
+				succeeded: true,
+				record: postActivationRecord,
+				activatedMessageIds,
+				cleanupMessageIds: activatedMessageIds
+			};
 		}
 		let obsResult;
 		try {
 			obsResult = await om.observe({
 				threadId,
 				resourceId,
-				messages: observableMessages,
+				messages: pendingMessages,
 				messageList,
 				trigger: "turn-sync",
 				agent: this.turn.agent,
@@ -28180,8 +28199,10 @@ var ObservationStep = class {
 		} catch (error) {
 			if (om.config.observation.failurePolicy !== "continue" || !isOmModelExecutionError(error) || error.failureKind !== "observer-model") throw error;
 			return {
-				succeeded: false,
-				record: freshStatus.record
+				succeeded: activated,
+				record: this.turn.record,
+				activatedMessageIds: activated ? activatedMessageIds : void 0,
+				cleanupMessageIds: activatedMessageIds
 			};
 		}
 		if (obsResult.observed) {
@@ -28208,8 +28229,10 @@ var ObservationStep = class {
 			if (messagesToSeal.length > 0) await om.persistMessages(messagesToSeal, threadId, resourceId);
 		}
 		return {
-			succeeded: obsResult.observed,
+			succeeded: activated || obsResult.observed,
 			record: obsResult.record,
+			activatedMessageIds: activated ? activatedMessageIds : void 0,
+			cleanupMessageIds: [.../* @__PURE__ */ new Set([...activatedMessageIds, ...obsResult.observed ? obsResult.record.observedMessageIds ?? [] : []])],
 			observerExchange: om.observer.lastExchange
 		};
 	}
@@ -31897,6 +31920,7 @@ ${formattedMessages}
 		let activateAfterIdleExpiredMs;
 		let previousModel;
 		let currentModel;
+		let livePendingTokens = opts.pendingTokens;
 		if (opts.checkThreshold) {
 			const thresholdMessages = opts.messages ?? await this.loadMessagesFromStorage(threadId, resourceId, record.lastObservedAt ? new Date(record.lastObservedAt) : void 0);
 			const activateAfterIdle = resolveActivationTTL(this.observationConfig.activateAfterIdle, opts.currentModel);
@@ -31925,6 +31949,7 @@ ${formattedMessages}
 					activated: false,
 					record
 				};
+				livePendingTokens ??= status.pendingTokens;
 			}
 		}
 		if (record.isBufferingObservation) {
@@ -31948,8 +31973,9 @@ ${formattedMessages}
 		const messageTokensThreshold = getMaxThreshold(this.getEffectiveMessageTokens(freshRecord));
 		const activationRatio = resolveActivationRatio(this.observationConfig.bufferActivation ?? .7, messageTokensThreshold);
 		const totalChunkMessageTokens = freshChunks.reduce((sum, c) => sum + (c.messageTokens ?? 0), 0);
-		const currentPendingTokens = freshRecord.pendingMessageTokens || totalChunkMessageTokens;
+		const currentPendingTokens = livePendingTokens ?? (freshRecord.pendingMessageTokens || totalChunkMessageTokens);
 		const forceMaxActivation = !!(this.observationConfig.blockAfter && currentPendingTokens >= this.observationConfig.blockAfter);
+		if (freshRecord.pendingMessageTokens !== currentPendingTokens) await this.storage.setPendingMessageTokens(freshRecord.id, currentPendingTokens);
 		const activationResult = await this.storage.swapBufferedToActive({
 			id: freshRecord.id,
 			activationRatio,
@@ -33294,22 +33320,32 @@ var Memory = class Memory extends _mastra_core_memory.MastraMemory {
 			if (resourceScope && !resourceId && config?.semanticRecall && vectorSearchString) throw new Error("Memory error: Resource-scoped semantic recall is enabled but no resourceId was provided. Either provide a resourceId or explicitly set semanticRecall.scope to 'thread'.");
 			let usage;
 			if (config?.semanticRecall && vectorSearchString && this.vector) {
-				const result = await this.embedMessageContent(vectorSearchString);
-				usage = result.usage;
-				const { embeddings, dimension } = result;
-				const { indexName } = await this.createEmbeddingIndex(dimension, config);
-				await Promise.all(embeddings.map(async (embedding) => {
-					if (typeof this.vector === `undefined`) throw new Error(`Tried to query vector index ${indexName} but this Memory instance doesn't have an attached vector db.`);
-					const scopeFilter = resourceScope ? { resource_id: resourceId } : { thread_id: threadId };
-					const userFilter = typeof config.semanticRecall === "object" ? config.semanticRecall.filter : void 0;
-					const combinedFilter = userFilter ? { $and: [scopeFilter, userFilter] } : scopeFilter;
+				const scopeFilter = resourceScope ? { resource_id: resourceId } : { thread_id: threadId };
+				const userFilter = typeof config.semanticRecall === "object" ? config.semanticRecall.filter : void 0;
+				const combinedFilter = userFilter ? { $and: [scopeFilter, userFilter] } : scopeFilter;
+				if (this.isSelfEmbedding) {
+					const { indexName } = await this.createEmbeddingIndex(void 0, config);
 					vectorResults.push(...await this.vector.query({
 						indexName,
-						queryVector: embedding,
+						queryText: vectorSearchString,
 						topK: vectorConfig.topK,
 						filter: combinedFilter
 					}));
-				}));
+				} else {
+					const result = await this.embedMessageContent(vectorSearchString);
+					usage = result.usage;
+					const { embeddings, dimension } = result;
+					const { indexName } = await this.createEmbeddingIndex(dimension, config);
+					await Promise.all(embeddings.map(async (embedding) => {
+						if (typeof this.vector === `undefined`) throw new Error(`Tried to query vector index ${indexName} but this Memory instance doesn't have an attached vector db.`);
+						vectorResults.push(...await this.vector.query({
+							indexName,
+							queryVector: embedding,
+							topK: vectorConfig.topK,
+							filter: combinedFilter
+						}));
+					}));
+				}
 			}
 			const threshold = (typeof config.semanticRecall === "object" ? config.semanticRecall : void 0)?.threshold;
 			const filteredVectorResults = threshold !== void 0 ? vectorResults.filter((r) => r.score >= threshold) : vectorResults;
@@ -33467,11 +33503,13 @@ var Memory = class Memory extends _mastra_core_memory.MastraMemory {
 		if (this.vector) this.trackVectorCleanup(this.deleteThreadVectors(threadId));
 	}
 	/**
-	* Prefix shared by every message index. The index for the default embedding
-	* dimension is named with the bare prefix; other dimensions add a suffix.
+	* Prefix shared by every message index. The index for the default embedding dimension is named
+	* with the bare prefix; other dimensions add a dimension suffix, and a store that embeds the
+	* text itself adds its own. Built here rather than read from `getEmbeddingIndexName()`, which
+	* answers for one configuration and so would not match indexes written under another.
 	*/
 	get messageIndexPrefix() {
-		return this.getEmbeddingIndexName();
+		return `memory${this.vector?.indexSeparator ?? "_"}messages`;
 	}
 	/**
 	* Prefix shared by every observation index. Each observation index adds a dimension suffix.
@@ -33756,13 +33794,24 @@ ${workingMemory}`;
 						if (joined) textForEmbedding = joined;
 					}
 					if (!textForEmbedding) return;
-					const result = await this.embedMessageContent(textForEmbedding);
-					dimension = result.dimension;
-					if (result.usage?.tokens) totalTokens += result.usage.tokens;
+					let embeddings = [];
+					let documents;
+					let chunks;
+					if (this.isSelfEmbedding) {
+						chunks = this.chunkText(textForEmbedding);
+						documents = chunks;
+					} else {
+						const result = await this.embedMessageContent(textForEmbedding);
+						dimension = result.dimension;
+						if (result.usage?.tokens) totalTokens += result.usage.tokens;
+						embeddings = result.embeddings;
+						chunks = result.chunks;
+					}
 					const threadMetadata = message.threadId ? threadMetadataMap.get(message.threadId) || {} : {};
 					embeddingData.push({
-						embeddings: result.embeddings,
-						metadata: result.chunks.map(() => ({
+						embeddings,
+						documents,
+						metadata: chunks.map(() => ({
 							...threadMetadata,
 							message_id: message.id,
 							thread_id: message.threadId,
@@ -33773,16 +33822,23 @@ ${workingMemory}`;
 						}))
 					});
 				}));
-				if (embeddingData.length > 0 && dimension !== void 0) {
+				if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 					if (typeof this.vector === `undefined`) throw new Error(`Tried to upsert embeddings but this Memory instance doesn't have an attached vector db.`);
 					const { indexName } = await this.createEmbeddingIndex(dimension, config);
 					const allVectors = [];
 					const allMetadata = [];
+					const allDocuments = [];
 					for (const data of embeddingData) {
 						allVectors.push(...data.embeddings);
+						if (data.documents) allDocuments.push(...data.documents);
 						allMetadata.push(...data.metadata);
 					}
-					await this.vector.upsert({
+					if (this.isSelfEmbedding) await this.vector.upsert({
+						indexName,
+						documents: allDocuments,
+						metadata: allMetadata
+					});
+					else await this.vector.upsert({
 						indexName,
 						vectors: allVectors,
 						metadata: allMetadata
@@ -34567,11 +34623,22 @@ Notes:
 						}
 					}
 					if (textForEmbedding) {
-						const result = await this.embedMessageContent(textForEmbedding);
-						dimension = result.dimension;
+						let embeddings = [];
+						let documents;
+						let chunks;
+						if (this.isSelfEmbedding) {
+							chunks = this.chunkText(textForEmbedding);
+							documents = chunks;
+						} else {
+							const result = await this.embedMessageContent(textForEmbedding);
+							dimension = result.dimension;
+							embeddings = result.embeddings;
+							chunks = result.chunks;
+						}
 						embeddingData.push({
-							embeddings: result.embeddings,
-							metadata: result.chunks.map(() => ({
+							embeddings,
+							documents,
+							metadata: chunks.map(() => ({
 								message_id: message.id,
 								thread_id: existingMessage.threadId,
 								resource_id: existingMessage.resourceId,
@@ -34606,15 +34673,22 @@ Notes:
 				} catch {
 					this.logger.debug("Failed to clean up old vectors during message update");
 				}
-				if (embeddingData.length > 0 && dimension !== void 0) {
+				if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 					const { indexName } = await this.createEmbeddingIndex(dimension, config);
 					const allVectors = [];
 					const allMetadata = [];
+					const allDocuments = [];
 					for (const data of embeddingData) {
 						allVectors.push(...data.embeddings);
+						if (data.documents) allDocuments.push(...data.documents);
 						allMetadata.push(...data.metadata);
 					}
-					await this.vector.upsert({
+					if (this.isSelfEmbedding) await this.vector.upsert({
+						indexName,
+						documents: allDocuments,
+						metadata: allMetadata
+					});
+					else await this.vector.upsert({
 						indexName,
 						vectors: allVectors,
 						metadata: allMetadata
@@ -34801,7 +34875,7 @@ Notes:
 			await this.rollbackCopiedThread(memoryStore, result.thread, "OM clone", false, priorDestinationResourceWm);
 			throw error;
 		}
-		if (this.vector && this.embedder && config.semanticRecall) try {
+		if (this.vector && (this.embedder || this.isSelfEmbedding) && config.semanticRecall) try {
 			await this.embedCopiedMessagesInBatches(memoryStore, result, config);
 		} catch (error) {
 			await this.rollbackCopiedThread(memoryStore, result.thread, "embedding", true, priorDestinationResourceWm);
@@ -34870,7 +34944,7 @@ Notes:
 	async updateThreadResourceId({ threadId, resourceId, memoryConfig }) {
 		const memoryStore = await this.getMemoryStore();
 		const config = this.getMergedThreadConfig(memoryConfig);
-		const migratesVectors = Boolean(this.vector && this.embedder && config.semanticRecall);
+		const migratesVectors = Boolean(this.vector && (this.embedder || this.isSelfEmbedding) && config.semanticRecall);
 		if (!migratesVectors) {
 			const existing = await memoryStore.getThreadById({ threadId });
 			if (existing && existing.resourceId === resourceId) return existing;
@@ -34976,7 +35050,7 @@ Notes:
 	* This is similar to the embedding logic in saveMessages but operates on already-saved messages.
 	*/
 	async embedClonedMessages(messages, config) {
-		if (!this.vector || !this.embedder) return;
+		if (!this.vector || !this.embedder && !this.isSelfEmbedding) return;
 		const embeddingData = [];
 		let dimension;
 		await Promise.all(messages.map(async (message) => {
@@ -34987,11 +35061,22 @@ Notes:
 				if (joined) textForEmbedding = joined;
 			}
 			if (!textForEmbedding) return;
-			const result = await this.embedMessageContent(textForEmbedding);
-			dimension = result.dimension;
+			let embeddings = [];
+			let documents;
+			let chunks;
+			if (this.isSelfEmbedding) {
+				chunks = this.chunkText(textForEmbedding);
+				documents = chunks;
+			} else {
+				const result = await this.embedMessageContent(textForEmbedding);
+				dimension = result.dimension;
+				embeddings = result.embeddings;
+				chunks = result.chunks;
+			}
 			embeddingData.push({
-				embeddings: result.embeddings,
-				metadata: result.chunks.map(() => ({
+				embeddings,
+				documents,
+				metadata: chunks.map(() => ({
 					message_id: message.id,
 					thread_id: message.threadId,
 					resource_id: message.resourceId,
@@ -35001,15 +35086,22 @@ Notes:
 				}))
 			});
 		}));
-		if (embeddingData.length > 0 && dimension !== void 0) {
+		if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 			const { indexName } = await this.createEmbeddingIndex(dimension, config);
 			const allVectors = [];
 			const allMetadata = [];
+			const allDocuments = [];
 			for (const data of embeddingData) {
 				allVectors.push(...data.embeddings);
+				if (data.documents) allDocuments.push(...data.documents);
 				allMetadata.push(...data.metadata);
 			}
-			await this.vector.upsert({
+			if (this.isSelfEmbedding) await this.vector.upsert({
+				indexName,
+				documents: allDocuments,
+				metadata: allMetadata
+			});
+			else await this.vector.upsert({
 				indexName,
 				vectors: allVectors,
 				metadata: allMetadata
@@ -35498,4 +35590,4 @@ Object.defineProperty(exports, "wrapInObservationGroup", {
 	}
 });
 
-//# sourceMappingURL=src-DhCg-LqR.cjs.map
+//# sourceMappingURL=src-OOB19zXe.cjs.map