@mastra/memory 1.36.0-alpha.0 → 1.36.0-alpha.2

package/dist/{src-BV9eXuMM.js → src-BxI-PDpC.js}

@@ -3712,7 +3712,7 @@ function trimStartOfStream() {
 	};
 }
 //#endregion
-//#region ../_vendored/ai_v5/dist/dist-DqVGDtnX.js
+//#region ../_vendored/ai_v5/dist/dist-CwCWl16R.js
 var marker$2$4 = "vercel.ai.error";
 var symbol$2$4 = Symbol.for(marker$2$4);
 var _a$2$4;
@@ -8999,7 +8999,7 @@ var object$1 = ({ schema: inputSchema }) => {
 	};
 };
 //#endregion
-//#region ../_vendored/ai_v6/dist/dist-BcG-LFmj.js
+//#region ../_vendored/ai_v6/dist/dist-B5Rt0qDx.js
 var marker$2 = "vercel.ai.error";
 var symbol$2 = Symbol.for(marker$2);
 var _a$2;
@@ -11544,7 +11544,7 @@ var _b14;
 	}
 });
 //#endregion
-//#region ../_vendored/ai_v6/dist/dist-t-eoX9xC.js
+//#region ../_vendored/ai_v6/dist/dist-BgUIUei9.js
 function getContext() {
 	return { headers: {} };
 }
@@ -22180,6 +22180,7 @@ function serializeNonImageFilePartForTokenCounting(part) {
 function isValidCacheEntry(entry, expectedKey, expectedSource) {
 	return Boolean(entry && entry.v === TOKEN_ESTIMATE_CACHE_VERSION && entry.source === expectedSource && entry.key === expectedKey && Number.isFinite(entry.tokens));
 }
+const modelContextStorage = new AsyncLocalStorage();
 function parseModelContext(model) {
 	if (!model) return void 0;
 	if (typeof model === "object") return model.provider || model.modelId ? {
@@ -22801,7 +22802,6 @@ async function fetchGoogleAttachmentTokenEstimate(modelId, part) {
 var TokenCounter = class TokenCounter {
 	cacheSource;
 	defaultModelContext;
-	modelContextStorage = new AsyncLocalStorage();
 	inFlightAttachmentCounts = /* @__PURE__ */ new Map();
 	multimodalToolResultCounts = /* @__PURE__ */ new WeakMap();
 	static TOKENS_PER_MESSAGE = 3.8;
@@ -22811,10 +22811,12 @@ var TokenCounter = class TokenCounter {
 		this.defaultModelContext = parseModelContext(options?.model);
 	}
 	runWithModelContext(model, fn) {
-		return this.modelContextStorage.run(parseModelContext(model), fn);
+		const contexts = new Map(modelContextStorage.getStore());
+		contexts.set(this, parseModelContext(model));
+		return modelContextStorage.run(contexts, fn);
 	}
 	getModelContext() {
-		return this.modelContextStorage.getStore() ?? this.defaultModelContext;
+		return modelContextStorage.getStore()?.get(this) ?? this.defaultModelContext;
 	}
 	/**
 	* Count tokens in a plain string
@@ -28018,7 +28020,7 @@ var ObservationStep = class {
 				if (obsResult.succeeded) {
 					observed = true;
 					didThresholdCleanup = true;
-					const observedIds = obsResult.activatedMessageIds ?? obsResult.record.observedMessageIds ?? [];
+					const observedIds = obsResult.cleanupMessageIds;
 					const minRemaining = resolveRetentionFloor(om.getObservationConfig().bufferActivation ?? 1, statusSnapshot.threshold);
 					await om.cleanupMessages({
 						threadId,
@@ -28085,8 +28087,8 @@ var ObservationStep = class {
 	}
 	/**
 	* Run the full threshold observation pipeline:
-	* waitForBuffering → re-check → activate → reflect → observe (sync fallback when
-	* buffered activation did not happen)
+	* waitForBuffering → re-check → activate buffered chunks → reflect → observe
+	* (sync fallback when pending tokens are still at or above the threshold)
 	*/
 	async runThresholdObservation() {
 		const { threadId, resourceId, messageList } = this.turn;
@@ -28094,59 +28096,76 @@ var ObservationStep = class {
 		await om.waitForBuffering(threadId, resourceId);
 		await this.turn.refreshRecord();
 		const observableMessages = this.seededResponseMessage ? getObservableMessages(messageList).filter((msg) => msg.id !== this.turn.responseMessageId) : getObservableMessages(messageList);
-		const freshStatus = await om.getStatus({
+		let status = await om.getStatus({
 			threadId,
 			resourceId,
 			record: this.turn.record,
 			messages: observableMessages
 		});
-		if (!freshStatus.shouldObserve) return {
+		if (!status.shouldObserve) return {
 			succeeded: false,
-			record: freshStatus.record
+			record: status.record,
+			cleanupMessageIds: []
 		};
-		if (freshStatus.canActivate) {
+		let pendingMessages = observableMessages;
+		const activatedMessageIds = [];
+		let activated = false;
+		while (status.shouldObserve && status.canActivate) {
 			const activation = await om.activate({
 				threadId,
 				resourceId,
 				record: this.turn.record,
-				messages: observableMessages,
+				messages: pendingMessages,
+				pendingTokens: status.pendingTokens,
 				currentModel: this.turn.actorModelContext,
 				writer: this.turn.writer,
 				messageList
 			});
 			this.turn.setRecord(activation.record);
-			if (activation.activated) {
-				const postActivationRecord = activation.record;
-				await om.reflector.maybeReflect({
-					record: postActivationRecord,
-					observationTokens: postActivationRecord.observationTokenCount ?? 0,
+			if (!activation.activated) break;
+			activated = true;
+			const ids = new Set(activation.activatedMessageIds ?? []);
+			activatedMessageIds.push(...ids);
+			pendingMessages = pendingMessages.filter((msg) => !ids.has(msg.id));
+			status = await om.getStatus({
+				threadId,
+				resourceId,
+				record: this.turn.record,
+				messages: pendingMessages
+			});
+		}
+		if (activated) {
+			const postActivationRecord = this.turn.record;
+			await om.reflector.maybeReflect({
+				record: postActivationRecord,
+				observationTokens: postActivationRecord.observationTokenCount ?? 0,
+				threadId,
+				writer: this.turn.writer,
+				messageList,
+				currentModel: this.turn.actorModelContext,
+				requestContext: this.turn.requestContext,
+				observabilityContext: this.turn.observabilityContext,
+				lastActivityAt: getLastActivityFromMessages(getObservableMessages(messageList)),
+				reflectionHooks: om.composeHooks(void 0, {
 					threadId,
-					writer: this.turn.writer,
-					messageList,
-					currentModel: this.turn.actorModelContext,
-					requestContext: this.turn.requestContext,
-					observabilityContext: this.turn.observabilityContext,
-					lastActivityAt: getLastActivityFromMessages(getObservableMessages(messageList)),
-					reflectionHooks: om.composeHooks(void 0, {
-						threadId,
-						resourceId,
-						trigger: "turn-sync"
-					}),
+					resourceId,
 					trigger: "turn-sync"
-				});
-				return {
-					succeeded: true,
-					record: activation.record,
-					activatedMessageIds: activation.activatedMessageIds
-				};
-			}
+				}),
+				trigger: "turn-sync"
+			});
+			if (!status.shouldObserve) return {
+				succeeded: true,
+				record: postActivationRecord,
+				activatedMessageIds,
+				cleanupMessageIds: activatedMessageIds
+			};
 		}
 		let obsResult;
 		try {
 			obsResult = await om.observe({
 				threadId,
 				resourceId,
-				messages: observableMessages,
+				messages: pendingMessages,
 				messageList,
 				trigger: "turn-sync",
 				agent: this.turn.agent,
@@ -28159,8 +28178,10 @@ var ObservationStep = class {
 		} catch (error) {
 			if (om.config.observation.failurePolicy !== "continue" || !isOmModelExecutionError(error) || error.failureKind !== "observer-model") throw error;
 			return {
-				succeeded: false,
-				record: freshStatus.record
+				succeeded: activated,
+				record: this.turn.record,
+				activatedMessageIds: activated ? activatedMessageIds : void 0,
+				cleanupMessageIds: activatedMessageIds
 			};
 		}
 		if (obsResult.observed) {
@@ -28187,8 +28208,10 @@ var ObservationStep = class {
 			if (messagesToSeal.length > 0) await om.persistMessages(messagesToSeal, threadId, resourceId);
 		}
 		return {
-			succeeded: obsResult.observed,
+			succeeded: activated || obsResult.observed,
 			record: obsResult.record,
+			activatedMessageIds: activated ? activatedMessageIds : void 0,
+			cleanupMessageIds: [.../* @__PURE__ */ new Set([...activatedMessageIds, ...obsResult.observed ? obsResult.record.observedMessageIds ?? [] : []])],
 			observerExchange: om.observer.lastExchange
 		};
 	}
@@ -31876,6 +31899,7 @@ ${formattedMessages}
 		let activateAfterIdleExpiredMs;
 		let previousModel;
 		let currentModel;
+		let livePendingTokens = opts.pendingTokens;
 		if (opts.checkThreshold) {
 			const thresholdMessages = opts.messages ?? await this.loadMessagesFromStorage(threadId, resourceId, record.lastObservedAt ? new Date(record.lastObservedAt) : void 0);
 			const activateAfterIdle = resolveActivationTTL(this.observationConfig.activateAfterIdle, opts.currentModel);
@@ -31904,6 +31928,7 @@ ${formattedMessages}
 					activated: false,
 					record
 				};
+				livePendingTokens ??= status.pendingTokens;
 			}
 		}
 		if (record.isBufferingObservation) {
@@ -31927,8 +31952,9 @@ ${formattedMessages}
 		const messageTokensThreshold = getMaxThreshold(this.getEffectiveMessageTokens(freshRecord));
 		const activationRatio = resolveActivationRatio(this.observationConfig.bufferActivation ?? .7, messageTokensThreshold);
 		const totalChunkMessageTokens = freshChunks.reduce((sum, c) => sum + (c.messageTokens ?? 0), 0);
-		const currentPendingTokens = freshRecord.pendingMessageTokens || totalChunkMessageTokens;
+		const currentPendingTokens = livePendingTokens ?? (freshRecord.pendingMessageTokens || totalChunkMessageTokens);
 		const forceMaxActivation = !!(this.observationConfig.blockAfter && currentPendingTokens >= this.observationConfig.blockAfter);
+		if (freshRecord.pendingMessageTokens !== currentPendingTokens) await this.storage.setPendingMessageTokens(freshRecord.id, currentPendingTokens);
 		const activationResult = await this.storage.swapBufferedToActive({
 			id: freshRecord.id,
 			activationRatio,
@@ -33273,22 +33299,32 @@ var Memory = class Memory extends MastraMemory {
 			if (resourceScope && !resourceId && config?.semanticRecall && vectorSearchString) throw new Error("Memory error: Resource-scoped semantic recall is enabled but no resourceId was provided. Either provide a resourceId or explicitly set semanticRecall.scope to 'thread'.");
 			let usage;
 			if (config?.semanticRecall && vectorSearchString && this.vector) {
-				const result = await this.embedMessageContent(vectorSearchString);
-				usage = result.usage;
-				const { embeddings, dimension } = result;
-				const { indexName } = await this.createEmbeddingIndex(dimension, config);
-				await Promise.all(embeddings.map(async (embedding) => {
-					if (typeof this.vector === `undefined`) throw new Error(`Tried to query vector index ${indexName} but this Memory instance doesn't have an attached vector db.`);
-					const scopeFilter = resourceScope ? { resource_id: resourceId } : { thread_id: threadId };
-					const userFilter = typeof config.semanticRecall === "object" ? config.semanticRecall.filter : void 0;
-					const combinedFilter = userFilter ? { $and: [scopeFilter, userFilter] } : scopeFilter;
+				const scopeFilter = resourceScope ? { resource_id: resourceId } : { thread_id: threadId };
+				const userFilter = typeof config.semanticRecall === "object" ? config.semanticRecall.filter : void 0;
+				const combinedFilter = userFilter ? { $and: [scopeFilter, userFilter] } : scopeFilter;
+				if (this.isSelfEmbedding) {
+					const { indexName } = await this.createEmbeddingIndex(void 0, config);
 					vectorResults.push(...await this.vector.query({
 						indexName,
-						queryVector: embedding,
+						queryText: vectorSearchString,
 						topK: vectorConfig.topK,
 						filter: combinedFilter
 					}));
-				}));
+				} else {
+					const result = await this.embedMessageContent(vectorSearchString);
+					usage = result.usage;
+					const { embeddings, dimension } = result;
+					const { indexName } = await this.createEmbeddingIndex(dimension, config);
+					await Promise.all(embeddings.map(async (embedding) => {
+						if (typeof this.vector === `undefined`) throw new Error(`Tried to query vector index ${indexName} but this Memory instance doesn't have an attached vector db.`);
+						vectorResults.push(...await this.vector.query({
+							indexName,
+							queryVector: embedding,
+							topK: vectorConfig.topK,
+							filter: combinedFilter
+						}));
+					}));
+				}
 			}
 			const threshold = (typeof config.semanticRecall === "object" ? config.semanticRecall : void 0)?.threshold;
 			const filteredVectorResults = threshold !== void 0 ? vectorResults.filter((r) => r.score >= threshold) : vectorResults;
@@ -33446,11 +33482,13 @@ var Memory = class Memory extends MastraMemory {
 		if (this.vector) this.trackVectorCleanup(this.deleteThreadVectors(threadId));
 	}
 	/**
-	* Prefix shared by every message index. The index for the default embedding
-	* dimension is named with the bare prefix; other dimensions add a suffix.
+	* Prefix shared by every message index. The index for the default embedding dimension is named
+	* with the bare prefix; other dimensions add a dimension suffix, and a store that embeds the
+	* text itself adds its own. Built here rather than read from `getEmbeddingIndexName()`, which
+	* answers for one configuration and so would not match indexes written under another.
 	*/
 	get messageIndexPrefix() {
-		return this.getEmbeddingIndexName();
+		return `memory${this.vector?.indexSeparator ?? "_"}messages`;
 	}
 	/**
 	* Prefix shared by every observation index. Each observation index adds a dimension suffix.
@@ -33735,13 +33773,24 @@ ${workingMemory}`;
 						if (joined) textForEmbedding = joined;
 					}
 					if (!textForEmbedding) return;
-					const result = await this.embedMessageContent(textForEmbedding);
-					dimension = result.dimension;
-					if (result.usage?.tokens) totalTokens += result.usage.tokens;
+					let embeddings = [];
+					let documents;
+					let chunks;
+					if (this.isSelfEmbedding) {
+						chunks = this.chunkText(textForEmbedding);
+						documents = chunks;
+					} else {
+						const result = await this.embedMessageContent(textForEmbedding);
+						dimension = result.dimension;
+						if (result.usage?.tokens) totalTokens += result.usage.tokens;
+						embeddings = result.embeddings;
+						chunks = result.chunks;
+					}
 					const threadMetadata = message.threadId ? threadMetadataMap.get(message.threadId) || {} : {};
 					embeddingData.push({
-						embeddings: result.embeddings,
-						metadata: result.chunks.map(() => ({
+						embeddings,
+						documents,
+						metadata: chunks.map(() => ({
 							...threadMetadata,
 							message_id: message.id,
 							thread_id: message.threadId,
@@ -33752,16 +33801,23 @@ ${workingMemory}`;
 						}))
 					});
 				}));
-				if (embeddingData.length > 0 && dimension !== void 0) {
+				if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 					if (typeof this.vector === `undefined`) throw new Error(`Tried to upsert embeddings but this Memory instance doesn't have an attached vector db.`);
 					const { indexName } = await this.createEmbeddingIndex(dimension, config);
 					const allVectors = [];
 					const allMetadata = [];
+					const allDocuments = [];
 					for (const data of embeddingData) {
 						allVectors.push(...data.embeddings);
+						if (data.documents) allDocuments.push(...data.documents);
 						allMetadata.push(...data.metadata);
 					}
-					await this.vector.upsert({
+					if (this.isSelfEmbedding) await this.vector.upsert({
+						indexName,
+						documents: allDocuments,
+						metadata: allMetadata
+					});
+					else await this.vector.upsert({
 						indexName,
 						vectors: allVectors,
 						metadata: allMetadata
@@ -34546,11 +34602,22 @@ Notes:
 						}
 					}
 					if (textForEmbedding) {
-						const result = await this.embedMessageContent(textForEmbedding);
-						dimension = result.dimension;
+						let embeddings = [];
+						let documents;
+						let chunks;
+						if (this.isSelfEmbedding) {
+							chunks = this.chunkText(textForEmbedding);
+							documents = chunks;
+						} else {
+							const result = await this.embedMessageContent(textForEmbedding);
+							dimension = result.dimension;
+							embeddings = result.embeddings;
+							chunks = result.chunks;
+						}
 						embeddingData.push({
-							embeddings: result.embeddings,
-							metadata: result.chunks.map(() => ({
+							embeddings,
+							documents,
+							metadata: chunks.map(() => ({
 								message_id: message.id,
 								thread_id: existingMessage.threadId,
 								resource_id: existingMessage.resourceId,
@@ -34585,15 +34652,22 @@ Notes:
 				} catch {
 					this.logger.debug("Failed to clean up old vectors during message update");
 				}
-				if (embeddingData.length > 0 && dimension !== void 0) {
+				if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 					const { indexName } = await this.createEmbeddingIndex(dimension, config);
 					const allVectors = [];
 					const allMetadata = [];
+					const allDocuments = [];
 					for (const data of embeddingData) {
 						allVectors.push(...data.embeddings);
+						if (data.documents) allDocuments.push(...data.documents);
 						allMetadata.push(...data.metadata);
 					}
-					await this.vector.upsert({
+					if (this.isSelfEmbedding) await this.vector.upsert({
+						indexName,
+						documents: allDocuments,
+						metadata: allMetadata
+					});
+					else await this.vector.upsert({
 						indexName,
 						vectors: allVectors,
 						metadata: allMetadata
@@ -34780,7 +34854,7 @@ Notes:
 			await this.rollbackCopiedThread(memoryStore, result.thread, "OM clone", false, priorDestinationResourceWm);
 			throw error;
 		}
-		if (this.vector && this.embedder && config.semanticRecall) try {
+		if (this.vector && (this.embedder || this.isSelfEmbedding) && config.semanticRecall) try {
 			await this.embedCopiedMessagesInBatches(memoryStore, result, config);
 		} catch (error) {
 			await this.rollbackCopiedThread(memoryStore, result.thread, "embedding", true, priorDestinationResourceWm);
@@ -34849,7 +34923,7 @@ Notes:
 	async updateThreadResourceId({ threadId, resourceId, memoryConfig }) {
 		const memoryStore = await this.getMemoryStore();
 		const config = this.getMergedThreadConfig(memoryConfig);
-		const migratesVectors = Boolean(this.vector && this.embedder && config.semanticRecall);
+		const migratesVectors = Boolean(this.vector && (this.embedder || this.isSelfEmbedding) && config.semanticRecall);
 		if (!migratesVectors) {
 			const existing = await memoryStore.getThreadById({ threadId });
 			if (existing && existing.resourceId === resourceId) return existing;
@@ -34955,7 +35029,7 @@ Notes:
 	* This is similar to the embedding logic in saveMessages but operates on already-saved messages.
 	*/
 	async embedClonedMessages(messages, config) {
-		if (!this.vector || !this.embedder) return;
+		if (!this.vector || !this.embedder && !this.isSelfEmbedding) return;
 		const embeddingData = [];
 		let dimension;
 		await Promise.all(messages.map(async (message) => {
@@ -34966,11 +35040,22 @@ Notes:
 				if (joined) textForEmbedding = joined;
 			}
 			if (!textForEmbedding) return;
-			const result = await this.embedMessageContent(textForEmbedding);
-			dimension = result.dimension;
+			let embeddings = [];
+			let documents;
+			let chunks;
+			if (this.isSelfEmbedding) {
+				chunks = this.chunkText(textForEmbedding);
+				documents = chunks;
+			} else {
+				const result = await this.embedMessageContent(textForEmbedding);
+				dimension = result.dimension;
+				embeddings = result.embeddings;
+				chunks = result.chunks;
+			}
 			embeddingData.push({
-				embeddings: result.embeddings,
-				metadata: result.chunks.map(() => ({
+				embeddings,
+				documents,
+				metadata: chunks.map(() => ({
 					message_id: message.id,
 					thread_id: message.threadId,
 					resource_id: message.resourceId,
@@ -34980,15 +35065,22 @@ Notes:
 				}))
 			});
 		}));
-		if (embeddingData.length > 0 && dimension !== void 0) {
+		if (embeddingData.length > 0 && (this.isSelfEmbedding || dimension !== void 0)) {
 			const { indexName } = await this.createEmbeddingIndex(dimension, config);
 			const allVectors = [];
 			const allMetadata = [];
+			const allDocuments = [];
 			for (const data of embeddingData) {
 				allVectors.push(...data.embeddings);
+				if (data.documents) allDocuments.push(...data.documents);
 				allMetadata.push(...data.metadata);
 			}
-			await this.vector.upsert({
+			if (this.isSelfEmbedding) await this.vector.upsert({
+				indexName,
+				documents: allDocuments,
+				metadata: allMetadata
+			});
+			else await this.vector.upsert({
 				indexName,
 				vectors: allVectors,
 				metadata: allMetadata
@@ -35184,4 +35276,4 @@ Notes:
 //#endregion
 export { formatMessagesForObserver as A, OBSERVATION_CONTEXT_PROMPT_THREAD as B, SUMMARIZE_THREAD_DEFAULTS as C, buildObserverPrompt as D, OBSERVER_SYSTEM_PROMPT as E, parseAnchorId as F, StaleKnowledgeSemanticIndexError as G, getObservationContextPrompt as H, stripEphemeralAnchorIds as I, SUBCONSCIOUS_ACTIVITY_STATE_ID as J, SubconsciousRemindExtractor as K, OBSERVATIONAL_MEMORY_DEFAULTS as L, optimizeObservationsForContext as M, parseObserverOutput as N, buildObserverSystemPrompt as O, injectAnchorIds as P, Extractor as Q, OBSERVATION_CONTEXT_INSTRUCTIONS as R, deepMergeWorkingMemory as S, TokenCounter as T, Subconscious as U, OBSERVATION_CONTINUATION_HINT as V, KnowledgeSemanticIndexCoordinator as W, publishSubconsciousActivity as X, buildSubconsciousActivitySnapshot as Y, renderSubconsciousActivity as Z, reconcileObservationGroupsFromReflection as _, extractWorkingMemoryContent as a, wrapInObservationGroup as b, getObservationsAsOf as c, WORKING_MEMORY_STATE_ID as d, WORKING_MEMORY_STATE_PROCESSOR_ID as f, parseObservationGroups as g, deriveObservationGroupProvenance as h, WorkingMemory as i, hasCurrentTaskSection as j, extractCurrentTask as k, ObservationalMemoryProcessor as l, combineObservationGroupRanges as m, MessageHistory$1 as n, extractWorkingMemoryTags as o, WorkingMemoryStateProcessor as p, ModelByInputTokens as q, SemanticRecall as r, removeWorkingMemoryTags as s, Memory as t, ObservationalMemory as u, renderObservationGroupsForReflection as v, summarizeConversation as w, WorkingMemoryExtractor as x, stripObservationGroups as y, OBSERVATION_CONTEXT_PROMPT as z };
 
-//# sourceMappingURL=src-BV9eXuMM.js.map
+//# sourceMappingURL=src-BxI-PDpC.js.map