@mastra/memory 1.39.0-alpha.3 → 1.39.0-alpha.4

package/dist/{src-Dg8w4Ud-.js → src--zkaCof1.js}

@@ -18664,7 +18664,15 @@ const RETRY_CONFIG = {
 	/** Cap on per-attempt delay (ms). */
 	maxDelayMs: 12e4,
 	/** Random jitter as a fraction of the computed delay (e.g. 0.2 = ±20%). */
-	jitter: .2
+	jitter: .2,
+	/**
+	* How long one observer, reflector, or extraction model call may go without streaming any
+	* output before it is aborted and retried. Without it a provider that accepts the request but
+	* never answers stalls the call forever, and the retry schedule above never gets a chance to run.
+	* It measures inactivity, not total duration, so a long reply that keeps streaming is never cut
+	* off. It is generous because reasoning models can think silently before their first token.
+	*/
+	modelCallIdleTimeoutMs: 3e5
 };
 const TRANSIENT_MESSAGE_SUBSTRINGS = [
 	"terminated",
@@ -18683,6 +18691,19 @@ const TRANSIENT_MESSAGE_SUBSTRINGS = [
 ];
 const INCOMPLETE_FINISH_REASONS = /* @__PURE__ */ new Set(["other", "unknown"]);
 /**
+* A model call that streamed nothing for `RETRY_CONFIG.modelCallIdleTimeoutMs`. Retryable, and
+* deliberately not an AbortError so `withRetry` doesn't treat it as cancellation.
+*
+* @internal
+*/
+var OmModelCallTimeoutError = class extends Error {
+	isRetryable = true;
+	constructor(label, idleTimeoutMs) {
+		super(`${label} call produced no output for ${idleTimeoutMs}ms`);
+		this.name = "OmModelCallTimeoutError";
+	}
+};
+/**
 * A model reply that ended before the model finished. Carries the AI SDK's
 * `isRetryable` flag, which `isTransientLLMError` and core's error processors
 * already recognize.
@@ -18825,6 +18846,51 @@ function sleep(ms, abortSignal) {
 	});
 }
 /**
+* Run one attempt, failing with a retryable `OmModelCallTimeoutError` if it
+* goes `idleTimeoutMs` without calling `onActivity`. `fn` gets a signal that
+* aborts on timeout or on the caller's signal, so the underlying request is
+* cancelled too.
+*/
+async function attemptWithIdleTimeout(fn, label, idleTimeoutMs, abortSignal) {
+	const controller = new AbortController();
+	let rejectOnAbort;
+	const callerAborted = new Promise((_, reject) => {
+		rejectOnAbort = reject;
+	});
+	const onAbort = () => {
+		controller.abort(abortSignal?.reason);
+		rejectOnAbort(abortSignal?.reason ?? /* @__PURE__ */ new Error("The operation was aborted."));
+	};
+	abortSignal?.addEventListener("abort", onAbort, { once: true });
+	let timer;
+	let rejectOnIdle;
+	const idle = new Promise((_, reject) => {
+		rejectOnIdle = reject;
+	});
+	const armIdleTimer = () => {
+		clearTimeout(timer);
+		timer = setTimeout(() => {
+			const error = new OmModelCallTimeoutError(label, idleTimeoutMs);
+			rejectOnIdle(error);
+			controller.abort(error);
+		}, idleTimeoutMs);
+	};
+	const onActivity = () => {
+		if (!controller.signal.aborted) armIdleTimer();
+	};
+	armIdleTimer();
+	try {
+		return await Promise.race([
+			fn(controller.signal, onActivity),
+			idle,
+			callerAborted
+		]);
+	} finally {
+		clearTimeout(timer);
+		abortSignal?.removeEventListener("abort", onAbort);
+	}
+}
+/**
 * Run `fn` with retries on transient transport-class errors.
 *
 * Non-transient errors (auth, validation, schema, etc.) are rethrown
@@ -18833,14 +18899,16 @@ function sleep(ms, abortSignal) {
 * @internal
 */
 async function withRetry(fn, opts) {
-	const { label, abortSignal, maxRetries = RETRY_CONFIG.maxRetries } = opts;
+	const { label, abortSignal, maxRetries = RETRY_CONFIG.maxRetries, idleTimeoutMs } = opts;
 	let attempt = 0;
 	while (true) {
 		if (abortSignal?.aborted) throw new Error("The operation was aborted.");
 		try {
-			return await fn();
+			if (idleTimeoutMs === void 0) return await fn(abortSignal ?? new AbortController().signal, () => {});
+			return await attemptWithIdleTimeout(fn, label, idleTimeoutMs, abortSignal);
 		} catch (error) {
-			if (hasAbortInChain(error) || abortSignal?.aborted) throw error;
+			if (abortSignal?.aborted) throw error;
+			if (!(error instanceof OmModelCallTimeoutError) && hasAbortInChain(error)) throw error;
 			if (attempt >= maxRetries || !isTransientLLMError(error)) {
 				if (attempt > 0) omDebug(`[OM:retry:${label}] giving up after ${attempt} retry/retries: ${error instanceof Error ? error.message : String(error)}`);
 				throw error;
@@ -18960,19 +19028,21 @@ If a prior value is still applicable and carry-forward is enabled, return that p
 ${extractorInstructions}${priorLines.length > 0 ? `\n\n## Prior Extracted Values\n\n${priorLines.join("\n\n")}` : ""}`;
 	const values = {};
 	const failures = [];
-	const streamWithStructuredOutput = async (jsonPromptInjection) => withRetry(() => streamOnce(jsonPromptInjection), {
+	const streamWithStructuredOutput = async (jsonPromptInjection) => withRetry((attemptSignal, onActivity) => streamOnce(jsonPromptInjection, attemptSignal, onActivity), {
 		label: `om-${opts.source}-structured-extraction`,
 		abortSignal: opts.abortSignal,
-		maxRetries: 1
+		maxRetries: 1,
+		idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs
 	});
-	const streamOnce = async (jsonPromptInjection) => {
+	const streamOnce = async (jsonPromptInjection, abortSignal, onActivity) => {
 		const object = await (await opts.agent.stream(prompt, {
 			structuredOutput: {
 				schema,
 				...jsonPromptInjection ? { jsonPromptInjection } : {}
 			},
 			...opts.memory ? { memory: opts.memory } : {},
-			...opts.abortSignal ? { abortSignal: opts.abortSignal } : {},
+			abortSignal,
+			onChunk: onActivity,
 			...opts.requestContext ? { requestContext: opts.requestContext } : {},
 			...opts.observabilityContext
 		})).object;
@@ -21740,7 +21810,7 @@ var ObserverRunner = class {
 			timeZone: options?.timeZone
 		})];
 		const doGenerate = async () => {
-			return withRetry(() => withOmTracingSpan({
+			return withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "observer",
 				model: resolvedModel.model,
 				inputTokens,
@@ -21763,7 +21833,8 @@ var ObserverRunner = class {
 							modelSettings: { ...this.observationConfig.modelSettings },
 							providerOptions: this.observationConfig.providerOptions,
 							...attemptMemory ? { memory: attemptMemory } : {},
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
+							onChunk: onActivity,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext
 						})).getFullOutput(), "OM observer");
@@ -21785,6 +21856,7 @@ var ObserverRunner = class {
 			}), {
 				label: "observer",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.observationConfig.maxRetries
 			});
 		};
@@ -21953,7 +22025,7 @@ var ObserverRunner = class {
 			timeZone
 		})];
 		const doGenerate = async () => {
-			return withRetry(() => withOmTracingSpan({
+			return withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "observer-multi-thread",
 				model: resolvedModel.model,
 				inputTokens,
@@ -21974,7 +22046,8 @@ var ObserverRunner = class {
 							maxSteps: 1,
 							modelSettings: { ...this.observationConfig.modelSettings },
 							providerOptions: this.observationConfig.providerOptions,
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
+							onChunk: onActivity,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext
 						})).getFullOutput(), "OM multi-thread observer");
@@ -21998,6 +22071,7 @@ var ObserverRunner = class {
 			}), {
 				label: "observer-multi-thread",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.observationConfig.maxRetries
 			});
 		};
@@ -28679,7 +28753,10 @@ var ObservationStep = class {
 					observabilityContext: this.turn.observabilityContext
 				}).catch((err) => {
 					omDebug(`[OM:buffer] fire-and-forget buffer failed: ${err?.message}`);
-				}));
+				}), {
+					threadId,
+					resourceId
+				});
 				buffered = true;
 			}
 		}
@@ -29232,7 +29309,10 @@ var ObservationTurn = class {
 				skipMinimumTokenCheck: true
 			}).catch((err) => {
 				omDebug(`[OM:turn.end] idle buffer failed: ${err?.message}`);
-			}));
+			}), {
+				threadId: this.threadId,
+				resourceId: this.resourceId
+			});
 		}
 		return { record: this._record };
 	}
@@ -29790,7 +29870,7 @@ var ReflectorRunner = class {
 			const prompt = buildReflectorPrompt(observations, manualPrompt, currentLevel, skipContinuationHints, activeExtractors, priorExtractedValues);
 			omDebug(`[OM:callReflector] ${isRetry ? `retry #${attemptNumber - 1}` : "first attempt"}: level=${currentLevel}, originalTokens=${originalTokens}, targetThreshold=${targetThreshold}, promptLen=${prompt.length}, skipContinuationHints=${skipContinuationHints}`);
 			let chunkCount = 0;
-			const result = await withRetry(() => withOmTracingSpan({
+			const result = await withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "reflector",
 				model: resolvedModel.model,
 				inputTokens: originalTokens,
@@ -29814,17 +29894,19 @@ var ReflectorRunner = class {
 							modelSettings: { ...this.reflectionConfig.modelSettings },
 							providerOptions: this.reflectionConfig.providerOptions,
 							...attemptMemory ? { memory: attemptMemory } : {},
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext,
+							onChunk(chunk) {
+								onActivity();
+								if (attemptNumber !== 1) return;
+								chunkCount++;
+								if (chunkCount === 1 || chunkCount % 50 === 0) {
+									const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
+									omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
+								}
+							},
 							...attemptNumber === 1 ? {
-								onChunk(chunk) {
-									chunkCount++;
-									if (chunkCount === 1 || chunkCount % 50 === 0) {
-										const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
-										omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
-									}
-								},
 								onFinish(event) {
 									omDebug(`[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`);
 								},
@@ -29844,6 +29926,7 @@ var ReflectorRunner = class {
 			}), {
 				label: "reflector",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.reflectionConfig.maxRetries
 			});
 			omDebug(`[OM:callReflector] attempt #${attemptNumber} returned: textLen=${result.text?.length}, textPreview="${result.text?.slice(0, 120)}...", inputTokens=${result.usage?.inputTokens ?? result.totalUsage?.inputTokens}, outputTokens=${result.usage?.outputTokens ?? result.totalUsage?.outputTokens}`);
@@ -30002,7 +30085,7 @@ var ReflectorRunner = class {
 			}
 		})();
 		BufferingCoordinator.asyncBufferingOps.set(bufferKey, asyncOp);
-		this.trackBackgroundWork?.(asyncOp);
+		this.trackBackgroundWork?.(asyncOp, lockKey);
 	}
 	/**
 	* Perform async buffered reflection — reflects observations and stores to bufferedReflection.
@@ -31030,32 +31113,44 @@ var ObservationalMemory = class ObservationalMemory {
 	* Tracked so callers can join them via `settled()` before closing a storage
 	* connection they own — otherwise a background cycle's tail statements race the close.
 	*/
-	pendingBackgroundWork = /* @__PURE__ */ new Set();
+	pendingBackgroundWork = /* @__PURE__ */ new Map();
 	/**
 	* Register fire-and-forget background work so `settled()` can join it.
+	* `scope` is the conversation the work belongs to (thread/resource ids, or a lock key from
+	* `buffering.getLockKey`),
+	* so a scoped `settled()` can wait for one conversation without waiting for every other.
 	* Returns the original promise so callers keep their own rejection handling.
 	*/
-	trackBackgroundWork(work) {
+	trackBackgroundWork(work, scope) {
+		const lockKey = typeof scope === "string" ? scope : this.buffering.getLockKey(scope.threadId, scope.resourceId);
 		const tracked = work.catch(() => {}).finally(() => {
 			this.pendingBackgroundWork.delete(tracked);
 		});
-		this.pendingBackgroundWork.add(tracked);
+		this.pendingBackgroundWork.set(tracked, lockKey);
 		return work;
 	}
 	/**
-	* Resolve once all background observational-memory work started so far has finished.
+	* Resolve once background observational-memory work started so far has finished.
+	*
+	* Pass `threadId`/`resourceId` to wait only for that conversation's work. An engine shared by
+	* many busy threads may never be fully idle, so an unscoped wait can last indefinitely there.
+	* A scope without the id this engine keys work on (`resourceId` for resource scope, `threadId`
+	* for thread scope) can't be matched to its work, so it falls back to waiting for all work.
 	*
 	* Background cycles enqueue further background work (a buffered observation can
 	* trigger a reflection, which runs nested agent streams), so this drains repeatedly
 	* until nothing is left rather than joining a single snapshot.
 	*/
-	async settled() {
+	async settled(scope) {
+		const lockKey = (this.scope === "resource" ? scope?.resourceId : scope?.threadId) ? this.buffering.getLockKey(scope.threadId, scope.resourceId) : void 0;
+		const pending = () => [...this.pendingBackgroundWork].filter(([, key]) => lockKey === void 0 || key === lockKey).map(([w]) => w);
 		const maxDrainRounds = 100;
 		for (let round = 0; round < maxDrainRounds; round++) {
-			if (this.pendingBackgroundWork.size === 0) return;
-			await Promise.allSettled([...this.pendingBackgroundWork]);
+			const work = pending();
+			if (work.length === 0) return;
+			await Promise.allSettled(work);
 		}
-		if (this.pendingBackgroundWork.size === 0) return;
+		if (pending().length === 0) return;
 		omError(`[OM:settled] background work still pending after ${maxDrainRounds} drain rounds; giving up waiting`);
 	}
 	/**
@@ -31212,7 +31307,7 @@ var ObservationalMemory = class ObservationalMemory {
 			memory: this.memory,
 			onReflectionCommitted: config.onReflectionCommitted,
 			hooks: this.hooks,
-			trackBackgroundWork: (work) => this.trackBackgroundWork(work)
+			trackBackgroundWork: (work, lockKey) => this.trackBackgroundWork(work, lockKey)
 		});
 		this.validateBufferConfig();
 		if (this.scope === "resource" && !hasWarnedResourceScopeDeprecation) {
@@ -32068,11 +32163,12 @@ ${formattedMessages}
 		}).finally(() => {
 			if (BufferingCoordinator.asyncBufferingOps.get(bufferKey) === asyncOp) BufferingCoordinator.asyncBufferingOps.delete(bufferKey);
 			unregisterOp(record.id, "bufferingObservation");
-			runOMCommit(lockKey, () => this.storage.setBufferingObservationFlag(record.id, false)).catch((err) => {
+			return runOMCommit(lockKey, () => this.storage.setBufferingObservationFlag(record.id, false)).catch((err) => {
 				omError("[OM] Failed to clear buffering observation flag", err);
 			});
 		});
 		BufferingCoordinator.asyncBufferingOps.set(bufferKey, asyncOp);
+		return { op: asyncOp };
 	}
 	/**
 	* Internal method that waits for existing buffering operation and then runs new buffering.
@@ -32172,7 +32268,7 @@ ${formattedMessages}
 		if (!this.buffering.isAsyncObservationEnabled()) return false;
 		const lockKey = this.buffering.getLockKey(opts.threadId, opts.resourceId);
 		const shouldTrigger = this.buffering.shouldTriggerAsyncObservation(opts.pendingTokens, lockKey, opts.record, this.storage, opts.threshold);
-		if (shouldTrigger) this.trackBackgroundWork(this.startAsyncBufferedObservation(opts.record, opts.threadId, opts.unobservedMessages, lockKey, opts.writer, opts.unbufferedPendingTokens, opts.requestContext, opts.observabilityContext));
+		if (shouldTrigger) this.trackBackgroundWork(this.startAsyncBufferedObservation(opts.record, opts.threadId, opts.unobservedMessages, lockKey, opts.writer, opts.unbufferedPendingTokens, opts.requestContext, opts.observabilityContext).then(({ op }) => op), lockKey);
 		return shouldTrigger;
 	}
 	isMessageList(value) {
@@ -34224,9 +34320,9 @@ var Memory = class Memory extends MastraMemory {
 	* await store.close();
 	* ```
 	*/
-	async settled() {
+	async settled(scope) {
 		await this.pendingVectorCleanup;
-		await (this._omEngine ? await this._omEngine : this._omEngineInstance)?.settled();
+		await (this._omEngine ? await this._omEngine : this._omEngineInstance)?.settled(scope);
 		await this.pendingVectorCleanup;
 	}
 	/** The shared ObservationalMemory engine. Lazily created on first access. */
@@ -36515,4 +36611,4 @@ Notes:
 //#endregion
 export { renderSubconsciousActivity as $, buildObserverSystemPrompt as A, OBSERVATION_CONTEXT_INSTRUCTIONS as B, WorkingMemoryExtractor as C, TokenCounter as D, summarizeConversation as E, parseObserverOutput as F, Subconscious as G, OBSERVATION_CONTEXT_PROMPT_THREAD as H, injectAnchorIds as I, SubconsciousRemindExtractor as J, KnowledgeSemanticIndexCoordinator as K, parseAnchorId as L, formatMessagesForObserver as M, hasCurrentTaskSection as N, OBSERVER_SYSTEM_PROMPT as O, optimizeObservationsForContext as P, publishSubconsciousActivity as Q, stripEphemeralAnchorIds as R, wrapInObservationGroup as S, SUMMARIZE_THREAD_DEFAULTS as T, OBSERVATION_CONTINUATION_HINT as U, OBSERVATION_CONTEXT_PROMPT as V, getObservationContextPrompt as W, SUBCONSCIOUS_ACTIVITY_STATE_ID as X, ModelByInputTokens as Y, buildSubconsciousActivitySnapshot as Z, deriveObservationGroupProvenance as _, extractWorkingMemoryContent as a, renderObservationGroupsForReflection as b, getObservationsAsOf as c, AUTO_MODEL_BY_PROVIDER as d, Extractor as et, resolveAutoModelId as f, combineObservationGroupRanges as g, WorkingMemoryStateProcessor as h, WorkingMemory as i, extractCurrentTask as j, buildObserverPrompt as k, ObservationalMemoryProcessor as l, WORKING_MEMORY_STATE_PROCESSOR_ID as m, MessageHistory$1 as n, extractWorkingMemoryTags as o, WORKING_MEMORY_STATE_ID as p, StaleKnowledgeSemanticIndexError as q, SemanticRecall as r, removeWorkingMemoryTags as s, Memory as t, ObservationalMemory as u, parseObservationGroups as v, deepMergeWorkingMemory as w, stripObservationGroups as x, reconcileObservationGroupsFromReflection as y, OBSERVATIONAL_MEMORY_DEFAULTS as z };
 
-//# sourceMappingURL=src-Dg8w4Ud-.js.map
+//# sourceMappingURL=src--zkaCof1.js.map