@mastra/memory 1.39.0-alpha.3 → 1.39.0-alpha.4

package/dist/{src-GHUZVCL8.cjs → src-BGgY5kgl.cjs}

@@ -18685,7 +18685,15 @@ const RETRY_CONFIG = {
 	/** Cap on per-attempt delay (ms). */
 	maxDelayMs: 12e4,
 	/** Random jitter as a fraction of the computed delay (e.g. 0.2 = ±20%). */
-	jitter: .2
+	jitter: .2,
+	/**
+	* How long one observer, reflector, or extraction model call may go without streaming any
+	* output before it is aborted and retried. Without it a provider that accepts the request but
+	* never answers stalls the call forever, and the retry schedule above never gets a chance to run.
+	* It measures inactivity, not total duration, so a long reply that keeps streaming is never cut
+	* off. It is generous because reasoning models can think silently before their first token.
+	*/
+	modelCallIdleTimeoutMs: 3e5
 };
 const TRANSIENT_MESSAGE_SUBSTRINGS = [
 	"terminated",
@@ -18704,6 +18712,19 @@ const TRANSIENT_MESSAGE_SUBSTRINGS = [
 ];
 const INCOMPLETE_FINISH_REASONS = /* @__PURE__ */ new Set(["other", "unknown"]);
 /**
+* A model call that streamed nothing for `RETRY_CONFIG.modelCallIdleTimeoutMs`. Retryable, and
+* deliberately not an AbortError so `withRetry` doesn't treat it as cancellation.
+*
+* @internal
+*/
+var OmModelCallTimeoutError = class extends Error {
+	isRetryable = true;
+	constructor(label, idleTimeoutMs) {
+		super(`${label} call produced no output for ${idleTimeoutMs}ms`);
+		this.name = "OmModelCallTimeoutError";
+	}
+};
+/**
 * A model reply that ended before the model finished. Carries the AI SDK's
 * `isRetryable` flag, which `isTransientLLMError` and core's error processors
 * already recognize.
@@ -18846,6 +18867,51 @@ function sleep(ms, abortSignal) {
 	});
 }
 /**
+* Run one attempt, failing with a retryable `OmModelCallTimeoutError` if it
+* goes `idleTimeoutMs` without calling `onActivity`. `fn` gets a signal that
+* aborts on timeout or on the caller's signal, so the underlying request is
+* cancelled too.
+*/
+async function attemptWithIdleTimeout(fn, label, idleTimeoutMs, abortSignal) {
+	const controller = new AbortController();
+	let rejectOnAbort;
+	const callerAborted = new Promise((_, reject) => {
+		rejectOnAbort = reject;
+	});
+	const onAbort = () => {
+		controller.abort(abortSignal?.reason);
+		rejectOnAbort(abortSignal?.reason ?? /* @__PURE__ */ new Error("The operation was aborted."));
+	};
+	abortSignal?.addEventListener("abort", onAbort, { once: true });
+	let timer;
+	let rejectOnIdle;
+	const idle = new Promise((_, reject) => {
+		rejectOnIdle = reject;
+	});
+	const armIdleTimer = () => {
+		clearTimeout(timer);
+		timer = setTimeout(() => {
+			const error = new OmModelCallTimeoutError(label, idleTimeoutMs);
+			rejectOnIdle(error);
+			controller.abort(error);
+		}, idleTimeoutMs);
+	};
+	const onActivity = () => {
+		if (!controller.signal.aborted) armIdleTimer();
+	};
+	armIdleTimer();
+	try {
+		return await Promise.race([
+			fn(controller.signal, onActivity),
+			idle,
+			callerAborted
+		]);
+	} finally {
+		clearTimeout(timer);
+		abortSignal?.removeEventListener("abort", onAbort);
+	}
+}
+/**
 * Run `fn` with retries on transient transport-class errors.
 *
 * Non-transient errors (auth, validation, schema, etc.) are rethrown
@@ -18854,14 +18920,16 @@ function sleep(ms, abortSignal) {
 * @internal
 */
 async function withRetry(fn, opts) {
-	const { label, abortSignal, maxRetries = RETRY_CONFIG.maxRetries } = opts;
+	const { label, abortSignal, maxRetries = RETRY_CONFIG.maxRetries, idleTimeoutMs } = opts;
 	let attempt = 0;
 	while (true) {
 		if (abortSignal?.aborted) throw new Error("The operation was aborted.");
 		try {
-			return await fn();
+			if (idleTimeoutMs === void 0) return await fn(abortSignal ?? new AbortController().signal, () => {});
+			return await attemptWithIdleTimeout(fn, label, idleTimeoutMs, abortSignal);
 		} catch (error) {
-			if (hasAbortInChain(error) || abortSignal?.aborted) throw error;
+			if (abortSignal?.aborted) throw error;
+			if (!(error instanceof OmModelCallTimeoutError) && hasAbortInChain(error)) throw error;
 			if (attempt >= maxRetries || !isTransientLLMError(error)) {
 				if (attempt > 0) omDebug(`[OM:retry:${label}] giving up after ${attempt} retry/retries: ${error instanceof Error ? error.message : String(error)}`);
 				throw error;
@@ -18981,19 +19049,21 @@ If a prior value is still applicable and carry-forward is enabled, return that p
 ${extractorInstructions}${priorLines.length > 0 ? `\n\n## Prior Extracted Values\n\n${priorLines.join("\n\n")}` : ""}`;
 	const values = {};
 	const failures = [];
-	const streamWithStructuredOutput = async (jsonPromptInjection) => withRetry(() => streamOnce(jsonPromptInjection), {
+	const streamWithStructuredOutput = async (jsonPromptInjection) => withRetry((attemptSignal, onActivity) => streamOnce(jsonPromptInjection, attemptSignal, onActivity), {
 		label: `om-${opts.source}-structured-extraction`,
 		abortSignal: opts.abortSignal,
-		maxRetries: 1
+		maxRetries: 1,
+		idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs
 	});
-	const streamOnce = async (jsonPromptInjection) => {
+	const streamOnce = async (jsonPromptInjection, abortSignal, onActivity) => {
 		const object = await (await opts.agent.stream(prompt, {
 			structuredOutput: {
 				schema,
 				...jsonPromptInjection ? { jsonPromptInjection } : {}
 			},
 			...opts.memory ? { memory: opts.memory } : {},
-			...opts.abortSignal ? { abortSignal: opts.abortSignal } : {},
+			abortSignal,
+			onChunk: onActivity,
 			...opts.requestContext ? { requestContext: opts.requestContext } : {},
 			...opts.observabilityContext
 		})).object;
@@ -21761,7 +21831,7 @@ var ObserverRunner = class {
 			timeZone: options?.timeZone
 		})];
 		const doGenerate = async () => {
-			return withRetry(() => withOmTracingSpan({
+			return withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "observer",
 				model: resolvedModel.model,
 				inputTokens,
@@ -21784,7 +21854,8 @@ var ObserverRunner = class {
 							modelSettings: { ...this.observationConfig.modelSettings },
 							providerOptions: this.observationConfig.providerOptions,
 							...attemptMemory ? { memory: attemptMemory } : {},
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
+							onChunk: onActivity,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext
 						})).getFullOutput(), "OM observer");
@@ -21806,6 +21877,7 @@ var ObserverRunner = class {
 			}), {
 				label: "observer",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.observationConfig.maxRetries
 			});
 		};
@@ -21974,7 +22046,7 @@ var ObserverRunner = class {
 			timeZone
 		})];
 		const doGenerate = async () => {
-			return withRetry(() => withOmTracingSpan({
+			return withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "observer-multi-thread",
 				model: resolvedModel.model,
 				inputTokens,
@@ -21995,7 +22067,8 @@ var ObserverRunner = class {
 							maxSteps: 1,
 							modelSettings: { ...this.observationConfig.modelSettings },
 							providerOptions: this.observationConfig.providerOptions,
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
+							onChunk: onActivity,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext
 						})).getFullOutput(), "OM multi-thread observer");
@@ -22019,6 +22092,7 @@ var ObserverRunner = class {
 			}), {
 				label: "observer-multi-thread",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.observationConfig.maxRetries
 			});
 		};
@@ -28700,7 +28774,10 @@ var ObservationStep = class {
 					observabilityContext: this.turn.observabilityContext
 				}).catch((err) => {
 					omDebug(`[OM:buffer] fire-and-forget buffer failed: ${err?.message}`);
-				}));
+				}), {
+					threadId,
+					resourceId
+				});
 				buffered = true;
 			}
 		}
@@ -29253,7 +29330,10 @@ var ObservationTurn = class {
 				skipMinimumTokenCheck: true
 			}).catch((err) => {
 				omDebug(`[OM:turn.end] idle buffer failed: ${err?.message}`);
-			}));
+			}), {
+				threadId: this.threadId,
+				resourceId: this.resourceId
+			});
 		}
 		return { record: this._record };
 	}
@@ -29811,7 +29891,7 @@ var ReflectorRunner = class {
 			const prompt = buildReflectorPrompt(observations, manualPrompt, currentLevel, skipContinuationHints, activeExtractors, priorExtractedValues);
 			omDebug(`[OM:callReflector] ${isRetry ? `retry #${attemptNumber - 1}` : "first attempt"}: level=${currentLevel}, originalTokens=${originalTokens}, targetThreshold=${targetThreshold}, promptLen=${prompt.length}, skipContinuationHints=${skipContinuationHints}`);
 			let chunkCount = 0;
-			const result = await withRetry(() => withOmTracingSpan({
+			const result = await withRetry((attemptSignal, onActivity) => withOmTracingSpan({
 				phase: "reflector",
 				model: resolvedModel.model,
 				inputTokens: originalTokens,
@@ -29835,17 +29915,19 @@ var ReflectorRunner = class {
 							modelSettings: { ...this.reflectionConfig.modelSettings },
 							providerOptions: this.reflectionConfig.providerOptions,
 							...attemptMemory ? { memory: attemptMemory } : {},
-							...abortSignal ? { abortSignal } : {},
+							abortSignal: attemptSignal,
 							...internalRequestContext ? { requestContext: internalRequestContext } : {},
 							...childObservabilityContext,
+							onChunk(chunk) {
+								onActivity();
+								if (attemptNumber !== 1) return;
+								chunkCount++;
+								if (chunkCount === 1 || chunkCount % 50 === 0) {
+									const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
+									omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
+								}
+							},
 							...attemptNumber === 1 ? {
-								onChunk(chunk) {
-									chunkCount++;
-									if (chunkCount === 1 || chunkCount % 50 === 0) {
-										const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
-										omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
-									}
-								},
 								onFinish(event) {
 									omDebug(`[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`);
 								},
@@ -29865,6 +29947,7 @@ var ReflectorRunner = class {
 			}), {
 				label: "reflector",
 				abortSignal,
+				idleTimeoutMs: RETRY_CONFIG.modelCallIdleTimeoutMs,
 				maxRetries: this.reflectionConfig.maxRetries
 			});
 			omDebug(`[OM:callReflector] attempt #${attemptNumber} returned: textLen=${result.text?.length}, textPreview="${result.text?.slice(0, 120)}...", inputTokens=${result.usage?.inputTokens ?? result.totalUsage?.inputTokens}, outputTokens=${result.usage?.outputTokens ?? result.totalUsage?.outputTokens}`);
@@ -30023,7 +30106,7 @@ var ReflectorRunner = class {
 			}
 		})();
 		BufferingCoordinator.asyncBufferingOps.set(bufferKey, asyncOp);
-		this.trackBackgroundWork?.(asyncOp);
+		this.trackBackgroundWork?.(asyncOp, lockKey);
 	}
 	/**
 	* Perform async buffered reflection — reflects observations and stores to bufferedReflection.
@@ -31051,32 +31134,44 @@ var ObservationalMemory = class ObservationalMemory {
 	* Tracked so callers can join them via `settled()` before closing a storage
 	* connection they own — otherwise a background cycle's tail statements race the close.
 	*/
-	pendingBackgroundWork = /* @__PURE__ */ new Set();
+	pendingBackgroundWork = /* @__PURE__ */ new Map();
 	/**
 	* Register fire-and-forget background work so `settled()` can join it.
+	* `scope` is the conversation the work belongs to (thread/resource ids, or a lock key from
+	* `buffering.getLockKey`),
+	* so a scoped `settled()` can wait for one conversation without waiting for every other.
 	* Returns the original promise so callers keep their own rejection handling.
 	*/
-	trackBackgroundWork(work) {
+	trackBackgroundWork(work, scope) {
+		const lockKey = typeof scope === "string" ? scope : this.buffering.getLockKey(scope.threadId, scope.resourceId);
 		const tracked = work.catch(() => {}).finally(() => {
 			this.pendingBackgroundWork.delete(tracked);
 		});
-		this.pendingBackgroundWork.add(tracked);
+		this.pendingBackgroundWork.set(tracked, lockKey);
 		return work;
 	}
 	/**
-	* Resolve once all background observational-memory work started so far has finished.
+	* Resolve once background observational-memory work started so far has finished.
+	*
+	* Pass `threadId`/`resourceId` to wait only for that conversation's work. An engine shared by
+	* many busy threads may never be fully idle, so an unscoped wait can last indefinitely there.
+	* A scope without the id this engine keys work on (`resourceId` for resource scope, `threadId`
+	* for thread scope) can't be matched to its work, so it falls back to waiting for all work.
 	*
 	* Background cycles enqueue further background work (a buffered observation can
 	* trigger a reflection, which runs nested agent streams), so this drains repeatedly
 	* until nothing is left rather than joining a single snapshot.
 	*/
-	async settled() {
+	async settled(scope) {
+		const lockKey = (this.scope === "resource" ? scope?.resourceId : scope?.threadId) ? this.buffering.getLockKey(scope.threadId, scope.resourceId) : void 0;
+		const pending = () => [...this.pendingBackgroundWork].filter(([, key]) => lockKey === void 0 || key === lockKey).map(([w]) => w);
 		const maxDrainRounds = 100;
 		for (let round = 0; round < maxDrainRounds; round++) {
-			if (this.pendingBackgroundWork.size === 0) return;
-			await Promise.allSettled([...this.pendingBackgroundWork]);
+			const work = pending();
+			if (work.length === 0) return;
+			await Promise.allSettled(work);
 		}
-		if (this.pendingBackgroundWork.size === 0) return;
+		if (pending().length === 0) return;
 		omError(`[OM:settled] background work still pending after ${maxDrainRounds} drain rounds; giving up waiting`);
 	}
 	/**
@@ -31233,7 +31328,7 @@ var ObservationalMemory = class ObservationalMemory {
 			memory: this.memory,
 			onReflectionCommitted: config.onReflectionCommitted,
 			hooks: this.hooks,
-			trackBackgroundWork: (work) => this.trackBackgroundWork(work)
+			trackBackgroundWork: (work, lockKey) => this.trackBackgroundWork(work, lockKey)
 		});
 		this.validateBufferConfig();
 		if (this.scope === "resource" && !hasWarnedResourceScopeDeprecation) {
@@ -32089,11 +32184,12 @@ ${formattedMessages}
 		}).finally(() => {
 			if (BufferingCoordinator.asyncBufferingOps.get(bufferKey) === asyncOp) BufferingCoordinator.asyncBufferingOps.delete(bufferKey);
 			unregisterOp(record.id, "bufferingObservation");
-			runOMCommit(lockKey, () => this.storage.setBufferingObservationFlag(record.id, false)).catch((err) => {
+			return runOMCommit(lockKey, () => this.storage.setBufferingObservationFlag(record.id, false)).catch((err) => {
 				omError("[OM] Failed to clear buffering observation flag", err);
 			});
 		});
 		BufferingCoordinator.asyncBufferingOps.set(bufferKey, asyncOp);
+		return { op: asyncOp };
 	}
 	/**
 	* Internal method that waits for existing buffering operation and then runs new buffering.
@@ -32193,7 +32289,7 @@ ${formattedMessages}
 		if (!this.buffering.isAsyncObservationEnabled()) return false;
 		const lockKey = this.buffering.getLockKey(opts.threadId, opts.resourceId);
 		const shouldTrigger = this.buffering.shouldTriggerAsyncObservation(opts.pendingTokens, lockKey, opts.record, this.storage, opts.threshold);
-		if (shouldTrigger) this.trackBackgroundWork(this.startAsyncBufferedObservation(opts.record, opts.threadId, opts.unobservedMessages, lockKey, opts.writer, opts.unbufferedPendingTokens, opts.requestContext, opts.observabilityContext));
+		if (shouldTrigger) this.trackBackgroundWork(this.startAsyncBufferedObservation(opts.record, opts.threadId, opts.unobservedMessages, lockKey, opts.writer, opts.unbufferedPendingTokens, opts.requestContext, opts.observabilityContext).then(({ op }) => op), lockKey);
 		return shouldTrigger;
 	}
 	isMessageList(value) {
@@ -34245,9 +34341,9 @@ var Memory = class Memory extends _mastra_core_memory.MastraMemory {
 	* await store.close();
 	* ```
 	*/
-	async settled() {
+	async settled(scope) {
 		await this.pendingVectorCleanup;
-		await (this._omEngine ? await this._omEngine : this._omEngineInstance)?.settled();
+		await (this._omEngine ? await this._omEngine : this._omEngineInstance)?.settled(scope);
 		await this.pendingVectorCleanup;
 	}
 	/** The shared ObservationalMemory engine. Lazily created on first access. */
@@ -36841,4 +36937,4 @@ Object.defineProperty(exports, "wrapInObservationGroup", {
 	}
 });
 
-//# sourceMappingURL=src-GHUZVCL8.cjs.map
+//# sourceMappingURL=src-BGgY5kgl.cjs.map