@mastra/memory 1.18.0-alpha.3 → 1.18.0-alpha.4

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @mastra/memory
 
+## 1.18.0-alpha.4
+
+### Patch Changes
+
+- Auto-recover from transient transport errors (e.g. undici `terminated`, `fetch failed`, `UND_ERR_*`, 5xx, 429) in the OM observer and reflector LLM calls. Adds an internal retry wrapper with exponential backoff (1s, 2s, 4s, 8s, 16s, 32s, 64s, 120s — per-attempt delay capped at 2 minutes, ~4 minute total budget per call) so a single network blip from any provider no longer kills the actor turn during long-running sessions. Non-transient errors (auth, validation, etc.) and user-initiated aborts still fail fast. No public API changes. ([#16454](https://github.com/mastra-ai/mastra/pull/16454))
+
+- Updated dependencies [[`b59316f`](https://github.com/mastra-ai/mastra/commit/b59316ffa0f7688165b0f9c81ccdf85da461e5b2), [`55f1e2d`](https://github.com/mastra-ai/mastra/commit/55f1e2d65425b95a49ae788053b266f256e38c96), [`d48a705`](https://github.com/mastra-ai/mastra/commit/d48a705ff3dfbdc7a996e07ecd8293b5effd9a2a)]:
+  - @mastra/core@1.33.0-alpha.12
+
 ## 1.18.0-alpha.3
 
 ### Patch Changes

package/dist/{chunk-XVVCS6R6.js → chunk-LPMZNXSF.js}

@@ -1013,8 +1013,6 @@ var ObservationStrategy = class _ObservationStrategy {
     this.scope = deps.scope;
     this.retrieval = deps.retrieval;
   }
-  deps;
-  opts;
   storage;
   messageHistory;
   tokenCounter;
@@ -2043,8 +2041,6 @@ var ObservationStep = class {
     this.turn = turn;
     this.stepNumber = stepNumber;
   }
-  turn;
-  stepNumber;
   _prepared = false;
   _context;
   /** Whether this step has been prepared. */
@@ -3791,6 +3787,135 @@ function optimizeObservationsForContext(observations) {
   optimized = optimized.replace(/\n{3,}/g, "\n\n");
   return optimized.trim();
 }
+
+// src/processors/observational-memory/retry.ts
+var RETRY_CONFIG = {
+  /** Maximum number of retry *attempts* (total tries = maxRetries + 1). */
+  maxRetries: 8,
+  /** Initial backoff delay in milliseconds. */
+  initialDelayMs: 1e3,
+  /** Multiplier applied to the delay after each failed attempt. */
+  backoffFactor: 2,
+  /** Cap on per-attempt delay (ms). */
+  maxDelayMs: 12e4,
+  /** Random jitter as a fraction of the computed delay (e.g. 0.2 = ±20%). */
+  jitter: 0.2
+};
+var TRANSIENT_MESSAGE_SUBSTRINGS = [
+  "terminated",
+  "fetch failed",
+  "econnreset",
+  "econnrefused",
+  "enotfound",
+  "eai_again",
+  "socket hang up",
+  "network error",
+  "request timed out",
+  "request timeout",
+  "connection reset",
+  "connection closed"
+];
+function isRecord2(value) {
+  return typeof value === "object" && value !== null;
+}
+function isAbortError(error) {
+  if (!isRecord2(error)) return false;
+  if (error.name === "AbortError") return true;
+  if (typeof error.code === "string" && error.code === "ABORT_ERR") return true;
+  return false;
+}
+function hasTransientMessage(value) {
+  if (!isRecord2(value)) return false;
+  const message = typeof value.message === "string" ? value.message.toLowerCase() : "";
+  if (message && TRANSIENT_MESSAGE_SUBSTRINGS.some((sub) => message.includes(sub))) return true;
+  if (typeof value.code === "string" && value.code.toUpperCase().startsWith("UND_ERR_")) return true;
+  return false;
+}
+function hasRetryableHttpStatus(value) {
+  if (!isRecord2(value)) return false;
+  const status = typeof value.statusCode === "number" ? value.statusCode : void 0;
+  if (status === void 0) return false;
+  if (status === 408 || status === 425 || status === 429) return true;
+  if (status >= 500 && status <= 599) return true;
+  return false;
+}
+function hasIsRetryableFlag(value) {
+  if (!isRecord2(value)) return false;
+  return value.isRetryable === true;
+}
+function isTransientLLMError(error) {
+  if (isAbortError(error)) return false;
+  const visited = /* @__PURE__ */ new WeakSet();
+  function visit(candidate) {
+    if (isRecord2(candidate)) {
+      if (visited.has(candidate)) return false;
+      visited.add(candidate);
+    }
+    if (hasTransientMessage(candidate)) return true;
+    if (hasRetryableHttpStatus(candidate)) return true;
+    if (hasIsRetryableFlag(candidate)) return true;
+    if (isRecord2(candidate)) {
+      if (visit(candidate.cause)) return true;
+      if (visit(candidate.error)) return true;
+    }
+    return false;
+  }
+  return visit(error);
+}
+function computeDelay(attempt) {
+  const base = RETRY_CONFIG.initialDelayMs * Math.pow(RETRY_CONFIG.backoffFactor, attempt);
+  const capped = Math.min(base, RETRY_CONFIG.maxDelayMs);
+  const jitterRange = capped * RETRY_CONFIG.jitter;
+  const offset = (Math.random() * 2 - 1) * jitterRange;
+  return Math.max(0, Math.round(capped + offset));
+}
+function sleep(ms, abortSignal) {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    if (abortSignal?.aborted) {
+      reject(new Error("The operation was aborted."));
+      return;
+    }
+    const timer = setTimeout(() => {
+      abortSignal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      abortSignal?.removeEventListener("abort", onAbort);
+      reject(new Error("The operation was aborted."));
+    };
+    abortSignal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+async function withRetry(fn, opts) {
+  const { label, abortSignal } = opts;
+  let attempt = 0;
+  while (true) {
+    if (abortSignal?.aborted) {
+      throw new Error("The operation was aborted.");
+    }
+    try {
+      return await fn();
+    } catch (error) {
+      if (isAbortError(error) || abortSignal?.aborted) throw error;
+      if (attempt >= RETRY_CONFIG.maxRetries || !isTransientLLMError(error)) {
+        if (attempt > 0) {
+          omDebug(
+            `[OM:retry:${label}] giving up after ${attempt} retry/retries: ${error instanceof Error ? error.message : String(error)}`
+          );
+        }
+        throw error;
+      }
+      const delay = computeDelay(attempt);
+      attempt++;
+      omDebug(
+        `[OM:retry:${label}] transient error on attempt ${attempt}, retrying in ${delay}ms: ${error instanceof Error ? error.message : String(error)}`
+      );
+      await sleep(delay, abortSignal);
+    }
+  }
+}
 var PHASE_CONFIG = {
   observer: {
     name: "om.observer",
@@ -3901,32 +4026,35 @@ var ObserverRunner = class {
       buildObserverHistoryMessage(messagesToObserve)
     ];
     const doGenerate = async () => {
-      return withOmTracingSpan({
-        phase: "observer",
-        model: resolvedModel.model,
-        inputTokens,
-        requestContext: options?.requestContext,
-        observabilityContext: options?.observabilityContext,
-        metadata: {
-          omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
-          omThreadTitleEnabled: this.observationConfig.threadTitle,
-          omSkipContinuationHints: options?.skipContinuationHints ?? false,
-          omWasTruncated: options?.wasTruncated ?? false,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => this.withAbortCheck(async () => {
-          const streamResult = await agent.stream(observerMessages, {
-            modelSettings: { ...this.observationConfig.modelSettings },
-            providerOptions: this.observationConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...options?.requestContext ? { requestContext: options.requestContext } : {},
-            ...childObservabilityContext
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+      return withRetry(
+        () => withOmTracingSpan({
+          phase: "observer",
+          model: resolvedModel.model,
+          inputTokens,
+          requestContext: options?.requestContext,
+          observabilityContext: options?.observabilityContext,
+          metadata: {
+            omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
+            omThreadTitleEnabled: this.observationConfig.threadTitle,
+            omSkipContinuationHints: options?.skipContinuationHints ?? false,
+            omWasTruncated: options?.wasTruncated ?? false,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => this.withAbortCheck(async () => {
+            const streamResult = await agent.stream(observerMessages, {
+              modelSettings: { ...this.observationConfig.modelSettings },
+              providerOptions: this.observationConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...options?.requestContext ? { requestContext: options.requestContext } : {},
+              ...childObservabilityContext
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "observer", abortSignal }
+      );
     };
     let result = await doGenerate();
     let parsed = parseObserverOutput(result.text);
@@ -4000,31 +4128,34 @@ var ObserverRunner = class {
       }
     }
     const doGenerate = async () => {
-      return withOmTracingSpan({
-        phase: "observer-multi-thread",
-        model: resolvedModel.model,
-        inputTokens,
-        requestContext,
-        observabilityContext,
-        metadata: {
-          omThreadCount: threadOrder.length,
-          omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
-          omThreadTitleEnabled: this.observationConfig.threadTitle,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => this.withAbortCheck(async () => {
-          const streamResult = await agent.stream(observerMessages, {
-            modelSettings: { ...this.observationConfig.modelSettings },
-            providerOptions: this.observationConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...requestContext ? { requestContext } : {},
-            ...childObservabilityContext
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+      return withRetry(
+        () => withOmTracingSpan({
+          phase: "observer-multi-thread",
+          model: resolvedModel.model,
+          inputTokens,
+          requestContext,
+          observabilityContext,
+          metadata: {
+            omThreadCount: threadOrder.length,
+            omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
+            omThreadTitleEnabled: this.observationConfig.threadTitle,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => this.withAbortCheck(async () => {
+            const streamResult = await agent.stream(observerMessages, {
+              modelSettings: { ...this.observationConfig.modelSettings },
+              providerOptions: this.observationConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...requestContext ? { requestContext } : {},
+              ...childObservabilityContext
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "observer-multi-thread", abortSignal }
+      );
     };
     let result = await doGenerate();
     let parsed = parseMultiThreadObserverOutput(result.text);
@@ -4468,54 +4599,60 @@ var ReflectorRunner = class {
         `[OM:callReflector] ${isRetry ? `retry #${attemptNumber - 1}` : "first attempt"}: level=${currentLevel}, originalTokens=${originalTokens}, targetThreshold=${targetThreshold}, promptLen=${prompt.length}, skipContinuationHints=${skipContinuationHints}`
       );
       let chunkCount = 0;
-      const result = await withOmTracingSpan({
-        phase: "reflector",
-        model: resolvedModel.model,
-        inputTokens: originalTokens,
-        requestContext,
-        observabilityContext,
-        metadata: {
-          omCompressionLevel: currentLevel,
-          omCompressionAttempt: attemptNumber,
-          omTargetThreshold: targetThreshold,
-          omSkipContinuationHints: skipContinuationHints ?? false,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => withAbortCheck(async () => {
-          const streamResult = await agent.stream(prompt, {
-            modelSettings: {
-              ...this.reflectionConfig.modelSettings
-            },
-            providerOptions: this.reflectionConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...requestContext ? { requestContext } : {},
-            ...childObservabilityContext,
-            ...attemptNumber === 1 ? {
-              onChunk(chunk) {
-                chunkCount++;
-                if (chunkCount === 1 || chunkCount % 50 === 0) {
-                  const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
-                  omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
-                }
-              },
-              onFinish(event) {
-                omDebug(
-                  `[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`
-                );
-              },
-              onAbort(event) {
-                omDebug(`[OM:callReflector] onAbort: chunks=${chunkCount}, reason=${event?.reason ?? "unknown"}`);
+      const result = await withRetry(
+        () => withOmTracingSpan({
+          phase: "reflector",
+          model: resolvedModel.model,
+          inputTokens: originalTokens,
+          requestContext,
+          observabilityContext,
+          metadata: {
+            omCompressionLevel: currentLevel,
+            omCompressionAttempt: attemptNumber,
+            omTargetThreshold: targetThreshold,
+            omSkipContinuationHints: skipContinuationHints ?? false,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => withAbortCheck(async () => {
+            chunkCount = 0;
+            const streamResult = await agent.stream(prompt, {
+              modelSettings: {
+                ...this.reflectionConfig.modelSettings
               },
-              onError({ error }) {
-                omError(`[OM:callReflector] onError after ${chunkCount} chunks`, error);
-              }
-            } : {}
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+              providerOptions: this.reflectionConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...requestContext ? { requestContext } : {},
+              ...childObservabilityContext,
+              ...attemptNumber === 1 ? {
+                onChunk(chunk) {
+                  chunkCount++;
+                  if (chunkCount === 1 || chunkCount % 50 === 0) {
+                    const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
+                    omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
+                  }
+                },
+                onFinish(event) {
+                  omDebug(
+                    `[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`
+                  );
+                },
+                onAbort(event) {
+                  omDebug(
+                    `[OM:callReflector] onAbort: chunks=${chunkCount}, reason=${event?.reason ?? "unknown"}`
+                  );
+                },
+                onError({ error }) {
+                  omError(`[OM:callReflector] onError after ${chunkCount} chunks`, error);
+                }
+              } : {}
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "reflector", abortSignal }
+      );
       omDebug(
         `[OM:callReflector] attempt #${attemptNumber} returned: textLen=${result.text?.length}, textPreview="${result.text?.slice(0, 120)}...", inputTokens=${result.usage?.inputTokens ?? result.totalUsage?.inputTokens}, outputTokens=${result.usage?.outputTokens ?? result.totalUsage?.outputTokens}`
       );
@@ -9496,5 +9633,5 @@ function getObservationsAsOf(activeObservations, asOf) {
 }
 
 export { ModelByInputTokens, OBSERVER_SYSTEM_PROMPT, ObservationalMemory, ObservationalMemoryProcessor, TokenCounter, buildObserverPrompt, buildObserverSystemPrompt, combineObservationGroupRanges, deriveObservationGroupProvenance, extractCurrentTask, formatMessagesForObserver, formatToolResultForObserver, getObservationsAsOf, hasCurrentTaskSection, injectAnchorIds, optimizeObservationsForContext, parseAnchorId, parseObservationGroups, parseObserverOutput, reconcileObservationGroupsFromReflection, renderObservationGroupsForReflection, resolveToolResultValue, stripEphemeralAnchorIds, stripObservationGroups, truncateStringByTokens, wrapInObservationGroup };
-//# sourceMappingURL=chunk-XVVCS6R6.js.map
-//# sourceMappingURL=chunk-XVVCS6R6.js.map
+//# sourceMappingURL=chunk-LPMZNXSF.js.map
+//# sourceMappingURL=chunk-LPMZNXSF.js.map