@mastra/memory 1.18.0-alpha.3 → 1.18.0

package/dist/{chunk-ET2TVAT3.cjs → chunk-NRYX4QWV.cjs}

@@ -1020,8 +1020,6 @@ var ObservationStrategy = class _ObservationStrategy {
     this.scope = deps.scope;
     this.retrieval = deps.retrieval;
   }
-  deps;
-  opts;
   storage;
   messageHistory;
   tokenCounter;
@@ -2050,8 +2048,6 @@ var ObservationStep = class {
     this.turn = turn;
     this.stepNumber = stepNumber;
   }
-  turn;
-  stepNumber;
   _prepared = false;
   _context;
   /** Whether this step has been prepared. */
@@ -3798,6 +3794,135 @@ function optimizeObservationsForContext(observations) {
   optimized = optimized.replace(/\n{3,}/g, "\n\n");
   return optimized.trim();
 }
+
+// src/processors/observational-memory/retry.ts
+var RETRY_CONFIG = {
+  /** Maximum number of retry *attempts* (total tries = maxRetries + 1). */
+  maxRetries: 8,
+  /** Initial backoff delay in milliseconds. */
+  initialDelayMs: 1e3,
+  /** Multiplier applied to the delay after each failed attempt. */
+  backoffFactor: 2,
+  /** Cap on per-attempt delay (ms). */
+  maxDelayMs: 12e4,
+  /** Random jitter as a fraction of the computed delay (e.g. 0.2 = ±20%). */
+  jitter: 0.2
+};
+var TRANSIENT_MESSAGE_SUBSTRINGS = [
+  "terminated",
+  "fetch failed",
+  "econnreset",
+  "econnrefused",
+  "enotfound",
+  "eai_again",
+  "socket hang up",
+  "network error",
+  "request timed out",
+  "request timeout",
+  "connection reset",
+  "connection closed"
+];
+function isRecord2(value) {
+  return typeof value === "object" && value !== null;
+}
+function isAbortError(error) {
+  if (!isRecord2(error)) return false;
+  if (error.name === "AbortError") return true;
+  if (typeof error.code === "string" && error.code === "ABORT_ERR") return true;
+  return false;
+}
+function hasTransientMessage(value) {
+  if (!isRecord2(value)) return false;
+  const message = typeof value.message === "string" ? value.message.toLowerCase() : "";
+  if (message && TRANSIENT_MESSAGE_SUBSTRINGS.some((sub) => message.includes(sub))) return true;
+  if (typeof value.code === "string" && value.code.toUpperCase().startsWith("UND_ERR_")) return true;
+  return false;
+}
+function hasRetryableHttpStatus(value) {
+  if (!isRecord2(value)) return false;
+  const status = typeof value.statusCode === "number" ? value.statusCode : void 0;
+  if (status === void 0) return false;
+  if (status === 408 || status === 425 || status === 429) return true;
+  if (status >= 500 && status <= 599) return true;
+  return false;
+}
+function hasIsRetryableFlag(value) {
+  if (!isRecord2(value)) return false;
+  return value.isRetryable === true;
+}
+function isTransientLLMError(error) {
+  if (isAbortError(error)) return false;
+  const visited = /* @__PURE__ */ new WeakSet();
+  function visit(candidate) {
+    if (isRecord2(candidate)) {
+      if (visited.has(candidate)) return false;
+      visited.add(candidate);
+    }
+    if (hasTransientMessage(candidate)) return true;
+    if (hasRetryableHttpStatus(candidate)) return true;
+    if (hasIsRetryableFlag(candidate)) return true;
+    if (isRecord2(candidate)) {
+      if (visit(candidate.cause)) return true;
+      if (visit(candidate.error)) return true;
+    }
+    return false;
+  }
+  return visit(error);
+}
+function computeDelay(attempt) {
+  const base = RETRY_CONFIG.initialDelayMs * Math.pow(RETRY_CONFIG.backoffFactor, attempt);
+  const capped = Math.min(base, RETRY_CONFIG.maxDelayMs);
+  const jitterRange = capped * RETRY_CONFIG.jitter;
+  const offset = (Math.random() * 2 - 1) * jitterRange;
+  return Math.max(0, Math.round(capped + offset));
+}
+function sleep(ms, abortSignal) {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    if (abortSignal?.aborted) {
+      reject(new Error("The operation was aborted."));
+      return;
+    }
+    const timer = setTimeout(() => {
+      abortSignal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      abortSignal?.removeEventListener("abort", onAbort);
+      reject(new Error("The operation was aborted."));
+    };
+    abortSignal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+async function withRetry(fn, opts) {
+  const { label, abortSignal } = opts;
+  let attempt = 0;
+  while (true) {
+    if (abortSignal?.aborted) {
+      throw new Error("The operation was aborted.");
+    }
+    try {
+      return await fn();
+    } catch (error) {
+      if (isAbortError(error) || abortSignal?.aborted) throw error;
+      if (attempt >= RETRY_CONFIG.maxRetries || !isTransientLLMError(error)) {
+        if (attempt > 0) {
+          omDebug(
+            `[OM:retry:${label}] giving up after ${attempt} retry/retries: ${error instanceof Error ? error.message : String(error)}`
+          );
+        }
+        throw error;
+      }
+      const delay = computeDelay(attempt);
+      attempt++;
+      omDebug(
+        `[OM:retry:${label}] transient error on attempt ${attempt}, retrying in ${delay}ms: ${error instanceof Error ? error.message : String(error)}`
+      );
+      await sleep(delay, abortSignal);
+    }
+  }
+}
 var PHASE_CONFIG = {
   observer: {
     name: "om.observer",
@@ -3908,32 +4033,35 @@ var ObserverRunner = class {
       buildObserverHistoryMessage(messagesToObserve)
     ];
     const doGenerate = async () => {
-      return withOmTracingSpan({
-        phase: "observer",
-        model: resolvedModel.model,
-        inputTokens,
-        requestContext: options?.requestContext,
-        observabilityContext: options?.observabilityContext,
-        metadata: {
-          omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
-          omThreadTitleEnabled: this.observationConfig.threadTitle,
-          omSkipContinuationHints: options?.skipContinuationHints ?? false,
-          omWasTruncated: options?.wasTruncated ?? false,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => this.withAbortCheck(async () => {
-          const streamResult = await agent.stream(observerMessages, {
-            modelSettings: { ...this.observationConfig.modelSettings },
-            providerOptions: this.observationConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...options?.requestContext ? { requestContext: options.requestContext } : {},
-            ...childObservabilityContext
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+      return withRetry(
+        () => withOmTracingSpan({
+          phase: "observer",
+          model: resolvedModel.model,
+          inputTokens,
+          requestContext: options?.requestContext,
+          observabilityContext: options?.observabilityContext,
+          metadata: {
+            omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
+            omThreadTitleEnabled: this.observationConfig.threadTitle,
+            omSkipContinuationHints: options?.skipContinuationHints ?? false,
+            omWasTruncated: options?.wasTruncated ?? false,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => this.withAbortCheck(async () => {
+            const streamResult = await agent.stream(observerMessages, {
+              modelSettings: { ...this.observationConfig.modelSettings },
+              providerOptions: this.observationConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...options?.requestContext ? { requestContext: options.requestContext } : {},
+              ...childObservabilityContext
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "observer", abortSignal }
+      );
     };
     let result = await doGenerate();
     let parsed = parseObserverOutput(result.text);
@@ -4007,31 +4135,34 @@ var ObserverRunner = class {
       }
     }
     const doGenerate = async () => {
-      return withOmTracingSpan({
-        phase: "observer-multi-thread",
-        model: resolvedModel.model,
-        inputTokens,
-        requestContext,
-        observabilityContext,
-        metadata: {
-          omThreadCount: threadOrder.length,
-          omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
-          omThreadTitleEnabled: this.observationConfig.threadTitle,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => this.withAbortCheck(async () => {
-          const streamResult = await agent.stream(observerMessages, {
-            modelSettings: { ...this.observationConfig.modelSettings },
-            providerOptions: this.observationConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...requestContext ? { requestContext } : {},
-            ...childObservabilityContext
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+      return withRetry(
+        () => withOmTracingSpan({
+          phase: "observer-multi-thread",
+          model: resolvedModel.model,
+          inputTokens,
+          requestContext,
+          observabilityContext,
+          metadata: {
+            omThreadCount: threadOrder.length,
+            omPreviousObserverTokens: this.observationConfig.previousObserverTokens,
+            omThreadTitleEnabled: this.observationConfig.threadTitle,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => this.withAbortCheck(async () => {
+            const streamResult = await agent.stream(observerMessages, {
+              modelSettings: { ...this.observationConfig.modelSettings },
+              providerOptions: this.observationConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...requestContext ? { requestContext } : {},
+              ...childObservabilityContext
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "observer-multi-thread", abortSignal }
+      );
     };
     let result = await doGenerate();
     let parsed = parseMultiThreadObserverOutput(result.text);
@@ -4475,54 +4606,60 @@ var ReflectorRunner = class {
         `[OM:callReflector] ${isRetry ? `retry #${attemptNumber - 1}` : "first attempt"}: level=${currentLevel}, originalTokens=${originalTokens}, targetThreshold=${targetThreshold}, promptLen=${prompt.length}, skipContinuationHints=${skipContinuationHints}`
       );
       let chunkCount = 0;
-      const result = await withOmTracingSpan({
-        phase: "reflector",
-        model: resolvedModel.model,
-        inputTokens: originalTokens,
-        requestContext,
-        observabilityContext,
-        metadata: {
-          omCompressionLevel: currentLevel,
-          omCompressionAttempt: attemptNumber,
-          omTargetThreshold: targetThreshold,
-          omSkipContinuationHints: skipContinuationHints ?? false,
-          ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
-          ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
-          ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
-        },
-        callback: (childObservabilityContext) => withAbortCheck(async () => {
-          const streamResult = await agent.stream(prompt, {
-            modelSettings: {
-              ...this.reflectionConfig.modelSettings
-            },
-            providerOptions: this.reflectionConfig.providerOptions,
-            ...abortSignal ? { abortSignal } : {},
-            ...requestContext ? { requestContext } : {},
-            ...childObservabilityContext,
-            ...attemptNumber === 1 ? {
-              onChunk(chunk) {
-                chunkCount++;
-                if (chunkCount === 1 || chunkCount % 50 === 0) {
-                  const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
-                  omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
-                }
-              },
-              onFinish(event) {
-                omDebug(
-                  `[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`
-                );
-              },
-              onAbort(event) {
-                omDebug(`[OM:callReflector] onAbort: chunks=${chunkCount}, reason=${event?.reason ?? "unknown"}`);
+      const result = await withRetry(
+        () => withOmTracingSpan({
+          phase: "reflector",
+          model: resolvedModel.model,
+          inputTokens: originalTokens,
+          requestContext,
+          observabilityContext,
+          metadata: {
+            omCompressionLevel: currentLevel,
+            omCompressionAttempt: attemptNumber,
+            omTargetThreshold: targetThreshold,
+            omSkipContinuationHints: skipContinuationHints ?? false,
+            ...resolvedModel.selectedThreshold !== void 0 ? { omSelectedThreshold: resolvedModel.selectedThreshold } : {},
+            ...resolvedModel.routingStrategy ? { omRoutingStrategy: resolvedModel.routingStrategy } : {},
+            ...resolvedModel.routingThresholds ? { omRoutingThresholds: resolvedModel.routingThresholds } : {}
+          },
+          callback: (childObservabilityContext) => withAbortCheck(async () => {
+            chunkCount = 0;
+            const streamResult = await agent.stream(prompt, {
+              modelSettings: {
+                ...this.reflectionConfig.modelSettings
               },
-              onError({ error }) {
-                omError(`[OM:callReflector] onError after ${chunkCount} chunks`, error);
-              }
-            } : {}
-          });
-          return streamResult.getFullOutput();
-        }, abortSignal)
-      });
+              providerOptions: this.reflectionConfig.providerOptions,
+              ...abortSignal ? { abortSignal } : {},
+              ...requestContext ? { requestContext } : {},
+              ...childObservabilityContext,
+              ...attemptNumber === 1 ? {
+                onChunk(chunk) {
+                  chunkCount++;
+                  if (chunkCount === 1 || chunkCount % 50 === 0) {
+                    const preview = chunk.type === "text-delta" ? ` text="${chunk.textDelta?.slice(0, 80)}..."` : chunk.type === "tool-call" ? ` tool=${chunk.toolName}` : "";
+                    omDebug(`[OM:callReflector] chunk#${chunkCount}: type=${chunk.type}${preview}`);
+                  }
+                },
+                onFinish(event) {
+                  omDebug(
+                    `[OM:callReflector] onFinish: chunks=${chunkCount}, finishReason=${event.finishReason}, inputTokens=${event.usage?.inputTokens}, outputTokens=${event.usage?.outputTokens}, textLen=${event.text?.length}`
+                  );
+                },
+                onAbort(event) {
+                  omDebug(
+                    `[OM:callReflector] onAbort: chunks=${chunkCount}, reason=${event?.reason ?? "unknown"}`
+                  );
+                },
+                onError({ error }) {
+                  omError(`[OM:callReflector] onError after ${chunkCount} chunks`, error);
+                }
+              } : {}
+            });
+            return streamResult.getFullOutput();
+          }, abortSignal)
+        }),
+        { label: "reflector", abortSignal }
+      );
       omDebug(
         `[OM:callReflector] attempt #${attemptNumber} returned: textLen=${result.text?.length}, textPreview="${result.text?.slice(0, 120)}...", inputTokens=${result.usage?.inputTokens ?? result.totalUsage?.inputTokens}, outputTokens=${result.usage?.outputTokens ?? result.totalUsage?.outputTokens}`
       );
@@ -9528,5 +9665,5 @@ exports.stripEphemeralAnchorIds = stripEphemeralAnchorIds;
 exports.stripObservationGroups = stripObservationGroups;
 exports.truncateStringByTokens = truncateStringByTokens;
 exports.wrapInObservationGroup = wrapInObservationGroup;
-//# sourceMappingURL=chunk-ET2TVAT3.cjs.map
-//# sourceMappingURL=chunk-ET2TVAT3.cjs.map
+//# sourceMappingURL=chunk-NRYX4QWV.cjs.map
+//# sourceMappingURL=chunk-NRYX4QWV.cjs.map