@mastra/observability 1.16.0 → 1.16.1-alpha.0

package/dist/index.js

@@ -5946,6 +5946,7 @@ function resolveTracingStorageStrategy(config, observabilityStorage, storageName
   }
   return observabilityStrategy.preferred;
 }
+var FLUSH_SUCCEEDED = { failed: false };
 var DefaultExporter = class extends BaseExporter {
   name = "mastra-default-observability-exporter";
   #config;
@@ -6040,17 +6041,37 @@ var DefaultExporter = class extends BaseExporter {
   /**
    * Schedules a flush using setTimeout
    */
-  scheduleFlush() {
+  scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
     if (this.#flushTimer) {
       clearTimeout(this.#flushTimer);
     }
     this.#flushTimer = setTimeout(() => {
+      this.#flushTimer = void 0;
       this.flushBuffer().catch((error) => {
         this.logger.error("Scheduled flush failed", {
           error: error instanceof Error ? error.message : String(error)
         });
       });
-    }, this.#config.maxBatchWaitMs);
+    }, delayMs);
+  }
+  retryDelay(retryAttempt) {
+    return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
+  }
+  retryAttempt(events) {
+    return events.reduce((earliestAttempt, event) => {
+      if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
+      return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
+    }, void 0);
+  }
+  logStorageFailure(signal, events, retryAttempt, error) {
+    this.logger.warn("Failed to persist observability events", {
+      signal,
+      eventCount: events.length,
+      retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
+      maxRetries: this.#config.maxRetries,
+      ...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
+      error: error instanceof Error ? error.message : String(error)
+    });
   }
   /**
    * Checks flush triggers and schedules/triggers flush as needed.
@@ -6097,22 +6118,26 @@ var DefaultExporter = class extends BaseExporter {
    * On other errors, re-adds events to the buffer for retry.
    */
   async flushCreates(signal, events, storageCall) {
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED;
     if (this.#unsupportedSignals.has(signal)) {
       this.emitDrop(signal, "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     try {
       await storageCall(events);
+      return FLUSH_SUCCEEDED;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add(signal);
         this.emitDrop(signal, "unsupported-storage", events.length, error);
-      } else {
-        const dropped = this.#eventBuffer.reAddCreates(events);
-        this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+        return { failed: true };
       }
+      const dropped = this.#eventBuffer.reAddCreates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure(signal, events, retryAttempt, error);
+      this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -6121,10 +6146,10 @@ var DefaultExporter = class extends BaseExporter {
    */
   async flushSpanUpdates(events, deferredUpdates, isEnd) {
     const deferredCountAtEntry = deferredUpdates.length;
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED;
     if (this.#unsupportedSignals.has("tracing")) {
       this.emitDrop("tracing", "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     const partials = [];
     for (const event of events) {
@@ -6139,26 +6164,30 @@ var DefaultExporter = class extends BaseExporter {
         deferredUpdates.push(event);
       }
     }
-    if (partials.length === 0) return;
+    if (partials.length === 0) return FLUSH_SUCCEEDED;
     try {
       await this.#observabilityStorage.batchUpdateSpans({ records: partials });
       if (isEnd) {
         this.#eventBuffer.endFinishedSpans({ records: partials });
       }
+      return FLUSH_SUCCEEDED;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add("tracing");
         deferredUpdates.length = 0;
         this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error);
-      } else {
-        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
-        if (newlyDeferred > 0) {
-          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
-        }
-        const dropped = this.#eventBuffer.reAddUpdates(events);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+        return { failed: true };
+      }
+      const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+      if (newlyDeferred > 0) {
+        deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
       }
+      const dropped = this.#eventBuffer.reAddUpdates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure("tracing", events, retryAttempt, error);
+      this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -6226,7 +6255,7 @@ var DefaultExporter = class extends BaseExporter {
           break;
       }
     }
-    await Promise.all([
+    const flushResults = await Promise.all([
       this.flushCreates(
         "feedback",
         createFeedbackEvents,
@@ -6254,25 +6283,36 @@ var DefaultExporter = class extends BaseExporter {
       })
     ]);
     const deferredUpdates = [];
-    await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
-    await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
+    flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
+    flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
+    let deferredDropCount = 0;
     if (deferredUpdates.length > 0) {
       if (this.#unsupportedSignals.has("tracing")) {
-        this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
+        deferredDropCount = deferredUpdates.length;
+        this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
         deferredUpdates.length = 0;
       } else {
         const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length);
+        deferredDropCount = dropped.length;
+        this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
       }
     }
-    const elapsed = Date.now() - startTime;
-    this.logger.debug("Batch flushed", {
-      strategy: this.#resolvedStrategy,
-      batchSize,
-      durationMs: elapsed,
-      deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
-    });
-    return;
+    const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
+      if (result.retryAttempt === void 0) return earliestAttempt;
+      return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
+    }, void 0);
+    if (retryAttempt !== void 0) {
+      this.scheduleFlush(this.retryDelay(retryAttempt));
+    }
+    if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
+      const elapsed = Date.now() - startTime;
+      this.logger.debug("Batch flushed", {
+        strategy: this.#resolvedStrategy,
+        batchSize,
+        durationMs: elapsed,
+        deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
+      });
+    }
   }
   async _exportTracingEvent(event) {
     await this.waitForInit();
@@ -6349,6 +6389,10 @@ var DefaultExporter = class extends BaseExporter {
       this.#flushTimer = void 0;
     }
     await this.flush();
+    if (this.#flushTimer) {
+      clearTimeout(this.#flushTimer);
+      this.#flushTimer = void 0;
+    }
     this.logger.info("DefaultExporter shutdown complete");
   }
 };
@@ -6881,6 +6925,7 @@ function resolveTracingStorageStrategy2(config, observabilityStorage, storageNam
   }
   return observabilityStrategy.preferred;
 }
+var FLUSH_SUCCEEDED2 = { failed: false };
 var MastraStorageExporter = class extends BaseExporter {
   name = "mastra-storage-exporter";
   #config;
@@ -6975,17 +7020,37 @@ var MastraStorageExporter = class extends BaseExporter {
   /**
    * Schedules a flush using setTimeout
    */
-  scheduleFlush() {
+  scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
     if (this.#flushTimer) {
       clearTimeout(this.#flushTimer);
     }
     this.#flushTimer = setTimeout(() => {
+      this.#flushTimer = void 0;
       this.flushBuffer().catch((error) => {
         this.logger.error("Scheduled flush failed", {
           error: error instanceof Error ? error.message : String(error)
         });
       });
-    }, this.#config.maxBatchWaitMs);
+    }, delayMs);
+  }
+  retryDelay(retryAttempt) {
+    return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
+  }
+  retryAttempt(events) {
+    return events.reduce((earliestAttempt, event) => {
+      if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
+      return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
+    }, void 0);
+  }
+  logStorageFailure(signal, events, retryAttempt, error) {
+    this.logger.warn("Failed to persist observability events", {
+      signal,
+      eventCount: events.length,
+      retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
+      maxRetries: this.#config.maxRetries,
+      ...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
+      error: error instanceof Error ? error.message : String(error)
+    });
   }
   /**
    * Checks flush triggers and schedules/triggers flush as needed.
@@ -7032,22 +7097,26 @@ var MastraStorageExporter = class extends BaseExporter {
    * On other errors, re-adds events to the buffer for retry.
    */
   async flushCreates(signal, events, storageCall) {
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED2;
     if (this.#unsupportedSignals.has(signal)) {
       this.emitDrop(signal, "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     try {
       await storageCall(events);
+      return FLUSH_SUCCEEDED2;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add(signal);
         this.emitDrop(signal, "unsupported-storage", events.length, error);
-      } else {
-        const dropped = this.#eventBuffer.reAddCreates(events);
-        this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+        return { failed: true };
       }
+      const dropped = this.#eventBuffer.reAddCreates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure(signal, events, retryAttempt, error);
+      this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -7056,10 +7125,10 @@ var MastraStorageExporter = class extends BaseExporter {
    */
   async flushSpanUpdates(events, deferredUpdates, isEnd) {
     const deferredCountAtEntry = deferredUpdates.length;
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED2;
     if (this.#unsupportedSignals.has("tracing")) {
       this.emitDrop("tracing", "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     const partials = [];
     for (const event of events) {
@@ -7074,26 +7143,30 @@ var MastraStorageExporter = class extends BaseExporter {
         deferredUpdates.push(event);
       }
     }
-    if (partials.length === 0) return;
+    if (partials.length === 0) return FLUSH_SUCCEEDED2;
     try {
       await this.#observabilityStorage.batchUpdateSpans({ records: partials });
       if (isEnd) {
         this.#eventBuffer.endFinishedSpans({ records: partials });
       }
+      return FLUSH_SUCCEEDED2;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add("tracing");
         deferredUpdates.length = 0;
         this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error);
-      } else {
-        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
-        if (newlyDeferred > 0) {
-          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
-        }
-        const dropped = this.#eventBuffer.reAddUpdates(events);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+        return { failed: true };
+      }
+      const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+      if (newlyDeferred > 0) {
+        deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
       }
+      const dropped = this.#eventBuffer.reAddUpdates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure("tracing", events, retryAttempt, error);
+      this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -7161,7 +7234,7 @@ var MastraStorageExporter = class extends BaseExporter {
           break;
       }
     }
-    await Promise.all([
+    const flushResults = await Promise.all([
       this.flushCreates(
         "feedback",
         createFeedbackEvents,
@@ -7189,25 +7262,36 @@ var MastraStorageExporter = class extends BaseExporter {
       })
     ]);
     const deferredUpdates = [];
-    await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
-    await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
+    flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
+    flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
+    let deferredDropCount = 0;
     if (deferredUpdates.length > 0) {
       if (this.#unsupportedSignals.has("tracing")) {
-        this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
+        deferredDropCount = deferredUpdates.length;
+        this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
         deferredUpdates.length = 0;
       } else {
         const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length);
+        deferredDropCount = dropped.length;
+        this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
       }
     }
-    const elapsed = Date.now() - startTime;
-    this.logger.debug("Batch flushed", {
-      strategy: this.#resolvedStrategy,
-      batchSize,
-      durationMs: elapsed,
-      deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
-    });
-    return;
+    const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
+      if (result.retryAttempt === void 0) return earliestAttempt;
+      return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
+    }, void 0);
+    if (retryAttempt !== void 0) {
+      this.scheduleFlush(this.retryDelay(retryAttempt));
+    }
+    if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
+      const elapsed = Date.now() - startTime;
+      this.logger.debug("Batch flushed", {
+        strategy: this.#resolvedStrategy,
+        batchSize,
+        durationMs: elapsed,
+        deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
+      });
+    }
   }
   async _exportTracingEvent(event) {
     await this.waitForInit();
@@ -7284,6 +7368,10 @@ var MastraStorageExporter = class extends BaseExporter {
       this.#flushTimer = void 0;
     }
     await this.flush();
+    if (this.#flushTimer) {
+      clearTimeout(this.#flushTimer);
+      this.#flushTimer = void 0;
+    }
     this.logger.info("MastraStorageExporter shutdown complete");
   }
 };