@mastra/observability 1.16.0 → 1.16.1-alpha.1

package/dist/index.js

@@ -998,6 +998,7 @@ var PricingModel = class {
 
 // src/metrics/pricing-registry.ts
 var DATA_FILE_NAME = "pricing-data.jsonl";
+var BEDROCK_GEOGRAPHY_PREFIXES = /* @__PURE__ */ new Set(["global", "us", "eu", "apac", "jp", "au"]);
 var MINIFIED_METER_TO_CANONICAL = {
   it: "input_tokens",
   ot: "output_tokens",
@@ -1032,7 +1033,7 @@ var PricingRegistry = class _PricingRegistry {
     return _PricingRegistry.globalRegistry;
   }
   get(args) {
-    const variants = getModelVariants(args.model);
+    const variants = getModelVariants(args.model, normalizeProvider(args.provider));
     for (const variant of variants) {
       const key = makePricingKey({ provider: args.provider, model: variant });
       const match = this.pricingModels.get(key);
@@ -1126,7 +1127,7 @@ function normalizeProvider(provider) {
   const dotIndex = normalized.indexOf(".");
   return dotIndex !== -1 ? normalized.substring(0, dotIndex) : normalized;
 }
-function getModelVariants(model) {
+function getModelVariants(model, provider) {
   const variants = /* @__PURE__ */ new Set();
   const add = (v) => {
     variants.add(v);
@@ -1141,6 +1142,22 @@ function getModelVariants(model) {
     add(withoutVendor);
     add(withoutVendor.replace(/\./g, "-"));
   }
+  if (provider === "amazon-bedrock") {
+    const addBedrockVariant = (v) => {
+      add(v);
+      add(v.replace(/-(?:v)?\d+(?::\d+)?$/, ""));
+    };
+    const segments = model.split(".");
+    const withoutGeography = BEDROCK_GEOGRAPHY_PREFIXES.has(segments[0] ?? "") ? segments.slice(1).join(".") : model;
+    addBedrockVariant(withoutGeography);
+    addBedrockVariant(withoutGeography.replace(/\./g, "-"));
+    const vendorSeparator = withoutGeography.indexOf(".");
+    if (vendorSeparator !== -1) {
+      const withoutVendor = withoutGeography.substring(vendorSeparator + 1);
+      addBedrockVariant(withoutVendor);
+      addBedrockVariant(withoutVendor.replace(/\./g, "-"));
+    }
+  }
   return [...variants];
 }
 function stripDateSuffix(model) {
@@ -1505,6 +1522,7 @@ function getDurationMetricName(span) {
       return "mastra_agent_duration_ms";
     case SpanType.TOOL_CALL:
     case SpanType.MCP_TOOL_CALL:
+    case SpanType.PROVIDER_TOOL_CALL:
       return "mastra_tool_duration_ms";
     case SpanType.CLIENT_TOOL_CALL:
       return null;
@@ -2464,6 +2482,7 @@ function isSpanInternal(spanType, flags) {
     // Tool-related spans
     case SpanType.TOOL_CALL:
     case SpanType.MCP_TOOL_CALL:
+    case SpanType.PROVIDER_TOOL_CALL:
       return (flags & InternalSpans.TOOL) !== 0;
     // Model-related spans
     case SpanType.MODEL_GENERATION:
@@ -5946,6 +5965,7 @@ function resolveTracingStorageStrategy(config, observabilityStorage, storageName
   }
   return observabilityStrategy.preferred;
 }
+var FLUSH_SUCCEEDED = { failed: false };
 var DefaultExporter = class extends BaseExporter {
   name = "mastra-default-observability-exporter";
   #config;
@@ -6040,17 +6060,37 @@ var DefaultExporter = class extends BaseExporter {
   /**
    * Schedules a flush using setTimeout
    */
-  scheduleFlush() {
+  scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
     if (this.#flushTimer) {
       clearTimeout(this.#flushTimer);
     }
     this.#flushTimer = setTimeout(() => {
+      this.#flushTimer = void 0;
       this.flushBuffer().catch((error) => {
         this.logger.error("Scheduled flush failed", {
           error: error instanceof Error ? error.message : String(error)
         });
       });
-    }, this.#config.maxBatchWaitMs);
+    }, delayMs);
+  }
+  retryDelay(retryAttempt) {
+    return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
+  }
+  retryAttempt(events) {
+    return events.reduce((earliestAttempt, event) => {
+      if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
+      return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
+    }, void 0);
+  }
+  logStorageFailure(signal, events, retryAttempt, error) {
+    this.logger.warn("Failed to persist observability events", {
+      signal,
+      eventCount: events.length,
+      retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
+      maxRetries: this.#config.maxRetries,
+      ...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
+      error: error instanceof Error ? error.message : String(error)
+    });
   }
   /**
    * Checks flush triggers and schedules/triggers flush as needed.
@@ -6097,22 +6137,26 @@ var DefaultExporter = class extends BaseExporter {
    * On other errors, re-adds events to the buffer for retry.
    */
   async flushCreates(signal, events, storageCall) {
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED;
     if (this.#unsupportedSignals.has(signal)) {
       this.emitDrop(signal, "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     try {
       await storageCall(events);
+      return FLUSH_SUCCEEDED;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add(signal);
         this.emitDrop(signal, "unsupported-storage", events.length, error);
-      } else {
-        const dropped = this.#eventBuffer.reAddCreates(events);
-        this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+        return { failed: true };
       }
+      const dropped = this.#eventBuffer.reAddCreates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure(signal, events, retryAttempt, error);
+      this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -6121,10 +6165,10 @@ var DefaultExporter = class extends BaseExporter {
    */
   async flushSpanUpdates(events, deferredUpdates, isEnd) {
     const deferredCountAtEntry = deferredUpdates.length;
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED;
     if (this.#unsupportedSignals.has("tracing")) {
       this.emitDrop("tracing", "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     const partials = [];
     for (const event of events) {
@@ -6139,26 +6183,30 @@ var DefaultExporter = class extends BaseExporter {
         deferredUpdates.push(event);
       }
     }
-    if (partials.length === 0) return;
+    if (partials.length === 0) return FLUSH_SUCCEEDED;
     try {
       await this.#observabilityStorage.batchUpdateSpans({ records: partials });
       if (isEnd) {
         this.#eventBuffer.endFinishedSpans({ records: partials });
       }
+      return FLUSH_SUCCEEDED;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add("tracing");
         deferredUpdates.length = 0;
         this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error);
-      } else {
-        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
-        if (newlyDeferred > 0) {
-          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
-        }
-        const dropped = this.#eventBuffer.reAddUpdates(events);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+        return { failed: true };
+      }
+      const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+      if (newlyDeferred > 0) {
+        deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
       }
+      const dropped = this.#eventBuffer.reAddUpdates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure("tracing", events, retryAttempt, error);
+      this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -6226,7 +6274,7 @@ var DefaultExporter = class extends BaseExporter {
           break;
       }
     }
-    await Promise.all([
+    const flushResults = await Promise.all([
       this.flushCreates(
         "feedback",
         createFeedbackEvents,
@@ -6254,25 +6302,36 @@ var DefaultExporter = class extends BaseExporter {
       })
     ]);
     const deferredUpdates = [];
-    await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
-    await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
+    flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
+    flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
+    let deferredDropCount = 0;
     if (deferredUpdates.length > 0) {
       if (this.#unsupportedSignals.has("tracing")) {
-        this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
+        deferredDropCount = deferredUpdates.length;
+        this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
         deferredUpdates.length = 0;
       } else {
         const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length);
+        deferredDropCount = dropped.length;
+        this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
       }
     }
-    const elapsed = Date.now() - startTime;
-    this.logger.debug("Batch flushed", {
-      strategy: this.#resolvedStrategy,
-      batchSize,
-      durationMs: elapsed,
-      deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
-    });
-    return;
+    const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
+      if (result.retryAttempt === void 0) return earliestAttempt;
+      return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
+    }, void 0);
+    if (retryAttempt !== void 0) {
+      this.scheduleFlush(this.retryDelay(retryAttempt));
+    }
+    if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
+      const elapsed = Date.now() - startTime;
+      this.logger.debug("Batch flushed", {
+        strategy: this.#resolvedStrategy,
+        batchSize,
+        durationMs: elapsed,
+        deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
+      });
+    }
   }
   async _exportTracingEvent(event) {
     await this.waitForInit();
@@ -6349,6 +6408,10 @@ var DefaultExporter = class extends BaseExporter {
       this.#flushTimer = void 0;
     }
     await this.flush();
+    if (this.#flushTimer) {
+      clearTimeout(this.#flushTimer);
+      this.#flushTimer = void 0;
+    }
     this.logger.info("DefaultExporter shutdown complete");
   }
 };
@@ -6881,6 +6944,7 @@ function resolveTracingStorageStrategy2(config, observabilityStorage, storageNam
   }
   return observabilityStrategy.preferred;
 }
+var FLUSH_SUCCEEDED2 = { failed: false };
 var MastraStorageExporter = class extends BaseExporter {
   name = "mastra-storage-exporter";
   #config;
@@ -6975,17 +7039,37 @@ var MastraStorageExporter = class extends BaseExporter {
   /**
    * Schedules a flush using setTimeout
    */
-  scheduleFlush() {
+  scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
     if (this.#flushTimer) {
       clearTimeout(this.#flushTimer);
     }
     this.#flushTimer = setTimeout(() => {
+      this.#flushTimer = void 0;
       this.flushBuffer().catch((error) => {
         this.logger.error("Scheduled flush failed", {
           error: error instanceof Error ? error.message : String(error)
         });
       });
-    }, this.#config.maxBatchWaitMs);
+    }, delayMs);
+  }
+  retryDelay(retryAttempt) {
+    return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
+  }
+  retryAttempt(events) {
+    return events.reduce((earliestAttempt, event) => {
+      if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
+      return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
+    }, void 0);
+  }
+  logStorageFailure(signal, events, retryAttempt, error) {
+    this.logger.warn("Failed to persist observability events", {
+      signal,
+      eventCount: events.length,
+      retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
+      maxRetries: this.#config.maxRetries,
+      ...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
+      error: error instanceof Error ? error.message : String(error)
+    });
   }
   /**
    * Checks flush triggers and schedules/triggers flush as needed.
@@ -7032,22 +7116,26 @@ var MastraStorageExporter = class extends BaseExporter {
    * On other errors, re-adds events to the buffer for retry.
    */
   async flushCreates(signal, events, storageCall) {
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED2;
     if (this.#unsupportedSignals.has(signal)) {
       this.emitDrop(signal, "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     try {
       await storageCall(events);
+      return FLUSH_SUCCEEDED2;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add(signal);
         this.emitDrop(signal, "unsupported-storage", events.length, error);
-      } else {
-        const dropped = this.#eventBuffer.reAddCreates(events);
-        this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+        return { failed: true };
       }
+      const dropped = this.#eventBuffer.reAddCreates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure(signal, events, retryAttempt, error);
+      this.emitDrop(signal, "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -7056,10 +7144,10 @@ var MastraStorageExporter = class extends BaseExporter {
    */
   async flushSpanUpdates(events, deferredUpdates, isEnd) {
     const deferredCountAtEntry = deferredUpdates.length;
-    if (events.length === 0) return;
+    if (events.length === 0) return FLUSH_SUCCEEDED2;
     if (this.#unsupportedSignals.has("tracing")) {
       this.emitDrop("tracing", "unsupported-storage", events.length);
-      return;
+      return { failed: true };
     }
     const partials = [];
     for (const event of events) {
@@ -7074,26 +7162,30 @@ var MastraStorageExporter = class extends BaseExporter {
         deferredUpdates.push(event);
       }
     }
-    if (partials.length === 0) return;
+    if (partials.length === 0) return FLUSH_SUCCEEDED2;
     try {
       await this.#observabilityStorage.batchUpdateSpans({ records: partials });
       if (isEnd) {
         this.#eventBuffer.endFinishedSpans({ records: partials });
       }
+      return FLUSH_SUCCEEDED2;
     } catch (error) {
       if (error instanceof MastraError && error.domain === ErrorDomain.MASTRA_OBSERVABILITY && error.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error.message);
         this.#unsupportedSignals.add("tracing");
         deferredUpdates.length = 0;
         this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error);
-      } else {
-        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
-        if (newlyDeferred > 0) {
-          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
-        }
-        const dropped = this.#eventBuffer.reAddUpdates(events);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+        return { failed: true };
+      }
+      const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+      if (newlyDeferred > 0) {
+        deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
       }
+      const dropped = this.#eventBuffer.reAddUpdates(events);
+      const retryAttempt = this.retryAttempt(events);
+      this.logStorageFailure("tracing", events, retryAttempt, error);
+      this.emitDrop("tracing", "retry-exhausted", dropped.length, error);
+      return { failed: true, retryAttempt };
     }
   }
   /**
@@ -7161,7 +7253,7 @@ var MastraStorageExporter = class extends BaseExporter {
           break;
       }
     }
-    await Promise.all([
+    const flushResults = await Promise.all([
       this.flushCreates(
         "feedback",
         createFeedbackEvents,
@@ -7189,25 +7281,36 @@ var MastraStorageExporter = class extends BaseExporter {
       })
     ]);
     const deferredUpdates = [];
-    await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
-    await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
+    flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
+    flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
+    let deferredDropCount = 0;
     if (deferredUpdates.length > 0) {
       if (this.#unsupportedSignals.has("tracing")) {
-        this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
+        deferredDropCount = deferredUpdates.length;
+        this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
         deferredUpdates.length = 0;
       } else {
         const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
-        this.emitDrop("tracing", "retry-exhausted", dropped.length);
+        deferredDropCount = dropped.length;
+        this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
       }
     }
-    const elapsed = Date.now() - startTime;
-    this.logger.debug("Batch flushed", {
-      strategy: this.#resolvedStrategy,
-      batchSize,
-      durationMs: elapsed,
-      deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
-    });
-    return;
+    const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
+      if (result.retryAttempt === void 0) return earliestAttempt;
+      return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
+    }, void 0);
+    if (retryAttempt !== void 0) {
+      this.scheduleFlush(this.retryDelay(retryAttempt));
+    }
+    if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
+      const elapsed = Date.now() - startTime;
+      this.logger.debug("Batch flushed", {
+        strategy: this.#resolvedStrategy,
+        batchSize,
+        durationMs: elapsed,
+        deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
+      });
+    }
   }
   async _exportTracingEvent(event) {
     await this.waitForInit();
@@ -7284,6 +7387,10 @@ var MastraStorageExporter = class extends BaseExporter {
       this.#flushTimer = void 0;
     }
     await this.flush();
+    if (this.#flushTimer) {
+      clearTimeout(this.#flushTimer);
+      this.#flushTimer = void 0;
+    }
     this.logger.info("MastraStorageExporter shutdown complete");
   }
 };