@mastra/observability 1.12.0 → 1.13.0-alpha.1

package/dist/index.js

@@ -16012,7 +16012,10 @@ var DefaultExporter = class extends BaseExporter {
         deferredUpdates.length = 0;
         this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error48);
       } else {
-        deferredUpdates.length = 0;
+        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+        if (newlyDeferred > 0) {
+          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
+        }
         const dropped = this.#eventBuffer.reAddUpdates(events);
         this.emitDrop("tracing", "retry-exhausted", dropped.length, error48);
       }
@@ -16711,6 +16714,7 @@ var MastraStorageExporter = class extends BaseExporter {
   #observabilityStorage;
   #resolvedStrategy;
   #flushTimer;
+  #emitDropEvent;
   // Signals whose storage methods threw "not implemented" — skip on future flushes
   #unsupportedSignals = /* @__PURE__ */ new Set();
   constructor(config2 = {}) {
@@ -16732,6 +16736,7 @@ var MastraStorageExporter = class extends BaseExporter {
   async init(options) {
     try {
       this.#isInitializing = true;
+      this.#emitDropEvent = options.emitDropEvent;
       this.#storage = options.mastra?.getStorage();
       if (!this.#storage) {
         this.logger.warn("MastraStorageExporter disabled: Storage not available. Traces will not be persisted.");
@@ -16817,21 +16822,54 @@ var MastraStorageExporter = class extends BaseExporter {
       this.scheduleFlush();
     }
   }
+  sanitizeDropError(error48) {
+    if (error48 instanceof MastraError) {
+      return {
+        id: error48.id,
+        domain: String(error48.domain),
+        message: error48.message
+      };
+    }
+    if (error48 instanceof Error) {
+      return { message: error48.message };
+    }
+    return { message: String(error48) };
+  }
+  emitDrop(signal, reason, count, error48) {
+    if (count === 0) return;
+    const dropEvent = {
+      type: "drop",
+      signal,
+      reason,
+      count,
+      timestamp: /* @__PURE__ */ new Date(),
+      exporterName: this.name,
+      ...this.#observabilityStorage ? { storageName: this.#observabilityStorage.constructor.name } : {},
+      ...error48 === void 0 ? {} : { error: this.sanitizeDropError(error48) }
+    };
+    this.#emitDropEvent?.(dropEvent);
+  }
   /**
    * Flush a batch of create events for a single signal type.
    * On "not implemented" errors, disables the signal for future flushes.
    * On other errors, re-adds events to the buffer for retry.
    */
   async flushCreates(signal, events, storageCall) {
-    if (this.#unsupportedSignals.has(signal) || events.length === 0) return;
+    if (events.length === 0) return;
+    if (this.#unsupportedSignals.has(signal)) {
+      this.emitDrop(signal, "unsupported-storage", events.length);
+      return;
+    }
     try {
       await storageCall(events);
     } catch (error48) {
       if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error48.message);
         this.#unsupportedSignals.add(signal);
+        this.emitDrop(signal, "unsupported-storage", events.length, error48);
       } else {
-        this.#eventBuffer.reAddCreates(events);
+        const dropped = this.#eventBuffer.reAddCreates(events);
+        this.emitDrop(signal, "retry-exhausted", dropped.length, error48);
       }
     }
   }
@@ -16840,7 +16878,12 @@ var MastraStorageExporter = class extends BaseExporter {
    * When `isEnd` is true, successfully flushed spans are removed from tracking.
    */
   async flushSpanUpdates(events, deferredUpdates, isEnd) {
-    if (this.#unsupportedSignals.has("tracing") || events.length === 0) return;
+    const deferredCountAtEntry = deferredUpdates.length;
+    if (events.length === 0) return;
+    if (this.#unsupportedSignals.has("tracing")) {
+      this.emitDrop("tracing", "unsupported-storage", events.length);
+      return;
+    }
     const partials = [];
     for (const event of events) {
       const span = event.exportedSpan;
@@ -16864,9 +16907,15 @@ var MastraStorageExporter = class extends BaseExporter {
       if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
         this.logger.warn(error48.message);
         this.#unsupportedSignals.add("tracing");
-      } else {
         deferredUpdates.length = 0;
-        this.#eventBuffer.reAddUpdates(events);
+        this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error48);
+      } else {
+        const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
+        if (newlyDeferred > 0) {
+          deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
+        }
+        const dropped = this.#eventBuffer.reAddUpdates(events);
+        this.emitDrop("tracing", "retry-exhausted", dropped.length, error48);
       }
     }
   }
@@ -16942,17 +16991,17 @@ var MastraStorageExporter = class extends BaseExporter {
         (events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => buildFeedbackRecord(f)) })
       ),
       this.flushCreates(
-        "logs",
+        "log",
         createLogEvents,
         (events) => this.#observabilityStorage.batchCreateLogs({ logs: events.map((l) => buildLogRecord(l)) })
       ),
       this.flushCreates(
-        "metrics",
+        "metric",
         createMetricEvents,
         (events) => this.#observabilityStorage.batchCreateMetrics({ metrics: events.map((m) => buildMetricRecord(m)) })
       ),
       this.flushCreates(
-        "scores",
+        "score",
         createScoreEvents,
         (events) => this.#observabilityStorage.batchCreateScores({ scores: events.map((s) => buildScoreRecord(s)) })
       ),
@@ -16966,7 +17015,13 @@ var MastraStorageExporter = class extends BaseExporter {
     await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
     await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
     if (deferredUpdates.length > 0) {
-      this.#eventBuffer.reAddUpdates(deferredUpdates);
+      if (this.#unsupportedSignals.has("tracing")) {
+        this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
+        deferredUpdates.length = 0;
+      } else {
+        const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
+        this.emitDrop("tracing", "retry-exhausted", dropped.length);
+      }
     }
     const elapsed = Date.now() - startTime;
     this.logger.debug("Batch flushed", {
@@ -19021,6 +19076,7 @@ var PricingRegistry = class _PricingRegistry {
   constructor(pricingModels) {
     this.pricingModels = pricingModels;
   }
+  pricingModels;
   static globalRegistry = null;
   static fromText(pricingModelText) {
     return new _PricingRegistry(parsePricingModelText(pricingModelText));
@@ -19424,6 +19480,9 @@ function emitTokenMetrics(span, metrics) {
   }
   emitUsageMetrics(attrs, attrs.usage, metrics);
 }
+function emitTokenMetricsForUsage(usage, provider, model, metrics) {
+  emitUsageMetrics({ provider, model }, usage, metrics);
+}
 function emitAutoExtractedMetrics(span, metrics) {
   emitDurationMetrics(span, metrics);
   emitTokenMetrics(span, metrics);
@@ -19585,6 +19644,46 @@ function extractUsageMetrics(usage, providerMetadata) {
 function sumDefinedValues(obj, keys) {
   return keys.reduce((sum, key) => sum + (obj[key] ?? 0), 0);
 }
+function addOptional(a, b) {
+  if (a === void 0 && b === void 0) return void 0;
+  return (a ?? 0) + (b ?? 0);
+}
+function mergeInputDetails(a, b) {
+  if (!a) return b ? { ...b } : void 0;
+  if (!b) return { ...a };
+  return {
+    text: addOptional(a.text, b.text),
+    cacheRead: addOptional(a.cacheRead, b.cacheRead),
+    cacheWrite: addOptional(a.cacheWrite, b.cacheWrite),
+    audio: addOptional(a.audio, b.audio),
+    image: addOptional(a.image, b.image)
+  };
+}
+function mergeOutputDetails(a, b) {
+  if (!a) return b ? { ...b } : void 0;
+  if (!b) return { ...a };
+  return {
+    text: addOptional(a.text, b.text),
+    reasoning: addOptional(a.reasoning, b.reasoning),
+    audio: addOptional(a.audio, b.audio),
+    image: addOptional(a.image, b.image)
+  };
+}
+function addUsageStats(a, b) {
+  if (!a) {
+    return {
+      ...b,
+      inputDetails: b.inputDetails ? { ...b.inputDetails } : void 0,
+      outputDetails: b.outputDetails ? { ...b.outputDetails } : void 0
+    };
+  }
+  return {
+    inputTokens: addOptional(a.inputTokens, b.inputTokens),
+    outputTokens: addOptional(a.outputTokens, b.outputTokens),
+    inputDetails: mergeInputDetails(a.inputDetails, b.inputDetails),
+    outputDetails: mergeOutputDetails(a.outputDetails, b.outputDetails)
+  };
+}
 
 // src/model-tracing.ts
 function supportsModelInference() {
@@ -21114,7 +21213,7 @@ var BaseObservabilityInstance = class extends MastraBase {
    * This ensures all spans emit events regardless of implementation
    */
   wireSpanLifecycle(span) {
-    if (!this.config.includeInternalSpans && span.isInternal) {
+    if (!this.config.includeInternalSpans && span.isInternal && span.type !== SpanType.MODEL_GENERATION) {
       return;
     }
     const originalEnd = span.end.bind(span);
@@ -21124,7 +21223,11 @@ var BaseObservabilityInstance = class extends MastraBase {
         this.logger.warn(`End event is not available on event spans`);
         return;
       }
+      const rollupTarget = this.captureModelUsageRollup(span, options);
       originalEnd(options);
+      if (rollupTarget) {
+        this.applyUsageRollup(rollupTarget);
+      }
       this.emitSpanEnded(span);
     };
     span.update = (options) => {
@@ -21298,6 +21401,70 @@ var BaseObservabilityInstance = class extends MastraBase {
       this.emitTracingEvent(event);
     }
   }
+  /**
+   * When an internal MODEL_GENERATION span ends, capture the rollup payload
+   * (usage, provider, model, target ancestor) needed to attribute its cost
+   * to the closest exported ancestor span. Returns undefined when no rollup
+   * applies — non-MODEL_GENERATION spans, spans that will be exported, or
+   * spans whose usage isn't available at end time.
+   */
+  captureModelUsageRollup(span, endOptions) {
+    if (span.type !== SpanType.MODEL_GENERATION) return void 0;
+    if (!span.isInternal || this.config.includeInternalSpans) return void 0;
+    const endAttrs = endOptions?.attributes ?? void 0;
+    const liveAttrs = span.attributes;
+    const usage = endAttrs?.usage ?? liveAttrs?.usage;
+    if (!usage) return void 0;
+    const ancestor = this.findExportedAncestor(span);
+    if (!ancestor) return void 0;
+    const provider = endAttrs?.provider ?? liveAttrs?.provider;
+    const model = endAttrs?.responseModel ?? endAttrs?.model ?? liveAttrs?.responseModel ?? liveAttrs?.model;
+    return { ancestor, usage, provider, model };
+  }
+  /**
+   * Accumulate usage onto the ancestor's `internalUsage` attribute (for trace
+   * UI visibility) and emit auto-extracted token metrics now, using the
+   * ancestor's metrics context so cost / token labels point at the visible
+   * span instead of the hidden agent that incurred them.
+   */
+  applyUsageRollup(target) {
+    const { ancestor, usage, provider, model } = target;
+    const attrs = ancestor.attributes;
+    attrs.internalUsage = addUsageStats(attrs.internalUsage, usage);
+    try {
+      emitTokenMetricsForUsage(usage, provider, model, this.getMetricsContext(ancestor));
+    } catch (err) {
+      this.logger.error("[Observability] Usage rollup metric emission error:", err);
+    }
+  }
+  /**
+   * Walk up the parent chain to find the closest ancestor that will actually
+   * reach exporters. Skips both internal-filtered ancestors and ancestors
+   * whose type matches `excludeSpanTypes`, so the rollup target is one whose
+   * mutated `internalUsage` attribute is visible in exported traces.
+   *
+   * Note: this does not preemptively run `spanFilter` — that filter can be
+   * async and have side effects, so the rare case of a `spanFilter`-dropped
+   * ancestor falls through.
+   */
+  findExportedAncestor(span) {
+    let ancestor = span.parent;
+    while (ancestor && this.isFilteredFromExport(ancestor)) {
+      ancestor = ancestor.parent;
+    }
+    return ancestor;
+  }
+  /**
+   * Returns true when a span would be dropped by `getSpanForExport` for a
+   * reason cheap to check up-front (internal-span filtering or
+   * `excludeSpanTypes`). Used by `findExportedAncestor` to skip rollup
+   * targets that would silently lose their `internalUsage` attribute.
+   */
+  isFilteredFromExport(span) {
+    if (span.isInternal && !this.config.includeInternalSpans) return true;
+    if (this.config.excludeSpanTypes?.includes(span.type)) return true;
+    return false;
+  }
   /**
    * Emit a tracing event through the bus.
    *
@@ -22248,7 +22415,7 @@ var Observability = class extends MastraBase {
 };
 
 // src/features.ts
-var observabilityFeatures = /* @__PURE__ */ new Set(["model-inference-span"]);
+var observabilityFeatures = /* @__PURE__ */ new Set(["model-inference-span", "internal-usage-rollup"]);
 
 // src/tracing-options.ts
 function buildTracingOptions(...updaters) {