@mastra/observability 1.0.0-beta.10 → 1.0.0-beta.12

package/dist/index.cjs

@@ -4163,12 +4163,19 @@ var observabilityRegistryConfigSchema = external_exports.object({
 var BaseExporter = class {
   /** Mastra logger instance */
   logger;
+  /** Base configuration (accessible by subclasses) */
+  baseConfig;
   /** Whether this exporter is disabled */
-  isDisabled = false;
+  #disabled = false;
+  /** Public getter for disabled state */
+  get isDisabled() {
+    return this.#disabled;
+  }
   /**
    * Initialize the base exporter with logger
    */
   constructor(config = {}) {
+    this.baseConfig = config;
     const logLevel = this.resolveLogLevel(config.logLevel);
     this.logger = config.logger ?? new logger.ConsoleLogger({ level: logLevel, name: this.constructor.name });
   }
@@ -4203,20 +4210,50 @@ var BaseExporter = class {
    * @param reason - Reason why the exporter is disabled
    */
   setDisabled(reason) {
-    this.isDisabled = true;
+    this.#disabled = true;
     this.logger.warn(`${this.name} disabled: ${reason}`);
   }
+  /**
+   * Apply the customSpanFormatter if configured.
+   * This is called automatically by exportTracingEvent before _exportTracingEvent.
+   *
+   * Supports both synchronous and asynchronous formatters. If the formatter
+   * returns a Promise, it will be awaited.
+   *
+   * @param event - The incoming tracing event
+   * @returns The (possibly modified) event to process
+   */
+  async applySpanFormatter(event) {
+    if (this.baseConfig.customSpanFormatter) {
+      try {
+        const formattedSpan = await this.baseConfig.customSpanFormatter(event.exportedSpan);
+        return {
+          ...event,
+          exportedSpan: formattedSpan
+        };
+      } catch (error) {
+        this.logger.error(`${this.name}: Error in customSpanFormatter`, {
+          error,
+          spanId: event.exportedSpan.id,
+          traceId: event.exportedSpan.traceId
+        });
+      }
+    }
+    return event;
+  }
   /**
    * Export a tracing event
    *
-   * This method checks if the exporter is disabled before calling _exportEvent.
-   * Subclasses should implement _exportEvent instead of overriding this method.
+   * This method checks if the exporter is disabled, applies the customSpanFormatter,
+   * then calls _exportTracingEvent.
+   * Subclasses should implement _exportTracingEvent instead of overriding this method.
    */
   async exportTracingEvent(event) {
     if (this.isDisabled) {
       return;
     }
-    await this._exportTracingEvent(event);
+    const processedEvent = await this.applySpanFormatter(event);
+    await this._exportTracingEvent(processedEvent);
   }
   /**
    * Shutdown the exporter and clean up resources
@@ -4227,9 +4264,957 @@ var BaseExporter = class {
     this.logger.info(`${this.name} shutdown complete`);
   }
 };
+var TraceData = class {
+  /** The vendor-specific root/trace object */
+  #rootSpan;
+  /** The span ID of the root span */
+  #rootSpanId;
+  /** Whether a span with isRootSpan=true has been successfully processed */
+  #rootSpanProcessed;
+  /** Maps eventId to vendor-specific event objects */
+  #events;
+  /** Maps spanId to vendor-specific span objects */
+  #spans;
+  /** Maps spanId to parentSpanId, representing the span hierarchy */
+  #tree;
+  /** Set of span IDs that have started but not yet ended */
+  #activeSpanIds;
+  /** Maps spanId to vendor-specific metadata */
+  #metadata;
+  /** Arbitrary key-value storage for per-trace data */
+  #extraData;
+  /** Events waiting for the root span to be processed */
+  #waitingForRoot;
+  /** Events waiting for specific parent spans, keyed by parentSpanId */
+  #waitingForParent;
+  /** When this trace data was created, used for cap enforcement */
+  createdAt;
+  constructor() {
+    this.#events = /* @__PURE__ */ new Map();
+    this.#spans = /* @__PURE__ */ new Map();
+    this.#activeSpanIds = /* @__PURE__ */ new Set();
+    this.#tree = /* @__PURE__ */ new Map();
+    this.#metadata = /* @__PURE__ */ new Map();
+    this.#extraData = /* @__PURE__ */ new Map();
+    this.#rootSpanProcessed = false;
+    this.#waitingForRoot = [];
+    this.#waitingForParent = /* @__PURE__ */ new Map();
+    this.createdAt = /* @__PURE__ */ new Date();
+  }
+  /**
+   * Check if this trace has a root span registered.
+   * @returns True if addRoot() has been called
+   */
+  hasRoot() {
+    return !!this.#rootSpanId;
+  }
+  /**
+   * Register the root span for this trace.
+   * @param args.rootId - The span ID of the root span
+   * @param args.rootData - The vendor-specific root object
+   */
+  addRoot(args) {
+    this.#rootSpanId = args.rootId;
+    this.#rootSpan = args.rootData;
+    this.#rootSpanProcessed = true;
+  }
+  /**
+   * Get the vendor-specific root object.
+   * @returns The root object, or undefined if not yet set
+   */
+  getRoot() {
+    return this.#rootSpan;
+  }
+  /**
+   * Check if a span with isRootSpan=true has been successfully processed.
+   * Set via addRoot() or markRootSpanProcessed().
+   * @returns True if the root span has been processed
+   */
+  isRootProcessed() {
+    return this.#rootSpanProcessed;
+  }
+  /**
+   * Mark that the root span has been processed.
+   * Used by exporters with skipBuildRootTask=true where root goes through _buildSpan
+   * instead of _buildRoot.
+   */
+  markRootSpanProcessed() {
+    this.#rootSpanProcessed = true;
+  }
+  /**
+   * Store an arbitrary value in per-trace storage.
+   * @param key - Storage key
+   * @param value - Value to store
+   */
+  setExtraValue(key, value) {
+    this.#extraData.set(key, value);
+  }
+  /**
+   * Check if a key exists in per-trace storage.
+   * @param key - Storage key
+   * @returns True if the key exists
+   */
+  hasExtraValue(key) {
+    return this.#extraData.has(key);
+  }
+  /**
+   * Get a value from per-trace storage.
+   * @param key - Storage key
+   * @returns The stored value, or undefined if not found
+   */
+  getExtraValue(key) {
+    return this.#extraData.get(key);
+  }
+  // ============================================================================
+  // Early Queue Methods
+  // ============================================================================
+  /**
+   * Add an event to the waiting queue.
+   * @param args.event - The tracing event to queue
+   * @param args.waitingFor - 'root' or a specific parentSpanId
+   * @param args.attempts - Optional: preserve attempts count when re-queuing
+   * @param args.queuedAt - Optional: preserve original queue time when re-queuing
+   */
+  addToWaitingQueue(args) {
+    const queuedEvent = {
+      event: args.event,
+      waitingFor: args.waitingFor,
+      attempts: args.attempts ?? 0,
+      queuedAt: args.queuedAt ?? /* @__PURE__ */ new Date()
+    };
+    if (args.waitingFor === "root") {
+      this.#waitingForRoot.push(queuedEvent);
+    } else {
+      const queue = this.#waitingForParent.get(args.waitingFor) ?? [];
+      queue.push(queuedEvent);
+      this.#waitingForParent.set(args.waitingFor, queue);
+    }
+  }
+  /**
+   * Get all events waiting for the root span.
+   * Returns a copy of the internal array.
+   */
+  getEventsWaitingForRoot() {
+    return [...this.#waitingForRoot];
+  }
+  /**
+   * Get all events waiting for a specific parent span.
+   * Returns a copy of the internal array.
+   */
+  getEventsWaitingFor(args) {
+    return [...this.#waitingForParent.get(args.spanId) ?? []];
+  }
+  /**
+   * Clear the waiting-for-root queue.
+   */
+  clearWaitingForRoot() {
+    this.#waitingForRoot = [];
+  }
+  /**
+   * Clear the waiting queue for a specific parent span.
+   */
+  clearWaitingFor(args) {
+    this.#waitingForParent.delete(args.spanId);
+  }
+  /**
+   * Get total count of events in all waiting queues.
+   */
+  waitingQueueSize() {
+    let count = this.#waitingForRoot.length;
+    for (const queue of this.#waitingForParent.values()) {
+      count += queue.length;
+    }
+    return count;
+  }
+  /**
+   * Get all queued events across all waiting queues.
+   * Used for cleanup and logging orphaned events.
+   * @returns Array of all queued events
+   */
+  getAllQueuedEvents() {
+    const all = [...this.#waitingForRoot];
+    for (const queue of this.#waitingForParent.values()) {
+      all.push(...queue);
+    }
+    return all;
+  }
+  // ============================================================================
+  // Span Tree Methods
+  // ============================================================================
+  /**
+   * Record the parent-child relationship for a span.
+   * @param args.spanId - The child span ID
+   * @param args.parentSpanId - The parent span ID, or undefined for root spans
+   */
+  addBranch(args) {
+    this.#tree.set(args.spanId, args.parentSpanId);
+  }
+  /**
+   * Get the parent span ID for a given span.
+   * @param args.spanId - The span ID to look up
+   * @returns The parent span ID, or undefined if root or not found
+   */
+  getParentId(args) {
+    return this.#tree.get(args.spanId);
+  }
+  // ============================================================================
+  // Span Management Methods
+  // ============================================================================
+  /**
+   * Register a span and mark it as active.
+   * @param args.spanId - The span ID
+   * @param args.spanData - The vendor-specific span object
+   */
+  addSpan(args) {
+    this.#spans.set(args.spanId, args.spanData);
+    this.#activeSpanIds.add(args.spanId);
+  }
+  /**
+   * Check if a span exists (regardless of active state).
+   * @param args.spanId - The span ID to check
+   * @returns True if the span exists
+   */
+  hasSpan(args) {
+    return this.#spans.has(args.spanId);
+  }
+  /**
+   * Get a span by ID.
+   * @param args.spanId - The span ID to look up
+   * @returns The vendor-specific span object, or undefined if not found
+   */
+  getSpan(args) {
+    return this.#spans.get(args.spanId);
+  }
+  /**
+   * Mark a span as ended (no longer active).
+   * @param args.spanId - The span ID to mark as ended
+   */
+  endSpan(args) {
+    this.#activeSpanIds.delete(args.spanId);
+  }
+  /**
+   * Check if a span is currently active (started but not ended).
+   * @param args.spanId - The span ID to check
+   * @returns True if the span is active
+   */
+  isActiveSpan(args) {
+    return this.#activeSpanIds.has(args.spanId);
+  }
+  /**
+   * Get the count of currently active spans.
+   * @returns Number of active spans
+   */
+  activeSpanCount() {
+    return this.#activeSpanIds.size;
+  }
+  /**
+   * Get all active span IDs.
+   * @returns Array of active span IDs
+   */
+  get activeSpanIds() {
+    return [...this.#activeSpanIds];
+  }
+  // ============================================================================
+  // Event Management Methods
+  // ============================================================================
+  /**
+   * Register an event.
+   * @param args.eventId - The event ID
+   * @param args.eventData - The vendor-specific event object
+   */
+  addEvent(args) {
+    this.#events.set(args.eventId, args.eventData);
+  }
+  // ============================================================================
+  // Metadata Methods
+  // ============================================================================
+  /**
+   * Store vendor-specific metadata for a span.
+   * Note: This overwrites any existing metadata for the span.
+   * @param args.spanId - The span ID
+   * @param args.metadata - The vendor-specific metadata
+   */
+  addMetadata(args) {
+    this.#metadata.set(args.spanId, args.metadata);
+  }
+  /**
+   * Get vendor-specific metadata for a span.
+   * @param args.spanId - The span ID
+   * @returns The metadata, or undefined if not found
+   */
+  getMetadata(args) {
+    return this.#metadata.get(args.spanId);
+  }
+  // ============================================================================
+  // Parent Lookup Methods
+  // ============================================================================
+  /**
+   * Get the parent span or event for a given span.
+   * Looks up in both spans and events maps.
+   * @param args.span - The span to find the parent for
+   * @returns The parent span/event object, or undefined if root or not found
+   */
+  getParent(args) {
+    const parentId = args.span.parentSpanId;
+    if (parentId) {
+      if (this.#spans.has(parentId)) {
+        return this.#spans.get(parentId);
+      }
+      if (this.#events.has(parentId)) {
+        return this.#events.get(parentId);
+      }
+    }
+    return void 0;
+  }
+  /**
+   * Get the parent span/event or fall back to the root object.
+   * Useful for vendors that attach child spans to either parent spans or the trace root.
+   * @param args.span - The span to find the parent for
+   * @returns The parent span/event, the root object, or undefined
+   */
+  getParentOrRoot(args) {
+    return this.getParent(args) ?? this.getRoot();
+  }
+};
+var DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS = 5;
+var DEFAULT_EARLY_QUEUE_TTL_MS = 3e4;
+var DEFAULT_TRACE_CLEANUP_DELAY_MS = 3e4;
+var DEFAULT_MAX_PENDING_CLEANUP_TRACES = 100;
+var DEFAULT_MAX_TOTAL_TRACES = 500;
+var TrackingExporter = class extends BaseExporter {
+  /** Map of traceId to per-trace data container */
+  #traceMap = /* @__PURE__ */ new Map();
+  /** Flag to prevent processing during shutdown */
+  #shutdownStarted = false;
+  /** Flag to prevent concurrent hard cap enforcement */
+  #hardCapEnforcementInProgress = false;
+  /** Map of traceId to scheduled cleanup timeout */
+  #pendingCleanups = /* @__PURE__ */ new Map();
+  // Note: #traceMap maintains insertion order (JS Map spec), so we use
+  // #traceMap.keys() to iterate traces oldest-first for cap enforcement.
+  /** Subclass configuration with resolved values */
+  config;
+  /** Maximum attempts to process a queued event before dropping */
+  #earlyQueueMaxAttempts;
+  /** TTL in milliseconds for queued events */
+  #earlyQueueTTLMs;
+  /** Delay before cleaning up completed traces */
+  #traceCleanupDelayMs;
+  /** Soft cap on traces awaiting cleanup */
+  #maxPendingCleanupTraces;
+  /** Hard cap on total traces (will abort active spans if exceeded) */
+  #maxTotalTraces;
+  constructor(config) {
+    super(config);
+    this.config = config;
+    this.#earlyQueueMaxAttempts = config.earlyQueueMaxAttempts ?? DEFAULT_EARLY_QUEUE_MAX_ATTEMPTS;
+    this.#earlyQueueTTLMs = config.earlyQueueTTLMs ?? DEFAULT_EARLY_QUEUE_TTL_MS;
+    this.#traceCleanupDelayMs = config.traceCleanupDelayMs ?? DEFAULT_TRACE_CLEANUP_DELAY_MS;
+    this.#maxPendingCleanupTraces = config.maxPendingCleanupTraces ?? DEFAULT_MAX_PENDING_CLEANUP_TRACES;
+    this.#maxTotalTraces = config.maxTotalTraces ?? DEFAULT_MAX_TOTAL_TRACES;
+  }
+  // ============================================================================
+  // Early Queue Processing
+  // ============================================================================
+  /**
+   * Schedule async processing of events waiting for root span.
+   * Called after root span is successfully processed.
+   */
+  #scheduleProcessWaitingForRoot(traceId) {
+    setImmediate(() => {
+      this.#processWaitingForRoot(traceId).catch((error) => {
+        this.logger.error(`${this.name}: Error processing waiting-for-root queue`, { error, traceId });
+      });
+    });
+  }
+  /**
+   * Schedule async processing of events waiting for a specific parent span.
+   * Called after a span/event is successfully created.
+   */
+  #scheduleProcessWaitingFor(traceId, spanId) {
+    setImmediate(() => {
+      this.#processWaitingFor(traceId, spanId).catch((error) => {
+        this.logger.error(`${this.name}: Error processing waiting queue`, { error, traceId, spanId });
+      });
+    });
+  }
+  /**
+   * Process all events waiting for root span.
+   */
+  async #processWaitingForRoot(traceId) {
+    if (this.#shutdownStarted) return;
+    const traceData = this.#traceMap.get(traceId);
+    if (!traceData) return;
+    const queue = traceData.getEventsWaitingForRoot();
+    if (queue.length === 0) return;
+    this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for root`, { traceId });
+    const toKeep = [];
+    const now = Date.now();
+    for (const queuedEvent of queue) {
+      if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
+        this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
+          traceId,
+          spanId: queuedEvent.event.exportedSpan.id,
+          waitingFor: queuedEvent.waitingFor,
+          queuedAt: queuedEvent.queuedAt,
+          attempts: queuedEvent.attempts
+        });
+        continue;
+      }
+      if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
+        this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
+          traceId,
+          spanId: queuedEvent.event.exportedSpan.id,
+          waitingFor: queuedEvent.waitingFor,
+          attempts: queuedEvent.attempts
+        });
+        continue;
+      }
+      queuedEvent.attempts++;
+      const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
+      if (!processed) {
+        const parentId = queuedEvent.event.exportedSpan.parentSpanId;
+        if (parentId && traceData.isRootProcessed()) {
+          traceData.addToWaitingQueue({
+            event: queuedEvent.event,
+            waitingFor: parentId,
+            attempts: queuedEvent.attempts,
+            queuedAt: queuedEvent.queuedAt
+          });
+        } else {
+          toKeep.push(queuedEvent);
+        }
+      }
+    }
+    traceData.clearWaitingForRoot();
+    for (const event of toKeep) {
+      traceData.addToWaitingQueue({
+        event: event.event,
+        waitingFor: "root",
+        attempts: event.attempts,
+        queuedAt: event.queuedAt
+      });
+    }
+  }
+  /**
+   * Process events waiting for a specific parent span.
+   */
+  async #processWaitingFor(traceId, spanId) {
+    if (this.#shutdownStarted) return;
+    const traceData = this.#traceMap.get(traceId);
+    if (!traceData) return;
+    const queue = traceData.getEventsWaitingFor({ spanId });
+    if (queue.length === 0) return;
+    this.logger.debug(`${this.name}: Processing ${queue.length} events waiting for span`, { traceId, spanId });
+    const toKeep = [];
+    const now = Date.now();
+    for (const queuedEvent of queue) {
+      if (now - queuedEvent.queuedAt.getTime() > this.#earlyQueueTTLMs) {
+        this.logger.warn(`${this.name}: Dropping event due to TTL expiry`, {
+          traceId,
+          spanId: queuedEvent.event.exportedSpan.id,
+          waitingFor: queuedEvent.waitingFor,
+          queuedAt: queuedEvent.queuedAt,
+          attempts: queuedEvent.attempts
+        });
+        continue;
+      }
+      if (queuedEvent.attempts >= this.#earlyQueueMaxAttempts) {
+        this.logger.warn(`${this.name}: Dropping event due to max attempts`, {
+          traceId,
+          spanId: queuedEvent.event.exportedSpan.id,
+          waitingFor: queuedEvent.waitingFor,
+          attempts: queuedEvent.attempts
+        });
+        continue;
+      }
+      queuedEvent.attempts++;
+      const processed = await this.#tryProcessQueuedEvent(queuedEvent, traceData);
+      if (!processed) {
+        toKeep.push(queuedEvent);
+      }
+    }
+    traceData.clearWaitingFor({ spanId });
+    for (const event of toKeep) {
+      traceData.addToWaitingQueue({
+        event: event.event,
+        waitingFor: spanId,
+        attempts: event.attempts,
+        queuedAt: event.queuedAt
+      });
+    }
+  }
+  /**
+   * Try to process a queued event.
+   * Returns true if successfully processed, false if still waiting for dependencies.
+   */
+  async #tryProcessQueuedEvent(queuedEvent, traceData) {
+    const { event } = queuedEvent;
+    const { exportedSpan } = event;
+    const method = this.getMethod(event);
+    try {
+      switch (method) {
+        case "handleEventSpan": {
+          traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
+          const eventData = await this._buildEvent({ span: exportedSpan, traceData });
+          if (eventData) {
+            if (!this.skipCachingEventSpans) {
+              traceData.addEvent({ eventId: exportedSpan.id, eventData });
+            }
+            this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
+            return true;
+          }
+          return false;
+        }
+        case "handleSpanStart": {
+          traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
+          const spanData = await this._buildSpan({ span: exportedSpan, traceData });
+          if (spanData) {
+            traceData.addSpan({ spanId: exportedSpan.id, spanData });
+            if (exportedSpan.isRootSpan) {
+              traceData.markRootSpanProcessed();
+            }
+            this.#scheduleProcessWaitingFor(exportedSpan.traceId, exportedSpan.id);
+            return true;
+          }
+          return false;
+        }
+        case "handleSpanUpdate": {
+          await this._updateSpan({ span: exportedSpan, traceData });
+          return true;
+        }
+        case "handleSpanEnd": {
+          traceData.endSpan({ spanId: exportedSpan.id });
+          await this._finishSpan({ span: exportedSpan, traceData });
+          if (traceData.activeSpanCount() === 0) {
+            this.#scheduleCleanup(exportedSpan.traceId);
+          }
+          return true;
+        }
+        default:
+          return false;
+      }
+    } catch (error) {
+      this.logger.error(`${this.name}: Error processing queued event`, { error, event, method });
+      return false;
+    }
+  }
+  // ============================================================================
+  // Delayed Cleanup
+  // ============================================================================
+  /**
+   * Schedule cleanup of trace data after a delay.
+   * Allows late-arriving data to still be processed.
+   */
+  #scheduleCleanup(traceId) {
+    this.#cancelScheduledCleanup(traceId);
+    this.logger.debug(`${this.name}: Scheduling cleanup in ${this.#traceCleanupDelayMs}ms`, { traceId });
+    const timeout = setTimeout(() => {
+      this.#pendingCleanups.delete(traceId);
+      this.#performCleanup(traceId);
+    }, this.#traceCleanupDelayMs);
+    this.#pendingCleanups.set(traceId, timeout);
+    this.#enforcePendingCleanupCap();
+  }
+  /**
+   * Cancel a scheduled cleanup for a trace.
+   */
+  #cancelScheduledCleanup(traceId) {
+    const existingTimeout = this.#pendingCleanups.get(traceId);
+    if (existingTimeout) {
+      clearTimeout(existingTimeout);
+      this.#pendingCleanups.delete(traceId);
+      this.logger.debug(`${this.name}: Cancelled scheduled cleanup`, { traceId });
+    }
+  }
+  /**
+   * Perform the actual cleanup of trace data.
+   */
+  #performCleanup(traceId) {
+    const traceData = this.#traceMap.get(traceId);
+    if (!traceData) return;
+    const orphanedEvents = traceData.getAllQueuedEvents();
+    if (orphanedEvents.length > 0) {
+      this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on cleanup`, {
+        traceId,
+        orphanedEvents: orphanedEvents.map((e) => ({
+          spanId: e.event.exportedSpan.id,
+          waitingFor: e.waitingFor,
+          attempts: e.attempts,
+          queuedAt: e.queuedAt
+        }))
+      });
+    }
+    this.#traceMap.delete(traceId);
+    this.logger.debug(`${this.name}: Cleaned up trace data`, { traceId });
+  }
+  // ============================================================================
+  // Cap Enforcement
+  // ============================================================================
+  /**
+   * Enforce soft cap on pending cleanup traces.
+   * Only removes traces with activeSpanCount == 0.
+   */
+  #enforcePendingCleanupCap() {
+    if (this.#pendingCleanups.size <= this.#maxPendingCleanupTraces) {
+      return;
+    }
+    const toRemove = this.#pendingCleanups.size - this.#maxPendingCleanupTraces;
+    this.logger.warn(`${this.name}: Pending cleanup cap exceeded, force-cleaning ${toRemove} traces`, {
+      pendingCount: this.#pendingCleanups.size,
+      cap: this.#maxPendingCleanupTraces
+    });
+    let removed = 0;
+    for (const traceId of this.#traceMap.keys()) {
+      if (removed >= toRemove) break;
+      if (this.#pendingCleanups.has(traceId)) {
+        this.#cancelScheduledCleanup(traceId);
+        this.#performCleanup(traceId);
+        removed++;
+      }
+    }
+  }
+  /**
+   * Enforce hard cap on total traces.
+   * Will kill even active traces if necessary.
+   * Uses a flag to prevent concurrent executions when called fire-and-forget.
+   */
+  async #enforceHardCap() {
+    if (this.#traceMap.size <= this.#maxTotalTraces || this.#hardCapEnforcementInProgress) {
+      return;
+    }
+    this.#hardCapEnforcementInProgress = true;
+    try {
+      if (this.#traceMap.size <= this.#maxTotalTraces) {
+        return;
+      }
+      const toRemove = this.#traceMap.size - this.#maxTotalTraces;
+      this.logger.warn(`${this.name}: Total trace cap exceeded, killing ${toRemove} oldest traces`, {
+        traceCount: this.#traceMap.size,
+        cap: this.#maxTotalTraces
+      });
+      const reason = {
+        id: "TRACE_CAP_EXCEEDED",
+        message: "Trace killed due to memory cap enforcement.",
+        domain: "MASTRA_OBSERVABILITY",
+        category: "SYSTEM"
+      };
+      let removed = 0;
+      for (const traceId of [...this.#traceMap.keys()]) {
+        if (removed >= toRemove) break;
+        const traceData = this.#traceMap.get(traceId);
+        if (traceData) {
+          for (const spanId of traceData.activeSpanIds) {
+            const span = traceData.getSpan({ spanId });
+            if (span) {
+              await this._abortSpan({ span, traceData, reason });
+            }
+          }
+          this.#cancelScheduledCleanup(traceId);
+          this.#performCleanup(traceId);
+          removed++;
+        }
+      }
+    } finally {
+      this.#hardCapEnforcementInProgress = false;
+    }
+  }
+  // ============================================================================
+  // Lifecycle Hooks (Override in subclass)
+  // ============================================================================
+  /**
+   * Hook called before processing each tracing event.
+   * Override to transform or enrich the event before processing.
+   *
+   * Note: The customSpanFormatter is applied at the BaseExporter level before this hook.
+   * Subclasses can override this to add additional pre-processing logic.
+   *
+   * @param event - The incoming tracing event
+   * @returns The (possibly modified) event to process
+   */
+  async _preExportTracingEvent(event) {
+    return event;
+  }
+  /**
+   * Hook called after processing each tracing event.
+   * Override to perform post-processing actions like flushing.
+   */
+  async _postExportTracingEvent() {
+  }
+  // ============================================================================
+  // Behavior Flags (Override in subclass as needed)
+  // ============================================================================
+  /**
+   * If true, skip calling _buildRoot and let root spans go through _buildSpan.
+   * Use when the vendor doesn't have a separate trace/root concept.
+   * @default false
+   */
+  skipBuildRootTask = false;
+  /**
+   * If true, skip processing span_updated events entirely.
+   * Use when the vendor doesn't support incremental span updates.
+   * @default false
+   */
+  skipSpanUpdateEvents = false;
+  /**
+   * If true, don't cache event spans in TraceData.
+   * Use when events can't be parents of other spans.
+   * @default false
+   */
+  skipCachingEventSpans = false;
+  getMethod(event) {
+    if (event.exportedSpan.isEvent) {
+      return "handleEventSpan";
+    }
+    const eventType = event.type;
+    switch (eventType) {
+      case observability.TracingEventType.SPAN_STARTED:
+        return "handleSpanStart";
+      case observability.TracingEventType.SPAN_UPDATED:
+        return "handleSpanUpdate";
+      case observability.TracingEventType.SPAN_ENDED:
+        return "handleSpanEnd";
+      default: {
+        const _exhaustiveCheck = eventType;
+        throw new Error(`Unhandled event type: ${_exhaustiveCheck}`);
+      }
+    }
+  }
+  async _exportTracingEvent(event) {
+    if (this.#shutdownStarted) {
+      return;
+    }
+    const method = this.getMethod(event);
+    if (method == "handleSpanUpdate" && this.skipSpanUpdateEvents) {
+      return;
+    }
+    const traceId = event.exportedSpan.traceId;
+    const traceData = this.getTraceData({ traceId, method });
+    const { exportedSpan } = await this._preExportTracingEvent(event);
+    if (!this.skipBuildRootTask && !traceData.hasRoot()) {
+      if (exportedSpan.isRootSpan) {
+        this.logger.debug(`${this.name}: Building root`, {
+          traceId: exportedSpan.traceId,
+          spanId: exportedSpan.id
+        });
+        const rootData = await this._buildRoot({ span: exportedSpan, traceData });
+        if (rootData) {
+          this.logger.debug(`${this.name}: Adding root`, {
+            traceId: exportedSpan.traceId,
+            spanId: exportedSpan.id
+          });
+          traceData.addRoot({ rootId: exportedSpan.id, rootData });
+          this.#scheduleProcessWaitingForRoot(traceId);
+        }
+      } else {
+        this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
+          traceId: exportedSpan.traceId,
+          spanId: exportedSpan.id
+        });
+        traceData.addToWaitingQueue({ event, waitingFor: "root" });
+        return;
+      }
+    }
+    if (exportedSpan.metadata && this.name in exportedSpan.metadata) {
+      const metadata = exportedSpan.metadata[this.name];
+      this.logger.debug(`${this.name}: Found provider metadata in span`, {
+        traceId: exportedSpan.traceId,
+        spanId: exportedSpan.id,
+        metadata
+      });
+      traceData.addMetadata({ spanId: exportedSpan.id, metadata });
+    }
+    try {
+      switch (method) {
+        case "handleEventSpan": {
+          this.logger.debug(`${this.name}: handling event`, {
+            traceId: exportedSpan.traceId,
+            spanId: exportedSpan.id
+          });
+          traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
+          const eventData = await this._buildEvent({ span: exportedSpan, traceData });
+          if (eventData) {
+            if (!this.skipCachingEventSpans) {
+              this.logger.debug(`${this.name}: adding event to traceData`, {
+                traceId: exportedSpan.traceId,
+                spanId: exportedSpan.id
+              });
+              traceData.addEvent({ eventId: exportedSpan.id, eventData });
+            }
+            this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
+          } else {
+            const parentId = exportedSpan.parentSpanId;
+            this.logger.debug(`${this.name}: adding event to waiting queue`, {
+              traceId: exportedSpan.traceId,
+              spanId: exportedSpan.id,
+              waitingFor: parentId ?? "root"
+            });
+            traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
+          }
+          break;
+        }
+        case "handleSpanStart": {
+          this.logger.debug(`${this.name}: handling span start`, {
+            traceId: exportedSpan.traceId,
+            spanId: exportedSpan.id
+          });
+          traceData.addBranch({ spanId: exportedSpan.id, parentSpanId: exportedSpan.parentSpanId });
+          const spanData = await this._buildSpan({ span: exportedSpan, traceData });
+          if (spanData) {
+            this.logger.debug(`${this.name}: adding span to traceData`, {
+              traceId: exportedSpan.traceId,
+              spanId: exportedSpan.id
+            });
+            traceData.addSpan({ spanId: exportedSpan.id, spanData });
+            if (exportedSpan.isRootSpan) {
+              traceData.markRootSpanProcessed();
+              this.#scheduleProcessWaitingForRoot(traceId);
+            }
+            this.#scheduleProcessWaitingFor(traceId, exportedSpan.id);
+          } else {
+            const parentId = exportedSpan.parentSpanId;
+            this.logger.debug(`${this.name}: adding span to waiting queue`, {
+              traceId: exportedSpan.traceId,
+              waitingFor: parentId ?? "root"
+            });
+            traceData.addToWaitingQueue({ event, waitingFor: parentId ?? "root" });
+          }
+          break;
+        }
+        case "handleSpanUpdate":
+          this.logger.debug(`${this.name}: handling span update`, {
+            traceId: exportedSpan.traceId,
+            spanId: exportedSpan.id
+          });
+          await this._updateSpan({ span: exportedSpan, traceData });
+          break;
+        case "handleSpanEnd":
+          this.logger.debug(`${this.name}: handling span end`, {
+            traceId: exportedSpan.traceId,
+            spanId: exportedSpan.id
+          });
+          traceData.endSpan({ spanId: exportedSpan.id });
+          await this._finishSpan({ span: exportedSpan, traceData });
+          if (traceData.activeSpanCount() === 0) {
+            this.#scheduleCleanup(traceId);
+          }
+          break;
+      }
+    } catch (error) {
+      this.logger.error(`${this.name}: exporter error`, { error, event, method });
+    }
+    if (traceData.activeSpanCount() === 0) {
+      this.#scheduleCleanup(traceId);
+    }
+    await this._postExportTracingEvent();
+  }
+  // ============================================================================
+  // Protected Helpers
+  // ============================================================================
+  /**
+   * Get or create the TraceData container for a trace.
+   * Also cancels any pending cleanup since new data has arrived.
+   *
+   * @param args.traceId - The trace ID
+   * @param args.method - The calling method name (for logging)
+   * @returns The TraceData container for this trace
+   */
+  getTraceData(args) {
+    const { traceId, method } = args;
+    this.#cancelScheduledCleanup(traceId);
+    if (!this.#traceMap.has(traceId)) {
+      this.#traceMap.set(traceId, new TraceData());
+      this.logger.debug(`${this.name}: Created new trace data cache`, {
+        traceId,
+        method
+      });
+      this.#enforceHardCap().catch((error) => {
+        this.logger.error(`${this.name}: Error enforcing hard cap`, { error });
+      });
+    }
+    return this.#traceMap.get(traceId);
+  }
+  /**
+   * Get the current number of traces being tracked.
+   * @returns The trace count
+   */
+  traceMapSize() {
+    return this.#traceMap.size;
+  }
+  // ============================================================================
+  // Shutdown Hooks (Override in subclass as needed)
+  // ============================================================================
+  /**
+   * Hook called at the start of shutdown, before cancelling timers and aborting spans.
+   * Override to perform vendor-specific pre-shutdown tasks.
+   */
+  async _preShutdown() {
+  }
+  /**
+   * Hook called at the end of shutdown, after all spans are aborted.
+   * Override to perform vendor-specific cleanup (e.g., flushing).
+   */
+  async _postShutdown() {
+  }
+  /**
+   * Gracefully shut down the exporter.
+   * Cancels all pending cleanup timers, aborts all active spans, and clears state.
+   */
+  async shutdown() {
+    if (this.isDisabled) {
+      return;
+    }
+    this.#shutdownStarted = true;
+    await this._preShutdown();
+    for (const [traceId, timeout] of this.#pendingCleanups) {
+      clearTimeout(timeout);
+      this.logger.debug(`${this.name}: Cancelled pending cleanup on shutdown`, { traceId });
+    }
+    this.#pendingCleanups.clear();
+    const reason = {
+      id: "SHUTDOWN",
+      message: "Observability is shutting down.",
+      domain: "MASTRA_OBSERVABILITY",
+      category: "SYSTEM"
+    };
+    for (const [traceId, traceData] of this.#traceMap) {
+      const orphanedEvents = traceData.getAllQueuedEvents();
+      if (orphanedEvents.length > 0) {
+        this.logger.warn(`${this.name}: Dropping ${orphanedEvents.length} orphaned events on shutdown`, {
+          traceId,
+          orphanedEvents: orphanedEvents.map((e) => ({
+            spanId: e.event.exportedSpan.id,
+            waitingFor: e.waitingFor,
+            attempts: e.attempts
+          }))
+        });
+      }
+      for (const spanId of traceData.activeSpanIds) {
+        const span = traceData.getSpan({ spanId });
+        if (span) {
+          await this._abortSpan({ span, traceData, reason });
+        }
+      }
+    }
+    this.#traceMap.clear();
+    await this._postShutdown();
+    await super.shutdown();
+  }
+};
+
+// src/exporters/span-formatters.ts
+function chainFormatters(formatters) {
+  return async (span) => {
+    let currentSpan = span;
+    for (const formatter of formatters) {
+      currentSpan = await formatter(currentSpan);
+    }
+    return currentSpan;
+  };
+}
 var CloudExporter = class extends BaseExporter {
   name = "mastra-cloud-observability-exporter";
-  config;
+  cloudConfig;
   buffer;
   flushTimer = null;
   constructor(config = {}) {
@@ -4239,7 +5224,7 @@ var CloudExporter = class extends BaseExporter {
       this.setDisabled("MASTRA_CLOUD_ACCESS_TOKEN environment variable not set.");
     }
     const endpoint = config.endpoint ?? process.env.MASTRA_CLOUD_TRACES_ENDPOINT ?? "https://api.mastra.ai/ai/spans/publish";
-    this.config = {
+    this.cloudConfig = {
       logger: this.logger,
       logLevel: config.logLevel ?? logger.LogLevel.INFO,
       maxBatchSize: config.maxBatchSize ?? 1e3,
@@ -4297,12 +5282,12 @@ var CloudExporter = class extends BaseExporter {
     return spanRecord;
   }
   shouldFlush() {
-    if (this.buffer.totalSize >= this.config.maxBatchSize) {
+    if (this.buffer.totalSize >= this.cloudConfig.maxBatchSize) {
       return true;
     }
     if (this.buffer.firstEventTime && this.buffer.totalSize > 0) {
       const elapsed = Date.now() - this.buffer.firstEventTime.getTime();
-      if (elapsed >= this.config.maxBatchWaitMs) {
+      if (elapsed >= this.cloudConfig.maxBatchWaitMs) {
         return true;
       }
     }
@@ -4325,7 +5310,7 @@ var CloudExporter = class extends BaseExporter {
         this.logger.trackException(mastraError);
         this.logger.error("Scheduled flush failed", mastraError);
       });
-    }, this.config.maxBatchWaitMs);
+    }, this.cloudConfig.maxBatchWaitMs);
   }
   async flush() {
     if (this.flushTimer) {
@@ -4337,7 +5322,7 @@ var CloudExporter = class extends BaseExporter {
     }
     const startTime = Date.now();
     const spansCopy = [...this.buffer.spans];
-    const flushReason = this.buffer.totalSize >= this.config.maxBatchSize ? "size" : "time";
+    const flushReason = this.buffer.totalSize >= this.cloudConfig.maxBatchSize ? "size" : "time";
     this.resetBuffer();
     try {
       await this.batchUpload(spansCopy);
@@ -4368,7 +5353,7 @@ var CloudExporter = class extends BaseExporter {
    */
   async batchUpload(spans) {
     const headers = {
-      Authorization: `Bearer ${this.config.accessToken}`,
+      Authorization: `Bearer ${this.cloudConfig.accessToken}`,
       "Content-Type": "application/json"
     };
     const options = {
@@ -4376,7 +5361,7 @@ var CloudExporter = class extends BaseExporter {
       headers,
       body: JSON.stringify({ spans })
     };
-    await utils.fetchWithRetry(this.config.endpoint, options, this.config.maxRetries);
+    await utils.fetchWithRetry(this.cloudConfig.endpoint, options, this.cloudConfig.maxRetries);
   }
   resetBuffer() {
     this.buffer.spans = [];
@@ -5480,10 +6465,39 @@ var ModelSpanTracker = class {
             case "step-finish":
               this.#endStepSpan(chunk.payload);
               break;
+            // Infrastructure chunks - skip creating spans for these
+            // They are either redundant, metadata-only, or error/control flow
             case "raw":
-            // Skip raw chunks as they're redundant
+            // Redundant raw data
             case "start":
+            // Stream start marker
             case "finish":
+            // Stream finish marker (step-finish already captures this)
+            case "response-metadata":
+            // Response metadata (not semantic content)
+            case "source":
+            // Source references (metadata)
+            case "file":
+            // Binary file data (too large/not semantic)
+            case "error":
+            // Error handling
+            case "abort":
+            // Abort signal
+            case "tripwire":
+            // Processor rejection
+            case "watch":
+            // Internal watch event
+            case "tool-error":
+            // Tool error handling
+            case "tool-call-approval":
+            // Approval request (not content)
+            case "tool-call-suspended":
+            // Suspension (not content)
+            case "reasoning-signature":
+            // Signature metadata
+            case "redacted-reasoning":
+            // Redacted content metadata
+            case "step-output":
               break;
             case "tool-output":
               this.#handleToolOutputChunk(chunk);
@@ -5498,20 +6512,6 @@ var ModelSpanTracker = class {
               this.#createEventSpan(chunk.type, cleanPayload);
               break;
             }
-            // Default: auto-create event span for all other chunk types
-            default: {
-              let outputPayload = chunk.payload;
-              if (outputPayload && typeof outputPayload === "object" && "data" in outputPayload) {
-                const typedPayload = outputPayload;
-                outputPayload = { ...typedPayload };
-                if (typedPayload.data) {
-                  outputPayload.size = typeof typedPayload.data === "string" ? typedPayload.data.length : typedPayload.data instanceof Uint8Array ? typedPayload.data.length : void 0;
-                  delete outputPayload.data;
-                }
-              }
-              this.#createEventSpan(chunk.type, outputPayload);
-              break;
-            }
           }
         }
       })
@@ -5712,15 +6712,15 @@ var BaseSpan = class {
     this.attributes = deepClean(options.attributes, this.deepCleanOptions) || {};
     this.metadata = deepClean(options.metadata, this.deepCleanOptions);
     this.parent = options.parent;
-    this.startTime = /* @__PURE__ */ new Date();
+    this.startTime = options.startTime ?? /* @__PURE__ */ new Date();
     this.observabilityInstance = observabilityInstance;
     this.isEvent = options.isEvent ?? false;
     this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
     this.traceState = options.traceState;
     this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
-    this.entityType = options.entityType;
-    this.entityId = options.entityId;
-    this.entityName = options.entityName;
+    this.entityType = options.entityType ?? options.parent?.entityType;
+    this.entityId = options.entityId ?? options.parent?.entityId;
+    this.entityName = options.entityName ?? options.parent?.entityName;
     if (this.isEvent) {
       this.output = deepClean(options.output, this.deepCleanOptions);
     } else {
@@ -5769,6 +6769,8 @@ var BaseSpan = class {
   }
   /** Returns a lightweight span ready for export */
   exportSpan(includeInternalSpans) {
+    const hideInput = this.traceState?.hideInput ?? false;
+    const hideOutput = this.traceState?.hideOutput ?? false;
     return {
       id: this.id,
       traceId: this.traceId,
@@ -5781,8 +6783,8 @@ var BaseSpan = class {
       metadata: this.metadata,
       startTime: this.startTime,
       endTime: this.endTime,
-      input: this.input,
-      output: this.output,
+      input: hideInput ? void 0 : this.input,
+      output: hideOutput ? void 0 : this.output,
       errorInfo: this.errorInfo,
       isEvent: this.isEvent,
       isRootSpan: this.isRootSpan,
@@ -5822,6 +6824,14 @@ var DefaultSpan = class extends BaseSpan {
   traceId;
   constructor(options, observabilityInstance) {
     super(options, observabilityInstance);
+    if (options.spanId && options.traceId) {
+      this.id = options.spanId;
+      this.traceId = options.traceId;
+      if (options.parentSpanId) {
+        this.parentSpanId = options.parentSpanId;
+      }
+      return;
+    }
     const bridge = observabilityInstance.getBridge();
     if (bridge && !this.isInternal) {
       const bridgeIds = bridge.createSpan(options);
@@ -6066,8 +7076,12 @@ var BaseObservabilityInstance = class extends base.MastraBase {
     const mergedMetadata = metadata || tracingMetadata ? { ...metadata, ...tracingMetadata } : void 0;
     const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, mergedMetadata, traceState);
     const tags = !options.parent ? tracingOptions?.tags : void 0;
+    const traceId = !options.parent ? options.traceId ?? tracingOptions?.traceId : options.traceId;
+    const parentSpanId = !options.parent ? options.parentSpanId ?? tracingOptions?.parentSpanId : options.parentSpanId;
     const span = this.createSpan({
       ...rest,
+      traceId,
+      parentSpanId,
       metadata: enrichedMetadata,
       traceState,
       tags
@@ -6080,6 +7094,37 @@ var BaseObservabilityInstance = class extends base.MastraBase {
     }
     return span;
   }
+  /**
+   * Rebuild a span from exported data for lifecycle operations.
+   * Used by durable execution engines (e.g., Inngest) to end/update spans
+   * that were created in a previous durable operation.
+   *
+   * The rebuilt span:
+   * - Does NOT emit SPAN_STARTED (assumes original span already did)
+   * - Can have end(), update(), error() called on it
+   * - Will emit SPAN_ENDED or SPAN_UPDATED when those methods are called
+   *
+   * @param cached - The exported span data to rebuild from
+   * @returns A span that can have lifecycle methods called on it
+   */
+  rebuildSpan(cached) {
+    const span = this.createSpan({
+      name: cached.name,
+      type: cached.type,
+      traceId: cached.traceId,
+      spanId: cached.id,
+      parentSpanId: cached.parentSpanId,
+      startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
+      input: cached.input,
+      attributes: cached.attributes,
+      metadata: cached.metadata,
+      entityType: cached.entityType,
+      entityId: cached.entityId,
+      entityName: cached.entityName
+    });
+    this.wireSpanLifecycle(span);
+    return span;
+  }
   // ============================================================================
   // Configuration Management
   // ============================================================================
@@ -6182,11 +7227,15 @@ var BaseObservabilityInstance = class extends base.MastraBase {
     const configuredKeys = this.config.requestContextKeys ?? [];
     const additionalKeys = tracingOptions?.requestContextKeys ?? [];
     const allKeys = [...configuredKeys, ...additionalKeys];
-    if (allKeys.length === 0) {
+    const hideInput = tracingOptions?.hideInput;
+    const hideOutput = tracingOptions?.hideOutput;
+    if (allKeys.length === 0 && !hideInput && !hideOutput) {
       return void 0;
     }
     return {
-      requestContextKeys: allKeys
+      requestContextKeys: allKeys,
+      ...hideInput !== void 0 && { hideInput },
+      ...hideOutput !== void 0 && { hideOutput }
     };
   }
   /**
@@ -6732,7 +7781,10 @@ exports.Observability = Observability;
 exports.SamplingStrategyType = SamplingStrategyType;
 exports.SensitiveDataFilter = SensitiveDataFilter;
 exports.TestExporter = TestExporter;
+exports.TraceData = TraceData;
+exports.TrackingExporter = TrackingExporter;
 exports.buildTracingOptions = buildTracingOptions;
+exports.chainFormatters = chainFormatters;
 exports.deepClean = deepClean;
 exports.getExternalParentId = getExternalParentId;
 exports.mergeSerializationOptions = mergeSerializationOptions;