@mastra/observability 1.21.1-alpha.2 → 1.22.0

package/dist/index.js

@@ -2,6 +2,7 @@ import { createRequire } from "module";
 import { MastraBase } from "@mastra/core/base";
 import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
 import { ConsoleLogger, LogLevel, RegisteredLogger } from "@mastra/core/logger";
+import * as coreObservability from "@mastra/core/observability";
 import { DEFAULT_BLOCKED_LABELS, EntityType, InternalSpans, SpanType, TracingEventType, noOpLoggerContext } from "@mastra/core/observability";
 import { fetchWithRetry, getNestedValue, setNestedValue } from "@mastra/core/utils";
 import { z } from "zod/v4";
@@ -2712,8 +2713,20 @@ var ModelSpanTracker = class {
 	* feature flag, or when called without an active step span. Auto-invoked from
 	* chunk handlers as a safety net; explicit callers get the most accurate
 	* start time.
+	*
+	* `payload.modelId` / `payload.provider` name the model the caller is about
+	* to invoke and are recorded on MODEL_STEP, also when a fallback attempt
+	* reuses the step's open inference span. The safety net never records them:
+	* a replayed step-start chunk names the model of the original call.
 	*/
 	startInference(payload) {
+		if (payload?.modelId && payload.provider) this.#currentStepSpan?.update({ attributes: {
+			model: payload.modelId,
+			provider: payload.provider
+		} });
+		this.#openInference(payload);
+	}
+	#openInference(payload) {
 		if (!supportsModelInference()) return;
 		if (!this.#currentStepSpan || this.#currentInferenceSpan) return;
 		this.#inferenceCompletionStartTime = void 0;
@@ -2838,7 +2851,7 @@ var ModelSpanTracker = class {
 	*/
 	#ensureStepAndInference() {
 		if (!this.#currentStepSpan) this.startStep();
-		if (!this.#currentInferenceSpan) this.startInference();
+		if (!this.#currentInferenceSpan) this.#openInference();
 	}
 	/**
 	* Create a new chunk span (for multi-part chunks like text-start/delta/end)
@@ -3038,7 +3051,7 @@ var ModelSpanTracker = class {
 					this.#inModelResponse = true;
 					if (this.#currentStepSpan) this.updateStep(chunk.payload);
 					else this.startStep(chunk.payload);
-					if (!this.#currentInferenceSpan) this.startInference(chunk.payload);
+					if (!this.#currentInferenceSpan) this.#openInference(chunk.payload);
 					break;
 				case "step-finish":
 					this.#inModelResponse = false;
@@ -3244,6 +3257,19 @@ function getExternalParentId(options) {
 	if (parent.isInternal || parent.isExcluded) return parent.getParentSpanId(false);
 	return parent.id;
 }
+/**
+* Vercel AI Gateway reports which provider served a step, and any provider
+* attempts that failed first, only in `providerMetadata.gateway.routing`.
+* Read it defensively so a throwing getter never skips the strip below.
+*/
+function readGatewayRouting(providerMetadata) {
+	try {
+		const gateway = isPlainRecord(providerMetadata) ? providerMetadata.gateway : void 0;
+		return isPlainRecord(gateway) ? gateway.routing : void 0;
+	} catch {
+		return;
+	}
+}
 var BaseSpan = class BaseSpan {
 	name;
 	type;
@@ -3273,6 +3299,11 @@ var BaseSpan = class BaseSpan {
 	parentSpanId;
 	/** Parent from an external tracing system (ambient OTel / dd-trace span) that Mastra did not create; carried for external correlation, not part of Mastra's own parent/child linkage */
 	externalParentSpanId;
+	/** Native span fields (see `SpanData`); carried as supplied and exported only when set */
+	resumedFromSpanId;
+	outcome;
+	stopReason;
+	stopReasonContext;
 	/** Deep clean options for serialization */
 	deepCleanOptions;
 	/**
@@ -3339,6 +3370,10 @@ var BaseSpan = class BaseSpan {
 		}
 		this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
 		this.links = validLinks(options.links);
+		this.resumedFromSpanId = options.resumedFromSpanId;
+		this.outcome = options.outcome;
+		this.stopReason = options.stopReason;
+		this.stopReasonContext = deepClean(options.stopReasonContext, this.deepCleanOptions);
 		const entityParent = this.getParentSpan(false);
 		this.entityType = options.entityType ?? entityParent?.entityType;
 		this.entityId = options.entityId ?? entityParent?.entityId;
@@ -3371,8 +3406,10 @@ var BaseSpan = class BaseSpan {
 		if (this.type !== SpanType.MODEL_STEP) return value;
 		try {
 			const prepared = { ...value };
+			const gatewayRouting = readGatewayRouting(prepared.providerMetadata);
 			delete prepared.providerMetadata;
 			delete prepared.experimental_providerMetadata;
+			if (gatewayRouting !== void 0) prepared.providerMetadata = { gateway: { routing: gatewayRouting } };
 			return prepared;
 		} catch {
 			return value;
@@ -3605,6 +3642,10 @@ var BaseSpan = class BaseSpan {
 			...definedLineage(correlation),
 			parentSpanId: this.getParentSpanId(includeInternalSpans),
 			externalParentSpanId: this.getExportedExternalParentSpanId(includeInternalSpans),
+			...this.resumedFromSpanId !== void 0 ? { resumedFromSpanId: this.resumedFromSpanId } : {},
+			...this.outcome !== void 0 ? { outcome: this.outcome } : {},
+			...this.stopReason !== void 0 ? { stopReason: this.stopReason } : {},
+			...this.stopReasonContext !== void 0 ? { stopReasonContext: this.stopReasonContext } : {},
 			...nestedUnderParent ? { nestedUnderParent: true } : {},
 			...links ? { links } : {},
 			...this.isRootSpan && !nestedUnderParent && this.tags?.length ? { tags: this.tags } : {}
@@ -3833,7 +3874,7 @@ function isValidSpanId(spanId) {
 function getOrCreateTraceId(options) {
 	if (options.traceId) {
 		if (isValidTraceId(options.traceId)) return options.traceId;
-		else console.error(`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`);
+		console.error(`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`);
 	}
 	return generateTraceId();
 }
@@ -4029,6 +4070,8 @@ var EventBuffer = class {
 	#creates = [];
 	#updates = [];
 	#allCreatedSpans = /* @__PURE__ */ new Set();
+	/** Spans whose SPAN_STARTED this buffer has seen but whose create isn't confirmed yet */
+	#pendingCreateSpans = /* @__PURE__ */ new Set();
 	#firstEventTime;
 	#storageStrategy;
 	#maxRetries;
@@ -4079,8 +4122,11 @@ var EventBuffer = class {
 			case TracingEventType.SPAN_STARTED:
 				switch (this.#storageStrategy) {
 					case "realtime":
-					case "event-sourced":
-					case "batch-with-updates": this.pushCreate(event);
+					case "batch-with-updates":
+						if (!event.exportedSpan.isEvent) this.#pendingCreateSpans.add(this.exportedSpanKey(event.exportedSpan));
+						this.pushCreate(event);
+						break;
+					case "event-sourced": this.pushCreate(event);
 				}
 				break;
 			case TracingEventType.SPAN_UPDATED:
@@ -4149,23 +4195,40 @@ var EventBuffer = class {
 	buildSpanKey(span) {
 		return `${span.traceId}:${span.spanId}`;
 	}
+	exportedSpanKey(span) {
+		return this.buildSpanKey({
+			traceId: span.traceId,
+			spanId: span.id
+		});
+	}
 	/** Track successfully created spans so updates can verify span existence before flushing. */
 	addCreatedSpans(args) {
 		if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") return;
-		for (const createRecord of args.records) if (!createRecord.isEvent) this.#allCreatedSpans.add(this.buildSpanKey(createRecord));
+		for (const createRecord of args.records) if (!createRecord.isEvent) {
+			const key = this.buildSpanKey(createRecord);
+			this.#pendingCreateSpans.delete(key);
+			this.#allCreatedSpans.add(key);
+		}
+	}
+	/** Stop waiting on creates that were dropped, so their updates aren't held back for them. */
+	forgetDroppedCreates(events) {
+		for (const event of events) if (event.type === TracingEventType.SPAN_STARTED) this.#pendingCreateSpans.delete(this.exportedSpanKey(event.exportedSpan));
 	}
 	/** Check whether a span's create record has already been flushed to storage. */
 	spanExists(span) {
-		return this.#allCreatedSpans?.has(this.buildSpanKey({
-			traceId: span.traceId,
-			spanId: span.id
-		}));
+		return this.#allCreatedSpans.has(this.exportedSpanKey(span));
+	}
+	/** Check whether this buffer saw the span's SPAN_STARTED but its create hasn't been confirmed yet. */
+	isAwaitingCreate(span) {
+		return this.#pendingCreateSpans.has(this.exportedSpanKey(span));
 	}
 	/** Remove completed spans from tracking after their SPAN_ENDED updates are flushed. */
 	endFinishedSpans(args) {
 		if (this.#storageStrategy === "event-sourced" || this.#storageStrategy === "insert-only") return;
 		args.records.forEach((r) => {
-			this.#allCreatedSpans.delete(this.buildSpanKey(r));
+			const key = this.buildSpanKey(r);
+			this.#allCreatedSpans.delete(key);
+			this.#pendingCreateSpans.delete(key);
 		});
 	}
 };
@@ -4607,6 +4670,21 @@ var DefaultExporter = class extends BaseExporter {
 		this.logger.info("DefaultExporter shutdown complete");
 	}
 };
+// src/exporters/span-previews.ts
+const buildSpanPreviews = coreObservability.buildSpanPreviews;
+/** Older core versions retain their existing storage preview fallback. */
+function withSpanPreviews(span) {
+	if (!coreFeatures.has("observability-span-previews")) return span;
+	try {
+		const previews = buildSpanPreviews?.(span);
+		return previews ? {
+			...span,
+			...previews
+		} : span;
+	} catch {
+		return span;
+	}
+}
 // src/exporters/mastra-storage.ts
 /**
 * Resolves the final tracing storage strategy based on config and observability store hints
@@ -4810,6 +4888,7 @@ var MastraStorageExporter = class extends BaseExporter {
 				return { failed: true };
 			}
 			const dropped = this.#eventBuffer.reAddCreates(events);
+			this.#eventBuffer.forgetDroppedCreates(dropped);
 			const retryAttempt = this.retryAttempt(events);
 			this.logStorageFailure(signal, events, retryAttempt, error);
 			this.emitDrop(signal, "retry-exhausted", dropped.length, error);
@@ -4820,7 +4899,15 @@ var MastraStorageExporter = class extends BaseExporter {
 		}
 	}
 	/**
-	* Flush span update/end events, deferring any whose span hasn't been created yet.
+	* Flush span update/end events.
+	*
+	* When the store supports conditional updates, only spans this exporter started
+	* and hasn't confirmed the create for are deferred. Everything else, including
+	* spans another exporter instance started (such as a durable agent ending spans
+	* it rebuilt after a crash), goes straight to storage, and updates that didn't
+	* apply are counted as drops. Without conditional updates, updates for spans this
+	* exporter hasn't created are deferred until their create is flushed.
+	*
 	* When `isEnd` is true, successfully flushed spans are removed from tracking.
 	*/
 	async flushSpanUpdates(events, deferredUpdates, isEnd) {
@@ -4830,10 +4917,12 @@ var MastraStorageExporter = class extends BaseExporter {
 			this.emitDrop("tracing", "unsupported-storage", events.length);
 			return { failed: true };
 		}
+		const storage = this.#observabilityStorage;
+		const batchUpdateUnendedSpans = typeof storage.batchUpdateUnendedSpans === "function" ? storage.batchUpdateUnendedSpans.bind(storage) : void 0;
 		const partials = [];
 		for (const event of events) {
 			const span = event.exportedSpan;
-			if (this.#eventBuffer.spanExists(span)) partials.push({
+			if (batchUpdateUnendedSpans ? !this.#eventBuffer.isAwaitingCreate(span) : this.#eventBuffer.spanExists(span)) partials.push({
 				traceId: span.traceId,
 				spanId: span.id,
 				updates: buildUpdateSpanRecord(span)
@@ -4842,7 +4931,10 @@ var MastraStorageExporter = class extends BaseExporter {
 		}
 		if (partials.length === 0) return FLUSH_SUCCEEDED;
 		try {
-			await this.#observabilityStorage.batchUpdateSpans({ records: partials });
+			if (batchUpdateUnendedSpans) {
+				const { updated } = await batchUpdateUnendedSpans({ records: partials });
+				this.emitNotApplied(partials, updated);
+			} else await storage.batchUpdateSpans({ records: partials });
 			if (isEnd) this.#eventBuffer.endFinishedSpans({ records: partials });
 			return FLUSH_SUCCEEDED;
 		} catch (error) {
@@ -4866,6 +4958,27 @@ var MastraStorageExporter = class extends BaseExporter {
 		}
 	}
 	/**
+	* Reports conditional span updates that storage didn't apply: the span wasn't
+	* stored, or it had already ended. Ended spans are immutable, so these are dropped.
+	*/
+	emitNotApplied(partials, updated) {
+		const appliedCounts = /* @__PURE__ */ new Map();
+		for (const span of updated) {
+			const key = `${span.traceId}:${span.spanId}`;
+			appliedCounts.set(key, (appliedCounts.get(key) ?? 0) + 1);
+		}
+		let notApplied = 0;
+		for (const partial of partials) {
+			const key = `${partial.traceId}:${partial.spanId}`;
+			const remaining = appliedCounts.get(key) ?? 0;
+			if (remaining > 0) appliedCounts.set(key, remaining - 1);
+			else notApplied++;
+		}
+		if (notApplied === 0) return;
+		this.logger.debug("Span updates not applied: span not stored or already ended", { count: notApplied });
+		this.emitDrop("tracing", "span-not-updatable", notApplied);
+	}
+	/**
 	* Runs flushBuffer() and tracks its promise so flush() can wait for writes
 	* that a timer or size trigger started on their own.
 	*/
@@ -4975,7 +5088,10 @@ var MastraStorageExporter = class extends BaseExporter {
 			this.logger.debug("Cannot store traces. Observability storage is not initialized");
 			return;
 		}
-		this.#eventBuffer.addEvent(event);
+		this.#eventBuffer.addEvent({
+			...event,
+			exportedSpan: withSpanPreviews(event.exportedSpan)
+		});
 		await this.handleBatchedFlush();
 	}
 	/**
@@ -5227,6 +5343,10 @@ var BaseObservabilityInstance = class extends MastraBase {
 			spanId: cached.id,
 			parentSpanId: cached.parentSpanId,
 			externalParentSpanId: cached.externalParentSpanId,
+			resumedFromSpanId: cached.resumedFromSpanId,
+			outcome: cached.outcome,
+			stopReason: cached.stopReason,
+			stopReasonContext: cached.stopReasonContext,
 			links: cached.links,
 			startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
 			input: cached.input,
@@ -8079,7 +8199,7 @@ var MastraPlatformExporter = class extends BaseExporter {
 	}
 	formatSpan(span) {
 		return {
-			...span,
+			...withSpanPreviews(span),
 			spanId: span.id,
 			spanType: span.type,
 			startedAt: span.startTime,