@mastra/observability 1.16.4 → 1.16.6-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @mastra/observability
 
+## 1.16.6-alpha.0
+
+### Patch Changes
+
+- Fixed an issue where live workflow scores and feedback could be lost before their trace data reached observability storage. Mastra now logs a warning when it cannot save an annotation. ([#20795](https://github.com/mastra-ai/mastra/pull/20795))
+
+- Updated dependencies [[`e7109ee`](https://github.com/mastra-ai/mastra/commit/e7109ee6f731bacc79c885906f3c7dca8d8f013a), [`772c0c8`](https://github.com/mastra-ai/mastra/commit/772c0c897cec383258de2e6178147f8014767c7b), [`578bf2e`](https://github.com/mastra-ai/mastra/commit/578bf2e6a88e9d5b8bf502204e15a95dfbb679ae), [`06b2d87`](https://github.com/mastra-ai/mastra/commit/06b2d87e63bcdd0ed59215c6789692b9b12de376), [`ac01d63`](https://github.com/mastra-ai/mastra/commit/ac01d6355974aec73fdb8781449ed12bac582094), [`a810a05`](https://github.com/mastra-ai/mastra/commit/a810a058f62ad407cfc1701e0be36ae91145d7cf), [`f8da216`](https://github.com/mastra-ai/mastra/commit/f8da21633e7eb0e31c9ce0fc30567870d19416d3), [`6104347`](https://github.com/mastra-ai/mastra/commit/61043473ba6bfd0a25156824e853e13165562e6c), [`45bfb88`](https://github.com/mastra-ai/mastra/commit/45bfb88fd52f1dd3be20e2a38905777c96499c90), [`e3b9307`](https://github.com/mastra-ai/mastra/commit/e3b9307098daefbfae2a52ae2ef51bc9fc701190), [`d6834c5`](https://github.com/mastra-ai/mastra/commit/d6834c5a7866b16734d23900163c2414ed70d791), [`c52d346`](https://github.com/mastra-ai/mastra/commit/c52d3462ec831a5d95926ecd3d3373f5928ad2e5), [`0023e79`](https://github.com/mastra-ai/mastra/commit/0023e7919431078280abd11c89d1edeae35fcc69), [`c2ad51e`](https://github.com/mastra-ai/mastra/commit/c2ad51e2467f901eecba8c9f4a45e22a50bd7c18), [`3dc97ea`](https://github.com/mastra-ai/mastra/commit/3dc97ea415fad353b48a13095fad1835933cc12a), [`3d01cd3`](https://github.com/mastra-ai/mastra/commit/3d01cd387321b6f9c5cac31d487c84bf51b19c78), [`7bf3086`](https://github.com/mastra-ai/mastra/commit/7bf308663f0115ca74ad20554ade740f06640859), [`a8dd139`](https://github.com/mastra-ai/mastra/commit/a8dd1391a9fe9a6632c25809ef236980afa9a020), [`e5786be`](https://github.com/mastra-ai/mastra/commit/e5786be02bb903073082bd9d6da880ebaacc343f), [`2093fbd`](https://github.com/mastra-ai/mastra/commit/2093fbd53bb744bae19ec89f6d73db9a66fbe8a7), [`e7a5da4`](https://github.com/mastra-ai/mastra/commit/e7a5da4ef8e4dd452d2f232961b4e682a85ffe43), [`7b4393d`](https://github.com/mastra-ai/mastra/commit/7b4393d557411fdcf07b0e30e5acaf7cc85154ae)]:
+  - @mastra/core@1.58.0-alpha.1
+
+## 1.16.5
+
+### Patch Changes
+
+- Fixed token metrics losing model labels and cost data when providers return blank response model IDs. ([#20742](https://github.com/mastra-ai/mastra/pull/20742))
+
+- Updated dependencies [[`8d2399b`](https://github.com/mastra-ai/mastra/commit/8d2399b638f8e0945cf2cda0187dbea8dcf0b784), [`c8002da`](https://github.com/mastra-ai/mastra/commit/c8002da7775c468e2965b6ff5f82045450fa8cb9), [`92be47f`](https://github.com/mastra-ai/mastra/commit/92be47fbd26ffccec0e2131ef7c1d9e70dd5ef4a), [`89200ba`](https://github.com/mastra-ai/mastra/commit/89200bafa05444bb7949b363ce7b743e29867561), [`c950138`](https://github.com/mastra-ai/mastra/commit/c950138e72e4f317a40187e3800588731ab790ce), [`810c7e7`](https://github.com/mastra-ai/mastra/commit/810c7e74929989d8b8b5db52cd3af22cd0998af4), [`063c8b2`](https://github.com/mastra-ai/mastra/commit/063c8b2eb14e4e5ca021779bc33e8c3c031c8604), [`f9f9884`](https://github.com/mastra-ai/mastra/commit/f9f98848ee194dc71a787a709ec430b065cdc41b), [`e0904dc`](https://github.com/mastra-ai/mastra/commit/e0904dc538792e54e1806b70172e5900ac49bff4), [`9672fab`](https://github.com/mastra-ai/mastra/commit/9672fabfbcadb961a35c22a2d6722e077f7b24b9), [`f4e964c`](https://github.com/mastra-ai/mastra/commit/f4e964cad57057301d6bed5c55bcdd730175b941), [`1f7bbd7`](https://github.com/mastra-ai/mastra/commit/1f7bbd7785a8d230aad02454ecabeb4a0b2cc96f), [`e47ff36`](https://github.com/mastra-ai/mastra/commit/e47ff36945720f4ee4caa09f6e83514d7d188608), [`64d6781`](https://github.com/mastra-ai/mastra/commit/64d67814bccddd314f7e09643243821e57cb87b6), [`fb9a6ac`](https://github.com/mastra-ai/mastra/commit/fb9a6ac11c9560518742ece60b49d6b062845fd3), [`aa2cec8`](https://github.com/mastra-ai/mastra/commit/aa2cec8501f634d51c2f3ebfb3dd3aa7af8d2ca2), [`c848e65`](https://github.com/mastra-ai/mastra/commit/c848e655a64ff10331a8ceafafe7f18e70a0f092), [`2adf8eb`](https://github.com/mastra-ai/mastra/commit/2adf8eb4a70ed2b6cff2dd39281496ea0e025fac), [`0494489`](https://github.com/mastra-ai/mastra/commit/049448906e4c3d2d615bbe865b073a0d890ddb7c), [`8d1aeb8`](https://github.com/mastra-ai/mastra/commit/8d1aeb8acf7c20c4bb8e4d8e4bdc6569c83ac561), [`8264611`](https://github.com/mastra-ai/mastra/commit/8264611510e421b818bc7395dc2ae4d9c2d518b2), [`d8fa243`](https://github.com/mastra-ai/mastra/commit/d8fa2430d21113e330c4e676ac65e1235cf44f81), [`44fc98b`](https://github.com/mastra-ai/mastra/commit/44fc98b9d1242aa87a3ab44bdce9e9f12c44d8c9), [`f933ba3`](https://github.com/mastra-ai/mastra/commit/f933ba32700e1d0bf143311c1a08f88300b840b6), [`83065bf`](https://github.com/mastra-ai/mastra/commit/83065bfee9e47c3c6f09132a9034501f6cfb69cf), [`0f2ef41`](https://github.com/mastra-ai/mastra/commit/0f2ef4118da022e4f30dac4e9856cc3a8c97671c), [`01b162f`](https://github.com/mastra-ai/mastra/commit/01b162fe435295881aa7ea55f1759407ad5175ad)]:
+  - @mastra/core@1.57.0
+
 ## 1.16.4
 
 ### Patch Changes

package/dist/default.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../src/default.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EACV,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,aAAa,EAEb,uBAAuB,EAEvB,qBAAqB,EACrB,aAAa,EACb,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAA+B,2BAA2B,EAAE,MAAM,UAAU,CAAC;AAuBzF;;;GAGG;AACH,qBAAa,aAAc,SAAQ,UAAW,YAAW,uBAAuB;;gBAKlE,MAAM,EAAE,2BAA2B;IAgJ/C,yFAAyF;IACzF,gBAAgB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsCnD,sFAAsF;IACtF,SAAS,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,GAAG,IAAI;IAOnD,0FAA0F;IAC1F,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,SAAS;IAIhF,gBAAgB,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA6B1E,QAAQ,CAAC,IAAI,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,KAAK,EAAE,UAAU,CAAC;KACnB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsCX,WAAW,CAAC,IAAI,EAAE;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,QAAQ,EAAE,aAAa,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsCjB,iFAAiF;IACjF,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,qBAAqB,EAAE,SAAS,UAAQ,GAAG,IAAI;IAWxF,yCAAyC;IACzC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAI5D,8CAA8C;IAC9C,kBAAkB,IAAI,qBAAqB,GAAG,SAAS;IAIvD,mDAAmD;IACnD,aAAa,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC;IAI3D,gFAAgF;IAChF,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIzC,mEAAmE;IACnE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC,qEAAqE;IACrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAIjD,8DAA8D;IAC9D,KAAK,IAAI,IAAI;IAIb,4DAA4D;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,qEAAqE;IAC/D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;;;OAQG;IACH,2BAA2B,IAAI,wBAAwB,GAAG,SAAS;CA2DpE"}
+{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../src/default.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EACV,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,aAAa,EAEb,uBAAuB,EAEvB,qBAAqB,EACrB,aAAa,EACb,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAA+B,2BAA2B,EAAE,MAAM,UAAU,CAAC;AAkCzF;;;GAGG;AACH,qBAAa,aAAc,SAAQ,UAAW,YAAW,uBAAuB;;gBAKlE,MAAM,EAAE,2BAA2B;IAgJ/C,yFAAyF;IACzF,gBAAgB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsCnD,sFAAsF;IACtF,SAAS,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,GAAG,IAAI;IAOnD,0FAA0F;IAC1F,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,SAAS;IAIhF,gBAAgB,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA6B1E,QAAQ,CAAC,IAAI,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,KAAK,EAAE,UAAU,CAAC;KACnB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCX,WAAW,CAAC,IAAI,EAAE;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,QAAQ,EAAE,aAAa,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCjB,iFAAiF;IACjF,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,qBAAqB,EAAE,SAAS,UAAQ,GAAG,IAAI;IAWxF,yCAAyC;IACzC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAI5D,8CAA8C;IAC9C,kBAAkB,IAAI,qBAAqB,GAAG,SAAS;IAIvD,mDAAmD;IACnD,aAAa,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC;IAI3D,gFAAgF;IAChF,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIzC,mEAAmE;IACnE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC,qEAAqE;IACrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAIjD,8DAA8D;IAC9D,KAAK,IAAI,IAAI;IAIb,4DAA4D;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,qEAAqE;IAC/D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;;;OAQG;IACH,2BAA2B,IAAI,wBAAwB,GAAG,SAAS;CAmFpE"}

package/dist/index.cjs

@@ -954,6 +954,12 @@ function cloneCostContext(costContext) {
 	};
 }
 //#endregion
+//#region src/model-id.ts
+/** Return the first model id that contains non-whitespace characters. */
+function resolveModelId(...candidates) {
+	return candidates.find((candidate) => candidate?.trim());
+}
+//#endregion
 //#region src/metrics/pricing-model.ts
 var PricingTier = class {
 	index;
@@ -1530,7 +1536,7 @@ function emitUsageMetrics(attrs, usage, metrics) {
 	if (providedCostContext) metricCosts = providedCostContext;
 	else try {
 		const provider = attrs.provider;
-		const model = attrs.responseModel ?? attrs.model;
+		const model = resolveModelId(attrs.responseModel, attrs.model);
 		if (provider && model) {
 			metricCosts = estimateCosts({
 				provider,
@@ -1567,7 +1573,7 @@ function getProvidedCostContext(attrs, usage) {
 	if (typeof costContext?.estimatedCost !== "number") return;
 	const carrierMetric = usage.inputTokens !== void 0 ? TokenMetrics.TOTAL_INPUT : TokenMetrics.TOTAL_OUTPUT;
 	const provider = costContext.provider ?? attrs.provider;
-	const model = costContext.model ?? attrs.responseModel ?? attrs.model;
+	const model = resolveModelId(costContext.model, attrs.responseModel, attrs.model);
 	const contexts = /* @__PURE__ */ new Map();
 	for (const sample of getTokenMetricSamples(usage)) contexts.set(sample.name, {
 		provider,
@@ -2999,7 +3005,7 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
 		});
 		if (rest.type === _mastra_core_observability.SpanType.MODEL_GENERATION && this.config.excludeSpanTypes?.includes(_mastra_core_observability.SpanType.MODEL_GENERATION)) {
 			const attrs = rest.attributes;
-			const model = attrs?.responseModel ?? attrs?.model;
+			const model = resolveModelId(attrs?.responseModel, attrs?.model);
 			if (attrs?.provider || model) this.#excludedModelMeta.set(span, {
 				provider: attrs?.provider,
 				model
@@ -3402,7 +3408,7 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
 			ancestor,
 			usage,
 			provider: endAttrs?.provider ?? liveAttrs?.provider,
-			model: endAttrs?.responseModel ?? endAttrs?.model ?? liveAttrs?.responseModel ?? liveAttrs?.model
+			model: resolveModelId(endAttrs?.responseModel, endAttrs?.model, liveAttrs?.responseModel, liveAttrs?.model)
 		};
 	}
 	/**
@@ -3424,7 +3430,7 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
 		return {
 			usage,
 			provider: endAttrs?.provider ?? liveAttrs?.provider ?? stashed?.provider,
-			model: endAttrs?.responseModel ?? endAttrs?.model ?? liveAttrs?.responseModel ?? liveAttrs?.model ?? stashed?.model
+			model: resolveModelId(endAttrs?.responseModel, endAttrs?.model, liveAttrs?.responseModel, liveAttrs?.model, stashed?.model)
 		};
 	}
 	/**
@@ -8715,6 +8721,23 @@ function isInstance(obj) {
 	return obj instanceof BaseObservabilityInstance;
 }
 /**
+* Delays (ms) between attempts to rehydrate a trace from storage when an
+* annotation (score/feedback) targets a span that has not been flushed by
+* the configured exporters yet. Exporters buffer and flush asynchronously:
+* by default they flush at 1000 spans or after a 5000ms wait
+* (`maxBatchWaitMs`), so an annotation emitted right after a span ends can
+* race the flush and be silently dropped. The schedule below sums to
+* ~5.85s, covering the default flush interval with margin.
+*/
+const RECORDED_TRACE_LOOKUP_RETRY_DELAYS_MS = [
+	100,
+	250,
+	500,
+	1e3,
+	2e3,
+	2e3
+];
+/**
 * Top-level observability entrypoint. Manages a registry of ObservabilityInstance
 * configurations and provides instance selection via config selectors.
 */
@@ -8871,14 +8894,15 @@ var Observability = class extends _mastra_core_base.MastraBase {
 			return;
 		}
 		if (!args.traceId) return;
-		const trace = await this.#getStoredTrace(args.traceId);
-		if (!trace) return;
-		const event = buildRecordedScoreEventFromTrace({
+		const event = await this.#buildRecordedEventWithRetry(args.traceId, (trace) => buildRecordedScoreEventFromTrace({
 			trace,
 			spanId: args.spanId,
 			score: args.score
-		});
-		if (!event) return;
+		}));
+		if (!event) {
+			this.logger?.warn(`Score event was dropped because the target trace/span was not found in observability storage (traceId: ${args.traceId}, spanId: ${args.spanId})`);
+			return;
+		}
 		await this.#emitRecordedEvent(event);
 	}
 	async addFeedback(args) {
@@ -8894,14 +8918,15 @@ var Observability = class extends _mastra_core_base.MastraBase {
 			return;
 		}
 		if (!args.traceId) return;
-		const trace = await this.#getStoredTrace(args.traceId);
-		if (!trace) return;
-		const event = buildRecordedFeedbackEventFromTrace({
+		const event = await this.#buildRecordedEventWithRetry(args.traceId, (trace) => buildRecordedFeedbackEventFromTrace({
 			trace,
 			spanId: args.spanId,
 			feedback: args.feedback
-		});
-		if (!event) return;
+		}));
+		if (!event) {
+			this.logger?.warn(`Feedback event was dropped because the target trace/span was not found in observability storage (traceId: ${args.traceId}, spanId: ${args.spanId})`);
+			return;
+		}
 		await this.#emitRecordedEvent(event);
 	}
 	/** Register a named observability instance, optionally marking it as default. */
@@ -8966,6 +8991,20 @@ var Observability = class extends _mastra_core_base.MastraBase {
 		if (!storage) return null;
 		return await storage.getStore("observability") ?? null;
 	}
+	/**
+	* Build a recorded score/feedback event from storage, retrying briefly to
+	* ride out the async exporter flush: annotations emitted right after a
+	* span ends can otherwise race the flush and be silently dropped.
+	*/
+	async #buildRecordedEventWithRetry(traceId, build) {
+		for (const delayMs of [0, ...RECORDED_TRACE_LOOKUP_RETRY_DELAYS_MS]) {
+			if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
+			const trace = await this.#getStoredTrace(traceId);
+			const event = trace ? build(trace) : null;
+			if (event) return event;
+		}
+		return null;
+	}
 	async #getStoredTrace(traceId) {
 		const observabilityStorage = await this.#getObservabilityStorage();
 		if (!observabilityStorage) return null;