@mastra/observability 1.16.5 → 1.16.6-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @mastra/observability
 
+## 1.16.6-alpha.0
+
+### Patch Changes
+
+- Fixed an issue where live workflow scores and feedback could be lost before their trace data reached observability storage. Mastra now logs a warning when it cannot save an annotation. ([#20795](https://github.com/mastra-ai/mastra/pull/20795))
+
+- Updated dependencies [[`e7109ee`](https://github.com/mastra-ai/mastra/commit/e7109ee6f731bacc79c885906f3c7dca8d8f013a), [`772c0c8`](https://github.com/mastra-ai/mastra/commit/772c0c897cec383258de2e6178147f8014767c7b), [`578bf2e`](https://github.com/mastra-ai/mastra/commit/578bf2e6a88e9d5b8bf502204e15a95dfbb679ae), [`06b2d87`](https://github.com/mastra-ai/mastra/commit/06b2d87e63bcdd0ed59215c6789692b9b12de376), [`ac01d63`](https://github.com/mastra-ai/mastra/commit/ac01d6355974aec73fdb8781449ed12bac582094), [`a810a05`](https://github.com/mastra-ai/mastra/commit/a810a058f62ad407cfc1701e0be36ae91145d7cf), [`f8da216`](https://github.com/mastra-ai/mastra/commit/f8da21633e7eb0e31c9ce0fc30567870d19416d3), [`6104347`](https://github.com/mastra-ai/mastra/commit/61043473ba6bfd0a25156824e853e13165562e6c), [`45bfb88`](https://github.com/mastra-ai/mastra/commit/45bfb88fd52f1dd3be20e2a38905777c96499c90), [`e3b9307`](https://github.com/mastra-ai/mastra/commit/e3b9307098daefbfae2a52ae2ef51bc9fc701190), [`d6834c5`](https://github.com/mastra-ai/mastra/commit/d6834c5a7866b16734d23900163c2414ed70d791), [`c52d346`](https://github.com/mastra-ai/mastra/commit/c52d3462ec831a5d95926ecd3d3373f5928ad2e5), [`0023e79`](https://github.com/mastra-ai/mastra/commit/0023e7919431078280abd11c89d1edeae35fcc69), [`c2ad51e`](https://github.com/mastra-ai/mastra/commit/c2ad51e2467f901eecba8c9f4a45e22a50bd7c18), [`3dc97ea`](https://github.com/mastra-ai/mastra/commit/3dc97ea415fad353b48a13095fad1835933cc12a), [`3d01cd3`](https://github.com/mastra-ai/mastra/commit/3d01cd387321b6f9c5cac31d487c84bf51b19c78), [`7bf3086`](https://github.com/mastra-ai/mastra/commit/7bf308663f0115ca74ad20554ade740f06640859), [`a8dd139`](https://github.com/mastra-ai/mastra/commit/a8dd1391a9fe9a6632c25809ef236980afa9a020), [`e5786be`](https://github.com/mastra-ai/mastra/commit/e5786be02bb903073082bd9d6da880ebaacc343f), [`2093fbd`](https://github.com/mastra-ai/mastra/commit/2093fbd53bb744bae19ec89f6d73db9a66fbe8a7), [`e7a5da4`](https://github.com/mastra-ai/mastra/commit/e7a5da4ef8e4dd452d2f232961b4e682a85ffe43), [`7b4393d`](https://github.com/mastra-ai/mastra/commit/7b4393d557411fdcf07b0e30e5acaf7cc85154ae)]:
+  - @mastra/core@1.58.0-alpha.1
+
 ## 1.16.5
 
 ### Patch Changes

package/dist/default.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../src/default.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EACV,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,aAAa,EAEb,uBAAuB,EAEvB,qBAAqB,EACrB,aAAa,EACb,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAA+B,2BAA2B,EAAE,MAAM,UAAU,CAAC;AAuBzF;;;GAGG;AACH,qBAAa,aAAc,SAAQ,UAAW,YAAW,uBAAuB;;gBAKlE,MAAM,EAAE,2BAA2B;IAgJ/C,yFAAyF;IACzF,gBAAgB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsCnD,sFAAsF;IACtF,SAAS,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,GAAG,IAAI;IAOnD,0FAA0F;IAC1F,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,SAAS;IAIhF,gBAAgB,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA6B1E,QAAQ,CAAC,IAAI,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,KAAK,EAAE,UAAU,CAAC;KACnB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsCX,WAAW,CAAC,IAAI,EAAE;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,QAAQ,EAAE,aAAa,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsCjB,iFAAiF;IACjF,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,qBAAqB,EAAE,SAAS,UAAQ,GAAG,IAAI;IAWxF,yCAAyC;IACzC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAI5D,8CAA8C;IAC9C,kBAAkB,IAAI,qBAAqB,GAAG,SAAS;IAIvD,mDAAmD;IACnD,aAAa,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC;IAI3D,gFAAgF;IAChF,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIzC,mEAAmE;IACnE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC,qEAAqE;IACrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAIjD,8DAA8D;IAC9D,KAAK,IAAI,IAAI;IAIb,4DAA4D;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,qEAAqE;IAC/D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;;;OAQG;IACH,2BAA2B,IAAI,wBAAwB,GAAG,SAAS;CA2DpE"}
+{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../src/default.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EACV,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,EACd,qBAAqB,EACrB,aAAa,EAEb,uBAAuB,EAEvB,qBAAqB,EACrB,aAAa,EACb,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAA+B,2BAA2B,EAAE,MAAM,UAAU,CAAC;AAkCzF;;;GAGG;AACH,qBAAa,aAAc,SAAQ,UAAW,YAAW,uBAAuB;;gBAKlE,MAAM,EAAE,2BAA2B;IAgJ/C,yFAAyF;IACzF,gBAAgB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsCnD,sFAAsF;IACtF,SAAS,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,GAAG,IAAI;IAOnD,0FAA0F;IAC1F,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,SAAS;IAIhF,gBAAgB,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA6B1E,QAAQ,CAAC,IAAI,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,KAAK,EAAE,UAAU,CAAC;KACnB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCX,WAAW,CAAC,IAAI,EAAE;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;QACxC,QAAQ,EAAE,aAAa,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCjB,iFAAiF;IACjF,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,qBAAqB,EAAE,SAAS,UAAQ,GAAG,IAAI;IAWxF,yCAAyC;IACzC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS;IAI5D,8CAA8C;IAC9C,kBAAkB,IAAI,qBAAqB,GAAG,SAAS;IAIvD,mDAAmD;IACnD,aAAa,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC;IAI3D,gFAAgF;IAChF,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIzC,mEAAmE;IACnE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC,qEAAqE;IACrE,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAIjD,8DAA8D;IAC9D,KAAK,IAAI,IAAI;IAIb,4DAA4D;IACtD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,qEAAqE;IAC/D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;;;OAQG;IACH,2BAA2B,IAAI,wBAAwB,GAAG,SAAS;CAmFpE"}

package/dist/index.cjs

@@ -8721,6 +8721,23 @@ function isInstance(obj) {
 	return obj instanceof BaseObservabilityInstance;
 }
 /**
+* Delays (ms) between attempts to rehydrate a trace from storage when an
+* annotation (score/feedback) targets a span that has not been flushed by
+* the configured exporters yet. Exporters buffer and flush asynchronously:
+* by default they flush at 1000 spans or after a 5000ms wait
+* (`maxBatchWaitMs`), so an annotation emitted right after a span ends can
+* race the flush and be silently dropped. The schedule below sums to
+* ~5.85s, covering the default flush interval with margin.
+*/
+const RECORDED_TRACE_LOOKUP_RETRY_DELAYS_MS = [
+	100,
+	250,
+	500,
+	1e3,
+	2e3,
+	2e3
+];
+/**
 * Top-level observability entrypoint. Manages a registry of ObservabilityInstance
 * configurations and provides instance selection via config selectors.
 */
@@ -8877,14 +8894,15 @@ var Observability = class extends _mastra_core_base.MastraBase {
 			return;
 		}
 		if (!args.traceId) return;
-		const trace = await this.#getStoredTrace(args.traceId);
-		if (!trace) return;
-		const event = buildRecordedScoreEventFromTrace({
+		const event = await this.#buildRecordedEventWithRetry(args.traceId, (trace) => buildRecordedScoreEventFromTrace({
 			trace,
 			spanId: args.spanId,
 			score: args.score
-		});
-		if (!event) return;
+		}));
+		if (!event) {
+			this.logger?.warn(`Score event was dropped because the target trace/span was not found in observability storage (traceId: ${args.traceId}, spanId: ${args.spanId})`);
+			return;
+		}
 		await this.#emitRecordedEvent(event);
 	}
 	async addFeedback(args) {
@@ -8900,14 +8918,15 @@ var Observability = class extends _mastra_core_base.MastraBase {
 			return;
 		}
 		if (!args.traceId) return;
-		const trace = await this.#getStoredTrace(args.traceId);
-		if (!trace) return;
-		const event = buildRecordedFeedbackEventFromTrace({
+		const event = await this.#buildRecordedEventWithRetry(args.traceId, (trace) => buildRecordedFeedbackEventFromTrace({
 			trace,
 			spanId: args.spanId,
 			feedback: args.feedback
-		});
-		if (!event) return;
+		}));
+		if (!event) {
+			this.logger?.warn(`Feedback event was dropped because the target trace/span was not found in observability storage (traceId: ${args.traceId}, spanId: ${args.spanId})`);
+			return;
+		}
 		await this.#emitRecordedEvent(event);
 	}
 	/** Register a named observability instance, optionally marking it as default. */
@@ -8972,6 +8991,20 @@ var Observability = class extends _mastra_core_base.MastraBase {
 		if (!storage) return null;
 		return await storage.getStore("observability") ?? null;
 	}
+	/**
+	* Build a recorded score/feedback event from storage, retrying briefly to
+	* ride out the async exporter flush: annotations emitted right after a
+	* span ends can otherwise race the flush and be silently dropped.
+	*/
+	async #buildRecordedEventWithRetry(traceId, build) {
+		for (const delayMs of [0, ...RECORDED_TRACE_LOOKUP_RETRY_DELAYS_MS]) {
+			if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
+			const trace = await this.#getStoredTrace(traceId);
+			const event = trace ? build(trace) : null;
+			if (event) return event;
+		}
+		return null;
+	}
 	async #getStoredTrace(traceId) {
 		const observabilityStorage = await this.#getObservabilityStorage();
 		if (!observabilityStorage) return null;