@mastra/observability 1.21.0-alpha.1 → 1.21.0

package/dist/index.js

@@ -51,9 +51,7 @@ function routeToHandler(handler, event, logger) {
 					}), handler.name, "score", logger);
 				}
 				break;
-			case "feedback":
-				if (handler.onFeedbackEvent) return catchAsyncResult(handler.onFeedbackEvent(event), handler.name, "feedback", logger);
-				break;
+			case "feedback": if (handler.onFeedbackEvent) return catchAsyncResult(handler.onFeedbackEvent(event), handler.name, "feedback", logger);
 		}
 	} catch (err) {
 		logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
@@ -202,7 +200,7 @@ const VALUE_STRIP_KEYS = /* @__PURE__ */ new Set([
 const DEFAULT_DEEP_CLEAN_OPTIONS = Object.freeze({
 	keysToStrip: [],
 	maxDepth: 8,
-	maxStringLength: 128 * 1024,
+	maxStringLength: 131072,
 	maxArrayLength: 50,
 	maxObjectKeys: 50
 });
@@ -1276,8 +1274,8 @@ function isRecord(value) {
 //#endregion
 //#region src/metrics/pricing-registry.ts
 const DATA_FILE_NAME = "pricing-data.json";
-const REFRESH_INTERVAL_MS = 1440 * 60 * 1e3;
-const MIN_REFRESH_ATTEMPT_INTERVAL_MS = 3600 * 1e3;
+const REFRESH_INTERVAL_MS = 864e5;
+const MIN_REFRESH_ATTEMPT_INTERVAL_MS = 36e5;
 const REFRESH_TIMEOUT_MS = 5e3;
 const CACHE_FILE = () => path.join(os.homedir(), ".cache", "mastra", "pricing-data.json");
 const CACHE_VERSION = 1;
@@ -2819,6 +2817,26 @@ var ModelSpanTracker = class {
 		this.#stepIndex++;
 	}
 	/**
+	* A suspended step emits no step-finish; its suspension chunk carries the
+	* step usage instead. Close MODEL_INFERENCE and MODEL_STEP with that usage so
+	* the suspended call is not reported without tokens.
+	*/
+	#endSuspendedStep(chunk) {
+		if (chunk.type !== "tool-call-suspended" && chunk.type !== "tool-call-approval") return;
+		const usage = chunk.payload.usage;
+		if (!usage) return;
+		this.#inModelResponse = false;
+		this.#endStepSpan({
+			output: { usage },
+			stepResult: {
+				reason: "suspended",
+				isContinued: false,
+				warnings: []
+			},
+			metadata: {}
+		});
+	}
+	/**
 	* Returns the parent span for chunks. Chunks parent under MODEL_INFERENCE
 	* (the provider call) when available, falling back to MODEL_STEP only if
 	* startStep() was bypassed.
@@ -2916,9 +2934,7 @@ var ModelSpanTracker = class {
 				if (this.#currentChunkType !== "text") this.#startChunkSpan("text");
 				this.#appendToAccumulator("text", chunk.payload.text);
 				break;
-			case "text-end":
-				this.#endChunkSpan();
-				break;
+			case "text-end": this.#endChunkSpan();
 		}
 	}
 	/**
@@ -2933,9 +2949,7 @@ var ModelSpanTracker = class {
 				if (this.#currentChunkType !== "reasoning") this.#startChunkSpan("reasoning");
 				this.#appendToAccumulator("text", chunk.payload.text);
 				break;
-			case "reasoning-end":
-				this.#endChunkSpan();
-				break;
+			case "reasoning-end": this.#endChunkSpan();
 		}
 	}
 	/**
@@ -2978,9 +2992,7 @@ var ModelSpanTracker = class {
 			case "object":
 				if (this.#currentChunkType !== "object") this.#startChunkSpan("object");
 				break;
-			case "object-result":
-				this.#endChunkSpan(chunk.object);
-				break;
+			case "object-result": this.#endChunkSpan(chunk.object);
 		}
 	}
 	/**
@@ -2990,8 +3002,8 @@ var ModelSpanTracker = class {
 	#handleToolApprovalChunk(chunk) {
 		if (chunk.type !== "tool-call-approval") return;
 		const payload = chunk.payload;
-		this.#ensureStepAndInference();
-		if (this.#chunkParent()?.createEventSpan({
+		if (payload.usage || this.#currentStepSpan) this.#ensureStepAndInference();
+		if ((this.#chunkParent() ?? this.#modelSpan)?.createEventSpan({
 			name: `chunk: 'tool-call-approval'`,
 			type: SpanType.MODEL_CHUNK,
 			attributes: {
@@ -3011,6 +3023,10 @@ var ModelSpanTracker = class {
 	wrapStream(stream) {
 		return stream.pipeThrough(new TransformStream({ transform: (chunk, controller) => {
 			if ((this.#inModelResponse || this.#deferStepClose) && MODEL_RESPONSE_CHUNK_TYPES.has(chunk.type)) this.#captureCompletionStartTime();
+			if (chunk.type === "tool-call-suspended" || chunk.type === "tool-call-approval") {
+				if (chunk.type === "tool-call-approval") this.#handleToolApprovalChunk(chunk);
+				this.#endSuspendedStep(chunk);
+			}
 			controller.enqueue(chunk);
 			switch (chunk.type) {
 				case "text-start":
@@ -3058,13 +3074,11 @@ var ModelSpanTracker = class {
 				case "tripwire":
 				case "watch":
 				case "tool-error":
-				case "tool-call-suspended":
 				case "reasoning-signature":
 				case "redacted-reasoning":
 				case "step-output": break;
-				case "tool-call-approval":
-					this.#handleToolApprovalChunk(chunk);
-					break;
+				case "tool-call-suspended":
+				case "tool-call-approval": break;
 				case "tool-output": break;
 				case "tool-result": {
 					const { toolCallId, toolName, isError, dynamic, providerExecuted, providerMetadata, result, args: _args } = chunk.payload || {};
@@ -3081,7 +3095,6 @@ var ModelSpanTracker = class {
 					this.#createEventSpan(chunk.type, spanOutput, { metadata });
 					break;
 				}
-				default: break;
 			}
 		} }));
 	}
@@ -3622,10 +3635,14 @@ var DefaultSpan = class extends BaseSpan {
 		}
 		this.traceId = getOrCreateTraceId(options);
 		this.id = generateSpanId();
-		if (options.parentSpanId) if (isValidSpanId(options.parentSpanId)) this.parentSpanId = options.parentSpanId;
-		else console.error(`[Mastra Tracing] Invalid parentSpanId: must be 1-16 hexadecimal characters, got "${options.parentSpanId}". Ignoring.`);
-		if (options.externalParentSpanId) if (isValidSpanId(options.externalParentSpanId)) this.externalParentSpanId = options.externalParentSpanId;
-		else console.error(`[Mastra Tracing] Invalid externalParentSpanId: must be 1-16 hexadecimal characters, got "${options.externalParentSpanId}". Ignoring.`);
+		if (options.parentSpanId) {
+			if (isValidSpanId(options.parentSpanId)) this.parentSpanId = options.parentSpanId;
+			else console.error(`[Mastra Tracing] Invalid parentSpanId: must be 1-16 hexadecimal characters, got "${options.parentSpanId}". Ignoring.`);
+		}
+		if (options.externalParentSpanId) {
+			if (isValidSpanId(options.externalParentSpanId)) this.externalParentSpanId = options.externalParentSpanId;
+			else console.error(`[Mastra Tracing] Invalid externalParentSpanId: must be 1-16 hexadecimal characters, got "${options.externalParentSpanId}". Ignoring.`);
+		}
 	}
 	end(options) {
 		if (this.isEvent) return;
@@ -3747,8 +3764,10 @@ function isValidSpanId(spanId) {
 	return /^[0-9a-f]{1,16}$/i.test(spanId);
 }
 function getOrCreateTraceId(options) {
-	if (options.traceId) if (isValidTraceId(options.traceId)) return options.traceId;
-	else console.error(`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`);
+	if (options.traceId) {
+		if (isValidTraceId(options.traceId)) return options.traceId;
+		else console.error(`[Mastra Tracing] Invalid traceId: must be 1-32 hexadecimal characters, got "${options.traceId}". Generating new trace ID.`);
+	}
 	return generateTraceId();
 }
 //#endregion
@@ -3997,17 +4016,13 @@ var EventBuffer = class {
 				switch (this.#storageStrategy) {
 					case "realtime":
 					case "event-sourced":
-					case "batch-with-updates":
-						this.pushCreate(event);
-						break;
+					case "batch-with-updates": this.pushCreate(event);
 				}
 				break;
 			case TracingEventType.SPAN_UPDATED:
 				switch (this.#storageStrategy) {
 					case "realtime":
-					case "batch-with-updates":
-						this.pushUpdate(event);
-						break;
+					case "batch-with-updates": this.pushUpdate(event);
 				}
 				break;
 			case TracingEventType.SPAN_ENDED:
@@ -4017,14 +4032,10 @@ var EventBuffer = class {
 					case "batch-with-updates":
 						this.pushUpdate(event);
 						break;
-					default:
-						this.pushCreate(event);
-						break;
+					default: this.pushCreate(event);
 				}
 				break;
-			default:
-				this.pushCreate(event);
-				break;
+			default: this.pushCreate(event);
 		}
 	}
 	/** Re-add failed create events to the buffer, returning events that exceed max retries. */
@@ -4399,17 +4410,13 @@ var DefaultExporter = class extends BaseExporter {
 			case "score":
 				createScoreEvents.push(createEvent);
 				break;
-			default:
-				createSpanEvents.push(createEvent);
-				break;
+			default: createSpanEvents.push(createEvent);
 		}
 		for (const updateEvent of updates) switch (updateEvent.type) {
 			case TracingEventType.SPAN_UPDATED:
 				updateSpanEvents.push(updateEvent);
 				break;
-			case TracingEventType.SPAN_ENDED:
-				endSpanEvents.push(updateEvent);
-				break;
+			case TracingEventType.SPAN_ENDED: endSpanEvents.push(updateEvent);
 		}
 		const flushResults = await Promise.all([
 			this.flushCreates("feedback", createFeedbackEvents, (events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => buildFeedbackRecord(f)) })),
@@ -4426,13 +4433,15 @@ var DefaultExporter = class extends BaseExporter {
 		flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
 		flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
 		let deferredDropCount = 0;
-		if (deferredUpdates.length > 0) if (this.#unsupportedSignals.has("tracing")) {
-			deferredDropCount = deferredUpdates.length;
-			this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
-			deferredUpdates.length = 0;
-		} else {
-			deferredDropCount = this.#eventBuffer.reAddUpdates(deferredUpdates).length;
-			this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
+		if (deferredUpdates.length > 0) {
+			if (this.#unsupportedSignals.has("tracing")) {
+				deferredDropCount = deferredUpdates.length;
+				this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
+				deferredUpdates.length = 0;
+			} else {
+				deferredDropCount = this.#eventBuffer.reAddUpdates(deferredUpdates).length;
+				this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
+			}
 		}
 		const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
 			if (result.retryAttempt === void 0) return earliestAttempt;
@@ -4850,17 +4859,13 @@ var MastraStorageExporter = class extends BaseExporter {
 			case "score":
 				createScoreEvents.push(createEvent);
 				break;
-			default:
-				createSpanEvents.push(createEvent);
-				break;
+			default: createSpanEvents.push(createEvent);
 		}
 		for (const updateEvent of updates) switch (updateEvent.type) {
 			case TracingEventType.SPAN_UPDATED:
 				updateSpanEvents.push(updateEvent);
 				break;
-			case TracingEventType.SPAN_ENDED:
-				endSpanEvents.push(updateEvent);
-				break;
+			case TracingEventType.SPAN_ENDED: endSpanEvents.push(updateEvent);
 		}
 		const flushResults = await Promise.all([
 			this.flushCreates("feedback", createFeedbackEvents, (events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => buildFeedbackRecord(f)) })),
@@ -4877,13 +4882,15 @@ var MastraStorageExporter = class extends BaseExporter {
 		flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
 		flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
 		let deferredDropCount = 0;
-		if (deferredUpdates.length > 0) if (this.#unsupportedSignals.has("tracing")) {
-			deferredDropCount = deferredUpdates.length;
-			this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
-			deferredUpdates.length = 0;
-		} else {
-			deferredDropCount = this.#eventBuffer.reAddUpdates(deferredUpdates).length;
-			this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
+		if (deferredUpdates.length > 0) {
+			if (this.#unsupportedSignals.has("tracing")) {
+				deferredDropCount = deferredUpdates.length;
+				this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
+				deferredUpdates.length = 0;
+			} else {
+				deferredDropCount = this.#eventBuffer.reAddUpdates(deferredUpdates).length;
+				this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
+			}
 		}
 		const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
 			if (result.retryAttempt === void 0) return earliestAttempt;
@@ -5403,10 +5410,13 @@ var BaseObservabilityInstance = class extends MastraBase {
 			const parts = key.split(".");
 			const rootKey = parts[0];
 			const value = requestContext.get(rootKey);
-			if (value !== void 0) if (parts.length > 1) {
-				const nestedValue = getNestedValue(value, parts.slice(1).join("."));
-				if (nestedValue !== void 0) setNestedValue(result, key, nestedValue);
-			} else setNestedValue(result, key, value);
+			if (value !== void 0) {
+				if (parts.length > 1) {
+					const nestedPath = parts.slice(1).join(".");
+					const nestedValue = getNestedValue(value, nestedPath);
+					if (nestedValue !== void 0) setNestedValue(result, key, nestedValue);
+				} else setNestedValue(result, key, value);
+			}
 		}
 		return result;
 	}
@@ -5954,7 +5964,7 @@ function formatBaggage(entries) {
 const DEFAULT_LIMITS = {
 	maxSpans: 10,
 	maxLogs: 10,
-	maxPayloadBytes: 1024 * 1024
+	maxPayloadBytes: 1048576
 };
 var ClientObservabilityProxyImpl = class {
 	#resolveInstance;
@@ -6889,36 +6899,38 @@ var TrackingExporter = class extends BaseExporter {
 			method
 		});
 		const { exportedSpan } = await this._preExportTracingEvent(event);
-		if (!this.skipBuildRootTask && !traceData.hasRoot()) if (exportedSpan.isRootSpan) {
-			this.logger.debug(`${this.name}: Building root`, {
-				traceId: exportedSpan.traceId,
-				spanId: exportedSpan.id
-			});
-			const rootData = await this._buildRoot({
-				span: exportedSpan,
-				traceData
-			});
-			if (rootData) {
-				this.logger.debug(`${this.name}: Adding root`, {
+		if (!this.skipBuildRootTask && !traceData.hasRoot()) {
+			if (exportedSpan.isRootSpan) {
+				this.logger.debug(`${this.name}: Building root`, {
+					traceId: exportedSpan.traceId,
+					spanId: exportedSpan.id
+				});
+				const rootData = await this._buildRoot({
+					span: exportedSpan,
+					traceData
+				});
+				if (rootData) {
+					this.logger.debug(`${this.name}: Adding root`, {
+						traceId: exportedSpan.traceId,
+						spanId: exportedSpan.id
+					});
+					traceData.addRoot({
+						rootId: exportedSpan.id,
+						rootData
+					});
+					this.#scheduleProcessWaitingForRoot(traceId);
+				}
+			} else {
+				this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
 					traceId: exportedSpan.traceId,
 					spanId: exportedSpan.id
 				});
-				traceData.addRoot({
-					rootId: exportedSpan.id,
-					rootData
+				traceData.addToWaitingQueue({
+					event,
+					waitingFor: "root"
 				});
-				this.#scheduleProcessWaitingForRoot(traceId);
+				return;
 			}
-		} else {
-			this.logger.debug(`${this.name}: Root does not exist, adding span to waiting queue.`, {
-				traceId: exportedSpan.traceId,
-				spanId: exportedSpan.id
-			});
-			traceData.addToWaitingQueue({
-				event,
-				waitingFor: "root"
-			});
-			return;
 		}
 		if (exportedSpan.metadata && this.name in exportedSpan.metadata) {
 			const metadata = exportedSpan.metadata[this.name];
@@ -7045,7 +7057,6 @@ var TrackingExporter = class extends BaseExporter {
 						traceData
 					});
 					if (traceData.activeSpanCount() === 0) this.#scheduleCleanup(traceId);
-					break;
 			}
 		} catch (error) {
 			this.logger.error(`${this.name}: exporter error`, {
@@ -7211,7 +7222,7 @@ function chainFormatters(formatters) {
 //#region src/exporters/auth-failure-cooldown.ts
 const AUTH_FAILURE_STATUSES = /* @__PURE__ */ new Set([401, 403]);
 const AUTH_COOLDOWN_BASE_MS = 6e4;
-const AUTH_COOLDOWN_MAX_MS = 15 * 6e4;
+const AUTH_COOLDOWN_MAX_MS = 9e5;
 const AUTH_COOLDOWN_JITTER_RATIO = .1;
 var AuthFailureError = class extends Error {
 	status;
@@ -7640,8 +7651,9 @@ var CloudExporter = class extends BaseExporter {
 				succeeded: false,
 				authFailureStatus: error.status
 			};
+			const errorId = `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD_${signal.toUpperCase()}`;
 			const mastraError = new MastraError({
-				id: `CLOUD_EXPORTER_FAILED_TO_BATCH_UPLOAD_${signal.toUpperCase()}`,
+				id: errorId,
 				domain: ErrorDomain.MASTRA_OBSERVABILITY,
 				category: ErrorCategory.USER,
 				details: {
@@ -8230,8 +8242,9 @@ var MastraPlatformExporter = class extends BaseExporter {
 				succeeded: false,
 				authFailureStatus: error.status
 			};
+			const errorId = `MASTRA_PLATFORM_EXPORTER_FAILED_TO_BATCH_UPLOAD_${signal.toUpperCase()}`;
 			const mastraError = new MastraError({
-				id: `MASTRA_PLATFORM_EXPORTER_FAILED_TO_BATCH_UPLOAD_${signal.toUpperCase()}`,
+				id: errorId,
 				domain: ErrorDomain.MASTRA_OBSERVABILITY,
 				category: ErrorCategory.USER,
 				details: {
@@ -9095,16 +9108,18 @@ var TestExporter = class extends BaseExporter {
 		for (let i = 0; i < maxLen; i++) {
 			const actualLine = actual[i];
 			const expectedLine = expected[i];
-			if (actualLine !== expectedLine) if (actualLine === void 0) {
-				diffs.push(`Line ${i + 1}: Missing in actual`);
-				diffs.push(`  Expected: ${expectedLine}`);
-			} else if (expectedLine === void 0) {
-				diffs.push(`Line ${i + 1}: Extra in actual`);
-				diffs.push(`  Actual: ${actualLine}`);
-			} else {
-				diffs.push(`Line ${i + 1}:`);
-				diffs.push(`  Expected: ${expectedLine}`);
-				diffs.push(`  Actual:   ${actualLine}`);
+			if (actualLine !== expectedLine) {
+				if (actualLine === void 0) {
+					diffs.push(`Line ${i + 1}: Missing in actual`);
+					diffs.push(`  Expected: ${expectedLine}`);
+				} else if (expectedLine === void 0) {
+					diffs.push(`Line ${i + 1}: Extra in actual`);
+					diffs.push(`  Actual: ${actualLine}`);
+				} else {
+					diffs.push(`Line ${i + 1}:`);
+					diffs.push(`  Expected: ${expectedLine}`);
+					diffs.push(`  Actual:   ${actualLine}`);
+				}
 			}
 		}
 		return diffs;
@@ -9867,9 +9882,10 @@ var SensitiveDataFilter = class {
 		const filtered = {};
 		for (const key of Object.keys(obj)) {
 			const normKey = this.normalizeKey(key);
-			if (this.isSensitive(normKey)) if (obj[key] && typeof obj[key] === "object") filtered[key] = this.deepFilter(obj[key], seen, indexedState);
-			else filtered[key] = this.redactValue(obj[key], normKey, indexedState);
-			else filtered[key] = this.deepFilter(obj[key], seen, indexedState);
+			if (this.isSensitive(normKey)) {
+				if (obj[key] && typeof obj[key] === "object") filtered[key] = this.deepFilter(obj[key], seen, indexedState);
+				else filtered[key] = this.redactValue(obj[key], normKey, indexedState);
+			} else filtered[key] = this.deepFilter(obj[key], seen, indexedState);
 		}
 		return filtered;
 	}