@mastra/observability 1.16.6-alpha.3 → 1.16.6

package/dist/index.js

@@ -5178,15 +5178,15 @@ function isAuthFailureError(error) {
 function isAuthFailureStatus(status) {
 	return AUTH_FAILURE_STATUSES.has(status);
 }
-async function fetchWithAuthFailureHandling(url, options, maxRetries) {
+async function fetchWithAuthFailureHandling(url, options, maxRetries, callerShouldRetry) {
 	let authFailureStatus;
 	try {
-		await fetchWithRetry(url, options, maxRetries, { shouldRetryResponse: (response) => {
+		return await fetchWithRetry(url, options, maxRetries, { shouldRetryResponse: (response) => {
 			if (isAuthFailureStatus(response.status)) {
 				authFailureStatus = response.status;
 				return false;
 			}
-			return true;
+			return callerShouldRetry?.(response) ?? true;
 		} });
 	} catch (error) {
 		if (authFailureStatus !== void 0) throw new AuthFailureError(authFailureStatus, error);
@@ -6316,6 +6316,25 @@ const SIGNAL_PUBLISH_SUFFIXES = {
 	feedback: "/feedback/publish"
 };
 const DEFAULT_PLATFORM_SPAN_FILTER = (span) => span.type !== SpanType.MODEL_CHUNK;
+const QUOTA_EXCEEDED_STATUS = 402;
+const OBSERVABILITY_STATUS_HEADER = "x-mastra-observability";
+const OBSERVABILITY_DISABLED_VALUE = "disabled";
+const OBSERVABILITY_RETRY_AFTER_HEADER = "x-mastra-observability-retry-after";
+const DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS = 300;
+const MAX_QUOTA_PROBE_INTERVAL_SECONDS = Math.floor(2147483647 / 1e3);
+function isObservabilityDisabled(response) {
+	return response.headers.get(OBSERVABILITY_STATUS_HEADER) === OBSERVABILITY_DISABLED_VALUE;
+}
+function isQuotaExceededResponse(response) {
+	return response.status === QUOTA_EXCEEDED_STATUS || isObservabilityDisabled(response);
+}
+function parseQuotaRetryAfterSeconds(response) {
+	const raw = response.headers.get(OBSERVABILITY_RETRY_AFTER_HEADER);
+	if (!raw || !/^\d+$/.test(raw.trim())) return DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
+	const parsed = Number.parseInt(raw, 10);
+	if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
+	return Math.min(parsed, MAX_QUOTA_PROBE_INTERVAL_SECONDS);
+}
 const SIGNAL_PUBLISH_SEGMENTS = {
 	traces: "spans",
 	logs: "logs",
@@ -6405,6 +6424,10 @@ var MastraPlatformExporter = class extends BaseExporter {
 	buffer;
 	flushTimer = null;
 	inFlightFlushes = /* @__PURE__ */ new Set();
+	quotaPaused = false;
+	quotaProbeTimer = null;
+	quotaProbeIntervalSeconds = DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
+	shuttingDown = false;
 	constructor(config = {}) {
 		super(config);
 		if (config.projectId !== void 0 && !VALID_PROJECT_ID.test(config.projectId)) throw createInvalidProjectIdError(config.projectId);
@@ -6453,31 +6476,31 @@ var MastraPlatformExporter = class extends BaseExporter {
 	async _exportTracingEvent(event) {
 		if (event.type !== TracingEventType.SPAN_ENDED) return;
 		if (!DEFAULT_PLATFORM_SPAN_FILTER(event.exportedSpan)) return;
-		if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
+		if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
 		this.addToBuffer(event);
 		await this.handleBufferedEvent();
 	}
 	async onLogEvent(event) {
 		if (this.isDisabled) return;
-		if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
+		if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
 		this.addLogToBuffer(event);
 		await this.handleBufferedEvent();
 	}
 	async onMetricEvent(event) {
 		if (this.isDisabled) return;
-		if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
+		if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
 		this.addMetricToBuffer(event);
 		await this.handleBufferedEvent();
 	}
 	async onScoreEvent(event) {
 		if (this.isDisabled) return;
-		if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
+		if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
 		this.addScoreToBuffer(event);
 		await this.handleBufferedEvent();
 	}
 	async onFeedbackEvent(event) {
 		if (this.isDisabled) return;
-		if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
+		if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
 		this.addFeedbackToBuffer(event);
 		await this.handleBufferedEvent();
 	}
@@ -6567,6 +6590,10 @@ var MastraPlatformExporter = class extends BaseExporter {
 			this.flushTimer = null;
 		}
 		if (this.buffer.totalSize === 0) return;
+		if (this.quotaPaused) {
+			this.resetBuffer();
+			return;
+		}
 		if (this.authFailureCooldown.dropEventsIfCoolingDown(this.buffer.totalSize)) {
 			this.resetBuffer();
 			return;
@@ -6616,11 +6643,16 @@ var MastraPlatformExporter = class extends BaseExporter {
 	/**
 	* Uploads a signal batch to the configured Mastra Observability API using fetchWithRetry.
 	*/
-	async batchUpload(signal, records) {
-		const headers = {
+	buildPublishHeaders() {
+		return {
 			Authorization: `Bearer ${this.platformConfig.accessToken}`,
 			"Content-Type": "application/json"
 		};
+	}
+	buildPublishBody(signal, records) {
+		return JSON.stringify({ [SIGNAL_PUBLISH_SEGMENTS[signal]]: records });
+	}
+	async batchUpload(signal, records) {
 		const endpointMap = {
 			traces: this.platformConfig.tracesEndpoint,
 			logs: this.platformConfig.logsEndpoint,
@@ -6630,10 +6662,85 @@ var MastraPlatformExporter = class extends BaseExporter {
 		};
 		const options = {
 			method: "POST",
-			headers,
-			body: JSON.stringify({ [SIGNAL_PUBLISH_SEGMENTS[signal]]: records })
+			headers: this.buildPublishHeaders(),
+			body: this.buildPublishBody(signal, records)
 		};
-		await fetchWithAuthFailureHandling(endpointMap[signal], options, this.platformConfig.maxRetries);
+		let quotaResponse;
+		let response;
+		try {
+			response = await fetchWithAuthFailureHandling(endpointMap[signal], options, this.platformConfig.maxRetries, (res) => {
+				if (isQuotaExceededResponse(res)) {
+					quotaResponse = res;
+					return false;
+				}
+				return true;
+			});
+		} catch (error) {
+			if (quotaResponse) {
+				this.enterQuotaPause(parseQuotaRetryAfterSeconds(quotaResponse));
+				return;
+			}
+			throw error;
+		}
+		if (isObservabilityDisabled(response)) this.enterQuotaPause(parseQuotaRetryAfterSeconds(response));
+	}
+	/**
+	* Enter the quota-exhausted paused state: drop buffered events, stop the
+	* flush timer, and start probing until the collector stops rejecting with
+	* 402. The gate is per-org, so a signal on any publish route pauses all
+	* five signal types.
+	*/
+	enterQuotaPause(retryAfterSeconds) {
+		this.quotaProbeIntervalSeconds = retryAfterSeconds;
+		if (this.quotaPaused) return;
+		this.quotaPaused = true;
+		if (this.flushTimer) {
+			clearTimeout(this.flushTimer);
+			this.flushTimer = null;
+		}
+		this.resetBuffer();
+		this.logger.warn(`Mastra observability paused: quota exhausted, dropping telemetry and probing every ${retryAfterSeconds}s`);
+		this.scheduleQuotaProbe();
+	}
+	scheduleQuotaProbe() {
+		if (this.shuttingDown) return;
+		if (this.quotaProbeTimer) clearTimeout(this.quotaProbeTimer);
+		this.quotaProbeTimer = setTimeout(() => {
+			this.quotaProbeTimer = null;
+			this.probeQuotaStatus();
+		}, this.quotaProbeIntervalSeconds * 1e3);
+		this.quotaProbeTimer.unref?.();
+	}
+	/**
+	* Send an empty spans batch as a free probe. The collector treats an empty
+	* batch as a no-op, and an exhausted org still gets the 402 on it. A 402
+	* throws out of fetchWithRetry without the Response, so the retry predicate
+	* captures it to keep the retry-after hint.
+	*/
+	async probeQuotaStatus() {
+		if (this.shuttingDown || !this.quotaPaused) return;
+		let quotaResponse;
+		try {
+			const response = await fetchWithRetry(this.platformConfig.tracesEndpoint, {
+				method: "POST",
+				headers: this.buildPublishHeaders(),
+				body: this.buildPublishBody("traces", [])
+			}, 1, { shouldRetryResponse: (res) => {
+				if (isQuotaExceededResponse(res)) quotaResponse = res;
+				return false;
+			} });
+			if (this.shuttingDown) return;
+			if (!isObservabilityDisabled(response)) {
+				this.quotaPaused = false;
+				this.quotaProbeIntervalSeconds = DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
+				this.logger.warn("Mastra observability resumed: quota restored, exports re-enabled");
+				return;
+			}
+			this.quotaProbeIntervalSeconds = parseQuotaRetryAfterSeconds(response);
+		} catch {
+			if (quotaResponse) this.quotaProbeIntervalSeconds = parseQuotaRetryAfterSeconds(quotaResponse);
+		}
+		this.scheduleQuotaProbe();
 	}
 	async flushSignalBatch(signal, records) {
 		if (records.length === 0) return {
@@ -6701,11 +6808,16 @@ var MastraPlatformExporter = class extends BaseExporter {
 		}
 	}
 	async shutdown() {
+		this.shuttingDown = true;
 		if (this.isDisabled) return;
 		if (this.flushTimer) {
 			clearTimeout(this.flushTimer);
 			this.flushTimer = null;
 		}
+		if (this.quotaProbeTimer) {
+			clearTimeout(this.quotaProbeTimer);
+			this.quotaProbeTimer = null;
+		}
 		try {
 			await this.flush();
 		} catch (error) {