@mastra/observability 1.19.0 → 1.20.0-alpha.1

package/dist/index.js

@@ -6,6 +6,7 @@ import { DEFAULT_BLOCKED_LABELS, EntityType, InternalSpans, SpanType, TracingEve
 import { fetchWithRetry, getNestedValue, setNestedValue } from "@mastra/core/utils";
 import { z } from "zod/v4";
 import fs from "fs";
+import os from "os";
 import path from "path";
 import { TransformStream } from "stream/web";
 import { coreFeatures } from "@mastra/core/features";
@@ -1123,14 +1124,12 @@ var PricingModel = class {
 	id;
 	provider;
 	model;
-	schema;
 	currency;
 	tiers;
 	constructor(args) {
 		this.id = args.id;
 		this.provider = args.provider;
 		this.model = args.model;
-		this.schema = args.schema;
 		this.currency = args.currency;
 		this.tiers = args.tiers;
 	}
@@ -1143,8 +1142,145 @@ var PricingModel = class {
 	}
 };
 //#endregion
+//#region src/metrics/models-dev.ts
+/** Same catalog the model router reads in `@mastra/core` (`gateways/models-dev.ts`). */
+const MODELS_DEV_API_URL = "https://models.dev/api.json";
+/**
+* A models.dev catalog with fewer prices than this has changed shape or is broken, so it must
+* not replace the pricing in use. The full catalog prices roughly 8,000 models.
+*/
+const MIN_PRICED_MODELS = 1e3;
+/** models.dev `cost` keys (USD per 1M tokens) mapped to pricing meters. Keyed by core's `ModelPricingRates` so the two cannot drift. */
+const COST_KEY_TO_METER = {
+	input: "input_tokens",
+	output: "output_tokens",
+	cache_read: "input_cache_read_tokens",
+	cache_write: "input_cache_write_tokens",
+	input_audio: "input_audio_tokens",
+	output_audio: "output_audio_tokens",
+	reasoning: "output_reasoning_tokens"
+};
+const PRICE_SIGNIFICANT_DIGITS = 12;
+/** Keep only each model's `cost` from a models.dev `api.json` payload, sorted by id so diffs stay small. */
+function pickModelsDevCosts(catalog) {
+	const costs = {};
+	for (const [providerId, providerInfo] of sortedEntries(catalog)) {
+		if (!isRecord(providerInfo)) continue;
+		for (const [modelId, modelInfo] of sortedEntries(providerInfo.models)) if (isRecord(modelInfo) && isRecord(modelInfo.cost)) (costs[providerId] ??= { models: {} }).models[modelId] = { cost: modelInfo.cost };
+	}
+	return costs;
+}
+/**
+* Build the pricing lookup from models.dev costs, keyed by `provider::model`.
+*
+* Each model is keyed by its own id (lowercased, dots and slashes flattened to dashes, matching
+* the lookup variants in `pricing-registry.ts`). Each model is also keyed by an alias with region,
+* vendor, version and date parts stripped, but only when no model owns that key itself, so the
+* dated `gpt-4o-2024-05-13` never replaces `gpt-4o`.
+*/
+function modelsDevToPricingModels(costs) {
+	const pricingModels = /* @__PURE__ */ new Map();
+	const aliases = [];
+	for (const [providerId, providerInfo] of Object.entries(isRecord(costs) ? costs : {})) {
+		if (!isRecord(providerInfo) || !isRecord(providerInfo.models)) continue;
+		const provider = normalizeProviderId(providerId);
+		for (const [modelId, modelInfo] of Object.entries(providerInfo.models)) {
+			const tiers = isRecord(modelInfo) ? toTiers(modelInfo.cost, modelId) : null;
+			if (!tiers) continue;
+			const model = modelId.toLowerCase().replace(/[./]/g, "-");
+			const pricingModel = new PricingModel({
+				id: `${providerId}/${modelId}`,
+				provider,
+				model,
+				currency: "USD",
+				tiers
+			});
+			pricingModels.set(`${provider}::${model}`, pricingModel);
+			const alias = toAliasModelKey(modelId);
+			if (alias && alias !== model) aliases.push({
+				alias,
+				modelId,
+				pricingModel
+			});
+		}
+	}
+	aliases.sort((a, b) => a.modelId.length - b.modelId.length || b.modelId.localeCompare(a.modelId));
+	for (const { alias, pricingModel } of aliases) {
+		const key = `${pricingModel.provider}::${alias}`;
+		if (!pricingModels.has(key)) {
+			const { id, provider, currency, tiers } = pricingModel;
+			pricingModels.set(key, new PricingModel({
+				id,
+				provider,
+				model: alias,
+				currency,
+				tiers
+			}));
+		}
+	}
+	return pricingModels;
+}
+function toTiers(cost, modelId) {
+	const isClaude = modelId.toLowerCase().includes("claude");
+	const baseRates = toRates(cost, isClaude);
+	if (!baseRates || !isRecord(cost)) return null;
+	const contextTiers = (Array.isArray(cost.tiers) ? cost.tiers : []).flatMap((tier) => {
+		const band = isRecord(tier) && isRecord(tier.tier) ? tier.tier : null;
+		const rates = toRates(tier, isClaude);
+		return band && (band.type ?? "context") === "context" && typeof band.size === "number" && band.size > 0 && rates ? [{
+			size: band.size,
+			rates
+		}] : [];
+	}).sort((a, b) => b.size - a.size);
+	return [new PricingTier({
+		index: 0,
+		rates: baseRates
+	}), ...contextTiers.map(({ size, rates }, i) => new PricingTier({
+		index: i + 1,
+		when: [{
+			field: "total_input_tokens",
+			op: "gte",
+			value: size
+		}],
+		rates
+	}))];
+}
+/** Zero prices are dropped (free endpoints and flat-fee plans), as the old platform pipeline did. */
+function toRates(cost, isClaude) {
+	if (!isRecord(cost)) return null;
+	const rates = {};
+	for (const [costKey, meter] of Object.entries(COST_KEY_TO_METER)) {
+		const pricePerMillion = cost[costKey];
+		if (typeof pricePerMillion === "number" && Number.isFinite(pricePerMillion) && pricePerMillion > 0) rates[meter] = Number((pricePerMillion / 1e6).toPrecision(PRICE_SIGNIFICANT_DIGITS));
+	}
+	const cacheWriteRate = rates.input_cache_write_tokens;
+	if (isClaude && cacheWriteRate !== void 0) {
+		rates.input_cache_write_5m_tokens = cacheWriteRate;
+		rates.input_cache_write_1h_tokens = cacheWriteRate * 1.6;
+	}
+	return rates.input_tokens && rates.output_tokens ? rates : null;
+}
+function normalizeProviderId(value) {
+	return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+/** Port of the old platform pipeline's model-key normalization, used only for alias keys. */
+function toAliasModelKey(modelId) {
+	return modelId.replace(/^accounts\/[^/]+\/models\//i, "").replace(/^(global|us|eu|apac)\./i, "").replace(/^[a-z0-9-]+\.(openai|anthropic|google|xai|minimax)\./i, "").replace(/^(openai|anthropic|google|xai|x-ai|minimax|minimaxai)[/:._-]+/i, "").replace(/-v\d+(?::0)?$/i, "").replace(/@\d{8}$/, "").replace(/@default$/i, "").replace(/-\d{4}-\d{2}-\d{2}$/, "").replace(/-20\d{6}$/, "").replace(/_20\d{6}$/, "").replace(/[/.:_\s]+/g, "-").replace(/[^a-zA-Z0-9-]+/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "").toLowerCase();
+}
+function sortedEntries(value) {
+	return Object.entries(isRecord(value) ? value : {}).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
 //#region src/metrics/pricing-registry.ts
-const DATA_FILE_NAME = "pricing-data.jsonl";
+const DATA_FILE_NAME = "pricing-data.json";
+const REFRESH_INTERVAL_MS = 1440 * 60 * 1e3;
+const MIN_REFRESH_ATTEMPT_INTERVAL_MS = 3600 * 1e3;
+const REFRESH_TIMEOUT_MS = 5e3;
+const CACHE_FILE = () => path.join(os.homedir(), ".cache", "mastra", "pricing-data.json");
+const CACHE_VERSION = 1;
 const BEDROCK_GEOGRAPHY_PREFIXES = /* @__PURE__ */ new Set([
 	"global",
 	"us",
@@ -1154,102 +1290,220 @@ const BEDROCK_GEOGRAPHY_PREFIXES = /* @__PURE__ */ new Set([
 	"au"
 ]);
 const AI_SDK_VERCEL_GATEWAY_PROVIDER_ID = "gateway";
+const AZURE_OPENAI_GATEWAY_ID = "azure-openai";
 const VERCEL_PRICING_PROVIDER_ID = "vercel";
 const AI_SDK_PROVIDER_NAMESPACE_ALIASES = /* @__PURE__ */ new Map([
+	["fireworks", "fireworks-ai"],
 	["google.vertex", "google-vertex"],
 	["vertex.anthropic", "google-vertex-anthropic"],
 	["vertex.maas", "google-vertex"]
 ]);
-const MINIFIED_METER_TO_CANONICAL = {
-	it: "input_tokens",
-	ot: "output_tokens",
-	icrt: "input_cache_read_tokens",
-	icwt: "input_cache_write_tokens",
-	icwt5m: "input_cache_write_5m_tokens",
-	icwt1h: "input_cache_write_1h_tokens",
-	iat: "input_audio_tokens",
-	oat: "output_audio_tokens",
-	ort: "output_reasoning_tokens"
-};
-const MINIFIED_CONDITION_FIELD_TO_CANONICAL = { tit: "total_input_tokens" };
 let cachedLoadError = null;
+let globalRegistry = null;
+let bundledRegistry = null;
+let gatewayRegistry = null;
+let gatewayCosts = {};
+let gatewaySource = null;
+const gatewayLoadAttempts = /* @__PURE__ */ new Map();
+let gatewayLoadInFlight = false;
+let refreshedAt = 0;
+let lastRefreshAttemptAt = 0;
+let refreshEtag;
+let refreshInFlight = false;
 var PricingRegistry = class PricingRegistry {
 	pricingModels;
-	static globalRegistry = null;
 	constructor(pricingModels) {
 		this.pricingModels = pricingModels;
 	}
-	static fromText(pricingModelText) {
-		return new PricingRegistry(parsePricingModelText(pricingModelText));
+	/** A registry from models.dev costs, in the shape of `pricing-data.json` or `api.json`. */
+	static fromModelsDev(costs) {
+		return new PricingRegistry(modelsDevToPricingModels(costs));
 	}
+	/**
+	* The registry used for cost estimates. It uses the bundled snapshot. When refresh is on (under
+	* `mastra dev`, or with `MASTRA_AUTO_REFRESH_PRICING=true`), it starts from the models.dev copy
+	* cached in `~/.cache/mastra` instead, and refreshes from models.dev in the background once a day
+	* and when a model has no price. Lookups never wait for a refresh.
+	*/
 	static getGlobal() {
-		if (PricingRegistry.globalRegistry) return PricingRegistry.globalRegistry;
-		const pricingModels = loadPricingModels();
-		if (!pricingModels) return null;
-		PricingRegistry.globalRegistry = new PricingRegistry(pricingModels);
-		return PricingRegistry.globalRegistry;
+		if (!globalRegistry) {
+			const cachedPricingModels = loadCachedPricingModels();
+			globalRegistry = cachedPricingModels ? new PricingRegistry(cachedPricingModels) : getBundledRegistry();
+		}
+		if (Date.now() - refreshedAt >= REFRESH_INTERVAL_MS) refreshGlobalPricing();
+		return globalRegistry;
 	}
 	get(args) {
-		for (const provider of getPricingProviderCandidates(args.provider, args.model)) {
-			const variants = getModelVariants(args.model, provider);
-			for (const variant of variants) {
-				const key = makePricingKey({
-					provider,
-					model: variant
-				});
-				const match = this.pricingModels.get(key);
-				if (match) return match;
-			}
+		if (this !== globalRegistry) return this.find(args);
+		const match = this.find(args) ?? (this === bundledRegistry ? null : getBundledRegistry()?.find(args)) ?? gatewayRegistry?.find(args);
+		if (match) return match;
+		loadGatewayPricing();
+		refreshGlobalPricing();
+		return null;
+	}
+	find(args) {
+		for (const provider of getPricingProviderCandidates(args.provider, args.model)) for (const variant of getModelVariants(args.model, provider)) {
+			const match = this.pricingModels.get(makePricingKey({
+				provider,
+				model: variant
+			}));
+			if (match) return match;
 		}
 		return null;
 	}
 };
+function getBundledRegistry() {
+	if (!bundledRegistry) {
+		const pricingModels = loadPricingModels();
+		if (pricingModels) bundledRegistry = new PricingRegistry(pricingModels);
+	}
+	return bundledRegistry;
+}
+/**
+* Where to find the model router gateways whose `pricing` fills the gaps in models.dev. Prices
+* load on the first lookup that has no price, so production does not call gateways at startup;
+* under the same switches as the models.dev refresh (`mastra dev`, or
+* `MASTRA_AUTO_REFRESH_PRICING=true`) they also load right away, off the hot path, so the first
+* model call is priced. Each gateway is read once; a failed read is retried at most once an hour.
+*/
+function setPricingGateways(source) {
+	gatewaySource = source;
+	if (isRefreshEnabled()) loadGatewayPricing();
+}
+/** Load `pricing` from every custom gateway not loaded yet. Off under `MASTRA_OFFLINE`. */
+async function loadGatewayPricing() {
+	if (!gatewaySource || gatewayLoadInFlight || isOffline()) return;
+	gatewayLoadInFlight = true;
+	try {
+		const builtInIds = await getBuiltInGatewayIds();
+		let changed = false;
+		for (const gateway of gatewaySource()) {
+			const gatewayId = gateway.getId?.() ?? gateway.id;
+			const lastAttempt = gatewayLoadAttempts.get(gateway) ?? 0;
+			if (builtInIds.has(gatewayId) || Date.now() - lastAttempt < MIN_REFRESH_ATTEMPT_INTERVAL_MS) continue;
+			try {
+				const providers = await gateway.fetchProviders();
+				for (const [providerId, config] of Object.entries(providers ?? {})) for (const [modelId, cost] of Object.entries(config?.pricing ?? {})) (gatewayCosts[providerId] ??= { models: {} }).models[modelId] = { cost };
+				gatewayLoadAttempts.set(gateway, Infinity);
+				changed = true;
+			} catch (error) {
+				gatewayLoadAttempts.set(gateway, Date.now());
+				console.warn(`Failed to load model prices from gateway ${gatewayId}:`, error);
+			}
+		}
+		if (changed) gatewayRegistry = new PricingRegistry(modelsDevToPricingModels(gatewayCosts));
+	} finally {
+		gatewayLoadInFlight = false;
+	}
+}
+/**
+* The gateways core ships: the ones Mastra registers on its own, plus Azure OpenAI, which users
+* construct themselves. Their `fetchProviders()` can hit the network and never carries prices.
+* Read from core when it exports the list (1.46+), else the known ids.
+*/
+async function getBuiltInGatewayIds() {
+	try {
+		const { defaultGateways, getGatewayId } = await import("@mastra/core/llm");
+		return /* @__PURE__ */ new Set([...defaultGateways.map(getGatewayId), AZURE_OPENAI_GATEWAY_ID]);
+	} catch {
+		return /* @__PURE__ */ new Set([
+			"models.dev",
+			"netlify",
+			"mastra",
+			AZURE_OPENAI_GATEWAY_ID
+		]);
+	}
+}
+function isOffline() {
+	const offline = process.env.MASTRA_OFFLINE;
+	return offline === "true" || offline === "1";
+}
+/**
+* Same switches as the model router's auto-refresh in `@mastra/core`: on by default only under
+* `mastra dev` (`MASTRA_DEV`), opt-in elsewhere, and always off with `MASTRA_OFFLINE`.
+*/
+function isRefreshEnabled() {
+	if (isOffline()) return false;
+	const isDev = process.env.MASTRA_DEV === "true" || process.env.MASTRA_DEV === "1";
+	const setting = process.env.MASTRA_AUTO_REFRESH_PRICING;
+	return setting === "true" || setting !== "false" && isDev;
+}
+/**
+* Start a background refresh from models.dev, unless refresh is off, one is already
+* running, or the last attempt was less than an hour ago.
+*/
+function refreshGlobalPricing() {
+	if (refreshInFlight || !isRefreshEnabled() || Date.now() - lastRefreshAttemptAt < MIN_REFRESH_ATTEMPT_INTERVAL_MS) return;
+	refreshInFlight = true;
+	lastRefreshAttemptAt = Date.now();
+	fetchModelsDevPricing().catch(() => {}).finally(() => {
+		refreshInFlight = false;
+	});
+}
+async function fetchModelsDevPricing() {
+	const response = await fetch(MODELS_DEV_API_URL, {
+		headers: refreshEtag ? { "if-none-match": refreshEtag } : void 0,
+		signal: AbortSignal.timeout(REFRESH_TIMEOUT_MS)
+	});
+	if (response.status === 304) {
+		refreshedAt = Date.now();
+		await touchCacheFile();
+		return;
+	}
+	if (!response.ok) throw new Error(`models.dev responded with ${response.status}`);
+	const costs = pickModelsDevCosts(await response.json());
+	const pricingModels = modelsDevToPricingModels(costs);
+	if (pricingModels.size < 1e3) throw new Error(`models.dev returned ${pricingModels.size} prices, expected at least ${MIN_PRICED_MODELS}`);
+	globalRegistry = new PricingRegistry(pricingModels);
+	refreshedAt = Date.now();
+	refreshEtag = response.headers.get("etag") ?? void 0;
+	await writeCacheFile(JSON.stringify({
+		version: CACHE_VERSION,
+		etag: refreshEtag,
+		costs
+	}));
+}
+/** The cache file's mtime records when its prices were last confirmed against models.dev. */
+function loadCachedPricingModels() {
+	if (!isRefreshEnabled()) return null;
+	try {
+		const cacheFile = CACHE_FILE();
+		const cached = JSON.parse(fs.readFileSync(cacheFile, "utf-8"));
+		if (cached.version !== CACHE_VERSION) return null;
+		const pricingModels = modelsDevToPricingModels(cached.costs);
+		if (pricingModels.size < 1e3) return null;
+		refreshEtag = typeof cached.etag === "string" ? cached.etag : void 0;
+		refreshedAt = fs.statSync(cacheFile).mtimeMs;
+		return pricingModels;
+	} catch {
+		return null;
+	}
+}
+/** Write-to-temp-then-rename, like the model router's cache writes, so readers never see a partial file. */
+async function writeCacheFile(content) {
+	try {
+		const cacheFile = CACHE_FILE();
+		const tempFile = `${cacheFile}.${process.pid}.${Date.now()}.tmp`;
+		await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true });
+		await fs.promises.writeFile(tempFile, content, "utf-8");
+		await fs.promises.rename(tempFile, cacheFile);
+	} catch {}
+}
+async function touchCacheFile() {
+	try {
+		const now = /* @__PURE__ */ new Date();
+		await fs.promises.utimes(CACHE_FILE(), now, now);
+	} catch {}
+}
 function loadPricingModels() {
 	if (cachedLoadError) return null;
 	try {
-		return parsePricingModelText(fs.readFileSync(resolvePricingModelPath(), "utf-8"));
+		return modelsDevToPricingModels(JSON.parse(fs.readFileSync(resolvePricingModelPath(), "utf-8")));
 	} catch (error) {
 		cachedLoadError = error instanceof Error ? error.message : String(error);
 		return null;
 	}
 }
-function parsePricingModelText(content) {
-	const pricingModels = /* @__PURE__ */ new Map();
-	for (const line of content.split("\n")) {
-		const trimmed = line.trim();
-		if (!trimmed) continue;
-		const pricingModel = expandPricingModelRow(JSON.parse(trimmed));
-		pricingModels.set(makePricingKey(pricingModel), pricingModel);
-	}
-	return pricingModels;
-}
-function expandRates(row, tier) {
-	const rates = Object.fromEntries(Object.entries(tier.r).map(([meter, value]) => [MINIFIED_METER_TO_CANONICAL[meter], value.c]));
-	const cacheWriteRate = rates.input_cache_write_tokens;
-	if (row.m.includes("claude") && typeof cacheWriteRate === "number") {
-		rates.input_cache_write_5m_tokens ??= cacheWriteRate;
-		rates.input_cache_write_1h_tokens ??= cacheWriteRate * 1.6;
-	}
-	return rates;
-}
-function expandPricingModelRow(row) {
-	return new PricingModel({
-		id: row.i,
-		provider: row.p,
-		model: row.m,
-		schema: row.s.v,
-		currency: row.s.d.u,
-		tiers: row.s.d.t.map((tier, index) => new PricingTier({
-			index,
-			when: tier.w?.map((condition) => ({
-				field: MINIFIED_CONDITION_FIELD_TO_CANONICAL[condition.f],
-				op: condition.op,
-				value: condition.value
-			})),
-			rates: expandRates(row, tier)
-		}))
-	});
-}
 function resolvePricingModelPath() {
 	const packageRoot = getPackageRoot();
 	const candidates = [
@@ -1259,7 +1513,7 @@ function resolvePricingModelPath() {
 		path.join(process.cwd(), "src", "metrics", DATA_FILE_NAME)
 	];
 	for (const candidate of candidates) if (fs.existsSync(candidate)) return candidate;
-	throw new Error(`Unable to locate pricing data JSONL at any known path: ${candidates.join(", ")}`);
+	throw new Error(`Unable to locate pricing data at any known path: ${candidates.join(", ")}`);
 }
 function getPackageRoot() {
 	try {
@@ -2880,6 +3134,17 @@ function isSpanInternal(spanType, flags) {
 	}
 }
 /**
+* Copies the links whose IDs are valid W3C trace and span IDs (32 and 16 lowercase
+* hex characters, not all zeros); `undefined` when none remain.
+*/
+function validLinks(links) {
+	const valid = links?.filter((link) => /^(?!0+$)[0-9a-f]{32}$/.test(link?.traceId ?? "") && /^(?!0+$)[0-9a-f]{16}$/.test(link?.spanId ?? ""));
+	return valid?.length ? valid.map(({ traceId, spanId }) => ({
+		traceId,
+		spanId
+	})) : void 0;
+}
+/**
 * Get the external parent span ID from CreateSpanOptions.
 *
 * If the parent is internal or excluded from export (`excludeSpanTypes`),
@@ -2929,6 +3194,7 @@ var BaseSpan = class BaseSpan {
 	metadata;
 	requestContext;
 	tags;
+	links;
 	traceState;
 	/** Entity type that created the span (e.g., agent, workflow) */
 	entityType;
@@ -2994,6 +3260,7 @@ var BaseSpan = class BaseSpan {
 			this.#openChildrenOwner = parent;
 		}
 		this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
+		this.links = validLinks(options.links);
 		const entityParent = this.getParentSpan(false);
 		this.entityType = options.entityType ?? entityParent?.entityType;
 		this.entityId = options.entityId ?? entityParent?.entityId;
@@ -3194,6 +3461,7 @@ var BaseSpan = class BaseSpan {
 		const hideInput = this.traceState?.hideInput ?? false;
 		const hideOutput = this.traceState?.hideOutput ?? false;
 		const nestedUnderParent = this.traceState?.nestedUnderParent ?? false;
+		const links = validLinks(this.links);
 		return {
 			id: this.id,
 			traceId: this.traceId,
@@ -3216,6 +3484,7 @@ var BaseSpan = class BaseSpan {
 			parentSpanId: this.getParentSpanId(includeInternalSpans),
 			externalParentSpanId: this.getExportedExternalParentSpanId(includeInternalSpans),
 			...nestedUnderParent ? { nestedUnderParent: true } : {},
+			...links ? { links } : {},
 			...this.isRootSpan && !nestedUnderParent && this.tags?.length ? { tags: this.tags } : {}
 		};
 	}
@@ -3795,6 +4064,8 @@ function resolveTracingStorageStrategy$1(config, observabilityStorage, storageNa
 	return observabilityStrategy.preferred;
 }
 const FLUSH_SUCCEEDED$1 = { failed: false };
+/** Max join/write rounds a single flush() makes before it returns under continuous traffic */
+const MAX_FLUSH_ROUNDS$1 = 3;
 /**
 * @deprecated Use `MastraStorageExporter` from `@mastra/observability` instead.
 * This class is preserved unchanged so existing integrations (including code
@@ -3814,6 +4085,8 @@ var DefaultExporter = class extends BaseExporter {
 	#observabilityStorage;
 	#resolvedStrategy;
 	#flushTimer;
+	/** Storage writes (timer, size trigger, or flush()) that haven't finished yet */
+	#inFlightFlushes = /* @__PURE__ */ new Set();
 	#emitDropEvent;
 	#unsupportedSignals = /* @__PURE__ */ new Set();
 	constructor(config = {}) {
@@ -3888,7 +4161,7 @@ var DefaultExporter = class extends BaseExporter {
 		if (this.#flushTimer) clearTimeout(this.#flushTimer);
 		this.#flushTimer = setTimeout(() => {
 			this.#flushTimer = void 0;
-			this.flushBuffer().catch((error) => {
+			this.trackFlushBuffer().catch((error) => {
 				this.logger.error("Scheduled flush failed", { error: error instanceof Error ? error.message : String(error) });
 			});
 		}, delayMs);
@@ -3918,7 +4191,7 @@ var DefaultExporter = class extends BaseExporter {
 	* Returns the flush promise when flushing so callers can await it.
 	*/
 	async handleBatchedFlush() {
-		if (this.shouldFlush()) await this.flushBuffer();
+		if (this.shouldFlush()) await this.trackFlushBuffer();
 		else if (this.#eventBuffer.totalSize === 1) this.scheduleFlush();
 	}
 	sanitizeDropError(error) {
@@ -4022,6 +4295,15 @@ var DefaultExporter = class extends BaseExporter {
 		}
 	}
 	/**
+	* Runs flushBuffer() and tracks its promise so flush() can wait for writes
+	* that a timer or size trigger started on their own.
+	*/
+	trackFlushBuffer() {
+		const promise = this.flushBuffer().finally(() => this.#inFlightFlushes.delete(promise));
+		this.#inFlightFlushes.add(promise);
+		return promise;
+	}
+	/**
 	* Flushes the current buffer to storage.
 	*
 	* Creates are flushed first, then their span keys are added to allCreatedSpans.
@@ -4180,10 +4462,16 @@ var DefaultExporter = class extends BaseExporter {
 	* are exported before the runtime instance is terminated.
 	*/
 	async flush() {
-		if (this.#eventBuffer.totalSize > 0) {
-			this.logger.debug("Flushing buffered events", { bufferedEvents: this.#eventBuffer.totalSize });
-			await this.flushBuffer();
+		for (let round = 0; round < MAX_FLUSH_ROUNDS$1; round++) {
+			if (this.#inFlightFlushes.size > 0) await Promise.allSettled([...this.#inFlightFlushes]);
+			if (this.#eventBuffer.totalSize > 0) {
+				this.logger.debug("Flushing buffered events", { bufferedEvents: this.#eventBuffer.totalSize });
+				await this.trackFlushBuffer();
+			}
+			if (this.#inFlightFlushes.size === 0) return;
 		}
+		await Promise.allSettled([...this.#inFlightFlushes]);
+		this.logger.warn(`flush() stopped after ${MAX_FLUSH_ROUNDS$1} rounds; new writes kept starting`, { bufferedEventsLeftBehind: this.#eventBuffer.totalSize });
 	}
 	async shutdown() {
 		if (this.#flushTimer) {
@@ -4217,6 +4505,8 @@ function resolveTracingStorageStrategy(config, observabilityStorage, storageName
 	return observabilityStrategy.preferred;
 }
 const FLUSH_SUCCEEDED = { failed: false };
+/** Max join/write rounds a single flush() makes before it returns under continuous traffic */
+const MAX_FLUSH_ROUNDS = 3;
 /**
 * Storage-backed exporter. Buffers observability events and flushes them in
 * batches to the configured ObservabilityStorage backend with retry support.
@@ -4231,6 +4521,8 @@ var MastraStorageExporter = class extends BaseExporter {
 	#observabilityStorage;
 	#resolvedStrategy;
 	#flushTimer;
+	/** Storage writes (timer, size trigger, or flush()) that haven't finished yet */
+	#inFlightFlushes = /* @__PURE__ */ new Set();
 	#emitDropEvent;
 	#unsupportedSignals = /* @__PURE__ */ new Set();
 	constructor(config = {}) {
@@ -4320,7 +4612,7 @@ var MastraStorageExporter = class extends BaseExporter {
 		if (this.#flushTimer) clearTimeout(this.#flushTimer);
 		this.#flushTimer = setTimeout(() => {
 			this.#flushTimer = void 0;
-			this.flushBuffer().catch((error) => {
+			this.trackFlushBuffer().catch((error) => {
 				this.logger.error("Scheduled flush failed", { error: error instanceof Error ? error.message : String(error) });
 			});
 		}, delayMs);
@@ -4350,7 +4642,7 @@ var MastraStorageExporter = class extends BaseExporter {
 	* Returns the flush promise when flushing so callers can await it.
 	*/
 	async handleBatchedFlush() {
-		if (this.shouldFlush()) await this.flushBuffer();
+		if (this.shouldFlush()) await this.trackFlushBuffer();
 		else if (this.#eventBuffer.totalSize === 1) this.scheduleFlush();
 	}
 	sanitizeDropError(error) {
@@ -4454,6 +4746,15 @@ var MastraStorageExporter = class extends BaseExporter {
 		}
 	}
 	/**
+	* Runs flushBuffer() and tracks its promise so flush() can wait for writes
+	* that a timer or size trigger started on their own.
+	*/
+	trackFlushBuffer() {
+		const promise = this.flushBuffer().finally(() => this.#inFlightFlushes.delete(promise));
+		this.#inFlightFlushes.add(promise);
+		return promise;
+	}
+	/**
 	* Flushes the current buffer to storage.
 	*
 	* Creates are flushed first, then their span keys are added to allCreatedSpans.
@@ -4601,10 +4902,16 @@ var MastraStorageExporter = class extends BaseExporter {
 	* are exported before the runtime instance is terminated.
 	*/
 	async flush() {
-		if (this.#eventBuffer.totalSize > 0) {
-			this.logger.debug("Flushing buffered events", { bufferedEvents: this.#eventBuffer.totalSize });
-			await this.flushBuffer();
+		for (let round = 0; round < MAX_FLUSH_ROUNDS; round++) {
+			if (this.#inFlightFlushes.size > 0) await Promise.allSettled([...this.#inFlightFlushes]);
+			if (this.#eventBuffer.totalSize > 0) {
+				this.logger.debug("Flushing buffered events", { bufferedEvents: this.#eventBuffer.totalSize });
+				await this.trackFlushBuffer();
+			}
+			if (this.#inFlightFlushes.size === 0) return;
 		}
+		await Promise.allSettled([...this.#inFlightFlushes]);
+		this.logger.warn(`flush() stopped after ${MAX_FLUSH_ROUNDS} rounds; new writes kept starting`, { bufferedEventsLeftBehind: this.#eventBuffer.totalSize });
 	}
 	async shutdown() {
 		if (this.#flushTimer) {
@@ -4804,6 +5111,7 @@ var BaseObservabilityInstance = class extends MastraBase {
 			spanId: cached.id,
 			parentSpanId: cached.parentSpanId,
 			externalParentSpanId: cached.externalParentSpanId,
+			links: cached.links,
 			startTime: cached.startTime instanceof Date ? cached.startTime : new Date(cached.startTime),
 			input: cached.input,
 			attributes: cached.attributes,
@@ -9725,6 +10033,7 @@ var Observability = class extends MastraBase {
 		const instances = this.listInstances();
 		const { mastra } = options;
 		this.#mastra = mastra;
+		setPricingGateways(() => Object.values(mastra.listGateways?.() ?? {}));
 		const mastraEnvironment = mastra.getEnvironment?.();
 		instances.forEach((instance) => {
 			instance.__setMastraEnvironment?.(mastraEnvironment);