@mastra/observability 1.16.6-alpha.2 → 1.16.6-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/exporters/auth-failure-cooldown.d.ts +1 -1
- package/dist/exporters/auth-failure-cooldown.d.ts.map +1 -1
- package/dist/exporters/mastra-platform.d.ts +21 -0
- package/dist/exporters/mastra-platform.d.ts.map +1 -1
- package/dist/index.cjs +128 -14
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +128 -14
- package/dist/index.js.map +1 -1
- package/dist/model-tracing.d.ts.map +1 -1
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,33 @@
|
|
|
1
1
|
# @mastra/observability
|
|
2
2
|
|
|
3
|
+
## 1.16.6-alpha.4
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Added automatic quota pause to MastraPlatformExporter. When the Mastra platform reports that an organization's observability quota is exhausted (by rejecting publish requests with `402 Payment Required` and the `x-mastra-observability: disabled` header), the exporter now stops uploading telemetry, drops events locally instead of retrying, and periodically probes the platform (honoring the `x-mastra-observability-retry-after` hint, defaulting to every 5 minutes). Exports resume automatically once the platform re-enables observability, with warnings logged on pause and resume. ([#21160](https://github.com/mastra-ai/mastra/pull/21160))
|
|
8
|
+
|
|
9
|
+
No quota-specific configuration is needed — any existing exporter setup gets this behavior automatically:
|
|
10
|
+
|
|
11
|
+
```typescript
|
|
12
|
+
import { MastraPlatformExporter } from '@mastra/observability';
|
|
13
|
+
|
|
14
|
+
const exporter = new MastraPlatformExporter({
|
|
15
|
+
accessToken: process.env.MASTRA_PLATFORM_ACCESS_TOKEN,
|
|
16
|
+
});
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
- Updated dependencies [[`210cb7a`](https://github.com/mastra-ai/mastra/commit/210cb7a167998c7bbf72cb3b93e6eb0563330239), [`5f798b3`](https://github.com/mastra-ai/mastra/commit/5f798b3362e9bdf4d690f85245606e146eef60b9), [`01a2943`](https://github.com/mastra-ai/mastra/commit/01a2943a7d886edefdff072bfa51f055bab54437), [`01a2943`](https://github.com/mastra-ai/mastra/commit/01a2943a7d886edefdff072bfa51f055bab54437), [`25ca73d`](https://github.com/mastra-ai/mastra/commit/25ca73d25dee7ce9f0ca72939e3a505c4db7257e), [`e1cead1`](https://github.com/mastra-ai/mastra/commit/e1cead17b5f3653cf00d2f90cc19b113119c02ba), [`01a2943`](https://github.com/mastra-ai/mastra/commit/01a2943a7d886edefdff072bfa51f055bab54437)]:
|
|
20
|
+
- @mastra/core@1.58.0-alpha.14
|
|
21
|
+
|
|
22
|
+
## 1.16.6-alpha.3
|
|
23
|
+
|
|
24
|
+
### Patch Changes
|
|
25
|
+
|
|
26
|
+
- Preserved the provider response model on completed model inference spans. ([#21154](https://github.com/mastra-ai/mastra/pull/21154))
|
|
27
|
+
|
|
28
|
+
- Updated dependencies [[`2e4624e`](https://github.com/mastra-ai/mastra/commit/2e4624edb6917e61249cb60ee377735e7af7e4a9), [`e6534fa`](https://github.com/mastra-ai/mastra/commit/e6534fab031216f6cb48c4c9907cbfdce9d60bc6), [`7fdcaa6`](https://github.com/mastra-ai/mastra/commit/7fdcaa66105d64290f9b14432a12ec99f39c4d3a), [`cfd0d9e`](https://github.com/mastra-ai/mastra/commit/cfd0d9ec77ec3c69dd96f79cdb579e03d79f22ce), [`d9d93b2`](https://github.com/mastra-ai/mastra/commit/d9d93b25e4a65ad5fa153fa35be7ed149c8d587f)]:
|
|
29
|
+
- @mastra/core@1.58.0-alpha.12
|
|
30
|
+
|
|
3
31
|
## 1.16.6-alpha.2
|
|
4
32
|
|
|
5
33
|
### Patch Changes
|
|
@@ -4,7 +4,7 @@ export declare class AuthFailureError extends Error {
|
|
|
4
4
|
constructor(status: number, cause?: unknown);
|
|
5
5
|
}
|
|
6
6
|
export declare function isAuthFailureError(error: unknown): error is AuthFailureError;
|
|
7
|
-
export declare function fetchWithAuthFailureHandling(url: string, options: RequestInit, maxRetries: number): Promise<
|
|
7
|
+
export declare function fetchWithAuthFailureHandling(url: string, options: RequestInit, maxRetries: number, callerShouldRetry?: (response: Response) => boolean): Promise<Response>;
|
|
8
8
|
export declare class AuthFailureCooldown {
|
|
9
9
|
private readonly exporterName;
|
|
10
10
|
private readonly getLogger;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"auth-failure-cooldown.d.ts","sourceRoot":"","sources":["../../src/exporters/auth-failure-cooldown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AASzD,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAK5C;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAMD,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"auth-failure-cooldown.d.ts","sourceRoot":"","sources":["../../src/exporters/auth-failure-cooldown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AASzD,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAEZ,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAK5C;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAMD,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,GAClD,OAAO,CAAC,QAAQ,CAAC,CAqBnB;AAED,qBAAa,mBAAmB;IAM5B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAN5B,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,2BAA2B,CAAK;gBAGrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,aAAa;IAGjD,OAAO,CAAC,gBAAgB;IAIxB,sBAAsB,IAAI,OAAO;IAIjC,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAS/C,KAAK,IAAI,MAAM;IAUf,aAAa,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;CAwBjG"}
|
|
@@ -21,6 +21,10 @@ export declare class MastraPlatformExporter extends BaseExporter {
|
|
|
21
21
|
private buffer;
|
|
22
22
|
private flushTimer;
|
|
23
23
|
private inFlightFlushes;
|
|
24
|
+
private quotaPaused;
|
|
25
|
+
private quotaProbeTimer;
|
|
26
|
+
private quotaProbeIntervalSeconds;
|
|
27
|
+
private shuttingDown;
|
|
24
28
|
constructor(config?: MastraPlatformExporterConfig);
|
|
25
29
|
protected _exportTracingEvent(event: TracingEvent): Promise<void>;
|
|
26
30
|
onLogEvent(event: LogEvent): Promise<void>;
|
|
@@ -45,7 +49,24 @@ export declare class MastraPlatformExporter extends BaseExporter {
|
|
|
45
49
|
/**
|
|
46
50
|
* Uploads a signal batch to the configured Mastra Observability API using fetchWithRetry.
|
|
47
51
|
*/
|
|
52
|
+
private buildPublishHeaders;
|
|
53
|
+
private buildPublishBody;
|
|
48
54
|
private batchUpload;
|
|
55
|
+
/**
|
|
56
|
+
* Enter the quota-exhausted paused state: drop buffered events, stop the
|
|
57
|
+
* flush timer, and start probing until the collector stops rejecting with
|
|
58
|
+
* 402. The gate is per-org, so a signal on any publish route pauses all
|
|
59
|
+
* five signal types.
|
|
60
|
+
*/
|
|
61
|
+
private enterQuotaPause;
|
|
62
|
+
private scheduleQuotaProbe;
|
|
63
|
+
/**
|
|
64
|
+
* Send an empty spans batch as a free probe. The collector treats an empty
|
|
65
|
+
* batch as a no-op, and an exhausted org still gets the 402 on it. A 402
|
|
66
|
+
* throws out of fetchWithRetry without the Response, so the retry predicate
|
|
67
|
+
* captures it to keep the retry-after hint.
|
|
68
|
+
*/
|
|
69
|
+
private probeQuotaStatus;
|
|
49
70
|
private flushSignalBatch;
|
|
50
71
|
private resetBuffer;
|
|
51
72
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mastra-platform.d.ts","sourceRoot":"","sources":["../../src/exporters/mastra-platform.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,YAAY,EAEZ,QAAQ,EACR,WAAW,EACX,UAAU,EACV,aAAa,EACd,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"mastra-platform.d.ts","sourceRoot":"","sources":["../../src/exporters/mastra-platform.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,YAAY,EAEZ,QAAQ,EACR,WAAW,EACX,UAAU,EACV,aAAa,EACd,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AAEjD,MAAM,WAAW,4BAA6B,SAAQ,kBAAkB;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAkOD,qBAAa,sBAAuB,SAAQ,YAAY;IACtD,IAAI,SAA8B;IAElC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAmC;IAClE,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAsB;IAC1D,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,eAAe,CAA4B;IACnD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAA+B;IACtD,OAAO,CAAC,yBAAyB,CAAwC;IACzE,OAAO,CAAC,YAAY,CAAS;gBAEjB,MAAM,GAAE,4BAAiC;cAyErC,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBjE,UAAU,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1C,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAahD,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9C,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1D,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,mBAAmB;IAO3B,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,UAAU;IAelB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,cAAc;YAMR,mBAAmB;IAYjC,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,aAAa;YAoBP,WAAW;IA4EzB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAO3B,OAAO,CAAC,gBAAgB;YAIV,WAAW;IAoDzB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAsBvB,OAAO,CAAC,kBAAkB;IAkB1B;;;;;OAKG;YACW,gBAAgB;YAoDhB,gBAAgB;IAmC9B,OAAO,CAAC,WAAW;IAUnB;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BtB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAsChC"}
|
package/dist/index.cjs
CHANGED
|
@@ -2081,13 +2081,15 @@ var ModelSpanTracker = class {
|
|
|
2081
2081
|
if (!this.#currentInferenceSpan) return;
|
|
2082
2082
|
const { usage: rawUsage, ...otherOutput } = payload.output;
|
|
2083
2083
|
const usage = extractUsageMetrics(rawUsage, payload.metadata?.providerMetadata);
|
|
2084
|
+
const responseModel = typeof payload.metadata?.modelId === "string" ? payload.metadata.modelId : void 0;
|
|
2084
2085
|
this.#currentInferenceSpan.end({
|
|
2085
2086
|
output: otherOutput,
|
|
2086
2087
|
attributes: {
|
|
2087
2088
|
usage,
|
|
2088
2089
|
finishReason: payload.stepResult.reason,
|
|
2089
2090
|
warnings: payload.stepResult.warnings,
|
|
2090
|
-
completionStartTime: this.#completionStartTime
|
|
2091
|
+
completionStartTime: this.#completionStartTime,
|
|
2092
|
+
...responseModel?.trim() ? { responseModel } : {}
|
|
2091
2093
|
}
|
|
2092
2094
|
});
|
|
2093
2095
|
this.#currentInferenceSpan = void 0;
|
|
@@ -5201,15 +5203,15 @@ function isAuthFailureError(error) {
|
|
|
5201
5203
|
function isAuthFailureStatus(status) {
|
|
5202
5204
|
return AUTH_FAILURE_STATUSES.has(status);
|
|
5203
5205
|
}
|
|
5204
|
-
async function fetchWithAuthFailureHandling(url, options, maxRetries) {
|
|
5206
|
+
async function fetchWithAuthFailureHandling(url, options, maxRetries, callerShouldRetry) {
|
|
5205
5207
|
let authFailureStatus;
|
|
5206
5208
|
try {
|
|
5207
|
-
await (0, _mastra_core_utils.fetchWithRetry)(url, options, maxRetries, { shouldRetryResponse: (response) => {
|
|
5209
|
+
return await (0, _mastra_core_utils.fetchWithRetry)(url, options, maxRetries, { shouldRetryResponse: (response) => {
|
|
5208
5210
|
if (isAuthFailureStatus(response.status)) {
|
|
5209
5211
|
authFailureStatus = response.status;
|
|
5210
5212
|
return false;
|
|
5211
5213
|
}
|
|
5212
|
-
return true;
|
|
5214
|
+
return callerShouldRetry?.(response) ?? true;
|
|
5213
5215
|
} });
|
|
5214
5216
|
} catch (error) {
|
|
5215
5217
|
if (authFailureStatus !== void 0) throw new AuthFailureError(authFailureStatus, error);
|
|
@@ -6339,6 +6341,25 @@ const SIGNAL_PUBLISH_SUFFIXES = {
|
|
|
6339
6341
|
feedback: "/feedback/publish"
|
|
6340
6342
|
};
|
|
6341
6343
|
const DEFAULT_PLATFORM_SPAN_FILTER = (span) => span.type !== _mastra_core_observability.SpanType.MODEL_CHUNK;
|
|
6344
|
+
const QUOTA_EXCEEDED_STATUS = 402;
|
|
6345
|
+
const OBSERVABILITY_STATUS_HEADER = "x-mastra-observability";
|
|
6346
|
+
const OBSERVABILITY_DISABLED_VALUE = "disabled";
|
|
6347
|
+
const OBSERVABILITY_RETRY_AFTER_HEADER = "x-mastra-observability-retry-after";
|
|
6348
|
+
const DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS = 300;
|
|
6349
|
+
const MAX_QUOTA_PROBE_INTERVAL_SECONDS = Math.floor(2147483647 / 1e3);
|
|
6350
|
+
function isObservabilityDisabled(response) {
|
|
6351
|
+
return response.headers.get(OBSERVABILITY_STATUS_HEADER) === OBSERVABILITY_DISABLED_VALUE;
|
|
6352
|
+
}
|
|
6353
|
+
function isQuotaExceededResponse(response) {
|
|
6354
|
+
return response.status === QUOTA_EXCEEDED_STATUS || isObservabilityDisabled(response);
|
|
6355
|
+
}
|
|
6356
|
+
function parseQuotaRetryAfterSeconds(response) {
|
|
6357
|
+
const raw = response.headers.get(OBSERVABILITY_RETRY_AFTER_HEADER);
|
|
6358
|
+
if (!raw || !/^\d+$/.test(raw.trim())) return DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
|
|
6359
|
+
const parsed = Number.parseInt(raw, 10);
|
|
6360
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
|
|
6361
|
+
return Math.min(parsed, MAX_QUOTA_PROBE_INTERVAL_SECONDS);
|
|
6362
|
+
}
|
|
6342
6363
|
const SIGNAL_PUBLISH_SEGMENTS = {
|
|
6343
6364
|
traces: "spans",
|
|
6344
6365
|
logs: "logs",
|
|
@@ -6428,6 +6449,10 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6428
6449
|
buffer;
|
|
6429
6450
|
flushTimer = null;
|
|
6430
6451
|
inFlightFlushes = /* @__PURE__ */ new Set();
|
|
6452
|
+
quotaPaused = false;
|
|
6453
|
+
quotaProbeTimer = null;
|
|
6454
|
+
quotaProbeIntervalSeconds = DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
|
|
6455
|
+
shuttingDown = false;
|
|
6431
6456
|
constructor(config = {}) {
|
|
6432
6457
|
super(config);
|
|
6433
6458
|
if (config.projectId !== void 0 && !VALID_PROJECT_ID.test(config.projectId)) throw createInvalidProjectIdError(config.projectId);
|
|
@@ -6476,31 +6501,31 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6476
6501
|
async _exportTracingEvent(event) {
|
|
6477
6502
|
if (event.type !== _mastra_core_observability.TracingEventType.SPAN_ENDED) return;
|
|
6478
6503
|
if (!DEFAULT_PLATFORM_SPAN_FILTER(event.exportedSpan)) return;
|
|
6479
|
-
if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6504
|
+
if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6480
6505
|
this.addToBuffer(event);
|
|
6481
6506
|
await this.handleBufferedEvent();
|
|
6482
6507
|
}
|
|
6483
6508
|
async onLogEvent(event) {
|
|
6484
6509
|
if (this.isDisabled) return;
|
|
6485
|
-
if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6510
|
+
if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6486
6511
|
this.addLogToBuffer(event);
|
|
6487
6512
|
await this.handleBufferedEvent();
|
|
6488
6513
|
}
|
|
6489
6514
|
async onMetricEvent(event) {
|
|
6490
6515
|
if (this.isDisabled) return;
|
|
6491
|
-
if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6516
|
+
if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6492
6517
|
this.addMetricToBuffer(event);
|
|
6493
6518
|
await this.handleBufferedEvent();
|
|
6494
6519
|
}
|
|
6495
6520
|
async onScoreEvent(event) {
|
|
6496
6521
|
if (this.isDisabled) return;
|
|
6497
|
-
if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6522
|
+
if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6498
6523
|
this.addScoreToBuffer(event);
|
|
6499
6524
|
await this.handleBufferedEvent();
|
|
6500
6525
|
}
|
|
6501
6526
|
async onFeedbackEvent(event) {
|
|
6502
6527
|
if (this.isDisabled) return;
|
|
6503
|
-
if (this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6528
|
+
if (this.quotaPaused || this.authFailureCooldown.dropEventIfCoolingDown()) return;
|
|
6504
6529
|
this.addFeedbackToBuffer(event);
|
|
6505
6530
|
await this.handleBufferedEvent();
|
|
6506
6531
|
}
|
|
@@ -6590,6 +6615,10 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6590
6615
|
this.flushTimer = null;
|
|
6591
6616
|
}
|
|
6592
6617
|
if (this.buffer.totalSize === 0) return;
|
|
6618
|
+
if (this.quotaPaused) {
|
|
6619
|
+
this.resetBuffer();
|
|
6620
|
+
return;
|
|
6621
|
+
}
|
|
6593
6622
|
if (this.authFailureCooldown.dropEventsIfCoolingDown(this.buffer.totalSize)) {
|
|
6594
6623
|
this.resetBuffer();
|
|
6595
6624
|
return;
|
|
@@ -6639,11 +6668,16 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6639
6668
|
/**
|
|
6640
6669
|
* Uploads a signal batch to the configured Mastra Observability API using fetchWithRetry.
|
|
6641
6670
|
*/
|
|
6642
|
-
|
|
6643
|
-
|
|
6671
|
+
buildPublishHeaders() {
|
|
6672
|
+
return {
|
|
6644
6673
|
Authorization: `Bearer ${this.platformConfig.accessToken}`,
|
|
6645
6674
|
"Content-Type": "application/json"
|
|
6646
6675
|
};
|
|
6676
|
+
}
|
|
6677
|
+
buildPublishBody(signal, records) {
|
|
6678
|
+
return JSON.stringify({ [SIGNAL_PUBLISH_SEGMENTS[signal]]: records });
|
|
6679
|
+
}
|
|
6680
|
+
async batchUpload(signal, records) {
|
|
6647
6681
|
const endpointMap = {
|
|
6648
6682
|
traces: this.platformConfig.tracesEndpoint,
|
|
6649
6683
|
logs: this.platformConfig.logsEndpoint,
|
|
@@ -6653,10 +6687,85 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6653
6687
|
};
|
|
6654
6688
|
const options = {
|
|
6655
6689
|
method: "POST",
|
|
6656
|
-
headers,
|
|
6657
|
-
body:
|
|
6690
|
+
headers: this.buildPublishHeaders(),
|
|
6691
|
+
body: this.buildPublishBody(signal, records)
|
|
6658
6692
|
};
|
|
6659
|
-
|
|
6693
|
+
let quotaResponse;
|
|
6694
|
+
let response;
|
|
6695
|
+
try {
|
|
6696
|
+
response = await fetchWithAuthFailureHandling(endpointMap[signal], options, this.platformConfig.maxRetries, (res) => {
|
|
6697
|
+
if (isQuotaExceededResponse(res)) {
|
|
6698
|
+
quotaResponse = res;
|
|
6699
|
+
return false;
|
|
6700
|
+
}
|
|
6701
|
+
return true;
|
|
6702
|
+
});
|
|
6703
|
+
} catch (error) {
|
|
6704
|
+
if (quotaResponse) {
|
|
6705
|
+
this.enterQuotaPause(parseQuotaRetryAfterSeconds(quotaResponse));
|
|
6706
|
+
return;
|
|
6707
|
+
}
|
|
6708
|
+
throw error;
|
|
6709
|
+
}
|
|
6710
|
+
if (isObservabilityDisabled(response)) this.enterQuotaPause(parseQuotaRetryAfterSeconds(response));
|
|
6711
|
+
}
|
|
6712
|
+
/**
|
|
6713
|
+
* Enter the quota-exhausted paused state: drop buffered events, stop the
|
|
6714
|
+
* flush timer, and start probing until the collector stops rejecting with
|
|
6715
|
+
* 402. The gate is per-org, so a signal on any publish route pauses all
|
|
6716
|
+
* five signal types.
|
|
6717
|
+
*/
|
|
6718
|
+
enterQuotaPause(retryAfterSeconds) {
|
|
6719
|
+
this.quotaProbeIntervalSeconds = retryAfterSeconds;
|
|
6720
|
+
if (this.quotaPaused) return;
|
|
6721
|
+
this.quotaPaused = true;
|
|
6722
|
+
if (this.flushTimer) {
|
|
6723
|
+
clearTimeout(this.flushTimer);
|
|
6724
|
+
this.flushTimer = null;
|
|
6725
|
+
}
|
|
6726
|
+
this.resetBuffer();
|
|
6727
|
+
this.logger.warn(`Mastra observability paused: quota exhausted, dropping telemetry and probing every ${retryAfterSeconds}s`);
|
|
6728
|
+
this.scheduleQuotaProbe();
|
|
6729
|
+
}
|
|
6730
|
+
scheduleQuotaProbe() {
|
|
6731
|
+
if (this.shuttingDown) return;
|
|
6732
|
+
if (this.quotaProbeTimer) clearTimeout(this.quotaProbeTimer);
|
|
6733
|
+
this.quotaProbeTimer = setTimeout(() => {
|
|
6734
|
+
this.quotaProbeTimer = null;
|
|
6735
|
+
this.probeQuotaStatus();
|
|
6736
|
+
}, this.quotaProbeIntervalSeconds * 1e3);
|
|
6737
|
+
this.quotaProbeTimer.unref?.();
|
|
6738
|
+
}
|
|
6739
|
+
/**
|
|
6740
|
+
* Send an empty spans batch as a free probe. The collector treats an empty
|
|
6741
|
+
* batch as a no-op, and an exhausted org still gets the 402 on it. A 402
|
|
6742
|
+
* throws out of fetchWithRetry without the Response, so the retry predicate
|
|
6743
|
+
* captures it to keep the retry-after hint.
|
|
6744
|
+
*/
|
|
6745
|
+
async probeQuotaStatus() {
|
|
6746
|
+
if (this.shuttingDown || !this.quotaPaused) return;
|
|
6747
|
+
let quotaResponse;
|
|
6748
|
+
try {
|
|
6749
|
+
const response = await (0, _mastra_core_utils.fetchWithRetry)(this.platformConfig.tracesEndpoint, {
|
|
6750
|
+
method: "POST",
|
|
6751
|
+
headers: this.buildPublishHeaders(),
|
|
6752
|
+
body: this.buildPublishBody("traces", [])
|
|
6753
|
+
}, 1, { shouldRetryResponse: (res) => {
|
|
6754
|
+
if (isQuotaExceededResponse(res)) quotaResponse = res;
|
|
6755
|
+
return false;
|
|
6756
|
+
} });
|
|
6757
|
+
if (this.shuttingDown) return;
|
|
6758
|
+
if (!isObservabilityDisabled(response)) {
|
|
6759
|
+
this.quotaPaused = false;
|
|
6760
|
+
this.quotaProbeIntervalSeconds = DEFAULT_QUOTA_PROBE_INTERVAL_SECONDS;
|
|
6761
|
+
this.logger.warn("Mastra observability resumed: quota restored, exports re-enabled");
|
|
6762
|
+
return;
|
|
6763
|
+
}
|
|
6764
|
+
this.quotaProbeIntervalSeconds = parseQuotaRetryAfterSeconds(response);
|
|
6765
|
+
} catch {
|
|
6766
|
+
if (quotaResponse) this.quotaProbeIntervalSeconds = parseQuotaRetryAfterSeconds(quotaResponse);
|
|
6767
|
+
}
|
|
6768
|
+
this.scheduleQuotaProbe();
|
|
6660
6769
|
}
|
|
6661
6770
|
async flushSignalBatch(signal, records) {
|
|
6662
6771
|
if (records.length === 0) return {
|
|
@@ -6724,11 +6833,16 @@ var MastraPlatformExporter = class extends BaseExporter {
|
|
|
6724
6833
|
}
|
|
6725
6834
|
}
|
|
6726
6835
|
async shutdown() {
|
|
6836
|
+
this.shuttingDown = true;
|
|
6727
6837
|
if (this.isDisabled) return;
|
|
6728
6838
|
if (this.flushTimer) {
|
|
6729
6839
|
clearTimeout(this.flushTimer);
|
|
6730
6840
|
this.flushTimer = null;
|
|
6731
6841
|
}
|
|
6842
|
+
if (this.quotaProbeTimer) {
|
|
6843
|
+
clearTimeout(this.quotaProbeTimer);
|
|
6844
|
+
this.quotaProbeTimer = null;
|
|
6845
|
+
}
|
|
6732
6846
|
try {
|
|
6733
6847
|
await this.flush();
|
|
6734
6848
|
} catch (error) {
|