@mastra/observability 1.16.0 → 1.16.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/exporters/default.d.ts +3 -0
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/mastra-storage.d.ts +3 -0
- package/dist/exporters/mastra-storage.d.ts.map +1 -1
- package/dist/index.cjs +148 -60
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +148 -60
- package/dist/index.js.map +1 -1
- package/package.json +9 -9
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @mastra/observability
|
|
2
2
|
|
|
3
|
+
## 1.16.1-alpha.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Fixed storage exporters to report persistence failures immediately and retry failed batches automatically with the configured exponential backoff. ([#19259](https://github.com/mastra-ai/mastra/pull/19259))
|
|
8
|
+
|
|
9
|
+
- Updated dependencies [[`fb8aea3`](https://github.com/mastra-ai/mastra/commit/fb8aea384291e77311be3a64ee1717320d5c3c73), [`4ce0163`](https://github.com/mastra-ai/mastra/commit/4ce0163dc86e675a86809685c8ce6c49f1aeb87e)]:
|
|
10
|
+
- @mastra/core@1.51.0-alpha.5
|
|
11
|
+
|
|
3
12
|
## 1.16.0
|
|
4
13
|
|
|
5
14
|
### Minor Changes
|
|
@@ -42,6 +42,9 @@ export declare class DefaultExporter extends BaseExporter {
|
|
|
42
42
|
* Schedules a flush using setTimeout
|
|
43
43
|
*/
|
|
44
44
|
private scheduleFlush;
|
|
45
|
+
private retryDelay;
|
|
46
|
+
private retryAttempt;
|
|
47
|
+
private logStorageFailure;
|
|
45
48
|
/**
|
|
46
49
|
* Checks flush triggers and schedules/triggers flush as needed.
|
|
47
50
|
* Called after adding any event to the buffer.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../../src/exporters/default.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,aAAa,EAId,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAwB,sBAAsB,EAAwB,MAAM,sBAAsB,CAAC;AAS/G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC;;;;;;GAMG;AACH,UAAU,qBAAsB,SAAQ,kBAAkB;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,sBAAsB,GAAG,MAAM,CAAC;CAC5C;
|
|
1
|
+
{"version":3,"file":"default.d.ts","sourceRoot":"","sources":["../../src/exporters/default.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,aAAa,EAId,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAwB,sBAAsB,EAAwB,MAAM,sBAAsB,CAAC;AAS/G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC;;;;;;GAMG;AACH,UAAU,qBAAsB,SAAQ,kBAAkB;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,sBAAsB,GAAG,MAAM,CAAC;CAC5C;AAoCD;;;;;;;;GAQG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;IAC/C,IAAI,SAA2C;gBAgBnC,MAAM,GAAE,qBAA0B;IAiB9C;;OAEG;IACG,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqDvD;;OAEG;IACH,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,iBAAiB;IAiBzB;;;;OAIG;YACW,kBAAkB;IAQhC,OAAO,CAAC,iBAAiB;IAgBzB,OAAO,CAAC,QAAQ;IAsBhB;;;;OAIG;YACW,YAAY;IAkC1B;;;OAGG;YACW,gBAAgB;IA+D9B;;;;;;;OAOG;YACW,WAAW;IA+HnB,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAW7D;;;;OAIG;YACW,WAAW;IAOzB;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhD;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpD;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ1D;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAiBhC"}
|
|
@@ -31,6 +31,9 @@ export declare class MastraStorageExporter extends BaseExporter {
|
|
|
31
31
|
* Schedules a flush using setTimeout
|
|
32
32
|
*/
|
|
33
33
|
private scheduleFlush;
|
|
34
|
+
private retryDelay;
|
|
35
|
+
private retryAttempt;
|
|
36
|
+
private logStorageFailure;
|
|
34
37
|
/**
|
|
35
38
|
* Checks flush triggers and schedules/triggers flush as needed.
|
|
36
39
|
* Called after adding any event to the buffer.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mastra-storage.d.ts","sourceRoot":"","sources":["../../src/exporters/mastra-storage.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,aAAa,EAId,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAwB,sBAAsB,EAAwB,MAAM,sBAAsB,CAAC;AAS/G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC,4FAA4F;AAC5F,MAAM,WAAW,2BAA4B,SAAQ,kBAAkB;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,sBAAsB,GAAG,MAAM,CAAC;CAC5C;
|
|
1
|
+
{"version":3,"file":"mastra-storage.d.ts","sourceRoot":"","sources":["../../src/exporters/mastra-storage.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,QAAQ,EACR,UAAU,EACV,aAAa,EAId,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAwB,sBAAsB,EAAwB,MAAM,sBAAsB,CAAC;AAS/G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAItC,4FAA4F;AAC5F,MAAM,WAAW,2BAA4B,SAAQ,kBAAkB;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,sBAAsB,GAAG,MAAM,CAAC;CAC5C;AAoCD;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,YAAY;;IACrD,IAAI,SAA6B;gBAgBrB,MAAM,GAAE,2BAAgC;IAiBpD;;OAEG;IACG,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqDvD;;OAEG;IACH,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,iBAAiB;IAiBzB;;;;OAIG;YACW,kBAAkB;IAQhC,OAAO,CAAC,iBAAiB;IAgBzB,OAAO,CAAC,QAAQ;IAsBhB;;;;OAIG;YACW,YAAY;IAkC1B;;;OAGG;YACW,gBAAgB;IA+D9B;;;;;;;OAOG;YACW,WAAW;IA+HnB,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAW7D;;;;OAIG;YACW,WAAW;IAOzB;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhD;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpD;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ1D;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAiBhC"}
|
package/dist/index.cjs
CHANGED
|
@@ -5954,6 +5954,7 @@ function resolveTracingStorageStrategy(config, observabilityStorage, storageName
|
|
|
5954
5954
|
}
|
|
5955
5955
|
return observabilityStrategy.preferred;
|
|
5956
5956
|
}
|
|
5957
|
+
var FLUSH_SUCCEEDED = { failed: false };
|
|
5957
5958
|
var DefaultExporter = class extends BaseExporter {
|
|
5958
5959
|
name = "mastra-default-observability-exporter";
|
|
5959
5960
|
#config;
|
|
@@ -6048,17 +6049,37 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6048
6049
|
/**
|
|
6049
6050
|
* Schedules a flush using setTimeout
|
|
6050
6051
|
*/
|
|
6051
|
-
scheduleFlush() {
|
|
6052
|
+
scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
|
|
6052
6053
|
if (this.#flushTimer) {
|
|
6053
6054
|
clearTimeout(this.#flushTimer);
|
|
6054
6055
|
}
|
|
6055
6056
|
this.#flushTimer = setTimeout(() => {
|
|
6057
|
+
this.#flushTimer = void 0;
|
|
6056
6058
|
this.flushBuffer().catch((error) => {
|
|
6057
6059
|
this.logger.error("Scheduled flush failed", {
|
|
6058
6060
|
error: error instanceof Error ? error.message : String(error)
|
|
6059
6061
|
});
|
|
6060
6062
|
});
|
|
6061
|
-
},
|
|
6063
|
+
}, delayMs);
|
|
6064
|
+
}
|
|
6065
|
+
retryDelay(retryAttempt) {
|
|
6066
|
+
return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
|
|
6067
|
+
}
|
|
6068
|
+
retryAttempt(events) {
|
|
6069
|
+
return events.reduce((earliestAttempt, event) => {
|
|
6070
|
+
if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
|
|
6071
|
+
return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
|
|
6072
|
+
}, void 0);
|
|
6073
|
+
}
|
|
6074
|
+
logStorageFailure(signal, events, retryAttempt, error) {
|
|
6075
|
+
this.logger.warn("Failed to persist observability events", {
|
|
6076
|
+
signal,
|
|
6077
|
+
eventCount: events.length,
|
|
6078
|
+
retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
|
|
6079
|
+
maxRetries: this.#config.maxRetries,
|
|
6080
|
+
...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
|
|
6081
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6082
|
+
});
|
|
6062
6083
|
}
|
|
6063
6084
|
/**
|
|
6064
6085
|
* Checks flush triggers and schedules/triggers flush as needed.
|
|
@@ -6105,22 +6126,26 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6105
6126
|
* On other errors, re-adds events to the buffer for retry.
|
|
6106
6127
|
*/
|
|
6107
6128
|
async flushCreates(signal, events, storageCall) {
|
|
6108
|
-
if (events.length === 0) return;
|
|
6129
|
+
if (events.length === 0) return FLUSH_SUCCEEDED;
|
|
6109
6130
|
if (this.#unsupportedSignals.has(signal)) {
|
|
6110
6131
|
this.emitDrop(signal, "unsupported-storage", events.length);
|
|
6111
|
-
return;
|
|
6132
|
+
return { failed: true };
|
|
6112
6133
|
}
|
|
6113
6134
|
try {
|
|
6114
6135
|
await storageCall(events);
|
|
6136
|
+
return FLUSH_SUCCEEDED;
|
|
6115
6137
|
} catch (error$1) {
|
|
6116
6138
|
if (error$1 instanceof error.MastraError && error$1.domain === error.ErrorDomain.MASTRA_OBSERVABILITY && error$1.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
6117
6139
|
this.logger.warn(error$1.message);
|
|
6118
6140
|
this.#unsupportedSignals.add(signal);
|
|
6119
6141
|
this.emitDrop(signal, "unsupported-storage", events.length, error$1);
|
|
6120
|
-
|
|
6121
|
-
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
6122
|
-
this.emitDrop(signal, "retry-exhausted", dropped.length, error$1);
|
|
6142
|
+
return { failed: true };
|
|
6123
6143
|
}
|
|
6144
|
+
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
6145
|
+
const retryAttempt = this.retryAttempt(events);
|
|
6146
|
+
this.logStorageFailure(signal, events, retryAttempt, error$1);
|
|
6147
|
+
this.emitDrop(signal, "retry-exhausted", dropped.length, error$1);
|
|
6148
|
+
return { failed: true, retryAttempt };
|
|
6124
6149
|
}
|
|
6125
6150
|
}
|
|
6126
6151
|
/**
|
|
@@ -6129,10 +6154,10 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6129
6154
|
*/
|
|
6130
6155
|
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
6131
6156
|
const deferredCountAtEntry = deferredUpdates.length;
|
|
6132
|
-
if (events.length === 0) return;
|
|
6157
|
+
if (events.length === 0) return FLUSH_SUCCEEDED;
|
|
6133
6158
|
if (this.#unsupportedSignals.has("tracing")) {
|
|
6134
6159
|
this.emitDrop("tracing", "unsupported-storage", events.length);
|
|
6135
|
-
return;
|
|
6160
|
+
return { failed: true };
|
|
6136
6161
|
}
|
|
6137
6162
|
const partials = [];
|
|
6138
6163
|
for (const event of events) {
|
|
@@ -6147,26 +6172,30 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6147
6172
|
deferredUpdates.push(event);
|
|
6148
6173
|
}
|
|
6149
6174
|
}
|
|
6150
|
-
if (partials.length === 0) return;
|
|
6175
|
+
if (partials.length === 0) return FLUSH_SUCCEEDED;
|
|
6151
6176
|
try {
|
|
6152
6177
|
await this.#observabilityStorage.batchUpdateSpans({ records: partials });
|
|
6153
6178
|
if (isEnd) {
|
|
6154
6179
|
this.#eventBuffer.endFinishedSpans({ records: partials });
|
|
6155
6180
|
}
|
|
6181
|
+
return FLUSH_SUCCEEDED;
|
|
6156
6182
|
} catch (error$1) {
|
|
6157
6183
|
if (error$1 instanceof error.MastraError && error$1.domain === error.ErrorDomain.MASTRA_OBSERVABILITY && error$1.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
6158
6184
|
this.logger.warn(error$1.message);
|
|
6159
6185
|
this.#unsupportedSignals.add("tracing");
|
|
6160
6186
|
deferredUpdates.length = 0;
|
|
6161
6187
|
this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error$1);
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
6165
|
-
|
|
6166
|
-
|
|
6167
|
-
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
6168
|
-
this.emitDrop("tracing", "retry-exhausted", dropped.length, error$1);
|
|
6188
|
+
return { failed: true };
|
|
6189
|
+
}
|
|
6190
|
+
const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
|
|
6191
|
+
if (newlyDeferred > 0) {
|
|
6192
|
+
deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
|
|
6169
6193
|
}
|
|
6194
|
+
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
6195
|
+
const retryAttempt = this.retryAttempt(events);
|
|
6196
|
+
this.logStorageFailure("tracing", events, retryAttempt, error$1);
|
|
6197
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length, error$1);
|
|
6198
|
+
return { failed: true, retryAttempt };
|
|
6170
6199
|
}
|
|
6171
6200
|
}
|
|
6172
6201
|
/**
|
|
@@ -6234,7 +6263,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6234
6263
|
break;
|
|
6235
6264
|
}
|
|
6236
6265
|
}
|
|
6237
|
-
await Promise.all([
|
|
6266
|
+
const flushResults = await Promise.all([
|
|
6238
6267
|
this.flushCreates(
|
|
6239
6268
|
"feedback",
|
|
6240
6269
|
createFeedbackEvents,
|
|
@@ -6262,25 +6291,36 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6262
6291
|
})
|
|
6263
6292
|
]);
|
|
6264
6293
|
const deferredUpdates = [];
|
|
6265
|
-
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
6266
|
-
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
6294
|
+
flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
|
|
6295
|
+
flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
|
|
6296
|
+
let deferredDropCount = 0;
|
|
6267
6297
|
if (deferredUpdates.length > 0) {
|
|
6268
6298
|
if (this.#unsupportedSignals.has("tracing")) {
|
|
6269
|
-
|
|
6299
|
+
deferredDropCount = deferredUpdates.length;
|
|
6300
|
+
this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
|
|
6270
6301
|
deferredUpdates.length = 0;
|
|
6271
6302
|
} else {
|
|
6272
6303
|
const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
6273
|
-
|
|
6304
|
+
deferredDropCount = dropped.length;
|
|
6305
|
+
this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
|
|
6274
6306
|
}
|
|
6275
6307
|
}
|
|
6276
|
-
const
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
}
|
|
6283
|
-
|
|
6308
|
+
const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
|
|
6309
|
+
if (result.retryAttempt === void 0) return earliestAttempt;
|
|
6310
|
+
return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
|
|
6311
|
+
}, void 0);
|
|
6312
|
+
if (retryAttempt !== void 0) {
|
|
6313
|
+
this.scheduleFlush(this.retryDelay(retryAttempt));
|
|
6314
|
+
}
|
|
6315
|
+
if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
|
|
6316
|
+
const elapsed = Date.now() - startTime;
|
|
6317
|
+
this.logger.debug("Batch flushed", {
|
|
6318
|
+
strategy: this.#resolvedStrategy,
|
|
6319
|
+
batchSize,
|
|
6320
|
+
durationMs: elapsed,
|
|
6321
|
+
deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
|
|
6322
|
+
});
|
|
6323
|
+
}
|
|
6284
6324
|
}
|
|
6285
6325
|
async _exportTracingEvent(event) {
|
|
6286
6326
|
await this.waitForInit();
|
|
@@ -6357,6 +6397,10 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
6357
6397
|
this.#flushTimer = void 0;
|
|
6358
6398
|
}
|
|
6359
6399
|
await this.flush();
|
|
6400
|
+
if (this.#flushTimer) {
|
|
6401
|
+
clearTimeout(this.#flushTimer);
|
|
6402
|
+
this.#flushTimer = void 0;
|
|
6403
|
+
}
|
|
6360
6404
|
this.logger.info("DefaultExporter shutdown complete");
|
|
6361
6405
|
}
|
|
6362
6406
|
};
|
|
@@ -6889,6 +6933,7 @@ function resolveTracingStorageStrategy2(config, observabilityStorage, storageNam
|
|
|
6889
6933
|
}
|
|
6890
6934
|
return observabilityStrategy.preferred;
|
|
6891
6935
|
}
|
|
6936
|
+
var FLUSH_SUCCEEDED2 = { failed: false };
|
|
6892
6937
|
var MastraStorageExporter = class extends BaseExporter {
|
|
6893
6938
|
name = "mastra-storage-exporter";
|
|
6894
6939
|
#config;
|
|
@@ -6983,17 +7028,37 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
6983
7028
|
/**
|
|
6984
7029
|
* Schedules a flush using setTimeout
|
|
6985
7030
|
*/
|
|
6986
|
-
scheduleFlush() {
|
|
7031
|
+
scheduleFlush(delayMs = this.#config.maxBatchWaitMs) {
|
|
6987
7032
|
if (this.#flushTimer) {
|
|
6988
7033
|
clearTimeout(this.#flushTimer);
|
|
6989
7034
|
}
|
|
6990
7035
|
this.#flushTimer = setTimeout(() => {
|
|
7036
|
+
this.#flushTimer = void 0;
|
|
6991
7037
|
this.flushBuffer().catch((error) => {
|
|
6992
7038
|
this.logger.error("Scheduled flush failed", {
|
|
6993
7039
|
error: error instanceof Error ? error.message : String(error)
|
|
6994
7040
|
});
|
|
6995
7041
|
});
|
|
6996
|
-
},
|
|
7042
|
+
}, delayMs);
|
|
7043
|
+
}
|
|
7044
|
+
retryDelay(retryAttempt) {
|
|
7045
|
+
return this.#config.retryDelayMs * 2 ** (retryAttempt - 1);
|
|
7046
|
+
}
|
|
7047
|
+
retryAttempt(events) {
|
|
7048
|
+
return events.reduce((earliestAttempt, event) => {
|
|
7049
|
+
if (event.retryCount > this.#config.maxRetries) return earliestAttempt;
|
|
7050
|
+
return earliestAttempt === void 0 ? event.retryCount : Math.min(earliestAttempt, event.retryCount);
|
|
7051
|
+
}, void 0);
|
|
7052
|
+
}
|
|
7053
|
+
logStorageFailure(signal, events, retryAttempt, error) {
|
|
7054
|
+
this.logger.warn("Failed to persist observability events", {
|
|
7055
|
+
signal,
|
|
7056
|
+
eventCount: events.length,
|
|
7057
|
+
retryAttempt: retryAttempt ?? events.reduce((latestAttempt, event) => Math.max(latestAttempt, event.retryCount), 0),
|
|
7058
|
+
maxRetries: this.#config.maxRetries,
|
|
7059
|
+
...retryAttempt === void 0 ? {} : { nextRetryDelayMs: this.retryDelay(retryAttempt) },
|
|
7060
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7061
|
+
});
|
|
6997
7062
|
}
|
|
6998
7063
|
/**
|
|
6999
7064
|
* Checks flush triggers and schedules/triggers flush as needed.
|
|
@@ -7040,22 +7105,26 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7040
7105
|
* On other errors, re-adds events to the buffer for retry.
|
|
7041
7106
|
*/
|
|
7042
7107
|
async flushCreates(signal, events, storageCall) {
|
|
7043
|
-
if (events.length === 0) return;
|
|
7108
|
+
if (events.length === 0) return FLUSH_SUCCEEDED2;
|
|
7044
7109
|
if (this.#unsupportedSignals.has(signal)) {
|
|
7045
7110
|
this.emitDrop(signal, "unsupported-storage", events.length);
|
|
7046
|
-
return;
|
|
7111
|
+
return { failed: true };
|
|
7047
7112
|
}
|
|
7048
7113
|
try {
|
|
7049
7114
|
await storageCall(events);
|
|
7115
|
+
return FLUSH_SUCCEEDED2;
|
|
7050
7116
|
} catch (error$1) {
|
|
7051
7117
|
if (error$1 instanceof error.MastraError && error$1.domain === error.ErrorDomain.MASTRA_OBSERVABILITY && error$1.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
7052
7118
|
this.logger.warn(error$1.message);
|
|
7053
7119
|
this.#unsupportedSignals.add(signal);
|
|
7054
7120
|
this.emitDrop(signal, "unsupported-storage", events.length, error$1);
|
|
7055
|
-
|
|
7056
|
-
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
7057
|
-
this.emitDrop(signal, "retry-exhausted", dropped.length, error$1);
|
|
7121
|
+
return { failed: true };
|
|
7058
7122
|
}
|
|
7123
|
+
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
7124
|
+
const retryAttempt = this.retryAttempt(events);
|
|
7125
|
+
this.logStorageFailure(signal, events, retryAttempt, error$1);
|
|
7126
|
+
this.emitDrop(signal, "retry-exhausted", dropped.length, error$1);
|
|
7127
|
+
return { failed: true, retryAttempt };
|
|
7059
7128
|
}
|
|
7060
7129
|
}
|
|
7061
7130
|
/**
|
|
@@ -7064,10 +7133,10 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7064
7133
|
*/
|
|
7065
7134
|
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
7066
7135
|
const deferredCountAtEntry = deferredUpdates.length;
|
|
7067
|
-
if (events.length === 0) return;
|
|
7136
|
+
if (events.length === 0) return FLUSH_SUCCEEDED2;
|
|
7068
7137
|
if (this.#unsupportedSignals.has("tracing")) {
|
|
7069
7138
|
this.emitDrop("tracing", "unsupported-storage", events.length);
|
|
7070
|
-
return;
|
|
7139
|
+
return { failed: true };
|
|
7071
7140
|
}
|
|
7072
7141
|
const partials = [];
|
|
7073
7142
|
for (const event of events) {
|
|
@@ -7082,26 +7151,30 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7082
7151
|
deferredUpdates.push(event);
|
|
7083
7152
|
}
|
|
7084
7153
|
}
|
|
7085
|
-
if (partials.length === 0) return;
|
|
7154
|
+
if (partials.length === 0) return FLUSH_SUCCEEDED2;
|
|
7086
7155
|
try {
|
|
7087
7156
|
await this.#observabilityStorage.batchUpdateSpans({ records: partials });
|
|
7088
7157
|
if (isEnd) {
|
|
7089
7158
|
this.#eventBuffer.endFinishedSpans({ records: partials });
|
|
7090
7159
|
}
|
|
7160
|
+
return FLUSH_SUCCEEDED2;
|
|
7091
7161
|
} catch (error$1) {
|
|
7092
7162
|
if (error$1 instanceof error.MastraError && error$1.domain === error.ErrorDomain.MASTRA_OBSERVABILITY && error$1.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
7093
7163
|
this.logger.warn(error$1.message);
|
|
7094
7164
|
this.#unsupportedSignals.add("tracing");
|
|
7095
7165
|
deferredUpdates.length = 0;
|
|
7096
7166
|
this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error$1);
|
|
7097
|
-
|
|
7098
|
-
|
|
7099
|
-
|
|
7100
|
-
|
|
7101
|
-
|
|
7102
|
-
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
7103
|
-
this.emitDrop("tracing", "retry-exhausted", dropped.length, error$1);
|
|
7167
|
+
return { failed: true };
|
|
7168
|
+
}
|
|
7169
|
+
const newlyDeferred = deferredUpdates.length - deferredCountAtEntry;
|
|
7170
|
+
if (newlyDeferred > 0) {
|
|
7171
|
+
deferredUpdates.splice(deferredUpdates.length - newlyDeferred, newlyDeferred);
|
|
7104
7172
|
}
|
|
7173
|
+
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
7174
|
+
const retryAttempt = this.retryAttempt(events);
|
|
7175
|
+
this.logStorageFailure("tracing", events, retryAttempt, error$1);
|
|
7176
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length, error$1);
|
|
7177
|
+
return { failed: true, retryAttempt };
|
|
7105
7178
|
}
|
|
7106
7179
|
}
|
|
7107
7180
|
/**
|
|
@@ -7169,7 +7242,7 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7169
7242
|
break;
|
|
7170
7243
|
}
|
|
7171
7244
|
}
|
|
7172
|
-
await Promise.all([
|
|
7245
|
+
const flushResults = await Promise.all([
|
|
7173
7246
|
this.flushCreates(
|
|
7174
7247
|
"feedback",
|
|
7175
7248
|
createFeedbackEvents,
|
|
@@ -7197,25 +7270,36 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7197
7270
|
})
|
|
7198
7271
|
]);
|
|
7199
7272
|
const deferredUpdates = [];
|
|
7200
|
-
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
7201
|
-
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
7273
|
+
flushResults.push(await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false));
|
|
7274
|
+
flushResults.push(await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true));
|
|
7275
|
+
let deferredDropCount = 0;
|
|
7202
7276
|
if (deferredUpdates.length > 0) {
|
|
7203
7277
|
if (this.#unsupportedSignals.has("tracing")) {
|
|
7204
|
-
|
|
7278
|
+
deferredDropCount = deferredUpdates.length;
|
|
7279
|
+
this.emitDrop("tracing", "unsupported-storage", deferredDropCount);
|
|
7205
7280
|
deferredUpdates.length = 0;
|
|
7206
7281
|
} else {
|
|
7207
7282
|
const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
7208
|
-
|
|
7283
|
+
deferredDropCount = dropped.length;
|
|
7284
|
+
this.emitDrop("tracing", "retry-exhausted", deferredDropCount);
|
|
7209
7285
|
}
|
|
7210
7286
|
}
|
|
7211
|
-
const
|
|
7212
|
-
|
|
7213
|
-
|
|
7214
|
-
|
|
7215
|
-
|
|
7216
|
-
|
|
7217
|
-
}
|
|
7218
|
-
|
|
7287
|
+
const retryAttempt = flushResults.reduce((earliestAttempt, result) => {
|
|
7288
|
+
if (result.retryAttempt === void 0) return earliestAttempt;
|
|
7289
|
+
return earliestAttempt === void 0 ? result.retryAttempt : Math.min(earliestAttempt, result.retryAttempt);
|
|
7290
|
+
}, void 0);
|
|
7291
|
+
if (retryAttempt !== void 0) {
|
|
7292
|
+
this.scheduleFlush(this.retryDelay(retryAttempt));
|
|
7293
|
+
}
|
|
7294
|
+
if (!flushResults.some((result) => result.failed) && deferredDropCount === 0) {
|
|
7295
|
+
const elapsed = Date.now() - startTime;
|
|
7296
|
+
this.logger.debug("Batch flushed", {
|
|
7297
|
+
strategy: this.#resolvedStrategy,
|
|
7298
|
+
batchSize,
|
|
7299
|
+
durationMs: elapsed,
|
|
7300
|
+
deferredUpdates: deferredUpdates.length > 0 ? deferredUpdates.length : void 0
|
|
7301
|
+
});
|
|
7302
|
+
}
|
|
7219
7303
|
}
|
|
7220
7304
|
async _exportTracingEvent(event) {
|
|
7221
7305
|
await this.waitForInit();
|
|
@@ -7292,6 +7376,10 @@ var MastraStorageExporter = class extends BaseExporter {
|
|
|
7292
7376
|
this.#flushTimer = void 0;
|
|
7293
7377
|
}
|
|
7294
7378
|
await this.flush();
|
|
7379
|
+
if (this.#flushTimer) {
|
|
7380
|
+
clearTimeout(this.#flushTimer);
|
|
7381
|
+
this.#flushTimer = void 0;
|
|
7382
|
+
}
|
|
7295
7383
|
this.logger.info("MastraStorageExporter shutdown complete");
|
|
7296
7384
|
}
|
|
7297
7385
|
};
|