@mastra/observability 1.11.1 → 1.12.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +3 -2
- package/dist/bus/observability-bus.d.ts +21 -4
- package/dist/bus/observability-bus.d.ts.map +1 -1
- package/dist/bus/route-event.d.ts +8 -1
- package/dist/bus/route-event.d.ts.map +1 -1
- package/dist/config.d.ts +28 -6
- package/dist/config.d.ts.map +1 -1
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +2 -0
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/event-buffer.d.ts +4 -4
- package/dist/exporters/event-buffer.d.ts.map +1 -1
- package/dist/features.d.ts +39 -0
- package/dist/features.d.ts.map +1 -0
- package/dist/index.cjs +344 -48
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +344 -49
- package/dist/index.js.map +1 -1
- package/dist/model-tracing.d.ts +34 -4
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/recorded.d.ts.map +1 -1
- package/dist/spans/base.d.ts.map +1 -1
- package/package.json +6 -6
package/dist/index.cjs
CHANGED
|
@@ -10,6 +10,7 @@ var fs = require('fs');
|
|
|
10
10
|
var module$1 = require('module');
|
|
11
11
|
var path = require('path');
|
|
12
12
|
var web = require('stream/web');
|
|
13
|
+
var features = require('@mastra/core/features');
|
|
13
14
|
|
|
14
15
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
15
16
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
@@ -45,6 +46,23 @@ function routeToHandler(handler, event, logger) {
|
|
|
45
46
|
if (handler.onScoreEvent) {
|
|
46
47
|
return catchAsyncResult(handler.onScoreEvent(event), handler.name, "score", logger);
|
|
47
48
|
}
|
|
49
|
+
if (handler.addScoreToTrace) {
|
|
50
|
+
const score = event.score;
|
|
51
|
+
if (!score.traceId) break;
|
|
52
|
+
return catchAsyncResult(
|
|
53
|
+
handler.addScoreToTrace({
|
|
54
|
+
traceId: score.traceId,
|
|
55
|
+
...score.spanId ? { spanId: score.spanId } : {},
|
|
56
|
+
score: score.score,
|
|
57
|
+
...score.reason ? { reason: score.reason } : {},
|
|
58
|
+
scorerName: score.scorerName ?? score.scorerId,
|
|
59
|
+
...score.metadata ? { metadata: score.metadata } : {}
|
|
60
|
+
}),
|
|
61
|
+
handler.name,
|
|
62
|
+
"score",
|
|
63
|
+
logger
|
|
64
|
+
);
|
|
65
|
+
}
|
|
48
66
|
break;
|
|
49
67
|
case "feedback":
|
|
50
68
|
if (handler.onFeedbackEvent) {
|
|
@@ -56,6 +74,15 @@ function routeToHandler(handler, event, logger) {
|
|
|
56
74
|
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
57
75
|
}
|
|
58
76
|
}
|
|
77
|
+
function routeDropToHandler(handler, event, logger) {
|
|
78
|
+
try {
|
|
79
|
+
if (handler.onDroppedEvent) {
|
|
80
|
+
return catchAsyncResult(handler.onDroppedEvent(event), handler.name, "drop", logger);
|
|
81
|
+
}
|
|
82
|
+
} catch (err) {
|
|
83
|
+
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
59
86
|
function catchAsyncResult(result, handlerName, signal, logger) {
|
|
60
87
|
if (result && typeof result.then === "function") {
|
|
61
88
|
return result.catch((err) => {
|
|
@@ -13905,12 +13932,18 @@ var observabilityConfigValueSchema = external_exports.object(observabilityInstan
|
|
|
13905
13932
|
message: "At least one exporter or a bridge is required"
|
|
13906
13933
|
}
|
|
13907
13934
|
);
|
|
13935
|
+
var sensitiveDataFilterOptionsSchema = external_exports.object({
|
|
13936
|
+
sensitiveFields: external_exports.array(external_exports.string()).optional(),
|
|
13937
|
+
redactionToken: external_exports.string().optional(),
|
|
13938
|
+
redactionStyle: external_exports.enum(["full", "partial"]).optional()
|
|
13939
|
+
}).strict();
|
|
13908
13940
|
var observabilityRegistryConfigSchema = external_exports.object({
|
|
13909
13941
|
default: external_exports.object({
|
|
13910
13942
|
enabled: external_exports.boolean().optional()
|
|
13911
13943
|
}).optional().nullable(),
|
|
13912
13944
|
configs: external_exports.union([external_exports.record(external_exports.string(), external_exports.any()), external_exports.array(external_exports.any()), external_exports.null()]).optional(),
|
|
13913
|
-
configSelector: external_exports.function().optional()
|
|
13945
|
+
configSelector: external_exports.function().optional(),
|
|
13946
|
+
sensitiveDataFilter: external_exports.union([external_exports.boolean(), sensitiveDataFilterOptionsSchema]).optional()
|
|
13914
13947
|
}).passthrough().refine(
|
|
13915
13948
|
(data) => {
|
|
13916
13949
|
const isDefaultEnabled = data.default?.enabled === true;
|
|
@@ -15680,31 +15713,39 @@ var EventBuffer = class {
|
|
|
15680
15713
|
break;
|
|
15681
15714
|
}
|
|
15682
15715
|
}
|
|
15683
|
-
/** Re-add failed create events to the buffer,
|
|
15716
|
+
/** Re-add failed create events to the buffer, returning events that exceed max retries. */
|
|
15684
15717
|
reAddCreates(events) {
|
|
15685
15718
|
const retryable = [];
|
|
15719
|
+
const dropped = [];
|
|
15686
15720
|
for (const e of events) {
|
|
15687
15721
|
if (++e.retryCount <= this.#maxRetries) {
|
|
15688
15722
|
retryable.push(e);
|
|
15723
|
+
} else {
|
|
15724
|
+
dropped.push(e);
|
|
15689
15725
|
}
|
|
15690
15726
|
}
|
|
15691
15727
|
if (retryable.length > 0) {
|
|
15692
15728
|
this.setFirstEventTime();
|
|
15693
15729
|
this.#creates.push(...retryable);
|
|
15694
15730
|
}
|
|
15731
|
+
return dropped;
|
|
15695
15732
|
}
|
|
15696
|
-
/** Re-add failed update events to the buffer,
|
|
15733
|
+
/** Re-add failed update events to the buffer, returning events that exceed max retries. */
|
|
15697
15734
|
reAddUpdates(events) {
|
|
15698
15735
|
const retryable = [];
|
|
15736
|
+
const dropped = [];
|
|
15699
15737
|
for (const e of events) {
|
|
15700
15738
|
if (++e.retryCount <= this.#maxRetries) {
|
|
15701
15739
|
retryable.push(e);
|
|
15740
|
+
} else {
|
|
15741
|
+
dropped.push(e);
|
|
15702
15742
|
}
|
|
15703
15743
|
}
|
|
15704
15744
|
if (retryable.length > 0) {
|
|
15705
15745
|
this.setFirstEventTime();
|
|
15706
15746
|
this.#updates.push(...retryable);
|
|
15707
15747
|
}
|
|
15748
|
+
return dropped;
|
|
15708
15749
|
}
|
|
15709
15750
|
/** Snapshot of buffered create events. */
|
|
15710
15751
|
get creates() {
|
|
@@ -15783,6 +15824,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15783
15824
|
#observabilityStorage;
|
|
15784
15825
|
#resolvedStrategy;
|
|
15785
15826
|
#flushTimer;
|
|
15827
|
+
#emitDropEvent;
|
|
15786
15828
|
// Signals whose storage methods threw "not implemented" — skip on future flushes
|
|
15787
15829
|
#unsupportedSignals = /* @__PURE__ */ new Set();
|
|
15788
15830
|
constructor(config2 = {}) {
|
|
@@ -15804,6 +15846,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15804
15846
|
async init(options) {
|
|
15805
15847
|
try {
|
|
15806
15848
|
this.#isInitializing = true;
|
|
15849
|
+
this.#emitDropEvent = options.emitDropEvent;
|
|
15807
15850
|
this.#storage = options.mastra?.getStorage();
|
|
15808
15851
|
if (!this.#storage) {
|
|
15809
15852
|
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
@@ -15889,21 +15932,54 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15889
15932
|
this.scheduleFlush();
|
|
15890
15933
|
}
|
|
15891
15934
|
}
|
|
15935
|
+
sanitizeDropError(error48) {
|
|
15936
|
+
if (error48 instanceof error$1.MastraError) {
|
|
15937
|
+
return {
|
|
15938
|
+
id: error48.id,
|
|
15939
|
+
domain: String(error48.domain),
|
|
15940
|
+
message: error48.message
|
|
15941
|
+
};
|
|
15942
|
+
}
|
|
15943
|
+
if (error48 instanceof Error) {
|
|
15944
|
+
return { message: error48.message };
|
|
15945
|
+
}
|
|
15946
|
+
return { message: String(error48) };
|
|
15947
|
+
}
|
|
15948
|
+
emitDrop(signal, reason, count, error48) {
|
|
15949
|
+
if (count === 0) return;
|
|
15950
|
+
const dropEvent = {
|
|
15951
|
+
type: "drop",
|
|
15952
|
+
signal,
|
|
15953
|
+
reason,
|
|
15954
|
+
count,
|
|
15955
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
15956
|
+
exporterName: this.name,
|
|
15957
|
+
...this.#observabilityStorage ? { storageName: this.#observabilityStorage.constructor.name } : {},
|
|
15958
|
+
...error48 === void 0 ? {} : { error: this.sanitizeDropError(error48) }
|
|
15959
|
+
};
|
|
15960
|
+
this.#emitDropEvent?.(dropEvent);
|
|
15961
|
+
}
|
|
15892
15962
|
/**
|
|
15893
15963
|
* Flush a batch of create events for a single signal type.
|
|
15894
15964
|
* On "not implemented" errors, disables the signal for future flushes.
|
|
15895
15965
|
* On other errors, re-adds events to the buffer for retry.
|
|
15896
15966
|
*/
|
|
15897
15967
|
async flushCreates(signal, events, storageCall) {
|
|
15898
|
-
if (
|
|
15968
|
+
if (events.length === 0) return;
|
|
15969
|
+
if (this.#unsupportedSignals.has(signal)) {
|
|
15970
|
+
this.emitDrop(signal, "unsupported-storage", events.length);
|
|
15971
|
+
return;
|
|
15972
|
+
}
|
|
15899
15973
|
try {
|
|
15900
15974
|
await storageCall(events);
|
|
15901
15975
|
} catch (error48) {
|
|
15902
15976
|
if (error48 instanceof error$1.MastraError && error48.domain === error$1.ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15903
15977
|
this.logger.warn(error48.message);
|
|
15904
15978
|
this.#unsupportedSignals.add(signal);
|
|
15979
|
+
this.emitDrop(signal, "unsupported-storage", events.length, error48);
|
|
15905
15980
|
} else {
|
|
15906
|
-
this.#eventBuffer.reAddCreates(events);
|
|
15981
|
+
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
15982
|
+
this.emitDrop(signal, "retry-exhausted", dropped.length, error48);
|
|
15907
15983
|
}
|
|
15908
15984
|
}
|
|
15909
15985
|
}
|
|
@@ -15912,7 +15988,12 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15912
15988
|
* When `isEnd` is true, successfully flushed spans are removed from tracking.
|
|
15913
15989
|
*/
|
|
15914
15990
|
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
15915
|
-
|
|
15991
|
+
const deferredCountAtEntry = deferredUpdates.length;
|
|
15992
|
+
if (events.length === 0) return;
|
|
15993
|
+
if (this.#unsupportedSignals.has("tracing")) {
|
|
15994
|
+
this.emitDrop("tracing", "unsupported-storage", events.length);
|
|
15995
|
+
return;
|
|
15996
|
+
}
|
|
15916
15997
|
const partials = [];
|
|
15917
15998
|
for (const event of events) {
|
|
15918
15999
|
const span = event.exportedSpan;
|
|
@@ -15936,9 +16017,12 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15936
16017
|
if (error48 instanceof error$1.MastraError && error48.domain === error$1.ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15937
16018
|
this.logger.warn(error48.message);
|
|
15938
16019
|
this.#unsupportedSignals.add("tracing");
|
|
16020
|
+
deferredUpdates.length = 0;
|
|
16021
|
+
this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error48);
|
|
15939
16022
|
} else {
|
|
15940
16023
|
deferredUpdates.length = 0;
|
|
15941
|
-
this.#eventBuffer.reAddUpdates(events);
|
|
16024
|
+
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
16025
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length, error48);
|
|
15942
16026
|
}
|
|
15943
16027
|
}
|
|
15944
16028
|
}
|
|
@@ -16014,17 +16098,17 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
16014
16098
|
(events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => storage.buildFeedbackRecord(f)) })
|
|
16015
16099
|
),
|
|
16016
16100
|
this.flushCreates(
|
|
16017
|
-
"
|
|
16101
|
+
"log",
|
|
16018
16102
|
createLogEvents,
|
|
16019
16103
|
(events) => this.#observabilityStorage.batchCreateLogs({ logs: events.map((l) => storage.buildLogRecord(l)) })
|
|
16020
16104
|
),
|
|
16021
16105
|
this.flushCreates(
|
|
16022
|
-
"
|
|
16106
|
+
"metric",
|
|
16023
16107
|
createMetricEvents,
|
|
16024
16108
|
(events) => this.#observabilityStorage.batchCreateMetrics({ metrics: events.map((m) => storage.buildMetricRecord(m)) })
|
|
16025
16109
|
),
|
|
16026
16110
|
this.flushCreates(
|
|
16027
|
-
"
|
|
16111
|
+
"score",
|
|
16028
16112
|
createScoreEvents,
|
|
16029
16113
|
(events) => this.#observabilityStorage.batchCreateScores({ scores: events.map((s) => storage.buildScoreRecord(s)) })
|
|
16030
16114
|
),
|
|
@@ -16038,7 +16122,13 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
16038
16122
|
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
16039
16123
|
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
16040
16124
|
if (deferredUpdates.length > 0) {
|
|
16041
|
-
this.#
|
|
16125
|
+
if (this.#unsupportedSignals.has("tracing")) {
|
|
16126
|
+
this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
|
|
16127
|
+
deferredUpdates.length = 0;
|
|
16128
|
+
} else {
|
|
16129
|
+
const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
16130
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length);
|
|
16131
|
+
}
|
|
16042
16132
|
}
|
|
16043
16133
|
const elapsed = Date.now() - startTime;
|
|
16044
16134
|
this.logger.debug("Batch flushed", {
|
|
@@ -17667,6 +17757,8 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17667
17757
|
bridge;
|
|
17668
17758
|
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
17669
17759
|
pendingHandlers = /* @__PURE__ */ new Set();
|
|
17760
|
+
handlerBufferFlushDepth = 0;
|
|
17761
|
+
dropEventsEmittedDuringHandlerFlush = 0;
|
|
17670
17762
|
/** Resolved deepClean options applied to non-tracing events before fan-out. */
|
|
17671
17763
|
deepCleanOptions;
|
|
17672
17764
|
constructor(opts) {
|
|
@@ -17753,6 +17845,23 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17753
17845
|
}
|
|
17754
17846
|
super.emit(cleaned);
|
|
17755
17847
|
}
|
|
17848
|
+
/**
|
|
17849
|
+
* Emit exporter pipeline drop events to exporters and the bridge.
|
|
17850
|
+
*
|
|
17851
|
+
* Drop events describe exporter health, not user observability data, so they
|
|
17852
|
+
* are intentionally not delivered to generic event-bus subscribers.
|
|
17853
|
+
*/
|
|
17854
|
+
emitDropEvent(event) {
|
|
17855
|
+
if (this.handlerBufferFlushDepth > 0) {
|
|
17856
|
+
this.dropEventsEmittedDuringHandlerFlush++;
|
|
17857
|
+
}
|
|
17858
|
+
for (const exporter of this.exporters) {
|
|
17859
|
+
this.trackPromise(routeDropToHandler(exporter, event, this.logger));
|
|
17860
|
+
}
|
|
17861
|
+
if (this.bridge) {
|
|
17862
|
+
this.trackPromise(routeDropToHandler(this.bridge, event, this.logger));
|
|
17863
|
+
}
|
|
17864
|
+
}
|
|
17756
17865
|
/**
|
|
17757
17866
|
* Track an async handler promise so flush() can await it.
|
|
17758
17867
|
* No-ops for sync (void) results.
|
|
@@ -17764,19 +17873,8 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17764
17873
|
void promise2.finally(() => this.pendingHandlers.delete(promise2));
|
|
17765
17874
|
}
|
|
17766
17875
|
}
|
|
17767
|
-
/**
|
|
17768
|
-
|
|
17769
|
-
*
|
|
17770
|
-
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
17771
|
-
* bridge, and base-class subscribers). After this resolves, all event data
|
|
17772
|
-
* has been delivered to handler methods.
|
|
17773
|
-
*
|
|
17774
|
-
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
17775
|
-
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
17776
|
-
* client queue). Phases are sequential — Phase 2 must not start until
|
|
17777
|
-
* Phase 1 completes, otherwise exporters would flush empty buffers.
|
|
17778
|
-
*/
|
|
17779
|
-
async flush() {
|
|
17876
|
+
/** Await in-flight routed handler promises, draining until empty. */
|
|
17877
|
+
async drainPendingHandlers() {
|
|
17780
17878
|
let iterations = 0;
|
|
17781
17879
|
while (this.pendingHandlers.size > 0) {
|
|
17782
17880
|
await Promise.allSettled([...this.pendingHandlers]);
|
|
@@ -17791,14 +17889,54 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17791
17889
|
break;
|
|
17792
17890
|
}
|
|
17793
17891
|
}
|
|
17794
|
-
|
|
17795
|
-
|
|
17796
|
-
|
|
17797
|
-
|
|
17892
|
+
}
|
|
17893
|
+
/** Drain exporter and bridge SDK-internal buffers. */
|
|
17894
|
+
async flushHandlerBuffers() {
|
|
17895
|
+
const initialDropCount = this.dropEventsEmittedDuringHandlerFlush;
|
|
17896
|
+
this.handlerBufferFlushDepth++;
|
|
17897
|
+
try {
|
|
17898
|
+
const bufferFlushPromises = this.exporters.map((e) => e.flush());
|
|
17899
|
+
if (this.bridge) {
|
|
17900
|
+
bufferFlushPromises.push(this.bridge.flush());
|
|
17901
|
+
}
|
|
17902
|
+
if (bufferFlushPromises.length > 0) {
|
|
17903
|
+
await Promise.allSettled(bufferFlushPromises);
|
|
17904
|
+
}
|
|
17905
|
+
return this.dropEventsEmittedDuringHandlerFlush > initialDropCount;
|
|
17906
|
+
} finally {
|
|
17907
|
+
this.handlerBufferFlushDepth--;
|
|
17798
17908
|
}
|
|
17799
|
-
|
|
17800
|
-
|
|
17909
|
+
}
|
|
17910
|
+
/**
|
|
17911
|
+
* Multi-phase flush to ensure all observability data is fully exported.
|
|
17912
|
+
*
|
|
17913
|
+
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
17914
|
+
* bridge, and base-class subscribers). After this resolves, all event data
|
|
17915
|
+
* has been delivered to handler methods.
|
|
17916
|
+
*
|
|
17917
|
+
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
17918
|
+
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
17919
|
+
* client queue). Phases are sequential — buffer drains must not start until
|
|
17920
|
+
* delivery completes, otherwise exporters would flush empty buffers.
|
|
17921
|
+
*
|
|
17922
|
+
* Exporter flushes can emit drop events. When that happens, flush loops
|
|
17923
|
+
* through delivery and buffer drain again so alerting integrations that buffer
|
|
17924
|
+
* drop notifications are drained before returning.
|
|
17925
|
+
*/
|
|
17926
|
+
async flush() {
|
|
17927
|
+
await this.drainPendingHandlers();
|
|
17928
|
+
await super.flush();
|
|
17929
|
+
for (let iterations = 0; iterations < MAX_FLUSH_ITERATIONS; iterations++) {
|
|
17930
|
+
const emittedDropEvents = await this.flushHandlerBuffers();
|
|
17931
|
+
if (!emittedDropEvents && this.pendingHandlers.size === 0) {
|
|
17932
|
+
return;
|
|
17933
|
+
}
|
|
17934
|
+
await this.drainPendingHandlers();
|
|
17935
|
+
await super.flush();
|
|
17801
17936
|
}
|
|
17937
|
+
this.logger.error(
|
|
17938
|
+
`[ObservabilityBus] flush() exceeded ${MAX_FLUSH_ITERATIONS} buffer drain iterations. Handlers may be emitting drop events during every flush.`
|
|
17939
|
+
);
|
|
17802
17940
|
}
|
|
17803
17941
|
/** Flush all pending events and exporter buffers, then clear subscribers. */
|
|
17804
17942
|
async shutdown() {
|
|
@@ -18157,8 +18295,19 @@ function normalizeProvider(provider) {
|
|
|
18157
18295
|
return dotIndex !== -1 ? normalized.substring(0, dotIndex) : normalized;
|
|
18158
18296
|
}
|
|
18159
18297
|
function getModelVariants(model) {
|
|
18160
|
-
const
|
|
18161
|
-
|
|
18298
|
+
const variants = /* @__PURE__ */ new Set();
|
|
18299
|
+
const add = (v) => {
|
|
18300
|
+
variants.add(v);
|
|
18301
|
+
variants.add(stripDateSuffix(v));
|
|
18302
|
+
};
|
|
18303
|
+
add(model);
|
|
18304
|
+
add(model.replace(/\./g, "-"));
|
|
18305
|
+
add(model.replace(/[./]/g, "-"));
|
|
18306
|
+
const slashIndex = model.indexOf("/");
|
|
18307
|
+
if (slashIndex !== -1) {
|
|
18308
|
+
add(model.substring(slashIndex + 1));
|
|
18309
|
+
}
|
|
18310
|
+
return [...variants];
|
|
18162
18311
|
}
|
|
18163
18312
|
function stripDateSuffix(model) {
|
|
18164
18313
|
let stripped = model.replace(/-20\d{2}-\d{2}-\d{2}$/, "");
|
|
@@ -18601,6 +18750,9 @@ function sumDefinedValues(obj, keys) {
|
|
|
18601
18750
|
}
|
|
18602
18751
|
|
|
18603
18752
|
// src/model-tracing.ts
|
|
18753
|
+
function supportsModelInference() {
|
|
18754
|
+
return features.coreFeatures.has("model-inference-span");
|
|
18755
|
+
}
|
|
18604
18756
|
function formatPreviewLabel(label, fallback) {
|
|
18605
18757
|
return typeof label === "string" && label.length > 0 ? label : fallback;
|
|
18606
18758
|
}
|
|
@@ -18759,6 +18911,7 @@ function extractStepInput(payload) {
|
|
|
18759
18911
|
var ModelSpanTracker = class {
|
|
18760
18912
|
#modelSpan;
|
|
18761
18913
|
#currentStepSpan;
|
|
18914
|
+
#currentInferenceSpan;
|
|
18762
18915
|
#currentChunkSpan;
|
|
18763
18916
|
#currentChunkType;
|
|
18764
18917
|
#accumulator = {};
|
|
@@ -18770,9 +18923,18 @@ var ModelSpanTracker = class {
|
|
|
18770
18923
|
#deferStepClose = false;
|
|
18771
18924
|
/** Stored step-finish payload when defer mode is enabled */
|
|
18772
18925
|
#pendingStepFinishPayload;
|
|
18926
|
+
/** Static request-side context applied to every MODEL_INFERENCE span */
|
|
18927
|
+
#inferenceContext;
|
|
18773
18928
|
constructor(modelSpan) {
|
|
18774
18929
|
this.#modelSpan = modelSpan;
|
|
18775
18930
|
}
|
|
18931
|
+
/**
|
|
18932
|
+
* Set request-side context applied to subsequent MODEL_INFERENCE spans.
|
|
18933
|
+
* No-op when paired with an older @mastra/core that lacks the feature flag.
|
|
18934
|
+
*/
|
|
18935
|
+
setInferenceContext(context) {
|
|
18936
|
+
this.#inferenceContext = context;
|
|
18937
|
+
}
|
|
18776
18938
|
/**
|
|
18777
18939
|
* Capture the completion start time (time to first token) when the first content chunk arrives.
|
|
18778
18940
|
*/
|
|
@@ -18854,6 +19016,11 @@ var ModelSpanTracker = class {
|
|
|
18854
19016
|
* Start a new Model execution step.
|
|
18855
19017
|
* This should be called at the beginning of LLM execution to capture accurate startTime.
|
|
18856
19018
|
* The step-start chunk payload can be passed later via updateStep() if needed.
|
|
19019
|
+
*
|
|
19020
|
+
* Note: this only opens MODEL_STEP. The MODEL_INFERENCE child span is opened
|
|
19021
|
+
* separately via startInference() so its duration excludes input processor work.
|
|
19022
|
+
* Callers that don't call startInference() explicitly will get one auto-created
|
|
19023
|
+
* when the first model chunk arrives.
|
|
18857
19024
|
*/
|
|
18858
19025
|
startStep(payload) {
|
|
18859
19026
|
if (this.#currentStepSpan) {
|
|
@@ -18873,6 +19040,71 @@ var ModelSpanTracker = class {
|
|
|
18873
19040
|
this.#currentStepInputIsFinal = Array.isArray(payload?.inputMessages);
|
|
18874
19041
|
this.#chunkSequence = 0;
|
|
18875
19042
|
}
|
|
19043
|
+
/**
|
|
19044
|
+
* End the current MODEL_INFERENCE span when the provider stream finishes.
|
|
19045
|
+
* Fields are duplicated onto MODEL_STEP (in #endStepSpan) so existing
|
|
19046
|
+
* integrations that read usage/finishReason from the step span continue
|
|
19047
|
+
* to work unchanged.
|
|
19048
|
+
*
|
|
19049
|
+
* Safe to call multiple times - no-ops if the span is already closed.
|
|
19050
|
+
*/
|
|
19051
|
+
#endInferenceSpan(payload) {
|
|
19052
|
+
if (!this.#currentInferenceSpan) return;
|
|
19053
|
+
const { usage: rawUsage, ...otherOutput } = payload.output;
|
|
19054
|
+
const usage = extractUsageMetrics(rawUsage, payload.metadata?.providerMetadata);
|
|
19055
|
+
this.#currentInferenceSpan.end({
|
|
19056
|
+
output: otherOutput,
|
|
19057
|
+
attributes: {
|
|
19058
|
+
usage,
|
|
19059
|
+
finishReason: payload.stepResult.reason,
|
|
19060
|
+
warnings: payload.stepResult.warnings,
|
|
19061
|
+
completionStartTime: this.#completionStartTime
|
|
19062
|
+
}
|
|
19063
|
+
});
|
|
19064
|
+
this.#currentInferenceSpan = void 0;
|
|
19065
|
+
}
|
|
19066
|
+
/**
|
|
19067
|
+
* Open the MODEL_INFERENCE span for the current step. Chunks (including tool-call
|
|
19068
|
+
* chunks emitted by the model) parent under this span so its duration reflects
|
|
19069
|
+
* pure model latency.
|
|
19070
|
+
*
|
|
19071
|
+
* Should be called immediately before invoking the model — after any input
|
|
19072
|
+
* processors / `prepareStep` work has completed — so the span's startTime
|
|
19073
|
+
* does not include processor time. The latest `#inferenceContext` (set via
|
|
19074
|
+
* setInferenceContext) is snapshotted onto the span at creation.
|
|
19075
|
+
*
|
|
19076
|
+
* No-ops when the installed @mastra/core lacks the `model-inference-span`
|
|
19077
|
+
* feature flag, or when called without an active step span. Auto-invoked from
|
|
19078
|
+
* chunk handlers as a safety net; explicit callers get the most accurate
|
|
19079
|
+
* start time.
|
|
19080
|
+
*/
|
|
19081
|
+
startInference(payload) {
|
|
19082
|
+
if (!supportsModelInference()) {
|
|
19083
|
+
return;
|
|
19084
|
+
}
|
|
19085
|
+
if (!this.#currentStepSpan || this.#currentInferenceSpan) {
|
|
19086
|
+
return;
|
|
19087
|
+
}
|
|
19088
|
+
const input = extractStepInput(payload);
|
|
19089
|
+
const generationAttrs = this.#modelSpan?.attributes;
|
|
19090
|
+
const ctx = this.#inferenceContext;
|
|
19091
|
+
this.#currentInferenceSpan = this.#currentStepSpan.createChildSpan({
|
|
19092
|
+
name: `inference: ${this.#stepIndex}`,
|
|
19093
|
+
type: observability.SpanType.MODEL_INFERENCE,
|
|
19094
|
+
attributes: {
|
|
19095
|
+
stepIndex: this.#stepIndex,
|
|
19096
|
+
model: generationAttrs?.model,
|
|
19097
|
+
provider: generationAttrs?.provider,
|
|
19098
|
+
streaming: generationAttrs?.streaming,
|
|
19099
|
+
...ctx?.parameters !== void 0 ? { parameters: ctx.parameters } : {},
|
|
19100
|
+
...ctx?.providerOptions !== void 0 ? { providerOptions: ctx.providerOptions } : {},
|
|
19101
|
+
...ctx?.availableTools !== void 0 ? { availableTools: ctx.availableTools } : {},
|
|
19102
|
+
...ctx?.toolChoice !== void 0 ? { toolChoice: ctx.toolChoice } : {},
|
|
19103
|
+
...ctx?.responseFormat !== void 0 ? { responseFormat: ctx.responseFormat } : {}
|
|
19104
|
+
},
|
|
19105
|
+
input
|
|
19106
|
+
});
|
|
19107
|
+
}
|
|
18876
19108
|
/**
|
|
18877
19109
|
* Update the current step span with additional payload data.
|
|
18878
19110
|
* Called when step-start chunk arrives with request/warnings info.
|
|
@@ -18911,6 +19143,7 @@ var ModelSpanTracker = class {
|
|
|
18911
19143
|
delete cleanMetadata[key];
|
|
18912
19144
|
}
|
|
18913
19145
|
}
|
|
19146
|
+
this.#endInferenceSpan(payload);
|
|
18914
19147
|
this.#currentStepSpan.end({
|
|
18915
19148
|
output: otherOutput,
|
|
18916
19149
|
attributes: {
|
|
@@ -18928,14 +19161,35 @@ var ModelSpanTracker = class {
|
|
|
18928
19161
|
this.#stepIndex++;
|
|
18929
19162
|
}
|
|
18930
19163
|
/**
|
|
18931
|
-
*
|
|
19164
|
+
* Returns the parent span for chunks. Chunks parent under MODEL_INFERENCE
|
|
19165
|
+
* (the provider call) when available, falling back to MODEL_STEP only if
|
|
19166
|
+
* startStep() was bypassed.
|
|
18932
19167
|
*/
|
|
18933
|
-
#
|
|
18934
|
-
this.#
|
|
19168
|
+
#chunkParent() {
|
|
19169
|
+
return this.#currentInferenceSpan ?? this.#currentStepSpan;
|
|
19170
|
+
}
|
|
19171
|
+
/**
|
|
19172
|
+
* Safety-net invoked from chunk handlers: auto-create MODEL_STEP and
|
|
19173
|
+
* MODEL_INFERENCE if a chunk arrives before the loop has explicitly opened
|
|
19174
|
+
* them, so chunks parent under MODEL_INFERENCE rather than falling through
|
|
19175
|
+
* to MODEL_STEP. Idempotent — each public start* method is itself a no-op
|
|
19176
|
+
* when its span is already live.
|
|
19177
|
+
*/
|
|
19178
|
+
#ensureStepAndInference() {
|
|
18935
19179
|
if (!this.#currentStepSpan) {
|
|
18936
19180
|
this.startStep();
|
|
18937
19181
|
}
|
|
18938
|
-
this.#
|
|
19182
|
+
if (!this.#currentInferenceSpan) {
|
|
19183
|
+
this.startInference();
|
|
19184
|
+
}
|
|
19185
|
+
}
|
|
19186
|
+
/**
|
|
19187
|
+
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
19188
|
+
*/
|
|
19189
|
+
#startChunkSpan(chunkType, initialData) {
|
|
19190
|
+
this.#endChunkSpan();
|
|
19191
|
+
this.#ensureStepAndInference();
|
|
19192
|
+
this.#currentChunkSpan = this.#chunkParent()?.createChildSpan({
|
|
18939
19193
|
name: `chunk: '${chunkType}'`,
|
|
18940
19194
|
type: observability.SpanType.MODEL_CHUNK,
|
|
18941
19195
|
attributes: {
|
|
@@ -18974,10 +19228,8 @@ var ModelSpanTracker = class {
|
|
|
18974
19228
|
* Create an event span (for single chunks like tool-call)
|
|
18975
19229
|
*/
|
|
18976
19230
|
#createEventSpan(chunkType, output, options) {
|
|
18977
|
-
|
|
18978
|
-
|
|
18979
|
-
}
|
|
18980
|
-
const span = this.#currentStepSpan?.createEventSpan({
|
|
19231
|
+
this.#ensureStepAndInference();
|
|
19232
|
+
const span = this.#chunkParent()?.createEventSpan({
|
|
18981
19233
|
name: `chunk: '${chunkType}'`,
|
|
18982
19234
|
type: observability.SpanType.MODEL_CHUNK,
|
|
18983
19235
|
attributes: {
|
|
@@ -19098,10 +19350,8 @@ var ModelSpanTracker = class {
|
|
|
19098
19350
|
#handleToolApprovalChunk(chunk) {
|
|
19099
19351
|
if (chunk.type !== "tool-call-approval") return;
|
|
19100
19352
|
const payload = chunk.payload;
|
|
19101
|
-
|
|
19102
|
-
|
|
19103
|
-
}
|
|
19104
|
-
const span = this.#currentStepSpan?.createEventSpan({
|
|
19353
|
+
this.#ensureStepAndInference();
|
|
19354
|
+
const span = this.#chunkParent()?.createEventSpan({
|
|
19105
19355
|
name: `chunk: 'tool-call-approval'`,
|
|
19106
19356
|
type: observability.SpanType.MODEL_CHUNK,
|
|
19107
19357
|
attributes: {
|
|
@@ -19159,10 +19409,15 @@ var ModelSpanTracker = class {
|
|
|
19159
19409
|
} else {
|
|
19160
19410
|
this.startStep(chunk.payload);
|
|
19161
19411
|
}
|
|
19412
|
+
if (!this.#currentInferenceSpan) {
|
|
19413
|
+
this.startInference(chunk.payload);
|
|
19414
|
+
}
|
|
19162
19415
|
break;
|
|
19163
19416
|
case "step-finish":
|
|
19164
19417
|
if (this.#deferStepClose) {
|
|
19165
19418
|
this.#pendingStepFinishPayload = chunk.payload;
|
|
19419
|
+
this.#endChunkSpan();
|
|
19420
|
+
this.#endInferenceSpan(chunk.payload);
|
|
19166
19421
|
} else {
|
|
19167
19422
|
this.#endStepSpan(chunk.payload);
|
|
19168
19423
|
}
|
|
@@ -19258,6 +19513,7 @@ function isSpanInternal(spanType, flags) {
|
|
|
19258
19513
|
// Model-related spans
|
|
19259
19514
|
case observability.SpanType.MODEL_GENERATION:
|
|
19260
19515
|
case observability.SpanType.MODEL_STEP:
|
|
19516
|
+
case observability.SpanType.MODEL_INFERENCE:
|
|
19261
19517
|
case observability.SpanType.MODEL_CHUNK:
|
|
19262
19518
|
return (flags & observability.InternalSpans.MODEL) !== 0;
|
|
19263
19519
|
// Default: never internal
|
|
@@ -20350,6 +20606,7 @@ function buildScoreEvent(args) {
|
|
|
20350
20606
|
traceId,
|
|
20351
20607
|
spanId,
|
|
20352
20608
|
scorerId: score.scorerId,
|
|
20609
|
+
scorerName: score.scorerName,
|
|
20353
20610
|
scorerVersion: score.scorerVersion,
|
|
20354
20611
|
source: score.source,
|
|
20355
20612
|
scoreSource: score.scoreSource,
|
|
@@ -20357,6 +20614,7 @@ function buildScoreEvent(args) {
|
|
|
20357
20614
|
reason: score.reason,
|
|
20358
20615
|
experimentId: score.experimentId,
|
|
20359
20616
|
scoreTraceId: score.scoreTraceId,
|
|
20617
|
+
targetEntityType: score.targetEntityType,
|
|
20360
20618
|
correlationContext,
|
|
20361
20619
|
metadata: mergeMetadata(inheritedMetadata, score.metadata)
|
|
20362
20620
|
}
|
|
@@ -20887,23 +21145,56 @@ var Observability = class extends base.MastraBase {
|
|
|
20887
21145
|
}
|
|
20888
21146
|
}
|
|
20889
21147
|
}
|
|
21148
|
+
const sensitiveDataFilterSetting = config2.sensitiveDataFilter ?? true;
|
|
21149
|
+
const shouldAutoApplySensitiveFilter = sensitiveDataFilterSetting !== false;
|
|
21150
|
+
const sensitiveDataFilterOptions = typeof sensitiveDataFilterSetting === "object" && sensitiveDataFilterSetting !== null ? sensitiveDataFilterSetting : void 0;
|
|
21151
|
+
const buildAutoSensitiveFilter = () => {
|
|
21152
|
+
if (!shouldAutoApplySensitiveFilter) {
|
|
21153
|
+
return void 0;
|
|
21154
|
+
}
|
|
21155
|
+
return new SensitiveDataFilter(sensitiveDataFilterOptions);
|
|
21156
|
+
};
|
|
20890
21157
|
if (config2.default?.enabled) {
|
|
20891
21158
|
console.warn(
|
|
20892
|
-
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter
|
|
21159
|
+
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter and CloudExporter instead. Sensitive data filtering is applied by default and can be controlled via the top-level "sensitiveDataFilter" option. See https://mastra.ai/docs/observability/tracing/overview for the recommended configuration.'
|
|
20893
21160
|
);
|
|
21161
|
+
const autoFilter = buildAutoSensitiveFilter();
|
|
20894
21162
|
const defaultInstance = new DefaultObservabilityInstance({
|
|
20895
21163
|
serviceName: "mastra",
|
|
20896
21164
|
name: "default",
|
|
20897
21165
|
sampling: { type: "always" /* ALWAYS */ },
|
|
20898
21166
|
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
20899
|
-
spanOutputProcessors: [
|
|
21167
|
+
spanOutputProcessors: autoFilter ? [autoFilter] : []
|
|
20900
21168
|
});
|
|
20901
21169
|
this.#registry.register("default", defaultInstance, true);
|
|
20902
21170
|
}
|
|
20903
21171
|
if (config2.configs) {
|
|
20904
21172
|
const instances = Object.entries(config2.configs);
|
|
20905
21173
|
instances.forEach(([name, tracingDef], index) => {
|
|
20906
|
-
|
|
21174
|
+
let instance;
|
|
21175
|
+
if (isInstance(tracingDef)) {
|
|
21176
|
+
instance = tracingDef;
|
|
21177
|
+
if (shouldAutoApplySensitiveFilter) {
|
|
21178
|
+
const processors = instance.getSpanOutputProcessors?.() ?? [];
|
|
21179
|
+
const hasFilter = processors.some((p) => p instanceof SensitiveDataFilter);
|
|
21180
|
+
if (!hasFilter) {
|
|
21181
|
+
this.logger?.warn(
|
|
21182
|
+
"[Mastra Observability] Pre-instantiated observability instance does not include a SensitiveDataFilter. Auto-applied filtering is skipped for pre-instantiated instances. Add a SensitiveDataFilter to spanOutputProcessors when constructing the instance to redact sensitive data.",
|
|
21183
|
+
{ instanceName: name }
|
|
21184
|
+
);
|
|
21185
|
+
}
|
|
21186
|
+
}
|
|
21187
|
+
} else {
|
|
21188
|
+
const userProcessors = tracingDef.spanOutputProcessors ?? [];
|
|
21189
|
+
const hasFilter = userProcessors.some((p) => p instanceof SensitiveDataFilter);
|
|
21190
|
+
const autoFilter = !hasFilter ? buildAutoSensitiveFilter() : void 0;
|
|
21191
|
+
const spanOutputProcessors = autoFilter ? [...userProcessors, autoFilter] : userProcessors;
|
|
21192
|
+
instance = new DefaultObservabilityInstance({
|
|
21193
|
+
...tracingDef,
|
|
21194
|
+
name,
|
|
21195
|
+
spanOutputProcessors
|
|
21196
|
+
});
|
|
21197
|
+
}
|
|
20907
21198
|
const isDefault = !config2.default?.enabled && index === 0;
|
|
20908
21199
|
this.#registry.register(name, instance, isDefault);
|
|
20909
21200
|
});
|
|
@@ -20922,10 +21213,11 @@ var Observability = class extends base.MastraBase {
|
|
|
20922
21213
|
instance.__setMastraEnvironment?.(mastraEnvironment);
|
|
20923
21214
|
const config2 = instance.getConfig();
|
|
20924
21215
|
const exporters = instance.getExporters();
|
|
21216
|
+
const emitDropEvent = instance instanceof BaseObservabilityInstance ? (event) => instance.getObservabilityBus().emitDropEvent(event) : void 0;
|
|
20925
21217
|
exporters.forEach((exporter) => {
|
|
20926
21218
|
if ("init" in exporter && typeof exporter.init === "function") {
|
|
20927
21219
|
try {
|
|
20928
|
-
exporter.init({ mastra, config: config2 });
|
|
21220
|
+
exporter.init({ mastra, config: config2, emitDropEvent });
|
|
20929
21221
|
} catch (error48) {
|
|
20930
21222
|
this.logger?.warn("Failed to initialize observability exporter", {
|
|
20931
21223
|
exporterName: exporter.name,
|
|
@@ -21113,6 +21405,9 @@ var Observability = class extends base.MastraBase {
|
|
|
21113
21405
|
}
|
|
21114
21406
|
};
|
|
21115
21407
|
|
|
21408
|
+
// src/features.ts
|
|
21409
|
+
var observabilityFeatures = /* @__PURE__ */ new Set(["model-inference-span"]);
|
|
21410
|
+
|
|
21116
21411
|
// src/tracing-options.ts
|
|
21117
21412
|
function buildTracingOptions(...updaters) {
|
|
21118
21413
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
@@ -21149,6 +21444,7 @@ exports.getExternalParentId = getExternalParentId;
|
|
|
21149
21444
|
exports.isSerializedMap = isSerializedMap;
|
|
21150
21445
|
exports.mergeSerializationOptions = mergeSerializationOptions;
|
|
21151
21446
|
exports.observabilityConfigValueSchema = observabilityConfigValueSchema;
|
|
21447
|
+
exports.observabilityFeatures = observabilityFeatures;
|
|
21152
21448
|
exports.observabilityInstanceConfigSchema = observabilityInstanceConfigSchema;
|
|
21153
21449
|
exports.observabilityRegistryConfigSchema = observabilityRegistryConfigSchema;
|
|
21154
21450
|
exports.reconstructSerializedMap = reconstructSerializedMap;
|