@mastra/observability 1.11.1 → 1.12.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +3 -2
- package/dist/bus/observability-bus.d.ts +21 -4
- package/dist/bus/observability-bus.d.ts.map +1 -1
- package/dist/bus/route-event.d.ts +8 -1
- package/dist/bus/route-event.d.ts.map +1 -1
- package/dist/config.d.ts +28 -6
- package/dist/config.d.ts.map +1 -1
- package/dist/default.d.ts.map +1 -1
- package/dist/exporters/default.d.ts +2 -0
- package/dist/exporters/default.d.ts.map +1 -1
- package/dist/exporters/event-buffer.d.ts +4 -4
- package/dist/exporters/event-buffer.d.ts.map +1 -1
- package/dist/features.d.ts +39 -0
- package/dist/features.d.ts.map +1 -0
- package/dist/index.cjs +344 -48
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +344 -49
- package/dist/index.js.map +1 -1
- package/dist/model-tracing.d.ts +34 -4
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/recorded.d.ts.map +1 -1
- package/dist/spans/base.d.ts.map +1 -1
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import fs from 'fs';
|
|
|
8
8
|
import { createRequire } from 'module';
|
|
9
9
|
import path from 'path';
|
|
10
10
|
import { TransformStream } from 'stream/web';
|
|
11
|
+
import { coreFeatures } from '@mastra/core/features';
|
|
11
12
|
|
|
12
13
|
var __defProp = Object.defineProperty;
|
|
13
14
|
var __export = (target, all) => {
|
|
@@ -37,6 +38,23 @@ function routeToHandler(handler, event, logger) {
|
|
|
37
38
|
if (handler.onScoreEvent) {
|
|
38
39
|
return catchAsyncResult(handler.onScoreEvent(event), handler.name, "score", logger);
|
|
39
40
|
}
|
|
41
|
+
if (handler.addScoreToTrace) {
|
|
42
|
+
const score = event.score;
|
|
43
|
+
if (!score.traceId) break;
|
|
44
|
+
return catchAsyncResult(
|
|
45
|
+
handler.addScoreToTrace({
|
|
46
|
+
traceId: score.traceId,
|
|
47
|
+
...score.spanId ? { spanId: score.spanId } : {},
|
|
48
|
+
score: score.score,
|
|
49
|
+
...score.reason ? { reason: score.reason } : {},
|
|
50
|
+
scorerName: score.scorerName ?? score.scorerId,
|
|
51
|
+
...score.metadata ? { metadata: score.metadata } : {}
|
|
52
|
+
}),
|
|
53
|
+
handler.name,
|
|
54
|
+
"score",
|
|
55
|
+
logger
|
|
56
|
+
);
|
|
57
|
+
}
|
|
40
58
|
break;
|
|
41
59
|
case "feedback":
|
|
42
60
|
if (handler.onFeedbackEvent) {
|
|
@@ -48,6 +66,15 @@ function routeToHandler(handler, event, logger) {
|
|
|
48
66
|
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
49
67
|
}
|
|
50
68
|
}
|
|
69
|
+
function routeDropToHandler(handler, event, logger) {
|
|
70
|
+
try {
|
|
71
|
+
if (handler.onDroppedEvent) {
|
|
72
|
+
return catchAsyncResult(handler.onDroppedEvent(event), handler.name, "drop", logger);
|
|
73
|
+
}
|
|
74
|
+
} catch (err) {
|
|
75
|
+
logger.error(`[Observability] Handler error [handler=${handler.name}]:`, err);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
51
78
|
function catchAsyncResult(result, handlerName, signal, logger) {
|
|
52
79
|
if (result && typeof result.then === "function") {
|
|
53
80
|
return result.catch((err) => {
|
|
@@ -13897,12 +13924,18 @@ var observabilityConfigValueSchema = external_exports.object(observabilityInstan
|
|
|
13897
13924
|
message: "At least one exporter or a bridge is required"
|
|
13898
13925
|
}
|
|
13899
13926
|
);
|
|
13927
|
+
var sensitiveDataFilterOptionsSchema = external_exports.object({
|
|
13928
|
+
sensitiveFields: external_exports.array(external_exports.string()).optional(),
|
|
13929
|
+
redactionToken: external_exports.string().optional(),
|
|
13930
|
+
redactionStyle: external_exports.enum(["full", "partial"]).optional()
|
|
13931
|
+
}).strict();
|
|
13900
13932
|
var observabilityRegistryConfigSchema = external_exports.object({
|
|
13901
13933
|
default: external_exports.object({
|
|
13902
13934
|
enabled: external_exports.boolean().optional()
|
|
13903
13935
|
}).optional().nullable(),
|
|
13904
13936
|
configs: external_exports.union([external_exports.record(external_exports.string(), external_exports.any()), external_exports.array(external_exports.any()), external_exports.null()]).optional(),
|
|
13905
|
-
configSelector: external_exports.function().optional()
|
|
13937
|
+
configSelector: external_exports.function().optional(),
|
|
13938
|
+
sensitiveDataFilter: external_exports.union([external_exports.boolean(), sensitiveDataFilterOptionsSchema]).optional()
|
|
13906
13939
|
}).passthrough().refine(
|
|
13907
13940
|
(data) => {
|
|
13908
13941
|
const isDefaultEnabled = data.default?.enabled === true;
|
|
@@ -15672,31 +15705,39 @@ var EventBuffer = class {
|
|
|
15672
15705
|
break;
|
|
15673
15706
|
}
|
|
15674
15707
|
}
|
|
15675
|
-
/** Re-add failed create events to the buffer,
|
|
15708
|
+
/** Re-add failed create events to the buffer, returning events that exceed max retries. */
|
|
15676
15709
|
reAddCreates(events) {
|
|
15677
15710
|
const retryable = [];
|
|
15711
|
+
const dropped = [];
|
|
15678
15712
|
for (const e of events) {
|
|
15679
15713
|
if (++e.retryCount <= this.#maxRetries) {
|
|
15680
15714
|
retryable.push(e);
|
|
15715
|
+
} else {
|
|
15716
|
+
dropped.push(e);
|
|
15681
15717
|
}
|
|
15682
15718
|
}
|
|
15683
15719
|
if (retryable.length > 0) {
|
|
15684
15720
|
this.setFirstEventTime();
|
|
15685
15721
|
this.#creates.push(...retryable);
|
|
15686
15722
|
}
|
|
15723
|
+
return dropped;
|
|
15687
15724
|
}
|
|
15688
|
-
/** Re-add failed update events to the buffer,
|
|
15725
|
+
/** Re-add failed update events to the buffer, returning events that exceed max retries. */
|
|
15689
15726
|
reAddUpdates(events) {
|
|
15690
15727
|
const retryable = [];
|
|
15728
|
+
const dropped = [];
|
|
15691
15729
|
for (const e of events) {
|
|
15692
15730
|
if (++e.retryCount <= this.#maxRetries) {
|
|
15693
15731
|
retryable.push(e);
|
|
15732
|
+
} else {
|
|
15733
|
+
dropped.push(e);
|
|
15694
15734
|
}
|
|
15695
15735
|
}
|
|
15696
15736
|
if (retryable.length > 0) {
|
|
15697
15737
|
this.setFirstEventTime();
|
|
15698
15738
|
this.#updates.push(...retryable);
|
|
15699
15739
|
}
|
|
15740
|
+
return dropped;
|
|
15700
15741
|
}
|
|
15701
15742
|
/** Snapshot of buffered create events. */
|
|
15702
15743
|
get creates() {
|
|
@@ -15775,6 +15816,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15775
15816
|
#observabilityStorage;
|
|
15776
15817
|
#resolvedStrategy;
|
|
15777
15818
|
#flushTimer;
|
|
15819
|
+
#emitDropEvent;
|
|
15778
15820
|
// Signals whose storage methods threw "not implemented" — skip on future flushes
|
|
15779
15821
|
#unsupportedSignals = /* @__PURE__ */ new Set();
|
|
15780
15822
|
constructor(config2 = {}) {
|
|
@@ -15796,6 +15838,7 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15796
15838
|
async init(options) {
|
|
15797
15839
|
try {
|
|
15798
15840
|
this.#isInitializing = true;
|
|
15841
|
+
this.#emitDropEvent = options.emitDropEvent;
|
|
15799
15842
|
this.#storage = options.mastra?.getStorage();
|
|
15800
15843
|
if (!this.#storage) {
|
|
15801
15844
|
this.logger.warn("DefaultExporter disabled: Storage not available. Traces will not be persisted.");
|
|
@@ -15881,21 +15924,54 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15881
15924
|
this.scheduleFlush();
|
|
15882
15925
|
}
|
|
15883
15926
|
}
|
|
15927
|
+
sanitizeDropError(error48) {
|
|
15928
|
+
if (error48 instanceof MastraError) {
|
|
15929
|
+
return {
|
|
15930
|
+
id: error48.id,
|
|
15931
|
+
domain: String(error48.domain),
|
|
15932
|
+
message: error48.message
|
|
15933
|
+
};
|
|
15934
|
+
}
|
|
15935
|
+
if (error48 instanceof Error) {
|
|
15936
|
+
return { message: error48.message };
|
|
15937
|
+
}
|
|
15938
|
+
return { message: String(error48) };
|
|
15939
|
+
}
|
|
15940
|
+
emitDrop(signal, reason, count, error48) {
|
|
15941
|
+
if (count === 0) return;
|
|
15942
|
+
const dropEvent = {
|
|
15943
|
+
type: "drop",
|
|
15944
|
+
signal,
|
|
15945
|
+
reason,
|
|
15946
|
+
count,
|
|
15947
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
15948
|
+
exporterName: this.name,
|
|
15949
|
+
...this.#observabilityStorage ? { storageName: this.#observabilityStorage.constructor.name } : {},
|
|
15950
|
+
...error48 === void 0 ? {} : { error: this.sanitizeDropError(error48) }
|
|
15951
|
+
};
|
|
15952
|
+
this.#emitDropEvent?.(dropEvent);
|
|
15953
|
+
}
|
|
15884
15954
|
/**
|
|
15885
15955
|
* Flush a batch of create events for a single signal type.
|
|
15886
15956
|
* On "not implemented" errors, disables the signal for future flushes.
|
|
15887
15957
|
* On other errors, re-adds events to the buffer for retry.
|
|
15888
15958
|
*/
|
|
15889
15959
|
async flushCreates(signal, events, storageCall) {
|
|
15890
|
-
if (
|
|
15960
|
+
if (events.length === 0) return;
|
|
15961
|
+
if (this.#unsupportedSignals.has(signal)) {
|
|
15962
|
+
this.emitDrop(signal, "unsupported-storage", events.length);
|
|
15963
|
+
return;
|
|
15964
|
+
}
|
|
15891
15965
|
try {
|
|
15892
15966
|
await storageCall(events);
|
|
15893
15967
|
} catch (error48) {
|
|
15894
15968
|
if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15895
15969
|
this.logger.warn(error48.message);
|
|
15896
15970
|
this.#unsupportedSignals.add(signal);
|
|
15971
|
+
this.emitDrop(signal, "unsupported-storage", events.length, error48);
|
|
15897
15972
|
} else {
|
|
15898
|
-
this.#eventBuffer.reAddCreates(events);
|
|
15973
|
+
const dropped = this.#eventBuffer.reAddCreates(events);
|
|
15974
|
+
this.emitDrop(signal, "retry-exhausted", dropped.length, error48);
|
|
15899
15975
|
}
|
|
15900
15976
|
}
|
|
15901
15977
|
}
|
|
@@ -15904,7 +15980,12 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15904
15980
|
* When `isEnd` is true, successfully flushed spans are removed from tracking.
|
|
15905
15981
|
*/
|
|
15906
15982
|
async flushSpanUpdates(events, deferredUpdates, isEnd) {
|
|
15907
|
-
|
|
15983
|
+
const deferredCountAtEntry = deferredUpdates.length;
|
|
15984
|
+
if (events.length === 0) return;
|
|
15985
|
+
if (this.#unsupportedSignals.has("tracing")) {
|
|
15986
|
+
this.emitDrop("tracing", "unsupported-storage", events.length);
|
|
15987
|
+
return;
|
|
15988
|
+
}
|
|
15908
15989
|
const partials = [];
|
|
15909
15990
|
for (const event of events) {
|
|
15910
15991
|
const span = event.exportedSpan;
|
|
@@ -15928,9 +16009,12 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
15928
16009
|
if (error48 instanceof MastraError && error48.domain === ErrorDomain.MASTRA_OBSERVABILITY && error48.id.endsWith("_NOT_IMPLEMENTED")) {
|
|
15929
16010
|
this.logger.warn(error48.message);
|
|
15930
16011
|
this.#unsupportedSignals.add("tracing");
|
|
16012
|
+
deferredUpdates.length = 0;
|
|
16013
|
+
this.emitDrop("tracing", "unsupported-storage", events.length + deferredCountAtEntry, error48);
|
|
15931
16014
|
} else {
|
|
15932
16015
|
deferredUpdates.length = 0;
|
|
15933
|
-
this.#eventBuffer.reAddUpdates(events);
|
|
16016
|
+
const dropped = this.#eventBuffer.reAddUpdates(events);
|
|
16017
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length, error48);
|
|
15934
16018
|
}
|
|
15935
16019
|
}
|
|
15936
16020
|
}
|
|
@@ -16006,17 +16090,17 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
16006
16090
|
(events) => this.#observabilityStorage.batchCreateFeedback({ feedbacks: events.map((f) => buildFeedbackRecord(f)) })
|
|
16007
16091
|
),
|
|
16008
16092
|
this.flushCreates(
|
|
16009
|
-
"
|
|
16093
|
+
"log",
|
|
16010
16094
|
createLogEvents,
|
|
16011
16095
|
(events) => this.#observabilityStorage.batchCreateLogs({ logs: events.map((l) => buildLogRecord(l)) })
|
|
16012
16096
|
),
|
|
16013
16097
|
this.flushCreates(
|
|
16014
|
-
"
|
|
16098
|
+
"metric",
|
|
16015
16099
|
createMetricEvents,
|
|
16016
16100
|
(events) => this.#observabilityStorage.batchCreateMetrics({ metrics: events.map((m) => buildMetricRecord(m)) })
|
|
16017
16101
|
),
|
|
16018
16102
|
this.flushCreates(
|
|
16019
|
-
"
|
|
16103
|
+
"score",
|
|
16020
16104
|
createScoreEvents,
|
|
16021
16105
|
(events) => this.#observabilityStorage.batchCreateScores({ scores: events.map((s) => buildScoreRecord(s)) })
|
|
16022
16106
|
),
|
|
@@ -16030,7 +16114,13 @@ var DefaultExporter = class extends BaseExporter {
|
|
|
16030
16114
|
await this.flushSpanUpdates(updateSpanEvents, deferredUpdates, false);
|
|
16031
16115
|
await this.flushSpanUpdates(endSpanEvents, deferredUpdates, true);
|
|
16032
16116
|
if (deferredUpdates.length > 0) {
|
|
16033
|
-
this.#
|
|
16117
|
+
if (this.#unsupportedSignals.has("tracing")) {
|
|
16118
|
+
this.emitDrop("tracing", "unsupported-storage", deferredUpdates.length);
|
|
16119
|
+
deferredUpdates.length = 0;
|
|
16120
|
+
} else {
|
|
16121
|
+
const dropped = this.#eventBuffer.reAddUpdates(deferredUpdates);
|
|
16122
|
+
this.emitDrop("tracing", "retry-exhausted", dropped.length);
|
|
16123
|
+
}
|
|
16034
16124
|
}
|
|
16035
16125
|
const elapsed = Date.now() - startTime;
|
|
16036
16126
|
this.logger.debug("Batch flushed", {
|
|
@@ -17659,6 +17749,8 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17659
17749
|
bridge;
|
|
17660
17750
|
/** In-flight handler promises from routeToHandler. Self-cleaning via .finally(). */
|
|
17661
17751
|
pendingHandlers = /* @__PURE__ */ new Set();
|
|
17752
|
+
handlerBufferFlushDepth = 0;
|
|
17753
|
+
dropEventsEmittedDuringHandlerFlush = 0;
|
|
17662
17754
|
/** Resolved deepClean options applied to non-tracing events before fan-out. */
|
|
17663
17755
|
deepCleanOptions;
|
|
17664
17756
|
constructor(opts) {
|
|
@@ -17745,6 +17837,23 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17745
17837
|
}
|
|
17746
17838
|
super.emit(cleaned);
|
|
17747
17839
|
}
|
|
17840
|
+
/**
|
|
17841
|
+
* Emit exporter pipeline drop events to exporters and the bridge.
|
|
17842
|
+
*
|
|
17843
|
+
* Drop events describe exporter health, not user observability data, so they
|
|
17844
|
+
* are intentionally not delivered to generic event-bus subscribers.
|
|
17845
|
+
*/
|
|
17846
|
+
emitDropEvent(event) {
|
|
17847
|
+
if (this.handlerBufferFlushDepth > 0) {
|
|
17848
|
+
this.dropEventsEmittedDuringHandlerFlush++;
|
|
17849
|
+
}
|
|
17850
|
+
for (const exporter of this.exporters) {
|
|
17851
|
+
this.trackPromise(routeDropToHandler(exporter, event, this.logger));
|
|
17852
|
+
}
|
|
17853
|
+
if (this.bridge) {
|
|
17854
|
+
this.trackPromise(routeDropToHandler(this.bridge, event, this.logger));
|
|
17855
|
+
}
|
|
17856
|
+
}
|
|
17748
17857
|
/**
|
|
17749
17858
|
* Track an async handler promise so flush() can await it.
|
|
17750
17859
|
* No-ops for sync (void) results.
|
|
@@ -17756,19 +17865,8 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17756
17865
|
void promise2.finally(() => this.pendingHandlers.delete(promise2));
|
|
17757
17866
|
}
|
|
17758
17867
|
}
|
|
17759
|
-
/**
|
|
17760
|
-
|
|
17761
|
-
*
|
|
17762
|
-
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
17763
|
-
* bridge, and base-class subscribers). After this resolves, all event data
|
|
17764
|
-
* has been delivered to handler methods.
|
|
17765
|
-
*
|
|
17766
|
-
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
17767
|
-
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
17768
|
-
* client queue). Phases are sequential — Phase 2 must not start until
|
|
17769
|
-
* Phase 1 completes, otherwise exporters would flush empty buffers.
|
|
17770
|
-
*/
|
|
17771
|
-
async flush() {
|
|
17868
|
+
/** Await in-flight routed handler promises, draining until empty. */
|
|
17869
|
+
async drainPendingHandlers() {
|
|
17772
17870
|
let iterations = 0;
|
|
17773
17871
|
while (this.pendingHandlers.size > 0) {
|
|
17774
17872
|
await Promise.allSettled([...this.pendingHandlers]);
|
|
@@ -17783,14 +17881,54 @@ var ObservabilityBus = class extends BaseObservabilityEventBus {
|
|
|
17783
17881
|
break;
|
|
17784
17882
|
}
|
|
17785
17883
|
}
|
|
17786
|
-
|
|
17787
|
-
|
|
17788
|
-
|
|
17789
|
-
|
|
17884
|
+
}
|
|
17885
|
+
/** Drain exporter and bridge SDK-internal buffers. */
|
|
17886
|
+
async flushHandlerBuffers() {
|
|
17887
|
+
const initialDropCount = this.dropEventsEmittedDuringHandlerFlush;
|
|
17888
|
+
this.handlerBufferFlushDepth++;
|
|
17889
|
+
try {
|
|
17890
|
+
const bufferFlushPromises = this.exporters.map((e) => e.flush());
|
|
17891
|
+
if (this.bridge) {
|
|
17892
|
+
bufferFlushPromises.push(this.bridge.flush());
|
|
17893
|
+
}
|
|
17894
|
+
if (bufferFlushPromises.length > 0) {
|
|
17895
|
+
await Promise.allSettled(bufferFlushPromises);
|
|
17896
|
+
}
|
|
17897
|
+
return this.dropEventsEmittedDuringHandlerFlush > initialDropCount;
|
|
17898
|
+
} finally {
|
|
17899
|
+
this.handlerBufferFlushDepth--;
|
|
17790
17900
|
}
|
|
17791
|
-
|
|
17792
|
-
|
|
17901
|
+
}
|
|
17902
|
+
/**
|
|
17903
|
+
* Multi-phase flush to ensure all observability data is fully exported.
|
|
17904
|
+
*
|
|
17905
|
+
* **Phase 1 — Delivery:** Await all in-flight handler promises (exporters,
|
|
17906
|
+
* bridge, and base-class subscribers). After this resolves, all event data
|
|
17907
|
+
* has been delivered to handler methods.
|
|
17908
|
+
*
|
|
17909
|
+
* **Phase 2 — Buffer drain:** Call flush() on each exporter and bridge to
|
|
17910
|
+
* drain their SDK-internal buffers (e.g., OTEL BatchSpanProcessor, Langfuse
|
|
17911
|
+
* client queue). Phases are sequential — buffer drains must not start until
|
|
17912
|
+
* delivery completes, otherwise exporters would flush empty buffers.
|
|
17913
|
+
*
|
|
17914
|
+
* Exporter flushes can emit drop events. When that happens, flush loops
|
|
17915
|
+
* through delivery and buffer drain again so alerting integrations that buffer
|
|
17916
|
+
* drop notifications are drained before returning.
|
|
17917
|
+
*/
|
|
17918
|
+
async flush() {
|
|
17919
|
+
await this.drainPendingHandlers();
|
|
17920
|
+
await super.flush();
|
|
17921
|
+
for (let iterations = 0; iterations < MAX_FLUSH_ITERATIONS; iterations++) {
|
|
17922
|
+
const emittedDropEvents = await this.flushHandlerBuffers();
|
|
17923
|
+
if (!emittedDropEvents && this.pendingHandlers.size === 0) {
|
|
17924
|
+
return;
|
|
17925
|
+
}
|
|
17926
|
+
await this.drainPendingHandlers();
|
|
17927
|
+
await super.flush();
|
|
17793
17928
|
}
|
|
17929
|
+
this.logger.error(
|
|
17930
|
+
`[ObservabilityBus] flush() exceeded ${MAX_FLUSH_ITERATIONS} buffer drain iterations. Handlers may be emitting drop events during every flush.`
|
|
17931
|
+
);
|
|
17794
17932
|
}
|
|
17795
17933
|
/** Flush all pending events and exporter buffers, then clear subscribers. */
|
|
17796
17934
|
async shutdown() {
|
|
@@ -18149,8 +18287,19 @@ function normalizeProvider(provider) {
|
|
|
18149
18287
|
return dotIndex !== -1 ? normalized.substring(0, dotIndex) : normalized;
|
|
18150
18288
|
}
|
|
18151
18289
|
function getModelVariants(model) {
|
|
18152
|
-
const
|
|
18153
|
-
|
|
18290
|
+
const variants = /* @__PURE__ */ new Set();
|
|
18291
|
+
const add = (v) => {
|
|
18292
|
+
variants.add(v);
|
|
18293
|
+
variants.add(stripDateSuffix(v));
|
|
18294
|
+
};
|
|
18295
|
+
add(model);
|
|
18296
|
+
add(model.replace(/\./g, "-"));
|
|
18297
|
+
add(model.replace(/[./]/g, "-"));
|
|
18298
|
+
const slashIndex = model.indexOf("/");
|
|
18299
|
+
if (slashIndex !== -1) {
|
|
18300
|
+
add(model.substring(slashIndex + 1));
|
|
18301
|
+
}
|
|
18302
|
+
return [...variants];
|
|
18154
18303
|
}
|
|
18155
18304
|
function stripDateSuffix(model) {
|
|
18156
18305
|
let stripped = model.replace(/-20\d{2}-\d{2}-\d{2}$/, "");
|
|
@@ -18593,6 +18742,9 @@ function sumDefinedValues(obj, keys) {
|
|
|
18593
18742
|
}
|
|
18594
18743
|
|
|
18595
18744
|
// src/model-tracing.ts
|
|
18745
|
+
function supportsModelInference() {
|
|
18746
|
+
return coreFeatures.has("model-inference-span");
|
|
18747
|
+
}
|
|
18596
18748
|
function formatPreviewLabel(label, fallback) {
|
|
18597
18749
|
return typeof label === "string" && label.length > 0 ? label : fallback;
|
|
18598
18750
|
}
|
|
@@ -18751,6 +18903,7 @@ function extractStepInput(payload) {
|
|
|
18751
18903
|
var ModelSpanTracker = class {
|
|
18752
18904
|
#modelSpan;
|
|
18753
18905
|
#currentStepSpan;
|
|
18906
|
+
#currentInferenceSpan;
|
|
18754
18907
|
#currentChunkSpan;
|
|
18755
18908
|
#currentChunkType;
|
|
18756
18909
|
#accumulator = {};
|
|
@@ -18762,9 +18915,18 @@ var ModelSpanTracker = class {
|
|
|
18762
18915
|
#deferStepClose = false;
|
|
18763
18916
|
/** Stored step-finish payload when defer mode is enabled */
|
|
18764
18917
|
#pendingStepFinishPayload;
|
|
18918
|
+
/** Static request-side context applied to every MODEL_INFERENCE span */
|
|
18919
|
+
#inferenceContext;
|
|
18765
18920
|
constructor(modelSpan) {
|
|
18766
18921
|
this.#modelSpan = modelSpan;
|
|
18767
18922
|
}
|
|
18923
|
+
/**
|
|
18924
|
+
* Set request-side context applied to subsequent MODEL_INFERENCE spans.
|
|
18925
|
+
* No-op when paired with an older @mastra/core that lacks the feature flag.
|
|
18926
|
+
*/
|
|
18927
|
+
setInferenceContext(context) {
|
|
18928
|
+
this.#inferenceContext = context;
|
|
18929
|
+
}
|
|
18768
18930
|
/**
|
|
18769
18931
|
* Capture the completion start time (time to first token) when the first content chunk arrives.
|
|
18770
18932
|
*/
|
|
@@ -18846,6 +19008,11 @@ var ModelSpanTracker = class {
|
|
|
18846
19008
|
* Start a new Model execution step.
|
|
18847
19009
|
* This should be called at the beginning of LLM execution to capture accurate startTime.
|
|
18848
19010
|
* The step-start chunk payload can be passed later via updateStep() if needed.
|
|
19011
|
+
*
|
|
19012
|
+
* Note: this only opens MODEL_STEP. The MODEL_INFERENCE child span is opened
|
|
19013
|
+
* separately via startInference() so its duration excludes input processor work.
|
|
19014
|
+
* Callers that don't call startInference() explicitly will get one auto-created
|
|
19015
|
+
* when the first model chunk arrives.
|
|
18849
19016
|
*/
|
|
18850
19017
|
startStep(payload) {
|
|
18851
19018
|
if (this.#currentStepSpan) {
|
|
@@ -18865,6 +19032,71 @@ var ModelSpanTracker = class {
|
|
|
18865
19032
|
this.#currentStepInputIsFinal = Array.isArray(payload?.inputMessages);
|
|
18866
19033
|
this.#chunkSequence = 0;
|
|
18867
19034
|
}
|
|
19035
|
+
/**
|
|
19036
|
+
* End the current MODEL_INFERENCE span when the provider stream finishes.
|
|
19037
|
+
* Fields are duplicated onto MODEL_STEP (in #endStepSpan) so existing
|
|
19038
|
+
* integrations that read usage/finishReason from the step span continue
|
|
19039
|
+
* to work unchanged.
|
|
19040
|
+
*
|
|
19041
|
+
* Safe to call multiple times - no-ops if the span is already closed.
|
|
19042
|
+
*/
|
|
19043
|
+
#endInferenceSpan(payload) {
|
|
19044
|
+
if (!this.#currentInferenceSpan) return;
|
|
19045
|
+
const { usage: rawUsage, ...otherOutput } = payload.output;
|
|
19046
|
+
const usage = extractUsageMetrics(rawUsage, payload.metadata?.providerMetadata);
|
|
19047
|
+
this.#currentInferenceSpan.end({
|
|
19048
|
+
output: otherOutput,
|
|
19049
|
+
attributes: {
|
|
19050
|
+
usage,
|
|
19051
|
+
finishReason: payload.stepResult.reason,
|
|
19052
|
+
warnings: payload.stepResult.warnings,
|
|
19053
|
+
completionStartTime: this.#completionStartTime
|
|
19054
|
+
}
|
|
19055
|
+
});
|
|
19056
|
+
this.#currentInferenceSpan = void 0;
|
|
19057
|
+
}
|
|
19058
|
+
/**
|
|
19059
|
+
* Open the MODEL_INFERENCE span for the current step. Chunks (including tool-call
|
|
19060
|
+
* chunks emitted by the model) parent under this span so its duration reflects
|
|
19061
|
+
* pure model latency.
|
|
19062
|
+
*
|
|
19063
|
+
* Should be called immediately before invoking the model — after any input
|
|
19064
|
+
* processors / `prepareStep` work has completed — so the span's startTime
|
|
19065
|
+
* does not include processor time. The latest `#inferenceContext` (set via
|
|
19066
|
+
* setInferenceContext) is snapshotted onto the span at creation.
|
|
19067
|
+
*
|
|
19068
|
+
* No-ops when the installed @mastra/core lacks the `model-inference-span`
|
|
19069
|
+
* feature flag, or when called without an active step span. Auto-invoked from
|
|
19070
|
+
* chunk handlers as a safety net; explicit callers get the most accurate
|
|
19071
|
+
* start time.
|
|
19072
|
+
*/
|
|
19073
|
+
startInference(payload) {
|
|
19074
|
+
if (!supportsModelInference()) {
|
|
19075
|
+
return;
|
|
19076
|
+
}
|
|
19077
|
+
if (!this.#currentStepSpan || this.#currentInferenceSpan) {
|
|
19078
|
+
return;
|
|
19079
|
+
}
|
|
19080
|
+
const input = extractStepInput(payload);
|
|
19081
|
+
const generationAttrs = this.#modelSpan?.attributes;
|
|
19082
|
+
const ctx = this.#inferenceContext;
|
|
19083
|
+
this.#currentInferenceSpan = this.#currentStepSpan.createChildSpan({
|
|
19084
|
+
name: `inference: ${this.#stepIndex}`,
|
|
19085
|
+
type: SpanType.MODEL_INFERENCE,
|
|
19086
|
+
attributes: {
|
|
19087
|
+
stepIndex: this.#stepIndex,
|
|
19088
|
+
model: generationAttrs?.model,
|
|
19089
|
+
provider: generationAttrs?.provider,
|
|
19090
|
+
streaming: generationAttrs?.streaming,
|
|
19091
|
+
...ctx?.parameters !== void 0 ? { parameters: ctx.parameters } : {},
|
|
19092
|
+
...ctx?.providerOptions !== void 0 ? { providerOptions: ctx.providerOptions } : {},
|
|
19093
|
+
...ctx?.availableTools !== void 0 ? { availableTools: ctx.availableTools } : {},
|
|
19094
|
+
...ctx?.toolChoice !== void 0 ? { toolChoice: ctx.toolChoice } : {},
|
|
19095
|
+
...ctx?.responseFormat !== void 0 ? { responseFormat: ctx.responseFormat } : {}
|
|
19096
|
+
},
|
|
19097
|
+
input
|
|
19098
|
+
});
|
|
19099
|
+
}
|
|
18868
19100
|
/**
|
|
18869
19101
|
* Update the current step span with additional payload data.
|
|
18870
19102
|
* Called when step-start chunk arrives with request/warnings info.
|
|
@@ -18903,6 +19135,7 @@ var ModelSpanTracker = class {
|
|
|
18903
19135
|
delete cleanMetadata[key];
|
|
18904
19136
|
}
|
|
18905
19137
|
}
|
|
19138
|
+
this.#endInferenceSpan(payload);
|
|
18906
19139
|
this.#currentStepSpan.end({
|
|
18907
19140
|
output: otherOutput,
|
|
18908
19141
|
attributes: {
|
|
@@ -18920,14 +19153,35 @@ var ModelSpanTracker = class {
|
|
|
18920
19153
|
this.#stepIndex++;
|
|
18921
19154
|
}
|
|
18922
19155
|
/**
|
|
18923
|
-
*
|
|
19156
|
+
* Returns the parent span for chunks. Chunks parent under MODEL_INFERENCE
|
|
19157
|
+
* (the provider call) when available, falling back to MODEL_STEP only if
|
|
19158
|
+
* startStep() was bypassed.
|
|
18924
19159
|
*/
|
|
18925
|
-
#
|
|
18926
|
-
this.#
|
|
19160
|
+
#chunkParent() {
|
|
19161
|
+
return this.#currentInferenceSpan ?? this.#currentStepSpan;
|
|
19162
|
+
}
|
|
19163
|
+
/**
|
|
19164
|
+
* Safety-net invoked from chunk handlers: auto-create MODEL_STEP and
|
|
19165
|
+
* MODEL_INFERENCE if a chunk arrives before the loop has explicitly opened
|
|
19166
|
+
* them, so chunks parent under MODEL_INFERENCE rather than falling through
|
|
19167
|
+
* to MODEL_STEP. Idempotent — each public start* method is itself a no-op
|
|
19168
|
+
* when its span is already live.
|
|
19169
|
+
*/
|
|
19170
|
+
#ensureStepAndInference() {
|
|
18927
19171
|
if (!this.#currentStepSpan) {
|
|
18928
19172
|
this.startStep();
|
|
18929
19173
|
}
|
|
18930
|
-
this.#
|
|
19174
|
+
if (!this.#currentInferenceSpan) {
|
|
19175
|
+
this.startInference();
|
|
19176
|
+
}
|
|
19177
|
+
}
|
|
19178
|
+
/**
|
|
19179
|
+
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
19180
|
+
*/
|
|
19181
|
+
#startChunkSpan(chunkType, initialData) {
|
|
19182
|
+
this.#endChunkSpan();
|
|
19183
|
+
this.#ensureStepAndInference();
|
|
19184
|
+
this.#currentChunkSpan = this.#chunkParent()?.createChildSpan({
|
|
18931
19185
|
name: `chunk: '${chunkType}'`,
|
|
18932
19186
|
type: SpanType.MODEL_CHUNK,
|
|
18933
19187
|
attributes: {
|
|
@@ -18966,10 +19220,8 @@ var ModelSpanTracker = class {
|
|
|
18966
19220
|
* Create an event span (for single chunks like tool-call)
|
|
18967
19221
|
*/
|
|
18968
19222
|
#createEventSpan(chunkType, output, options) {
|
|
18969
|
-
|
|
18970
|
-
|
|
18971
|
-
}
|
|
18972
|
-
const span = this.#currentStepSpan?.createEventSpan({
|
|
19223
|
+
this.#ensureStepAndInference();
|
|
19224
|
+
const span = this.#chunkParent()?.createEventSpan({
|
|
18973
19225
|
name: `chunk: '${chunkType}'`,
|
|
18974
19226
|
type: SpanType.MODEL_CHUNK,
|
|
18975
19227
|
attributes: {
|
|
@@ -19090,10 +19342,8 @@ var ModelSpanTracker = class {
|
|
|
19090
19342
|
#handleToolApprovalChunk(chunk) {
|
|
19091
19343
|
if (chunk.type !== "tool-call-approval") return;
|
|
19092
19344
|
const payload = chunk.payload;
|
|
19093
|
-
|
|
19094
|
-
|
|
19095
|
-
}
|
|
19096
|
-
const span = this.#currentStepSpan?.createEventSpan({
|
|
19345
|
+
this.#ensureStepAndInference();
|
|
19346
|
+
const span = this.#chunkParent()?.createEventSpan({
|
|
19097
19347
|
name: `chunk: 'tool-call-approval'`,
|
|
19098
19348
|
type: SpanType.MODEL_CHUNK,
|
|
19099
19349
|
attributes: {
|
|
@@ -19151,10 +19401,15 @@ var ModelSpanTracker = class {
|
|
|
19151
19401
|
} else {
|
|
19152
19402
|
this.startStep(chunk.payload);
|
|
19153
19403
|
}
|
|
19404
|
+
if (!this.#currentInferenceSpan) {
|
|
19405
|
+
this.startInference(chunk.payload);
|
|
19406
|
+
}
|
|
19154
19407
|
break;
|
|
19155
19408
|
case "step-finish":
|
|
19156
19409
|
if (this.#deferStepClose) {
|
|
19157
19410
|
this.#pendingStepFinishPayload = chunk.payload;
|
|
19411
|
+
this.#endChunkSpan();
|
|
19412
|
+
this.#endInferenceSpan(chunk.payload);
|
|
19158
19413
|
} else {
|
|
19159
19414
|
this.#endStepSpan(chunk.payload);
|
|
19160
19415
|
}
|
|
@@ -19250,6 +19505,7 @@ function isSpanInternal(spanType, flags) {
|
|
|
19250
19505
|
// Model-related spans
|
|
19251
19506
|
case SpanType.MODEL_GENERATION:
|
|
19252
19507
|
case SpanType.MODEL_STEP:
|
|
19508
|
+
case SpanType.MODEL_INFERENCE:
|
|
19253
19509
|
case SpanType.MODEL_CHUNK:
|
|
19254
19510
|
return (flags & InternalSpans.MODEL) !== 0;
|
|
19255
19511
|
// Default: never internal
|
|
@@ -20342,6 +20598,7 @@ function buildScoreEvent(args) {
|
|
|
20342
20598
|
traceId,
|
|
20343
20599
|
spanId,
|
|
20344
20600
|
scorerId: score.scorerId,
|
|
20601
|
+
scorerName: score.scorerName,
|
|
20345
20602
|
scorerVersion: score.scorerVersion,
|
|
20346
20603
|
source: score.source,
|
|
20347
20604
|
scoreSource: score.scoreSource,
|
|
@@ -20349,6 +20606,7 @@ function buildScoreEvent(args) {
|
|
|
20349
20606
|
reason: score.reason,
|
|
20350
20607
|
experimentId: score.experimentId,
|
|
20351
20608
|
scoreTraceId: score.scoreTraceId,
|
|
20609
|
+
targetEntityType: score.targetEntityType,
|
|
20352
20610
|
correlationContext,
|
|
20353
20611
|
metadata: mergeMetadata(inheritedMetadata, score.metadata)
|
|
20354
20612
|
}
|
|
@@ -20879,23 +21137,56 @@ var Observability = class extends MastraBase {
|
|
|
20879
21137
|
}
|
|
20880
21138
|
}
|
|
20881
21139
|
}
|
|
21140
|
+
const sensitiveDataFilterSetting = config2.sensitiveDataFilter ?? true;
|
|
21141
|
+
const shouldAutoApplySensitiveFilter = sensitiveDataFilterSetting !== false;
|
|
21142
|
+
const sensitiveDataFilterOptions = typeof sensitiveDataFilterSetting === "object" && sensitiveDataFilterSetting !== null ? sensitiveDataFilterSetting : void 0;
|
|
21143
|
+
const buildAutoSensitiveFilter = () => {
|
|
21144
|
+
if (!shouldAutoApplySensitiveFilter) {
|
|
21145
|
+
return void 0;
|
|
21146
|
+
}
|
|
21147
|
+
return new SensitiveDataFilter(sensitiveDataFilterOptions);
|
|
21148
|
+
};
|
|
20882
21149
|
if (config2.default?.enabled) {
|
|
20883
21150
|
console.warn(
|
|
20884
|
-
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter
|
|
21151
|
+
'[Mastra Observability] The "default: { enabled: true }" configuration is deprecated and will be removed in a future version. Please use explicit configs with DefaultExporter and CloudExporter instead. Sensitive data filtering is applied by default and can be controlled via the top-level "sensitiveDataFilter" option. See https://mastra.ai/docs/observability/tracing/overview for the recommended configuration.'
|
|
20885
21152
|
);
|
|
21153
|
+
const autoFilter = buildAutoSensitiveFilter();
|
|
20886
21154
|
const defaultInstance = new DefaultObservabilityInstance({
|
|
20887
21155
|
serviceName: "mastra",
|
|
20888
21156
|
name: "default",
|
|
20889
21157
|
sampling: { type: "always" /* ALWAYS */ },
|
|
20890
21158
|
exporters: [new DefaultExporter(), new CloudExporter()],
|
|
20891
|
-
spanOutputProcessors: [
|
|
21159
|
+
spanOutputProcessors: autoFilter ? [autoFilter] : []
|
|
20892
21160
|
});
|
|
20893
21161
|
this.#registry.register("default", defaultInstance, true);
|
|
20894
21162
|
}
|
|
20895
21163
|
if (config2.configs) {
|
|
20896
21164
|
const instances = Object.entries(config2.configs);
|
|
20897
21165
|
instances.forEach(([name, tracingDef], index) => {
|
|
20898
|
-
|
|
21166
|
+
let instance;
|
|
21167
|
+
if (isInstance(tracingDef)) {
|
|
21168
|
+
instance = tracingDef;
|
|
21169
|
+
if (shouldAutoApplySensitiveFilter) {
|
|
21170
|
+
const processors = instance.getSpanOutputProcessors?.() ?? [];
|
|
21171
|
+
const hasFilter = processors.some((p) => p instanceof SensitiveDataFilter);
|
|
21172
|
+
if (!hasFilter) {
|
|
21173
|
+
this.logger?.warn(
|
|
21174
|
+
"[Mastra Observability] Pre-instantiated observability instance does not include a SensitiveDataFilter. Auto-applied filtering is skipped for pre-instantiated instances. Add a SensitiveDataFilter to spanOutputProcessors when constructing the instance to redact sensitive data.",
|
|
21175
|
+
{ instanceName: name }
|
|
21176
|
+
);
|
|
21177
|
+
}
|
|
21178
|
+
}
|
|
21179
|
+
} else {
|
|
21180
|
+
const userProcessors = tracingDef.spanOutputProcessors ?? [];
|
|
21181
|
+
const hasFilter = userProcessors.some((p) => p instanceof SensitiveDataFilter);
|
|
21182
|
+
const autoFilter = !hasFilter ? buildAutoSensitiveFilter() : void 0;
|
|
21183
|
+
const spanOutputProcessors = autoFilter ? [...userProcessors, autoFilter] : userProcessors;
|
|
21184
|
+
instance = new DefaultObservabilityInstance({
|
|
21185
|
+
...tracingDef,
|
|
21186
|
+
name,
|
|
21187
|
+
spanOutputProcessors
|
|
21188
|
+
});
|
|
21189
|
+
}
|
|
20899
21190
|
const isDefault = !config2.default?.enabled && index === 0;
|
|
20900
21191
|
this.#registry.register(name, instance, isDefault);
|
|
20901
21192
|
});
|
|
@@ -20914,10 +21205,11 @@ var Observability = class extends MastraBase {
|
|
|
20914
21205
|
instance.__setMastraEnvironment?.(mastraEnvironment);
|
|
20915
21206
|
const config2 = instance.getConfig();
|
|
20916
21207
|
const exporters = instance.getExporters();
|
|
21208
|
+
const emitDropEvent = instance instanceof BaseObservabilityInstance ? (event) => instance.getObservabilityBus().emitDropEvent(event) : void 0;
|
|
20917
21209
|
exporters.forEach((exporter) => {
|
|
20918
21210
|
if ("init" in exporter && typeof exporter.init === "function") {
|
|
20919
21211
|
try {
|
|
20920
|
-
exporter.init({ mastra, config: config2 });
|
|
21212
|
+
exporter.init({ mastra, config: config2, emitDropEvent });
|
|
20921
21213
|
} catch (error48) {
|
|
20922
21214
|
this.logger?.warn("Failed to initialize observability exporter", {
|
|
20923
21215
|
exporterName: exporter.name,
|
|
@@ -21105,11 +21397,14 @@ var Observability = class extends MastraBase {
|
|
|
21105
21397
|
}
|
|
21106
21398
|
};
|
|
21107
21399
|
|
|
21400
|
+
// src/features.ts
|
|
21401
|
+
var observabilityFeatures = /* @__PURE__ */ new Set(["model-inference-span"]);
|
|
21402
|
+
|
|
21108
21403
|
// src/tracing-options.ts
|
|
21109
21404
|
function buildTracingOptions(...updaters) {
|
|
21110
21405
|
return updaters.reduce((opts, updater) => updater(opts), {});
|
|
21111
21406
|
}
|
|
21112
21407
|
|
|
21113
|
-
export { BaseExporter, BaseObservabilityEventBus, BaseObservabilityInstance, BaseSpan, CardinalityFilter, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, LoggerContextImpl, MetricsContextImpl, ModelSpanTracker, NoOpSpan, Observability, ObservabilityBus, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, isSerializedMap, mergeSerializationOptions, observabilityConfigValueSchema, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, reconstructSerializedMap, routeToHandler, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
21408
|
+
export { BaseExporter, BaseObservabilityEventBus, BaseObservabilityInstance, BaseSpan, CardinalityFilter, CloudExporter, ConsoleExporter, DEFAULT_DEEP_CLEAN_OPTIONS, DEFAULT_KEYS_TO_STRIP, DefaultExporter, DefaultObservabilityInstance, DefaultSpan, JsonExporter, LoggerContextImpl, MetricsContextImpl, ModelSpanTracker, NoOpSpan, Observability, ObservabilityBus, SamplingStrategyType, SensitiveDataFilter, TestExporter, TraceData, TrackingExporter, buildTracingOptions, chainFormatters, deepClean, getExternalParentId, isSerializedMap, mergeSerializationOptions, observabilityConfigValueSchema, observabilityFeatures, observabilityInstanceConfigSchema, observabilityRegistryConfigSchema, reconstructSerializedMap, routeToHandler, samplingStrategySchema, serializationOptionsSchema, truncateString };
|
|
21114
21409
|
//# sourceMappingURL=index.js.map
|
|
21115
21410
|
//# sourceMappingURL=index.js.map
|