@mastra/observability 1.22.0 → 1.23.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/exporters/test.d.ts.map +1 -1
- package/dist/index.cjs +277 -69
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +277 -69
- package/dist/index.js.map +1 -1
- package/dist/instances/base.d.ts.map +1 -1
- package/dist/metrics/pricing-data.json +418 -542
- package/dist/model-tracing.d.ts +24 -9
- package/dist/model-tracing.d.ts.map +1 -1
- package/dist/span_processors/sensitive-data-filter.d.ts.map +1 -1
- package/dist/spans/base.d.ts +13 -9
- package/dist/spans/base.d.ts.map +1 -1
- package/dist/spans/default.d.ts.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -2317,9 +2317,10 @@ function addUsageStats(a, b) {
|
|
|
2317
2317
|
* Provides span tracking for Model generations, including:
|
|
2318
2318
|
* - MODEL_STEP spans (one per Model API call - includes processors and tool executions)
|
|
2319
2319
|
* - MODEL_INFERENCE spans (the provider call itself - model latency only)
|
|
2320
|
-
* - MODEL_CHUNK spans (
|
|
2320
|
+
* - MODEL_CHUNK spans (processed streaming content within a logical step)
|
|
2321
2321
|
*
|
|
2322
|
-
* Hierarchy: MODEL_GENERATION -> MODEL_STEP -> MODEL_INFERENCE
|
|
2322
|
+
* Hierarchy: MODEL_GENERATION -> MODEL_STEP -> MODEL_INFERENCE (provider timing)
|
|
2323
|
+
* -> MODEL_CHUNK (processed content)
|
|
2323
2324
|
*
|
|
2324
2325
|
* Processors and tool executions remain children of MODEL_STEP (siblings of
|
|
2325
2326
|
* MODEL_INFERENCE), so MODEL_INFERENCE measures pure model time.
|
|
@@ -2336,6 +2337,12 @@ function addUsageStats(a, b) {
|
|
|
2336
2337
|
function supportsModelInference() {
|
|
2337
2338
|
return _mastra_core_features.coreFeatures.has("model-inference-span");
|
|
2338
2339
|
}
|
|
2340
|
+
function interruptedByRun(termination) {
|
|
2341
|
+
return termination.outcome === "abort" || termination.stopReason === "timeout" && termination.stopReasonContext && "timeoutType" in termination.stopReasonContext && termination.stopReasonContext.timeoutType === "total" ? {
|
|
2342
|
+
...termination,
|
|
2343
|
+
outcome: "interrupted"
|
|
2344
|
+
} : termination;
|
|
2345
|
+
}
|
|
2339
2346
|
function parseGatewayCost(providerMetadata) {
|
|
2340
2347
|
const rawCost = providerMetadata?.gateway?.cost;
|
|
2341
2348
|
const cost = typeof rawCost === "number" ? rawCost : typeof rawCost === "string" && rawCost.trim().length > 0 ? Number(rawCost) : void 0;
|
|
@@ -2526,13 +2533,17 @@ var ModelSpanTracker = class {
|
|
|
2526
2533
|
#modelSpan;
|
|
2527
2534
|
#currentStepSpan;
|
|
2528
2535
|
#currentInferenceSpan;
|
|
2536
|
+
#stepHasInference = false;
|
|
2537
|
+
#tracksProviderStream = false;
|
|
2529
2538
|
#currentChunkSpan;
|
|
2530
2539
|
#currentChunkType;
|
|
2531
2540
|
#accumulator = {};
|
|
2532
2541
|
#stepIndex = 0;
|
|
2533
2542
|
#chunkSequence = 0;
|
|
2534
2543
|
#completionStartTime;
|
|
2544
|
+
#timeToFirstTokenMs;
|
|
2535
2545
|
#inferenceCompletionStartTime;
|
|
2546
|
+
#inferenceTimeToFirstTokenMs;
|
|
2536
2547
|
/**
|
|
2537
2548
|
* True between a step's `step-start` and `step-finish` chunks, the window in which
|
|
2538
2549
|
* the provider responds. Chunks outside it, such as a `tool-call` replayed for a
|
|
@@ -2561,22 +2572,41 @@ var ModelSpanTracker = class {
|
|
|
2561
2572
|
this.#inferenceContext = context;
|
|
2562
2573
|
}
|
|
2563
2574
|
/**
|
|
2564
|
-
* Record when the provider's first chunk arrived
|
|
2565
|
-
*
|
|
2566
|
-
*
|
|
2567
|
-
*
|
|
2575
|
+
* Record when the provider's first chunk arrived. Time to first token is measured
|
|
2576
|
+
* once here, from the start of the inference that streamed the chunk; the inference
|
|
2577
|
+
* reports it when it ends, and the number is copied up the agent's own chain
|
|
2578
|
+
* (step → generation → agent run) with the chunk's clock, first write wins at each
|
|
2579
|
+
* level. Time to first chunk is `completionStartTime - startTime` on whichever span
|
|
2580
|
+
* is being read.
|
|
2568
2581
|
*/
|
|
2569
2582
|
#captureCompletionStartTime() {
|
|
2570
2583
|
this.#ensureStepAndInference();
|
|
2571
2584
|
if (this.#inferenceCompletionStartTime) return;
|
|
2572
2585
|
const now = /* @__PURE__ */ new Date();
|
|
2586
|
+
const inference = this.#currentInferenceSpan;
|
|
2587
|
+
const timeToFirstTokenMs = inference ? now.getTime() - inference.startTime.getTime() : void 0;
|
|
2573
2588
|
this.#inferenceCompletionStartTime = now;
|
|
2589
|
+
this.#inferenceTimeToFirstTokenMs = timeToFirstTokenMs;
|
|
2574
2590
|
this.#completionStartTime ??= now;
|
|
2591
|
+
this.#timeToFirstTokenMs ??= timeToFirstTokenMs;
|
|
2575
2592
|
for (const span of [
|
|
2576
2593
|
this.#currentStepSpan,
|
|
2577
2594
|
this.#modelSpan,
|
|
2578
2595
|
this.#agentRunSpan
|
|
2579
|
-
])
|
|
2596
|
+
]) {
|
|
2597
|
+
if (!span || span.attributes?.completionStartTime) continue;
|
|
2598
|
+
span.update({ attributes: {
|
|
2599
|
+
completionStartTime: now,
|
|
2600
|
+
...timeToFirstTokenMs !== void 0 ? { timeToFirstTokenMs } : {}
|
|
2601
|
+
} });
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
/** What the current inference streamed so far, for whichever way it ends. */
|
|
2605
|
+
#inferenceStreamAttributes() {
|
|
2606
|
+
return {
|
|
2607
|
+
...this.#inferenceCompletionStartTime ? { completionStartTime: this.#inferenceCompletionStartTime } : {},
|
|
2608
|
+
...this.#inferenceTimeToFirstTokenMs !== void 0 ? { timeToFirstTokenMs: this.#inferenceTimeToFirstTokenMs } : {}
|
|
2609
|
+
};
|
|
2580
2610
|
}
|
|
2581
2611
|
/**
|
|
2582
2612
|
* Get the tracing context for creating child spans.
|
|
@@ -2591,21 +2621,33 @@ var ModelSpanTracker = class {
|
|
|
2591
2621
|
* doesn't leave them dangling. No-op if they were already closed.
|
|
2592
2622
|
*/
|
|
2593
2623
|
reportGenerationError(options) {
|
|
2624
|
+
this.#endChunkSpan();
|
|
2625
|
+
const termination = options.outcome != null ? {
|
|
2626
|
+
outcome: options.outcome,
|
|
2627
|
+
stopReason: options.stopReason,
|
|
2628
|
+
stopReasonContext: options.stopReasonContext
|
|
2629
|
+
} : _mastra_core_observability.executionFailureTermination?.(options.error);
|
|
2594
2630
|
if (this.#currentInferenceSpan) {
|
|
2595
2631
|
this.#currentInferenceSpan.error({
|
|
2596
2632
|
error: options.error,
|
|
2597
|
-
|
|
2633
|
+
...termination ? interruptedByRun(termination) : {},
|
|
2634
|
+
endSpan: true,
|
|
2635
|
+
attributes: this.#inferenceStreamAttributes()
|
|
2598
2636
|
});
|
|
2599
2637
|
this.#currentInferenceSpan = void 0;
|
|
2600
2638
|
}
|
|
2601
2639
|
if (this.#currentStepSpan) {
|
|
2602
2640
|
this.#currentStepSpan.error({
|
|
2603
2641
|
error: options.error,
|
|
2642
|
+
...termination ? interruptedByRun(termination) : {},
|
|
2604
2643
|
endSpan: true
|
|
2605
2644
|
});
|
|
2606
2645
|
this.#currentStepSpan = void 0;
|
|
2607
2646
|
}
|
|
2608
|
-
this.#modelSpan?.error(
|
|
2647
|
+
this.#modelSpan?.error({
|
|
2648
|
+
...options,
|
|
2649
|
+
...termination
|
|
2650
|
+
});
|
|
2609
2651
|
}
|
|
2610
2652
|
/**
|
|
2611
2653
|
* End the generation span with optional raw usage data.
|
|
@@ -2620,16 +2662,37 @@ var ModelSpanTracker = class {
|
|
|
2620
2662
|
}, model);
|
|
2621
2663
|
if (providerCostContext && !spanOptions.attributes) spanOptions.attributes = {};
|
|
2622
2664
|
if (spanOptions.attributes) {
|
|
2623
|
-
|
|
2665
|
+
if (this.#completionStartTime) {
|
|
2666
|
+
spanOptions.attributes.completionStartTime = this.#completionStartTime;
|
|
2667
|
+
spanOptions.attributes.timeToFirstTokenMs = this.#timeToFirstTokenMs;
|
|
2668
|
+
}
|
|
2624
2669
|
spanOptions.attributes.usage = extractUsageMetrics(usage, providerMetadata);
|
|
2625
2670
|
spanOptions.attributes.costContext = providerCostContext ?? spanOptions.attributes.costContext;
|
|
2626
2671
|
if (!spanOptions.attributes.costContext) spanOptions.attributes.costContext = getGatewayCostContext({ stepProviderMetadata });
|
|
2627
2672
|
}
|
|
2673
|
+
if (spanOptions.outcome === "abort" || spanOptions.stopReason === "timeout") {
|
|
2674
|
+
const termination = {
|
|
2675
|
+
outcome: spanOptions.outcome,
|
|
2676
|
+
stopReason: spanOptions.stopReason,
|
|
2677
|
+
stopReasonContext: spanOptions.stopReasonContext
|
|
2678
|
+
};
|
|
2679
|
+
this.#endChunkSpan();
|
|
2680
|
+
this.#currentStepSpan?.end({
|
|
2681
|
+
...interruptedByRun(termination),
|
|
2682
|
+
endTree: true
|
|
2683
|
+
});
|
|
2684
|
+
this.#currentStepSpan = void 0;
|
|
2685
|
+
this.#currentInferenceSpan = void 0;
|
|
2686
|
+
this.#currentStepInputIsFinal = false;
|
|
2687
|
+
}
|
|
2628
2688
|
this.#modelSpan?.end(spanOptions);
|
|
2629
2689
|
}
|
|
2630
2690
|
getCompletionStartTime() {
|
|
2631
2691
|
return this.#completionStartTime;
|
|
2632
2692
|
}
|
|
2693
|
+
getTimeToFirstTokenMs() {
|
|
2694
|
+
return this.#timeToFirstTokenMs;
|
|
2695
|
+
}
|
|
2633
2696
|
/**
|
|
2634
2697
|
* Update the generation span
|
|
2635
2698
|
*/
|
|
@@ -2683,10 +2746,12 @@ var ModelSpanTracker = class {
|
|
|
2683
2746
|
*/
|
|
2684
2747
|
startStep(payload) {
|
|
2685
2748
|
if (this.#currentStepSpan) return;
|
|
2749
|
+
this.#stepHasInference = false;
|
|
2686
2750
|
const input = extractStepInput(payload);
|
|
2687
2751
|
this.#currentStepSpan = this.#modelSpan?.createChildSpan({
|
|
2688
2752
|
name: `step: ${this.#stepIndex}`,
|
|
2689
2753
|
type: _mastra_core_observability.SpanType.MODEL_STEP,
|
|
2754
|
+
...typeof _mastra_core_observability.agentLoopTermination === "function" ? { outcome: null } : {},
|
|
2690
2755
|
attributes: {
|
|
2691
2756
|
stepIndex: this.#stepIndex,
|
|
2692
2757
|
...payload?.messageId ? { messageId: payload.messageId } : {},
|
|
@@ -2698,6 +2763,7 @@ var ModelSpanTracker = class {
|
|
|
2698
2763
|
this.#currentStepInputIsFinal = Array.isArray(payload?.inputMessages);
|
|
2699
2764
|
this.#chunkSequence = 0;
|
|
2700
2765
|
this.#inferenceCompletionStartTime = void 0;
|
|
2766
|
+
this.#inferenceTimeToFirstTokenMs = void 0;
|
|
2701
2767
|
}
|
|
2702
2768
|
/**
|
|
2703
2769
|
* End the current MODEL_INFERENCE span when the provider stream finishes.
|
|
@@ -2707,22 +2773,31 @@ var ModelSpanTracker = class {
|
|
|
2707
2773
|
*
|
|
2708
2774
|
* Safe to call multiple times - no-ops if the span is already closed.
|
|
2709
2775
|
*/
|
|
2710
|
-
#endInferenceSpan(payload) {
|
|
2776
|
+
#endInferenceSpan(payload, captureOutput = true) {
|
|
2711
2777
|
if (!this.#currentInferenceSpan) return;
|
|
2712
2778
|
const { usage: rawUsage, ...otherOutput } = payload.output;
|
|
2713
2779
|
const usage = extractUsageMetrics(rawUsage, payload.metadata?.providerMetadata);
|
|
2714
2780
|
const responseModel = typeof payload.metadata?.modelId === "string" ? payload.metadata.modelId : void 0;
|
|
2715
2781
|
this.#currentInferenceSpan.end({
|
|
2716
|
-
|
|
2782
|
+
..._mastra_core_observability.agentLoopTermination?.({
|
|
2783
|
+
stopCause: "model_finished",
|
|
2784
|
+
finishReason: payload.stepResult.reason
|
|
2785
|
+
}),
|
|
2786
|
+
...payload.stepResult.reason === "error" ? {
|
|
2787
|
+
outcome: "error",
|
|
2788
|
+
stopReason: "model_error"
|
|
2789
|
+
} : {},
|
|
2790
|
+
...captureOutput ? { output: otherOutput } : {},
|
|
2717
2791
|
attributes: {
|
|
2718
2792
|
usage,
|
|
2719
2793
|
finishReason: payload.stepResult.reason,
|
|
2720
2794
|
warnings: payload.stepResult.warnings,
|
|
2721
|
-
|
|
2795
|
+
...this.#inferenceStreamAttributes(),
|
|
2722
2796
|
...responseModel?.trim() ? { responseModel } : {}
|
|
2723
2797
|
}
|
|
2724
2798
|
});
|
|
2725
2799
|
this.#currentInferenceSpan = void 0;
|
|
2800
|
+
this.#inferenceCompletionStartTime = void 0;
|
|
2726
2801
|
}
|
|
2727
2802
|
/**
|
|
2728
2803
|
* Open the MODEL_INFERENCE span for the current step. Chunks (including tool-call
|
|
@@ -2740,31 +2815,56 @@ var ModelSpanTracker = class {
|
|
|
2740
2815
|
* start time.
|
|
2741
2816
|
*
|
|
2742
2817
|
* `payload.modelId` / `payload.provider` name the model the caller is about
|
|
2743
|
-
* to invoke and are recorded on MODEL_STEP
|
|
2744
|
-
*
|
|
2745
|
-
* a
|
|
2818
|
+
* to invoke and are recorded on MODEL_STEP and the inference. A named call
|
|
2819
|
+
* while an inference is still open is the next attempt of the same step (a
|
|
2820
|
+
* fallback model after a failure): that inference ends as failed and the
|
|
2821
|
+
* attempt gets its own, so its timing and model are its own. The safety net
|
|
2822
|
+
* never records them: a replayed step-start chunk names the model of the
|
|
2823
|
+
* original call.
|
|
2746
2824
|
*/
|
|
2747
2825
|
startInference(payload) {
|
|
2748
|
-
if (payload?.modelId && payload.provider)
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2826
|
+
if (payload?.modelId && payload.provider) {
|
|
2827
|
+
this.#currentStepSpan?.update({ attributes: {
|
|
2828
|
+
model: payload.modelId,
|
|
2829
|
+
provider: payload.provider
|
|
2830
|
+
} });
|
|
2831
|
+
this.failInference(/* @__PURE__ */ new Error("Model call ended without a result before the next attempt started"));
|
|
2832
|
+
}
|
|
2752
2833
|
this.#openInference(payload);
|
|
2753
2834
|
}
|
|
2835
|
+
/** End the current attempt only; generation and agent spans belong to their owners. */
|
|
2836
|
+
endAttempt(termination) {
|
|
2837
|
+
termination = interruptedByRun(termination);
|
|
2838
|
+
const payload = this.#pendingStepFinishPayload;
|
|
2839
|
+
if (payload) this.#endStepSpan(payload, termination);
|
|
2840
|
+
else {
|
|
2841
|
+
this.#endChunkSpan();
|
|
2842
|
+
this.#currentInferenceSpan?.end(termination);
|
|
2843
|
+
this.#currentStepSpan?.end(termination);
|
|
2844
|
+
this.#currentStepSpan = void 0;
|
|
2845
|
+
this.#currentInferenceSpan = void 0;
|
|
2846
|
+
}
|
|
2847
|
+
this.#pendingStepFinishPayload = void 0;
|
|
2848
|
+
this.#inferenceCompletionStartTime = void 0;
|
|
2849
|
+
this.#currentStepInputIsFinal = false;
|
|
2850
|
+
}
|
|
2754
2851
|
#openInference(payload) {
|
|
2755
2852
|
if (!supportsModelInference()) return;
|
|
2756
2853
|
if (!this.#currentStepSpan || this.#currentInferenceSpan) return;
|
|
2757
2854
|
this.#inferenceCompletionStartTime = void 0;
|
|
2855
|
+
this.#inferenceTimeToFirstTokenMs = void 0;
|
|
2758
2856
|
const input = extractStepInput(payload);
|
|
2759
2857
|
const generationAttrs = this.#modelSpan?.attributes;
|
|
2760
2858
|
const ctx = this.#inferenceContext;
|
|
2859
|
+
this.#stepHasInference = true;
|
|
2761
2860
|
this.#currentInferenceSpan = this.#currentStepSpan.createChildSpan({
|
|
2762
2861
|
name: `inference: ${this.#stepIndex}`,
|
|
2763
2862
|
type: _mastra_core_observability.SpanType.MODEL_INFERENCE,
|
|
2863
|
+
...typeof _mastra_core_observability.agentLoopTermination === "function" ? { outcome: null } : {},
|
|
2764
2864
|
attributes: {
|
|
2765
2865
|
stepIndex: this.#stepIndex,
|
|
2766
|
-
model: generationAttrs?.model,
|
|
2767
|
-
provider: generationAttrs?.provider,
|
|
2866
|
+
model: payload?.modelId ?? generationAttrs?.model,
|
|
2867
|
+
provider: payload?.provider ?? generationAttrs?.provider,
|
|
2768
2868
|
streaming: generationAttrs?.streaming,
|
|
2769
2869
|
...ctx?.parameters !== void 0 ? { parameters: ctx.parameters } : {},
|
|
2770
2870
|
...ctx?.providerOptions !== void 0 ? { providerOptions: ctx.providerOptions } : {},
|
|
@@ -2781,11 +2881,37 @@ var ModelSpanTracker = class {
|
|
|
2781
2881
|
* End the open MODEL_INFERENCE span as interrupted while keeping MODEL_STEP open,
|
|
2782
2882
|
* so a replacement request for the same step gets its own inference span.
|
|
2783
2883
|
*/
|
|
2784
|
-
interruptInference() {
|
|
2884
|
+
interruptInference(termination) {
|
|
2785
2885
|
this.#endChunkSpan();
|
|
2786
2886
|
if (!this.#currentInferenceSpan) return;
|
|
2787
|
-
this.#currentInferenceSpan.end({
|
|
2887
|
+
this.#currentInferenceSpan.end({
|
|
2888
|
+
attributes: {
|
|
2889
|
+
finishReason: "interrupted",
|
|
2890
|
+
...this.#inferenceStreamAttributes()
|
|
2891
|
+
},
|
|
2892
|
+
...termination ?? (typeof _mastra_core_observability.agentLoopTermination === "function" ? {
|
|
2893
|
+
outcome: "interrupted",
|
|
2894
|
+
stopReason: null,
|
|
2895
|
+
stopReasonContext: null
|
|
2896
|
+
} : {})
|
|
2897
|
+
});
|
|
2788
2898
|
this.#currentInferenceSpan = void 0;
|
|
2899
|
+
this.#inferenceCompletionStartTime = void 0;
|
|
2900
|
+
this.#inferenceTimeToFirstTokenMs = void 0;
|
|
2901
|
+
}
|
|
2902
|
+
/** End the open inference span, if any, with `error` and what it streamed. */
|
|
2903
|
+
failInference(error) {
|
|
2904
|
+
this.#endChunkSpan();
|
|
2905
|
+
if (!this.#currentInferenceSpan) return;
|
|
2906
|
+
this.#currentInferenceSpan.error({
|
|
2907
|
+
error,
|
|
2908
|
+
..._mastra_core_observability.executionFailureTermination?.(error, "model_error"),
|
|
2909
|
+
endSpan: true,
|
|
2910
|
+
attributes: this.#inferenceStreamAttributes()
|
|
2911
|
+
});
|
|
2912
|
+
this.#inferenceTimeToFirstTokenMs = void 0;
|
|
2913
|
+
this.#currentInferenceSpan = void 0;
|
|
2914
|
+
this.#inferenceCompletionStartTime = void 0;
|
|
2789
2915
|
}
|
|
2790
2916
|
/**
|
|
2791
2917
|
* Update the current step span with additional payload data.
|
|
@@ -2807,10 +2933,11 @@ var ModelSpanTracker = class {
|
|
|
2807
2933
|
/**
|
|
2808
2934
|
* End the current Model execution step with token usage, finish reason, output, and metadata
|
|
2809
2935
|
*/
|
|
2810
|
-
#endStepSpan(payload) {
|
|
2936
|
+
#endStepSpan(payload, termination) {
|
|
2811
2937
|
this.#endChunkSpan();
|
|
2812
2938
|
if (!this.#currentStepSpan) return;
|
|
2813
|
-
const
|
|
2939
|
+
const output = payload.output;
|
|
2940
|
+
const { usage: rawUsage, ...otherOutput } = output;
|
|
2814
2941
|
const stepResult = payload.stepResult;
|
|
2815
2942
|
const metadata = payload.metadata;
|
|
2816
2943
|
const usage = extractUsageMetrics(rawUsage, metadata?.providerMetadata);
|
|
@@ -2826,6 +2953,11 @@ var ModelSpanTracker = class {
|
|
|
2826
2953
|
]) delete cleanMetadata[key];
|
|
2827
2954
|
this.#endInferenceSpan(payload);
|
|
2828
2955
|
this.#currentStepSpan.end({
|
|
2956
|
+
...termination ?? _mastra_core_observability.agentLoopTermination?.({
|
|
2957
|
+
stopCause: "model_finished",
|
|
2958
|
+
finishReason: stepResult.reason,
|
|
2959
|
+
tripwire: output.steps?.at(-1)?.tripwire
|
|
2960
|
+
}),
|
|
2829
2961
|
output: otherOutput,
|
|
2830
2962
|
attributes: {
|
|
2831
2963
|
usage,
|
|
@@ -2857,26 +2989,23 @@ var ModelSpanTracker = class {
|
|
|
2857
2989
|
warnings: []
|
|
2858
2990
|
},
|
|
2859
2991
|
metadata: {}
|
|
2860
|
-
}
|
|
2992
|
+
}, _mastra_core_observability.agentLoopTermination?.({
|
|
2993
|
+
finishReason: "suspended",
|
|
2994
|
+
suspension: {
|
|
2995
|
+
approvalRequired: chunk.type === "tool-call-approval",
|
|
2996
|
+
toolCallId: chunk.payload.toolCallId,
|
|
2997
|
+
toolName: chunk.payload.toolName
|
|
2998
|
+
}
|
|
2999
|
+
}));
|
|
2861
3000
|
}
|
|
2862
|
-
/**
|
|
2863
|
-
* Returns the parent span for chunks. Chunks parent under MODEL_INFERENCE
|
|
2864
|
-
* (the provider call) when available, falling back to MODEL_STEP only if
|
|
2865
|
-
* startStep() was bypassed.
|
|
2866
|
-
*/
|
|
3001
|
+
/** Processed content belongs to the logical step, independently of provider timing. */
|
|
2867
3002
|
#chunkParent() {
|
|
2868
|
-
return this.#
|
|
3003
|
+
return this.#currentStepSpan?.endTime ? void 0 : this.#currentStepSpan;
|
|
2869
3004
|
}
|
|
2870
|
-
/**
|
|
2871
|
-
* Safety-net invoked from chunk handlers: auto-create MODEL_STEP and
|
|
2872
|
-
* MODEL_INFERENCE if a chunk arrives before the loop has explicitly opened
|
|
2873
|
-
* them, so chunks parent under MODEL_INFERENCE rather than falling through
|
|
2874
|
-
* to MODEL_STEP. Idempotent — each public start* method is itself a no-op
|
|
2875
|
-
* when its span is already live.
|
|
2876
|
-
*/
|
|
3005
|
+
/** Ensure a logical step and provider timing span exist for legacy stream callers. */
|
|
2877
3006
|
#ensureStepAndInference() {
|
|
2878
3007
|
if (!this.#currentStepSpan) this.startStep();
|
|
2879
|
-
if (!this.#currentInferenceSpan) this.#openInference();
|
|
3008
|
+
if (!this.#currentInferenceSpan && !this.#stepHasInference) this.#openInference();
|
|
2880
3009
|
}
|
|
2881
3010
|
/**
|
|
2882
3011
|
* Create a new chunk span (for multi-part chunks like text-start/delta/end)
|
|
@@ -3037,20 +3166,39 @@ var ModelSpanTracker = class {
|
|
|
3037
3166
|
tracingPolicy: this.#modelSpan?.tracingPolicy
|
|
3038
3167
|
})) this.#chunkSequence++;
|
|
3039
3168
|
}
|
|
3169
|
+
/** Track provider timing and completion without recording unprocessed content. */
|
|
3170
|
+
wrapInferenceStream(stream) {
|
|
3171
|
+
this.#tracksProviderStream = true;
|
|
3172
|
+
return this.wrapStream(stream, true);
|
|
3173
|
+
}
|
|
3040
3174
|
/**
|
|
3041
3175
|
* Wraps a stream with model tracing transform to track MODEL_STEP and MODEL_CHUNK spans.
|
|
3042
3176
|
*
|
|
3043
3177
|
* This should be added to the stream pipeline to automatically
|
|
3044
3178
|
* create MODEL_STEP and MODEL_CHUNK spans for each semantic unit in the stream.
|
|
3045
3179
|
*/
|
|
3046
|
-
wrapStream(stream) {
|
|
3180
|
+
wrapStream(stream, inferenceOnly = false) {
|
|
3047
3181
|
return stream.pipeThrough(new stream_web.TransformStream({ transform: (chunk, controller) => {
|
|
3048
|
-
if (
|
|
3182
|
+
if (inferenceOnly) {
|
|
3183
|
+
try {
|
|
3184
|
+
if (MODEL_RESPONSE_CHUNK_TYPES.has(chunk.type)) this.#captureCompletionStartTime();
|
|
3185
|
+
if (chunk.type === "finish") this.#endInferenceSpan(chunk.payload, false);
|
|
3186
|
+
} catch {}
|
|
3187
|
+
controller.enqueue(chunk);
|
|
3188
|
+
return;
|
|
3189
|
+
}
|
|
3190
|
+
this.recordChunk(chunk);
|
|
3191
|
+
controller.enqueue(chunk);
|
|
3192
|
+
} }));
|
|
3193
|
+
}
|
|
3194
|
+
/** @internal Record a chunk only after output processors accept it. */
|
|
3195
|
+
recordChunk(chunk) {
|
|
3196
|
+
try {
|
|
3197
|
+
if (!this.#tracksProviderStream && (this.#inModelResponse || this.#deferStepClose) && MODEL_RESPONSE_CHUNK_TYPES.has(chunk.type)) this.#captureCompletionStartTime();
|
|
3049
3198
|
if (chunk.type === "tool-call-suspended" || chunk.type === "tool-call-approval") {
|
|
3050
3199
|
if (chunk.type === "tool-call-approval") this.#handleToolApprovalChunk(chunk);
|
|
3051
3200
|
this.#endSuspendedStep(chunk);
|
|
3052
3201
|
}
|
|
3053
|
-
controller.enqueue(chunk);
|
|
3054
3202
|
switch (chunk.type) {
|
|
3055
3203
|
case "text-start":
|
|
3056
3204
|
case "text-delta":
|
|
@@ -3076,7 +3224,7 @@ var ModelSpanTracker = class {
|
|
|
3076
3224
|
this.#inModelResponse = true;
|
|
3077
3225
|
if (this.#currentStepSpan) this.updateStep(chunk.payload);
|
|
3078
3226
|
else this.startStep(chunk.payload);
|
|
3079
|
-
if (!this.#currentInferenceSpan) this.#openInference(chunk.payload);
|
|
3227
|
+
if (!this.#currentInferenceSpan && !this.#stepHasInference) this.#openInference(chunk.payload);
|
|
3080
3228
|
break;
|
|
3081
3229
|
case "step-finish":
|
|
3082
3230
|
this.#inModelResponse = false;
|
|
@@ -3086,15 +3234,20 @@ var ModelSpanTracker = class {
|
|
|
3086
3234
|
this.#endInferenceSpan(chunk.payload);
|
|
3087
3235
|
} else this.#endStepSpan(chunk.payload);
|
|
3088
3236
|
break;
|
|
3237
|
+
case "tripwire": {
|
|
3238
|
+
const termination = _mastra_core_observability.agentLoopTermination?.({ tripwire: chunk.payload });
|
|
3239
|
+
if (!termination) break;
|
|
3240
|
+
this.endAttempt(termination);
|
|
3241
|
+
break;
|
|
3242
|
+
}
|
|
3243
|
+
case "finish": break;
|
|
3089
3244
|
case "raw":
|
|
3090
3245
|
case "start":
|
|
3091
|
-
case "finish":
|
|
3092
3246
|
case "response-metadata":
|
|
3093
3247
|
case "source":
|
|
3094
3248
|
case "file":
|
|
3095
3249
|
case "error":
|
|
3096
3250
|
case "abort":
|
|
3097
|
-
case "tripwire":
|
|
3098
3251
|
case "watch":
|
|
3099
3252
|
case "tool-error":
|
|
3100
3253
|
case "reasoning-signature":
|
|
@@ -3119,7 +3272,7 @@ var ModelSpanTracker = class {
|
|
|
3119
3272
|
break;
|
|
3120
3273
|
}
|
|
3121
3274
|
}
|
|
3122
|
-
} }
|
|
3275
|
+
} catch {}
|
|
3123
3276
|
}
|
|
3124
3277
|
};
|
|
3125
3278
|
// src/spans/metadata.ts
|
|
@@ -3308,6 +3461,9 @@ var BaseSpan = class BaseSpan {
|
|
|
3308
3461
|
observabilityInstance;
|
|
3309
3462
|
input;
|
|
3310
3463
|
output;
|
|
3464
|
+
outcome;
|
|
3465
|
+
stopReason;
|
|
3466
|
+
stopReasonContext;
|
|
3311
3467
|
errorInfo;
|
|
3312
3468
|
metadata;
|
|
3313
3469
|
requestContext;
|
|
@@ -3324,11 +3480,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3324
3480
|
parentSpanId;
|
|
3325
3481
|
/** Parent from an external tracing system (ambient OTel / dd-trace span) that Mastra did not create; carried for external correlation, not part of Mastra's own parent/child linkage */
|
|
3326
3482
|
externalParentSpanId;
|
|
3327
|
-
/** Native span fields (see `SpanData`); carried as supplied and exported only when set */
|
|
3328
3483
|
resumedFromSpanId;
|
|
3329
|
-
outcome;
|
|
3330
|
-
stopReason;
|
|
3331
|
-
stopReasonContext;
|
|
3332
3484
|
/** Deep clean options for serialization */
|
|
3333
3485
|
deepCleanOptions;
|
|
3334
3486
|
/**
|
|
@@ -3356,6 +3508,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3356
3508
|
* live `parent`, so its original ancestors are only known from the cached export.
|
|
3357
3509
|
*/
|
|
3358
3510
|
restoredLineage;
|
|
3511
|
+
restoredIsRootSpan;
|
|
3359
3512
|
/**
|
|
3360
3513
|
* Version declared for this span's own entity, or inherited from a parent span
|
|
3361
3514
|
* of the same entity. Unlike `metadata.entityVersionId`, never an ancestor's
|
|
@@ -3376,6 +3529,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3376
3529
|
constructor(options, observabilityInstance) {
|
|
3377
3530
|
const observabilityConfig = observabilityInstance.getConfig();
|
|
3378
3531
|
this.deepCleanOptions = mergeSerializationOptions(observabilityConfig.serializationOptions);
|
|
3532
|
+
this.applyOutcome(options);
|
|
3379
3533
|
this.name = options.name;
|
|
3380
3534
|
this.type = options.type;
|
|
3381
3535
|
this.isInternal = isSpanInternal(this.type, options.tracingPolicy?.internal);
|
|
@@ -3396,9 +3550,6 @@ var BaseSpan = class BaseSpan {
|
|
|
3396
3550
|
this.tags = !options.parent && options.tags?.length ? options.tags : void 0;
|
|
3397
3551
|
this.links = validLinks(options.links);
|
|
3398
3552
|
this.resumedFromSpanId = options.resumedFromSpanId;
|
|
3399
|
-
this.outcome = options.outcome;
|
|
3400
|
-
this.stopReason = options.stopReason;
|
|
3401
|
-
this.stopReasonContext = deepClean(options.stopReasonContext, this.deepCleanOptions);
|
|
3402
3553
|
const entityParent = this.getParentSpan(false);
|
|
3403
3554
|
this.entityType = options.entityType ?? entityParent?.entityType;
|
|
3404
3555
|
this.entityId = options.entityId ?? entityParent?.entityId;
|
|
@@ -3440,14 +3591,49 @@ var BaseSpan = class BaseSpan {
|
|
|
3440
3591
|
return value;
|
|
3441
3592
|
}
|
|
3442
3593
|
}
|
|
3443
|
-
/**
|
|
3594
|
+
/** Preserve explicitly supplied native fields; never derive results from payloads/errors. */
|
|
3595
|
+
applyOutcome(options) {
|
|
3596
|
+
if (this.endTime) return;
|
|
3597
|
+
if (options.outcome !== void 0) {
|
|
3598
|
+
this.outcome = options.outcome;
|
|
3599
|
+
this.stopReason = options.stopReason ?? null;
|
|
3600
|
+
this.stopReasonContext = options.stopReasonContext == null ? null : deepClean(options.stopReasonContext, this.deepCleanOptions);
|
|
3601
|
+
return;
|
|
3602
|
+
}
|
|
3603
|
+
if (options.stopReason !== void 0 && options.stopReason !== this.stopReason) this.stopReasonContext = null;
|
|
3604
|
+
if (options.stopReason !== void 0) this.stopReason = options.stopReason;
|
|
3605
|
+
if (options.stopReasonContext !== void 0) this.stopReasonContext = options.stopReasonContext == null ? null : deepClean(options.stopReasonContext, this.deepCleanOptions);
|
|
3606
|
+
}
|
|
3607
|
+
/** Forward options to open descendants, with interruption outcomes for cascading termination. */
|
|
3444
3608
|
endTree(options) {
|
|
3445
|
-
|
|
3609
|
+
this.endOpenDescendants(options, true);
|
|
3446
3610
|
this.end(options);
|
|
3447
3611
|
}
|
|
3448
|
-
/**
|
|
3449
|
-
endOpenDescendants() {
|
|
3450
|
-
|
|
3612
|
+
/** Preserve completed children; only known cascading termination interrupts open work. */
|
|
3613
|
+
endOpenDescendants(options, forwardOptions = false) {
|
|
3614
|
+
const outcome = options?.outcome !== void 0 ? options.outcome : this.outcome;
|
|
3615
|
+
const stopReason = options?.stopReason !== void 0 ? options.stopReason : this.stopReason;
|
|
3616
|
+
const stopReasonContext = options?.stopReasonContext !== void 0 ? options.stopReasonContext : this.stopReasonContext;
|
|
3617
|
+
const interrupts = outcome === "abort" || outcome === "interrupted" || outcome === "limit" && stopReason === "timeout";
|
|
3618
|
+
const forwarded = forwardOptions ? { ...options } : {};
|
|
3619
|
+
delete forwarded.outcome;
|
|
3620
|
+
delete forwarded.stopReason;
|
|
3621
|
+
delete forwarded.stopReasonContext;
|
|
3622
|
+
if (this.#openChildren) for (const child of [...this.#openChildren]) {
|
|
3623
|
+
const childOptions = {
|
|
3624
|
+
...forwarded,
|
|
3625
|
+
...interrupts ? {
|
|
3626
|
+
outcome: "interrupted",
|
|
3627
|
+
stopReason,
|
|
3628
|
+
stopReasonContext
|
|
3629
|
+
} : {}
|
|
3630
|
+
};
|
|
3631
|
+
if (forwardOptions) child.endTree(childOptions);
|
|
3632
|
+
else child.end({
|
|
3633
|
+
...childOptions,
|
|
3634
|
+
endTree: true
|
|
3635
|
+
});
|
|
3636
|
+
}
|
|
3451
3637
|
}
|
|
3452
3638
|
/**
|
|
3453
3639
|
* Release this span from its parent's open-child set once it has ended.
|
|
@@ -3498,7 +3684,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3498
3684
|
}
|
|
3499
3685
|
/** Returns `TRUE` if the span is the root span of a trace */
|
|
3500
3686
|
get isRootSpan() {
|
|
3501
|
-
return !this.parent;
|
|
3687
|
+
return this.restoredIsRootSpan ?? !this.parent;
|
|
3502
3688
|
}
|
|
3503
3689
|
/**
|
|
3504
3690
|
* Get the closest parent span.
|
|
@@ -3562,6 +3748,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3562
3748
|
*/
|
|
3563
3749
|
restoreEntityLineage(cached) {
|
|
3564
3750
|
this.ownEntityVersionId = cached.ownEntityVersionId ?? void 0;
|
|
3751
|
+
this.restoredIsRootSpan = cached.isRootSpan;
|
|
3565
3752
|
const lineage = definedLineage(cached);
|
|
3566
3753
|
if (Object.keys(lineage).length > 0) {
|
|
3567
3754
|
this.restoredLineage = lineage;
|
|
@@ -3644,6 +3831,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3644
3831
|
const nestedUnderParent = this.traceState?.nestedUnderParent ?? false;
|
|
3645
3832
|
const links = validLinks(this.links);
|
|
3646
3833
|
const correlation = this.getCorrelationContext();
|
|
3834
|
+
const timeToFirstTokenMs = this.attributes?.timeToFirstTokenMs;
|
|
3647
3835
|
return {
|
|
3648
3836
|
id: this.id,
|
|
3649
3837
|
traceId: this.traceId,
|
|
@@ -3659,6 +3847,9 @@ var BaseSpan = class BaseSpan {
|
|
|
3659
3847
|
input: hideInput ? void 0 : this.input,
|
|
3660
3848
|
output: hideOutput ? void 0 : this.output,
|
|
3661
3849
|
errorInfo: this.errorInfo,
|
|
3850
|
+
...this.outcome !== void 0 ? { outcome: this.outcome } : {},
|
|
3851
|
+
...this.stopReason !== void 0 ? { stopReason: this.stopReason } : {},
|
|
3852
|
+
...this.stopReasonContext !== void 0 ? { stopReasonContext: this.stopReasonContext } : {},
|
|
3662
3853
|
requestContext: this.requestContext,
|
|
3663
3854
|
isEvent: this.isEvent,
|
|
3664
3855
|
isRootSpan: this.isRootSpan,
|
|
@@ -3668,9 +3859,7 @@ var BaseSpan = class BaseSpan {
|
|
|
3668
3859
|
parentSpanId: this.getParentSpanId(includeInternalSpans),
|
|
3669
3860
|
externalParentSpanId: this.getExportedExternalParentSpanId(includeInternalSpans),
|
|
3670
3861
|
...this.resumedFromSpanId !== void 0 ? { resumedFromSpanId: this.resumedFromSpanId } : {},
|
|
3671
|
-
...
|
|
3672
|
-
...this.stopReason !== void 0 ? { stopReason: this.stopReason } : {},
|
|
3673
|
-
...this.stopReasonContext !== void 0 ? { stopReasonContext: this.stopReasonContext } : {},
|
|
3862
|
+
...typeof timeToFirstTokenMs === "number" ? { timeToFirstTokenMs } : {},
|
|
3674
3863
|
...nestedUnderParent ? { nestedUnderParent: true } : {},
|
|
3675
3864
|
...links ? { links } : {},
|
|
3676
3865
|
...this.isRootSpan && !nestedUnderParent && this.tags?.length ? { tags: this.tags } : {}
|
|
@@ -3780,7 +3969,9 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
3780
3969
|
end(options) {
|
|
3781
3970
|
if (this.isEvent) return;
|
|
3782
3971
|
if (this.endTime) return;
|
|
3783
|
-
if (options?.endTree) this.endOpenDescendants();
|
|
3972
|
+
if (options?.endTree) this.endOpenDescendants(options);
|
|
3973
|
+
if (options) this.applyOutcome(options);
|
|
3974
|
+
if (this.outcome === null) this.outcome = "unknown";
|
|
3784
3975
|
this.endTime = /* @__PURE__ */ new Date();
|
|
3785
3976
|
this.detachFromParent();
|
|
3786
3977
|
if (options?.metadata) this.metadata = {
|
|
@@ -3795,8 +3986,9 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
3795
3986
|
};
|
|
3796
3987
|
}
|
|
3797
3988
|
error(options) {
|
|
3798
|
-
if (this.isEvent) return;
|
|
3989
|
+
if (this.isEvent || this.endTime) return;
|
|
3799
3990
|
const { error, endSpan = true, endTree, attributes, metadata } = options;
|
|
3991
|
+
this.applyOutcome(options);
|
|
3800
3992
|
if (metadata) this.metadata = {
|
|
3801
3993
|
...this.metadata,
|
|
3802
3994
|
...deepClean(this.prepareSpanMetadata(metadata), this.deepCleanOptions)
|
|
@@ -3831,6 +4023,7 @@ var DefaultSpan = class extends BaseSpan {
|
|
|
3831
4023
|
}
|
|
3832
4024
|
update(options) {
|
|
3833
4025
|
if (this.isEvent) return;
|
|
4026
|
+
this.applyOutcome(options);
|
|
3834
4027
|
if (options.name !== void 0) this.name = options.name;
|
|
3835
4028
|
if (options.metadata) this.metadata = {
|
|
3836
4029
|
...this.metadata,
|
|
@@ -5300,13 +5493,13 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
|
|
|
5300
5493
|
...rest,
|
|
5301
5494
|
metadata
|
|
5302
5495
|
}, this);
|
|
5303
|
-
} else if (!this.shouldSample(customSamplerOptions)) return new NoOpSpan({
|
|
5496
|
+
} else if (!(options.samplingDecision ?? this.shouldSample(customSamplerOptions))) return new NoOpSpan({
|
|
5304
5497
|
...rest,
|
|
5305
5498
|
metadata
|
|
5306
5499
|
}, this);
|
|
5307
5500
|
let traceState;
|
|
5308
5501
|
if (options.parent) traceState = options.parent.traceState;
|
|
5309
|
-
else traceState = this.computeTraceState(tracingOptions);
|
|
5502
|
+
else traceState = options.traceState ?? this.computeTraceState(tracingOptions);
|
|
5310
5503
|
const tracingMetadata = !options.parent ? tracingOptions?.metadata : void 0;
|
|
5311
5504
|
const mergedMetadata = mergeMetadata$1(stripUndefined(metadata), stripUndefined(tracingMetadata));
|
|
5312
5505
|
const enrichedMetadata = this.extractMetadataFromRequestContext(requestContext, mergedMetadata, traceState);
|
|
@@ -5361,6 +5554,11 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
|
|
|
5361
5554
|
* @returns A span that can have lifecycle methods called on it
|
|
5362
5555
|
*/
|
|
5363
5556
|
rebuildSpan(cached) {
|
|
5557
|
+
if (cached.id === "no-op" && cached.traceId === "no-op-trace") return new NoOpSpan({
|
|
5558
|
+
name: cached.name,
|
|
5559
|
+
type: cached.type,
|
|
5560
|
+
traceState: cached.traceState
|
|
5561
|
+
}, this);
|
|
5364
5562
|
const span = this.createSpan({
|
|
5365
5563
|
name: cached.name,
|
|
5366
5564
|
type: cached.type,
|
|
@@ -5384,6 +5582,11 @@ var BaseObservabilityInstance = class extends _mastra_core_base.MastraBase {
|
|
|
5384
5582
|
traceState: this.rebuildTraceState(cached)
|
|
5385
5583
|
});
|
|
5386
5584
|
if (span instanceof BaseSpan) span.restoreEntityLineage(cached);
|
|
5585
|
+
if (cached.endTime) {
|
|
5586
|
+
span.endTime = cached.endTime instanceof Date ? cached.endTime : new Date(cached.endTime);
|
|
5587
|
+
span.output = cached.output;
|
|
5588
|
+
span.errorInfo = cached.errorInfo;
|
|
5589
|
+
}
|
|
5387
5590
|
this.wireSpanLifecycle(span);
|
|
5388
5591
|
return span;
|
|
5389
5592
|
}
|
|
@@ -9148,7 +9351,11 @@ var TestExporter = class extends BaseExporter {
|
|
|
9148
9351
|
if (span.parentSpanId && spanIdMap.has(span.parentSpanId)) normalizedSpan.parentId = spanIdMap.get(span.parentSpanId);
|
|
9149
9352
|
if (span.entityType) normalizedSpan.entityType = span.entityType;
|
|
9150
9353
|
if (span.entityId) normalizedSpan.entityId = normalizeValue(span.entityId, "entityId");
|
|
9151
|
-
if (span.attributes && Object.keys(span.attributes).length > 0)
|
|
9354
|
+
if (span.attributes && Object.keys(span.attributes).length > 0) {
|
|
9355
|
+
const attributes = normalizeValue(span.attributes);
|
|
9356
|
+
if (typeof attributes.timeToFirstTokenMs === "number") attributes.timeToFirstTokenMs = "<ms>";
|
|
9357
|
+
normalizedSpan.attributes = attributes;
|
|
9358
|
+
}
|
|
9152
9359
|
if (span.metadata && Object.keys(span.metadata).length > 0) normalizedSpan.metadata = normalizeValue(span.metadata);
|
|
9153
9360
|
if (span.input !== void 0) normalizedSpan.input = normalizeValue(span.input);
|
|
9154
9361
|
if (span.output !== void 0) normalizedSpan.output = normalizeValue(span.output);
|
|
@@ -10040,6 +10247,7 @@ var SensitiveDataFilter = class {
|
|
|
10040
10247
|
span.input = this.tryFilter(span.input, indexedState);
|
|
10041
10248
|
span.output = this.tryFilter(span.output, indexedState);
|
|
10042
10249
|
span.errorInfo = this.tryFilter(span.errorInfo, indexedState);
|
|
10250
|
+
span.stopReasonContext = this.tryFilter(span.stopReasonContext, indexedState);
|
|
10043
10251
|
span.requestContext = this.tryFilter(span.requestContext, indexedState);
|
|
10044
10252
|
return span;
|
|
10045
10253
|
}
|