@mastra/datadog 1.3.6-alpha.0 → 1.3.7-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.cjs +1113 -1178
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1087 -1170
- package/dist/index.js.map +1 -1
- package/package.json +7 -7
package/dist/index.cjs
CHANGED
|
@@ -1,1233 +1,1168 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var
|
|
6
|
-
var
|
|
7
|
-
var
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
//#region \0rolldown/runtime.js
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
11
|
+
key = keys[i];
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
13
|
+
get: ((k) => from[k]).bind(null, key),
|
|
14
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
return to;
|
|
18
|
+
};
|
|
19
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
20
|
+
value: mod,
|
|
21
|
+
enumerable: true
|
|
22
|
+
}) : target, mod));
|
|
23
|
+
//#endregion
|
|
24
|
+
let _mastra_core_observability = require("@mastra/core/observability");
|
|
25
|
+
let _mastra_core_utils = require("@mastra/core/utils");
|
|
26
|
+
let _mastra_observability = require("@mastra/observability");
|
|
27
|
+
let dd_trace = require("dd-trace");
|
|
28
|
+
dd_trace = __toESM(dd_trace, 1);
|
|
29
|
+
let _mastra_core_features = require("@mastra/core/features");
|
|
30
|
+
//#region src/features.ts
|
|
31
|
+
/**
|
|
32
|
+
* Feature detection for paired @mastra/core and @mastra/observability versions.
|
|
33
|
+
*
|
|
34
|
+
* `coreFeatures` is a hard peer dependency, so it can be imported statically.
|
|
35
|
+
* `observabilityFeatures` is loaded via dynamic import so this exporter degrades
|
|
36
|
+
* gracefully against older `@mastra/observability` versions that don't export it
|
|
37
|
+
* (per the pattern documented in `observability/mastra/src/features.ts`).
|
|
38
|
+
*
|
|
39
|
+
* Detection runs once at module load. Callers consume the result through the
|
|
40
|
+
* sync `isModelInferenceEnabled()` accessor, which conservatively returns
|
|
41
|
+
* `false` until detection settles — long enough only to cover the microtask
|
|
42
|
+
* window before any spans are emitted in practice.
|
|
43
|
+
*/
|
|
44
|
+
const FEATURE = "model-inference-span";
|
|
45
|
+
let observabilityFeatures;
|
|
46
|
+
let featureLoadPromise;
|
|
17
47
|
function loadObservabilityFeatures() {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
});
|
|
23
|
-
}
|
|
24
|
-
return featureLoadPromise;
|
|
48
|
+
if (!featureLoadPromise) featureLoadPromise = import("@mastra/observability").then((mod) => {
|
|
49
|
+
observabilityFeatures = mod.observabilityFeatures;
|
|
50
|
+
}).catch(() => {});
|
|
51
|
+
return featureLoadPromise;
|
|
25
52
|
}
|
|
26
|
-
|
|
53
|
+
loadObservabilityFeatures();
|
|
54
|
+
/**
|
|
55
|
+
* Returns true when both packages report the `model-inference-span` feature,
|
|
56
|
+
* meaning MODEL_INFERENCE spans are emitted by the tracker. Drives the
|
|
57
|
+
* Datadog kind mapping and usage-source switch (legacy: MODEL_STEP carries
|
|
58
|
+
* 'llm' kind + usage; new: MODEL_INFERENCE does).
|
|
59
|
+
*/
|
|
27
60
|
function isModelInferenceEnabled() {
|
|
28
|
-
|
|
61
|
+
return observabilityFeatures?.has(FEATURE) === true && _mastra_core_features.coreFeatures.has(FEATURE);
|
|
29
62
|
}
|
|
30
|
-
|
|
31
|
-
|
|
63
|
+
//#endregion
|
|
64
|
+
//#region src/metrics.ts
|
|
32
65
|
function formatUsageMetrics(usage) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
if (cacheReadTokens !== void 0) {
|
|
47
|
-
result.cacheReadTokens = cacheReadTokens;
|
|
48
|
-
}
|
|
49
|
-
const cacheWriteTokens = usage?.inputDetails?.cacheWrite;
|
|
50
|
-
if (cacheWriteTokens !== void 0) {
|
|
51
|
-
result.cacheWriteTokens = cacheWriteTokens;
|
|
52
|
-
}
|
|
53
|
-
return Object.keys(result).length > 0 ? result : void 0;
|
|
66
|
+
if (!usage) return void 0;
|
|
67
|
+
const result = {};
|
|
68
|
+
const inputTokens = usage.inputTokens;
|
|
69
|
+
if (inputTokens !== void 0) result.inputTokens = inputTokens;
|
|
70
|
+
const outputTokens = usage.outputTokens;
|
|
71
|
+
if (outputTokens !== void 0) result.outputTokens = outputTokens;
|
|
72
|
+
if (inputTokens !== void 0 && outputTokens !== void 0) result.totalTokens = inputTokens + outputTokens;
|
|
73
|
+
if (usage?.outputDetails?.reasoning !== void 0) result.reasoningOutputTokens = usage.outputDetails.reasoning;
|
|
74
|
+
const cacheReadTokens = usage?.inputDetails?.cacheRead;
|
|
75
|
+
if (cacheReadTokens !== void 0) result.cacheReadTokens = cacheReadTokens;
|
|
76
|
+
const cacheWriteTokens = usage?.inputDetails?.cacheWrite;
|
|
77
|
+
if (cacheWriteTokens !== void 0) result.cacheWriteTokens = cacheWriteTokens;
|
|
78
|
+
return Object.keys(result).length > 0 ? result : void 0;
|
|
54
79
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
80
|
+
//#endregion
|
|
81
|
+
//#region src/utils.ts
|
|
82
|
+
/**
|
|
83
|
+
* Utility functions for Datadog LLM Observability Exporter
|
|
84
|
+
*/
|
|
85
|
+
/**
|
|
86
|
+
* Maps Mastra SpanTypes to Datadog LLMObs span kinds for the legacy hierarchy
|
|
87
|
+
* (no `model-inference-span` feature). MODEL_STEP is the actual API call here
|
|
88
|
+
* because MODEL_INFERENCE doesn't exist.
|
|
89
|
+
*
|
|
90
|
+
* Unmapped types fall back to 'task'.
|
|
91
|
+
*/
|
|
92
|
+
const SPAN_TYPE_TO_KIND_LEGACY = {
|
|
93
|
+
[_mastra_core_observability.SpanType.AGENT_RUN]: "agent",
|
|
94
|
+
[_mastra_core_observability.SpanType.MODEL_GENERATION]: "workflow",
|
|
95
|
+
[_mastra_core_observability.SpanType.MODEL_STEP]: "llm",
|
|
96
|
+
[_mastra_core_observability.SpanType.TOOL_CALL]: "tool",
|
|
97
|
+
[_mastra_core_observability.SpanType.MCP_TOOL_CALL]: "tool",
|
|
98
|
+
[_mastra_core_observability.SpanType.PROVIDER_TOOL_CALL]: "tool",
|
|
99
|
+
[_mastra_core_observability.SpanType.WORKFLOW_RUN]: "workflow"
|
|
63
100
|
};
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
101
|
+
/**
|
|
102
|
+
* Maps Mastra SpanTypes to Datadog LLMObs span kinds for the new hierarchy
|
|
103
|
+
* (`model-inference-span` feature). MODEL_INFERENCE is the LLM API call;
|
|
104
|
+
* MODEL_STEP wraps processors + inference + tool execution as a workflow.
|
|
105
|
+
*
|
|
106
|
+
* Unmapped types fall back to 'task'.
|
|
107
|
+
*/
|
|
108
|
+
const SPAN_TYPE_TO_KIND_INFERENCE = {
|
|
109
|
+
[_mastra_core_observability.SpanType.AGENT_RUN]: "agent",
|
|
110
|
+
[_mastra_core_observability.SpanType.MODEL_GENERATION]: "workflow",
|
|
111
|
+
[_mastra_core_observability.SpanType.MODEL_STEP]: "workflow",
|
|
112
|
+
[_mastra_core_observability.SpanType.MODEL_INFERENCE]: "llm",
|
|
113
|
+
[_mastra_core_observability.SpanType.TOOL_CALL]: "tool",
|
|
114
|
+
[_mastra_core_observability.SpanType.MCP_TOOL_CALL]: "tool",
|
|
115
|
+
[_mastra_core_observability.SpanType.PROVIDER_TOOL_CALL]: "tool",
|
|
116
|
+
[_mastra_core_observability.SpanType.WORKFLOW_RUN]: "workflow"
|
|
73
117
|
};
|
|
118
|
+
/**
|
|
119
|
+
* Resolves the active span-type → Datadog kind mapping based on whether the
|
|
120
|
+
* paired @mastra/core + @mastra/observability emit MODEL_INFERENCE spans.
|
|
121
|
+
* Re-evaluated on each call so tests can flip the feature flag at runtime.
|
|
122
|
+
*/
|
|
74
123
|
function getSpanTypeToKind() {
|
|
75
|
-
|
|
124
|
+
return isModelInferenceEnabled() ? SPAN_TYPE_TO_KIND_INFERENCE : SPAN_TYPE_TO_KIND_LEGACY;
|
|
76
125
|
}
|
|
77
|
-
|
|
126
|
+
/**
|
|
127
|
+
* Singleton flag to prevent multiple tracer initializations.
|
|
128
|
+
* dd-trace should only be initialized once per process.
|
|
129
|
+
*/
|
|
130
|
+
const tracerInitFlag = { done: false };
|
|
131
|
+
/**
|
|
132
|
+
* Ensures dd-trace is initialized exactly once.
|
|
133
|
+
* Respects any existing tracer initialization by the application.
|
|
134
|
+
*/
|
|
78
135
|
function ensureTracer(config) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
plugins: config.integrationsEnabled ?? false
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
tracer3__default.default.llmobs.enable({
|
|
96
|
-
mlApp: config.mlApp,
|
|
97
|
-
agentlessEnabled: config.agentless
|
|
98
|
-
});
|
|
99
|
-
tracerInitFlag.done = true;
|
|
136
|
+
if (tracerInitFlag.done) return;
|
|
137
|
+
if (config.site) process.env.DD_SITE = config.site;
|
|
138
|
+
if (config.apiKey) process.env.DD_API_KEY = config.apiKey;
|
|
139
|
+
if (!dd_trace.default._tracer?.started) dd_trace.default.init({
|
|
140
|
+
service: config.service || config.mlApp,
|
|
141
|
+
env: config.env || process.env.DD_ENV,
|
|
142
|
+
plugins: config.integrationsEnabled ?? false
|
|
143
|
+
});
|
|
144
|
+
dd_trace.default.llmobs.enable({
|
|
145
|
+
mlApp: config.mlApp,
|
|
146
|
+
agentlessEnabled: config.agentless
|
|
147
|
+
});
|
|
148
|
+
tracerInitFlag.done = true;
|
|
100
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* Returns the Datadog kind for a Mastra span type, using the mapping that
|
|
152
|
+
* matches the active span hierarchy (legacy vs MODEL_INFERENCE).
|
|
153
|
+
*/
|
|
101
154
|
function kindFor(spanType) {
|
|
102
|
-
|
|
155
|
+
return getSpanTypeToKind()[spanType] || "task";
|
|
103
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* Converts a value to a Date object.
|
|
159
|
+
*/
|
|
104
160
|
function toDate(value) {
|
|
105
|
-
|
|
161
|
+
return value instanceof Date ? value : new Date(value);
|
|
106
162
|
}
|
|
163
|
+
/**
|
|
164
|
+
* Safely stringifies data, handling circular references.
|
|
165
|
+
*/
|
|
107
166
|
function safeStringify(data) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
return String(data);
|
|
115
|
-
}
|
|
167
|
+
try {
|
|
168
|
+
return JSON.stringify(data) ?? "";
|
|
169
|
+
} catch {
|
|
170
|
+
if (typeof data === "object" && data !== null) return `[Non-serializable ${data.constructor?.name || "Object"}]`;
|
|
171
|
+
return String(data);
|
|
172
|
+
}
|
|
116
173
|
}
|
|
174
|
+
/**
|
|
175
|
+
* Checks if data is already in message array format ({role, content}[]).
|
|
176
|
+
*/
|
|
117
177
|
function isMessageArray(data) {
|
|
118
|
-
|
|
178
|
+
return Array.isArray(data) && data.every((m) => m?.role && (m?.content !== void 0 || Array.isArray(m.toolCalls)));
|
|
119
179
|
}
|
|
120
180
|
function isModelDataSpan(spanType) {
|
|
121
|
-
|
|
181
|
+
return spanType === _mastra_core_observability.SpanType.MODEL_GENERATION || spanType === _mastra_core_observability.SpanType.MODEL_STEP || spanType === _mastra_core_observability.SpanType.MODEL_INFERENCE;
|
|
122
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* Checks if data is in Gemini content array format ({role, parts}[]).
|
|
185
|
+
*/
|
|
123
186
|
function isGeminiContentArray(data) {
|
|
124
|
-
|
|
187
|
+
return Array.isArray(data) && data.every((m) => m?.role && Array.isArray(m?.parts));
|
|
125
188
|
}
|
|
126
189
|
function toMessageContent(content) {
|
|
127
|
-
|
|
190
|
+
return typeof content === "string" ? content : safeStringify(content);
|
|
128
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* Maps a {role, content}[] message array into the Datadog message shape,
|
|
194
|
+
* stringifying any non-string content (e.g. multimodal part arrays).
|
|
195
|
+
*/
|
|
129
196
|
function toDatadogMessages(messages) {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
return message;
|
|
139
|
-
}).filter((m) => !(m.role === "user" && m.content.trim().length === 0));
|
|
197
|
+
return messages.map((m) => {
|
|
198
|
+
const message = {
|
|
199
|
+
role: m.role,
|
|
200
|
+
content: m.content == null ? "" : toMessageContent(m.content)
|
|
201
|
+
};
|
|
202
|
+
if (Array.isArray(m.toolCalls) && m.toolCalls.length > 0) message.toolCalls = toDatadogToolCalls(m.toolCalls);
|
|
203
|
+
return message;
|
|
204
|
+
}).filter((m) => !(m.role === "user" && m.content.trim().length === 0));
|
|
140
205
|
}
|
|
141
206
|
function parseToolArguments(args) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
-
if (typeof args === "object" && !Array.isArray(args)) return args;
|
|
154
|
-
return { value: args };
|
|
207
|
+
if (args == null) return void 0;
|
|
208
|
+
if (typeof args === "string") {
|
|
209
|
+
try {
|
|
210
|
+
const parsed = JSON.parse(args);
|
|
211
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
|
|
212
|
+
} catch {}
|
|
213
|
+
return { value: args };
|
|
214
|
+
}
|
|
215
|
+
if (typeof args === "object" && !Array.isArray(args)) return args;
|
|
216
|
+
return { value: args };
|
|
155
217
|
}
|
|
156
218
|
function toDatadogToolCalls(toolCalls) {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
219
|
+
return toolCalls.map((c) => ({
|
|
220
|
+
name: c?.toolName ?? c?.name ?? c?.function?.name ?? "unknown",
|
|
221
|
+
arguments: parseToolArguments(c?.args ?? c?.input ?? c?.arguments ?? c?.function?.arguments),
|
|
222
|
+
toolId: c?.toolCallId ?? c?.toolId ?? c?.id,
|
|
223
|
+
type: c?.type === "function" ? c.type : "function"
|
|
224
|
+
}));
|
|
163
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* Converts a Gemini content item to Datadog message format.
|
|
228
|
+
* Extracts text from parts, skips binary data to avoid bloating traces.
|
|
229
|
+
*/
|
|
164
230
|
function geminiContentToMessage(item) {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
231
|
+
const text = item.parts.map((p) => {
|
|
232
|
+
if (typeof p === "string") return p;
|
|
233
|
+
if (p?.text) return p.text;
|
|
234
|
+
if (p?.inlineData) return `[${p.inlineData.mimeType ?? "binary"}]`;
|
|
235
|
+
if (p?.functionCall) return `[tool: ${p.functionCall.name ?? "unknown"}]`;
|
|
236
|
+
return safeStringify(p);
|
|
237
|
+
}).join("");
|
|
238
|
+
return {
|
|
239
|
+
role: item.role,
|
|
240
|
+
content: text
|
|
241
|
+
};
|
|
173
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* Formats input data for Datadog annotations.
|
|
245
|
+
* Model spans use message array format; others use raw or stringified data.
|
|
246
|
+
*/
|
|
174
247
|
function formatInput(input, spanType) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
return toDatadogMessages([{ role: "user", content: input }]);
|
|
195
|
-
}
|
|
196
|
-
return toDatadogMessages([{ role: "user", content: safeStringify(input) }]);
|
|
197
|
-
}
|
|
198
|
-
if (typeof input === "string" || Array.isArray(input)) return input;
|
|
199
|
-
return safeStringify(input);
|
|
248
|
+
if (isModelDataSpan(spanType)) {
|
|
249
|
+
if (isMessageArray(input)) return toDatadogMessages(input);
|
|
250
|
+
if (isGeminiContentArray(input)) return toDatadogMessages(input.map(geminiContentToMessage));
|
|
251
|
+
if (input && typeof input === "object" && !Array.isArray(input)) {
|
|
252
|
+
if (isMessageArray(input.messages)) return toDatadogMessages(input.messages);
|
|
253
|
+
if (isGeminiContentArray(input.messages)) return toDatadogMessages(input.messages.map(geminiContentToMessage));
|
|
254
|
+
if (isGeminiContentArray(input.contents)) return toDatadogMessages(input.contents.map(geminiContentToMessage));
|
|
255
|
+
}
|
|
256
|
+
if (typeof input === "string") return toDatadogMessages([{
|
|
257
|
+
role: "user",
|
|
258
|
+
content: input
|
|
259
|
+
}]);
|
|
260
|
+
return toDatadogMessages([{
|
|
261
|
+
role: "user",
|
|
262
|
+
content: safeStringify(input)
|
|
263
|
+
}]);
|
|
264
|
+
}
|
|
265
|
+
if (typeof input === "string" || Array.isArray(input)) return input;
|
|
266
|
+
return safeStringify(input);
|
|
200
267
|
}
|
|
268
|
+
/**
|
|
269
|
+
* Formats output data for Datadog annotations.
|
|
270
|
+
* Model spans use message array format; others use raw or stringified data.
|
|
271
|
+
*/
|
|
201
272
|
function formatOutput(output, spanType) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
273
|
+
if (isModelDataSpan(spanType)) {
|
|
274
|
+
if (isMessageArray(output)) return toDatadogMessages(output);
|
|
275
|
+
if (typeof output === "string") return [{
|
|
276
|
+
role: "assistant",
|
|
277
|
+
content: output
|
|
278
|
+
}];
|
|
279
|
+
if (output && typeof output === "object") {
|
|
280
|
+
if (Array.isArray(output.toolCalls) && output.toolCalls.length > 0) return [{
|
|
281
|
+
role: "assistant",
|
|
282
|
+
content: typeof output.text === "string" ? output.text : "",
|
|
283
|
+
toolCalls: toDatadogToolCalls(output.toolCalls)
|
|
284
|
+
}];
|
|
285
|
+
if (typeof output.text === "string" && output.text.length > 0) return [{
|
|
286
|
+
role: "assistant",
|
|
287
|
+
content: output.text
|
|
288
|
+
}];
|
|
289
|
+
if (output.object !== void 0) return [{
|
|
290
|
+
role: "assistant",
|
|
291
|
+
content: safeStringify(output.object)
|
|
292
|
+
}];
|
|
293
|
+
}
|
|
294
|
+
return [{
|
|
295
|
+
role: "assistant",
|
|
296
|
+
content: safeStringify(output)
|
|
297
|
+
}];
|
|
298
|
+
}
|
|
299
|
+
if (typeof output === "string") return output;
|
|
300
|
+
return safeStringify(output);
|
|
230
301
|
}
|
|
231
|
-
|
|
232
|
-
|
|
302
|
+
//#endregion
|
|
303
|
+
//#region src/bridge.ts
|
|
233
304
|
function flushApmExporter() {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
}
|
|
254
|
-
});
|
|
305
|
+
const exporterFlush = dd_trace.default?._tracer?._exporter?.flush;
|
|
306
|
+
if (typeof exporterFlush !== "function") return Promise.resolve();
|
|
307
|
+
return new Promise((resolve, reject) => {
|
|
308
|
+
let settled = false;
|
|
309
|
+
const done = (error) => {
|
|
310
|
+
if (settled) return;
|
|
311
|
+
settled = true;
|
|
312
|
+
if (error) {
|
|
313
|
+
reject(error);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
resolve();
|
|
317
|
+
};
|
|
318
|
+
try {
|
|
319
|
+
exporterFlush.call(dd_trace.default._tracer._exporter, done);
|
|
320
|
+
} catch (error) {
|
|
321
|
+
reject(error);
|
|
322
|
+
}
|
|
323
|
+
});
|
|
255
324
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
}
|
|
616
|
-
if (span.errorInfo.category) {
|
|
617
|
-
tags["error.category"] = span.errorInfo.category;
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
if (Object.keys(tags).length > 0) {
|
|
621
|
-
annotations.tags = tags;
|
|
622
|
-
}
|
|
623
|
-
return annotations;
|
|
624
|
-
}
|
|
625
|
-
setErrorTags(ddSpan, errorInfo) {
|
|
626
|
-
ddSpan.setTag("error", true);
|
|
627
|
-
ddSpan.setTag("error.message", errorInfo.message);
|
|
628
|
-
ddSpan.setTag("error.type", errorInfo.name ?? errorInfo.category ?? "Error");
|
|
629
|
-
if (errorInfo.stack) {
|
|
630
|
-
ddSpan.setTag("error.stack", errorInfo.stack);
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
async flush() {
|
|
634
|
-
if (this.isDisabled) return;
|
|
635
|
-
if (tracer3__default.default.llmobs?.flush) {
|
|
636
|
-
try {
|
|
637
|
-
await tracer3__default.default.llmobs.flush();
|
|
638
|
-
this.logger.debug("Datadog llmobs flushed");
|
|
639
|
-
} catch (e) {
|
|
640
|
-
this.logger.error("Error flushing llmobs", { error: e });
|
|
641
|
-
}
|
|
642
|
-
}
|
|
643
|
-
try {
|
|
644
|
-
await flushApmExporter();
|
|
645
|
-
this.logger.debug("Datadog APM exporter flushed");
|
|
646
|
-
} catch (e) {
|
|
647
|
-
this.logger.error("Error flushing Datadog APM exporter", { error: e });
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
async shutdown() {
|
|
651
|
-
for (const [spanId, ddSpan] of this.ddSpanMap) {
|
|
652
|
-
this.logger.warn(`[DatadogBridge] Force-finishing APM span that was not properly closed [id=${spanId}]`);
|
|
653
|
-
try {
|
|
654
|
-
if (typeof ddSpan.finish === "function") {
|
|
655
|
-
ddSpan.finish();
|
|
656
|
-
}
|
|
657
|
-
} catch {
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
this.ddSpanMap.clear();
|
|
661
|
-
this.openSpanCounts.clear();
|
|
662
|
-
this.traceContext.clear();
|
|
663
|
-
await this.flush();
|
|
664
|
-
if (tracer3__default.default.llmobs?.disable) {
|
|
665
|
-
try {
|
|
666
|
-
tracer3__default.default.llmobs.disable();
|
|
667
|
-
} catch (e) {
|
|
668
|
-
this.logger.error("Error disabling llmobs", { error: e });
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
await super.shutdown();
|
|
672
|
-
}
|
|
325
|
+
/**
|
|
326
|
+
* Datadog Bridge for Mastra Observability.
|
|
327
|
+
*
|
|
328
|
+
* Creates native dd-trace spans in real time during execution for proper APM
|
|
329
|
+
* context propagation, and uses the same live dd span for Datadog LLMObs
|
|
330
|
+
* tagging and annotations.
|
|
331
|
+
*/
|
|
332
|
+
var DatadogBridge = class extends _mastra_observability.BaseExporter {
|
|
333
|
+
name = "datadog-bridge";
|
|
334
|
+
config;
|
|
335
|
+
ddSpanMap = /* @__PURE__ */ new Map();
|
|
336
|
+
traceContext = /* @__PURE__ */ new Map();
|
|
337
|
+
openSpanCounts = /* @__PURE__ */ new Map();
|
|
338
|
+
constructor(config = {}) {
|
|
339
|
+
super(config);
|
|
340
|
+
const mlApp = config.mlApp ?? process.env.DD_LLMOBS_ML_APP;
|
|
341
|
+
const apiKey = config.apiKey ?? process.env.DD_API_KEY;
|
|
342
|
+
const site = config.site ?? process.env.DD_SITE ?? "datadoghq.com";
|
|
343
|
+
const env = config.env ?? process.env.DD_ENV;
|
|
344
|
+
const envAgentless = process.env.DD_LLMOBS_AGENTLESS_ENABLED?.toLowerCase();
|
|
345
|
+
const agentless = config.agentless ?? (envAgentless === "true" || envAgentless === "1" ? true : false);
|
|
346
|
+
if (!mlApp) {
|
|
347
|
+
this.setDisabled(`Missing required mlApp. Set DD_LLMOBS_ML_APP environment variable or pass mlApp in config.`);
|
|
348
|
+
this.config = config;
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
if (agentless && !apiKey) {
|
|
352
|
+
this.setDisabled(`Missing required apiKey for agentless mode. Set DD_API_KEY environment variable or pass apiKey in config.`);
|
|
353
|
+
this.config = config;
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
this.config = {
|
|
357
|
+
...config,
|
|
358
|
+
mlApp,
|
|
359
|
+
site,
|
|
360
|
+
apiKey,
|
|
361
|
+
agentless,
|
|
362
|
+
env
|
|
363
|
+
};
|
|
364
|
+
ensureTracer({
|
|
365
|
+
mlApp,
|
|
366
|
+
site,
|
|
367
|
+
apiKey,
|
|
368
|
+
agentless,
|
|
369
|
+
service: config.service,
|
|
370
|
+
env,
|
|
371
|
+
integrationsEnabled: config.integrationsEnabled ?? true
|
|
372
|
+
});
|
|
373
|
+
this.logger.info("Datadog bridge initialized", {
|
|
374
|
+
mlApp,
|
|
375
|
+
site,
|
|
376
|
+
agentless
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Create a dd-trace span eagerly when a Mastra span is constructed.
|
|
381
|
+
*/
|
|
382
|
+
createSpan(options) {
|
|
383
|
+
if (this.isDisabled) return void 0;
|
|
384
|
+
try {
|
|
385
|
+
let apmParentDdSpan = void 0;
|
|
386
|
+
let llmobsParentDdSpan = void 0;
|
|
387
|
+
let parentSource = "none";
|
|
388
|
+
const externalParentId = (0, _mastra_observability.getExternalParentId)(options);
|
|
389
|
+
if (externalParentId) {
|
|
390
|
+
apmParentDdSpan = this.ddSpanMap.get(externalParentId);
|
|
391
|
+
llmobsParentDdSpan = apmParentDdSpan;
|
|
392
|
+
if (apmParentDdSpan) parentSource = "external-parent";
|
|
393
|
+
}
|
|
394
|
+
if (!apmParentDdSpan) {
|
|
395
|
+
apmParentDdSpan = dd_trace.default.scope().active() ?? void 0;
|
|
396
|
+
if (apmParentDdSpan) parentSource = "active-scope";
|
|
397
|
+
}
|
|
398
|
+
if (!llmobsParentDdSpan && externalParentId) llmobsParentDdSpan = apmParentDdSpan;
|
|
399
|
+
const ddSpan = dd_trace.default.startSpan(options.name, {
|
|
400
|
+
...apmParentDdSpan ? { childOf: apmParentDdSpan } : {},
|
|
401
|
+
...options.startTime ? { startTime: toDate(options.startTime).getTime() } : {}
|
|
402
|
+
});
|
|
403
|
+
const ddContext = ddSpan.context?.();
|
|
404
|
+
const spanId = ddContext?.toSpanId?.(true) ?? generateSpanId();
|
|
405
|
+
const traceId = ddContext?.toTraceId?.(true) ?? (externalParentId ? options.parent?.traceId ?? generateTraceId() : generateTraceId());
|
|
406
|
+
const parentSpanId = (apmParentDdSpan?.context?.())?.toSpanId?.(true) ?? externalParentId;
|
|
407
|
+
this.captureTraceContext(traceId, options);
|
|
408
|
+
this.openSpanCounts.set(traceId, (this.openSpanCounts.get(traceId) ?? 0) + 1);
|
|
409
|
+
this.ddSpanMap.set(spanId, ddSpan);
|
|
410
|
+
this.registerLlmObsSpan(ddSpan, traceId, options, llmobsParentDdSpan);
|
|
411
|
+
this.logger.debug(`[DatadogBridge.createSpan] Created APM span [spanId=${spanId}] [traceId=${traceId}] [parentSpanId=${parentSpanId}] [type=${options.type}] [mapSize=${this.ddSpanMap.size}] [parentSource=${parentSource}] [externalParentId=${externalParentId ?? "none"}]`);
|
|
412
|
+
return {
|
|
413
|
+
spanId,
|
|
414
|
+
traceId,
|
|
415
|
+
parentSpanId
|
|
416
|
+
};
|
|
417
|
+
} catch (error) {
|
|
418
|
+
this.logger.error("[DatadogBridge] Failed to create span:", error);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Execute an async function within the dd-trace context of a Mastra span.
|
|
424
|
+
*/
|
|
425
|
+
executeInContext(spanId, fn) {
|
|
426
|
+
return this.executeWithSpanContext(spanId, fn);
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Execute a synchronous function within the dd-trace context of a Mastra span.
|
|
430
|
+
*/
|
|
431
|
+
executeInContextSync(spanId, fn) {
|
|
432
|
+
return this.executeWithSpanContext(spanId, fn);
|
|
433
|
+
}
|
|
434
|
+
async onScoreEvent(event) {
|
|
435
|
+
if (this.isDisabled || !dd_trace.default.llmobs?.submitEvaluation) return;
|
|
436
|
+
const { score } = event;
|
|
437
|
+
if (!score.traceId || !score.spanId) {
|
|
438
|
+
this.logger.warn("Datadog bridge: dropping score with no traceId/spanId", { scorerId: score.scorerId });
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
try {
|
|
442
|
+
dd_trace.default.llmobs.submitEvaluation({
|
|
443
|
+
traceId: score.traceId,
|
|
444
|
+
spanId: toDatadogSpanId(score.spanId)
|
|
445
|
+
}, {
|
|
446
|
+
label: score.scorerName ?? score.scorerId,
|
|
447
|
+
value: score.score,
|
|
448
|
+
metricType: "score",
|
|
449
|
+
mlApp: this.config.mlApp,
|
|
450
|
+
timestampMs: score.timestamp instanceof Date ? score.timestamp.getTime() : Date.now(),
|
|
451
|
+
...score.reason ? { reasoning: score.reason } : {},
|
|
452
|
+
...score.metadata ? { metadata: score.metadata } : {}
|
|
453
|
+
});
|
|
454
|
+
} catch (error) {
|
|
455
|
+
this.logger.error("Datadog bridge: Failed to submit evaluation", {
|
|
456
|
+
error,
|
|
457
|
+
traceId: score.traceId,
|
|
458
|
+
spanId: score.spanId,
|
|
459
|
+
scorerId: score.scorerId
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
executeWithSpanContext(spanId, fn) {
|
|
464
|
+
const ddSpan = this.ddSpanMap.get(spanId);
|
|
465
|
+
if (ddSpan) return dd_trace.default.scope().activate(ddSpan, () => {
|
|
466
|
+
const llmobs = dd_trace.default.llmobs;
|
|
467
|
+
if (typeof llmobs?._activate === "function") return llmobs._activate(ddSpan, void 0, fn);
|
|
468
|
+
return fn();
|
|
469
|
+
});
|
|
470
|
+
return fn();
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Handle tracing events from the observability bus.
|
|
474
|
+
*/
|
|
475
|
+
async _exportTracingEvent(event) {
|
|
476
|
+
if (this.isDisabled) return;
|
|
477
|
+
try {
|
|
478
|
+
const span = event.exportedSpan;
|
|
479
|
+
if (span.isEvent) {
|
|
480
|
+
if (event.type === "span_started" || event.type === "span_ended") this.annotateAndFinishSpan(span);
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
switch (event.type) {
|
|
484
|
+
case "span_started":
|
|
485
|
+
case "span_updated": return;
|
|
486
|
+
case "span_ended":
|
|
487
|
+
this.annotateAndFinishSpan(span);
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
} catch (error) {
|
|
491
|
+
this.logger.error("Datadog bridge error", {
|
|
492
|
+
error,
|
|
493
|
+
eventType: event.type,
|
|
494
|
+
spanId: event.exportedSpan?.id,
|
|
495
|
+
spanName: event.exportedSpan?.name
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
registerLlmObsSpan(ddSpan, traceId, options, parentDdSpan) {
|
|
500
|
+
const tagger = this.getLlmObsTagger();
|
|
501
|
+
if (!tagger) return;
|
|
502
|
+
try {
|
|
503
|
+
const kind = kindFor(options.type);
|
|
504
|
+
const ownAttrs = options.attributes;
|
|
505
|
+
const inheritedModelAttrs = options.parent?.type === _mastra_core_observability.SpanType.MODEL_GENERATION ? options.parent.attributes : void 0;
|
|
506
|
+
const traceCtx = this.resolveTraceContext(traceId, options);
|
|
507
|
+
tagger.registerLLMObsSpan(ddSpan, {
|
|
508
|
+
parent: parentDdSpan,
|
|
509
|
+
kind,
|
|
510
|
+
name: options.name,
|
|
511
|
+
userId: traceCtx.userId,
|
|
512
|
+
sessionId: traceCtx.sessionId,
|
|
513
|
+
...kind === "llm" && (ownAttrs?.model ?? inheritedModelAttrs?.model) ? { modelName: ownAttrs?.model ?? inheritedModelAttrs?.model } : {},
|
|
514
|
+
...kind === "llm" && (ownAttrs?.provider ?? inheritedModelAttrs?.provider) ? { modelProvider: ownAttrs?.provider ?? inheritedModelAttrs?.provider } : {}
|
|
515
|
+
});
|
|
516
|
+
} catch (error) {
|
|
517
|
+
this.logger.warn("[DatadogBridge] Failed to register LLMObs span", {
|
|
518
|
+
error,
|
|
519
|
+
spanName: options.name,
|
|
520
|
+
spanType: options.type
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
getLlmObsTagger() {
|
|
525
|
+
const tagger = dd_trace.default.llmobs?._tagger;
|
|
526
|
+
if (tagger && typeof tagger.registerLLMObsSpan === "function") return tagger;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Annotate the eagerly-created dd span and finish it using the final Mastra
|
|
530
|
+
* span state.
|
|
531
|
+
*/
|
|
532
|
+
annotateAndFinishSpan(span) {
|
|
533
|
+
const ddSpan = this.ddSpanMap.get(span.id);
|
|
534
|
+
if (!ddSpan) {
|
|
535
|
+
this.logger.warn("[DatadogBridge] No dd span found when finalizing Mastra span", {
|
|
536
|
+
spanId: span.id,
|
|
537
|
+
spanName: span.name,
|
|
538
|
+
spanType: span.type,
|
|
539
|
+
mapSize: this.ddSpanMap.size
|
|
540
|
+
});
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
const endTime = span.endTime ? toDate(span.endTime) : span.isEvent ? toDate(span.startTime) : /* @__PURE__ */ new Date();
|
|
544
|
+
const annotations = this.buildAnnotations(span);
|
|
545
|
+
try {
|
|
546
|
+
if (Object.keys(annotations).length > 0 && dd_trace.default.llmobs?.annotate) dd_trace.default.llmobs.annotate(ddSpan, annotations);
|
|
547
|
+
} catch (error) {
|
|
548
|
+
this.logger.error("[DatadogBridge] Failed to annotate span before finish", {
|
|
549
|
+
error,
|
|
550
|
+
spanId: span.id,
|
|
551
|
+
spanName: span.name
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
try {
|
|
555
|
+
if (span.errorInfo) this.setErrorTags(ddSpan, span.errorInfo);
|
|
556
|
+
} catch (error) {
|
|
557
|
+
this.logger.error("[DatadogBridge] Failed to set error tags on dd span", {
|
|
558
|
+
error,
|
|
559
|
+
spanId: span.id,
|
|
560
|
+
spanName: span.name
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
try {
|
|
564
|
+
if (typeof ddSpan.finish === "function") ddSpan.finish(endTime.getTime());
|
|
565
|
+
} catch (error) {
|
|
566
|
+
this.logger.error("[DatadogBridge] Failed to finish dd span", {
|
|
567
|
+
error,
|
|
568
|
+
spanId: span.id,
|
|
569
|
+
spanName: span.name
|
|
570
|
+
});
|
|
571
|
+
} finally {
|
|
572
|
+
this.ddSpanMap.delete(span.id);
|
|
573
|
+
this.releaseTraceContext(span.traceId);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
captureTraceContext(traceId, options) {
|
|
577
|
+
const existing = this.traceContext.get(traceId);
|
|
578
|
+
const next = {
|
|
579
|
+
userId: firstString(options.metadata?.userId, options.parent?.metadata?.userId, existing?.userId),
|
|
580
|
+
sessionId: firstString(options.metadata?.sessionId, options.parent?.metadata?.sessionId, existing?.sessionId)
|
|
581
|
+
};
|
|
582
|
+
if (next.userId || next.sessionId) this.traceContext.set(traceId, next);
|
|
583
|
+
}
|
|
584
|
+
resolveTraceContext(traceId, options) {
|
|
585
|
+
const stored = this.traceContext.get(traceId);
|
|
586
|
+
return {
|
|
587
|
+
userId: firstString(options.metadata?.userId, options.parent?.metadata?.userId, stored?.userId),
|
|
588
|
+
sessionId: firstString(options.metadata?.sessionId, options.parent?.metadata?.sessionId, stored?.sessionId)
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
releaseTraceContext(traceId) {
|
|
592
|
+
const nextCount = (this.openSpanCounts.get(traceId) ?? 1) - 1;
|
|
593
|
+
if (nextCount > 0) {
|
|
594
|
+
this.openSpanCounts.set(traceId, nextCount);
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
this.openSpanCounts.delete(traceId);
|
|
598
|
+
this.traceContext.delete(traceId);
|
|
599
|
+
}
|
|
600
|
+
buildAnnotations(span) {
|
|
601
|
+
const annotations = {};
|
|
602
|
+
if (span.input !== void 0) annotations.inputData = formatInput(span.input, span.type);
|
|
603
|
+
if (span.output !== void 0) annotations.outputData = formatOutput(span.output, span.type);
|
|
604
|
+
const usageSpanType = isModelInferenceEnabled() ? _mastra_core_observability.SpanType.MODEL_INFERENCE : _mastra_core_observability.SpanType.MODEL_STEP;
|
|
605
|
+
if (span.type === usageSpanType) {
|
|
606
|
+
const usage = span.attributes?.usage;
|
|
607
|
+
const metrics = formatUsageMetrics(usage);
|
|
608
|
+
if (metrics) annotations.metrics = metrics;
|
|
609
|
+
}
|
|
610
|
+
const otherAttributes = (0, _mastra_core_utils.omitKeys)(span.attributes ?? {}, [
|
|
611
|
+
"usage",
|
|
612
|
+
"model",
|
|
613
|
+
"provider"
|
|
614
|
+
]);
|
|
615
|
+
const contextKeySet = new Set(this.config.requestContextKeys ?? []);
|
|
616
|
+
const flatContextTags = {};
|
|
617
|
+
const remainingMetadata = {};
|
|
618
|
+
for (const [key, value] of Object.entries(span.metadata ?? {})) if (contextKeySet.has(key)) flatContextTags[key] = value;
|
|
619
|
+
else remainingMetadata[key] = value;
|
|
620
|
+
const remainingAttributes = {};
|
|
621
|
+
for (const [key, value] of Object.entries(otherAttributes)) if (contextKeySet.has(key)) {
|
|
622
|
+
if (!(key in flatContextTags)) flatContextTags[key] = value;
|
|
623
|
+
} else remainingAttributes[key] = value;
|
|
624
|
+
const combinedMetadata = {
|
|
625
|
+
...remainingMetadata,
|
|
626
|
+
...remainingAttributes
|
|
627
|
+
};
|
|
628
|
+
if (span.errorInfo) combinedMetadata["error.message"] = span.errorInfo.message;
|
|
629
|
+
if (Object.keys(combinedMetadata).length > 0) annotations.metadata = combinedMetadata;
|
|
630
|
+
const tags = { ...flatContextTags };
|
|
631
|
+
if (span.tags?.length) for (const tag of span.tags) {
|
|
632
|
+
const colonIndex = tag.indexOf(":");
|
|
633
|
+
if (colonIndex > 0) tags[tag.substring(0, colonIndex)] = tag.substring(colonIndex + 1);
|
|
634
|
+
else tags[tag] = true;
|
|
635
|
+
}
|
|
636
|
+
if (span.errorInfo) {
|
|
637
|
+
tags.error = true;
|
|
638
|
+
if (span.errorInfo.id) tags["error.id"] = span.errorInfo.id;
|
|
639
|
+
if (span.errorInfo.domain) tags["error.domain"] = span.errorInfo.domain;
|
|
640
|
+
if (span.errorInfo.category) tags["error.category"] = span.errorInfo.category;
|
|
641
|
+
}
|
|
642
|
+
if (Object.keys(tags).length > 0) annotations.tags = tags;
|
|
643
|
+
return annotations;
|
|
644
|
+
}
|
|
645
|
+
setErrorTags(ddSpan, errorInfo) {
|
|
646
|
+
ddSpan.setTag("error", true);
|
|
647
|
+
ddSpan.setTag("error.message", errorInfo.message);
|
|
648
|
+
ddSpan.setTag("error.type", errorInfo.name ?? errorInfo.category ?? "Error");
|
|
649
|
+
if (errorInfo.stack) ddSpan.setTag("error.stack", errorInfo.stack);
|
|
650
|
+
}
|
|
651
|
+
async flush() {
|
|
652
|
+
if (this.isDisabled) return;
|
|
653
|
+
if (dd_trace.default.llmobs?.flush) try {
|
|
654
|
+
await dd_trace.default.llmobs.flush();
|
|
655
|
+
this.logger.debug("Datadog llmobs flushed");
|
|
656
|
+
} catch (e) {
|
|
657
|
+
this.logger.error("Error flushing llmobs", { error: e });
|
|
658
|
+
}
|
|
659
|
+
try {
|
|
660
|
+
await flushApmExporter();
|
|
661
|
+
this.logger.debug("Datadog APM exporter flushed");
|
|
662
|
+
} catch (e) {
|
|
663
|
+
this.logger.error("Error flushing Datadog APM exporter", { error: e });
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
async shutdown() {
|
|
667
|
+
for (const [spanId, ddSpan] of this.ddSpanMap) {
|
|
668
|
+
this.logger.warn(`[DatadogBridge] Force-finishing APM span that was not properly closed [id=${spanId}]`);
|
|
669
|
+
try {
|
|
670
|
+
if (typeof ddSpan.finish === "function") ddSpan.finish();
|
|
671
|
+
} catch {}
|
|
672
|
+
}
|
|
673
|
+
this.ddSpanMap.clear();
|
|
674
|
+
this.openSpanCounts.clear();
|
|
675
|
+
this.traceContext.clear();
|
|
676
|
+
await this.flush();
|
|
677
|
+
if (dd_trace.default.llmobs?.disable) try {
|
|
678
|
+
dd_trace.default.llmobs.disable();
|
|
679
|
+
} catch (e) {
|
|
680
|
+
this.logger.error("Error disabling llmobs", { error: e });
|
|
681
|
+
}
|
|
682
|
+
await super.shutdown();
|
|
683
|
+
}
|
|
673
684
|
};
|
|
674
685
|
function firstString(...values) {
|
|
675
|
-
|
|
676
|
-
if (typeof value === "string") {
|
|
677
|
-
return value;
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
return void 0;
|
|
686
|
+
for (const value of values) if (typeof value === "string") return value;
|
|
681
687
|
}
|
|
682
688
|
function fillRandomBytes(bytes) {
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
692
|
-
bytes[i] = Math.floor(Math.random() * 256);
|
|
693
|
-
}
|
|
689
|
+
try {
|
|
690
|
+
const webCrypto = globalThis.crypto;
|
|
691
|
+
if (webCrypto?.getRandomValues) {
|
|
692
|
+
webCrypto.getRandomValues.call(webCrypto, bytes);
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
} catch {}
|
|
696
|
+
for (let i = 0; i < bytes.length; i++) bytes[i] = Math.floor(Math.random() * 256);
|
|
694
697
|
}
|
|
695
698
|
function toDatadogSpanId(spanId) {
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
}
|
|
699
|
-
return spanId;
|
|
699
|
+
if (/^[0-9a-f]{16}$/i.test(spanId)) return BigInt(`0x${spanId}`).toString(10);
|
|
700
|
+
return spanId;
|
|
700
701
|
}
|
|
701
702
|
function generateSpanId() {
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
703
|
+
const bytes = /* @__PURE__ */ new Uint8Array(8);
|
|
704
|
+
fillRandomBytes(bytes);
|
|
705
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
705
706
|
}
|
|
706
707
|
function generateTraceId() {
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
708
|
+
const bytes = /* @__PURE__ */ new Uint8Array(16);
|
|
709
|
+
fillRandomBytes(bytes);
|
|
710
|
+
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
710
711
|
}
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
userId: traceCtx.userId,
|
|
1163
|
-
startTime,
|
|
1164
|
-
...kind === "llm" && attrs?.model ? { modelName: attrs.model } : {},
|
|
1165
|
-
...kind === "llm" && attrs?.provider ? { modelProvider: attrs.provider } : {}
|
|
1166
|
-
},
|
|
1167
|
-
// endTime as milliseconds for ddSpan.finish() — dd-trace's llmobs.trace() does not
|
|
1168
|
-
// honor endTime in options, so we must call finish(ms) explicitly on the span.
|
|
1169
|
-
endTimeMs: endTime.getTime()
|
|
1170
|
-
};
|
|
1171
|
-
}
|
|
1172
|
-
/**
|
|
1173
|
-
* Recursively emits a span tree using nested llmobs.trace() calls.
|
|
1174
|
-
* This ensures parent-child relationships are properly established in Datadog
|
|
1175
|
-
* because child spans are created while their parent span is active in scope.
|
|
1176
|
-
*/
|
|
1177
|
-
emitSpanTree(node, state, inheritedModelAttrs) {
|
|
1178
|
-
const span = node.span;
|
|
1179
|
-
const { traceOptions, endTimeMs } = this.buildSpanOptions(span, inheritedModelAttrs);
|
|
1180
|
-
const childInheritedModelAttrs = span.type === observability$1.SpanType.MODEL_GENERATION ? {
|
|
1181
|
-
model: span.attributes?.model,
|
|
1182
|
-
provider: span.attributes?.provider
|
|
1183
|
-
} : inheritedModelAttrs;
|
|
1184
|
-
tracer3__default.default.llmobs.trace(traceOptions, (ddSpan) => {
|
|
1185
|
-
const annotations = this.buildAnnotations(span);
|
|
1186
|
-
if (Object.keys(annotations).length > 0) {
|
|
1187
|
-
tracer3__default.default.llmobs.annotate(ddSpan, annotations);
|
|
1188
|
-
}
|
|
1189
|
-
if (span.errorInfo) {
|
|
1190
|
-
this.setErrorTags(ddSpan, span.errorInfo);
|
|
1191
|
-
}
|
|
1192
|
-
const exported = tracer3__default.default.llmobs.exportSpan ? tracer3__default.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
1193
|
-
state.contexts.set(span.id, { ddSpan, exported });
|
|
1194
|
-
for (const child of node.children) {
|
|
1195
|
-
this.emitSpanTree(child, state, childInheritedModelAttrs);
|
|
1196
|
-
}
|
|
1197
|
-
if (typeof ddSpan.finish === "function") {
|
|
1198
|
-
ddSpan.finish(endTimeMs);
|
|
1199
|
-
}
|
|
1200
|
-
});
|
|
1201
|
-
}
|
|
1202
|
-
/**
|
|
1203
|
-
* Emit a single span with the proper Datadog parent context.
|
|
1204
|
-
* Used for late-arriving spans after the main tree has been emitted.
|
|
1205
|
-
*/
|
|
1206
|
-
emitSingleSpan(span, state, parent) {
|
|
1207
|
-
const { traceOptions, endTimeMs } = this.buildSpanOptions(span);
|
|
1208
|
-
const runTrace = () => tracer3__default.default.llmobs.trace(traceOptions, (ddSpan) => {
|
|
1209
|
-
const annotations = this.buildAnnotations(span);
|
|
1210
|
-
if (Object.keys(annotations).length > 0) {
|
|
1211
|
-
tracer3__default.default.llmobs.annotate(ddSpan, annotations);
|
|
1212
|
-
}
|
|
1213
|
-
if (span.errorInfo) {
|
|
1214
|
-
this.setErrorTags(ddSpan, span.errorInfo);
|
|
1215
|
-
}
|
|
1216
|
-
const exported = tracer3__default.default.llmobs.exportSpan ? tracer3__default.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
1217
|
-
state.contexts.set(span.id, { ddSpan, exported });
|
|
1218
|
-
if (typeof ddSpan.finish === "function") {
|
|
1219
|
-
ddSpan.finish(endTimeMs);
|
|
1220
|
-
}
|
|
1221
|
-
});
|
|
1222
|
-
if (parent) {
|
|
1223
|
-
tracer3__default.default.scope().activate(parent, runTrace);
|
|
1224
|
-
} else {
|
|
1225
|
-
runTrace();
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
712
|
+
//#endregion
|
|
713
|
+
//#region src/tracing.ts
|
|
714
|
+
/**
|
|
715
|
+
* Maximum lifetime for a trace state entry (30 minutes).
|
|
716
|
+
* This is a fallback cleanup mechanism for traces that never receive a root span
|
|
717
|
+
* or have all spans marked as non-root, preventing unbounded memory growth.
|
|
718
|
+
*/
|
|
719
|
+
const MAX_TRACE_LIFETIME_MS = 1800 * 1e3;
|
|
720
|
+
/**
|
|
721
|
+
* Regular cleanup interval for trace state entries (1 minute).
|
|
722
|
+
*/
|
|
723
|
+
const REGULAR_CLEANUP_INTERVAL_MS = 60 * 1e3;
|
|
724
|
+
/**
|
|
725
|
+
* Datadog LLM Observability Exporter for Mastra.
|
|
726
|
+
*
|
|
727
|
+
* Exports observability data to Datadog's LLM Observability product using
|
|
728
|
+
* a completion-only pattern where spans are emitted on span_ended events.
|
|
729
|
+
*/
|
|
730
|
+
var DatadogExporter = class extends _mastra_observability.BaseExporter {
|
|
731
|
+
name = "datadog";
|
|
732
|
+
config;
|
|
733
|
+
traceContext = /* @__PURE__ */ new Map();
|
|
734
|
+
traceState = /* @__PURE__ */ new Map();
|
|
735
|
+
constructor(config = {}) {
|
|
736
|
+
super(config);
|
|
737
|
+
const mlApp = config.mlApp ?? process.env.DD_LLMOBS_ML_APP;
|
|
738
|
+
const apiKey = config.apiKey ?? process.env.DD_API_KEY;
|
|
739
|
+
const site = config.site ?? process.env.DD_SITE ?? "datadoghq.com";
|
|
740
|
+
const env = config.env ?? process.env.DD_ENV;
|
|
741
|
+
const envAgentless = process.env.DD_LLMOBS_AGENTLESS_ENABLED?.toLowerCase();
|
|
742
|
+
const agentless = config.agentless ?? (envAgentless === "false" || envAgentless === "0" ? false : true);
|
|
743
|
+
if (!mlApp) {
|
|
744
|
+
this.setDisabled(`Missing required mlApp. Set DD_LLMOBS_ML_APP environment variable or pass mlApp in config.`);
|
|
745
|
+
this.config = config;
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
if (agentless && !apiKey) {
|
|
749
|
+
this.setDisabled(`Missing required apiKey for agentless mode. Set DD_API_KEY environment variable or pass apiKey in config.`);
|
|
750
|
+
this.config = config;
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
this.config = {
|
|
754
|
+
...config,
|
|
755
|
+
mlApp,
|
|
756
|
+
site,
|
|
757
|
+
apiKey,
|
|
758
|
+
agentless,
|
|
759
|
+
env
|
|
760
|
+
};
|
|
761
|
+
ensureTracer({
|
|
762
|
+
mlApp,
|
|
763
|
+
site,
|
|
764
|
+
apiKey,
|
|
765
|
+
agentless,
|
|
766
|
+
service: config.service,
|
|
767
|
+
env,
|
|
768
|
+
integrationsEnabled: config.integrationsEnabled
|
|
769
|
+
});
|
|
770
|
+
this.logger.info("Datadog exporter initialized", {
|
|
771
|
+
mlApp,
|
|
772
|
+
site,
|
|
773
|
+
agentless
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Main entry point for tracing events from Mastra.
|
|
778
|
+
*/
|
|
779
|
+
async _exportTracingEvent(event) {
|
|
780
|
+
if (this.isDisabled || !dd_trace.default.llmobs) return;
|
|
781
|
+
try {
|
|
782
|
+
const span = event.exportedSpan;
|
|
783
|
+
if (span.isEvent) {
|
|
784
|
+
if (event.type === "span_started") {
|
|
785
|
+
this.captureTraceContext(span);
|
|
786
|
+
this.enqueueSpan(span);
|
|
787
|
+
}
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
switch (event.type) {
|
|
791
|
+
case "span_started":
|
|
792
|
+
this.captureTraceContext(span);
|
|
793
|
+
return;
|
|
794
|
+
case "span_updated": return;
|
|
795
|
+
case "span_ended":
|
|
796
|
+
this.enqueueSpan(span);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
} catch (error) {
|
|
800
|
+
this.logger.error("Datadog exporter error", {
|
|
801
|
+
error,
|
|
802
|
+
eventType: event.type,
|
|
803
|
+
spanId: event.exportedSpan?.id,
|
|
804
|
+
spanName: event.exportedSpan?.name
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* Captures user/session context from root spans for tagging all spans in the trace.
|
|
810
|
+
*/
|
|
811
|
+
captureTraceContext(span) {
|
|
812
|
+
if (span.isRootSpan && !this.traceContext.has(span.traceId)) this.traceContext.set(span.traceId, {
|
|
813
|
+
userId: span.metadata?.userId,
|
|
814
|
+
sessionId: span.metadata?.sessionId
|
|
815
|
+
});
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Queue span until its parent context is available, then emit spans parent-first.
|
|
819
|
+
*/
|
|
820
|
+
enqueueSpan(span) {
|
|
821
|
+
const state = this.getOrCreateTraceState(span.traceId);
|
|
822
|
+
if (span.isRootSpan) state.rootEnded = true;
|
|
823
|
+
state.buffer.set(span.id, span);
|
|
824
|
+
this.tryEmitReadySpans(span.traceId);
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Sets native dd-trace error tags required by Datadog's Error Tracking UI.
|
|
828
|
+
*/
|
|
829
|
+
setErrorTags(ddSpan, errorInfo) {
|
|
830
|
+
ddSpan.setTag("error", true);
|
|
831
|
+
ddSpan.setTag("error.message", errorInfo.message);
|
|
832
|
+
ddSpan.setTag("error.type", errorInfo.name ?? errorInfo.category ?? "Error");
|
|
833
|
+
if (errorInfo.stack) ddSpan.setTag("error.stack", errorInfo.stack);
|
|
834
|
+
}
|
|
835
|
+
/**
|
|
836
|
+
* Builds annotations object for llmobs.annotate().
|
|
837
|
+
* Uses dd-trace's expected property names: inputData, outputData, metadata, tags, metrics.
|
|
838
|
+
*/
|
|
839
|
+
buildAnnotations(span) {
|
|
840
|
+
const annotations = {};
|
|
841
|
+
if (span.input !== void 0) annotations.inputData = formatInput(span.input, span.type);
|
|
842
|
+
if (span.output !== void 0) annotations.outputData = formatOutput(span.output, span.type);
|
|
843
|
+
const usageSpanType = isModelInferenceEnabled() ? _mastra_core_observability.SpanType.MODEL_INFERENCE : _mastra_core_observability.SpanType.MODEL_STEP;
|
|
844
|
+
if (span.type === usageSpanType) {
|
|
845
|
+
const usage = span.attributes?.usage;
|
|
846
|
+
const metrics = formatUsageMetrics(usage);
|
|
847
|
+
if (metrics) annotations.metrics = metrics;
|
|
848
|
+
}
|
|
849
|
+
const otherAttributes = (0, _mastra_core_utils.omitKeys)(span.attributes ?? {}, [
|
|
850
|
+
"usage",
|
|
851
|
+
"model",
|
|
852
|
+
"provider"
|
|
853
|
+
]);
|
|
854
|
+
const contextKeySet = new Set(this.config.requestContextKeys ?? []);
|
|
855
|
+
const flatContextTags = {};
|
|
856
|
+
const remainingMetadata = {};
|
|
857
|
+
for (const [key, value] of Object.entries(span.metadata ?? {})) if (contextKeySet.has(key)) flatContextTags[key] = value;
|
|
858
|
+
else remainingMetadata[key] = value;
|
|
859
|
+
const remainingAttributes = {};
|
|
860
|
+
for (const [key, value] of Object.entries(otherAttributes)) if (contextKeySet.has(key)) {
|
|
861
|
+
if (!(key in flatContextTags)) flatContextTags[key] = value;
|
|
862
|
+
} else remainingAttributes[key] = value;
|
|
863
|
+
const combinedMetadata = {
|
|
864
|
+
...remainingMetadata,
|
|
865
|
+
...remainingAttributes
|
|
866
|
+
};
|
|
867
|
+
if (span.errorInfo) combinedMetadata["error.message"] = span.errorInfo.message;
|
|
868
|
+
if (Object.keys(combinedMetadata).length > 0) annotations.metadata = combinedMetadata;
|
|
869
|
+
const tags = { ...flatContextTags };
|
|
870
|
+
if (span.tags?.length) for (const tag of span.tags) {
|
|
871
|
+
const colonIndex = tag.indexOf(":");
|
|
872
|
+
if (colonIndex > 0) tags[tag.substring(0, colonIndex)] = tag.substring(colonIndex + 1);
|
|
873
|
+
else tags[tag] = true;
|
|
874
|
+
}
|
|
875
|
+
if (span.errorInfo) {
|
|
876
|
+
tags.error = true;
|
|
877
|
+
if (span.errorInfo.id) tags["error.id"] = span.errorInfo.id;
|
|
878
|
+
if (span.errorInfo.domain) tags["error.domain"] = span.errorInfo.domain;
|
|
879
|
+
if (span.errorInfo.category) tags["error.category"] = span.errorInfo.category;
|
|
880
|
+
}
|
|
881
|
+
if (Object.keys(tags).length > 0) annotations.tags = tags;
|
|
882
|
+
return annotations;
|
|
883
|
+
}
|
|
884
|
+
/**
|
|
885
|
+
* Submit an eval score to Datadog LLM Observability for the matching ddSpan.
|
|
886
|
+
*
|
|
887
|
+
* Ordering constraint: the matching span must have already been emitted to dd-trace
|
|
888
|
+
* (i.e. its `SPAN_ENDED` event must have been processed and the trace tree flushed).
|
|
889
|
+
* On Mastra's normal scoring path this is always true — scorer hooks fire after the
|
|
890
|
+
* scored entity completes, so the root span has ended by the time `onScoreEvent` runs.
|
|
891
|
+
*
|
|
892
|
+
* If a score arrives for an unexported span (either before `SPAN_ENDED` or after the
|
|
893
|
+
* `traceState` entry has been cleaned up), the event is dropped and a warning is logged
|
|
894
|
+
* so the misuse is observable. Scores must therefore only be submitted for spans whose
|
|
895
|
+
* lifecycle has completed.
|
|
896
|
+
*/
|
|
897
|
+
async onScoreEvent(event) {
|
|
898
|
+
if (this.isDisabled || !dd_trace.default.llmobs?.submitEvaluation) return;
|
|
899
|
+
const { score } = event;
|
|
900
|
+
if (!score.traceId || !score.spanId) {
|
|
901
|
+
this.logger.warn("Datadog exporter: dropping score with no traceId/spanId", { scorerId: score.scorerId });
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
const exported = (this.traceState.get(score.traceId)?.contexts.get(score.spanId))?.exported;
|
|
905
|
+
if (!exported) {
|
|
906
|
+
this.logger.warn("Datadog exporter: dropping score for span that has not been emitted to dd-trace yet (span_ended must be processed before submitting a score for it)", {
|
|
907
|
+
traceId: score.traceId,
|
|
908
|
+
spanId: score.spanId,
|
|
909
|
+
scorerId: score.scorerId
|
|
910
|
+
});
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
try {
|
|
914
|
+
dd_trace.default.llmobs.submitEvaluation({
|
|
915
|
+
traceId: exported.traceId,
|
|
916
|
+
spanId: exported.spanId
|
|
917
|
+
}, {
|
|
918
|
+
label: score.scorerName ?? score.scorerId,
|
|
919
|
+
value: score.score,
|
|
920
|
+
metricType: "score",
|
|
921
|
+
mlApp: this.config.mlApp,
|
|
922
|
+
timestampMs: score.timestamp instanceof Date ? score.timestamp.getTime() : Date.now(),
|
|
923
|
+
...score.reason ? { reasoning: score.reason } : {},
|
|
924
|
+
...score.metadata ? { metadata: score.metadata } : {}
|
|
925
|
+
});
|
|
926
|
+
} catch (err) {
|
|
927
|
+
this.logger.error("Datadog exporter: Failed to submit evaluation", {
|
|
928
|
+
error: err,
|
|
929
|
+
traceId: score.traceId,
|
|
930
|
+
spanId: score.spanId,
|
|
931
|
+
scorerId: score.scorerId
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
/**
|
|
936
|
+
* Force flush any buffered spans without shutting down the exporter.
|
|
937
|
+
* This is useful in serverless environments where you need to ensure spans
|
|
938
|
+
* are exported before the runtime instance is terminated.
|
|
939
|
+
*/
|
|
940
|
+
async flush() {
|
|
941
|
+
if (this.isDisabled || !dd_trace.default.llmobs) return;
|
|
942
|
+
if (dd_trace.default.llmobs?.flush) try {
|
|
943
|
+
await dd_trace.default.llmobs.flush();
|
|
944
|
+
this.logger.debug("Datadog llmobs flushed");
|
|
945
|
+
} catch (e) {
|
|
946
|
+
this.logger.error("Error flushing llmobs", { error: e });
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* Gracefully shuts down the exporter.
|
|
951
|
+
*/
|
|
952
|
+
async shutdown() {
|
|
953
|
+
for (const [traceId, state] of this.traceState) {
|
|
954
|
+
if (state.cleanupTimer) clearTimeout(state.cleanupTimer);
|
|
955
|
+
if (state.maxLifetimeTimer) clearTimeout(state.maxLifetimeTimer);
|
|
956
|
+
if (state.buffer.size > 0) this.logger.warn("Shutdown with pending spans", {
|
|
957
|
+
traceId,
|
|
958
|
+
pendingCount: state.buffer.size,
|
|
959
|
+
spanIds: Array.from(state.buffer.keys())
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
this.traceState.clear();
|
|
963
|
+
await this.flush();
|
|
964
|
+
if (dd_trace.default.llmobs?.disable) try {
|
|
965
|
+
dd_trace.default.llmobs.disable();
|
|
966
|
+
} catch (e) {
|
|
967
|
+
this.logger.error("Error disabling llmobs", { error: e });
|
|
968
|
+
}
|
|
969
|
+
this.traceContext.clear();
|
|
970
|
+
await super.shutdown();
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* Retrieve or initialize trace state for buffering and parent tracking.
|
|
974
|
+
*/
|
|
975
|
+
getOrCreateTraceState(traceId) {
|
|
976
|
+
const existing = this.traceState.get(traceId);
|
|
977
|
+
if (existing) {
|
|
978
|
+
if (existing.cleanupTimer) {
|
|
979
|
+
clearTimeout(existing.cleanupTimer);
|
|
980
|
+
existing.cleanupTimer = void 0;
|
|
981
|
+
}
|
|
982
|
+
return existing;
|
|
983
|
+
}
|
|
984
|
+
const created = {
|
|
985
|
+
buffer: /* @__PURE__ */ new Map(),
|
|
986
|
+
contexts: /* @__PURE__ */ new Map(),
|
|
987
|
+
rootEnded: false,
|
|
988
|
+
treeEmitted: false,
|
|
989
|
+
createdAt: Date.now(),
|
|
990
|
+
cleanupTimer: void 0,
|
|
991
|
+
maxLifetimeTimer: void 0
|
|
992
|
+
};
|
|
993
|
+
const maxLifetimeTimer = setTimeout(() => {
|
|
994
|
+
const state = this.traceState.get(traceId);
|
|
995
|
+
if (state) {
|
|
996
|
+
if (state.buffer.size > 0 || state.contexts.size > 0) this.logger.warn("Discarding trace due to max lifetime exceeded", {
|
|
997
|
+
traceId,
|
|
998
|
+
bufferedSpans: state.buffer.size,
|
|
999
|
+
emittedSpans: state.contexts.size,
|
|
1000
|
+
lifetimeMs: Date.now() - state.createdAt
|
|
1001
|
+
});
|
|
1002
|
+
if (state.cleanupTimer) clearTimeout(state.cleanupTimer);
|
|
1003
|
+
this.traceState.delete(traceId);
|
|
1004
|
+
this.traceContext.delete(traceId);
|
|
1005
|
+
}
|
|
1006
|
+
}, MAX_TRACE_LIFETIME_MS);
|
|
1007
|
+
maxLifetimeTimer.unref?.();
|
|
1008
|
+
created.maxLifetimeTimer = maxLifetimeTimer;
|
|
1009
|
+
this.traceState.set(traceId, created);
|
|
1010
|
+
return created;
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Attempt to emit spans from the buffer.
|
|
1014
|
+
*
|
|
1015
|
+
* Two modes of operation:
|
|
1016
|
+
* 1. Initial tree emission: When root span ends and tree hasn't been emitted yet,
|
|
1017
|
+
* build a tree from all buffered spans and emit recursively using nested
|
|
1018
|
+
* llmobs.trace() calls. This ensures proper parent-child relationships in Datadog.
|
|
1019
|
+
* 2. Late-arriving spans: After the tree has been emitted, emit individual spans
|
|
1020
|
+
* with their parent context for proper linking.
|
|
1021
|
+
*/
|
|
1022
|
+
tryEmitReadySpans(traceId) {
|
|
1023
|
+
const state = this.traceState.get(traceId);
|
|
1024
|
+
if (!state) return;
|
|
1025
|
+
if (!state.treeEmitted) {
|
|
1026
|
+
if (!state.rootEnded) return;
|
|
1027
|
+
const tree = this.buildSpanTree(state.buffer);
|
|
1028
|
+
if (tree) this.emitSpanTree(tree, state);
|
|
1029
|
+
state.buffer.clear();
|
|
1030
|
+
state.treeEmitted = true;
|
|
1031
|
+
} else {
|
|
1032
|
+
let emitted = false;
|
|
1033
|
+
do {
|
|
1034
|
+
emitted = false;
|
|
1035
|
+
for (const [spanId, span] of state.buffer) {
|
|
1036
|
+
const parentCtx = span.parentSpanId ? state.contexts.get(span.parentSpanId) : void 0;
|
|
1037
|
+
if (span.parentSpanId && !parentCtx) continue;
|
|
1038
|
+
this.emitSingleSpan(span, state, parentCtx?.ddSpan);
|
|
1039
|
+
state.buffer.delete(spanId);
|
|
1040
|
+
emitted = true;
|
|
1041
|
+
}
|
|
1042
|
+
} while (emitted);
|
|
1043
|
+
}
|
|
1044
|
+
if (state.rootEnded && state.buffer.size === 0 && !state.cleanupTimer) {
|
|
1045
|
+
const timer = setTimeout(() => {
|
|
1046
|
+
const currentState = this.traceState.get(traceId);
|
|
1047
|
+
if (currentState) {
|
|
1048
|
+
if (currentState.buffer.size > 0) this.logger.warn("Discarding orphaned spans during cleanup", {
|
|
1049
|
+
traceId,
|
|
1050
|
+
orphanedCount: currentState.buffer.size,
|
|
1051
|
+
spanIds: Array.from(currentState.buffer.keys())
|
|
1052
|
+
});
|
|
1053
|
+
if (currentState.maxLifetimeTimer) clearTimeout(currentState.maxLifetimeTimer);
|
|
1054
|
+
}
|
|
1055
|
+
this.traceState.delete(traceId);
|
|
1056
|
+
this.traceContext.delete(traceId);
|
|
1057
|
+
}, REGULAR_CLEANUP_INTERVAL_MS);
|
|
1058
|
+
timer.unref?.();
|
|
1059
|
+
state.cleanupTimer = timer;
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Builds a tree structure from buffered spans based on parentSpanId relationships.
|
|
1064
|
+
* Returns the root node of the tree, or undefined if no root span is found.
|
|
1065
|
+
*/
|
|
1066
|
+
buildSpanTree(buffer) {
|
|
1067
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1068
|
+
let rootNode;
|
|
1069
|
+
for (const span of buffer.values()) nodes.set(span.id, {
|
|
1070
|
+
span,
|
|
1071
|
+
children: []
|
|
1072
|
+
});
|
|
1073
|
+
for (const node of nodes.values()) if (node.span.isRootSpan) rootNode = node;
|
|
1074
|
+
else if (node.span.parentSpanId) {
|
|
1075
|
+
const parentNode = nodes.get(node.span.parentSpanId);
|
|
1076
|
+
if (parentNode) parentNode.children.push(node);
|
|
1077
|
+
else this.logger.warn("Orphaned span detected during tree build", {
|
|
1078
|
+
spanId: node.span.id,
|
|
1079
|
+
parentSpanId: node.span.parentSpanId,
|
|
1080
|
+
traceId: node.span.traceId
|
|
1081
|
+
});
|
|
1082
|
+
}
|
|
1083
|
+
for (const node of nodes.values()) node.children.sort((a, b) => {
|
|
1084
|
+
return (a.span.startTime instanceof Date ? a.span.startTime.getTime() : new Date(a.span.startTime).getTime()) - (b.span.startTime instanceof Date ? b.span.startTime.getTime() : new Date(b.span.startTime).getTime());
|
|
1085
|
+
});
|
|
1086
|
+
return rootNode;
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Builds LLMObs span options from a Mastra span.
|
|
1090
|
+
* Handles trace context, timestamps, and conditional model information for LLM spans.
|
|
1091
|
+
*/
|
|
1092
|
+
buildSpanOptions(span, inheritedModelAttrs) {
|
|
1093
|
+
const traceCtx = this.traceContext.get(span.traceId) || {
|
|
1094
|
+
userId: span.metadata?.userId,
|
|
1095
|
+
sessionId: span.metadata?.sessionId
|
|
1096
|
+
};
|
|
1097
|
+
const kind = kindFor(span.type);
|
|
1098
|
+
const ownAttrs = span.attributes;
|
|
1099
|
+
const attrs = {
|
|
1100
|
+
model: ownAttrs?.model ?? inheritedModelAttrs?.model,
|
|
1101
|
+
provider: ownAttrs?.provider ?? inheritedModelAttrs?.provider
|
|
1102
|
+
};
|
|
1103
|
+
const startTime = toDate(span.startTime);
|
|
1104
|
+
const endTime = span.endTime ? toDate(span.endTime) : span.isEvent ? startTime : /* @__PURE__ */ new Date();
|
|
1105
|
+
return {
|
|
1106
|
+
traceOptions: {
|
|
1107
|
+
kind,
|
|
1108
|
+
name: span.name,
|
|
1109
|
+
sessionId: traceCtx.sessionId,
|
|
1110
|
+
userId: traceCtx.userId,
|
|
1111
|
+
startTime,
|
|
1112
|
+
...kind === "llm" && attrs?.model ? { modelName: attrs.model } : {},
|
|
1113
|
+
...kind === "llm" && attrs?.provider ? { modelProvider: attrs.provider } : {}
|
|
1114
|
+
},
|
|
1115
|
+
endTimeMs: endTime.getTime()
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Recursively emits a span tree using nested llmobs.trace() calls.
|
|
1120
|
+
* This ensures parent-child relationships are properly established in Datadog
|
|
1121
|
+
* because child spans are created while their parent span is active in scope.
|
|
1122
|
+
*/
|
|
1123
|
+
emitSpanTree(node, state, inheritedModelAttrs) {
|
|
1124
|
+
const span = node.span;
|
|
1125
|
+
const { traceOptions, endTimeMs } = this.buildSpanOptions(span, inheritedModelAttrs);
|
|
1126
|
+
const childInheritedModelAttrs = span.type === _mastra_core_observability.SpanType.MODEL_GENERATION ? {
|
|
1127
|
+
model: span.attributes?.model,
|
|
1128
|
+
provider: span.attributes?.provider
|
|
1129
|
+
} : inheritedModelAttrs;
|
|
1130
|
+
dd_trace.default.llmobs.trace(traceOptions, (ddSpan) => {
|
|
1131
|
+
const annotations = this.buildAnnotations(span);
|
|
1132
|
+
if (Object.keys(annotations).length > 0) dd_trace.default.llmobs.annotate(ddSpan, annotations);
|
|
1133
|
+
if (span.errorInfo) this.setErrorTags(ddSpan, span.errorInfo);
|
|
1134
|
+
const exported = dd_trace.default.llmobs.exportSpan ? dd_trace.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
1135
|
+
state.contexts.set(span.id, {
|
|
1136
|
+
ddSpan,
|
|
1137
|
+
exported
|
|
1138
|
+
});
|
|
1139
|
+
for (const child of node.children) this.emitSpanTree(child, state, childInheritedModelAttrs);
|
|
1140
|
+
if (typeof ddSpan.finish === "function") ddSpan.finish(endTimeMs);
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Emit a single span with the proper Datadog parent context.
|
|
1145
|
+
* Used for late-arriving spans after the main tree has been emitted.
|
|
1146
|
+
*/
|
|
1147
|
+
emitSingleSpan(span, state, parent) {
|
|
1148
|
+
const { traceOptions, endTimeMs } = this.buildSpanOptions(span);
|
|
1149
|
+
const runTrace = () => dd_trace.default.llmobs.trace(traceOptions, (ddSpan) => {
|
|
1150
|
+
const annotations = this.buildAnnotations(span);
|
|
1151
|
+
if (Object.keys(annotations).length > 0) dd_trace.default.llmobs.annotate(ddSpan, annotations);
|
|
1152
|
+
if (span.errorInfo) this.setErrorTags(ddSpan, span.errorInfo);
|
|
1153
|
+
const exported = dd_trace.default.llmobs.exportSpan ? dd_trace.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
1154
|
+
state.contexts.set(span.id, {
|
|
1155
|
+
ddSpan,
|
|
1156
|
+
exported
|
|
1157
|
+
});
|
|
1158
|
+
if (typeof ddSpan.finish === "function") ddSpan.finish(endTimeMs);
|
|
1159
|
+
});
|
|
1160
|
+
if (parent) dd_trace.default.scope().activate(parent, runTrace);
|
|
1161
|
+
else runTrace();
|
|
1162
|
+
}
|
|
1228
1163
|
};
|
|
1229
|
-
|
|
1164
|
+
//#endregion
|
|
1230
1165
|
exports.DatadogBridge = DatadogBridge;
|
|
1231
1166
|
exports.DatadogExporter = DatadogExporter;
|
|
1232
|
-
|
|
1167
|
+
|
|
1233
1168
|
//# sourceMappingURL=index.cjs.map
|