@mastra/datadog 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +103 -0
- package/dist/index.cjs +519 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +513 -0
- package/dist/index.js.map +1 -0
- package/dist/metrics.d.ts +6 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/tracing.d.ts +129 -0
- package/dist/tracing.d.ts.map +1 -0
- package/dist/utils.d.ts +49 -0
- package/dist/utils.d.ts.map +1 -0
- package/package.json +65 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# @mastra/datadog
|
|
2
|
+
|
|
3
|
+
## 1.0.0-beta.2
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Added a Datadog LLM Observability exporter for Mastra applications. ([#11305](https://github.com/mastra-ai/mastra/pull/11305))
|
|
8
|
+
|
|
9
|
+
This exporter integrates with Datadog's LLM Observability product to provide comprehensive tracing and monitoring for AI/LLM applications built with Mastra.
|
|
10
|
+
- **LLM Observability Integration**: Exports traces to Datadog's dedicated LLM Observability product
|
|
11
|
+
- **Dual Mode Support**: Works with direct HTTPS (agentless) or through a local Datadog Agent
|
|
12
|
+
- **Span Type Mapping**: Automatically maps Mastra span types to Datadog LLMObs kinds (llm, agent, tool, workflow, task)
|
|
13
|
+
- **Message Formatting**: LLM inputs/outputs are formatted as message arrays for proper visualization in Datadog
|
|
14
|
+
- **Token Metrics**: Captures inputTokens, outputTokens, totalTokens, reasoningTokens, and cached tokens
|
|
15
|
+
- **Error Tracking**: Error spans include detailed error info (message, ID, domain, category)
|
|
16
|
+
- **Hierarchical Traces**: Tree-based span emission preserves parent-child relationships
|
|
17
|
+
|
|
18
|
+
Required settings:
|
|
19
|
+
- `mlApp`: Groups traces under an ML application name (required)
|
|
20
|
+
- `apiKey`: Datadog API key (required for agentless mode)
|
|
21
|
+
|
|
22
|
+
Optional settings:
|
|
23
|
+
- `site`: Datadog site (datadoghq.com, datadoghq.eu, us3.datadoghq.com)
|
|
24
|
+
- `agentless`: true for direct HTTPS (default), false for local agent
|
|
25
|
+
- `service`, `env`: APM tagging
|
|
26
|
+
- `integrationsEnabled`: Enable dd-trace auto-instrumentation (default: false)
|
|
27
|
+
|
|
28
|
+
```typescript
|
|
29
|
+
import { Mastra } from '@mastra/core';
|
|
30
|
+
import { Observability } from '@mastra/observability';
|
|
31
|
+
import { DatadogExporter } from '@mastra/datadog';
|
|
32
|
+
|
|
33
|
+
const mastra = new Mastra({
|
|
34
|
+
observability: new Observability({
|
|
35
|
+
configs: {
|
|
36
|
+
datadog: {
|
|
37
|
+
serviceName: 'my-service',
|
|
38
|
+
exporters: [
|
|
39
|
+
new DatadogExporter({
|
|
40
|
+
mlApp: 'my-llm-app',
|
|
41
|
+
apiKey: process.env.DD_API_KEY,
|
|
42
|
+
}),
|
|
43
|
+
],
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
}),
|
|
47
|
+
});
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
This is an initial experimental beta release. Breaking changes may occur in future versions as the API evolves.
|
|
51
|
+
|
|
52
|
+
### Patch Changes
|
|
53
|
+
|
|
54
|
+
- Updated dependencies [[`08766f1`](https://github.com/mastra-ai/mastra/commit/08766f15e13ac0692fde2a8bd366c2e16e4321df), [`ae8baf7`](https://github.com/mastra-ai/mastra/commit/ae8baf7d8adcb0ff9dac11880400452bc49b33ff), [`cfabdd4`](https://github.com/mastra-ai/mastra/commit/cfabdd4aae7a726b706942d6836eeca110fb6267), [`a0e437f`](https://github.com/mastra-ai/mastra/commit/a0e437fac561b28ee719e0302d72b2f9b4c138f0), [`bec5efd`](https://github.com/mastra-ai/mastra/commit/bec5efde96653ccae6604e68c696d1bc6c1a0bf5), [`9eedf7d`](https://github.com/mastra-ai/mastra/commit/9eedf7de1d6e0022a2f4e5e9e6fe1ec468f9b43c)]:
|
|
55
|
+
- @mastra/core@1.0.0-beta.21
|
|
56
|
+
|
|
57
|
+
## 1.0.0-beta.1
|
|
58
|
+
|
|
59
|
+
### Major Changes
|
|
60
|
+
|
|
61
|
+
- Initial release of DatadogExporter for Mastra LLM Observability
|
package/README.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# @mastra/datadog
|
|
2
|
+
|
|
3
|
+
Datadog LLM Observability exporter for Mastra. Exports observability data to [Datadog's LLM Observability](https://docs.datadoghq.com/llm_observability/) product.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pnpm add @mastra/datadog
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Requirements
|
|
12
|
+
|
|
13
|
+
- Datadog account with LLM Observability enabled
|
|
14
|
+
- Datadog API key (available in your Datadog account settings)
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
### Basic Setup
|
|
19
|
+
|
|
20
|
+
```typescript
|
|
21
|
+
import { Mastra } from '@mastra/core';
|
|
22
|
+
import { DatadogExporter } from '@mastra/datadog';
|
|
23
|
+
|
|
24
|
+
const datadog = new DatadogExporter({
|
|
25
|
+
mlApp: 'my-llm-app',
|
|
26
|
+
apiKey: process.env.DD_API_KEY,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
const mastra = new Mastra({
|
|
30
|
+
observability: {
|
|
31
|
+
configs: {
|
|
32
|
+
default: {
|
|
33
|
+
serviceName: 'my-service',
|
|
34
|
+
exporters: [datadog],
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### With Local Datadog Agent (Optional)
|
|
42
|
+
|
|
43
|
+
If you have a Datadog Agent running locally, you can use agent mode instead:
|
|
44
|
+
|
|
45
|
+
```typescript
|
|
46
|
+
const datadog = new DatadogExporter({
|
|
47
|
+
mlApp: 'my-llm-app',
|
|
48
|
+
agentless: false, // Use local Datadog Agent instead of direct HTTPS
|
|
49
|
+
env: 'production',
|
|
50
|
+
});
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Configuration Options
|
|
54
|
+
|
|
55
|
+
| Option | Description | Default |
|
|
56
|
+
| --------------------- | ---------------------------------------------------- | ------------------------------ |
|
|
57
|
+
| `apiKey` | Datadog API key (required) | `DD_API_KEY` env var |
|
|
58
|
+
| `mlApp` | ML application name for grouping traces (required) | `DD_LLMOBS_ML_APP` env var |
|
|
59
|
+
| `site` | Datadog site (e.g., 'datadoghq.com', 'datadoghq.eu') | `DD_SITE` or `'datadoghq.com'` |
|
|
60
|
+
| `agentless` | Use direct HTTPS intake (no local agent required) | `true` |
|
|
61
|
+
| `service` | Service name for the application | Uses `mlApp` value |
|
|
62
|
+
| `env` | Environment name (e.g., 'production', 'staging') | `DD_ENV` env var |
|
|
63
|
+
| `integrationsEnabled` | Enable dd-trace automatic integrations | `false` |
|
|
64
|
+
|
|
65
|
+
Note that the `site` is also used to specify non-default regions, e.g. `us3.datadoghq.com` instead of `us1.datadoghq.com`.
|
|
66
|
+
|
|
67
|
+
### Environment Variables
|
|
68
|
+
|
|
69
|
+
The exporter reads configuration from environment variables:
|
|
70
|
+
|
|
71
|
+
- `DD_API_KEY` - Datadog API key (required)
|
|
72
|
+
- `DD_LLMOBS_ML_APP` - ML application name
|
|
73
|
+
- `DD_SITE` - Datadog site
|
|
74
|
+
- `DD_ENV` - Environment name
|
|
75
|
+
- `DD_LLMOBS_AGENTLESS_ENABLED` - Set to 'false' or '0' to use local Datadog Agent
|
|
76
|
+
|
|
77
|
+
## Span Type Mapping
|
|
78
|
+
|
|
79
|
+
Mastra span types are mapped to Datadog LLMObs span kinds:
|
|
80
|
+
|
|
81
|
+
| Mastra SpanType | Datadog Kind |
|
|
82
|
+
| ------------------ | ------------ |
|
|
83
|
+
| `AGENT_RUN` | `agent` |
|
|
84
|
+
| `MODEL_GENERATION` | `workflow` |
|
|
85
|
+
| `MODEL_STEP` | `llm` |
|
|
86
|
+
| `TOOL_CALL` | `tool` |
|
|
87
|
+
| `MCP_TOOL_CALL` | `tool` |
|
|
88
|
+
| `WORKFLOW_RUN` | `workflow` |
|
|
89
|
+
| All other types | `task` |
|
|
90
|
+
|
|
91
|
+
All unmapped span types (including `MODEL_CHUNK`, `GENERIC`, etc., and future span types) automatically default to `task`.
|
|
92
|
+
|
|
93
|
+
## Features
|
|
94
|
+
|
|
95
|
+
- **Completion-only pattern**: Spans are emitted at completion for efficient tracing
|
|
96
|
+
- **Message formatting**: LLM inputs/outputs formatted as message arrays
|
|
97
|
+
- **Metadata as tags**: Span metadata is flattened into searchable Datadog tags
|
|
98
|
+
- **Error tracking**: Error spans include error tags with message, ID, and category
|
|
99
|
+
- **Parent/child hierarchy**: Spans are emitted parent-first to preserve trace trees in Datadog
|
|
100
|
+
|
|
101
|
+
## License
|
|
102
|
+
|
|
103
|
+
Apache-2.0
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var observability$1 = require('@mastra/core/observability');
|
|
4
|
+
var utils = require('@mastra/core/utils');
|
|
5
|
+
var observability = require('@mastra/observability');
|
|
6
|
+
var tracer2 = require('dd-trace');
|
|
7
|
+
|
|
8
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
9
|
+
|
|
10
|
+
var tracer2__default = /*#__PURE__*/_interopDefault(tracer2);
|
|
11
|
+
|
|
12
|
+
// src/tracing.ts
|
|
13
|
+
|
|
14
|
+
// src/metrics.ts
|
|
15
|
+
function formatUsageMetrics(usage) {
|
|
16
|
+
if (!usage) return void 0;
|
|
17
|
+
const result = {};
|
|
18
|
+
const inputTokens = usage.inputTokens;
|
|
19
|
+
if (inputTokens !== void 0) result.inputTokens = inputTokens;
|
|
20
|
+
const outputTokens = usage.outputTokens;
|
|
21
|
+
if (outputTokens !== void 0) result.outputTokens = outputTokens;
|
|
22
|
+
if (inputTokens !== void 0 && outputTokens !== void 0) {
|
|
23
|
+
result.totalTokens = inputTokens + outputTokens;
|
|
24
|
+
}
|
|
25
|
+
if (usage?.outputDetails?.reasoning !== void 0) {
|
|
26
|
+
result.reasoningTokens = usage.outputDetails.reasoning;
|
|
27
|
+
}
|
|
28
|
+
const cachedTokens = usage?.inputDetails?.cacheRead;
|
|
29
|
+
if (cachedTokens !== void 0) {
|
|
30
|
+
result.cachedInputTokens = cachedTokens;
|
|
31
|
+
}
|
|
32
|
+
const cachedOutputTokens = usage?.inputDetails?.cacheWrite;
|
|
33
|
+
if (cachedOutputTokens !== void 0) {
|
|
34
|
+
result.cachedOutputTokens = cachedOutputTokens;
|
|
35
|
+
}
|
|
36
|
+
return Object.keys(result).length > 0 ? result : void 0;
|
|
37
|
+
}
|
|
38
|
+
var SPAN_TYPE_TO_KIND = {
|
|
39
|
+
[observability$1.SpanType.AGENT_RUN]: "agent",
|
|
40
|
+
[observability$1.SpanType.MODEL_GENERATION]: "workflow",
|
|
41
|
+
[observability$1.SpanType.MODEL_STEP]: "llm",
|
|
42
|
+
[observability$1.SpanType.TOOL_CALL]: "tool",
|
|
43
|
+
[observability$1.SpanType.MCP_TOOL_CALL]: "tool",
|
|
44
|
+
[observability$1.SpanType.WORKFLOW_RUN]: "workflow"
|
|
45
|
+
};
|
|
46
|
+
var tracerInitFlag = { done: false };
|
|
47
|
+
function ensureTracer(config) {
|
|
48
|
+
if (tracerInitFlag.done) return;
|
|
49
|
+
if (config.site) {
|
|
50
|
+
process.env.DD_SITE = config.site;
|
|
51
|
+
}
|
|
52
|
+
if (config.apiKey) {
|
|
53
|
+
process.env.DD_API_KEY = config.apiKey;
|
|
54
|
+
}
|
|
55
|
+
const alreadyStarted = tracer2__default.default._tracer?.started;
|
|
56
|
+
if (!alreadyStarted) {
|
|
57
|
+
tracer2__default.default.init({
|
|
58
|
+
service: config.service || config.mlApp,
|
|
59
|
+
env: config.env || process.env.DD_ENV,
|
|
60
|
+
// Disable automatic integrations by default to avoid surprise instrumentation
|
|
61
|
+
plugins: config.integrationsEnabled ?? false
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
tracer2__default.default.llmobs.enable({
|
|
65
|
+
mlApp: config.mlApp,
|
|
66
|
+
agentlessEnabled: config.agentless
|
|
67
|
+
});
|
|
68
|
+
tracerInitFlag.done = true;
|
|
69
|
+
}
|
|
70
|
+
function kindFor(spanType) {
|
|
71
|
+
return SPAN_TYPE_TO_KIND[spanType] || "task";
|
|
72
|
+
}
|
|
73
|
+
function toDate(value) {
|
|
74
|
+
return value instanceof Date ? value : new Date(value);
|
|
75
|
+
}
|
|
76
|
+
function safeStringify(data) {
|
|
77
|
+
try {
|
|
78
|
+
return JSON.stringify(data);
|
|
79
|
+
} catch {
|
|
80
|
+
if (typeof data === "object" && data !== null) {
|
|
81
|
+
return `[Non-serializable ${data.constructor?.name || "Object"}]`;
|
|
82
|
+
}
|
|
83
|
+
return String(data);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
function isMessageArray(data) {
|
|
87
|
+
return Array.isArray(data) && data.every((m) => m?.role && m?.content !== void 0);
|
|
88
|
+
}
|
|
89
|
+
function formatInput(input, spanType) {
|
|
90
|
+
if (spanType === observability$1.SpanType.MODEL_GENERATION || spanType === observability$1.SpanType.MODEL_STEP) {
|
|
91
|
+
if (isMessageArray(input)) {
|
|
92
|
+
return input.map((m) => ({
|
|
93
|
+
role: m.role,
|
|
94
|
+
content: typeof m.content === "string" ? m.content : safeStringify(m.content)
|
|
95
|
+
}));
|
|
96
|
+
}
|
|
97
|
+
if (typeof input === "string") {
|
|
98
|
+
return [{ role: "user", content: input }];
|
|
99
|
+
}
|
|
100
|
+
return [{ role: "user", content: safeStringify(input) }];
|
|
101
|
+
}
|
|
102
|
+
if (typeof input === "string" || Array.isArray(input)) return input;
|
|
103
|
+
return safeStringify(input);
|
|
104
|
+
}
|
|
105
|
+
function formatOutput(output, spanType) {
|
|
106
|
+
if (spanType === observability$1.SpanType.MODEL_GENERATION || spanType === observability$1.SpanType.MODEL_STEP) {
|
|
107
|
+
if (isMessageArray(output)) {
|
|
108
|
+
return output.map((m) => ({
|
|
109
|
+
role: m.role,
|
|
110
|
+
content: typeof m.content === "string" ? m.content : safeStringify(m.content)
|
|
111
|
+
}));
|
|
112
|
+
}
|
|
113
|
+
if (typeof output === "string") {
|
|
114
|
+
return [{ role: "assistant", content: output }];
|
|
115
|
+
}
|
|
116
|
+
if (output?.text) {
|
|
117
|
+
return [{ role: "assistant", content: output.text }];
|
|
118
|
+
}
|
|
119
|
+
return [{ role: "assistant", content: safeStringify(output) }];
|
|
120
|
+
}
|
|
121
|
+
if (typeof output === "string") return output;
|
|
122
|
+
return safeStringify(output);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// src/tracing.ts
|
|
126
|
+
var MAX_TRACE_LIFETIME_MS = 30 * 60 * 1e3;
|
|
127
|
+
var REGULAR_CLEANUP_INTERVAL_MS = 1 * 60 * 1e3;
|
|
128
|
+
var DatadogExporter = class extends observability.BaseExporter {
|
|
129
|
+
name = "datadog";
|
|
130
|
+
config;
|
|
131
|
+
traceContext = /* @__PURE__ */ new Map();
|
|
132
|
+
traceState = /* @__PURE__ */ new Map();
|
|
133
|
+
constructor(config = {}) {
|
|
134
|
+
super(config);
|
|
135
|
+
const mlApp = config.mlApp || process.env.DD_LLMOBS_ML_APP;
|
|
136
|
+
const apiKey = config.apiKey || process.env.DD_API_KEY;
|
|
137
|
+
const site = config.site || process.env.DD_SITE || "datadoghq.com";
|
|
138
|
+
const envAgentless = process.env.DD_LLMOBS_AGENTLESS_ENABLED?.toLowerCase();
|
|
139
|
+
const agentless = config.agentless ?? (envAgentless === "false" || envAgentless === "0" ? false : true);
|
|
140
|
+
if (!mlApp) {
|
|
141
|
+
this.setDisabled("Missing required mlApp (set config.mlApp or DD_LLMOBS_ML_APP)");
|
|
142
|
+
this.config = config;
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (agentless && !apiKey) {
|
|
146
|
+
this.setDisabled("Missing required apiKey (set config.apiKey or DD_API_KEY)");
|
|
147
|
+
this.config = config;
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
this.config = { ...config, mlApp, site, apiKey, agentless };
|
|
151
|
+
ensureTracer({
|
|
152
|
+
mlApp,
|
|
153
|
+
site,
|
|
154
|
+
apiKey,
|
|
155
|
+
agentless,
|
|
156
|
+
service: config.service,
|
|
157
|
+
env: config.env,
|
|
158
|
+
integrationsEnabled: config.integrationsEnabled
|
|
159
|
+
});
|
|
160
|
+
this.logger.info("Datadog exporter initialized", { mlApp, site, agentless });
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Main entry point for tracing events from Mastra.
|
|
164
|
+
*/
|
|
165
|
+
async _exportTracingEvent(event) {
|
|
166
|
+
if (this.isDisabled || !tracer2__default.default.llmobs) return;
|
|
167
|
+
try {
|
|
168
|
+
const span = event.exportedSpan;
|
|
169
|
+
if (span.isEvent) {
|
|
170
|
+
if (event.type === "span_started") {
|
|
171
|
+
this.captureTraceContext(span);
|
|
172
|
+
this.enqueueSpan(span);
|
|
173
|
+
}
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
switch (event.type) {
|
|
177
|
+
case "span_started":
|
|
178
|
+
this.captureTraceContext(span);
|
|
179
|
+
return;
|
|
180
|
+
case "span_updated":
|
|
181
|
+
return;
|
|
182
|
+
case "span_ended":
|
|
183
|
+
this.enqueueSpan(span);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
} catch (error) {
|
|
187
|
+
this.logger.error("Datadog exporter error", {
|
|
188
|
+
error,
|
|
189
|
+
eventType: event.type,
|
|
190
|
+
spanId: event.exportedSpan?.id,
|
|
191
|
+
spanName: event.exportedSpan?.name
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Captures user/session context from root spans for tagging all spans in the trace.
|
|
197
|
+
*/
|
|
198
|
+
captureTraceContext(span) {
|
|
199
|
+
if (span.isRootSpan && !this.traceContext.has(span.traceId)) {
|
|
200
|
+
this.traceContext.set(span.traceId, {
|
|
201
|
+
userId: span.metadata?.userId,
|
|
202
|
+
sessionId: span.metadata?.sessionId
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Queue span until its parent context is available, then emit spans parent-first.
|
|
208
|
+
*/
|
|
209
|
+
enqueueSpan(span) {
|
|
210
|
+
const state = this.getOrCreateTraceState(span.traceId);
|
|
211
|
+
if (span.isRootSpan) {
|
|
212
|
+
state.rootEnded = true;
|
|
213
|
+
}
|
|
214
|
+
state.buffer.set(span.id, span);
|
|
215
|
+
this.tryEmitReadySpans(span.traceId);
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Builds annotations object for llmobs.annotate().
|
|
219
|
+
* Uses dd-trace's expected property names: inputData, outputData, metadata, tags, metrics.
|
|
220
|
+
*/
|
|
221
|
+
buildAnnotations(span) {
|
|
222
|
+
const annotations = {};
|
|
223
|
+
if (span.input !== void 0) {
|
|
224
|
+
annotations.inputData = formatInput(span.input, span.type);
|
|
225
|
+
}
|
|
226
|
+
if (span.output !== void 0) {
|
|
227
|
+
annotations.outputData = formatOutput(span.output, span.type);
|
|
228
|
+
}
|
|
229
|
+
if (span.type === observability$1.SpanType.MODEL_GENERATION || span.type === observability$1.SpanType.MODEL_STEP) {
|
|
230
|
+
const usage = span.attributes?.usage;
|
|
231
|
+
const metrics = formatUsageMetrics(usage);
|
|
232
|
+
if (metrics) {
|
|
233
|
+
annotations.metrics = metrics;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const knownFields = ["usage", "model", "provider", "parameters"];
|
|
237
|
+
const otherAttributes = utils.omitKeys(span.attributes ?? {}, knownFields);
|
|
238
|
+
const combinedMetadata = {
|
|
239
|
+
...span.metadata,
|
|
240
|
+
...otherAttributes
|
|
241
|
+
};
|
|
242
|
+
if (Object.keys(combinedMetadata).length > 0) {
|
|
243
|
+
annotations.metadata = combinedMetadata;
|
|
244
|
+
}
|
|
245
|
+
const tags = {};
|
|
246
|
+
if (span.tags?.length) {
|
|
247
|
+
for (const tag of span.tags) {
|
|
248
|
+
tags[tag] = true;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (span.errorInfo) {
|
|
252
|
+
tags.error = true;
|
|
253
|
+
tags.errorInfo = {
|
|
254
|
+
message: span.errorInfo.message,
|
|
255
|
+
...span.errorInfo.id ? { id: span.errorInfo.id } : {},
|
|
256
|
+
...span.errorInfo.domain ? { domain: span.errorInfo.domain } : {},
|
|
257
|
+
...span.errorInfo.category ? { category: span.errorInfo.category } : {}
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
if (Object.keys(tags).length > 0) {
|
|
261
|
+
annotations.tags = tags;
|
|
262
|
+
}
|
|
263
|
+
return annotations;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Gracefully shuts down the exporter.
|
|
267
|
+
*/
|
|
268
|
+
async shutdown() {
|
|
269
|
+
for (const [traceId, state] of this.traceState) {
|
|
270
|
+
if (state.cleanupTimer) {
|
|
271
|
+
clearTimeout(state.cleanupTimer);
|
|
272
|
+
}
|
|
273
|
+
if (state.maxLifetimeTimer) {
|
|
274
|
+
clearTimeout(state.maxLifetimeTimer);
|
|
275
|
+
}
|
|
276
|
+
if (state.buffer.size > 0) {
|
|
277
|
+
this.logger.warn("Shutdown with pending spans", {
|
|
278
|
+
traceId,
|
|
279
|
+
pendingCount: state.buffer.size,
|
|
280
|
+
spanIds: Array.from(state.buffer.keys())
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
this.traceState.clear();
|
|
285
|
+
if (tracer2__default.default.llmobs?.flush) {
|
|
286
|
+
try {
|
|
287
|
+
await tracer2__default.default.llmobs.flush();
|
|
288
|
+
} catch (e) {
|
|
289
|
+
this.logger.error("Error flushing llmobs", { error: e });
|
|
290
|
+
}
|
|
291
|
+
} else if (tracer2__default.default.flush) {
|
|
292
|
+
try {
|
|
293
|
+
await tracer2__default.default.flush();
|
|
294
|
+
} catch (e) {
|
|
295
|
+
this.logger.error("Error flushing tracer", { error: e });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (tracer2__default.default.llmobs?.disable) {
|
|
299
|
+
try {
|
|
300
|
+
tracer2__default.default.llmobs.disable();
|
|
301
|
+
} catch (e) {
|
|
302
|
+
this.logger.error("Error disabling llmobs", { error: e });
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
this.traceContext.clear();
|
|
306
|
+
await super.shutdown();
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Retrieve or initialize trace state for buffering and parent tracking.
|
|
310
|
+
*/
|
|
311
|
+
getOrCreateTraceState(traceId) {
|
|
312
|
+
const existing = this.traceState.get(traceId);
|
|
313
|
+
if (existing) {
|
|
314
|
+
if (existing.cleanupTimer) {
|
|
315
|
+
clearTimeout(existing.cleanupTimer);
|
|
316
|
+
existing.cleanupTimer = void 0;
|
|
317
|
+
}
|
|
318
|
+
return existing;
|
|
319
|
+
}
|
|
320
|
+
const created = {
|
|
321
|
+
buffer: /* @__PURE__ */ new Map(),
|
|
322
|
+
contexts: /* @__PURE__ */ new Map(),
|
|
323
|
+
rootEnded: false,
|
|
324
|
+
treeEmitted: false,
|
|
325
|
+
createdAt: Date.now(),
|
|
326
|
+
cleanupTimer: void 0,
|
|
327
|
+
maxLifetimeTimer: void 0
|
|
328
|
+
};
|
|
329
|
+
const maxLifetimeTimer = setTimeout(() => {
|
|
330
|
+
const state = this.traceState.get(traceId);
|
|
331
|
+
if (state) {
|
|
332
|
+
if (state.buffer.size > 0 || state.contexts.size > 0) {
|
|
333
|
+
this.logger.warn("Discarding trace due to max lifetime exceeded", {
|
|
334
|
+
traceId,
|
|
335
|
+
bufferedSpans: state.buffer.size,
|
|
336
|
+
emittedSpans: state.contexts.size,
|
|
337
|
+
lifetimeMs: Date.now() - state.createdAt
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
if (state.cleanupTimer) {
|
|
341
|
+
clearTimeout(state.cleanupTimer);
|
|
342
|
+
}
|
|
343
|
+
this.traceState.delete(traceId);
|
|
344
|
+
this.traceContext.delete(traceId);
|
|
345
|
+
}
|
|
346
|
+
}, MAX_TRACE_LIFETIME_MS);
|
|
347
|
+
maxLifetimeTimer.unref?.();
|
|
348
|
+
created.maxLifetimeTimer = maxLifetimeTimer;
|
|
349
|
+
this.traceState.set(traceId, created);
|
|
350
|
+
return created;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Attempt to emit spans from the buffer.
|
|
354
|
+
*
|
|
355
|
+
* Two modes of operation:
|
|
356
|
+
* 1. Initial tree emission: When root span ends and tree hasn't been emitted yet,
|
|
357
|
+
* build a tree from all buffered spans and emit recursively using nested
|
|
358
|
+
* llmobs.trace() calls. This ensures proper parent-child relationships in Datadog.
|
|
359
|
+
* 2. Late-arriving spans: After the tree has been emitted, emit individual spans
|
|
360
|
+
* with their parent context for proper linking.
|
|
361
|
+
*/
|
|
362
|
+
tryEmitReadySpans(traceId) {
|
|
363
|
+
const state = this.traceState.get(traceId);
|
|
364
|
+
if (!state) return;
|
|
365
|
+
if (!state.treeEmitted) {
|
|
366
|
+
if (!state.rootEnded) return;
|
|
367
|
+
const tree = this.buildSpanTree(state.buffer);
|
|
368
|
+
if (tree) {
|
|
369
|
+
this.emitSpanTree(tree, state);
|
|
370
|
+
}
|
|
371
|
+
state.buffer.clear();
|
|
372
|
+
state.treeEmitted = true;
|
|
373
|
+
} else {
|
|
374
|
+
let emitted = false;
|
|
375
|
+
do {
|
|
376
|
+
emitted = false;
|
|
377
|
+
for (const [spanId, span] of state.buffer) {
|
|
378
|
+
const parentCtx = span.parentSpanId ? state.contexts.get(span.parentSpanId) : void 0;
|
|
379
|
+
if (span.parentSpanId && !parentCtx) {
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
this.emitSingleSpan(span, state, parentCtx?.ddSpan);
|
|
383
|
+
state.buffer.delete(spanId);
|
|
384
|
+
emitted = true;
|
|
385
|
+
}
|
|
386
|
+
} while (emitted);
|
|
387
|
+
}
|
|
388
|
+
if (state.rootEnded && state.buffer.size === 0 && !state.cleanupTimer) {
|
|
389
|
+
const timer = setTimeout(() => {
|
|
390
|
+
const currentState = this.traceState.get(traceId);
|
|
391
|
+
if (currentState) {
|
|
392
|
+
if (currentState.buffer.size > 0) {
|
|
393
|
+
this.logger.warn("Discarding orphaned spans during cleanup", {
|
|
394
|
+
traceId,
|
|
395
|
+
orphanedCount: currentState.buffer.size,
|
|
396
|
+
spanIds: Array.from(currentState.buffer.keys())
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
if (currentState.maxLifetimeTimer) {
|
|
400
|
+
clearTimeout(currentState.maxLifetimeTimer);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
this.traceState.delete(traceId);
|
|
404
|
+
this.traceContext.delete(traceId);
|
|
405
|
+
}, REGULAR_CLEANUP_INTERVAL_MS);
|
|
406
|
+
timer.unref?.();
|
|
407
|
+
state.cleanupTimer = timer;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Builds a tree structure from buffered spans based on parentSpanId relationships.
|
|
412
|
+
* Returns the root node of the tree, or undefined if no root span is found.
|
|
413
|
+
*/
|
|
414
|
+
buildSpanTree(buffer) {
|
|
415
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
416
|
+
let rootNode;
|
|
417
|
+
for (const span of buffer.values()) {
|
|
418
|
+
nodes.set(span.id, { span, children: [] });
|
|
419
|
+
}
|
|
420
|
+
for (const node of nodes.values()) {
|
|
421
|
+
if (node.span.isRootSpan) {
|
|
422
|
+
rootNode = node;
|
|
423
|
+
} else if (node.span.parentSpanId) {
|
|
424
|
+
const parentNode = nodes.get(node.span.parentSpanId);
|
|
425
|
+
if (parentNode) {
|
|
426
|
+
parentNode.children.push(node);
|
|
427
|
+
} else {
|
|
428
|
+
this.logger.warn("Orphaned span detected during tree build", {
|
|
429
|
+
spanId: node.span.id,
|
|
430
|
+
parentSpanId: node.span.parentSpanId,
|
|
431
|
+
traceId: node.span.traceId
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
for (const node of nodes.values()) {
|
|
437
|
+
node.children.sort((a, b) => {
|
|
438
|
+
const aTime = a.span.startTime instanceof Date ? a.span.startTime.getTime() : new Date(a.span.startTime).getTime();
|
|
439
|
+
const bTime = b.span.startTime instanceof Date ? b.span.startTime.getTime() : new Date(b.span.startTime).getTime();
|
|
440
|
+
return aTime - bTime;
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
return rootNode;
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Builds LLMObs span options from a Mastra span.
|
|
447
|
+
* Handles trace context, timestamps, and conditional model information for LLM spans.
|
|
448
|
+
*/
|
|
449
|
+
buildSpanOptions(span) {
|
|
450
|
+
const traceCtx = this.traceContext.get(span.traceId) || {
|
|
451
|
+
userId: span.metadata?.userId,
|
|
452
|
+
sessionId: span.metadata?.sessionId
|
|
453
|
+
};
|
|
454
|
+
const kind = kindFor(span.type);
|
|
455
|
+
const attrs = span.attributes;
|
|
456
|
+
const startTime = toDate(span.startTime);
|
|
457
|
+
const endTime = span.endTime ? toDate(span.endTime) : span.isEvent ? startTime : /* @__PURE__ */ new Date();
|
|
458
|
+
return {
|
|
459
|
+
kind,
|
|
460
|
+
name: span.name,
|
|
461
|
+
sessionId: traceCtx.sessionId,
|
|
462
|
+
userId: traceCtx.userId,
|
|
463
|
+
startTime,
|
|
464
|
+
endTime,
|
|
465
|
+
...kind === "llm" && attrs?.model ? { modelName: attrs.model } : {},
|
|
466
|
+
...kind === "llm" && attrs?.provider ? { modelProvider: attrs.provider } : {}
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Recursively emits a span tree using nested llmobs.trace() calls.
|
|
471
|
+
* This ensures parent-child relationships are properly established in Datadog
|
|
472
|
+
* because child spans are created while their parent span is active in scope.
|
|
473
|
+
*/
|
|
474
|
+
emitSpanTree(node, state) {
|
|
475
|
+
const span = node.span;
|
|
476
|
+
const options = this.buildSpanOptions(span);
|
|
477
|
+
tracer2__default.default.llmobs.trace(options, (ddSpan) => {
|
|
478
|
+
const annotations = this.buildAnnotations(span);
|
|
479
|
+
if (Object.keys(annotations).length > 0) {
|
|
480
|
+
tracer2__default.default.llmobs.annotate(ddSpan, annotations);
|
|
481
|
+
}
|
|
482
|
+
if (span.errorInfo) {
|
|
483
|
+
ddSpan.setTag("error", true);
|
|
484
|
+
}
|
|
485
|
+
const exported = tracer2__default.default.llmobs.exportSpan ? tracer2__default.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
486
|
+
state.contexts.set(span.id, { ddSpan, exported });
|
|
487
|
+
for (const child of node.children) {
|
|
488
|
+
this.emitSpanTree(child, state);
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Emit a single span with the proper Datadog parent context.
|
|
494
|
+
* Used for late-arriving spans after the main tree has been emitted.
|
|
495
|
+
*/
|
|
496
|
+
emitSingleSpan(span, state, parent) {
|
|
497
|
+
const options = this.buildSpanOptions(span);
|
|
498
|
+
const runTrace = () => tracer2__default.default.llmobs.trace(options, (ddSpan) => {
|
|
499
|
+
const annotations = this.buildAnnotations(span);
|
|
500
|
+
if (Object.keys(annotations).length > 0) {
|
|
501
|
+
tracer2__default.default.llmobs.annotate(ddSpan, annotations);
|
|
502
|
+
}
|
|
503
|
+
if (span.errorInfo) {
|
|
504
|
+
ddSpan.setTag("error", true);
|
|
505
|
+
}
|
|
506
|
+
const exported = tracer2__default.default.llmobs.exportSpan ? tracer2__default.default.llmobs.exportSpan(ddSpan) : void 0;
|
|
507
|
+
state.contexts.set(span.id, { ddSpan, exported });
|
|
508
|
+
});
|
|
509
|
+
if (parent) {
|
|
510
|
+
tracer2__default.default.scope().activate(parent, runTrace);
|
|
511
|
+
} else {
|
|
512
|
+
runTrace();
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
exports.DatadogExporter = DatadogExporter;
|
|
518
|
+
//# sourceMappingURL=index.cjs.map
|
|
519
|
+
//# sourceMappingURL=index.cjs.map
|