dd-trace 5.117.0 → 5.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ci/diagnose.js +162 -21
- package/ci/init.js +0 -1
- package/ci/runbook.md +187 -180
- package/ci/test-optimization-validation/approval-artifacts.js +3 -1
- package/ci/test-optimization-validation/approval.js +52 -15
- package/ci/test-optimization-validation/ci-command-candidate.js +26 -32
- package/ci/test-optimization-validation/ci-discovery.js +1 -1
- package/ci/test-optimization-validation/ci-package-scripts.js +141 -0
- package/ci/test-optimization-validation/ci-remediation.js +112 -39
- package/ci/test-optimization-validation/cli.js +516 -681
- package/ci/test-optimization-validation/command-blocker.js +188 -33
- package/ci/test-optimization-validation/command-output-policy.js +3 -26
- package/ci/test-optimization-validation/command-runner.js +189 -245
- package/ci/test-optimization-validation/environment.js +90 -0
- package/ci/test-optimization-validation/executable.js +159 -388
- package/ci/test-optimization-validation/framework-adapters/cucumber.js +119 -0
- package/ci/test-optimization-validation/framework-adapters/cypress.js +107 -0
- package/ci/test-optimization-validation/framework-adapters/playwright.js +181 -0
- package/ci/test-optimization-validation/generated-files.js +75 -13
- package/ci/test-optimization-validation/generated-test-contract.js +283 -0
- package/ci/test-optimization-validation/generated-verifier.js +49 -16
- package/ci/test-optimization-validation/literal-environment.js +57 -0
- package/ci/test-optimization-validation/manifest-loader.js +2 -22
- package/ci/test-optimization-validation/manifest-scaffold.js +816 -515
- package/ci/test-optimization-validation/manifest-schema.js +494 -702
- package/ci/test-optimization-validation/offline-fixtures.js +4 -1
- package/ci/test-optimization-validation/plan-writer.js +243 -984
- package/ci/test-optimization-validation/preflight-runner.js +49 -55
- package/ci/test-optimization-validation/redaction.js +2 -1
- package/ci/test-optimization-validation/report-writer.js +391 -1357
- package/ci/test-optimization-validation/result-semantics.js +86 -0
- package/ci/test-optimization-validation/runner-command.js +249 -0
- package/ci/test-optimization-validation/runner-contract.js +664 -0
- package/ci/test-optimization-validation/scenarios/auto-test-retries.js +29 -3
- package/ci/test-optimization-validation/scenarios/basic-reporting.js +240 -543
- package/ci/test-optimization-validation/scenarios/ci-wiring.js +450 -670
- package/ci/test-optimization-validation/scenarios/early-flake-detection.js +4 -3
- package/ci/test-optimization-validation/scenarios/helpers.js +67 -9
- package/ci/test-optimization-validation/scenarios/test-management.js +4 -3
- package/ci/test-optimization-validation/source-text.js +87 -0
- package/ci/test-optimization-validation/test-output.js +12 -22
- package/ext/tags.d.ts +1 -0
- package/index.d.ts +7 -0
- package/package.json +2 -2
- package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +1 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/openai-agents.js +31 -0
- package/packages/datadog-instrumentations/src/openai-agents.js +159 -0
- package/packages/datadog-plugin-openai-agents/src/index.js +74 -0
- package/packages/datadog-plugin-openai-agents/src/integration.js +503 -0
- package/packages/datadog-plugin-openai-agents/src/processor.js +108 -0
- package/packages/datadog-plugin-openai-agents/src/util.js +60 -0
- package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +48 -49
- package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +161 -33
- package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js +6 -4
- package/packages/dd-trace/src/config/generated-config-types.d.ts +2 -0
- package/packages/dd-trace/src/config/supported-configurations.json +7 -0
- package/packages/dd-trace/src/encode/span-stats.js +7 -1
- package/packages/dd-trace/src/exporter.js +16 -9
- package/packages/dd-trace/src/llmobs/index.js +9 -1
- package/packages/dd-trace/src/llmobs/plugins/ai/util.js +104 -22
- package/packages/dd-trace/src/llmobs/plugins/openai/index.js +5 -9
- package/packages/dd-trace/src/llmobs/plugins/openai/utils.js +20 -0
- package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js +321 -0
- package/packages/dd-trace/src/llmobs/sdk.js +2 -1
- package/packages/dd-trace/src/llmobs/span_processor.js +7 -2
- package/packages/dd-trace/src/llmobs/tagger.js +23 -3
- package/packages/dd-trace/src/llmobs/util.js +56 -3
- package/packages/dd-trace/src/openfeature/agentless_configuration_source.js +3 -6
- package/packages/dd-trace/src/openfeature/configuration_source.js +5 -8
- package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_transformer.js +5 -1
- package/packages/dd-trace/src/plugins/ci_plugin.js +9 -3
- package/packages/dd-trace/src/plugins/index.js +1 -0
- package/packages/dd-trace/src/plugins/outbound.js +4 -1
- package/packages/dd-trace/src/span_processor.js +1 -1
- package/packages/dd-trace/src/span_stats.js +22 -4
- package/ci/test-optimization-validation/command-suitability.js +0 -471
- package/ci/test-optimization-validation/init-probe-preload.js +0 -163
- package/ci/test-optimization-validation/init-probe.js +0 -246
- package/ci/test-optimization-validation/late-initialization.js +0 -63
- package/ci/test-optimization-validation/local-command.js +0 -163
- package/ci/test-optimization-validation/setup-runner.js +0 -97
- package/ci/test-optimization-validation-manifest.schema.json +0 -1
|
@@ -5,7 +5,6 @@ const {
|
|
|
5
5
|
PROMPT_TRACKING_INSTRUMENTATION_METHOD,
|
|
6
6
|
PROMPT_MULTIMODAL,
|
|
7
7
|
INSTRUMENTATION_METHOD_AUTO,
|
|
8
|
-
UNKNOWN_MODEL_PROVIDER,
|
|
9
8
|
} = require('../../constants/tags')
|
|
10
9
|
const { audioMimeTypeFromFormat, formatAudioPart, safeJsonParse } = require('../../util')
|
|
11
10
|
const { AUDIO_MIME_TYPES } = require('./constants')
|
|
@@ -15,6 +14,7 @@ const {
|
|
|
15
14
|
extractTextFromContentItem,
|
|
16
15
|
extractContentParts,
|
|
17
16
|
hasMultimodalInputs,
|
|
17
|
+
getOpenAIModelProvider,
|
|
18
18
|
} = require('./utils')
|
|
19
19
|
|
|
20
20
|
const allowedParamKeys = new Set([
|
|
@@ -108,14 +108,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
_getModelProviderAndClient (baseUrl = '') {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
} else if (baseUrl.includes('deepseek')) {
|
|
116
|
-
return { modelProvider: 'deepseek', client: 'DeepSeek' }
|
|
117
|
-
}
|
|
118
|
-
return { modelProvider: UNKNOWN_MODEL_PROVIDER, client: 'OpenAI' }
|
|
111
|
+
const modelProvider = getOpenAIModelProvider(baseUrl)
|
|
112
|
+
if (modelProvider === 'azure_openai') return { modelProvider, client: 'AzureOpenAI' }
|
|
113
|
+
if (modelProvider === 'deepseek') return { modelProvider, client: 'DeepSeek' }
|
|
114
|
+
return { modelProvider, client: 'OpenAI' }
|
|
119
115
|
}
|
|
120
116
|
|
|
121
117
|
_extractMetrics (response) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
+
const { UNKNOWN_MODEL_PROVIDER } = require('../../constants/tags')
|
|
3
4
|
const { audioMimeTypeFromFormat, formatAudioPart } = require('../../util')
|
|
4
5
|
const {
|
|
5
6
|
INPUT_TYPE_IMAGE,
|
|
@@ -160,10 +161,29 @@ function extractContentParts (parts) {
|
|
|
160
161
|
return { content: extracted.join('\n'), audioParts }
|
|
161
162
|
}
|
|
162
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Maps an OpenAI-compatible base URL to a model provider string. Covers
|
|
166
|
+
* OpenAI, Azure OpenAI, and DeepSeek; falls back to UNKNOWN_MODEL_PROVIDER
|
|
167
|
+
* for unrecognised hosts (e.g. local proxies or custom deployments).
|
|
168
|
+
*
|
|
169
|
+
* Shared with the openai-agents integration since both consume the same
|
|
170
|
+
* client baseURL convention.
|
|
171
|
+
*
|
|
172
|
+
* @param {string} baseUrl
|
|
173
|
+
* @returns {string}
|
|
174
|
+
*/
|
|
175
|
+
function getOpenAIModelProvider (baseUrl = '') {
|
|
176
|
+
if (baseUrl.includes('azure')) return 'azure_openai'
|
|
177
|
+
if (baseUrl.includes('deepseek')) return 'deepseek'
|
|
178
|
+
if (baseUrl.includes('openai')) return 'openai'
|
|
179
|
+
return UNKNOWN_MODEL_PROVIDER
|
|
180
|
+
}
|
|
181
|
+
|
|
163
182
|
module.exports = {
|
|
164
183
|
extractChatTemplateFromInstructions,
|
|
165
184
|
normalizePromptVariables,
|
|
166
185
|
extractTextFromContentItem,
|
|
167
186
|
extractContentParts,
|
|
168
187
|
hasMultimodalInputs,
|
|
188
|
+
getOpenAIModelProvider,
|
|
169
189
|
}
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const { extractContentParts, extractTextFromContentItem } = require('../openai/utils')
|
|
4
|
+
const { safeJsonParse } = require('../../util')
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Flatten Responses and Chat Completions content parts without dropping
|
|
8
|
+
* provider-specific image or audio inputs.
|
|
9
|
+
*
|
|
10
|
+
* @param {Array<object>} parts
|
|
11
|
+
* @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
|
|
12
|
+
*/
|
|
13
|
+
function extractMessageContent (parts) {
|
|
14
|
+
const contentParts = []
|
|
15
|
+
const audioParts = []
|
|
16
|
+
|
|
17
|
+
for (const part of parts) {
|
|
18
|
+
if (!part) continue
|
|
19
|
+
const text = extractTextFromContentItem(part)
|
|
20
|
+
if (text) {
|
|
21
|
+
contentParts.push(text)
|
|
22
|
+
continue
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const extracted = extractContentParts([part])
|
|
26
|
+
if (extracted.content) contentParts.push(extracted.content)
|
|
27
|
+
if (extracted.audioParts.length > 0) audioParts.push(...extracted.audioParts)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return { content: contentParts.join(''), audioParts }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Normalize OpenAI Chat Completions tool calls to the LLMObs message schema.
|
|
35
|
+
*
|
|
36
|
+
* @param {object} message
|
|
37
|
+
* @returns {Array<{ toolId?: string, name?: string, arguments: object, type?: string }>}
|
|
38
|
+
*/
|
|
39
|
+
function extractChatCompletionToolCalls (message) {
|
|
40
|
+
if (message.function_call) {
|
|
41
|
+
return [{
|
|
42
|
+
name: message.function_call.name,
|
|
43
|
+
arguments: safeJsonParse(message.function_call.arguments, {}),
|
|
44
|
+
type: 'function',
|
|
45
|
+
}]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const toolCalls = []
|
|
49
|
+
if (!Array.isArray(message.tool_calls)) return toolCalls
|
|
50
|
+
|
|
51
|
+
for (const toolCall of message.tool_calls) {
|
|
52
|
+
if (!toolCall) continue
|
|
53
|
+
toolCalls.push({
|
|
54
|
+
toolId: toolCall.id,
|
|
55
|
+
name: toolCall.function?.name,
|
|
56
|
+
arguments: safeJsonParse(toolCall.function?.arguments, {}),
|
|
57
|
+
type: toolCall.type,
|
|
58
|
+
})
|
|
59
|
+
}
|
|
60
|
+
return toolCalls
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Normalize a Chat Completions message to the LLMObs message schema.
|
|
65
|
+
*
|
|
66
|
+
* @param {object} message
|
|
67
|
+
* @returns {object}
|
|
68
|
+
*/
|
|
69
|
+
function normalizeChatCompletionMessage (message) {
|
|
70
|
+
const normalized = {
|
|
71
|
+
role: message.role || 'assistant',
|
|
72
|
+
content: message.content ?? '',
|
|
73
|
+
}
|
|
74
|
+
if (message.audioParts?.length > 0) normalized.audioParts = message.audioParts
|
|
75
|
+
const toolCalls = extractChatCompletionToolCalls(message)
|
|
76
|
+
if (toolCalls.length > 0) normalized.toolCalls = toolCalls
|
|
77
|
+
if (message.tool_call_id) normalized.toolId = message.tool_call_id
|
|
78
|
+
return normalized
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Extracts input messages for an LLM span. agents-openai stores only
|
|
83
|
+
* `request.input` on `spanData._input` (string or message-array), and the
|
|
84
|
+
* system instructions are echoed back on the response as `instructions`.
|
|
85
|
+
*
|
|
86
|
+
* @param {string|Array<unknown>} input - The raw `request.input` (`spanData._input`).
|
|
87
|
+
* @param {string} [instructions] - System instructions echoed on `response.instructions`.
|
|
88
|
+
* @returns {Array<{ role: string, content: string }>}
|
|
89
|
+
*/
|
|
90
|
+
function extractInputMessages (input, instructions) {
|
|
91
|
+
const messages = []
|
|
92
|
+
|
|
93
|
+
if (instructions) {
|
|
94
|
+
messages.push({ role: 'system', content: instructions })
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (typeof input === 'string') {
|
|
98
|
+
messages.push({ role: 'user', content: input })
|
|
99
|
+
} else if (Array.isArray(input)) {
|
|
100
|
+
for (const item of input) {
|
|
101
|
+
if (!item) continue
|
|
102
|
+
if (item.type === 'message' || item.role) {
|
|
103
|
+
const role = item.role
|
|
104
|
+
if (!role) continue
|
|
105
|
+
|
|
106
|
+
let content = ''
|
|
107
|
+
let audioParts
|
|
108
|
+
if (Array.isArray(item.content)) {
|
|
109
|
+
const extracted = extractMessageContent(item.content)
|
|
110
|
+
content = extracted.content
|
|
111
|
+
audioParts = extracted.audioParts
|
|
112
|
+
} else if (typeof item.content === 'string') {
|
|
113
|
+
content = item.content
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const message = normalizeChatCompletionMessage({ ...item, content, role, audioParts })
|
|
117
|
+
if (content || message.audioParts || message.toolCalls || message.toolId) {
|
|
118
|
+
messages.push(message)
|
|
119
|
+
}
|
|
120
|
+
} else if (item.type === 'function_call') {
|
|
121
|
+
let args = item.arguments
|
|
122
|
+
if (typeof args === 'string') {
|
|
123
|
+
try {
|
|
124
|
+
args = JSON.parse(args)
|
|
125
|
+
} catch {
|
|
126
|
+
args = {}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
messages.push({
|
|
130
|
+
role: 'assistant',
|
|
131
|
+
toolCalls: [{
|
|
132
|
+
toolId: item.call_id,
|
|
133
|
+
name: item.name,
|
|
134
|
+
arguments: args,
|
|
135
|
+
type: item.type,
|
|
136
|
+
}],
|
|
137
|
+
})
|
|
138
|
+
} else if (item.type === 'function_call_output') {
|
|
139
|
+
messages.push({
|
|
140
|
+
role: 'user',
|
|
141
|
+
toolResults: [{
|
|
142
|
+
toolId: item.call_id,
|
|
143
|
+
result: item.output,
|
|
144
|
+
name: item.name || '',
|
|
145
|
+
type: item.type,
|
|
146
|
+
}],
|
|
147
|
+
})
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return messages.length > 0 ? messages : [{ role: 'user', content: '' }]
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Extracts output messages from the model response.
|
|
157
|
+
*
|
|
158
|
+
* @param {{ output?: Array<unknown> }} result - The model response
|
|
159
|
+
* @returns {Array<{ role: string, content: string }>}
|
|
160
|
+
*/
|
|
161
|
+
function extractOutputMessages (result) {
|
|
162
|
+
const messages = []
|
|
163
|
+
|
|
164
|
+
if (result?.output) {
|
|
165
|
+
for (const item of result.output) {
|
|
166
|
+
if (!item) continue
|
|
167
|
+
if (item.type === 'message') {
|
|
168
|
+
let content = ''
|
|
169
|
+
if (Array.isArray(item.content)) {
|
|
170
|
+
const textParts = []
|
|
171
|
+
for (const contentItem of item.content) {
|
|
172
|
+
if (contentItem?.type === 'output_text' && contentItem.text) {
|
|
173
|
+
textParts.push(contentItem.text)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
content = textParts.join('')
|
|
177
|
+
} else if (typeof item.content === 'string') {
|
|
178
|
+
content = item.content
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
messages.push({ role: item.role || 'assistant', content })
|
|
182
|
+
} else if (item.type === 'function_call') {
|
|
183
|
+
let args = item.arguments
|
|
184
|
+
if (typeof args === 'string') {
|
|
185
|
+
try {
|
|
186
|
+
args = JSON.parse(args)
|
|
187
|
+
} catch {
|
|
188
|
+
args = {}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
messages.push({
|
|
192
|
+
role: 'assistant',
|
|
193
|
+
toolCalls: [{
|
|
194
|
+
toolId: item.call_id,
|
|
195
|
+
name: item.name,
|
|
196
|
+
arguments: args,
|
|
197
|
+
type: item.type,
|
|
198
|
+
}],
|
|
199
|
+
})
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return messages.length > 0 ? messages : [{ content: '', role: '' }]
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Extracts output messages from an OpenAI Chat Completions response.
|
|
209
|
+
*
|
|
210
|
+
* @param {Array<{ choices?: Array<{ message?: object }> }>} result - The model responses
|
|
211
|
+
* @returns {Array<object>}
|
|
212
|
+
*/
|
|
213
|
+
function extractGenerationOutputMessages (result) {
|
|
214
|
+
const messages = []
|
|
215
|
+
|
|
216
|
+
if (Array.isArray(result)) {
|
|
217
|
+
for (const response of result) {
|
|
218
|
+
if (!Array.isArray(response?.choices)) continue
|
|
219
|
+
for (const choice of response.choices) {
|
|
220
|
+
const message = choice?.message
|
|
221
|
+
if (!message) continue
|
|
222
|
+
messages.push(normalizeChatCompletionMessage(message))
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return messages.length > 0 ? messages : [{ content: '', role: '' }]
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Extracts token usage metrics from the model response. Returns `undefined`
|
|
232
|
+
* when there's nothing to tag, so callers can skip the tagger call without
|
|
233
|
+
* allocating an Object.keys array.
|
|
234
|
+
*
|
|
235
|
+
* @param {{ usage?: { inputTokens?: number, outputTokens?: number, totalTokens?: number,
|
|
236
|
+
* outputTokensDetails?: { reasoningTokens?: number },
|
|
237
|
+
* completion_tokens_details?: { reasoning_tokens?: number } } }} result
|
|
238
|
+
* @returns {{ inputTokens?: number, outputTokens?: number, totalTokens?: number,
|
|
239
|
+
* reasoningOutputTokens?: number } | undefined}
|
|
240
|
+
*/
|
|
241
|
+
function extractMetrics (result) {
|
|
242
|
+
const usage = result?.usage
|
|
243
|
+
if (!usage) return
|
|
244
|
+
|
|
245
|
+
const inputTokens = usage.inputTokens ?? usage.input_tokens ?? usage.prompt_tokens
|
|
246
|
+
const outputTokens = usage.outputTokens ?? usage.output_tokens ?? usage.completion_tokens
|
|
247
|
+
const totalTokens = usage.totalTokens ?? usage.total_tokens
|
|
248
|
+
const reasoningTokens = usage.outputTokensDetails?.reasoningTokens ??
|
|
249
|
+
usage.output_tokens_details?.reasoning_tokens ??
|
|
250
|
+
usage.completion_tokens_details?.reasoning_tokens
|
|
251
|
+
|
|
252
|
+
if (inputTokens === undefined && outputTokens === undefined &&
|
|
253
|
+
totalTokens === undefined && !reasoningTokens) return
|
|
254
|
+
|
|
255
|
+
const metrics = {}
|
|
256
|
+
if (inputTokens !== undefined) metrics.inputTokens = inputTokens
|
|
257
|
+
if (outputTokens !== undefined) metrics.outputTokens = outputTokens
|
|
258
|
+
// Tagger maps `reasoningOutputTokens` → `reasoning_output_tokens` in the
|
|
259
|
+
// LLMObs span event. Skip when zero — emitting a zero just adds noise.
|
|
260
|
+
if (reasoningTokens) metrics.reasoningOutputTokens = reasoningTokens
|
|
261
|
+
|
|
262
|
+
if (totalTokens !== undefined) {
|
|
263
|
+
metrics.totalTokens = totalTokens
|
|
264
|
+
} else if (metrics.inputTokens !== undefined && metrics.outputTokens !== undefined) {
|
|
265
|
+
metrics.totalTokens = metrics.inputTokens + metrics.outputTokens
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
return metrics
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Fields the OpenAI Responses API echoes back from the request configuration.
|
|
272
|
+
// agents-openai only stores `request.input` on the span — the user's
|
|
273
|
+
// `modelSettings` aren't directly observable, so we read the response-echoed
|
|
274
|
+
// values. Matches dd-trace-py's openai-agents integration (see
|
|
275
|
+
// `OaiSpanAdapter.llmobs_metadata`); both ship without filtering OpenAI's
|
|
276
|
+
// default values.
|
|
277
|
+
const RESPONSE_METADATA_FIELDS = [
|
|
278
|
+
'temperature',
|
|
279
|
+
'max_output_tokens',
|
|
280
|
+
'top_p',
|
|
281
|
+
'tools',
|
|
282
|
+
'tool_choice',
|
|
283
|
+
'truncation',
|
|
284
|
+
]
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Extracts metadata from the model response. Mirrors Python's
|
|
288
|
+
* `OaiSpanAdapter.llmobs_metadata` — emits all response-echoed configuration
|
|
289
|
+
* fields plus `text` when present. Returns `undefined` when nothing was
|
|
290
|
+
* captured, so callers can skip the tagger call without allocating.
|
|
291
|
+
*
|
|
292
|
+
* @param {object | undefined} response
|
|
293
|
+
* @returns {object | undefined}
|
|
294
|
+
*/
|
|
295
|
+
function extractMetadata (response) {
|
|
296
|
+
if (!response) return
|
|
297
|
+
|
|
298
|
+
let metadata
|
|
299
|
+
for (const field of RESPONSE_METADATA_FIELDS) {
|
|
300
|
+
const value = response[field]
|
|
301
|
+
if (value !== undefined && value !== null) {
|
|
302
|
+
metadata ??= {}
|
|
303
|
+
metadata[field] = value
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
if (response.text) {
|
|
308
|
+
metadata ??= {}
|
|
309
|
+
metadata.text = response.text
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return metadata
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
module.exports = {
|
|
316
|
+
extractInputMessages,
|
|
317
|
+
extractOutputMessages,
|
|
318
|
+
extractGenerationOutputMessages,
|
|
319
|
+
extractMetrics,
|
|
320
|
+
extractMetadata,
|
|
321
|
+
}
|
|
@@ -11,6 +11,7 @@ const {
|
|
|
11
11
|
SPAN_KIND,
|
|
12
12
|
OUTPUT_VALUE,
|
|
13
13
|
INPUT_VALUE,
|
|
14
|
+
TRACE_ID,
|
|
14
15
|
} = require('./constants/tags')
|
|
15
16
|
const {
|
|
16
17
|
getFunctionArguments,
|
|
@@ -345,7 +346,7 @@ class LLMObs extends NoopLLMObs {
|
|
|
345
346
|
}
|
|
346
347
|
try {
|
|
347
348
|
return {
|
|
348
|
-
traceId:
|
|
349
|
+
traceId: LLMObsTagger.tagMap.get(span)[TRACE_ID],
|
|
349
350
|
spanId: span.context().toSpanId(),
|
|
350
351
|
}
|
|
351
352
|
} catch {
|
|
@@ -35,6 +35,7 @@ const {
|
|
|
35
35
|
LLMOBS_SUBMITTED_TAG_KEY,
|
|
36
36
|
SAMPLE_RATE,
|
|
37
37
|
SAMPLING_DECISION,
|
|
38
|
+
TRACE_ID,
|
|
38
39
|
} = require('./constants/tags')
|
|
39
40
|
const { UNSERIALIZABLE_VALUE_TEXT } = require('./constants/text')
|
|
40
41
|
const telemetry = require('./telemetry')
|
|
@@ -236,8 +237,11 @@ class LLMObsSpanProcessor {
|
|
|
236
237
|
meta.input.prompt = prompt
|
|
237
238
|
}
|
|
238
239
|
|
|
240
|
+
const apmTraceId = span.context().toTraceId(true)
|
|
241
|
+
const llmobsTraceId = mlObsTags[TRACE_ID] ?? apmTraceId
|
|
242
|
+
|
|
239
243
|
const llmObsSpanEvent = {
|
|
240
|
-
trace_id:
|
|
244
|
+
trace_id: llmobsTraceId,
|
|
241
245
|
span_id: span.context().toSpanId(),
|
|
242
246
|
parent_id: parentId,
|
|
243
247
|
name,
|
|
@@ -249,9 +253,10 @@ class LLMObsSpanProcessor {
|
|
|
249
253
|
metrics,
|
|
250
254
|
_dd: {
|
|
251
255
|
span_id: span.context().toSpanId(),
|
|
252
|
-
trace_id:
|
|
256
|
+
trace_id: apmTraceId,
|
|
253
257
|
sample_rate: mlObsTags[SAMPLE_RATE],
|
|
254
258
|
sampling_decision: mlObsTags[SAMPLING_DECISION],
|
|
259
|
+
apm_trace_id: apmTraceId,
|
|
255
260
|
},
|
|
256
261
|
}
|
|
257
262
|
|
|
@@ -51,9 +51,18 @@ const {
|
|
|
51
51
|
SAMPLING_DECISION_DROPPED,
|
|
52
52
|
PROPAGATED_SAMPLE_RATE_KEY,
|
|
53
53
|
PROPAGATED_SAMPLING_DECISION_KEY,
|
|
54
|
+
TRACE_ID,
|
|
55
|
+
PROPAGATED_TRACE_ID_KEY,
|
|
54
56
|
} = require('./constants/tags')
|
|
55
57
|
const { storage } = require('./storage')
|
|
56
|
-
const {
|
|
58
|
+
const {
|
|
59
|
+
findGenAIAncestorSpanId,
|
|
60
|
+
validateCostTags,
|
|
61
|
+
writeBridgeTags,
|
|
62
|
+
validateToolDefinitions,
|
|
63
|
+
generateLlmObsTraceId,
|
|
64
|
+
normalizeLlmObsTraceId,
|
|
65
|
+
} = require('./util')
|
|
57
66
|
|
|
58
67
|
// global registry of LLMObs spans
|
|
59
68
|
// maps LLMObs spans to their annotations
|
|
@@ -126,12 +135,20 @@ class LLMObsTagger {
|
|
|
126
135
|
|
|
127
136
|
this._register(span)
|
|
128
137
|
|
|
138
|
+
const traceTags = span.context()._trace.tags
|
|
139
|
+
|
|
140
|
+
const llmobsTraceId =
|
|
141
|
+
registry.get(parent)?.[TRACE_ID] ??
|
|
142
|
+
normalizeLlmObsTraceId(traceTags[PROPAGATED_TRACE_ID_KEY]) ??
|
|
143
|
+
generateLlmObsTraceId(span._startTime)
|
|
144
|
+
this._setTag(span, TRACE_ID, llmobsTraceId)
|
|
145
|
+
|
|
129
146
|
// When the registering span sits below an OTel `gen_ai.*` ancestor, use
|
|
130
147
|
// that ancestor as the parent_id fallback and suppress the bridge
|
|
131
148
|
// parent_id tag so the indexer doesn't invert the trace.
|
|
132
149
|
const genAIAncestorSpanId = findGenAIAncestorSpanId(span)
|
|
133
150
|
|
|
134
|
-
writeBridgeTags(span, { includeParentId: genAIAncestorSpanId === null })
|
|
151
|
+
writeBridgeTags(span, { includeParentId: genAIAncestorSpanId === null, llmobsTraceId })
|
|
135
152
|
|
|
136
153
|
this._setTag(span, ML_APP, spanMlApp)
|
|
137
154
|
|
|
@@ -141,7 +158,6 @@ class LLMObsTagger {
|
|
|
141
158
|
if (modelName) this.tagModelName(span, modelName)
|
|
142
159
|
if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider)
|
|
143
160
|
|
|
144
|
-
const traceTags = span.context()._trace.tags
|
|
145
161
|
sessionId = sessionId ||
|
|
146
162
|
registry.get(parent)?.[SESSION_ID] ||
|
|
147
163
|
traceTags[SESSION_ID_TRACE_DEFAULT_KEY] ||
|
|
@@ -500,6 +516,10 @@ class LLMObsTagger {
|
|
|
500
516
|
this._setTag(span, MODEL_NAME, modelName)
|
|
501
517
|
}
|
|
502
518
|
|
|
519
|
+
setName (span, name) {
|
|
520
|
+
this._setTag(span, NAME, name)
|
|
521
|
+
}
|
|
522
|
+
|
|
503
523
|
#tagText (span, data, key) {
|
|
504
524
|
if (data) {
|
|
505
525
|
if (typeof data === 'string') {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
+
const id = require('../id')
|
|
3
4
|
const log = require('../log')
|
|
4
5
|
const {
|
|
5
6
|
LLMOBS_PARENT_ID_BRIDGE_KEY,
|
|
@@ -7,6 +8,10 @@ const {
|
|
|
7
8
|
SPAN_KINDS,
|
|
8
9
|
} = require('./constants/tags')
|
|
9
10
|
|
|
11
|
+
const DECIMAL_TRACE_ID_REGEX = /^\d+$/
|
|
12
|
+
const HEX_TRACE_ID_REGEX = /^[0-9a-f]{32}$/i
|
|
13
|
+
const MAX_UINT_64 = (1n << 64n) - 1n
|
|
14
|
+
|
|
10
15
|
// LLM I/O is overwhelmingly ASCII (English prompts and code). Walk once
|
|
11
16
|
// looking for the first non-ASCII char; if there is none, hand the input
|
|
12
17
|
// straight back. Otherwise pick up the slow path from the byte that needed
|
|
@@ -312,12 +317,12 @@ function safeJsonParse (value, fallback) {
|
|
|
312
317
|
// LLMObs root and hoists the gen_ai ancestors under it, inverting the trace.
|
|
313
318
|
/**
|
|
314
319
|
* @param {import('../opentracing/span')} span
|
|
315
|
-
* @param {{ includeParentId?: boolean }} [opts]
|
|
320
|
+
* @param {{ includeParentId?: boolean, llmobsTraceId?: string }} [opts]
|
|
316
321
|
*/
|
|
317
|
-
function writeBridgeTags (span, { includeParentId = true } = {}) {
|
|
322
|
+
function writeBridgeTags (span, { includeParentId = true, llmobsTraceId } = {}) {
|
|
318
323
|
const traceTags = span?.context?.()._trace?.tags
|
|
319
324
|
if (!traceTags || traceTags[LLMOBS_TRACE_ID_BRIDGE_KEY]) return
|
|
320
|
-
traceTags[LLMOBS_TRACE_ID_BRIDGE_KEY] = span.context().toTraceId(true)
|
|
325
|
+
traceTags[LLMOBS_TRACE_ID_BRIDGE_KEY] = llmobsTraceId ?? span.context().toTraceId(true)
|
|
321
326
|
if (includeParentId) {
|
|
322
327
|
traceTags[LLMOBS_PARENT_ID_BRIDGE_KEY] = span.context().toSpanId()
|
|
323
328
|
}
|
|
@@ -362,6 +367,51 @@ function findGenAIAncestorSpanId (span) {
|
|
|
362
367
|
return null
|
|
363
368
|
}
|
|
364
369
|
|
|
370
|
+
/**
|
|
371
|
+
* Generate a 128-bit LLMObs trace ID with the span start time encoded in its high bits.
|
|
372
|
+
* @param {number} startTime
|
|
373
|
+
* @returns {string}
|
|
374
|
+
*/
|
|
375
|
+
function generateLlmObsTraceId (startTime) {
|
|
376
|
+
const identifier = id()
|
|
377
|
+
const traceIdHigh = Math.floor(startTime / 1000)
|
|
378
|
+
.toString(16)
|
|
379
|
+
.padStart(8, '0')
|
|
380
|
+
.padEnd(16, '0')
|
|
381
|
+
|
|
382
|
+
return identifier.toTraceIdHex(traceIdHigh).padStart(32, '0')
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Convert an internally stored hexadecimal LLMObs trace ID to its distributed wire representation.
|
|
387
|
+
* @param {string | undefined} traceId
|
|
388
|
+
* @returns {string | undefined}
|
|
389
|
+
*/
|
|
390
|
+
function llmObsTraceIdToWire (traceId) {
|
|
391
|
+
if (!traceId) return
|
|
392
|
+
if (!HEX_TRACE_ID_REGEX.test(traceId)) return traceId
|
|
393
|
+
|
|
394
|
+
return BigInt(`0x${traceId}`).toString(10)
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Normalize a distributed LLMObs trace ID to the representation expected by LLMObs span events.
|
|
399
|
+
* @param {string | undefined} traceId
|
|
400
|
+
* @returns {string | undefined}
|
|
401
|
+
*/
|
|
402
|
+
function normalizeLlmObsTraceId (traceId) {
|
|
403
|
+
if (!traceId) return
|
|
404
|
+
|
|
405
|
+
if (HEX_TRACE_ID_REGEX.test(traceId) && (traceId[0] === '0' || !DECIMAL_TRACE_ID_REGEX.test(traceId))) {
|
|
406
|
+
return traceId
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
if (!DECIMAL_TRACE_ID_REGEX.test(traceId)) return traceId
|
|
410
|
+
|
|
411
|
+
const identifier = BigInt(traceId)
|
|
412
|
+
return identifier > MAX_UINT_64 ? identifier.toString(16).padStart(32, '0') : traceId
|
|
413
|
+
}
|
|
414
|
+
|
|
365
415
|
// Maps an audio `format` (e.g. "wav", "mp3") to a MIME type. Defaults to `audio/wav` when the
|
|
366
416
|
// format is missing. Provider-specific overrides (e.g. OpenAI's mp3 -> audio/mpeg) are passed in
|
|
367
417
|
// via `mimeTypeLookup` so this stays provider-agnostic. A non-string `format` is treated as missing
|
|
@@ -396,6 +446,9 @@ module.exports = {
|
|
|
396
446
|
audioMimeTypeFromFormat,
|
|
397
447
|
encodeUnicode,
|
|
398
448
|
findGenAIAncestorSpanId,
|
|
449
|
+
generateLlmObsTraceId,
|
|
450
|
+
llmObsTraceIdToWire,
|
|
451
|
+
normalizeLlmObsTraceId,
|
|
399
452
|
formatAudioPart,
|
|
400
453
|
validateCostTags,
|
|
401
454
|
validateKind,
|
|
@@ -20,7 +20,7 @@ const RETRY_JITTER = 0.2
|
|
|
20
20
|
* @property {URL} endpoint
|
|
21
21
|
* @property {number} pollIntervalMs
|
|
22
22
|
* @property {number} requestTimeoutMs
|
|
23
|
-
* @property {string} apiKey
|
|
23
|
+
* @property {string | undefined} apiKey
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
26
|
/**
|
|
@@ -138,7 +138,7 @@ class AgentlessConfigurationSource {
|
|
|
138
138
|
#request (signal) {
|
|
139
139
|
const headers = getClientLibraryHeaders()
|
|
140
140
|
headers['Accept-Encoding'] = 'gzip'
|
|
141
|
-
headers['DD-API-KEY'] = this.#config.apiKey
|
|
141
|
+
if (this.#config.apiKey) headers['DD-API-KEY'] = this.#config.apiKey
|
|
142
142
|
if (this.#etag) headers['If-None-Match'] = this.#etag
|
|
143
143
|
|
|
144
144
|
/**
|
|
@@ -228,10 +228,7 @@ class AgentlessConfigurationSource {
|
|
|
228
228
|
this.#failureWarnings.add(category)
|
|
229
229
|
|
|
230
230
|
if (statusCode === 401 || statusCode === 403) {
|
|
231
|
-
log.warn(
|
|
232
|
-
'Feature Flagging agentless endpoint returned HTTP %d; verify DD_API_KEY is configured and valid',
|
|
233
|
-
statusCode
|
|
234
|
-
)
|
|
231
|
+
log.warn('Feature Flagging agentless endpoint returned HTTP %d; verify endpoint authentication', statusCode)
|
|
235
232
|
} else if (statusCode) {
|
|
236
233
|
log.warn('Feature Flagging agentless endpoint returned HTTP %d after %d attempts', statusCode, attempts)
|
|
237
234
|
} else if (attempts > 1) {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
-
const { isLoopbackHost } = require('../exporters/common/url')
|
|
4
3
|
const log = require('../log')
|
|
5
4
|
|
|
6
5
|
const DEFAULT_AGENTLESS_PATH = '/api/v2/feature-flagging/config/rules-based/server'
|
|
@@ -27,9 +26,11 @@ function create (config, applyConfiguration) {
|
|
|
27
26
|
return
|
|
28
27
|
}
|
|
29
28
|
|
|
29
|
+
const hasCustomEndpoint = Boolean(baseUrl?.trim())
|
|
30
|
+
|
|
30
31
|
try {
|
|
31
|
-
if (!config.DD_API_KEY) {
|
|
32
|
-
throw new Error('DD_API_KEY is required for Feature Flagging
|
|
32
|
+
if (!hasCustomEndpoint && !config.DD_API_KEY) {
|
|
33
|
+
throw new Error('DD_API_KEY is required for the default Datadog Feature Flagging endpoint')
|
|
33
34
|
}
|
|
34
35
|
|
|
35
36
|
const AgentlessConfigurationSource = require('./agentless_configuration_source')
|
|
@@ -37,7 +38,7 @@ function create (config, applyConfiguration) {
|
|
|
37
38
|
endpoint: endpoint(config, baseUrl),
|
|
38
39
|
pollIntervalMs: Math.min(pollIntervalSeconds, MAX_POLL_INTERVAL_SECONDS) * 1000,
|
|
39
40
|
requestTimeoutMs: requestTimeoutSeconds * 1000,
|
|
40
|
-
apiKey: config.DD_API_KEY,
|
|
41
|
+
apiKey: hasCustomEndpoint ? undefined : config.DD_API_KEY,
|
|
41
42
|
}, applyConfiguration)
|
|
42
43
|
} catch (error) {
|
|
43
44
|
log.error('Unable to configure Feature Flagging configuration source', error)
|
|
@@ -74,10 +75,6 @@ function endpoint (config, configuredBaseUrl) {
|
|
|
74
75
|
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
|
75
76
|
throw new Error('Feature Flagging agentless URL must use HTTP or HTTPS')
|
|
76
77
|
}
|
|
77
|
-
if (url.protocol === 'http:' && !isLoopbackHost(url.hostname)) {
|
|
78
|
-
throw new Error('Feature Flagging agentless URL must use HTTPS unless it targets loopback')
|
|
79
|
-
}
|
|
80
|
-
|
|
81
78
|
if (url.pathname === '' || url.pathname === '/') {
|
|
82
79
|
url.pathname = DEFAULT_AGENTLESS_PATH
|
|
83
80
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const { LogCollapsingLowestDenseDDSketch } = require('../../../../../vendor/dist/@datadog/sketches-js')
|
|
4
4
|
const OtlpTransformerBase = require('../otlp/otlp_transformer_base')
|
|
5
5
|
const { getProtobufTypes } = require('../otlp/protobuf_loader')
|
|
6
|
+
const { GRPC_STATUS_NAMES } = require('../../constants')
|
|
6
7
|
|
|
7
8
|
const NS_PER_S = 1e9
|
|
8
9
|
|
|
@@ -152,7 +153,10 @@ class OtlpStatsTransformer extends OtlpTransformerBase {
|
|
|
152
153
|
if (aggKey.method) raw['http.request.method'] = aggKey.method
|
|
153
154
|
if (aggKey.endpoint) raw['http.route'] = aggKey.endpoint
|
|
154
155
|
if (aggKey.rpcStatusCode !== '') {
|
|
155
|
-
|
|
156
|
+
const n = Number(aggKey.rpcStatusCode)
|
|
157
|
+
raw['rpc.response.status_code'] = Number.isInteger(n) && n >= 0 && n < GRPC_STATUS_NAMES.length
|
|
158
|
+
? GRPC_STATUS_NAMES[n]
|
|
159
|
+
: String(aggKey.rpcStatusCode).toUpperCase()
|
|
156
160
|
}
|
|
157
161
|
|
|
158
162
|
if (!this.#otelSemanticsEnabled) {
|