dd-trace 5.116.0 → 5.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ci/diagnose.js +162 -21
- package/ci/init.js +0 -1
- package/ci/runbook.md +187 -180
- package/ci/test-optimization-validation/approval-artifacts.js +3 -1
- package/ci/test-optimization-validation/approval.js +52 -15
- package/ci/test-optimization-validation/ci-command-candidate.js +26 -32
- package/ci/test-optimization-validation/ci-discovery.js +1 -1
- package/ci/test-optimization-validation/ci-package-scripts.js +141 -0
- package/ci/test-optimization-validation/ci-remediation.js +112 -39
- package/ci/test-optimization-validation/cli.js +516 -681
- package/ci/test-optimization-validation/command-blocker.js +188 -33
- package/ci/test-optimization-validation/command-output-policy.js +3 -26
- package/ci/test-optimization-validation/command-runner.js +189 -245
- package/ci/test-optimization-validation/environment.js +90 -0
- package/ci/test-optimization-validation/executable.js +159 -388
- package/ci/test-optimization-validation/framework-adapters/cucumber.js +119 -0
- package/ci/test-optimization-validation/framework-adapters/cypress.js +107 -0
- package/ci/test-optimization-validation/framework-adapters/playwright.js +181 -0
- package/ci/test-optimization-validation/generated-files.js +75 -13
- package/ci/test-optimization-validation/generated-test-contract.js +283 -0
- package/ci/test-optimization-validation/generated-verifier.js +49 -16
- package/ci/test-optimization-validation/literal-environment.js +57 -0
- package/ci/test-optimization-validation/manifest-loader.js +2 -22
- package/ci/test-optimization-validation/manifest-scaffold.js +816 -515
- package/ci/test-optimization-validation/manifest-schema.js +494 -702
- package/ci/test-optimization-validation/offline-fixtures.js +4 -1
- package/ci/test-optimization-validation/plan-writer.js +243 -984
- package/ci/test-optimization-validation/preflight-runner.js +49 -55
- package/ci/test-optimization-validation/redaction.js +2 -1
- package/ci/test-optimization-validation/report-writer.js +391 -1357
- package/ci/test-optimization-validation/result-semantics.js +86 -0
- package/ci/test-optimization-validation/runner-command.js +249 -0
- package/ci/test-optimization-validation/runner-contract.js +664 -0
- package/ci/test-optimization-validation/scenarios/auto-test-retries.js +29 -3
- package/ci/test-optimization-validation/scenarios/basic-reporting.js +240 -543
- package/ci/test-optimization-validation/scenarios/ci-wiring.js +450 -670
- package/ci/test-optimization-validation/scenarios/early-flake-detection.js +4 -3
- package/ci/test-optimization-validation/scenarios/helpers.js +67 -9
- package/ci/test-optimization-validation/scenarios/test-management.js +4 -3
- package/ci/test-optimization-validation/source-text.js +87 -0
- package/ci/test-optimization-validation/test-output.js +12 -22
- package/ext/tags.d.ts +1 -0
- package/index.d.ts +7 -0
- package/package.json +3 -3
- package/packages/datadog-instrumentations/src/cucumber.js +11 -12
- package/packages/datadog-instrumentations/src/helpers/channel.js +74 -0
- package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +1 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/openai-agents.js +31 -0
- package/packages/datadog-instrumentations/src/jest.js +18 -17
- package/packages/datadog-instrumentations/src/mocha/main.js +79 -30
- package/packages/datadog-instrumentations/src/openai-agents.js +159 -0
- package/packages/datadog-instrumentations/src/playwright.js +3 -24
- package/packages/datadog-instrumentations/src/vitest-main-no-worker-init.js +1 -1
- package/packages/datadog-instrumentations/src/vitest-main.js +14 -26
- package/packages/datadog-instrumentations/src/vitest-util.js +0 -7
- package/packages/datadog-instrumentations/src/vitest-worker.js +5 -8
- package/packages/datadog-plugin-cucumber/src/index.js +2 -1
- package/packages/datadog-plugin-mocha/src/index.js +2 -1
- package/packages/datadog-plugin-openai-agents/src/index.js +74 -0
- package/packages/datadog-plugin-openai-agents/src/integration.js +503 -0
- package/packages/datadog-plugin-openai-agents/src/processor.js +108 -0
- package/packages/datadog-plugin-openai-agents/src/util.js +60 -0
- package/packages/datadog-plugin-playwright/src/index.js +4 -1
- package/packages/datadog-plugin-vitest/src/index.js +5 -5
- package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +48 -49
- package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +161 -33
- package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js +6 -4
- package/packages/dd-trace/src/config/generated-config-types.d.ts +2 -0
- package/packages/dd-trace/src/config/supported-configurations.json +7 -0
- package/packages/dd-trace/src/encode/span-stats.js +7 -1
- package/packages/dd-trace/src/exporter.js +16 -9
- package/packages/dd-trace/src/llmobs/index.js +9 -1
- package/packages/dd-trace/src/llmobs/plugins/ai/util.js +104 -22
- package/packages/dd-trace/src/llmobs/plugins/openai/index.js +5 -9
- package/packages/dd-trace/src/llmobs/plugins/openai/utils.js +20 -0
- package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js +321 -0
- package/packages/dd-trace/src/llmobs/sdk.js +2 -1
- package/packages/dd-trace/src/llmobs/span_processor.js +7 -2
- package/packages/dd-trace/src/llmobs/tagger.js +23 -3
- package/packages/dd-trace/src/llmobs/util.js +56 -3
- package/packages/dd-trace/src/openfeature/agentless_configuration_source.js +3 -6
- package/packages/dd-trace/src/openfeature/configuration_source.js +5 -8
- package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_transformer.js +5 -1
- package/packages/dd-trace/src/plugins/ci_plugin.js +9 -3
- package/packages/dd-trace/src/plugins/index.js +1 -0
- package/packages/dd-trace/src/plugins/outbound.js +4 -1
- package/packages/dd-trace/src/profiling/profilers/space.js +1 -1
- package/packages/dd-trace/src/profiling/profilers/wall.js +1 -0
- package/packages/dd-trace/src/span_processor.js +1 -1
- package/packages/dd-trace/src/span_stats.js +22 -4
- package/ci/test-optimization-validation/command-suitability.js +0 -471
- package/ci/test-optimization-validation/init-probe-preload.js +0 -163
- package/ci/test-optimization-validation/init-probe.js +0 -246
- package/ci/test-optimization-validation/late-initialization.js +0 -63
- package/ci/test-optimization-validation/local-command.js +0 -163
- package/ci/test-optimization-validation/setup-runner.js +0 -97
- package/ci/test-optimization-validation-manifest.schema.json +0 -1
|
@@ -4,6 +4,7 @@ const fs = require('fs')
|
|
|
4
4
|
const exporters = require('../../../ext/exporters')
|
|
5
5
|
const { getEnvironmentVariable } = require('../../dd-trace/src/config/helper')
|
|
6
6
|
const constants = require('./constants')
|
|
7
|
+
const { isTrue } = require('./util')
|
|
7
8
|
|
|
8
9
|
module.exports = function getExporter (name) {
|
|
9
10
|
switch (name) {
|
|
@@ -20,20 +21,26 @@ module.exports = function getExporter (name) {
|
|
|
20
21
|
case exporters.AGENT_PROXY:
|
|
21
22
|
return require('./ci-visibility/exporters/agent-proxy')
|
|
22
23
|
case exporters.CI_VALIDATION:
|
|
23
|
-
return require('./ci-visibility/exporters/ci-validation')
|
|
24
|
+
if (hasCiValidationEnvironment()) return require('./ci-visibility/exporters/ci-validation')
|
|
25
|
+
break
|
|
24
26
|
case exporters.JEST_WORKER:
|
|
25
27
|
case exporters.CUCUMBER_WORKER:
|
|
26
28
|
case exporters.MOCHA_WORKER:
|
|
27
29
|
case exporters.PLAYWRIGHT_WORKER:
|
|
28
30
|
case exporters.VITEST_WORKER:
|
|
29
31
|
return require('./ci-visibility/exporters/test-worker')
|
|
30
|
-
default: {
|
|
31
|
-
const inAWSLambda = getEnvironmentVariable('AWS_LAMBDA_FUNCTION_NAME') !== undefined
|
|
32
|
-
const usingAgent = inAWSLambda && (
|
|
33
|
-
fs.existsSync(constants.DATADOG_LAMBDA_EXTENSION_PATH) ||
|
|
34
|
-
fs.existsSync(constants.DATADOG_MINI_AGENT_PATH)
|
|
35
|
-
)
|
|
36
|
-
return inAWSLambda && !usingAgent ? require('./exporters/log') : require('./exporters/agent')
|
|
37
|
-
}
|
|
38
32
|
}
|
|
33
|
+
|
|
34
|
+
const inAWSLambda = getEnvironmentVariable('AWS_LAMBDA_FUNCTION_NAME') !== undefined
|
|
35
|
+
const usingAgent = inAWSLambda && (
|
|
36
|
+
fs.existsSync(constants.DATADOG_LAMBDA_EXTENSION_PATH) ||
|
|
37
|
+
fs.existsSync(constants.DATADOG_MINI_AGENT_PATH)
|
|
38
|
+
)
|
|
39
|
+
return inAWSLambda && !usingAgent ? require('./exporters/log') : require('./exporters/agent')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function hasCiValidationEnvironment () {
|
|
43
|
+
return isTrue(getEnvironmentVariable('_DD_TEST_OPTIMIZATION_VALIDATION_MODE')) &&
|
|
44
|
+
getEnvironmentVariable('_DD_TEST_OPTIMIZATION_VALIDATION_MANIFEST_FILE') &&
|
|
45
|
+
getEnvironmentVariable('_DD_TEST_OPTIMIZATION_VALIDATION_OUTPUT_DIR')
|
|
39
46
|
}
|
|
@@ -16,6 +16,8 @@ const {
|
|
|
16
16
|
SAMPLING_DECISION,
|
|
17
17
|
PROPAGATED_SAMPLE_RATE_KEY,
|
|
18
18
|
PROPAGATED_SAMPLING_DECISION_KEY,
|
|
19
|
+
TRACE_ID,
|
|
20
|
+
PROPAGATED_TRACE_ID_KEY,
|
|
19
21
|
} = require('./constants/tags')
|
|
20
22
|
const { storage } = require('./storage')
|
|
21
23
|
const telemetry = require('./telemetry')
|
|
@@ -25,6 +27,7 @@ const LLMObsTagger = require('./tagger')
|
|
|
25
27
|
const LLMObsSpanWriter = require('./writers/spans')
|
|
26
28
|
const { setAgentStrategy } = require('./writers/util')
|
|
27
29
|
const { INCOMPATIBLE_INITIALIZATION } = require('./constants/text')
|
|
30
|
+
const { llmObsTraceIdToWire } = require('./util')
|
|
28
31
|
|
|
29
32
|
const spanFinishCh = channel('dd-trace:span:finish')
|
|
30
33
|
const evalMetricAppendCh = channel('llmobs:eval-metric:append')
|
|
@@ -137,8 +140,12 @@ function handleLLMObsInjection ({ carrier }) {
|
|
|
137
140
|
mlObsSpanTags?.[SESSION_ID] ??
|
|
138
141
|
parentContext?._trace?.tags?.[SESSION_ID_TRACE_DEFAULT_KEY] ??
|
|
139
142
|
parentContext?._trace?.tags?.[PROPAGATED_SESSION_ID_KEY]
|
|
143
|
+
const llmobsTraceId = mlObsSpanTags?.[TRACE_ID]
|
|
144
|
+
const propagatedTraceId = llmobsTraceId === undefined
|
|
145
|
+
? parentContext?._trace?.tags?.[PROPAGATED_TRACE_ID_KEY]
|
|
146
|
+
: llmObsTraceIdToWire(llmobsTraceId)
|
|
140
147
|
|
|
141
|
-
if (!parentId && !mlApp && samplingDecision == null && !sessionId) return
|
|
148
|
+
if (!parentId && !mlApp && samplingDecision == null && !sessionId && !propagatedTraceId) return
|
|
142
149
|
|
|
143
150
|
// `_injectTags` only writes `x-datadog-tags` when the trace has `_dd.p.*`
|
|
144
151
|
// tags, so it may be undefined here — coalesce before appending.
|
|
@@ -149,6 +156,7 @@ function handleLLMObsInjection ({ carrier }) {
|
|
|
149
156
|
if (sessionId) tags += `${tags ? ',' : ''}${PROPAGATED_SESSION_ID_KEY}=${sessionId}`
|
|
150
157
|
if (sampleRate != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLE_RATE_KEY}=${sampleRate}`
|
|
151
158
|
if (samplingDecision != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLING_DECISION_KEY}=${samplingDecision}`
|
|
159
|
+
if (propagatedTraceId != null) tags += `${tags ? ',' : ''}${PROPAGATED_TRACE_ID_KEY}=${propagatedTraceId}`
|
|
152
160
|
if (tags !== existing) carrier['x-datadog-tags'] = tags
|
|
153
161
|
}
|
|
154
162
|
|
|
@@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([
|
|
|
13
13
|
const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.'
|
|
14
14
|
const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.'
|
|
15
15
|
const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
|
|
16
|
+
const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]'
|
|
17
|
+
const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]'
|
|
16
18
|
|
|
17
19
|
/**
|
|
18
20
|
* @typedef {import('../../../opentracing/span')} Span
|
|
@@ -26,8 +28,31 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
|
|
|
26
28
|
|
|
27
29
|
/**
|
|
28
30
|
* @typedef {{
|
|
29
|
-
* type:
|
|
30
|
-
*
|
|
31
|
+
* type: 'text',
|
|
32
|
+
* text: string
|
|
33
|
+
* } | {
|
|
34
|
+
* type: 'media' | 'file',
|
|
35
|
+
* mediaType: string
|
|
36
|
+
* } | {
|
|
37
|
+
* type: 'file-data' | 'file-url' | 'file-id' | 'file-reference'
|
|
38
|
+
* } | {
|
|
39
|
+
* type: 'image-data' | 'image-url' | 'image-file-id' | 'image-file-reference'
|
|
40
|
+
* } | {
|
|
41
|
+
* type: 'custom'
|
|
42
|
+
* }} ToolCallContentPart
|
|
43
|
+
*
|
|
44
|
+
* @typedef {{
|
|
45
|
+
* type: 'text' | 'error-text',
|
|
46
|
+
* value: string
|
|
47
|
+
* } | {
|
|
48
|
+
* type: 'json' | 'error-json',
|
|
49
|
+
* value: unknown
|
|
50
|
+
* } | {
|
|
51
|
+
* type: 'content',
|
|
52
|
+
* value: ToolCallContentPart[]
|
|
53
|
+
* } | {
|
|
54
|
+
* type: 'execution-denied',
|
|
55
|
+
* reason?: string
|
|
31
56
|
* }} ToolCallOutput
|
|
32
57
|
*
|
|
33
58
|
* @typedef {{ output?: ToolCallOutput, result?: unknown } & Record<string, unknown>} ToolCallResultContent
|
|
@@ -304,32 +329,89 @@ function getToolNameFromTags (tags) {
|
|
|
304
329
|
}
|
|
305
330
|
|
|
306
331
|
/**
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
|
|
332
|
+
* @param {unknown} value
|
|
333
|
+
* @returns {string}
|
|
334
|
+
*/
|
|
335
|
+
function stringifyToolCallResult (value) {
|
|
336
|
+
return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* @param {ToolCallContentPart[]} value
|
|
341
|
+
* @returns {string}
|
|
342
|
+
*/
|
|
343
|
+
function formatToolCallContent (value) {
|
|
344
|
+
if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT
|
|
345
|
+
|
|
346
|
+
let result = ''
|
|
347
|
+
for (const part of value) {
|
|
348
|
+
if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT
|
|
349
|
+
|
|
350
|
+
const { type } = part
|
|
351
|
+
if (type === 'text') {
|
|
352
|
+
if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT
|
|
353
|
+
result += part.text
|
|
354
|
+
} else if (type === 'media' || type === 'file') {
|
|
355
|
+
const { mediaType } = part
|
|
356
|
+
if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT
|
|
357
|
+
result += mediaType === 'image' || mediaType.startsWith('image/') ? '[Image]' : '[File]'
|
|
358
|
+
} else if (
|
|
359
|
+
type === 'file-data' ||
|
|
360
|
+
type === 'file-url' ||
|
|
361
|
+
type === 'file-id' ||
|
|
362
|
+
type === 'file-reference'
|
|
363
|
+
) {
|
|
364
|
+
result += '[File]'
|
|
365
|
+
} else if (
|
|
366
|
+
type === 'image-data' ||
|
|
367
|
+
type === 'image-url' ||
|
|
368
|
+
type === 'image-file-id' ||
|
|
369
|
+
type === 'image-file-reference'
|
|
370
|
+
) {
|
|
371
|
+
result += '[Image]'
|
|
372
|
+
} else if (type === 'custom') {
|
|
373
|
+
result += '[Custom Content]'
|
|
374
|
+
} else {
|
|
375
|
+
return UNPARSABLE_TOOL_RESULT
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
return result
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* @param {ToolCallResultContent | null | undefined} content
|
|
310
384
|
* @returns {string}
|
|
311
385
|
*/
|
|
312
386
|
function getToolCallResultContent (content) {
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
387
|
+
try {
|
|
388
|
+
if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT
|
|
389
|
+
|
|
390
|
+
const { output, result } = content
|
|
391
|
+
if (output !== undefined) {
|
|
392
|
+
if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT
|
|
393
|
+
|
|
394
|
+
const { type, value } = output
|
|
395
|
+
if (type === 'text' || type === 'error-text') {
|
|
396
|
+
return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT
|
|
397
|
+
} else if (type === 'json' || type === 'error-json') {
|
|
398
|
+
return stringifyToolCallResult(value)
|
|
399
|
+
} else if (type === 'content') {
|
|
400
|
+
return formatToolCallContent(value)
|
|
401
|
+
} else if (type === 'execution-denied') {
|
|
402
|
+
const { reason } = output
|
|
403
|
+
if (reason === undefined) return '[Tool Execution Denied]'
|
|
404
|
+
return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT
|
|
405
|
+
}
|
|
406
|
+
return UNPARSABLE_TOOL_RESULT
|
|
319
407
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (typeof result === 'string') {
|
|
323
|
-
return result
|
|
408
|
+
if (result !== undefined) {
|
|
409
|
+
return typeof result === 'string' ? result : stringifyToolCallResult(result)
|
|
324
410
|
}
|
|
325
411
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
return '[Unparsable Tool Result]'
|
|
330
|
-
}
|
|
331
|
-
} else {
|
|
332
|
-
return '[Unsupported Tool Result]'
|
|
412
|
+
return UNSUPPORTED_TOOL_RESULT
|
|
413
|
+
} catch {
|
|
414
|
+
return UNPARSABLE_TOOL_RESULT
|
|
333
415
|
}
|
|
334
416
|
}
|
|
335
417
|
|
|
@@ -5,7 +5,6 @@ const {
|
|
|
5
5
|
PROMPT_TRACKING_INSTRUMENTATION_METHOD,
|
|
6
6
|
PROMPT_MULTIMODAL,
|
|
7
7
|
INSTRUMENTATION_METHOD_AUTO,
|
|
8
|
-
UNKNOWN_MODEL_PROVIDER,
|
|
9
8
|
} = require('../../constants/tags')
|
|
10
9
|
const { audioMimeTypeFromFormat, formatAudioPart, safeJsonParse } = require('../../util')
|
|
11
10
|
const { AUDIO_MIME_TYPES } = require('./constants')
|
|
@@ -15,6 +14,7 @@ const {
|
|
|
15
14
|
extractTextFromContentItem,
|
|
16
15
|
extractContentParts,
|
|
17
16
|
hasMultimodalInputs,
|
|
17
|
+
getOpenAIModelProvider,
|
|
18
18
|
} = require('./utils')
|
|
19
19
|
|
|
20
20
|
const allowedParamKeys = new Set([
|
|
@@ -108,14 +108,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
_getModelProviderAndClient (baseUrl = '') {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
} else if (baseUrl.includes('deepseek')) {
|
|
116
|
-
return { modelProvider: 'deepseek', client: 'DeepSeek' }
|
|
117
|
-
}
|
|
118
|
-
return { modelProvider: UNKNOWN_MODEL_PROVIDER, client: 'OpenAI' }
|
|
111
|
+
const modelProvider = getOpenAIModelProvider(baseUrl)
|
|
112
|
+
if (modelProvider === 'azure_openai') return { modelProvider, client: 'AzureOpenAI' }
|
|
113
|
+
if (modelProvider === 'deepseek') return { modelProvider, client: 'DeepSeek' }
|
|
114
|
+
return { modelProvider, client: 'OpenAI' }
|
|
119
115
|
}
|
|
120
116
|
|
|
121
117
|
_extractMetrics (response) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
+
const { UNKNOWN_MODEL_PROVIDER } = require('../../constants/tags')
|
|
3
4
|
const { audioMimeTypeFromFormat, formatAudioPart } = require('../../util')
|
|
4
5
|
const {
|
|
5
6
|
INPUT_TYPE_IMAGE,
|
|
@@ -160,10 +161,29 @@ function extractContentParts (parts) {
|
|
|
160
161
|
return { content: extracted.join('\n'), audioParts }
|
|
161
162
|
}
|
|
162
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Maps an OpenAI-compatible base URL to a model provider string. Covers
|
|
166
|
+
* OpenAI, Azure OpenAI, and DeepSeek; falls back to UNKNOWN_MODEL_PROVIDER
|
|
167
|
+
* for unrecognised hosts (e.g. local proxies or custom deployments).
|
|
168
|
+
*
|
|
169
|
+
* Shared with the openai-agents integration since both consume the same
|
|
170
|
+
* client baseURL convention.
|
|
171
|
+
*
|
|
172
|
+
* @param {string} baseUrl
|
|
173
|
+
* @returns {string}
|
|
174
|
+
*/
|
|
175
|
+
function getOpenAIModelProvider (baseUrl = '') {
|
|
176
|
+
if (baseUrl.includes('azure')) return 'azure_openai'
|
|
177
|
+
if (baseUrl.includes('deepseek')) return 'deepseek'
|
|
178
|
+
if (baseUrl.includes('openai')) return 'openai'
|
|
179
|
+
return UNKNOWN_MODEL_PROVIDER
|
|
180
|
+
}
|
|
181
|
+
|
|
163
182
|
module.exports = {
|
|
164
183
|
extractChatTemplateFromInstructions,
|
|
165
184
|
normalizePromptVariables,
|
|
166
185
|
extractTextFromContentItem,
|
|
167
186
|
extractContentParts,
|
|
168
187
|
hasMultimodalInputs,
|
|
188
|
+
getOpenAIModelProvider,
|
|
169
189
|
}
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const { extractContentParts, extractTextFromContentItem } = require('../openai/utils')
|
|
4
|
+
const { safeJsonParse } = require('../../util')
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Flatten Responses and Chat Completions content parts without dropping
|
|
8
|
+
* provider-specific image or audio inputs.
|
|
9
|
+
*
|
|
10
|
+
* @param {Array<object>} parts
|
|
11
|
+
* @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
|
|
12
|
+
*/
|
|
13
|
+
function extractMessageContent (parts) {
|
|
14
|
+
const contentParts = []
|
|
15
|
+
const audioParts = []
|
|
16
|
+
|
|
17
|
+
for (const part of parts) {
|
|
18
|
+
if (!part) continue
|
|
19
|
+
const text = extractTextFromContentItem(part)
|
|
20
|
+
if (text) {
|
|
21
|
+
contentParts.push(text)
|
|
22
|
+
continue
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const extracted = extractContentParts([part])
|
|
26
|
+
if (extracted.content) contentParts.push(extracted.content)
|
|
27
|
+
if (extracted.audioParts.length > 0) audioParts.push(...extracted.audioParts)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return { content: contentParts.join(''), audioParts }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Normalize OpenAI Chat Completions tool calls to the LLMObs message schema.
|
|
35
|
+
*
|
|
36
|
+
* @param {object} message
|
|
37
|
+
* @returns {Array<{ toolId?: string, name?: string, arguments: object, type?: string }>}
|
|
38
|
+
*/
|
|
39
|
+
function extractChatCompletionToolCalls (message) {
|
|
40
|
+
if (message.function_call) {
|
|
41
|
+
return [{
|
|
42
|
+
name: message.function_call.name,
|
|
43
|
+
arguments: safeJsonParse(message.function_call.arguments, {}),
|
|
44
|
+
type: 'function',
|
|
45
|
+
}]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const toolCalls = []
|
|
49
|
+
if (!Array.isArray(message.tool_calls)) return toolCalls
|
|
50
|
+
|
|
51
|
+
for (const toolCall of message.tool_calls) {
|
|
52
|
+
if (!toolCall) continue
|
|
53
|
+
toolCalls.push({
|
|
54
|
+
toolId: toolCall.id,
|
|
55
|
+
name: toolCall.function?.name,
|
|
56
|
+
arguments: safeJsonParse(toolCall.function?.arguments, {}),
|
|
57
|
+
type: toolCall.type,
|
|
58
|
+
})
|
|
59
|
+
}
|
|
60
|
+
return toolCalls
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Normalize a Chat Completions message to the LLMObs message schema.
|
|
65
|
+
*
|
|
66
|
+
* @param {object} message
|
|
67
|
+
* @returns {object}
|
|
68
|
+
*/
|
|
69
|
+
function normalizeChatCompletionMessage (message) {
|
|
70
|
+
const normalized = {
|
|
71
|
+
role: message.role || 'assistant',
|
|
72
|
+
content: message.content ?? '',
|
|
73
|
+
}
|
|
74
|
+
if (message.audioParts?.length > 0) normalized.audioParts = message.audioParts
|
|
75
|
+
const toolCalls = extractChatCompletionToolCalls(message)
|
|
76
|
+
if (toolCalls.length > 0) normalized.toolCalls = toolCalls
|
|
77
|
+
if (message.tool_call_id) normalized.toolId = message.tool_call_id
|
|
78
|
+
return normalized
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Extracts input messages for an LLM span. agents-openai stores only
|
|
83
|
+
* `request.input` on `spanData._input` (string or message-array), and the
|
|
84
|
+
* system instructions are echoed back on the response as `instructions`.
|
|
85
|
+
*
|
|
86
|
+
* @param {string|Array<unknown>} input - The raw `request.input` (`spanData._input`).
|
|
87
|
+
* @param {string} [instructions] - System instructions echoed on `response.instructions`.
|
|
88
|
+
* @returns {Array<{ role: string, content: string }>}
|
|
89
|
+
*/
|
|
90
|
+
function extractInputMessages (input, instructions) {
|
|
91
|
+
const messages = []
|
|
92
|
+
|
|
93
|
+
if (instructions) {
|
|
94
|
+
messages.push({ role: 'system', content: instructions })
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (typeof input === 'string') {
|
|
98
|
+
messages.push({ role: 'user', content: input })
|
|
99
|
+
} else if (Array.isArray(input)) {
|
|
100
|
+
for (const item of input) {
|
|
101
|
+
if (!item) continue
|
|
102
|
+
if (item.type === 'message' || item.role) {
|
|
103
|
+
const role = item.role
|
|
104
|
+
if (!role) continue
|
|
105
|
+
|
|
106
|
+
let content = ''
|
|
107
|
+
let audioParts
|
|
108
|
+
if (Array.isArray(item.content)) {
|
|
109
|
+
const extracted = extractMessageContent(item.content)
|
|
110
|
+
content = extracted.content
|
|
111
|
+
audioParts = extracted.audioParts
|
|
112
|
+
} else if (typeof item.content === 'string') {
|
|
113
|
+
content = item.content
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const message = normalizeChatCompletionMessage({ ...item, content, role, audioParts })
|
|
117
|
+
if (content || message.audioParts || message.toolCalls || message.toolId) {
|
|
118
|
+
messages.push(message)
|
|
119
|
+
}
|
|
120
|
+
} else if (item.type === 'function_call') {
|
|
121
|
+
let args = item.arguments
|
|
122
|
+
if (typeof args === 'string') {
|
|
123
|
+
try {
|
|
124
|
+
args = JSON.parse(args)
|
|
125
|
+
} catch {
|
|
126
|
+
args = {}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
messages.push({
|
|
130
|
+
role: 'assistant',
|
|
131
|
+
toolCalls: [{
|
|
132
|
+
toolId: item.call_id,
|
|
133
|
+
name: item.name,
|
|
134
|
+
arguments: args,
|
|
135
|
+
type: item.type,
|
|
136
|
+
}],
|
|
137
|
+
})
|
|
138
|
+
} else if (item.type === 'function_call_output') {
|
|
139
|
+
messages.push({
|
|
140
|
+
role: 'user',
|
|
141
|
+
toolResults: [{
|
|
142
|
+
toolId: item.call_id,
|
|
143
|
+
result: item.output,
|
|
144
|
+
name: item.name || '',
|
|
145
|
+
type: item.type,
|
|
146
|
+
}],
|
|
147
|
+
})
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return messages.length > 0 ? messages : [{ role: 'user', content: '' }]
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Extracts output messages from the model response.
|
|
157
|
+
*
|
|
158
|
+
* @param {{ output?: Array<unknown> }} result - The model response
|
|
159
|
+
* @returns {Array<{ role: string, content: string }>}
|
|
160
|
+
*/
|
|
161
|
+
function extractOutputMessages (result) {
|
|
162
|
+
const messages = []
|
|
163
|
+
|
|
164
|
+
if (result?.output) {
|
|
165
|
+
for (const item of result.output) {
|
|
166
|
+
if (!item) continue
|
|
167
|
+
if (item.type === 'message') {
|
|
168
|
+
let content = ''
|
|
169
|
+
if (Array.isArray(item.content)) {
|
|
170
|
+
const textParts = []
|
|
171
|
+
for (const contentItem of item.content) {
|
|
172
|
+
if (contentItem?.type === 'output_text' && contentItem.text) {
|
|
173
|
+
textParts.push(contentItem.text)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
content = textParts.join('')
|
|
177
|
+
} else if (typeof item.content === 'string') {
|
|
178
|
+
content = item.content
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
messages.push({ role: item.role || 'assistant', content })
|
|
182
|
+
} else if (item.type === 'function_call') {
|
|
183
|
+
let args = item.arguments
|
|
184
|
+
if (typeof args === 'string') {
|
|
185
|
+
try {
|
|
186
|
+
args = JSON.parse(args)
|
|
187
|
+
} catch {
|
|
188
|
+
args = {}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
messages.push({
|
|
192
|
+
role: 'assistant',
|
|
193
|
+
toolCalls: [{
|
|
194
|
+
toolId: item.call_id,
|
|
195
|
+
name: item.name,
|
|
196
|
+
arguments: args,
|
|
197
|
+
type: item.type,
|
|
198
|
+
}],
|
|
199
|
+
})
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return messages.length > 0 ? messages : [{ content: '', role: '' }]
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Extracts output messages from an OpenAI Chat Completions response.
|
|
209
|
+
*
|
|
210
|
+
* @param {Array<{ choices?: Array<{ message?: object }> }>} result - The model responses
|
|
211
|
+
* @returns {Array<object>}
|
|
212
|
+
*/
|
|
213
|
+
function extractGenerationOutputMessages (result) {
|
|
214
|
+
const messages = []
|
|
215
|
+
|
|
216
|
+
if (Array.isArray(result)) {
|
|
217
|
+
for (const response of result) {
|
|
218
|
+
if (!Array.isArray(response?.choices)) continue
|
|
219
|
+
for (const choice of response.choices) {
|
|
220
|
+
const message = choice?.message
|
|
221
|
+
if (!message) continue
|
|
222
|
+
messages.push(normalizeChatCompletionMessage(message))
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return messages.length > 0 ? messages : [{ content: '', role: '' }]
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Extracts token usage metrics from the model response. Returns `undefined`
|
|
232
|
+
* when there's nothing to tag, so callers can skip the tagger call without
|
|
233
|
+
* allocating an Object.keys array.
|
|
234
|
+
*
|
|
235
|
+
* @param {{ usage?: { inputTokens?: number, outputTokens?: number, totalTokens?: number,
|
|
236
|
+
* outputTokensDetails?: { reasoningTokens?: number },
|
|
237
|
+
* completion_tokens_details?: { reasoning_tokens?: number } } }} result
|
|
238
|
+
* @returns {{ inputTokens?: number, outputTokens?: number, totalTokens?: number,
|
|
239
|
+
* reasoningOutputTokens?: number } | undefined}
|
|
240
|
+
*/
|
|
241
|
+
function extractMetrics (result) {
|
|
242
|
+
const usage = result?.usage
|
|
243
|
+
if (!usage) return
|
|
244
|
+
|
|
245
|
+
const inputTokens = usage.inputTokens ?? usage.input_tokens ?? usage.prompt_tokens
|
|
246
|
+
const outputTokens = usage.outputTokens ?? usage.output_tokens ?? usage.completion_tokens
|
|
247
|
+
const totalTokens = usage.totalTokens ?? usage.total_tokens
|
|
248
|
+
const reasoningTokens = usage.outputTokensDetails?.reasoningTokens ??
|
|
249
|
+
usage.output_tokens_details?.reasoning_tokens ??
|
|
250
|
+
usage.completion_tokens_details?.reasoning_tokens
|
|
251
|
+
|
|
252
|
+
if (inputTokens === undefined && outputTokens === undefined &&
|
|
253
|
+
totalTokens === undefined && !reasoningTokens) return
|
|
254
|
+
|
|
255
|
+
const metrics = {}
|
|
256
|
+
if (inputTokens !== undefined) metrics.inputTokens = inputTokens
|
|
257
|
+
if (outputTokens !== undefined) metrics.outputTokens = outputTokens
|
|
258
|
+
// Tagger maps `reasoningOutputTokens` → `reasoning_output_tokens` in the
|
|
259
|
+
// LLMObs span event. Skip when zero — emitting a zero just adds noise.
|
|
260
|
+
if (reasoningTokens) metrics.reasoningOutputTokens = reasoningTokens
|
|
261
|
+
|
|
262
|
+
if (totalTokens !== undefined) {
|
|
263
|
+
metrics.totalTokens = totalTokens
|
|
264
|
+
} else if (metrics.inputTokens !== undefined && metrics.outputTokens !== undefined) {
|
|
265
|
+
metrics.totalTokens = metrics.inputTokens + metrics.outputTokens
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
return metrics
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Fields the OpenAI Responses API echoes back from the request configuration.
|
|
272
|
+
// agents-openai only stores `request.input` on the span — the user's
|
|
273
|
+
// `modelSettings` aren't directly observable, so we read the response-echoed
|
|
274
|
+
// values. Matches dd-trace-py's openai-agents integration (see
|
|
275
|
+
// `OaiSpanAdapter.llmobs_metadata`); both ship without filtering OpenAI's
|
|
276
|
+
// default values.
|
|
277
|
+
const RESPONSE_METADATA_FIELDS = [
|
|
278
|
+
'temperature',
|
|
279
|
+
'max_output_tokens',
|
|
280
|
+
'top_p',
|
|
281
|
+
'tools',
|
|
282
|
+
'tool_choice',
|
|
283
|
+
'truncation',
|
|
284
|
+
]
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Extracts metadata from the model response. Mirrors Python's
|
|
288
|
+
* `OaiSpanAdapter.llmobs_metadata` — emits all response-echoed configuration
|
|
289
|
+
* fields plus `text` when present. Returns `undefined` when nothing was
|
|
290
|
+
* captured, so callers can skip the tagger call without allocating.
|
|
291
|
+
*
|
|
292
|
+
* @param {object | undefined} response
|
|
293
|
+
* @returns {object | undefined}
|
|
294
|
+
*/
|
|
295
|
+
function extractMetadata (response) {
|
|
296
|
+
if (!response) return
|
|
297
|
+
|
|
298
|
+
let metadata
|
|
299
|
+
for (const field of RESPONSE_METADATA_FIELDS) {
|
|
300
|
+
const value = response[field]
|
|
301
|
+
if (value !== undefined && value !== null) {
|
|
302
|
+
metadata ??= {}
|
|
303
|
+
metadata[field] = value
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
if (response.text) {
|
|
308
|
+
metadata ??= {}
|
|
309
|
+
metadata.text = response.text
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
return metadata
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
module.exports = {
|
|
316
|
+
extractInputMessages,
|
|
317
|
+
extractOutputMessages,
|
|
318
|
+
extractGenerationOutputMessages,
|
|
319
|
+
extractMetrics,
|
|
320
|
+
extractMetadata,
|
|
321
|
+
}
|
|
@@ -11,6 +11,7 @@ const {
|
|
|
11
11
|
SPAN_KIND,
|
|
12
12
|
OUTPUT_VALUE,
|
|
13
13
|
INPUT_VALUE,
|
|
14
|
+
TRACE_ID,
|
|
14
15
|
} = require('./constants/tags')
|
|
15
16
|
const {
|
|
16
17
|
getFunctionArguments,
|
|
@@ -345,7 +346,7 @@ class LLMObs extends NoopLLMObs {
|
|
|
345
346
|
}
|
|
346
347
|
try {
|
|
347
348
|
return {
|
|
348
|
-
traceId:
|
|
349
|
+
traceId: LLMObsTagger.tagMap.get(span)[TRACE_ID],
|
|
349
350
|
spanId: span.context().toSpanId(),
|
|
350
351
|
}
|
|
351
352
|
} catch {
|
|
@@ -35,6 +35,7 @@ const {
|
|
|
35
35
|
LLMOBS_SUBMITTED_TAG_KEY,
|
|
36
36
|
SAMPLE_RATE,
|
|
37
37
|
SAMPLING_DECISION,
|
|
38
|
+
TRACE_ID,
|
|
38
39
|
} = require('./constants/tags')
|
|
39
40
|
const { UNSERIALIZABLE_VALUE_TEXT } = require('./constants/text')
|
|
40
41
|
const telemetry = require('./telemetry')
|
|
@@ -236,8 +237,11 @@ class LLMObsSpanProcessor {
|
|
|
236
237
|
meta.input.prompt = prompt
|
|
237
238
|
}
|
|
238
239
|
|
|
240
|
+
const apmTraceId = span.context().toTraceId(true)
|
|
241
|
+
const llmobsTraceId = mlObsTags[TRACE_ID] ?? apmTraceId
|
|
242
|
+
|
|
239
243
|
const llmObsSpanEvent = {
|
|
240
|
-
trace_id:
|
|
244
|
+
trace_id: llmobsTraceId,
|
|
241
245
|
span_id: span.context().toSpanId(),
|
|
242
246
|
parent_id: parentId,
|
|
243
247
|
name,
|
|
@@ -249,9 +253,10 @@ class LLMObsSpanProcessor {
|
|
|
249
253
|
metrics,
|
|
250
254
|
_dd: {
|
|
251
255
|
span_id: span.context().toSpanId(),
|
|
252
|
-
trace_id:
|
|
256
|
+
trace_id: apmTraceId,
|
|
253
257
|
sample_rate: mlObsTags[SAMPLE_RATE],
|
|
254
258
|
sampling_decision: mlObsTags[SAMPLING_DECISION],
|
|
259
|
+
apm_trace_id: apmTraceId,
|
|
255
260
|
},
|
|
256
261
|
}
|
|
257
262
|
|