dd-trace 6.6.0 → 6.8.0

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -1,8 +1,90 @@
 'use strict'
 
+const log = require('../../log')
+
+class NoopDataset {
+  #name
+  #records
+
+  constructor (name = '', options = {}) {
+    this.#name = name
+    this.#records = (typeof options === 'string' ? [] : (options.records ?? [])).map(record => ({
+      id: record.id ?? null,
+      input: record.inputData,
+      expectedOutput: record.expectedOutput ?? null,
+      metadata: record.metadata ?? {},
+    }))
+  }
+
+  addRecord (input, expectedOutput, metadata) {
+    this.#records.push({ id: null, input, expectedOutput: expectedOutput ?? null, metadata: metadata ?? {} })
+    return this
+  }
+
+  push () {
+    return Promise.resolve({ pushedCount: 0, totalCount: 0 })
+  }
+
+  name () {
+    return this.#name
+  }
+
+  id () {
+    return null
+  }
+
+  projectId () {
+    return null
+  }
+
+  version () {
+    return null
+  }
+
+  latestVersion () {
+    return null
+  }
+
+  records () {
+    return [...this.#records]
+  }
+
+  recordIds () {
+    return []
+  }
+
+  url () {
+    return null
+  }
+}
+
+class NoopExperiment {
+  #name
+
+  constructor (name = '') {
+    this.#name = name
+  }
+
+  name () {
+    return this.#name
+  }
+
+  experimentId () {
+    return null
+  }
+
+  url () {
+    return null
+  }
+
+  run () {
+    return Promise.resolve({ experimentId: null, rows: [], url: null })
+  }
+}
+
 // No-op Experiments used when LLM Observability is disabled or the API/APP keys
-// are not configured. Every operation throws a clear, actionable error rather
-// than silently doing nothing, so misconfiguration surfaces immediately.
+// are not configured. Operations warn and return inert objects rather than
+// throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
 
@@ -10,20 +92,23 @@ class NoopExperiments {
     this.#reason = reason || 'LLMObs experiments are not available'
   }
 
-  #unavailable () {
-    return new Error(`LLMObs experiments unavailable: ${this.#reason}`)
+  #warn () {
+    log.warn('LLMObs experiments unavailable: %s', this.#reason)
   }
 
-  createDataset () {
-    throw this.#unavailable()
+  createDataset (name, options = {}) {
+    this.#warn()
+    return new NoopDataset(name, options)
   }
 
-  pullDataset () {
-    return Promise.reject(this.#unavailable())
+  pullDataset (name) {
+    this.#warn()
+    return Promise.resolve(new NoopDataset(name))
   }
 
-  experiment () {
-    throw this.#unavailable()
+  experiment (options = {}) {
+    this.#warn()
+    return new NoopExperiment(options.name)
   }
 }
 

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -22,13 +22,24 @@ class Row {
   }
 }
 
+class ExperimentRun {
+  constructor (fields) {
+    this.runId = fields.runId
+    this.runIteration = fields.runIteration
+    this.rows = fields.rows
+    this.summaryEvaluations = fields.summaryEvaluations
+  }
+}
+
 // Returned by Experiment.run().
 class ExperimentResult {
-  constructor (experimentId, rows, url) {
+  constructor (experimentId, rows, url, runs = [], summaryEvaluations = {}) {
     this.experimentId = experimentId
     this.rows = rows
     this.url = url
+    this.runs = runs
+    this.summaryEvaluations = summaryEvaluations
   }
 }
 
-module.exports = { Row, ExperimentResult }
+module.exports = { Row, ExperimentRun, ExperimentResult }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -0,0 +1,218 @@
+'use strict'
+
+const log = require('../../log')
+
+// Matches the backend and dd-trace-py evaluator metric label contract.
+const EVALUATOR_NAME_PATTERN = /^[a-zA-Z0-9_-]+$/
+
+/**
+ * @typedef {null | string | number | boolean | NormalizedJsonValue[] | { [key: string]: NormalizedJsonValue }}
+ *   NormalizedJsonValue
+ */
+
+/**
+ * @param {object | null | undefined} value
+ * @returns {boolean}
+ */
+function hasEntries (value) {
+  if (!value) return false
+  for (const key of Object.keys(value)) {
+    if (Object.hasOwn(value, key)) return true
+  }
+  return false
+}
+
+/**
+ * @param {string} name
+ */
+function validateEvaluatorName (name) {
+  if (typeof name !== 'string') throw new TypeError('Evaluator name must be a string')
+  if (name.length === 0) throw new Error('Evaluator name cannot be empty')
+  if (!EVALUATOR_NAME_PATTERN.test(name)) {
+    throw new Error(
+      `Evaluator name '${name}' is invalid. Name must contain only alphanumeric characters, underscores, and hyphens.`
+    )
+  }
+}
+
+/**
+ * @param {(...args: unknown[]) => unknown} fn
+ * @param {string} fallback
+ * @returns {string}
+ */
+function functionName (fn, fallback) {
+  return typeof fn.name === 'string' && fn.name.length > 0 ? fn.name : fallback
+}
+
+/**
+ * @param {unknown} evaluators
+ * @param {string} kind
+ * @returns {Array<[string, (...args: unknown[]) => unknown]>}
+ */
+function normalizeEvaluators (evaluators, kind) {
+  if (evaluators == null) return []
+
+  const normalized = []
+  if (Array.isArray(evaluators)) {
+    const indexesByName = new Map()
+    for (let i = 0; i < evaluators.length; i++) {
+      const evaluator = evaluators[i]
+      if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator must be a function`)
+      const name = functionName(evaluator, `${kind}_evaluator_${i}`)
+      validateEvaluatorName(name)
+      if (indexesByName.has(name)) {
+        log.warn('Duplicate %s evaluator name %s; previous evaluator will be overwritten', kind, name)
+        normalized[indexesByName.get(name)] = [name, evaluator]
+      } else {
+        indexesByName.set(name, normalized.length)
+        normalized.push([name, evaluator])
+      }
+    }
+    return normalized
+  }
+
+  if (typeof evaluators !== 'object') {
+    throw new TypeError(`${kind} evaluators must be an array of functions or an object keyed by evaluator name`)
+  }
+
+  for (const [name, evaluator] of Object.entries(evaluators)) {
+    validateEvaluatorName(name)
+    if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator '${name}' must be a function`)
+    normalized.push([name, evaluator])
+  }
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function inferMetricType (value) {
+  if (typeof value === 'boolean') return 'boolean'
+  if (typeof value === 'number' && Number.isFinite(value)) return 'score'
+  if (typeof value === 'string') return 'categorical'
+  return 'json'
+}
+
+/**
+ * @param {unknown} value
+ * @param {WeakSet<object>} seen
+ * @returns {NormalizedJsonValue}
+ */
+function normalizeJsonValue (value, seen) {
+  if (value === null) return null
+
+  const valueType = typeof value
+  if (valueType === 'string' || valueType === 'boolean') return value
+  if (valueType === 'number') return Number.isFinite(value) ? value : String(value)
+  if (valueType === 'bigint') return value.toString()
+  if (valueType === 'undefined') return null
+  if (valueType === 'symbol') return value.toString()
+  if (valueType === 'function') return { type: 'function', name: value.name || '<anonymous>' }
+
+  if (value instanceof Date) return value.toISOString()
+  if (value instanceof Error) {
+    return { type: value.name, message: value.message, stack: value.stack ?? '' }
+  }
+
+  if (seen.has(value)) return '[Circular]'
+  seen.add(value)
+
+  if (Array.isArray(value)) {
+    const normalized = new Array(value.length)
+    for (let i = 0; i < value.length; i++) normalized[i] = normalizeJsonValue(value[i], seen)
+    seen.delete(value)
+    return normalized
+  }
+
+  const normalized = {}
+  for (const [key, nestedValue] of Object.entries(value)) {
+    normalized[key] = normalizeJsonValue(nestedValue, seen)
+  }
+  seen.delete(value)
+  return normalized
+}
+
+/**
+ * @param {unknown} value
+ * @returns {{ [key: string]: NormalizedJsonValue }}
+ */
+function normalizeJsonMetricValue (value) {
+  const normalized = normalizeJsonValue(value, new WeakSet())
+  if (normalized !== null && typeof normalized === 'object' && !Array.isArray(normalized)) return normalized
+  return { value: normalized }
+}
+
+/**
+ * @param {unknown} value
+ * @returns {string}
+ */
+function stringify (value) {
+  if (value == null) return ''
+  if (typeof value === 'string') return value.toLowerCase()
+  if (typeof value === 'object') {
+    try {
+      const stringified = JSON.stringify(value)
+      if (stringified !== undefined) return stringified.toLowerCase()
+    } catch {}
+  }
+  return String(value).toLowerCase()
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {string[]}
+ */
+function buildTags (userTags, autoTags) {
+  const tags = new Map()
+  for (const [key, value] of Object.entries(userTags ?? {})) {
+    tags.set(key, `${key}:${value}`)
+  }
+  for (const [key, value] of Object.entries(autoTags)) {
+    if (value !== undefined && value !== null && value !== '') tags.set(key, `${key}:${value}`)
+  }
+  return [...tags.values()]
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} userTags
+ * @param {Record<string, unknown>} autoTags
+ * @returns {Record<string, unknown>}
+ */
+function buildExperimentTagObject (userTags, autoTags) {
+  return userTags ? { ...userTags, ...autoTags } : { ...autoTags }
+}
+
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function sleep (ms) {
+  if (ms <= 0) return Promise.resolve()
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * @param {Record<string, unknown> | undefined} recordMetadata
+ * @param {Record<string, unknown>} config
+ * @returns {Record<string, unknown>}
+ */
+function buildSpanMetadata (recordMetadata, config) {
+  return recordMetadata
+    ? { ...recordMetadata, experiment_config: config }
+    : { experiment_config: config }
+}
+
+module.exports = {
+  buildExperimentTagObject,
+  buildSpanMetadata,
+  buildTags,
+  hasEntries,
+  inferMetricType,
+  normalizeEvaluators,
+  normalizeJsonMetricValue,
+  sleep,
+  stringify,
+  validateEvaluatorName,
+}

package/packages/dd-trace/src/llmobs/index.js

@@ -16,6 +16,8 @@ const {
   SAMPLING_DECISION,
   PROPAGATED_SAMPLE_RATE_KEY,
   PROPAGATED_SAMPLING_DECISION_KEY,
+  TRACE_ID,
+  PROPAGATED_TRACE_ID_KEY,
 } = require('./constants/tags')
 const { storage } = require('./storage')
 const telemetry = require('./telemetry')
@@ -25,6 +27,7 @@ const LLMObsTagger = require('./tagger')
 const LLMObsSpanWriter = require('./writers/spans')
 const { setAgentStrategy } = require('./writers/util')
 const { INCOMPATIBLE_INITIALIZATION } = require('./constants/text')
+const { llmObsTraceIdToWire } = require('./util')
 
 const spanFinishCh = channel('dd-trace:span:finish')
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
@@ -137,8 +140,12 @@ function handleLLMObsInjection ({ carrier }) {
     mlObsSpanTags?.[SESSION_ID] ??
     parentContext?._trace?.tags?.[SESSION_ID_TRACE_DEFAULT_KEY] ??
     parentContext?._trace?.tags?.[PROPAGATED_SESSION_ID_KEY]
+  const llmobsTraceId = mlObsSpanTags?.[TRACE_ID]
+  const propagatedTraceId = llmobsTraceId === undefined
+    ? parentContext?._trace?.tags?.[PROPAGATED_TRACE_ID_KEY]
+    : llmObsTraceIdToWire(llmobsTraceId)
 
-  if (!parentId && !mlApp && samplingDecision == null && !sessionId) return
+  if (!parentId && !mlApp && samplingDecision == null && !sessionId && !propagatedTraceId) return
 
   // `_injectTags` only writes `x-datadog-tags` when the trace has `_dd.p.*`
   // tags, so it may be undefined here — coalesce before appending.
@@ -149,6 +156,7 @@ function handleLLMObsInjection ({ carrier }) {
   if (sessionId) tags += `${tags ? ',' : ''}${PROPAGATED_SESSION_ID_KEY}=${sessionId}`
   if (sampleRate != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLE_RATE_KEY}=${sampleRate}`
   if (samplingDecision != null) tags += `${tags ? ',' : ''}${PROPAGATED_SAMPLING_DECISION_KEY}=${samplingDecision}`
+  if (propagatedTraceId != null) tags += `${tags ? ',' : ''}${PROPAGATED_TRACE_ID_KEY}=${propagatedTraceId}`
   if (tags !== existing) carrier['x-datadog-tags'] = tags
 }
 

package/packages/dd-trace/src/llmobs/plugins/ai/util.js

@@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([
 const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.'
 const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.'
 const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
+const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]'
+const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]'
 
 /**
  * @typedef {import('../../../opentracing/span')} Span
@@ -26,8 +28,31 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
 
 /**
  * @typedef {{
- *   type: string,
- *   value?: unknown
+ *   type: 'text',
+ *   text: string
+ * } | {
+ *   type: 'media' | 'file',
+ *   mediaType: string
+ * } | {
+ *   type: 'file-data' | 'file-url' | 'file-id' | 'file-reference'
+ * } | {
+ *   type: 'image-data' | 'image-url' | 'image-file-id' | 'image-file-reference'
+ * } | {
+ *   type: 'custom'
+ * }} ToolCallContentPart
+ *
+ * @typedef {{
+ *   type: 'text' | 'error-text',
+ *   value: string
+ * } | {
+ *   type: 'json' | 'error-json',
+ *   value: unknown
+ * } | {
+ *   type: 'content',
+ *   value: ToolCallContentPart[]
+ * } | {
+ *   type: 'execution-denied',
+ *   reason?: string
  * }} ToolCallOutput
  *
  * @typedef {{ output?: ToolCallOutput, result?: unknown } & Record<string, unknown>} ToolCallResultContent
@@ -304,32 +329,89 @@ function getToolNameFromTags (tags) {
 }
 
 /**
- * Get the content of a tool call result.
- * Version 5 of the ai sdk sets this tag as `content.output`, with a `
- * @param {{ output?: { type: string, value: unknown }, result?: unknown }} content
+ * @param {unknown} value
+ * @returns {string}
+ */
+function stringifyToolCallResult (value) {
+  return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT
+}
+
+/**
+ * @param {ToolCallContentPart[]} value
+ * @returns {string}
+ */
+function formatToolCallContent (value) {
+  if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT
+
+  let result = ''
+  for (const part of value) {
+    if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT
+
+    const { type } = part
+    if (type === 'text') {
+      if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT
+      result += part.text
+    } else if (type === 'media' || type === 'file') {
+      const { mediaType } = part
+      if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT
+      result += mediaType === 'image' || mediaType.startsWith('image/') ? '[Image]' : '[File]'
+    } else if (
+      type === 'file-data' ||
+      type === 'file-url' ||
+      type === 'file-id' ||
+      type === 'file-reference'
+    ) {
+      result += '[File]'
+    } else if (
+      type === 'image-data' ||
+      type === 'image-url' ||
+      type === 'image-file-id' ||
+      type === 'image-file-reference'
+    ) {
+      result += '[Image]'
+    } else if (type === 'custom') {
+      result += '[Custom Content]'
+    } else {
+      return UNPARSABLE_TOOL_RESULT
+    }
+  }
+
+  return result
+}
+
+/**
+ * @param {ToolCallResultContent | null | undefined} content
  * @returns {string}
  */
 function getToolCallResultContent (content) {
-  const { output, result } = content
-  if (output) {
-    if (output.type === 'text') {
-      return output.value
-    } else if (output.type === 'json') {
-      return JSON.stringify(output.value)
+  try {
+    if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT
+
+    const { output, result } = content
+    if (output !== undefined) {
+      if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT
+
+      const { type, value } = output
+      if (type === 'text' || type === 'error-text') {
+        return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT
+      } else if (type === 'json' || type === 'error-json') {
+        return stringifyToolCallResult(value)
+      } else if (type === 'content') {
+        return formatToolCallContent(value)
+      } else if (type === 'execution-denied') {
+        const { reason } = output
+        if (reason === undefined) return '[Tool Execution Denied]'
+        return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT
+      }
+      return UNPARSABLE_TOOL_RESULT
     }
-    return '[Unparsable Tool Result]'
-  } else if (result) {
-    if (typeof result === 'string') {
-      return result
+    if (result !== undefined) {
+      return typeof result === 'string' ? result : stringifyToolCallResult(result)
     }
 
-    try {
-      return JSON.stringify(result)
-    } catch {
-      return '[Unparsable Tool Result]'
-    }
-  } else {
-    return '[Unsupported Tool Result]'
+    return UNSUPPORTED_TOOL_RESULT
+  } catch {
+    return UNPARSABLE_TOOL_RESULT
   }
 }
 

package/packages/dd-trace/src/llmobs/plugins/anthropic/index.js

@@ -93,7 +93,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin {
 
             const { usage } = chunk
             if (usage) {
-              const responseUsage = response.usage ?? (response.usage = { input_tokens: 0, output_tokens: 0 })
+              const responseUsage = (response.usage ??= { input_tokens: 0, output_tokens: 0 })
               responseUsage.output_tokens = usage.output_tokens
 
               const cacheCreationTokens = usage.cache_creation_input_tokens

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -8,7 +8,7 @@ const { splitModel } = require('../../../../../datadog-plugin-claude-agent-sdk/s
 const subagentToolIds = new Set()
 
 function normalizeToolOutputString (raw) {
-  const footerIndex = raw.search(/\s*agentId: [^\s]+ \(use SendMessage\b/)
+  const footerIndex = raw.search(/agentId: \S+ \(use SendMessage\b/)
   if (footerIndex === -1) return raw
 
   return raw.slice(0, footerIndex).trimEnd()

package/packages/dd-trace/src/llmobs/plugins/genai/index.js

@@ -24,7 +24,7 @@ class GenAiLLMObsPlugin extends LLMObsPlugin {
     // Subscribe to streaming chunk events
     this.addSub('apm:google:genai:request:chunk', ({ ctx, chunk, done }) => {
       ctx.isStreaming = true
-      ctx.chunks = ctx.chunks || []
+      ctx.chunks ||= []
 
       if (chunk) ctx.chunks.push(chunk)
       if (!done) return

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js

@@ -36,7 +36,7 @@ class LangChainLLMObsHandler {
     const runIdBase = runId ? runId.split('-').slice(0, -1).join('-') : ''
 
     const responseMetadata = message.response_metadata || {}
-    usage = usage || responseMetadata.usage || responseMetadata.tokenUsage || {}
+    usage ||= responseMetadata.usage || responseMetadata.tokenUsage || {}
 
     const inputTokens = usage.promptTokens || usage.inputTokens || usage.prompt_tokens || usage.input_tokens || 0
     const outputTokens =

package/packages/dd-trace/src/llmobs/plugins/openai/index.js

@@ -5,7 +5,6 @@ const {
   PROMPT_TRACKING_INSTRUMENTATION_METHOD,
   PROMPT_MULTIMODAL,
   INSTRUMENTATION_METHOD_AUTO,
-  UNKNOWN_MODEL_PROVIDER,
 } = require('../../constants/tags')
 const { audioMimeTypeFromFormat, formatAudioPart, safeJsonParse } = require('../../util')
 const { AUDIO_MIME_TYPES } = require('./constants')
@@ -15,6 +14,7 @@ const {
   extractTextFromContentItem,
   extractContentParts,
   hasMultimodalInputs,
+  getOpenAIModelProvider,
 } = require('./utils')
 
 const allowedParamKeys = new Set([
@@ -108,14 +108,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
   }
 
   _getModelProviderAndClient (baseUrl = '') {
-    if (baseUrl.includes('azure')) {
-      return { modelProvider: 'azure_openai', client: 'AzureOpenAI' }
-    } else if (baseUrl.includes('openai')) {
-      return { modelProvider: 'openai', client: 'OpenAI' }
-    } else if (baseUrl.includes('deepseek')) {
-      return { modelProvider: 'deepseek', client: 'DeepSeek' }
-    }
-    return { modelProvider: UNKNOWN_MODEL_PROVIDER, client: 'OpenAI' }
+    const modelProvider = getOpenAIModelProvider(baseUrl)
+    if (modelProvider === 'azure_openai') return { modelProvider, client: 'AzureOpenAI' }
+    if (modelProvider === 'deepseek') return { modelProvider, client: 'DeepSeek' }
+    return { modelProvider, client: 'OpenAI' }
   }
 
   _extractMetrics (response) {

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { UNKNOWN_MODEL_PROVIDER } = require('../../constants/tags')
 const { audioMimeTypeFromFormat, formatAudioPart } = require('../../util')
 const {
   INPUT_TYPE_IMAGE,
@@ -160,10 +161,29 @@ function extractContentParts (parts) {
   return { content: extracted.join('\n'), audioParts }
 }
 
+/**
+ * Maps an OpenAI-compatible base URL to a model provider string. Covers
+ * OpenAI, Azure OpenAI, and DeepSeek; falls back to UNKNOWN_MODEL_PROVIDER
+ * for unrecognised hosts (e.g. local proxies or custom deployments).
+ *
+ * Shared with the openai-agents integration since both consume the same
+ * client baseURL convention.
+ *
+ * @param {string} baseUrl
+ * @returns {string}
+ */
+function getOpenAIModelProvider (baseUrl = '') {
+  if (baseUrl.includes('azure')) return 'azure_openai'
+  if (baseUrl.includes('deepseek')) return 'deepseek'
+  if (baseUrl.includes('openai')) return 'openai'
+  return UNKNOWN_MODEL_PROVIDER
+}
+
 module.exports = {
   extractChatTemplateFromInstructions,
   normalizePromptVariables,
   extractTextFromContentItem,
   extractContentParts,
   hasMultimodalInputs,
+  getOpenAIModelProvider,
 }