dd-trace 6.3.0 → 6.5.0

package/packages/dd-trace/src/llmobs/plugins/openai/index.js

@@ -7,11 +7,13 @@ const {
   INSTRUMENTATION_METHOD_AUTO,
   UNKNOWN_MODEL_PROVIDER,
 } = require('../../constants/tags')
-const { safeJsonParse } = require('../../util')
+const { audioMimeTypeFromFormat, formatAudioPart, safeJsonParse } = require('../../util')
+const { AUDIO_MIME_TYPES } = require('./constants')
 const {
   extractChatTemplateFromInstructions,
   normalizePromptVariables,
   extractTextFromContentItem,
+  extractContentParts,
   hasMultimodalInputs,
 } = require('./utils')
 
@@ -29,6 +31,22 @@ function isIterable (obj) {
   return typeof obj[Symbol.iterator] === 'function'
 }
 
+// Flattens multimodal chat input messages (array `content`) into readable text plus structured
+// `audioParts`, leaving plain-string messages untouched. Model-agnostic: keys off message
+// structure, so it works for any audio-capable chat model (gpt-audio*, gpt-4o-audio-preview, ...).
+function flattenChatInputMessages (messages) {
+  if (!Array.isArray(messages)) return messages
+
+  return messages.map(message => {
+    if (!Array.isArray(message?.content)) return message
+
+    const { content, audioParts } = extractContentParts(message.content)
+    const flattenedMessage = { ...message, content }
+    if (audioParts.length) flattenedMessage.audioParts = audioParts
+    return flattenedMessage
+  })
+}
+
 class OpenAiLLMObsPlugin extends LLMObsPlugin {
   static id = 'openai'
   static integration = 'openai'
@@ -193,47 +211,63 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
 
     this._tagger.tagMetadata(span, metadata)
 
+    const inputMessages = flattenChatInputMessages(messages)
+
     if (error) {
-      this._tagger.tagLLMIO(span, messages, [{ content: '' }])
+      this._tagger.tagLLMIO(span, inputMessages, [{ content: '' }])
       return
     }
 
     const outputMessages = []
     const { choices } = response
     if (!isIterable(choices)) {
-      this._tagger.tagLLMIO(span, messages, [{ content: '' }])
+      this._tagger.tagLLMIO(span, inputMessages, [{ content: '' }])
       return
     }
 
+    // Output audio (non-streamed) is returned in the requested format; chat-completions
+    const outputAudioFormat = inputs.audio?.format
+
     for (const choice of choices) {
       const message = choice.message || choice.delta
-      const content = message.content || ''
+      let content = message.content || ''
       const role = message.role
 
+      const audio = message.audio
+      let audioParts
+      if (audio) {
+        if (audio.data) {
+          audioParts = [formatAudioPart(audio.data, audioMimeTypeFromFormat(outputAudioFormat, AUDIO_MIME_TYPES))]
+        }
+        // gpt-audio* / gpt-4o-audio-preview return null content; surface the transcript as text.
+        if (!content) content = audio.transcript || ''
+      }
+
+      const outputMessage = { content, role }
+      if (audioParts) outputMessage.audioParts = audioParts
+
       if (message.function_call) {
-        const functionCallInfo = {
+        outputMessage.toolCalls = [{
           name: message.function_call.name,
           arguments: safeJsonParse(message.function_call.arguments),
-        }
-        outputMessages.push({ content, role, toolCalls: [functionCallInfo] })
+        }]
       } else if (message.tool_calls) {
         const toolCallsInfo = []
         for (const toolCall of message.tool_calls) {
-          const toolCallInfo = {
+          toolCallsInfo.push({
             arguments: safeJsonParse(toolCall.function.arguments),
             name: toolCall.function.name,
             toolId: toolCall.id,
             type: toolCall.type,
-          }
-          toolCallsInfo.push(toolCallInfo)
+          })
         }
-        outputMessages.push({ content, role, toolCalls: toolCallsInfo })
-      } else {
-        outputMessages.push({ content, role })
+        outputMessage.toolCalls = toolCallsInfo
       }
+
+      outputMessages.push(outputMessage)
     }
 
-    this._tagger.tagLLMIO(span, messages, outputMessages)
+    this._tagger.tagLLMIO(span, inputMessages, outputMessages)
   }
 
   #tagResponse (span, inputs, response, error) {

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -1,10 +1,13 @@
 'use strict'
 
+const { audioMimeTypeFromFormat, formatAudioPart } = require('../../util')
 const {
   INPUT_TYPE_IMAGE,
   INPUT_TYPE_FILE,
   IMAGE_FALLBACK,
   FILE_FALLBACK,
+  AUDIO_FALLBACK,
+  AUDIO_MIME_TYPES,
 } = require('./constants')
 
 const REGEX_SPECIAL_CHARS = /[.*+?^${}()|[\]\\]/g
@@ -118,9 +121,49 @@ function hasMultimodalInputs (variables) {
   )
 }
 
+/**
+ * Flattens an array of OpenAI chat message content parts into readable text plus structured audio.
+ *
+ * Text parts are concatenated (newline-joined),
+ * images collapse to an `[image]` marker, and `input_audio` parts with data are captured as audio
+ * parts (rendered as a player). The `[audio]` marker is only emitted as a fallback when an audio
+ * part carries no data.
+ *
+ * @param {Array<object>} parts - Array of content parts from a chat message `content`
+ * @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
+ */
+function extractContentParts (parts) {
+  const extracted = []
+  const audioParts = []
+
+  for (const part of parts) {
+    const partType = part?.type ?? ''
+    if (partType === 'text') {
+      extracted.push(part.text ?? '')
+    } else if (partType === 'image_url') {
+      extracted.push(IMAGE_FALLBACK)
+    } else if (partType === 'input_audio') {
+      const inputAudio = part.input_audio ?? {}
+      const data = inputAudio.data
+      if (data) {
+        // Audio is captured as a structured audio part (rendered as a player), so no text marker
+        // is needed. Only fall back to "[audio]" when there's no audio to capture.
+        audioParts.push(formatAudioPart(data, audioMimeTypeFromFormat(inputAudio.format, AUDIO_MIME_TYPES)))
+      } else {
+        extracted.push(AUDIO_FALLBACK)
+      }
+    } else {
+      extracted.push(`[${partType}]`)
+    }
+  }
+
+  return { content: extracted.join('\n'), audioParts }
+}
+
 module.exports = {
   extractChatTemplateFromInstructions,
   normalizePromptVariables,
   extractTextFromContentItem,
+  extractContentParts,
   hasMultimodalInputs,
 }

package/packages/dd-trace/src/llmobs/sdk.js

@@ -19,6 +19,7 @@ const {
 const { storage } = require('./storage')
 const telemetry = require('./telemetry')
 const LLMObsTagger = require('./tagger')
+const { createExperiments } = require('./experiments')
 
 // communicating with writer
 const evalMetricAppendCh = channel('llmobs:eval-metric:append')
@@ -33,6 +34,12 @@ class LLMObs extends NoopLLMObs {
    */
   #hasUserSpanProcessor = false
 
+  /**
+   * Lazily-created experiments facade (see ./experiments).
+   * @type {import('./experiments').Experiments | undefined}
+   */
+  #experiments
+
   /**
    * @param {import('../tracer')} tracer - Tracer instance
    * @param {import('./index')} llmobsModule - LLMObs module instance
@@ -52,6 +59,16 @@ class LLMObs extends NoopLLMObs {
     return this._config.llmobs.DD_LLMOBS_ENABLED ?? false
   }
 
+  /**
+   * Datasets & Experiments API. Requires LLM Observability to be enabled and
+   * DD_API_KEY / DD_APP_KEY to be set; otherwise the returned facade throws with
+   * a clear message on use.
+   */
+  get experiments () {
+    this.#experiments ??= createExperiments(this._config)
+    return this.#experiments
+  }
+
   enable (options = {}) {
     logger.warn(
       'Enabling LLM Observability via `llmobs.enable()` is deprecated and will be removed in dd-trace@7.0.0. ' +

package/packages/dd-trace/src/llmobs/tagger.js

@@ -7,6 +7,7 @@ const {
   MODEL_NAME,
   MODEL_PROVIDER,
   SESSION_ID,
+  SESSION_ID_TRACE_DEFAULT_KEY,
   ML_APP,
   SPAN_KIND,
   INPUT_VALUE,
@@ -141,7 +142,10 @@ class LLMObsTagger {
     if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider)
 
     const traceTags = span.context()._trace.tags
-    sessionId = sessionId || registry.get(parent)?.[SESSION_ID] || traceTags[PROPAGATED_SESSION_ID_KEY]
+    sessionId = sessionId ||
+      registry.get(parent)?.[SESSION_ID] ||
+      traceTags[SESSION_ID_TRACE_DEFAULT_KEY] ||
+      traceTags[PROPAGATED_SESSION_ID_KEY]
     if (sessionId) this._setTag(span, SESSION_ID, sessionId)
     if (integration) this._setTag(span, INTEGRATION, integration)
     if (_decorator) this._setTag(span, DECORATOR, _decorator)
@@ -614,6 +618,65 @@ class LLMObsTagger {
     return filteredToolResults
   }
 
+  // Validates audio segments on a message and emits the snake_case wire shape
+  // `{ mime_type, content | attachment_key }`. Mirrors dd-trace-py's
+  // `_extract_audio_part`: a part requires `mimeType` and exactly one of
+  // `content` (inline base64) or `attachmentKey` (backend-offloaded).
+  #filterAudioParts (audioParts) {
+    if (!Array.isArray(audioParts)) {
+      audioParts = [audioParts]
+    }
+
+    const filteredAudioParts = []
+    for (const audioPart of audioParts) {
+      if (audioPart == null || typeof audioPart !== 'object') {
+        this.#handleFailure('Audio part must be an object.', 'invalid_io_messages')
+        continue
+      }
+
+      const { mimeType, content, attachmentKey } = audioPart
+
+      if (typeof mimeType !== 'string' || !mimeType) {
+        this.#handleFailure('Audio part mimeType must be a non-empty string.', 'invalid_io_messages')
+        continue
+      }
+
+      if (content == null && attachmentKey == null) {
+        this.#handleFailure("Audio part must have either 'content' or 'attachmentKey'.", 'invalid_io_messages')
+        continue
+      }
+
+      if (content != null && attachmentKey != null) {
+        this.#handleFailure(
+          "Audio part must have only one of 'content' or 'attachmentKey', not both.", 'invalid_io_messages'
+        )
+        continue
+      }
+
+      const audioPartObj = { mime_type: mimeType }
+
+      // Exactly one of content / attachmentKey is set here (guarded above). Validate its type
+      // explicitly so the failure carries the `invalid_io_messages` tag for telemetry, instead
+      // of routing through `#tagConditionalString` which omits it.
+      if (content == null) {
+        if (typeof attachmentKey !== 'string') {
+          this.#handleFailure('Audio part attachmentKey must be a string.', 'invalid_io_messages')
+          continue
+        }
+        audioPartObj.attachment_key = attachmentKey
+      } else {
+        if (typeof content !== 'string') {
+          this.#handleFailure('Audio part content must be a base64-encoded string.', 'invalid_io_messages')
+          continue
+        }
+        audioPartObj.content = content
+      }
+
+      filteredAudioParts.push(audioPartObj)
+    }
+    return filteredAudioParts
+  }
+
   #tagMessages (span, data, key) {
     if (!data) {
       return
@@ -640,6 +703,7 @@ class LLMObsTagger {
         toolCalls,
         toolResults,
         toolId,
+        audioParts,
       } = message
       const messageObj = {}
 
@@ -673,6 +737,14 @@ class LLMObsTagger {
         }
       }
 
+      if (audioParts != null) {
+        const filteredAudioParts = this.#filterAudioParts(audioParts)
+
+        if (filteredAudioParts.length) {
+          messageObj.audio_parts = filteredAudioParts
+        }
+      }
+
       if (toolId) {
         if (role === 'tool') {
           condition = this.#tagConditionalString(toolId, 'Tool ID', messageObj, 'tool_id') && condition
@@ -756,14 +828,14 @@ class LLMObsTagger {
     tagsCarrier[key] = value
 
     // The first session set in a trace becomes the trace-level default, stored on the trace-shared
-    // propagating tags so later spans (incl. those under a session-less parent) inherit it and it
-    // rides `x-datadog-tags` across service boundaries. Established here, the single choke point for
-    // session writes, so sessions post-populated by integrations after span start also seed it.
-    // First-writer wins, so an explicit session still overrides locally.
+    // tags so later spans (incl. those under a session-less parent) inherit it in-process.
+    // Established here, the single choke point for session writes, so sessions post-populated by
+    // integrations after span start also seed it. First-writer wins, so an explicit session still
+    // overrides locally. Cross-service injection is handled centrally in `handleLLMObsInjection`.
     if (key === SESSION_ID && value) {
       const traceTags = span.context()._trace.tags
-      if (traceTags[PROPAGATED_SESSION_ID_KEY] === undefined) {
-        traceTags[PROPAGATED_SESSION_ID_KEY] = value
+      if (traceTags[SESSION_ID_TRACE_DEFAULT_KEY] === undefined) {
+        traceTags[SESSION_ID_TRACE_DEFAULT_KEY] = value
       }
     }
   }

package/packages/dd-trace/src/llmobs/telemetry.js

@@ -3,6 +3,7 @@
 const ERROR_TYPE = require('../constants')
 
 const telemetryMetrics = require('../telemetry/metrics')
+const { getSegment } = require('../util')
 const {
   SPAN_KIND,
   MODEL_PROVIDER,
@@ -22,7 +23,7 @@ function extractIntegrationFromTags (tags) {
   if (!Array.isArray(tags)) return null
   const integrationTag = tags.find(tag => tag.startsWith('integration:'))
   if (!integrationTag) return null
-  return integrationTag.split(':')[1] || null
+  return getSegment(integrationTag, ':', 1) || null
 }
 
 function extractTagsFromSpanEvent (event) {

package/packages/dd-trace/src/llmobs/util.js

@@ -362,9 +362,41 @@ function findGenAIAncestorSpanId (span) {
   return null
 }
 
+// Maps an audio `format` (e.g. "wav", "mp3") to a MIME type. Defaults to `audio/wav` when the
+// format is missing. Provider-specific overrides (e.g. OpenAI's mp3 -> audio/mpeg) are passed in
+// via `mimeTypeLookup` so this stays provider-agnostic. A non-string `format` is treated as missing
+// so a malformed auto-instrumented payload can't throw and disable the plugin.
+/**
+ * @param {string} fmt
+ * @param {Record<string, string>} [mimeTypeLookup]
+ * @returns {string}
+ */
+function audioMimeTypeFromFormat (fmt, mimeTypeLookup = {}) {
+  fmt = typeof fmt === 'string' ? fmt.trim().toLowerCase() : ''
+  if (!fmt) return 'audio/wav'
+  return mimeTypeLookup[fmt] ?? `audio/${fmt}`
+}
+
+// Builds an audio part from raw audio bytes (base64-encoded) or an existing base64 string. Only
+// Buffer/Uint8Array inputs are base64-encoded; any other shape is passed through so a malformed
+// auto-instrumented payload can't throw (the tagger soft-skips a non-string `content`).
+/**
+ * @param {Buffer | Uint8Array | string} data
+ * @param {string} mimeType
+ * @returns {{ mimeType: string, content: string }}
+ */
+function formatAudioPart (data, mimeType) {
+  const content = Buffer.isBuffer(data) || ArrayBuffer.isView(data)
+    ? Buffer.from(data).toString('base64')
+    : data
+  return { mimeType, content }
+}
+
 module.exports = {
+  audioMimeTypeFromFormat,
   encodeUnicode,
   findGenAIAncestorSpanId,
+  formatAudioPart,
   validateCostTags,
   validateKind,
   getFunctionArguments,

package/packages/dd-trace/src/noop/proxy.js

@@ -1,8 +1,8 @@
 'use strict'
 
+const { features } = require('../feature-registry')
 const NoopAppsecSdk = require('../appsec/sdk/noop')
 const NoopLLMObsSDK = require('../llmobs/noop')
-const NoopFlaggingProvider = require('../openfeature/noop')
 const NoopAIGuardSDK = require('../aiguard/noop')
 const NoopDogStatsDClient = require('./dogstatsd')
 const NoopTracer = require('./tracer')
@@ -11,7 +11,6 @@ const noop = new NoopTracer()
 const noopAppsec = new NoopAppsecSdk()
 const noopDogStatsDClient = new NoopDogStatsDClient()
 const noopLLMObs = new NoopLLMObsSDK(noop)
-const noopOpenFeatureProvider = new NoopFlaggingProvider()
 const noopAIGuard = new NoopAIGuardSDK()
 const noopProfiling = {
   setCustomLabelKeys () {},
@@ -25,8 +24,10 @@ class NoopProxy {
     this.appsec = noopAppsec
     this.dogstatsd = noopDogStatsDClient
     this.llmobs = noopLLMObs
-    this.openfeature = noopOpenFeatureProvider
     this.aiguard = noopAIGuard
+    for (const { name, noop } of Object.values(features)) {
+      this[name] = noop
+    }
     this.setBaggageItem = (key, value) => {}
     this.getBaggageItem = (key) => {}
     this.getAllBaggageItems = () => {}
@@ -82,7 +83,7 @@ class NoopProxy {
   }
 
   inject () {
-    return this._tracer.inject.apply(this._tracer, arguments)
+    this._tracer.inject.apply(this._tracer, arguments)
   }
 
   extract () {