dd-trace 6.0.0-pre-0bb1f17 → 6.0.0

package/packages/dd-trace/src/llmobs/plugins/{openai.js → openai/index.js}

@@ -1,12 +1,25 @@
 'use strict'
 
-const LLMObsPlugin = require('./base')
+const LLMObsPlugin = require('../base')
+const {
+  PROMPT_TRACKING_INSTRUMENTATION_METHOD,
+  PROMPT_MULTIMODAL,
+  INSTRUMENTATION_METHOD_AUTO,
+  UNKNOWN_MODEL_PROVIDER,
+} = require('../../constants/tags')
+const { safeJsonParse } = require('../../util')
+const {
+  extractChatTemplateFromInstructions,
+  normalizePromptVariables,
+  extractTextFromContentItem,
+  hasMultimodalInputs,
+} = require('./utils')
 
 const allowedParamKeys = new Set([
   'max_output_tokens',
   'temperature',
   'stream',
-  'reasoning'
+  'reasoning',
 ])
 
 function isIterable (obj) {
@@ -38,7 +51,7 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
       modelProvider,
       modelName: inputs.model,
       kind,
-      name
+      name,
     }
   }
 
@@ -50,7 +63,7 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
 
     const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument
     const response = ctx.result?.data // no result if error
-    const error = !!span.context()._tags.error
+    const error = !!span.context().getTag('error')
 
     const operation = getOperation(methodName)
 
@@ -79,10 +92,12 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
   _getModelProviderAndClient (baseUrl = '') {
     if (baseUrl.includes('azure')) {
       return { modelProvider: 'azure_openai', client: 'AzureOpenAI' }
+    } else if (baseUrl.includes('openai')) {
+      return { modelProvider: 'openai', client: 'OpenAI' }
     } else if (baseUrl.includes('deepseek')) {
       return { modelProvider: 'deepseek', client: 'DeepSeek' }
     }
-    return { modelProvider: 'openai', client: 'OpenAI' }
+    return { modelProvider: UNKNOWN_MODEL_PROVIDER, client: 'OpenAI' }
   }
 
   _extractMetrics (response) {
@@ -115,6 +130,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
           metrics.cacheReadTokens = cacheReadTokens
         }
       }
+      // Reasoning tokens - Responses API returns `output_tokens_details`, `completion_tokens_details`
+      const reasoningOutputObject = tokenUsage.output_tokens_details ?? tokenUsage.completion_tokens_details
+      const reasoningOutputTokens = reasoningOutputObject?.reasoning_tokens ?? 0
+      if (reasoningOutputTokens !== undefined) metrics.reasoningOutputTokens = reasoningOutputTokens
     }
 
     return metrics
@@ -124,7 +143,7 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
     const { model, ...parameters } = inputs
 
     const metadata = {
-      encoding_format: parameters.encoding_format || 'float'
+      encoding_format: parameters.encoding_format || 'float',
     }
     if (inputs.dimensions) metadata.dimensions = inputs.dimensions
     this._tagger.tagMetadata(span, metadata)
@@ -165,13 +184,12 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
   _tagChatCompletion (span, inputs, response, error) {
     const { messages, model, ...parameters } = inputs
 
-    const metadata = Object.entries(parameters).reduce((obj, [key, value]) => {
-      if (!['tools', 'functions'].includes(key)) {
-        obj[key] = value
+    const metadata = {}
+    for (const key of Object.keys(parameters)) {
+      if (key !== 'tools' && key !== 'functions') {
+        metadata[key] = parameters[key]
       }
-
-      return obj
-    }, {})
+    }
 
     this._tagger.tagMetadata(span, metadata)
 
@@ -195,17 +213,17 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
       if (message.function_call) {
         const functionCallInfo = {
           name: message.function_call.name,
-          arguments: JSON.parse(message.function_call.arguments)
+          arguments: safeJsonParse(message.function_call.arguments),
         }
         outputMessages.push({ content, role, toolCalls: [functionCallInfo] })
       } else if (message.tool_calls) {
         const toolCallsInfo = []
         for (const toolCall of message.tool_calls) {
           const toolCallInfo = {
-            arguments: JSON.parse(toolCall.function.arguments),
+            arguments: safeJsonParse(toolCall.function.arguments),
             name: toolCall.function.name,
             toolId: toolCall.id,
-            type: toolCall.type
+            type: toolCall.type,
           }
           toolCallsInfo.push(toolCallInfo)
         }
@@ -221,7 +239,8 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
   #tagResponse (span, inputs, response, error) {
     // Tag metadata - use allowlist approach for request parameters
 
-    const { input, model, ...parameters } = inputs
+    const { model, ...parameters } = inputs
+    let input = inputs.input
 
     // Create input messages
     const inputMessages = []
@@ -231,28 +250,41 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
       inputMessages.push({ role: 'system', content: inputs.instructions })
     }
 
+    // For reusable prompts, use response.instructions if no explicit input is provided
+    if (!input && inputs.prompt && response?.instructions) {
+      input = response.instructions
+    }
+
     // Handle input - can be string or array of mixed messages
     if (Array.isArray(input)) {
       for (const item of input) {
-        if (item.type === 'function_call') {
-          // Function call: convert to message with tool_calls
-          // Parse arguments if it's a JSON string
-          let parsedArgs = item.arguments
-          if (typeof parsedArgs === 'string') {
-            try {
-              parsedArgs = JSON.parse(parsedArgs)
-            } catch {
-              parsedArgs = {}
-            }
+        if (item.type === 'message') {
+          // Handle instruction messages (from response.instructions for reusable prompts)
+          const role = item.role
+          if (!role) continue
+
+          let content = ''
+          if (Array.isArray(item.content)) {
+            const textParts = item.content
+              .map(extractTextFromContentItem)
+              .filter(Boolean)
+            content = textParts.join('')
+          } else if (typeof item.content === 'string') {
+            content = item.content
           }
+
+          if (content) {
+            inputMessages.push({ role, content })
+          }
+        } else if (item.type === 'function_call') {
           inputMessages.push({
             role: 'assistant',
             toolCalls: [{
               toolId: item.call_id,
               name: item.name,
-              arguments: parsedArgs,
-              type: item.type
-            }]
+              arguments: safeJsonParse(item.arguments, {}),
+              type: item.type,
+            }],
           })
         } else if (item.type === 'function_call_output') {
           // Function output: convert to user message with tool_results
@@ -262,8 +294,8 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
               toolId: item.call_id,
               result: item.output,
               name: item.name || '',
-              type: item.type
-            }]
+              type: item.type,
+            }],
           })
         } else if (item.role && item.content) {
           // Regular message
@@ -275,12 +307,12 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
       inputMessages.push({ role: 'user', content: input })
     }
 
-    const inputMetadata = Object.entries(parameters).reduce((obj, [key, value]) => {
+    const inputMetadata = {}
+    for (const key of Object.keys(parameters)) {
       if (allowedParamKeys.has(key)) {
-        obj[key] = value
+        inputMetadata[key] = parameters[key]
       }
-      return obj
-    }, {})
+    }
 
     this._tagger.tagMetadata(span, inputMetadata)
 
@@ -307,28 +339,19 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
             content: JSON.stringify({
               summary: item.summary ?? [],
               encrypted_content: item.encrypted_content ?? null,
-              id: item.id ?? ''
-            })
+              id: item.id ?? '',
+            }),
           })
         } else if (item.type === 'function_call') {
           // Handle function_call type (responses API tool calls)
-          let args = item.arguments
-          // Parse arguments if it's a JSON string
-          if (typeof args === 'string') {
-            try {
-              args = JSON.parse(args)
-            } catch {
-              args = {}
-            }
-          }
           outputMessages.push({
             role: 'assistant',
             toolCalls: [{
               toolId: item.call_id,
               name: item.name,
-              arguments: args,
-              type: item.type
-            }]
+              arguments: safeJsonParse(item.arguments, {}),
+              type: item.type,
+            }],
           })
         } else {
           // Handle regular message objects
@@ -348,23 +371,12 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
 
           // Extract tool calls if present in message.tool_calls
           if (Array.isArray(item.tool_calls)) {
-            outputMsg.toolCalls = item.tool_calls.map(tc => {
-              let args = tc.function?.arguments || tc.arguments
-              // Parse arguments if it's a JSON string
-              if (typeof args === 'string') {
-                try {
-                  args = JSON.parse(args)
-                } catch {
-                  args = {}
-                }
-              }
-              return {
-                toolId: tc.id,
-                name: tc.function?.name || tc.name,
-                arguments: args,
-                type: tc.type || 'function_call'
-              }
-            })
+            outputMsg.toolCalls = item.tool_calls.map(tc => ({
+              toolId: tc.id,
+              name: tc.function?.name || tc.name,
+              arguments: safeJsonParse(tc.function?.arguments || tc.arguments, {}),
+              type: tc.type || 'function_call',
+            }))
           }
 
           outputMessages.push(outputMsg)
@@ -380,6 +392,26 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
 
     this._tagger.tagLLMIO(span, inputMessages, outputMessages)
 
+    // Handle prompt tracking for reusable prompts
+    if (inputs.prompt && response?.prompt) {
+      const { id, version } = response.prompt // ResponsePrompt
+      if (id && version) {
+        const normalizedVariables = normalizePromptVariables(inputs.prompt.variables)
+        const chatTemplate = extractChatTemplateFromInstructions(response.instructions, normalizedVariables)
+        this._tagger.tagPrompt(span, {
+          id,
+          version,
+          variables: normalizedVariables,
+          template: chatTemplate,
+        }, true)
+        const tags = { [PROMPT_TRACKING_INSTRUMENTATION_METHOD]: INSTRUMENTATION_METHOD_AUTO }
+        if (hasMultimodalInputs(inputs.prompt.variables)) {
+          tags[PROMPT_MULTIMODAL] = 'true'
+        }
+        this._tagger.tagSpanTags(span, tags)
+      }
+    }
+
     const outputMetadata = {}
 
     // Add fields from response object (convert numbers to floats)
@@ -388,9 +420,6 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
     if (response.tool_choice !== undefined) outputMetadata.tool_choice = response.tool_choice
     if (response.truncation !== undefined) outputMetadata.truncation = response.truncation
     if (response.text !== undefined) outputMetadata.text = response.text
-    if (response.usage?.output_tokens_details?.reasoning_tokens !== undefined) {
-      outputMetadata.reasoning_tokens = response.usage.output_tokens_details.reasoning_tokens
-    }
 
     this._tagger.tagMetadata(span, outputMetadata) // update the metadata with the output metadata
   }

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -0,0 +1,126 @@
+'use strict'
+
+const {
+  INPUT_TYPE_IMAGE,
+  INPUT_TYPE_FILE,
+  IMAGE_FALLBACK,
+  FILE_FALLBACK,
+} = require('./constants')
+
+const REGEX_SPECIAL_CHARS = /[.*+?^${}()|[\]\\]/g
+
+/**
+ * Extracts chat templates from OpenAI response instructions by replacing variable values with placeholders.
+ *
+ * Performs reverse templating: reconstructs the template by replacing actual values with {{variable_name}}.
+ * For images/files: uses {{variable_name}} when values are available, falls back to [image]/[file] when stripped.
+ *
+ * @param {Array<object>} instructions - From Response.instructions (array of ResponseInputMessageItem)
+ * @param {Record<string, string>} variables - Normalized variables (output of normalizePromptVariables)
+ * @returns {Array<{role: string, content: string}>} Chat template with placeholders
+ */
+function extractChatTemplateFromInstructions (instructions, variables) {
+  if (!Array.isArray(instructions) || !variables) return []
+
+  const chatTemplate = []
+
+  // Build map of values to placeholders - exclude fallback markers so they remain as-is
+  const valueToPlaceholder = {}
+  for (const [varName, varValue] of Object.entries(variables)) {
+    // Exclude fallback markers - they should remain as [image]/[file] in the template
+    if (varValue && varValue !== IMAGE_FALLBACK && varValue !== FILE_FALLBACK) {
+      valueToPlaceholder[varValue] = `{{${varName}}}`
+    }
+  }
+
+  // Sort values by length (longest first) to handle overlapping values correctly
+  const sortedValues = Object.keys(valueToPlaceholder).sort((a, b) => b.length - a.length)
+
+  for (const instruction of instructions) {
+    const role = instruction.role
+    if (!role) continue
+
+    const contentItems = instruction.content
+    if (!Array.isArray(contentItems)) continue
+
+    // Extract text from all content items (uses actual values for images/files when available)
+    const textParts = contentItems
+      .map(extractTextFromContentItem)
+      .filter(Boolean)
+
+    if (textParts.length === 0) continue
+
+    // Combine text and replace variable values with placeholders (longest first)
+    let fullText = textParts.join('')
+    for (const valueStr of sortedValues) {
+      const placeholder = valueToPlaceholder[valueStr]
+      const escapedValue = valueStr.replaceAll(REGEX_SPECIAL_CHARS, String.raw`\$&`)
+      fullText = fullText.replaceAll(new RegExp(escapedValue, 'g'), placeholder)
+    }
+
+    chatTemplate.push({ role, content: fullText })
+  }
+
+  return chatTemplate
+}
+
+/**
+ * Extracts text content from a content item, using actual image_url/file_id values when available.
+ *
+ * Used for both input messages and chat template extraction. Falls back to [image]/[file] markers
+ * when the actual values are stripped (e.g., by OpenAI's default URL stripping behavior).
+ *
+ * @param {object} contentItem - Content item from Response.instructions[].content (ResponseInputContentItem)
+ * @returns {string|null} Text content, URL/file reference, or [image]/[file] fallback marker
+ */
+function extractTextFromContentItem (contentItem) {
+  if (!contentItem) return null
+
+  if (contentItem.text) {
+    return contentItem.text
+  }
+
+  // For image/file items, extract the actual reference value
+  if (contentItem.type === INPUT_TYPE_IMAGE) {
+    return contentItem.image_url || contentItem.file_id || IMAGE_FALLBACK
+  }
+
+  if (contentItem.type === INPUT_TYPE_FILE) {
+    return contentItem.file_id || contentItem.file_url || contentItem.filename || FILE_FALLBACK
+  }
+
+  return null
+}
+
+/**
+ * Normalizes prompt variables by extracting meaningful values from OpenAI SDK response objects.
+ *
+ * Converts ResponseInputText, ResponseInputImage, and ResponseInputFile objects to simple string values.
+ *
+ * @param {Record<string, string | object>} variables - From ResponsePrompt.variables
+ * @returns {Record<string, string>} Normalized variables with simple string values
+ */
+function normalizePromptVariables (variables) {
+  if (!variables) return {}
+
+  return Object.fromEntries(
+    Object.entries(variables).map(([key, value]) => [
+      key,
+      extractTextFromContentItem(value) ?? String(value ?? ''),
+    ])
+  )
+}
+
+function hasMultimodalInputs (variables) {
+  if (!variables) return false
+  return Object.values(variables).some(value =>
+    value?.type === INPUT_TYPE_IMAGE || value?.type === INPUT_TYPE_FILE
+  )
+}
+
+module.exports = {
+  extractChatTemplateFromInstructions,
+  normalizePromptVariables,
+  extractTextFromContentItem,
+  hasMultimodalInputs,
+}

package/packages/dd-trace/src/llmobs/plugins/vertexai.js

@@ -1,10 +1,10 @@
 'use strict'
 
-const LLMObsPlugin = require('./base')
 const {
   extractModel,
-  extractSystemInstructions
+  extractSystemInstructions,
 } = require('../../../../datadog-plugin-google-cloud-vertexai/src/utils')
+const LLMObsPlugin = require('./base')
 
 class VertexAILLMObsPlugin extends LLMObsPlugin {
   static integration = 'vertexai' // used for llmobs telemetry
@@ -19,7 +19,7 @@ class VertexAILLMObsPlugin extends LLMObsPlugin {
       kind: 'llm',
       modelName: extractModel(ctx.instance),
       modelProvider: 'google',
-      name: ctx.resource
+      name: ctx.resource,
     }
   }
 
@@ -53,7 +53,7 @@ function getMetadata (instance) {
     ['maxOutputTokens', 'max_output_tokens'],
     ['candidateCount', 'candidate_count'],
     ['topP', 'top_p'],
-    ['topK', 'top_k']
+    ['topK', 'top_k'],
   ]) {
     if (modelConfig[parameter]) {
       metadata[parameterKey] = modelConfig[parameter]
@@ -90,7 +90,7 @@ function extractInputMessages (request, history, systemInstructions) {
   if (!Array.isArray(contents)) {
     messages.push({
       content: '[Non-array content object: ' +
-      `${(typeof contents.toString === 'function' ? contents.toString() : String(contents))}]`
+      `${(typeof contents.toString === 'function' ? contents.toString() : String(contents))}]`,
     })
     return messages
   }
@@ -136,7 +136,7 @@ function extractMessagesFromContent (content) {
   if (parts == null || parts.length === 0 || !Array.isArray(parts)) {
     const message = {
       content:
-      `[Non-text content object: ${(typeof content.toString === 'function' ? content.toString() : String(content))}]`
+      `[Non-text content object: ${(typeof content.toString === 'function' ? content.toString() : String(content))}]`,
     }
     if (role) message.role = role
     messages.push(message)
@@ -161,7 +161,7 @@ function extractMessageFromPart (part, role) {
   if (functionCall) {
     message.toolCalls = [{
       name: functionCall.name,
-      arguments: functionCall.args
+      arguments: functionCall.args,
     }]
   }
   if (functionResponse) {