dd-trace 6.11.0 → 6.13.0

package/packages/dd-trace/src/llmobs/noop.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const NoopExperiments = require('./experiments/noop')
+let NoopExperiments
 
 class NoopLLMObs {
   constructor (noopTracer) {
@@ -12,6 +12,7 @@ class NoopLLMObs {
   }
 
   get experiments () {
+    NoopExperiments ??= require('./experiments/noop')
     return new NoopExperiments('LLM Observability is not enabled')
   }
 

package/packages/dd-trace/src/llmobs/plugins/anthropic/util.js

@@ -38,16 +38,20 @@ function formatAnthropicToolResultContent (content) {
   if (typeof content === 'string') {
     return content
   } else if (Array.isArray(content)) {
-    const formattedContent = []
+    let formattedContent = ''
     for (const toolResultBlock of content) {
+      let part
       if (toolResultBlock.text) {
-        formattedContent.push(toolResultBlock.text)
+        part = toolResultBlock.text
       } else if (toolResultBlock.type === 'image') {
-        formattedContent.push('([IMAGE DETECTED])')
+        part = '([IMAGE DETECTED])'
       }
+      if (part === undefined) continue
+      if (formattedContent) formattedContent += ','
+      formattedContent += part
     }
 
-    return formattedContent.join(',')
+    return formattedContent
   }
   return JSON.stringify(content)
 }

package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js

@@ -18,12 +18,15 @@ function getToolOutputText (raw) {
   if (raw == null) return
 
   if (Array.isArray(raw)) {
-    const output = []
+    let output = ''
+    let separator = ''
     for (const block of raw) {
       const text = getToolOutputText(block)
-      if (text) output.push(text)
+      if (!text) continue
+      output += separator + text
+      separator = '\n'
     }
-    return output.join('\n') || undefined
+    return output
   }
 
   if (raw.type === 'tool_result') return getToolOutputText(raw.content)

package/packages/dd-trace/src/llmobs/plugins/openai/utils.js

@@ -134,15 +134,17 @@ function hasMultimodalInputs (variables) {
  * @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
  */
 function extractContentParts (parts) {
-  const extracted = []
+  let content = ''
+  let hasContent = false
   const audioParts = []
 
   for (const part of parts) {
     const partType = part?.type ?? ''
+    let extracted
     if (partType === 'text') {
-      extracted.push(part.text ?? '')
+      extracted = part.text ?? ''
     } else if (partType === 'image_url') {
-      extracted.push(IMAGE_FALLBACK)
+      extracted = IMAGE_FALLBACK
     } else if (partType === 'input_audio') {
       const inputAudio = part.input_audio ?? {}
       const data = inputAudio.data
@@ -151,14 +153,19 @@ function extractContentParts (parts) {
         // is needed. Only fall back to "[audio]" when there's no audio to capture.
         audioParts.push(formatAudioPart(data, audioMimeTypeFromFormat(inputAudio.format, AUDIO_MIME_TYPES)))
       } else {
-        extracted.push(AUDIO_FALLBACK)
+        extracted = AUDIO_FALLBACK
       }
     } else {
-      extracted.push(`[${partType}]`)
+      extracted = `[${partType}]`
     }
+
+    if (extracted === undefined) continue
+    if (hasContent) content += '\n'
+    content += extracted
+    hasContent = true
   }
 
-  return { content: extracted.join('\n'), audioParts }
+  return { content, audioParts }
 }
 
 /**

package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js

@@ -11,23 +11,23 @@ const { safeJsonParse } = require('../../util')
  * @returns {{ content: string, audioParts: Array<{ mimeType: string, content: string }> }}
  */
 function extractMessageContent (parts) {
-  const contentParts = []
+  let content = ''
   const audioParts = []
 
   for (const part of parts) {
     if (!part) continue
     const text = extractTextFromContentItem(part)
     if (text) {
-      contentParts.push(text)
+      content += text
       continue
     }
 
     const extracted = extractContentParts([part])
-    if (extracted.content) contentParts.push(extracted.content)
+    if (extracted.content) content += extracted.content
     if (extracted.audioParts.length > 0) audioParts.push(...extracted.audioParts)
   }
 
-  return { content: contentParts.join(''), audioParts }
+  return { content, audioParts }
 }
 
 /**
@@ -167,13 +167,11 @@ function extractOutputMessages (result) {
       if (item.type === 'message') {
         let content = ''
         if (Array.isArray(item.content)) {
-          const textParts = []
           for (const contentItem of item.content) {
             if (contentItem?.type === 'output_text' && contentItem.text) {
-              textParts.push(contentItem.text)
+              content += contentItem.text
             }
           }
-          content = textParts.join('')
         } else if (typeof item.content === 'string') {
           content = item.content
         }

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -4,6 +4,7 @@ const { URL, format } = require('node:url')
 const path = require('node:path')
 const request = require('../../exporters/common/request')
 const { getEnvironmentVariable } = require('../../config/helper')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 
 const logger = require('../../log')
 
@@ -33,6 +34,9 @@ class LLMObsBuffer {
 
 class BaseLLMObsWriter {
   #destroyer
+  /** @type {Function[]} */
+  #pendingFlushes = []
+  #serverlessDeliveryTracker = createServerlessDeliveryTracker()
   /** @type {Map<string, LLMObsBuffer>} */
   #multiTenantBuffers = new Map()
 
@@ -111,35 +115,52 @@ class BaseLLMObsWriter {
     return true
   }
 
-  flush () {
+  /**
+   * Drains buffered events and joins requests active at this flush boundary.
+   * @param {Function} [done]
+   */
+  flush (done) {
     if (this._agentless == null) {
+      if (done) this.#pendingFlushes.push(done)
       return
     }
 
-    // Flush default buffer
-    if (this._buffer.events.length > 0) {
-      const events = this._buffer.events
-      this._buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
+    const requests = this.#drainBuffers()
+    if (!this.#serverlessDeliveryTracker) {
+      // Only invocation-retaining platforms need completion-aware delivery.
+      for (const request of requests) this.#sendSafely(request)
+      done?.()
+      return
+    }
 
-      log.debug('Encoded LLMObs payload: %s', payload)
+    for (const request of requests) this.#sendSafely(request)
+    this.#serverlessDeliveryTracker.waitForIdle(done)
+  }
 
-      const options = this._getOptions()
+  #sendSafely (requestToSend) {
+    try {
+      if (this.#serverlessDeliveryTracker) {
+        this.#serverlessDeliveryTracker.track(done => this.#send(requestToSend, done))
+      } else {
+        this.#send(requestToSend)
+      }
+    } catch (error) {
+      logger.error('Failed to send LLMObs %s events: %s', this._eventType, error.message)
+    }
+  }
 
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, this.url, this._eventType)
-      })
+  #drainBuffers () {
+    const requests = []
+    if (this._buffer.events.length > 0) {
+      const events = this._buffer.events
+      this._buffer.clear()
+      requests.push({ events, options: this._getOptions(), url: this.url })
     }
 
-    // Flush multi-tenant buffers
     for (const [apiKey, buffer] of this.#multiTenantBuffers) {
       if (buffer.events.length === 0) continue
-
       const events = buffer.events
       buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
       const site = buffer.routing.site || this._config.site
       const options = {
         headers: {
@@ -156,16 +177,21 @@ class BaseLLMObsWriter {
       }
       const url = this.#buildUrl(options.url.href, options.path)
       const maskedApiKey = apiKey ? `****${apiKey.slice(-4)}` : ''
-
       log.debug('Encoding and flushing multi-tenant buffer for %s', maskedApiKey)
-      log.debug('Encoded LLMObs payload: %s', payload)
-
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, url, this._eventType)
-      })
+      requests.push({ events, options, url })
     }
 
     this.#cleanupEmptyBuffers()
+    return requests
+  }
+
+  #send ({ events, options, url }, done) {
+    const payload = this._encode(this.makePayload(events))
+    log.debug('Encoded LLMObs payload: %s', payload)
+    request(payload, options, (err, resp, code) => {
+      parseResponseAndLog(err, code, events.length, url, this._eventType)
+      done?.()
+    })
   }
 
   #cleanupEmptyBuffers () {
@@ -178,13 +204,19 @@ class BaseLLMObsWriter {
 
   makePayload (events) {}
 
-  destroy () {
+  /**
+   * Stops periodic flushing and drains buffered events.
+   * @param {Function} [done] Called after queued and active deliveries complete.
+   */
+  destroy (done) {
     if (this.#destroyer) {
       logger.debug(`Stopping ${this.constructor.name}`)
       clearInterval(this._periodic)
       globalThis[Symbol.for('dd-trace')].beforeExitHandlers.delete(this.#destroyer)
-      this.flush()
+      this.flush(done)
       this.#destroyer = undefined
+    } else {
+      done?.()
     }
   }
 
@@ -196,6 +228,10 @@ class BaseLLMObsWriter {
     this._endpoint = endpoint
 
     logger.debug(`Configuring ${this.constructor.name} to ${this.url}`)
+
+    const pendingFlushes = this.#pendingFlushes
+    this.#pendingFlushes = []
+    for (const done of pendingFlushes) this.flush(done)
   }
 
   _getUrlAndPath () {

package/packages/dd-trace/src/noop/span.js

@@ -26,6 +26,11 @@ class NoopSpan {
   addLink (link) { return this }
   addLinks (links) { return this }
   addSpanPointer (ptrKind, ptrDir, ptrHash) { return this }
+  /**
+   * @param {import('../../../../index').Exception} exception
+   * @param {import('../../../../index').SpanEventAttributes} [attributes]
+   */
+  recordException (exception, attributes) {}
   log () { return this }
   logEvent () {}
   finish (finishTime) {}

package/packages/dd-trace/src/openfeature/constants/constants.js

@@ -1,24 +1,6 @@
 'use strict'
 
 module.exports = {
-  /**
-   * @constant
-   * @type {string} Base path for EVP proxy agent endpoint
-   */
-  EVP_PROXY_AGENT_BASE_PATH: '/evp_proxy/v2/',
-
-  /**
-   * @constant
-   * @type {string} HTTP header name for EVP subdomain routing
-   */
-  EVP_SUBDOMAIN_HEADER_NAME: 'X-Datadog-EVP-Subdomain',
-
-  /**
-   * @constant
-   * @type {string} EVP subdomain value for event platform intake
-   */
-  EVP_SUBDOMAIN_VALUE: 'event-platform-intake',
-
   /**
    * @constant
    * @type {string} API endpoint for exposure events EVP track

package/packages/dd-trace/src/openfeature/index.js

@@ -3,7 +3,7 @@
 const { channel } = require('dc-polyfill')
 const log = require('../log')
 const ExposuresWriter = require('./writers/exposures')
-const { setAgentStrategy } = require('./writers/util')
+const { setExposureDeliveryStrategy } = require('./writers/util')
 
 const exposureSubmitCh = channel('ffe:exposure:submit')
 const flushCh = channel('ffe:writers:flush')
@@ -40,12 +40,15 @@ function enable (config) {
     return
   }
 
-  exposuresWriter = new ExposuresWriter(config)
+  const writer = new ExposuresWriter(config)
+  exposuresWriter = writer
   exposureSubmitCh.subscribe(_handleExposureSubmit)
   flushCh.subscribe(_handleFlush)
 
-  setAgentStrategy(config, hasAgent => {
-    exposuresWriter?.setEnabled(hasAgent)
+  setExposureDeliveryStrategy(config, (enabled, route) => {
+    if (exposuresWriter !== writer) return
+
+    writer.setEnabled(enabled, route)
   })
 }
 

package/packages/dd-trace/src/openfeature/writers/base.js

@@ -11,14 +11,52 @@ const log = require('../../log')
  * @property {number} [timeout] - Request timeout in milliseconds
  * @property {object} config - Tracer configuration object
  * @property {string} endpoint - API endpoint path
- * @property {URL} [agentUrl] - Base URL for the agent
+ * @property {URL} [agentUrl] - Initial delivery URL
  * @property {number} [payloadSizeLimit] - Maximum payload size in bytes
  * @property {number} [eventSizeLimit] - Maximum individual event size in bytes
  * @property {object} [headers] - Additional HTTP headers
  */
 
 /**
- * BaseFFEWriter is the base class for sending Feature Flagging & Exposure Events payloads to the Datadog Agent.
+ * @typedef {object} WriterRoute
+ * @property {URL} url - Route base URL
+ * @property {string} endpoint - Route endpoint
+ * @property {object} headers - Route-specific headers
+ * @property {import('node:https').Agent} [agent] - Optional HTTPS proxy agent
+ */
+
+/**
+ * @typedef {object} ActiveWriterRoute
+ * @property {URL} url - Route base URL
+ * @property {string} endpoint - Route endpoint
+ * @property {object} requestOptions - HTTP request options
+ */
+
+/**
+ * Tests whether a local route definitively rejected an event batch.
+ *
+ * @param {Error | null} error - Request error
+ * @param {number | undefined} statusCode - HTTP response status
+ * @returns {boolean} Whether direct retry is safe
+ */
+function isDefinitiveRejection (error, statusCode) {
+  return error?.code === 'EAI_AGAIN' || error?.code === 'ECONNREFUSED' ||
+    error?.code === 'ENOENT' || error?.code === 'ENOTFOUND' ||
+    statusCode === 403 || statusCode === 404 || statusCode === 405
+}
+
+/**
+ * Tests whether a local route can have accepted an event batch before failing.
+ *
+ * @param {Error | null} error - Request error
+ * @returns {boolean} Whether the delivery result is ambiguous
+ */
+function isAmbiguousNetworkFailure (error) {
+  return error?.code === 'ECONNRESET' || error?.code === 'EPIPE' || error?.code === 'ETIMEDOUT'
+}
+
+/**
+ * Base writer for Feature Flagging and Experimentation event delivery.
  * @class BaseFFEWriter
  */
 class BaseFFEWriter {
@@ -40,6 +78,7 @@ class BaseFFEWriter {
     this._payloadSizeLimit = payloadSizeLimit
     this._eventSizeLimit = eventSizeLimit
     this._headers = headers || {}
+    this._fallbackRoute = undefined
 
     this._requestOptions = {
       headers: {
@@ -47,6 +86,7 @@ class BaseFFEWriter {
         'Content-Type': 'application/json',
       },
       method: 'POST',
+      retry: true,
       timeout: this._timeout,
       url: this._baseUrl,
       path: this._endpoint,
@@ -115,15 +155,8 @@ class BaseFFEWriter {
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
     log.debug(() => `${this.constructor.name} flushing payload: ${safeJSONStringify(payload)}`)
 
-    request(payload, this._requestOptions, (err, resp, code) => {
-      if (err) {
-        log.error('Failed to send events to %s%s', this._baseUrl.href, this._endpoint, err)
-      } else if (code >= 200 && code < 300) {
-        log.debug('Successfully sent %d events', events.length)
-      } else {
-        log.warn('Events request returned status %d', code)
-      }
-    })
+    const route = this.#createActiveRoute()
+    this.#sendRequest(payload, events.length, route, this._fallbackRoute)
   }
 
   /**
@@ -161,6 +194,115 @@ class BaseFFEWriter {
   _encode (payload) {
     return JSON.stringify(payload)
   }
+
+  /**
+   * Applies the active route and an optional direct fallback route.
+   *
+   * @param {WriterRoute} route - Active route
+   * @param {WriterRoute} [fallbackRoute] - Direct fallback route
+   * @returns {void}
+   */
+  _setRoutes (route, fallbackRoute) {
+    this.#activateRoute(this.#createRoute(route))
+    this._fallbackRoute = fallbackRoute ? this.#createRoute(fallbackRoute) : undefined
+  }
+
+  /**
+   * Creates request state for a configured writer route.
+   *
+   * @param {WriterRoute} route - Configured route
+   * @returns {ActiveWriterRoute} Active route state
+   */
+  #createRoute (route) {
+    return {
+      url: route.url,
+      endpoint: route.endpoint,
+      requestOptions: {
+        ...(route.agent && { agent: route.agent }),
+        headers: {
+          ...route.headers,
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+        retry: true,
+        timeout: this._timeout,
+        url: route.url,
+        path: route.endpoint,
+      },
+    }
+  }
+
+  /**
+   * Captures the current route for one event batch.
+   *
+   * @returns {ActiveWriterRoute} Active route state
+   */
+  #createActiveRoute () {
+    return {
+      url: this._baseUrl,
+      endpoint: this._endpoint,
+      requestOptions: this._requestOptions,
+    }
+  }
+
+  /**
+   * Makes a route active for future event batches.
+   *
+   * @param {ActiveWriterRoute} route - Route state
+   * @returns {void}
+   */
+  #activateRoute (route) {
+    this._baseUrl = route.url
+    this._endpoint = route.endpoint
+    this._requestOptions = route.requestOptions
+  }
+
+  /**
+   * Sends an encoded batch and retries it through direct intake after a local route failure.
+   *
+   * @param {string} payload - Encoded event batch
+   * @param {number} eventCount - Event count
+   * @param {ActiveWriterRoute} route - Selected route
+   * @param {ActiveWriterRoute} [fallbackRoute] - Direct fallback route
+   * @returns {void}
+   */
+  #sendRequest (payload, eventCount, route, fallbackRoute) {
+    request(payload, route.requestOptions, (error, response, statusCode) => {
+      if (fallbackRoute && isDefinitiveRejection(error, statusCode)) {
+        log.debug(
+          '%s switching from %s%s to direct intake after definitive rejection',
+          this.constructor.name,
+          route.url.href,
+          route.endpoint
+        )
+        this.#activateRoute(fallbackRoute)
+        this._fallbackRoute = undefined
+        this.#sendRequest(payload, eventCount, fallbackRoute)
+        return
+      }
+
+      if (fallbackRoute && isAmbiguousNetworkFailure(error)) {
+        log.debug(
+          '%s retrying through direct intake and switching future batches from %s%s after ambiguous failure',
+          this.constructor.name,
+          route.url.href,
+          route.endpoint
+        )
+        this.#activateRoute(fallbackRoute)
+        this._fallbackRoute = undefined
+        this.#sendRequest(payload, eventCount, fallbackRoute)
+        return
+      }
+
+      if (error) {
+        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
+      } else if (statusCode >= 200 && statusCode < 300) {
+        log.debug('Successfully sent %d events', eventCount)
+      } else {
+        log.warn('Events request returned status %d', statusCode)
+      }
+    })
+  }
 }
 
 module.exports = BaseFFEWriter