dd-trace 5.86.0 → 5.88.0

package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js

@@ -6,6 +6,24 @@ const { normalizeName, REDACTED_IDENTIFIERS } = require('./redaction')
 
 module.exports = {
   processRawState: processProperties,
+  processRemoteObject,
+}
+
+/**
+ * A RemoteObject with collected properties attached.
+ *
+ * @typedef {import('inspector').Runtime.RemoteObject & { properties?: object[] }} RemoteObjectWithProperties
+ */
+
+/**
+ * Process a RemoteObject into the snapshot format.
+ *
+ * @param {RemoteObjectWithProperties} remoteObject
+ * @param {number} maxLength - Maximum string length
+ * @returns {object} The processed value in snapshot format
+ */
+function processRemoteObject (remoteObject, maxLength) {
+  return getPropertyValueRaw({ value: remoteObject }, maxLength)
 }
 
 // Matches classes in source code, no matter how it's written:

package/packages/dd-trace/src/encode/agentless-json.js

@@ -0,0 +1,141 @@
+'use strict'
+
+const log = require('../log')
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+
+/**
+ * Formats a span for JSON encoding.
+ * @param {object} span - The span to format
+ * @returns {object} The formatted span
+ */
+function formatSpan (span) {
+  span = normalizeSpan(truncateSpan(span, false))
+
+  if (span.span_events) {
+    span.meta.events = JSON.stringify(span.span_events)
+    delete span.span_events
+  }
+
+  return span
+}
+
+/**
+ * Converts a span to JSON-serializable format.
+ * IDs are converted to lowercase hex strings. Start time is converted from
+ * nanoseconds to seconds for the intake format.
+ * @param {object} span - The formatted span
+ * @returns {object} JSON-serializable span object
+ */
+function spanToJSON (span) {
+  const result = {
+    trace_id: span.trace_id.toString(16).toLowerCase(),
+    span_id: span.span_id.toString(16).toLowerCase(),
+    parent_id: span.parent_id.toString(16).toLowerCase(),
+    name: span.name,
+    resource: span.resource,
+    service: span.service,
+    error: span.error,
+    start: Math.floor(span.start / 1e9),
+    duration: span.duration,
+    meta: span.meta,
+    metrics: span.metrics,
+  }
+
+  if (span.type) {
+    result.type = span.type
+  }
+
+  if (span.meta_struct) {
+    result.meta_struct = span.meta_struct
+  }
+
+  if (span.links && span.links.length > 0) {
+    result.links = span.links
+  }
+
+  return result
+}
+
+/**
+ * JSON encoder for agentless span intake.
+ * Encodes a single trace as JSON with the payload format: {"spans": [...]}
+ *
+ * This encoder handles one trace at a time since each trace must be sent as a
+ * separate request to the intake. -- bengl
+ */
+class AgentlessJSONEncoder {
+  constructor () {
+    this._reset()
+  }
+
+  /**
+   * Returns the number of spans encoded.
+   * @returns {number}
+   */
+  count () {
+    return this._spanCount
+  }
+
+  /**
+   * Encodes a trace (array of spans) into the buffer.
+   * @param {object[]} trace - Array of spans to encode
+   */
+  encode (trace) {
+    for (const span of trace) {
+      try {
+        const formattedSpan = formatSpan(span)
+        const jsonSpan = spanToJSON(formattedSpan)
+
+        this._spans.push(jsonSpan)
+        this._spanCount++
+      } catch (err) {
+        log.error(
+          'Failed to encode span (name: %s, service: %s). Span will be dropped. Error: %s\n%s',
+          span?.name || 'unknown',
+          span?.service || 'unknown',
+          err.message,
+          err.stack
+        )
+      }
+    }
+  }
+
+  /**
+   * Creates the JSON payload for the encoded trace.
+   * @returns {Buffer} JSON payload as a buffer, or empty buffer if no spans
+   */
+  makePayload () {
+    if (this._spans.length === 0) {
+      this._reset()
+      return Buffer.alloc(0)
+    }
+
+    try {
+      const payload = JSON.stringify({ spans: this._spans })
+      this._reset()
+      return Buffer.from(payload, 'utf8')
+    } catch (err) {
+      log.error(
+        'Failed to encode trace as JSON (%d spans). Trace will be dropped. Error: %s',
+        this._spans.length,
+        err.message
+      )
+      this._reset()
+      return Buffer.alloc(0)
+    }
+  }
+
+  /**
+   * Resets the encoder state.
+   */
+  reset () {
+    this._reset()
+  }
+
+  _reset () {
+    this._spans = []
+    this._spanCount = 0
+  }
+}
+
+module.exports = { AgentlessJSONEncoder }

package/packages/dd-trace/src/exporter.js

@@ -11,6 +11,8 @@ module.exports = function getExporter (name) {
       return require('./exporters/log')
     case exporters.AGENT:
       return require('./exporters/agent')
+    case exporters.AGENTLESS:
+      return require('./exporters/agentless')
     case exporters.DATADOG:
       return require('./ci-visibility/exporters/agentless')
     case exporters.AGENT_PROXY:

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -1,13 +1,12 @@
 'use strict'
 
-const { inspect } = require('util')
-
 const request = require('../common/request')
 const { startupLog } = require('../../startup-log')
 const runtimeMetrics = require('../../runtime_metrics')
 const log = require('../../log')
 const tracerVersion = require('../../../../../package.json').version
 const BaseWriter = require('../common/writer')
+const propagationHash = require('../../propagation-hash')
 
 const METRIC_PREFIX = 'datadog.tracer.node.exporter.agent'
 
@@ -29,10 +28,7 @@ class AgentWriter extends BaseWriter {
     runtimeMetrics.increment(`${METRIC_PREFIX}.requests`, true)
 
     const { _headers, _lookup, _protocolVersion, _url } = this
-    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, (err, res, status) => {
-      // Note that logging will only happen once, regardless of how many times this is called.
-      startupLog(status !== 404 && status !== 200 ? { status, message: err?.message ?? inspect(err) } : undefined)
-
+    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, true, (err, res, status, headers) => {
       if (status) {
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses`, true)
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses.by.status`, `status:${status}`, true)
@@ -53,6 +49,16 @@ class AgentWriter extends BaseWriter {
 
       log.debug('Response from the agent: %s', res)
 
+      // Capture container tags hash from agent response headers
+      // The hash is sent by the agent only when Datadog-Container-ID is present in the request
+      // (Datadog-Container-ID is automatically injected by docker.inject() in exporters/common/request.js)
+      if (headers) {
+        const containerTagsHash = headers['Datadog-Container-Tags-Hash']
+        if (containerTagsHash) {
+          propagationHash.updateContainerTagsHash(containerTagsHash)
+        }
+      }
+
       try {
         this._prioritySampler.update(JSON.parse(res).rate_by_service)
       } catch (e) {
@@ -72,7 +78,7 @@ function getEncoder (protocolVersion) {
     : require('../../encode/0.4').AgentEncoder
 }
 
-function makeRequest (version, data, count, url, headers, lookup, cb) {
+function makeRequest (version, data, count, url, headers, lookup, needsStartupLog, cb) {
   const options = {
     path: `/v${version}/traces`,
     method: 'PUT',
@@ -91,7 +97,15 @@ function makeRequest (version, data, count, url, headers, lookup, cb) {
 
   log.debug('Request to the agent: %j', options)
 
-  request(data, options, cb)
+  request(data, options, (err, res, status, headers) => {
+    if (needsStartupLog) {
+      // Note that logging will only happen once, regardless of how many times this is called.
+      startupLog({
+        agentError: status !== 404 && status !== 200 ? err : undefined,
+      })
+    }
+    cb(err, res, status, headers)
+  })
 }
 
 module.exports = AgentWriter

package/packages/dd-trace/src/exporters/agentless/index.js

@@ -0,0 +1,89 @@
+'use strict'
+
+const { URL } = require('node:url')
+
+const log = require('../../log')
+const Writer = require('./writer')
+
+/**
+ * Agentless exporter for APM span intake.
+ * Sends spans directly to the Datadog intake without requiring a local agent.
+ *
+ * Each trace is sent immediately as a separate request. The intake only accepts one trace
+ * per request - requests with spans from different traces return HTTP 200 but silently
+ * drop all spans. By flushing immediately after each export (which contains one trace),
+ * we avoid this limitation entirely. -- bengl
+ */
+class AgentlessExporter {
+  /**
+   * @param {object} config - Configuration object
+   * @param {string} [config.site='datadoghq.com'] - The Datadog site
+   * @param {string} [config.url] - Override intake URL
+   */
+  constructor (config) {
+    this._config = config
+    const { site = 'datadoghq.com', url } = config
+
+    try {
+      this._url = url ? new URL(url) : new URL(`https://public-trace-http-intake.logs.${site}`)
+    } catch (err) {
+      log.error(
+        'Invalid URL configuration for agentless exporter. url=%s, site=%s. Error: %s',
+        url || 'not set',
+        site,
+        err.message
+      )
+      this._url = null
+    }
+
+    this._writer = new Writer({
+      url: this._url,
+      site,
+    })
+
+    const ddTrace = globalThis[Symbol.for('dd-trace')]
+    if (ddTrace?.beforeExitHandlers) {
+      ddTrace.beforeExitHandlers.add(this.flush.bind(this))
+    } else {
+      log.error('dd-trace global not properly initialized. beforeExit handler not registered for agentless exporter.')
+    }
+  }
+
+  /**
+   * Sets the intake URL.
+   * @param {string} urlString - The new intake URL
+   * @returns {boolean} True if URL was set successfully
+   */
+  setUrl (urlString) {
+    try {
+      const url = new URL(urlString)
+      this._url = url
+      this._writer.setUrl(url)
+      return true
+    } catch {
+      log.error('Invalid URL for agentless exporter: %s. Using previous URL: %s', urlString, this._url?.href || 'none')
+      return false
+    }
+  }
+
+  /**
+   * Exports a trace to the intake. Flushes immediately since each trace must be
+   * sent as a separate request.
+   * @param {object[]} spans - Array of spans (all from the same trace)
+   */
+  export (spans) {
+    this._writer.append(spans)
+    this._writer.flush()
+  }
+
+  /**
+   * Flushes any pending spans. With immediate flush per trace, this is mainly
+   * used for the beforeExit handler to ensure nothing is left unsent.
+   * @param {Function} [done] - Callback when flush is complete
+   */
+  flush (done = () => {}) {
+    this._writer.flush(done)
+  }
+}
+
+module.exports = AgentlessExporter

package/packages/dd-trace/src/exporters/agentless/writer.js

@@ -0,0 +1,184 @@
+'use strict'
+
+const { getValueFromEnvSources } = require('../../config/helper')
+const log = require('../../log')
+const request = require('../common/request')
+const tracerVersion = require('../../../../../package.json').version
+
+const BaseWriter = require('../common/writer')
+const { AgentlessJSONEncoder } = require('../../encode/agentless-json')
+
+/**
+ * Writer for agentless APM span intake.
+ * Sends spans directly to the Datadog intake endpoint without an agent.
+ */
+class AgentlessWriter extends BaseWriter {
+  #apiKeyMissing = false
+
+  /**
+   * @param {object} options - Writer options
+   * @param {URL} [options.url] - The intake URL. If not provided, constructed from site.
+   * @param {string} [options.site='datadoghq.com'] - The Datadog site
+   */
+  constructor ({ url, site = 'datadoghq.com' }) {
+    super({ url })
+    this._encoder = new AgentlessJSONEncoder()
+
+    if (!url) {
+      try {
+        this._url = new URL(`https://public-trace-http-intake.logs.${site}`)
+      } catch (err) {
+        log.error(
+          'Invalid site value for agentless intake: %s. Cannot construct URL. Error: %s',
+          site,
+          err.message
+        )
+        this._url = null
+      }
+    }
+
+    if (!getValueFromEnvSources('DD_API_KEY')) {
+      this.#apiKeyMissing = true
+      log.error('DD_API_KEY is required for agentless span intake. Set DD_API_KEY. Spans will not be sent.')
+    }
+  }
+
+  /**
+   * Flushes the current trace. Since we flush after each trace, this sends
+   * a single request.
+   * @param {Function} [done] - Callback when send completes
+   */
+  flush (done = () => {}) {
+    if (!request.writable) {
+      this._encoder.reset()
+      done()
+      return
+    }
+
+    const count = this._encoder.count()
+
+    if (count === 0) {
+      done()
+      return
+    }
+
+    const payload = this._encoder.makePayload()
+
+    if (payload.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    this._sendPayload(payload, count, done)
+  }
+
+  /**
+   * Sends the encoded payload to the intake endpoint.
+   * @param {Buffer} data - The encoded JSON payload
+   * @param {number} count - Number of spans in the payload
+   * @param {Function} done - Callback when complete
+   */
+  _sendPayload (data, count, done) {
+    if (!data || data.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    if (!this._url) {
+      log.debug('Skipping send due to invalid URL configuration')
+      done()
+      return
+    }
+
+    const apiKey = getValueFromEnvSources('DD_API_KEY')
+    if (!apiKey) {
+      if (!this.#apiKeyMissing) {
+        this.#apiKeyMissing = true
+        log.error('DD_API_KEY is required for agentless span intake. Set DD_API_KEY. Spans will not be sent.')
+      }
+      log.debug('Dropping %d span(s) due to missing DD_API_KEY', count)
+      done()
+      return
+    }
+    this.#apiKeyMissing = false
+
+    const options = {
+      path: '/v1/input',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'dd-api-key': apiKey,
+        'Datadog-Meta-Lang': 'nodejs',
+        'Datadog-Meta-Lang-Version': process.version,
+        'Datadog-Meta-Lang-Interpreter': process.versions.bun ? 'JavaScriptCore' : 'v8',
+        'Datadog-Meta-Tracer-Version': tracerVersion,
+      },
+      timeout: 15_000,
+      url: this._url,
+    }
+
+    log.debug('Request to the agentless intake: %j', options)
+
+    request(data, options, (err, res, statusCode) => {
+      if (err) {
+        this._logRequestError(err, statusCode, count)
+        done()
+        return
+      }
+
+      log.debug('Response from the agentless intake: %s', res)
+      done()
+    })
+  }
+
+  /**
+   * Logs request errors with status-specific guidance.
+   * @param {Error} err - The error object
+   * @param {number} statusCode - HTTP status code (if available)
+   * @param {number} count - Number of spans that were being sent
+   */
+  _logRequestError (err, statusCode, count) {
+    if (statusCode === 401 || statusCode === 403) {
+      log.error(
+        'Authentication failed sending %d span(s) (status %s). Verify DD_API_KEY is valid.',
+        count,
+        statusCode
+      )
+    } else if (statusCode === 404) {
+      log.error(
+        'Span intake endpoint not found (status %s). Verify DD_SITE is correctly configured. %d span(s) dropped.',
+        statusCode,
+        count
+      )
+    } else if (statusCode === 429) {
+      log.error(
+        'Rate limited by span intake (status 429). %d span(s) dropped.',
+        count
+      )
+    } else if (statusCode >= 500) {
+      log.error(
+        'Span intake server error (status %s). %d span(s) dropped. This may be transient.',
+        statusCode,
+        count
+      )
+    } else if (statusCode) {
+      log.error(
+        'Error sending agentless payload (status %s): %s. %d span(s) dropped.',
+        statusCode,
+        err.message,
+        count
+      )
+    } else {
+      log.error(
+        'Network error sending %d span(s) to %s: %s',
+        count,
+        this._url?.hostname || 'unknown',
+        err.message
+      )
+    }
+  }
+}
+
+module.exports = AgentlessWriter

package/packages/dd-trace/src/exporters/common/agents.js

@@ -38,5 +38,5 @@ const HttpsAgent = createAgentClass(https.Agent)
 
 module.exports = {
   httpAgent: new HttpAgent(),
-  HttpsAgent: new HttpsAgent(),
+  httpsAgent: new HttpsAgent(),
 }

package/packages/dd-trace/src/exporters/common/request.js

@@ -88,13 +88,13 @@ function request (data, options, callback) {
           zlib.gunzip(buffer, (err, result) => {
             if (err) {
               log.error('Could not gunzip response: %s', err.message)
-              callback(null, '', res.statusCode)
+              callback(null, '', res.statusCode, res.headers)
             } else {
-              callback(null, result.toString(), res.statusCode)
+              callback(null, result.toString(), res.statusCode, res.headers)
             }
           })
         } else {
-          callback(null, buffer.toString(), res.statusCode)
+          callback(null, buffer.toString(), res.statusCode, res.headers)
         }
       } else {
         let errorMessage = ''
@@ -115,7 +115,7 @@ function request (data, options, callback) {
         const error = new log.NoTransmitError(errorMessage)
         error.status = res.statusCode
 
-        callback(error, null, res.statusCode)
+        callback(error, null, res.statusCode, res.headers)
       }
     })
   }

package/packages/dd-trace/src/llmobs/constants/writers.js

@@ -10,7 +10,7 @@ module.exports = {
 
   EVALUATIONS_INTAKE: 'api',
   EVALUATIONS_EVENT_TYPE: 'evaluation_metric',
-  EVALUATIONS_ENDPOINT: '/api/intake/llm-obs/v1/eval-metric',
+  EVALUATIONS_ENDPOINT: '/api/intake/llm-obs/v2/eval-metric',
 
   EVP_PAYLOAD_SIZE_LIMIT: 5 << 20, // 5MB (actual limit is 5.1MB)
   EVP_EVENT_SIZE_LIMIT: (1 << 20) - 1024, // 999KB (actual limit is 1MB)

package/packages/dd-trace/src/llmobs/plugins/ai/index.js

@@ -4,7 +4,7 @@ const { channel } = require('dc-polyfill')
 const BaseLLMObsPlugin = require('../base')
 const { getModelProvider } = require('../../../../../datadog-plugin-ai/src/utils')
 
-const toolCreationCh = channel('dd-trace:vercel-ai:tool')
+const toolCreationCh = channel('tracing:orchestrion:ai:tool:start')
 const setAttributesCh = channel('dd-trace:vercel-ai:span:setAttributes')
 
 const { MODEL_NAME, MODEL_PROVIDER, NAME } = require('../../constants/tags')
@@ -94,8 +94,10 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin {
 
     this.#toolCallIdsToName = {}
     this.#availableTools = new Set()
-    toolCreationCh.subscribe(toolArgs => {
-      this.#availableTools.add(toolArgs)
+    toolCreationCh.subscribe(ctx => {
+      const toolArgs = ctx.arguments
+      const tool = toolArgs[0] ?? {}
+      this.#availableTools.add(tool)
     })
 
     setAttributesCh.subscribe(({ ctx, attributes }) => {

package/packages/dd-trace/src/llmobs/sdk.js

@@ -359,15 +359,15 @@ class LLMObs extends NoopLLMObs {
         throw new Error('timestampMs must be a non-negative integer. Evaluation metric data will not be sent')
       }
 
-      const { label, value, tags } = options
+      const { label, value, tags, reasoning, assessment, metadata } = options
       const metricType = options.metricType?.toLowerCase()
       if (!label) {
         err = 'invalid_metric_label'
         throw new Error('label must be the specified name of the evaluation metric')
       }
-      if (!metricType || !['categorical', 'score', 'boolean'].includes(metricType)) {
+      if (!metricType || !['categorical', 'score', 'boolean', 'json'].includes(metricType)) {
         err = 'invalid_metric_type'
-        throw new Error('metricType must be one of "categorical" or "score"')
+        throw new Error('metricType must be one of "categorical", "score", "boolean" or "json"')
       }
       if (metricType === 'categorical' && typeof value !== 'string') {
         err = 'invalid_metric_value'
@@ -381,6 +381,22 @@ class LLMObs extends NoopLLMObs {
         err = 'invalid_metric_value'
         throw new Error('value must be a boolean for a boolean metric')
       }
+      if (metricType === 'json' && !(typeof value === 'object' && value != null && !Array.isArray(value))) {
+        err = 'invalid_metric_value'
+        throw new Error('value must be a JSON object for a json metric')
+      }
+      if (assessment != null && assessment !== 'pass' && assessment !== 'fail') {
+        err = 'invalid_assessment'
+        throw new Error('assessment must be pass or fail')
+      }
+      if (reasoning != null && typeof reasoning !== 'string') {
+        err = 'invalid_reasoning'
+        throw new Error('reasoning must be a string')
+      }
+      if (metadata != null && (typeof metadata !== 'object' || Array.isArray(metadata))) {
+        err = 'invalid_metadata'
+        throw new Error('metadata must be a JSON object')
+      }
 
       const evaluationTags = {
         'ddtrace.version': tracerVersion,
@@ -412,8 +428,12 @@ class LLMObs extends NoopLLMObs {
       }
 
       const payload = {
-        span_id: spanId,
-        trace_id: traceId,
+        join_on: {
+          span: {
+            span_id: spanId,
+            trace_id: traceId,
+          },
+        },
         label,
         metric_type: metricType,
         ml_app: mlApp,
@@ -421,6 +441,15 @@ class LLMObs extends NoopLLMObs {
         timestamp_ms: timestampMs,
         tags: Object.entries(evaluationTags).map(([key, value]) => `${key}:${value}`),
       }
+      if (reasoning != null) {
+        payload.reasoning = reasoning
+      }
+      if (metadata != null) {
+        payload.metadata = metadata
+      }
+      if (assessment != null) {
+        payload.assessment = assessment
+      }
       const currentStore = storage.getStore()
       const routing = currentStore?.routingContext
       evalMetricAppendCh.publish({ payload, routing })