dd-trace 5.87.0 → 5.88.0

package/packages/dd-trace/src/constants.js

@@ -46,8 +46,6 @@ module.exports = {
   SCHEMA_TOPIC: 'schema.topic',
   SCHEMA_OPERATION: 'schema.operation',
   SCHEMA_NAME: 'schema.name',
-  GRPC_CLIENT_ERROR_STATUSES: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
-  GRPC_SERVER_ERROR_STATUSES: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
   DYNAMODB_PTR_KIND: 'aws.dynamodb.item',
   S3_PTR_KIND: 'aws.s3.object',
   WEBSOCKET_PTR_KIND: 'websocket',

package/packages/dd-trace/src/crashtracking/crashtracker.js

@@ -52,6 +52,12 @@ class Crashtracker {
   #getConfig (config) {
     const url = getAgentUrl(config)
 
+    // Out-of-process symbolication currently (crashtracker 27.0.0) works on
+    // Linux only, does not work on Mac.
+    const resolveMode = require('os').platform === 'linux'
+      ? 'EnabledWithSymbolsInReceiver'
+      : 'EnabledWithInprocessSymbols'
+
     return {
       additional_files: [],
       create_alt_stack: true,
@@ -67,9 +73,10 @@ class Crashtracker {
         },
         timeout_ms: 3000,
       },
-      timeout_ms: 5000,
-      // TODO: Use `EnabledWithSymbolsInReceiver` instead for Linux when fixed.
-      resolve_frames: 'EnabledWithInprocessSymbols',
+      timeout: { secs: 5, nanos: 0 },
+      demangle_names: false,
+      signals: [],
+      resolve_frames: resolveMode,
     }
   }
 

package/packages/dd-trace/src/datastreams/pathway.js

@@ -26,17 +26,36 @@ function shaHash (checkpointString) {
  * @param {string} env
  * @param {string[]} edgeTags
  * @param {Buffer} parentHash
+ * @param {bigint | null} propagationHashBigInt - Optional propagation hash for process/container tags
  */
-function computeHash (service, env, edgeTags, parentHash) {
+function computeHash (service, env, edgeTags, parentHash, propagationHashBigInt = null) {
   edgeTags.sort()
   const hashableEdgeTags = edgeTags.filter(item => item !== 'manual_checkpoint:true')
 
-  const key = `${service}${env}${hashableEdgeTags.join('')}${parentHash}`
+  // Cache key includes parentHash to handle fan-in/fan-out scenarios where the same
+  // service+env+tags+propagationHash can have different parents. This ensures we cache
+  // the complete pathway context, not just the current node's identity.
+  const propagationPart = propagationHashBigInt ? `:${propagationHashBigInt.toString(16)}` : ''
+  const key = `${service}${env}${hashableEdgeTags.join('')}${parentHash}${propagationPart}`
+
   let value = cache.get(key)
   if (value) {
     return value
   }
-  const currentHash = shaHash(`${service}${env}` + hashableEdgeTags.join(''))
+
+  // Key vs hashInput distinction:
+  // - 'key' (above) is used for caching and includes parentHash to differentiate pathways
+  //   with the same node but different parents (e.g., multiple queues feeding one consumer)
+  // - 'hashInput' (below) excludes parentHash to compute only the current node's identity hash,
+  //   which is then XORed with parentHash (line 54) to build the complete pathway hash
+  // This two-step approach (hash current node independently, then combine with parent) is
+  // required for proper pathway construction in the DSM protocol.
+  const baseString = `${service}${env}` + hashableEdgeTags.join('')
+  const hashInput = propagationHashBigInt
+    ? `${baseString}:${propagationHashBigInt.toString(16)}`
+    : baseString
+
+  const currentHash = shaHash(hashInput)
   const buf = Buffer.concat([currentHash, parentHash], 16)
   value = shaHash(buf.toString())
   cache.set(key, value)

package/packages/dd-trace/src/datastreams/processor.js

@@ -6,6 +6,8 @@ const pkg = require('../../../../package.json')
 const { LogCollapsingLowestDenseDDSketch } = require('../../../../vendor/dist/@datadog/sketches-js')
 const { PATHWAY_HASH } = require('../../../../ext/tags')
 const log = require('../log')
+const processTags = require('../process-tags')
+const propagationHash = require('../propagation-hash')
 const { DsmPathwayCodec } = require('./pathway')
 const { DataStreamsWriter } = require('./writer')
 const { computePathwayHash } = require('./pathway')
@@ -162,6 +164,7 @@ class DataStreamsProcessor {
   onInterval () {
     const { Stats } = this._serializeBuckets()
     if (Stats.length === 0) return
+
     const payload = {
       Env: this.env,
       Service: this.service,
@@ -171,6 +174,12 @@ class DataStreamsProcessor {
       Lang: 'javascript',
       Tags: Object.entries(this.tags).map(([key, value]) => `${key}:${value}`),
     }
+
+    // Add ProcessTags only if feature is enabled and process tags exist
+    if (propagationHash.isEnabled() && processTags.serialized) {
+      payload.ProcessTags = processTags.serialized.split(',')
+    }
+
     this.writer.flush(payload)
   }
 
@@ -234,7 +243,11 @@ class DataStreamsProcessor {
         edgeTags
       )
     }
-    const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash)
+
+    // Get propagation hash if enabled
+    const propagationHashValue = propagationHash.isEnabled() ? propagationHash.getHash() : null
+
+    const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash, propagationHashValue)
     const edgeLatencyNs = nowNs - edgeStartNs
     const pathwayLatencyNs = nowNs - pathwayStartNs
     const dataStreamsContext = {

package/packages/dd-trace/src/encode/agentless-json.js

@@ -0,0 +1,141 @@
+'use strict'
+
+const log = require('../log')
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+
+/**
+ * Formats a span for JSON encoding.
+ * @param {object} span - The span to format
+ * @returns {object} The formatted span
+ */
+function formatSpan (span) {
+  span = normalizeSpan(truncateSpan(span, false))
+
+  if (span.span_events) {
+    span.meta.events = JSON.stringify(span.span_events)
+    delete span.span_events
+  }
+
+  return span
+}
+
+/**
+ * Converts a span to JSON-serializable format.
+ * IDs are converted to lowercase hex strings. Start time is converted from
+ * nanoseconds to seconds for the intake format.
+ * @param {object} span - The formatted span
+ * @returns {object} JSON-serializable span object
+ */
+function spanToJSON (span) {
+  const result = {
+    trace_id: span.trace_id.toString(16).toLowerCase(),
+    span_id: span.span_id.toString(16).toLowerCase(),
+    parent_id: span.parent_id.toString(16).toLowerCase(),
+    name: span.name,
+    resource: span.resource,
+    service: span.service,
+    error: span.error,
+    start: Math.floor(span.start / 1e9),
+    duration: span.duration,
+    meta: span.meta,
+    metrics: span.metrics,
+  }
+
+  if (span.type) {
+    result.type = span.type
+  }
+
+  if (span.meta_struct) {
+    result.meta_struct = span.meta_struct
+  }
+
+  if (span.links && span.links.length > 0) {
+    result.links = span.links
+  }
+
+  return result
+}
+
+/**
+ * JSON encoder for agentless span intake.
+ * Encodes a single trace as JSON with the payload format: {"spans": [...]}
+ *
+ * This encoder handles one trace at a time since each trace must be sent as a
+ * separate request to the intake. -- bengl
+ */
+class AgentlessJSONEncoder {
+  constructor () {
+    this._reset()
+  }
+
+  /**
+   * Returns the number of spans encoded.
+   * @returns {number}
+   */
+  count () {
+    return this._spanCount
+  }
+
+  /**
+   * Encodes a trace (array of spans) into the buffer.
+   * @param {object[]} trace - Array of spans to encode
+   */
+  encode (trace) {
+    for (const span of trace) {
+      try {
+        const formattedSpan = formatSpan(span)
+        const jsonSpan = spanToJSON(formattedSpan)
+
+        this._spans.push(jsonSpan)
+        this._spanCount++
+      } catch (err) {
+        log.error(
+          'Failed to encode span (name: %s, service: %s). Span will be dropped. Error: %s\n%s',
+          span?.name || 'unknown',
+          span?.service || 'unknown',
+          err.message,
+          err.stack
+        )
+      }
+    }
+  }
+
+  /**
+   * Creates the JSON payload for the encoded trace.
+   * @returns {Buffer} JSON payload as a buffer, or empty buffer if no spans
+   */
+  makePayload () {
+    if (this._spans.length === 0) {
+      this._reset()
+      return Buffer.alloc(0)
+    }
+
+    try {
+      const payload = JSON.stringify({ spans: this._spans })
+      this._reset()
+      return Buffer.from(payload, 'utf8')
+    } catch (err) {
+      log.error(
+        'Failed to encode trace as JSON (%d spans). Trace will be dropped. Error: %s',
+        this._spans.length,
+        err.message
+      )
+      this._reset()
+      return Buffer.alloc(0)
+    }
+  }
+
+  /**
+   * Resets the encoder state.
+   */
+  reset () {
+    this._reset()
+  }
+
+  _reset () {
+    this._spans = []
+    this._spanCount = 0
+  }
+}
+
+module.exports = { AgentlessJSONEncoder }

package/packages/dd-trace/src/exporter.js

@@ -11,6 +11,8 @@ module.exports = function getExporter (name) {
       return require('./exporters/log')
     case exporters.AGENT:
       return require('./exporters/agent')
+    case exporters.AGENTLESS:
+      return require('./exporters/agentless')
     case exporters.DATADOG:
       return require('./ci-visibility/exporters/agentless')
     case exporters.AGENT_PROXY:

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -1,13 +1,12 @@
 'use strict'
 
-const { inspect } = require('util')
-
 const request = require('../common/request')
 const { startupLog } = require('../../startup-log')
 const runtimeMetrics = require('../../runtime_metrics')
 const log = require('../../log')
 const tracerVersion = require('../../../../../package.json').version
 const BaseWriter = require('../common/writer')
+const propagationHash = require('../../propagation-hash')
 
 const METRIC_PREFIX = 'datadog.tracer.node.exporter.agent'
 
@@ -29,10 +28,7 @@ class AgentWriter extends BaseWriter {
     runtimeMetrics.increment(`${METRIC_PREFIX}.requests`, true)
 
     const { _headers, _lookup, _protocolVersion, _url } = this
-    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, (err, res, status) => {
-      // Note that logging will only happen once, regardless of how many times this is called.
-      startupLog(status !== 404 && status !== 200 ? { status, message: err?.message ?? inspect(err) } : undefined)
-
+    makeRequest(_protocolVersion, data, count, _url, _headers, _lookup, true, (err, res, status, headers) => {
       if (status) {
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses`, true)
         runtimeMetrics.increment(`${METRIC_PREFIX}.responses.by.status`, `status:${status}`, true)
@@ -53,6 +49,16 @@ class AgentWriter extends BaseWriter {
 
       log.debug('Response from the agent: %s', res)
 
+      // Capture container tags hash from agent response headers
+      // The hash is sent by the agent only when Datadog-Container-ID is present in the request
+      // (Datadog-Container-ID is automatically injected by docker.inject() in exporters/common/request.js)
+      if (headers) {
+        const containerTagsHash = headers['Datadog-Container-Tags-Hash']
+        if (containerTagsHash) {
+          propagationHash.updateContainerTagsHash(containerTagsHash)
+        }
+      }
+
       try {
         this._prioritySampler.update(JSON.parse(res).rate_by_service)
       } catch (e) {
@@ -72,7 +78,7 @@ function getEncoder (protocolVersion) {
     : require('../../encode/0.4').AgentEncoder
 }
 
-function makeRequest (version, data, count, url, headers, lookup, cb) {
+function makeRequest (version, data, count, url, headers, lookup, needsStartupLog, cb) {
   const options = {
     path: `/v${version}/traces`,
     method: 'PUT',
@@ -91,7 +97,15 @@ function makeRequest (version, data, count, url, headers, lookup, cb) {
 
   log.debug('Request to the agent: %j', options)
 
-  request(data, options, cb)
+  request(data, options, (err, res, status, headers) => {
+    if (needsStartupLog) {
+      // Note that logging will only happen once, regardless of how many times this is called.
+      startupLog({
+        agentError: status !== 404 && status !== 200 ? err : undefined,
+      })
+    }
+    cb(err, res, status, headers)
+  })
 }
 
 module.exports = AgentWriter

package/packages/dd-trace/src/exporters/agentless/index.js

@@ -0,0 +1,89 @@
+'use strict'
+
+const { URL } = require('node:url')
+
+const log = require('../../log')
+const Writer = require('./writer')
+
+/**
+ * Agentless exporter for APM span intake.
+ * Sends spans directly to the Datadog intake without requiring a local agent.
+ *
+ * Each trace is sent immediately as a separate request. The intake only accepts one trace
+ * per request - requests with spans from different traces return HTTP 200 but silently
+ * drop all spans. By flushing immediately after each export (which contains one trace),
+ * we avoid this limitation entirely. -- bengl
+ */
+class AgentlessExporter {
+  /**
+   * @param {object} config - Configuration object
+   * @param {string} [config.site='datadoghq.com'] - The Datadog site
+   * @param {string} [config.url] - Override intake URL
+   */
+  constructor (config) {
+    this._config = config
+    const { site = 'datadoghq.com', url } = config
+
+    try {
+      this._url = url ? new URL(url) : new URL(`https://public-trace-http-intake.logs.${site}`)
+    } catch (err) {
+      log.error(
+        'Invalid URL configuration for agentless exporter. url=%s, site=%s. Error: %s',
+        url || 'not set',
+        site,
+        err.message
+      )
+      this._url = null
+    }
+
+    this._writer = new Writer({
+      url: this._url,
+      site,
+    })
+
+    const ddTrace = globalThis[Symbol.for('dd-trace')]
+    if (ddTrace?.beforeExitHandlers) {
+      ddTrace.beforeExitHandlers.add(this.flush.bind(this))
+    } else {
+      log.error('dd-trace global not properly initialized. beforeExit handler not registered for agentless exporter.')
+    }
+  }
+
+  /**
+   * Sets the intake URL.
+   * @param {string} urlString - The new intake URL
+   * @returns {boolean} True if URL was set successfully
+   */
+  setUrl (urlString) {
+    try {
+      const url = new URL(urlString)
+      this._url = url
+      this._writer.setUrl(url)
+      return true
+    } catch {
+      log.error('Invalid URL for agentless exporter: %s. Using previous URL: %s', urlString, this._url?.href || 'none')
+      return false
+    }
+  }
+
+  /**
+   * Exports a trace to the intake. Flushes immediately since each trace must be
+   * sent as a separate request.
+   * @param {object[]} spans - Array of spans (all from the same trace)
+   */
+  export (spans) {
+    this._writer.append(spans)
+    this._writer.flush()
+  }
+
+  /**
+   * Flushes any pending spans. With immediate flush per trace, this is mainly
+   * used for the beforeExit handler to ensure nothing is left unsent.
+   * @param {Function} [done] - Callback when flush is complete
+   */
+  flush (done = () => {}) {
+    this._writer.flush(done)
+  }
+}
+
+module.exports = AgentlessExporter

package/packages/dd-trace/src/exporters/agentless/writer.js

@@ -0,0 +1,184 @@
+'use strict'
+
+const { getValueFromEnvSources } = require('../../config/helper')
+const log = require('../../log')
+const request = require('../common/request')
+const tracerVersion = require('../../../../../package.json').version
+
+const BaseWriter = require('../common/writer')
+const { AgentlessJSONEncoder } = require('../../encode/agentless-json')
+
+/**
+ * Writer for agentless APM span intake.
+ * Sends spans directly to the Datadog intake endpoint without an agent.
+ */
+class AgentlessWriter extends BaseWriter {
+  #apiKeyMissing = false
+
+  /**
+   * @param {object} options - Writer options
+   * @param {URL} [options.url] - The intake URL. If not provided, constructed from site.
+   * @param {string} [options.site='datadoghq.com'] - The Datadog site
+   */
+  constructor ({ url, site = 'datadoghq.com' }) {
+    super({ url })
+    this._encoder = new AgentlessJSONEncoder()
+
+    if (!url) {
+      try {
+        this._url = new URL(`https://public-trace-http-intake.logs.${site}`)
+      } catch (err) {
+        log.error(
+          'Invalid site value for agentless intake: %s. Cannot construct URL. Error: %s',
+          site,
+          err.message
+        )
+        this._url = null
+      }
+    }
+
+    if (!getValueFromEnvSources('DD_API_KEY')) {
+      this.#apiKeyMissing = true
+      log.error('DD_API_KEY is required for agentless span intake. Set DD_API_KEY. Spans will not be sent.')
+    }
+  }
+
+  /**
+   * Flushes the current trace. Since we flush after each trace, this sends
+   * a single request.
+   * @param {Function} [done] - Callback when send completes
+   */
+  flush (done = () => {}) {
+    if (!request.writable) {
+      this._encoder.reset()
+      done()
+      return
+    }
+
+    const count = this._encoder.count()
+
+    if (count === 0) {
+      done()
+      return
+    }
+
+    const payload = this._encoder.makePayload()
+
+    if (payload.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    this._sendPayload(payload, count, done)
+  }
+
+  /**
+   * Sends the encoded payload to the intake endpoint.
+   * @param {Buffer} data - The encoded JSON payload
+   * @param {number} count - Number of spans in the payload
+   * @param {Function} done - Callback when complete
+   */
+  _sendPayload (data, count, done) {
+    if (!data || data.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    if (!this._url) {
+      log.debug('Skipping send due to invalid URL configuration')
+      done()
+      return
+    }
+
+    const apiKey = getValueFromEnvSources('DD_API_KEY')
+    if (!apiKey) {
+      if (!this.#apiKeyMissing) {
+        this.#apiKeyMissing = true
+        log.error('DD_API_KEY is required for agentless span intake. Set DD_API_KEY. Spans will not be sent.')
+      }
+      log.debug('Dropping %d span(s) due to missing DD_API_KEY', count)
+      done()
+      return
+    }
+    this.#apiKeyMissing = false
+
+    const options = {
+      path: '/v1/input',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'dd-api-key': apiKey,
+        'Datadog-Meta-Lang': 'nodejs',
+        'Datadog-Meta-Lang-Version': process.version,
+        'Datadog-Meta-Lang-Interpreter': process.versions.bun ? 'JavaScriptCore' : 'v8',
+        'Datadog-Meta-Tracer-Version': tracerVersion,
+      },
+      timeout: 15_000,
+      url: this._url,
+    }
+
+    log.debug('Request to the agentless intake: %j', options)
+
+    request(data, options, (err, res, statusCode) => {
+      if (err) {
+        this._logRequestError(err, statusCode, count)
+        done()
+        return
+      }
+
+      log.debug('Response from the agentless intake: %s', res)
+      done()
+    })
+  }
+
+  /**
+   * Logs request errors with status-specific guidance.
+   * @param {Error} err - The error object
+   * @param {number} statusCode - HTTP status code (if available)
+   * @param {number} count - Number of spans that were being sent
+   */
+  _logRequestError (err, statusCode, count) {
+    if (statusCode === 401 || statusCode === 403) {
+      log.error(
+        'Authentication failed sending %d span(s) (status %s). Verify DD_API_KEY is valid.',
+        count,
+        statusCode
+      )
+    } else if (statusCode === 404) {
+      log.error(
+        'Span intake endpoint not found (status %s). Verify DD_SITE is correctly configured. %d span(s) dropped.',
+        statusCode,
+        count
+      )
+    } else if (statusCode === 429) {
+      log.error(
+        'Rate limited by span intake (status 429). %d span(s) dropped.',
+        count
+      )
+    } else if (statusCode >= 500) {
+      log.error(
+        'Span intake server error (status %s). %d span(s) dropped. This may be transient.',
+        statusCode,
+        count
+      )
+    } else if (statusCode) {
+      log.error(
+        'Error sending agentless payload (status %s): %s. %d span(s) dropped.',
+        statusCode,
+        err.message,
+        count
+      )
+    } else {
+      log.error(
+        'Network error sending %d span(s) to %s: %s',
+        count,
+        this._url?.hostname || 'unknown',
+        err.message
+      )
+    }
+  }
+}
+
+module.exports = AgentlessWriter

package/packages/dd-trace/src/exporters/common/request.js

@@ -88,13 +88,13 @@ function request (data, options, callback) {
           zlib.gunzip(buffer, (err, result) => {
             if (err) {
               log.error('Could not gunzip response: %s', err.message)
-              callback(null, '', res.statusCode)
+              callback(null, '', res.statusCode, res.headers)
             } else {
-              callback(null, result.toString(), res.statusCode)
+              callback(null, result.toString(), res.statusCode, res.headers)
             }
           })
         } else {
-          callback(null, buffer.toString(), res.statusCode)
+          callback(null, buffer.toString(), res.statusCode, res.headers)
         }
       } else {
         let errorMessage = ''
@@ -115,7 +115,7 @@ function request (data, options, callback) {
         const error = new log.NoTransmitError(errorMessage)
         error.status = res.statusCode
 
-        callback(error, null, res.statusCode)
+        callback(error, null, res.statusCode, res.headers)
       }
     })
   }

package/packages/dd-trace/src/llmobs/plugins/ai/index.js

@@ -4,7 +4,7 @@ const { channel } = require('dc-polyfill')
 const BaseLLMObsPlugin = require('../base')
 const { getModelProvider } = require('../../../../../datadog-plugin-ai/src/utils')
 
-const toolCreationCh = channel('dd-trace:vercel-ai:tool')
+const toolCreationCh = channel('tracing:orchestrion:ai:tool:start')
 const setAttributesCh = channel('dd-trace:vercel-ai:span:setAttributes')
 
 const { MODEL_NAME, MODEL_PROVIDER, NAME } = require('../../constants/tags')
@@ -94,8 +94,10 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin {
 
     this.#toolCallIdsToName = {}
     this.#availableTools = new Set()
-    toolCreationCh.subscribe(toolArgs => {
-      this.#availableTools.add(toolArgs)
+    toolCreationCh.subscribe(ctx => {
+      const toolArgs = ctx.arguments
+      const tool = toolArgs[0] ?? {}
+      this.#availableTools.add(tool)
     })
 
     setAttributesCh.subscribe(({ ctx, attributes }) => {