dd-trace 6.13.0 → 6.15.0

package/packages/dd-trace/src/exporters/common/agents.js

@@ -7,9 +7,15 @@ const { storage } = require('../../../../datadog-core')
 const legacyStorage = storage('legacy')
 
 const keepAlive = true
-const maxSockets = 1
 
-function createAgentClass (BaseAgent) {
+/**
+ * Creates an agent class that suppresses tracing around socket lifecycle operations.
+ *
+ * @param {typeof http.Agent|typeof https.Agent} BaseAgent
+ * @param {number} maxSockets
+ * @returns {typeof http.Agent|typeof https.Agent}
+ */
+function createAgentClass (BaseAgent, maxSockets) {
   class CustomAgent extends BaseAgent {
     constructor () {
       super({ keepAlive, maxSockets })
@@ -35,10 +41,26 @@ function createAgentClass (BaseAgent) {
   return CustomAgent
 }
 
-const HttpAgent = createAgentClass(http.Agent)
-const HttpsAgent = createAgentClass(https.Agent)
+/**
+ * Creates isolated HTTP and HTTPS agents with a bounded connection pool.
+ *
+ * @param {number} maxSockets
+ * @returns {{ httpAgent: http.Agent, httpsAgent: https.Agent }}
+ */
+function createAgents (maxSockets) {
+  const HttpAgent = createAgentClass(http.Agent, maxSockets)
+  const HttpsAgent = createAgentClass(https.Agent, maxSockets)
+
+  return {
+    httpAgent: new HttpAgent(),
+    httpsAgent: new HttpsAgent(),
+  }
+}
+
+const { httpAgent, httpsAgent } = createAgents(1)
 
 module.exports = {
-  httpAgent: new HttpAgent(),
-  httpsAgent: new HttpsAgent(),
+  createAgents,
+  httpAgent,
+  httpsAgent,
 }

package/packages/dd-trace/src/exporters/common/buffering-exporter.js

@@ -25,10 +25,10 @@ class BufferingExporter {
   }
 
   _export (payload, writer = this._writer, timerKey = '_timer', deferImmediateFlush = false) {
-    if (this._config.isCiVisibility) {
+    const appended = writer.append(payload)
+    if (this._config.isCiVisibility && appended !== false) {
       incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, {}, payload.length)
     }
-    const appended = writer.append(payload)
 
     const { flushInterval } = this._config
 

package/packages/dd-trace/src/exporters/common/final-flush-request-tracker.js

@@ -0,0 +1,191 @@
+'use strict'
+
+/**
+ * @typedef {Record<string, unknown> & { deadline?: number, signal?: AbortSignal }} RequestOptions
+ */
+
+/**
+ * @typedef {object} FinalFlush
+ * @property {number} deadline
+ * @property {(error?: Error) => void} done
+ * @property {Error | undefined} error
+ * @property {Set<PendingRequest>} requests
+ * @property {ReturnType<typeof setTimeout> | undefined} timeoutId
+ * @property {boolean} writerDone
+ */
+
+/**
+ * @typedef {object} PendingRequest
+ * @property {AbortController} controller
+ * @property {Set<FinalFlush>} finalFlushes
+ * @property {RequestOptions} options
+ */
+
+class FinalFlushRequestTracker {
+  #flush
+  #createTimeoutError
+  /** @type {Set<PendingRequest>} */
+  #pendingRequests = new Set()
+  /** @type {Set<FinalFlush>} */
+  #finalFlushes = new Set()
+  /** @type {FinalFlush | undefined} */
+  #activeFinalFlush
+
+  /**
+   * @param {(done?: (error?: Error) => void, options?: { deadline?: number }) => void} flush
+   * @param {() => Error} createTimeoutError
+   */
+  constructor (flush, createTimeoutError) {
+    this.#flush = flush
+    this.#createTimeoutError = createTimeoutError
+  }
+
+  /**
+   * Flushes queued payloads and waits for requests that were already in flight.
+   *
+   * @param {(error?: Error) => void} [done]
+   * @param {{ deadline?: number }} [options]
+   * @returns {void}
+   */
+  flush (done, options) {
+    if (options?.deadline === undefined) {
+      this.#flush(done, options)
+      return
+    }
+
+    /** @type {FinalFlush} */
+    const finalFlush = {
+      deadline: options.deadline,
+      done: done || (() => {}),
+      error: undefined,
+      requests: new Set(),
+      timeoutId: undefined,
+      writerDone: false,
+    }
+    this.#finalFlushes.add(finalFlush)
+
+    for (const pendingRequest of this.#pendingRequests) {
+      this.#attachRequest(finalFlush, pendingRequest)
+    }
+
+    const remaining = Math.max(0, options.deadline - Date.now())
+    finalFlush.timeoutId = setTimeout(() => {
+      const error = this.#createTimeoutError()
+
+      finalFlush.error ||= error
+      finalFlush.writerDone = true
+
+      for (const pendingRequest of finalFlush.requests) {
+        pendingRequest.finalFlushes.delete(finalFlush)
+        if (pendingRequest.finalFlushes.size === 0) {
+          pendingRequest.controller.abort(error)
+          this.#pendingRequests.delete(pendingRequest)
+        } else {
+          this.#updateRequestDeadline(pendingRequest)
+        }
+      }
+      finalFlush.requests.clear()
+      this.#finishFinalFlush(finalFlush)
+    }, remaining)
+
+    const previousFinalFlush = this.#activeFinalFlush
+    this.#activeFinalFlush = finalFlush
+    try {
+      this.#flush((error) => {
+        finalFlush.error ||= error
+        finalFlush.writerDone = true
+        this.#finishFinalFlush(finalFlush)
+      }, options)
+    } finally {
+      this.#activeFinalFlush = previousFinalFlush
+    }
+    this.#finishFinalFlush(finalFlush)
+  }
+
+  /**
+   * Sends and tracks a request so a later final flush can wait for or abort it.
+   *
+   * @param {(data: Buffer|string|object, options: RequestOptions,
+   *   callback: (error: Error|null, result?: string|null, statusCode?: number,
+   *   headers?: import('node:http').IncomingHttpHeaders) => void) => void} request
+   * @param {Buffer|string|object} data
+   * @param {RequestOptions} options
+   * @param {(error: Error|null, result?: string|null, statusCode?: number,
+   *   headers?: import('node:http').IncomingHttpHeaders) => void} callback
+   * @returns {void}
+   */
+  send (request, data, options, callback) {
+    const controller = new AbortController()
+    const requestOptions = { ...options, signal: controller.signal }
+    const pendingRequest = { controller, finalFlushes: new Set(), options: requestOptions }
+    this.#pendingRequests.add(pendingRequest)
+    if (this.#activeFinalFlush) this.#attachRequest(this.#activeFinalFlush, pendingRequest)
+
+    try {
+      request(data, requestOptions, (error, result, statusCode, headers) => {
+        if (error) {
+          for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
+        }
+
+        try {
+          callback(error, result, statusCode, headers)
+        } finally {
+          this.#dropRequest(pendingRequest)
+        }
+      })
+    } catch (error) {
+      this.#dropRequest(pendingRequest)
+      throw error
+    }
+  }
+
+  /**
+   * @param {PendingRequest} pendingRequest
+   * @returns {void}
+   */
+  #dropRequest (pendingRequest) {
+    this.#pendingRequests.delete(pendingRequest)
+    for (const finalFlush of pendingRequest.finalFlushes) {
+      finalFlush.requests.delete(pendingRequest)
+      this.#finishFinalFlush(finalFlush)
+    }
+    pendingRequest.finalFlushes.clear()
+  }
+
+  /**
+   * @param {FinalFlush} finalFlush
+   * @param {PendingRequest} pendingRequest
+   * @returns {void}
+   */
+  #attachRequest (finalFlush, pendingRequest) {
+    finalFlush.requests.add(pendingRequest)
+    pendingRequest.finalFlushes.add(finalFlush)
+    this.#updateRequestDeadline(pendingRequest)
+  }
+
+  /**
+   * @param {PendingRequest} pendingRequest
+   * @returns {void}
+   */
+  #updateRequestDeadline (pendingRequest) {
+    let deadline = 0
+    for (const finalFlush of pendingRequest.finalFlushes) {
+      deadline = Math.max(deadline, finalFlush.deadline)
+    }
+    pendingRequest.options.deadline = deadline
+  }
+
+  /**
+   * @param {FinalFlush} finalFlush
+   * @returns {void}
+   */
+  #finishFinalFlush (finalFlush) {
+    if (!this.#finalFlushes.has(finalFlush) || !finalFlush.writerDone || finalFlush.requests.size !== 0) return
+
+    this.#finalFlushes.delete(finalFlush)
+    clearTimeout(finalFlush.timeoutId)
+    finalFlush.done(finalFlush.error)
+  }
+}
+
+module.exports = FinalFlushRequestTracker

package/packages/dd-trace/src/exporters/common/proxy.js

@@ -0,0 +1,52 @@
+'use strict'
+
+const { isIPv6 } = require('node:net')
+
+let defaultAgentKey
+let getProxyForUrl
+let proxyAgents
+
+/**
+ * Selects a proxy agent for an HTTPS target while preserving the direct agent's pool boundaries.
+ *
+ * @param {string|URL|object} url
+ * @param {import('node:http').Agent|false} [directAgent]
+ * @returns {import('node:http').Agent|false|undefined}
+ */
+function getHttpsProxyAgent (url, directAgent) {
+  getProxyForUrl ??= require('../../../../../vendor/dist/proxy-from-env').getProxyForUrl
+
+  const host = typeof url === 'string' ? undefined : url.host ?? url.hostname
+  const isUnbracketedIPv6 = typeof host === 'string' && host.includes(':') && isIPv6(host)
+  const target = typeof url === 'string'
+    ? url
+    : { protocol: url.protocol, host: isUnbracketedIPv6 ? `[${host}]` : host, port: url.port }
+  const proxyUrl = getProxyForUrl(target)
+  if (!proxyUrl) return directAgent
+
+  directAgent ??= require('node:https').globalAgent
+
+  if (proxyAgents === undefined) {
+    defaultAgentKey = {}
+    proxyAgents = new WeakMap()
+  }
+  const cacheKey = directAgent || defaultAgentKey
+  let agents = proxyAgents.get(cacheKey)
+  if (agents === undefined) {
+    agents = new Map()
+    proxyAgents.set(cacheKey, agents)
+  }
+
+  let agent = agents.get(proxyUrl)
+  if (agent === undefined) {
+    const { HttpsProxyAgent } = require('../../../../../vendor/dist/https-proxy-agent')
+    const options = directAgent
+      ? { keepAlive: directAgent.keepAlive, maxSockets: directAgent.maxSockets }
+      : undefined
+    agent = new HttpsProxyAgent(proxyUrl, options)
+    agents.set(proxyUrl, agent)
+  }
+  return agent
+}
+
+module.exports = { getHttpsProxyAgent }

package/packages/dd-trace/src/exporters/common/request.js

@@ -10,9 +10,10 @@ const zlib = require('zlib')
 
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
-const { isLoopbackHost, parseUrl } = require('./url')
+const { canSendApiKey, parseUrl } = require('./url')
 const docker = require('./docker')
 const { httpAgent, httpsAgent } = require('./agents')
+const { getHttpsProxyAgent } = require('./proxy')
 const {
   getMaxAttempts,
   getRetryDelay,
@@ -30,7 +31,7 @@ let activeBufferSize = 0
  * @param {Buffer|string|Readable|Array<Buffer|string>} data
  * @param {object} options
  * @param {(error: Error|null, result?: string|null, statusCode?: number,
- *   headers?: import('node:http').IncomingHttpHeaders) => void} callback
+ *   headers?: import('node:http').IncomingHttpHeaders, dropped?: boolean) => void} callback
  */
 function request (data, options, callback) {
   if (!options.headers) {
@@ -54,7 +55,7 @@ function request (data, options, callback) {
   // rather than drop the request: the agent proxies telemetry with its own key, while an https
   // intake URL is required to authenticate agentless traffic.
   const hasApiKey = options.headers['dd-api-key'] !== undefined || options.headers['DD-API-KEY'] !== undefined
-  if (hasApiKey && options.protocol === 'http:' && !isLoopbackHost(options.hostname)) {
+  if (hasApiKey && !canSendApiKey(options.protocol, options.hostname)) {
     log.error(
       'Not sending the Datadog API key over a non-TLS connection to %s. Configure an https intake URL.',
       options.hostname
@@ -89,15 +90,23 @@ function request (data, options, callback) {
   if (!Array.isArray(data)) {
     dataArray = [data]
   }
-  options.headers['Content-Length'] = byteLength(dataArray)
+  const contentLength = byteLength(dataArray)
+  options.headers['Content-Length'] = contentLength
 
   docker.inject(options.headers)
 
-  const connectionOptions = {
-    ...options,
-    agent: options.agent ?? (isSecure ? httpsAgent : httpAgent),
+  let agent = options.agent ?? (isSecure ? httpsAgent : httpAgent)
+  if (hasApiKey && isSecure) {
+    try {
+      agent = getHttpsProxyAgent(options, agent)
+    } catch (error) {
+      callback(error)
+      return
+    }
   }
 
+  const connectionOptions = { ...options, agent }
+
   /**
    * @param {import('node:http').IncomingMessage} res
    * @param {(error: Error|null, result?: string|null, statusCode?: number,
@@ -172,20 +181,23 @@ function request (data, options, callback) {
   // outside AsyncContextFrame, so a synchronous re-entry would lose the store.
   /** @param {number} attemptIndex */
   const attempt = attemptIndex => {
-    if (!request.writable) {
-      log.debug('Maximum number of active requests reached: payload is discarded.')
-      return callback(null)
+    if (activeBufferSize + contentLength > maxActiveBufferSize) {
+      const error = new log.NoTransmitError('Maximum active request buffer size reached: payload is discarded.')
+      error.code = 'ERR_DD_REQUEST_BUFFER_FULL'
+      log.debug(error.message)
+      return callback(error, undefined, undefined, undefined, true)
     }
 
-    activeBufferSize += options.headers['Content-Length'] ?? 0
+    activeBufferSize += contentLength
 
     legacyStorage.run({ noop: true }, () => {
       let finished = false
       let settled = false
+      let timeoutImmediate
       const finalize = () => {
         if (finished) return
         finished = true
-        activeBufferSize -= options.headers['Content-Length'] ?? 0
+        activeBufferSize -= contentLength
       }
 
       /**
@@ -197,6 +209,7 @@ function request (data, options, callback) {
       const complete = (error, result, statusCode, headers) => {
         if (settled) return
         settled = true
+        clearImmediate(timeoutImmediate)
         finalize()
         callback(error, result, statusCode, headers)
       }
@@ -206,6 +219,7 @@ function request (data, options, callback) {
        */
       const handleError = (error) => {
         if (settled) return
+        clearImmediate(timeoutImmediate)
 
         if (options.retry !== false &&
             attemptIndex < getMaxAttempts(options) &&
@@ -224,10 +238,11 @@ function request (data, options, callback) {
       const req = client.request(connectionOptions, (res) => onResponse(res, complete, handleError))
 
       req.once('close', finalize)
-      req.once('timeout', finalize)
+      if (!options.deferTimeoutAbort) req.once('timeout', finalize)
       req.once('error', handleError)
 
-      req.setTimeout(timeout, () => {
+      const abortRequest = () => {
+        if (settled) return
         try {
           if (typeof req.abort === 'function') {
             req.abort()
@@ -237,6 +252,19 @@ function request (data, options, callback) {
         } catch {
           // ignore
         }
+      }
+
+      req.setTimeout(timeout, () => {
+        if (!options.deferTimeoutAbort) {
+          abortRequest()
+          return
+        }
+
+        timeoutImmediate = setImmediate(() => {
+          abortRequest()
+          finalize()
+        })
+        if (!options.keepProcessAlive) timeoutImmediate.unref?.()
       })
 
       for (const buffer of dataArray) req.write(buffer)

package/packages/dd-trace/src/exporters/common/url.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const net = require('node:net')
+const { format } = require('node:url')
 
 const { urlToHttpOptions } = require('./url-to-http-options-polyfill')
 
@@ -16,6 +17,44 @@ function isLoopbackHost (hostname) {
     (hostname.startsWith('127.') && net.isIPv4(hostname))
 }
 
+/**
+ * @param {string|undefined} protocol
+ * @param {string|undefined} hostname
+ * @returns {boolean}
+ */
+function canSendApiKey (protocol, hostname) {
+  return protocol === 'https:' || protocol === 'unix:' ||
+    typeof hostname === 'string' && isLoopbackHost(hostname)
+}
+
+/**
+ * @param {string} site
+ * @param {string} [intake]
+ * @returns {URL | undefined}
+ */
+function createSiteUrl (site, intake) {
+  const hostname = `${intake === undefined ? '' : `${intake}.`}${site}`.toLowerCase()
+
+  try {
+    const url = new URL(format({
+      protocol: 'https:',
+      hostname,
+    }))
+    if (
+      url.hostname !== hostname ||
+      url.username ||
+      url.password ||
+      url.port ||
+      url.pathname !== '/' ||
+      url.search ||
+      url.hash
+    ) {
+      return
+    }
+    return url
+  } catch {}
+}
+
 /**
  * Convert an agent/intake URL into Node http(s) request options.
  *
@@ -43,4 +82,4 @@ function parseUrl (urlObjOrString) {
   return url
 }
 
-module.exports = { isLoopbackHost, parseUrl }
+module.exports = { canSendApiKey, createSiteUrl, isLoopbackHost, parseUrl }

package/packages/dd-trace/src/exporters/common/writer.js

@@ -8,13 +8,16 @@ const request = require('./request')
 const { safeJSONStringify } = require('./util')
 
 const firstFlushChannel = channel('dd-trace:exporter:first-flush')
+const noop = () => {}
 
 class Writer {
   #deliveryTracker
+  #retainOnBackpressure
 
-  constructor ({ url, beforeFirstFlush, deliveryTracker }) {
+  constructor ({ url, beforeFirstFlush, deliveryTracker, retainOnBackpressure = false }) {
     this._url = url
     this._beforeFirstFlush = beforeFirstFlush
+    this.#retainOnBackpressure = retainOnBackpressure
     this.#deliveryTracker = deliveryTracker
   }
 
@@ -39,10 +42,10 @@ class Writer {
    * @param {{ deadline?: number }} [options]
    * @returns {void}
    */
-  flushDirect (done = () => {}, options) {
+  flushDirect (done = noop, options) {
     const count = this._encoder.count()
 
-    if (!request.writable && options?.deadline === undefined) {
+    if (!request.writable && options?.deadline === undefined && !this.#retainOnBackpressure) {
       this._encoder.reset()
       done()
     } else if (count > 0) {
@@ -77,7 +80,7 @@ class Writer {
   }
 
   append (payload, options) {
-    if (!request.writable && options?.deadline === undefined) {
+    if (!request.writable && options?.deadline === undefined && !this.#retainOnBackpressure) {
       // eslint-disable-next-line eslint-rules/eslint-log-printf-style
       log.debug(() => `Maximum number of active requests reached. Payload discarded: ${safeJSONStringify(payload)}`)
       return false
@@ -86,12 +89,11 @@ class Writer {
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
     log.debug(() => `Encoding payload: ${safeJSONStringify(payload)}`)
 
-    this._encode(payload)
-    return true
+    return this._encode(payload) !== false
   }
 
   _encode (payload) {
-    this._encoder.encode(payload)
+    return this._encoder.encode(payload)
   }
 
   setUrl (url) {

package/packages/dd-trace/src/flush.js

@@ -43,12 +43,16 @@ function registerTelemetryFlusher (flusher, options) {
  * @param {{ timeout?: number }} [options]
  * @param {TraceFlushers} [traceFlushers]
  */
-function flushServerlessTelemetry (done, options, traceFlushers = {}) {
-  const { trace: traceFlusher, spanStats: spanStatsFlusher } = traceFlushers
+function flushServerlessTelemetry (done, options, traceFlushers) {
+  const traceFlusher = traceFlushers?.trace
+  const spanStatsFlusher = traceFlushers?.spanStats
   // TODO: Include DSM after DataStreamsProcessor exposes a completion-aware flush API.
   let pending = telemetryFlushers.size + postTraceTelemetryFlushers.size +
     (typeof traceFlusher === 'function' ? 1 : 0) +
     (typeof spanStatsFlusher === 'function' ? 1 : 0)
+
+  if (pending === 0) return done?.()
+
   let completed = false
   let timeout
 
@@ -62,7 +66,6 @@ function flushServerlessTelemetry (done, options, traceFlushers = {}) {
     if (--pending === 0) finish()
   }
 
-  if (pending === 0) return finish()
   if (options?.timeout) {
     timeout = setTimeout(() => {
       log.warn('Timed out waiting for telemetry flush after %dms', options.timeout)

package/packages/dd-trace/src/id.js

@@ -2,6 +2,8 @@
 
 const { randomFillSync } = require('crypto')
 
+const { channel } = require('dc-polyfill')
+
 const UINT_MAX = 4_294_967_296
 
 const data = new Uint8Array(8 * 8192)
@@ -9,6 +11,8 @@ const zeroId = new Uint8Array(8)
 
 let batch = 0
 
+channel('datadog:identity:update').subscribe(reseed)
+
 // Internal representation of a trace or span ID.
 class Identifier {
   /** @type {number[] | Uint8Array} */
@@ -254,6 +258,16 @@ function writeUInt32BE (buffer, value, offset) {
   buffer[0 + offset] = value & 255
 }
 
+/**
+ * Resets the batch cursor, forcing the next ID batch to draw a fresh
+ * randomFillSync() call on MicroVM clone resume. Node's crypto RNG is
+ * re-seeded from the kernel CSPRNG on snapshot resume, so re-invoking it
+ * is sufficient — no need to read /dev/urandom directly.
+ */
+function reseed () {
+  batch = 0
+}
+
 /**
  * @param {string} [value]
  * @param {number} [radix]

package/packages/dd-trace/src/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
-const { getValueFromEnvSources } = require('./config/helper')
+const { getEnvironmentVariable, getValueFromEnvSources } = require('./config/helper')
+const { isTrue } = require('./util')
 
 // Global `jest` is only present in Jest workers.
 const inJestWorker = typeof jest !== 'undefined'
@@ -8,8 +9,10 @@ const inJestWorker = typeof jest !== 'undefined'
 // skipDefault: distinguish an unset DD_TRACE_ENABLED (fall back to the OTel signal) from an
 // explicit value; the registered default would otherwise mask the OTEL_TRACES_EXPORTER check.
 const ddTraceEnabled = getValueFromEnvSources('DD_TRACE_ENABLED', true)
+const agentlessTracingEnabled = getValueFromEnvSources('DD_AGENTLESS_ENABLED') ||
+  isTrue(getEnvironmentVariable('_DD_APM_TRACING_AGENTLESS_ENABLED'))
 const ddTraceDisabled = ddTraceEnabled === undefined
-  ? getValueFromEnvSources('OTEL_TRACES_EXPORTER') === 'none'
+  ? !agentlessTracingEnabled && getValueFromEnvSources('OTEL_TRACES_EXPORTER') === 'none'
   : ddTraceEnabled === false
 
 module.exports = ddTraceDisabled || inJestWorker

package/packages/dd-trace/src/llmobs/experiments/dataset.js

@@ -1,7 +1,8 @@
 'use strict'
 
 const { randomUUID } = require('node:crypto')
-const snapshotPayload = require('../../../../../vendor/dist/rfdc')({ proto: false, circles: false })
+const createRfdc = require('../../../../../vendor/dist/rfdc')
+const snapshotPayload = createRfdc({ proto: false, circles: false })
 
 /** @typedef {{add?: string[], remove?: string[], replace?: string[]}} TagOperations */
 /**