dd-trace 5.124.0 → 5.126.0

package/packages/dd-trace/src/appsec/lambda.js

@@ -3,7 +3,9 @@
 const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
 
 const log = require('../log')
+const { isEmpty } = require('../util')
 const addresses = require('./addresses')
+const apiSecurity = require('./api_security')
 const Reporter = require('./reporter')
 const waf = require('./waf')
 
@@ -22,7 +24,7 @@ const activeInvocations = new WeakMap()
  */
 function onLambdaStartInvocation (data) {
   try {
-    const { span, headers, method, path, query, body, clientIp, pathParams, cookies } = data
+    const { span, headers, method, path, query, body, clientIp, pathParams, cookies, route } = data
 
     if (!span) {
       log.warn('[ASM] No span provided in Lambda start invocation')
@@ -30,7 +32,7 @@ function onLambdaStartInvocation (data) {
     }
 
     const req = { headers: headers ?? {} }
-    activeInvocations.set(span, req)
+    activeInvocations.set(span, { req, method, route })
 
     span.addTags({
       '_dd.appsec.enabled': 1,
@@ -80,8 +82,8 @@ function onLambdaStartInvocation (data) {
 }
 
 /**
- * Maps response data to WAF addresses, runs a final WAF pass,
- * disposes the WAF context, and finishes the request report.
+ * Maps response data to WAF addresses, takes the API Security sampling decision, runs a final
+ * WAF pass, disposes the WAF context, and finishes the request report.
  *
  * @param {{ span: object, statusCode: string | undefined,
  *           responseHeaders: Record<string, string> | undefined }} data
@@ -99,31 +101,47 @@ function onLambdaEndInvocation (data) {
       return
     }
 
-    const req = activeInvocations.get(span)
+    const { req, method, route } = activeInvocations.get(span)
     activeInvocations.delete(span)
 
-    let hasPersistentData = false
-    const persistent = {}
-
-    if (statusCode) {
-      persistent[addresses.HTTP_INCOMING_RESPONSE_CODE] = String(statusCode)
-      hasPersistentData = true
-    }
-
-    if (responseHeaders) {
-      const filteredHeaders = { ...responseHeaders }
-      delete filteredHeaders['set-cookie']
-      persistent[addresses.HTTP_INCOMING_RESPONSE_HEADERS] = filteredHeaders
-      hasPersistentData = true
+    try {
+      const persistent = {}
+
+      if (statusCode) {
+        persistent[addresses.HTTP_INCOMING_RESPONSE_CODE] = String(statusCode)
+      }
+
+      if (responseHeaders) {
+        const filteredHeaders = { ...responseHeaders }
+        delete filteredHeaders['set-cookie']
+        persistent[addresses.HTTP_INCOMING_RESPONSE_HEADERS] = filteredHeaders
+      }
+
+      const samplingDecision = apiSecurity.sampleRootSpanRequest(span, {
+        method,
+        statusCode,
+        route,
+        // The tracer does not block in Lambda yet, so a response is never a blocked one.
+        blocked: false,
+      }, true)
+
+      if (samplingDecision === apiSecurity.SamplingDecision.SAMPLE) {
+        persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true }
+      }
+
+      let wafResult
+      if (!isEmpty(persistent)) {
+        wafResult = waf.run({ persistent }, req, undefined, span)
+      }
+
+      apiSecurity.reportRootSpanRequest(span, samplingDecision, wafResult)
+    } finally {
+      // The execution environment outlives the invocation, so the native WAF context and the
+      // module-level metrics queue must be released even if the work above threw.
+      waf.disposeContext(req)
+
+      Reporter.finishRequest(req, null, {}, undefined, span)
     }
-
-    if (hasPersistentData) {
-      waf.run({ persistent }, req, undefined, span)
-    }
-
-    waf.disposeContext(req)
-
-    Reporter.finishRequest(req, null, {}, undefined, span)
   } catch (err) {
     log.error('[ASM] Error in Lambda end-invocation handler', err)
   }

package/packages/dd-trace/src/ci-visibility/encode/json-encoder.js

@@ -1,12 +1,29 @@
 'use strict'
 
 class JSONEncoder {
-  constructor () {
+  #bytes = 2
+  #maxBytes
+
+  /**
+   * @param {number} [maxBytes]
+   */
+  constructor (maxBytes) {
+    this.#maxBytes = maxBytes
     this.payloads = []
   }
 
   encode (payload) {
+    if (this.#maxBytes !== undefined) {
+      const serializedPayload = JSON.stringify(payload) ?? 'null'
+      const payloadBytes = Buffer.byteLength(serializedPayload)
+      const separatorBytes = this.payloads.length === 0 ? 0 : 1
+      if (payloadBytes + separatorBytes > this.#maxBytes - this.#bytes) return false
+      this.#bytes += payloadBytes + separatorBytes
+      this.payloads.push(serializedPayload)
+      return true
+    }
     this.payloads.push(payload)
+    return true
   }
 
   count () {
@@ -14,11 +31,24 @@ class JSONEncoder {
   }
 
   reset () {
+    this.#bytes = 2
     this.payloads = []
   }
 
   makePayload () {
-    const data = JSON.stringify(this.payloads)
+    let data
+    if (this.#maxBytes === undefined) {
+      data = JSON.stringify(this.payloads)
+    } else if (this.payloads.length === 0) {
+      data = ['[', ']']
+    } else {
+      data = new Array(this.payloads.length * 2 + 1)
+      data[0] = '['
+      for (let index = 0; index < this.payloads.length; index++) {
+        data[index * 2 + 1] = this.payloads[index]
+        data[index * 2 + 2] = index === this.payloads.length - 1 ? ']' : ','
+      }
+    }
     this.reset()
     return data
   }

package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -10,6 +10,7 @@ const { DEBUGGER_INPUT_V1 } = require('../../../debugger/constants')
 
 // Product-specific discovery: newest advertised version, skip v3 (citestcycle), gzip if >= v4.
 // Shared `evp_proxy` discovery is an explicit path allowlist and does not cover this contract.
+const AGENT_INFO_TIMEOUT_MS = 5000
 const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
 const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
 
@@ -47,14 +48,19 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
     } = config
 
     const initializationController = new AbortController()
-    const initializationOptions = { signal: initializationController.signal }
+    const initializationOptions = {
+      deadline: Date.now() + AGENT_INFO_TIMEOUT_MS,
+      signal: initializationController.signal,
+      // Test runners await agent discovery before starting. A detached retry can let Node exit
+      // while that promise is still pending because promises alone do not keep the event loop alive.
+      keepProcessAlive: true,
+    }
     this._initializationRequest = {
       controller: initializationController,
       options: initializationOptions,
     }
 
     fetchAgentInfo(this._url, (err, agentInfo) => {
-      const initializationFinalFlush = this._initializationRequest?.finalFlush
       this._initializationRequest = undefined
       const initializationAborted = initializationController.signal.aborted
       const agentInfoError = err || (initializationAborted ? initializationController.signal.reason : undefined)
@@ -112,14 +118,8 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
       this._resolveCanUseCiVisProtocol(isEvpCompatible)
       if (initializationAborted) {
         this.resetUncodedTraces()
-        this.resetDeferredTestSuiteSpans()
         return
       }
-      if (isEvpCompatible) {
-        if (initializationFinalFlush) this.exportDeferredTestSuiteSpans()
-      } else {
-        this.resetDeferredTestSuiteSpans()
-      }
       this.exportUncodedTraces()
       this.exportUncodedCoverages()
     }, initializationOptions, request)

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -16,6 +16,7 @@ const {
 } = require('../../../ci-visibility/telemetry')
 const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const { getAgent } = require('../agents')
 const request = require('../request')
 const TestOptimizationRequestTracker = require('./request-tracker')
 
@@ -23,7 +24,7 @@ class Writer extends BaseWriter {
   #requestTracker
 
   constructor ({ url, evpProxyPrefix = '' }) {
-    super(...arguments)
+    super({ ...arguments[0], retainOnBackpressure: true })
     this.#requestTracker = new TestOptimizationRequestTracker(this)
     this._url = url
     this._encoder = new CoverageCIVisibilityEncoder(this)
@@ -51,6 +52,7 @@ class Writer extends BaseWriter {
       },
       timeout: 15_000,
       url: this._url,
+      agent: getAgent(this._url),
       deadline: flushOptions?.deadline,
     }
 
@@ -61,7 +63,7 @@ class Writer extends BaseWriter {
     }
 
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
-    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    log.debug(() => `Request to the intake: ${safeJSONStringify({ ...options, agent: undefined })}`)
 
     const startRequestTime = Date.now()
 
@@ -77,11 +79,11 @@ class Writer extends BaseWriter {
       if (err) {
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'code_coverage', statusCode }
+          { endpoint: 'code_coverage', statusCode, errorType: statusCode ? undefined : err.code }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-          { endpoint: 'code_coverage' }
+          { endpoint: 'code_coverage', statusCode, errorType: statusCode ? undefined : err.code }
         )
         log.error('Error sending CI coverage payload', err)
         done(err)

package/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js

@@ -5,7 +5,14 @@ const { safeJSONStringify } = require('../../../exporters/common/util')
 const { JSONEncoder } = require('../../encode/json-encoder')
 const { DEBUGGER_INPUT_V1 } = require('../../../debugger/constants')
 const BaseWriter = require('../../../exporters/common/writer')
+const {
+  incrementCountMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+} = require('../../../ci-visibility/telemetry')
 
+const { getAgent } = require('../agents')
+const { MAX_DI_LOG_BUFFERED_BYTES } = require('../limits')
 const request = require('../request')
 const TestOptimizationRequestTracker = require('./request-tracker')
 
@@ -17,10 +24,10 @@ class DynamicInstrumentationLogsWriter extends BaseWriter {
 
   // TODO: what's a good value for timeout for the logs intake?
   constructor ({ url, timeout = 15_000, isAgentProxy = false }) {
-    super(...arguments)
+    super({ ...arguments[0], retainOnBackpressure: true })
     this.#requestTracker = new TestOptimizationRequestTracker(this)
     this._url = url
-    this._encoder = new JSONEncoder()
+    this._encoder = new JSONEncoder(MAX_DI_LOG_BUFFERED_BYTES)
     this._isAgentProxy = isAgentProxy
     this.timeout = timeout
   }
@@ -46,6 +53,7 @@ class DynamicInstrumentationLogsWriter extends BaseWriter {
       },
       timeout: this.timeout,
       url: this._url,
+      agent: getAgent(this._url),
       deadline: flushOptions?.deadline,
     }
 
@@ -55,10 +63,18 @@ class DynamicInstrumentationLogsWriter extends BaseWriter {
     }
 
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
-    log.debug(() => `Request to the logs intake: ${safeJSONStringify(options)}`)
+    log.debug(() => `Request to the logs intake: ${safeJSONStringify({ ...options, agent: undefined })}`)
 
-    this.#requestTracker.send(request, data, options, (err, res) => {
+    this.#requestTracker.send(request, data, options, (err, res, statusCode) => {
       if (err) {
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+          { endpoint: 'di_logs', statusCode, errorType: statusCode ? undefined : err.code }
+        )
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+          { endpoint: 'di_logs', statusCode, errorType: statusCode ? undefined : err.code }
+        )
         log.error('Error sending DI logs payload', err)
         done(err)
         return

package/packages/dd-trace/src/ci-visibility/exporters/agentless/request-tracker.js

@@ -1,179 +1,9 @@
 'use strict'
 
 const BaseWriter = require('../../../exporters/common/writer')
+const FinalFlushRequestTracker = require('../../../exporters/common/final-flush-request-tracker')
 const { createFinalFlushTimeoutError } = require('../../final-flush')
 
-class FinalFlushRequestTracker {
-  #flush
-  #pendingRequests = new Set()
-  #finalFlushes = new Set()
-  #activeFinalFlush
-
-  /**
-   * Creates request tracking for a Test Optimization exporter.
-   *
-   * @param {(done?: (error?: Error) => void, options?: { deadline?: number }) => void} flush
-   */
-  constructor (flush) {
-    this.#flush = flush
-  }
-
-  /**
-   * Flushes queued payloads and, for a final flush, waits for requests that were
-   * already in flight. The absolute deadline prevents Test Optimization from
-   * keeping the test process alive indefinitely.
-   *
-   * @param {(error?: Error) => void} [done]
-   * @param {{ deadline?: number }} [options]
-   * @returns {void}
-   */
-  flush (done, options) {
-    if (options?.deadline === undefined) {
-      this.#flush(done, options)
-      return
-    }
-
-    const finalFlush = {
-      deadline: options.deadline,
-      done: done || (() => {}),
-      error: undefined,
-      requests: new Set(),
-      timeoutId: undefined,
-      writerDone: false,
-    }
-    this.#finalFlushes.add(finalFlush)
-
-    for (const pendingRequest of this.#pendingRequests) {
-      this.#attachRequest(finalFlush, pendingRequest)
-    }
-
-    const remaining = Math.max(0, options.deadline - Date.now())
-    finalFlush.timeoutId = setTimeout(() => {
-      const error = createFinalFlushTimeoutError()
-
-      finalFlush.error ||= error
-      finalFlush.writerDone = true
-
-      for (const pendingRequest of finalFlush.requests) {
-        pendingRequest.finalFlushes.delete(finalFlush)
-        if (pendingRequest.finalFlushes.size === 0) {
-          pendingRequest.controller.abort(error)
-          this.#pendingRequests.delete(pendingRequest)
-        } else {
-          this.#updateRequestDeadline(pendingRequest)
-        }
-      }
-      finalFlush.requests.clear()
-      this.#finishFinalFlush(finalFlush)
-    }, remaining)
-
-    const previousFinalFlush = this.#activeFinalFlush
-    this.#activeFinalFlush = finalFlush
-    try {
-      this.#flush((error) => {
-        finalFlush.error ||= error
-        finalFlush.writerDone = true
-        this.#finishFinalFlush(finalFlush)
-      }, options)
-    } finally {
-      this.#activeFinalFlush = previousFinalFlush
-    }
-    this.#finishFinalFlush(finalFlush)
-  }
-
-  /**
-   * Sends and tracks a request so a later final flush can wait for or abort it.
-   *
-   * @param {Function} request
-   * @param {Buffer|string|object} data
-   * @param {object} options
-   * @param {(error: Error|null, result?: string|null, statusCode?: number,
-   *   headers?: import('node:http').IncomingHttpHeaders) => void} callback
-   * @returns {void}
-   */
-  send (request, data, options, callback) {
-    const controller = new AbortController()
-    const requestOptions = { ...options, signal: controller.signal }
-    const pendingRequest = { controller, finalFlushes: new Set(), options: requestOptions }
-    this.#pendingRequests.add(pendingRequest)
-    if (this.#activeFinalFlush) this.#attachRequest(this.#activeFinalFlush, pendingRequest)
-
-    try {
-      request(data, requestOptions, (error, result, statusCode, headers) => {
-        if (error) {
-          for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
-        }
-
-        try {
-          callback(error, result, statusCode, headers)
-        } finally {
-          this.#dropRequest(pendingRequest)
-        }
-      })
-    } catch (error) {
-      this.#dropRequest(pendingRequest)
-      throw error
-    }
-  }
-
-  /**
-   * Stops tracking a settled request and releases final flushes waiting for it.
-   *
-   * @param {object} pendingRequest
-   * @returns {void}
-   */
-  #dropRequest (pendingRequest) {
-    this.#pendingRequests.delete(pendingRequest)
-    for (const finalFlush of pendingRequest.finalFlushes) {
-      finalFlush.requests.delete(pendingRequest)
-      this.#finishFinalFlush(finalFlush)
-    }
-    pendingRequest.finalFlushes.clear()
-  }
-
-  /**
-   * Associates a request with the final flush boundary that must wait for it.
-   *
-   * @param {object} finalFlush
-   * @param {object} pendingRequest
-   * @returns {void}
-   */
-  #attachRequest (finalFlush, pendingRequest) {
-    finalFlush.requests.add(pendingRequest)
-    pendingRequest.finalFlushes.add(finalFlush)
-    this.#updateRequestDeadline(pendingRequest)
-  }
-
-  /**
-   * Gives a shared request the latest deadline of the flushes waiting for it.
-   *
-   * @param {object} pendingRequest
-   * @returns {void}
-   */
-  #updateRequestDeadline (pendingRequest) {
-    let deadline = 0
-    for (const finalFlush of pendingRequest.finalFlushes) {
-      deadline = Math.max(deadline, finalFlush.deadline)
-    }
-    pendingRequest.options.deadline = deadline
-  }
-
-  /**
-   * Completes a final flush callback once its writer and associated requests
-   * have settled.
-   *
-   * @param {object} finalFlush
-   * @returns {void}
-   */
-  #finishFinalFlush (finalFlush) {
-    if (!this.#finalFlushes.has(finalFlush) || !finalFlush.writerDone || finalFlush.requests.size !== 0) return
-
-    this.#finalFlushes.delete(finalFlush)
-    clearTimeout(finalFlush.timeoutId)
-    finalFlush.done(finalFlush.error)
-  }
-}
-
 class TestOptimizationRequestTracker extends FinalFlushRequestTracker {
   /**
    * Creates request tracking for a Test Optimization writer.
@@ -181,9 +11,11 @@ class TestOptimizationRequestTracker extends FinalFlushRequestTracker {
    * @param {BaseWriter} writer
    */
   constructor (writer) {
-    super((done, options) => BaseWriter.prototype.flushDirect.call(writer, done, options))
+    super(
+      (done, options) => BaseWriter.prototype.flushDirect.call(writer, done, options),
+      createFinalFlushTimeoutError
+    )
   }
 }
 
 module.exports = TestOptimizationRequestTracker
-module.exports.FinalFlushRequestTracker = FinalFlushRequestTracker

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -16,6 +16,7 @@ const {
 } = require('../../../ci-visibility/telemetry')
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const { getAgent } = require('../agents')
 const request = require('../request')
 const TestOptimizationRequestTracker = require('./request-tracker')
 
@@ -23,11 +24,10 @@ class Writer extends BaseWriter {
   #requestTracker
 
   constructor ({ url, tags, evpProxyPrefix = '' }) {
-    super(...arguments)
+    super({ ...arguments[0], retainOnBackpressure: true })
     this.#requestTracker = new TestOptimizationRequestTracker(this)
-    const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
-    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { tags })
     this._evpProxyPrefix = evpProxyPrefix
   }
 
@@ -52,6 +52,7 @@ class Writer extends BaseWriter {
       },
       timeout: 15_000,
       url: this._url,
+      agent: getAgent(this._url),
       deadline: flushOptions?.deadline,
     }
 
@@ -62,7 +63,7 @@ class Writer extends BaseWriter {
     }
 
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
-    log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+    log.debug(() => `Request to the intake: ${safeJSONStringify({ ...options, agent: undefined })}`)
 
     const startRequestTime = Date.now()
 
@@ -78,11 +79,11 @@ class Writer extends BaseWriter {
       if (err) {
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
-          { endpoint: 'test_cycle', statusCode }
+          { endpoint: 'test_cycle', statusCode, errorType: statusCode ? undefined : err.code }
         )
         incrementCountMetric(
           TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
-          { endpoint: 'test_cycle' }
+          { endpoint: 'test_cycle', statusCode, errorType: statusCode ? undefined : err.code }
         )
         log.error('Error sending CI agentless payload', err)
         done(err)

package/packages/dd-trace/src/ci-visibility/exporters/agents.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+
+const { createAgents } = require('../../exporters/common/agents')
+
+// Finalization can flush several Test Optimization payload types together. Keep media on a separate
+// bounded pool so long uploads cannot consume the sockets needed by trace, coverage, and log writers.
+const maxSockets = 16
+
+const payloadAgents = createAgents(maxSockets)
+const mediaAgents = createAgents(maxSockets)
+
+/**
+ * Selects an agent by intake URL protocol.
+ *
+ * @param {string|URL|object} url
+ * @param {{httpAgent: http.Agent, httpsAgent: https.Agent}} agents
+ * @returns {http.Agent|https.Agent}
+ */
+function selectAgent (url, { httpAgent, httpsAgent }) {
+  const protocol = url?.protocol
+  if (protocol === 'https:' || protocol === 'https') return httpsAgent
+  if (protocol === 'http:' || protocol === 'http') return httpAgent
+
+  try {
+    return new URL(url).protocol === 'https:' ? httpsAgent : httpAgent
+  } catch {
+    return String(url).startsWith('https:') ? httpsAgent : httpAgent
+  }
+}
+
+/**
+ * Selects the dedicated Test Optimization payload agent for an intake URL.
+ *
+ * @param {string|URL|object} url
+ * @returns {http.Agent|https.Agent}
+ */
+function getAgent (url) {
+  return selectAgent(url, payloadAgents)
+}
+
+/**
+ * Selects the isolated Test Optimization media agent for an intake URL.
+ *
+ * @param {string|URL|object} url
+ * @returns {http.Agent|https.Agent}
+ */
+function getMediaAgent (url) {
+  return selectAgent(url, mediaAgents)
+}
+
+module.exports = { getAgent, getMediaAgent }

package/packages/dd-trace/src/ci-visibility/exporters/ci-validation/writer.js

@@ -11,9 +11,8 @@ class CiValidationWriter {
    * @param {object} options.tags tracer tags
    */
   constructor ({ sink, tags }) {
-    const { 'runtime-id': runtimeId, env, service } = tags
     this._sink = sink
-    this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
+    this._encoder = new AgentlessCiVisibilityEncoder(this, { tags })
   }
 
   /**