dd-trace 5.54.0 → 5.56.0

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -8,6 +8,7 @@ const dc = require('dc-polyfill')
 const { fileURLToPath } = require('url')
 const { isTrue } = require('../../src/util')
 
+/** @type {Set<string>} */
 const savedDependenciesToSend = new Set()
 const detectedDependencyKeys = new Set()
 const detectedDependencyVersions = new Set()
@@ -15,63 +16,66 @@ const detectedDependencyVersions = new Set()
 const FILE_URI_START = 'file://'
 const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
 
-let immediate, config, application, host, initialLoad
+let config, application, host, initialLoad
 let isFirstModule = true
 let getRetryData
 let updateRetryData
 
-function createBatchPayload (payload) {
-  const batchPayload = payload.map(item => {
-    return {
-      request_type: item.reqType,
-      payload: item.payload
-    }
-  })
-
-  return batchPayload
-}
 function waitAndSend (config, application, host) {
-  if (!immediate) {
-    immediate = setImmediate(() => {
-      immediate = null
-      if (savedDependenciesToSend.size > 0) {
-        const dependencies = [...savedDependenciesToSend.values()]
-          // if a dependency is from the initial load, *always* send the event
-          // Otherwise, only send if dependencyCollection is enabled
-          .filter(dep => {
-            const initialLoadModule = isTrue(dep.split(' ')[2])
-            const sendModule = initialLoadModule || (config.telemetry?.dependencyCollection)
-
-            if (!sendModule) savedDependenciesToSend.delete(dep) // we'll never send it
-            return sendModule
-          })
-          .splice(0, 2000) // v2 documentation specifies up to 2000 dependencies can be sent at once
-          .map(pair => {
-            savedDependenciesToSend.delete(pair)
-            const [name, version] = pair.split(' ')
-            return { name, version }
-          })
-        let currPayload
-        const retryData = getRetryData()
-        if (retryData) {
-          currPayload = { reqType: 'app-dependencies-loaded', payload: { dependencies } }
-        } else {
-          if (!dependencies.length) return // no retry data and no dependencies, nothing to send
-          currPayload = { dependencies }
+  setImmediate(() => {
+    if (savedDependenciesToSend.size === 0) {
+      return
+    }
+    const dependencies = []
+    let send = 0
+    for (const dependency of savedDependenciesToSend) {
+      const [name, version, initialLoadModule] = dependency.split(' ')
+      // If a dependency is from the initial load, *always* send the event
+      // Otherwise, only send if dependencyCollection is enabled
+      const sendModule = isTrue(initialLoadModule) || config.telemetry?.dependencyCollection
+
+      savedDependenciesToSend.delete(dependency)
+
+      if (sendModule) {
+        dependencies.push({ name, version })
+        send++
+        if (send === 2000) {
+          // v2 documentation specifies up to 2000 dependencies can be sent at once
+          break
         }
+      }
+    }
 
-        const payload = retryData ? createBatchPayload([currPayload, retryData]) : currPayload
-        const reqType = retryData ? 'message-batch' : 'app-dependencies-loaded'
+    /**
+     * @type { { dependencies: typeof dependencies } | {
+     *   request_type: string,
+     *   payload: typeof dependencies
+     * }[]}
+     */
+    let payload = { dependencies }
+    let reqType = 'app-dependencies-loaded'
+    const retryData = getRetryData()
+
+    if (retryData) {
+      payload = [{
+        request_type: 'app-dependencies-loaded',
+        payload
+      }, {
+        request_type: retryData.reqType,
+        payload: retryData.payload
+      }]
+      reqType = 'message-batch'
+    } else if (!dependencies.length) {
+      // No retry data and no dependencies, nothing to send
+      return
+    }
 
-        sendData(config, application, host, reqType, payload, updateRetryData)
+    sendData(config, application, host, reqType, payload, updateRetryData)
 
-        if (savedDependenciesToSend.size > 0) {
-          waitAndSend(config, application, host)
-        }
-      }
-    })
-    immediate.unref()
-  }
+    if (savedDependenciesToSend.size > 0) {
+      waitAndSend(config, application, host)
+    }
+  }).unref()
 }
 
 function loadAllTheLoadedModules () {
@@ -91,7 +95,7 @@ function onModuleLoad (data) {
 
   if (data) {
     let filename = data.filename
-    if (filename && filename.startsWith(FILE_URI_START)) {
+    if (filename?.startsWith(FILE_URI_START)) {
       try {
         filename = fileURLToPath(filename)
       } catch {
@@ -99,10 +103,10 @@ function onModuleLoad (data) {
       }
     }
     const parseResult = filename && parse(filename)
-    const request = data.request || (parseResult && parseResult.name)
-    const dependencyKey = parseResult && parseResult.basedir ? parseResult.basedir : request
+    const request = data.request || parseResult?.name
+    const dependencyKey = parseResult?.basedir ?? request
 
-    if (filename && request && isDependency(filename, request) && !detectedDependencyKeys.has(dependencyKey)) {
+    if (filename && request && isDependency(request) && !detectedDependencyKeys.has(dependencyKey)) {
       detectedDependencyKeys.add(dependencyKey)
 
       if (parseResult) {
@@ -135,20 +139,21 @@ function start (_config = {}, _application, _host, getRetryDataFunction, updateR
   updateRetryData = updateRetryDatafunction
   moduleLoadStartChannel.subscribe(onModuleLoad)
 
-  // try and capture intially loaded modules in the first tick
+  // Try and capture initially loaded modules in the first tick
   // since, ideally, the tracer (and this module) should be loaded first,
   // this should capture any first-tick dependencies
   queueMicrotask(() => { initialLoad = false })
 }
 
-function isDependency (filename, request) {
-  const isDependencyWithSlash = isDependencyWithSeparator(filename, request, '/')
+function isDependency (request) {
+  const isDependencyWithSlash = isDependencyWithSeparator(request, '/')
   if (isDependencyWithSlash && process.platform === 'win32') {
-    return isDependencyWithSeparator(filename, request, path.sep)
+    return isDependencyWithSeparator(request, path.sep)
   }
   return isDependencyWithSlash
 }
-function isDependencyWithSeparator (filename, request, sep) {
+
+function isDependencyWithSeparator (request, sep) {
   return request.indexOf(`..${sep}`) !== 0 &&
     request.indexOf(`.${sep}`) !== 0 &&
     request.indexOf(sep) !== 0 &&

package/packages/dd-trace/src/telemetry/send-data.js

@@ -1,6 +1,7 @@
 const request = require('../exporters/common/request')
 const log = require('../log')
 const { isTrue } = require('../util')
+const { getEnvironmentVariable } = require('../config-helper')
 
 let agentTelemetry = true
 
@@ -36,10 +37,9 @@ function getPayload (payload) {
   // 'logs' request type payload is meant to send library logs to Datadog’s backend.
   if (Array.isArray(payload)) {
     return payload
-  } else {
-    const { logger, tags, serviceMapping, ...trimmedPayload } = payload
-    return trimmedPayload
   }
+  const { logger, tags, serviceMapping, ...trimmedPayload } = payload
+  return trimmedPayload
 }
 
 function sendData (config, application, host, reqType, payload = {}, cb = () => {}) {
@@ -51,7 +51,8 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
 
   let url = config.url
 
-  const isCiVisibilityAgentlessMode = isCiVisibility && isTrue(process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED)
+  const isCiVisibilityAgentlessMode = isCiVisibility &&
+                                      isTrue(getEnvironmentVariable('DD_CIVISIBILITY_AGENTLESS_ENABLED'))
 
   if (isCiVisibilityAgentlessMode) {
     try {
@@ -85,14 +86,14 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
   })
 
   request(data, options, (error) => {
-    if (error && process.env.DD_API_KEY && config.site) {
+    if (error && getEnvironmentVariable('DD_API_KEY') && config.site) {
       if (agentTelemetry) {
         log.warn('Agent telemetry failed, started agentless telemetry')
         agentTelemetry = false
       }
       // figure out which data center to send to
       const backendUrl = getAgentlessTelemetryEndpoint(config.site)
-      const backendHeader = { ...options.headers, 'DD-API-KEY': process.env.DD_API_KEY }
+      const backendHeader = { ...options.headers, 'DD-API-KEY': getEnvironmentVariable('DD_API_KEY') }
       const backendOptions = {
         ...options,
         url: backendUrl,

package/packages/dd-trace/src/telemetry/telemetry.js

@@ -22,12 +22,14 @@ let heartbeatTimeout
 let heartbeatInterval
 let extendedInterval
 let integrations
-let configWithOrigin = []
+const configWithOrigin = new Map()
 let retryData = null
 const extendedHeartbeatPayload = {}
 
 const sentIntegrations = new Set()
 
+let seqId = 0
+
 function getRetryData () {
   return retryData
 }
@@ -113,7 +115,7 @@ function getInstallSignature (config) {
 function appStarted (config) {
   const app = {
     products: getProducts(config),
-    configuration: configWithOrigin
+    configuration: [...configWithOrigin.values()]
   }
   const installSignature = getInstallSignature(config)
   if (installSignature) {
@@ -282,7 +284,7 @@ function stop () {
 }
 
 function updateIntegrations () {
-  if (!config || !config.telemetry.enabled) {
+  if (!config?.telemetry.enabled) {
     return
   }
   const integrations = getIntegrations()
@@ -322,6 +324,8 @@ const nameMapping = {
   traceId128BitLoggingEnabled: 'DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED'
 }
 
+const namesNeedFormatting = new Set(['DD_TAGS', 'peerServiceMapping', 'serviceMapping'])
+
 function updateConfig (changes, config) {
   if (!config.telemetry.enabled) return
   if (changes.length === 0) return
@@ -331,17 +335,12 @@ function updateConfig (changes, config) {
   const application = createAppObject(config)
   const host = createHostObject()
 
-  const namesNeedFormatting = new Set(['DD_TAGS', 'peerServiceMapping', 'serviceMapping'])
-
-  const configuration = []
-  const names = [] // list of config names whose values have been changed
+  const changed = configWithOrigin.size > 0
 
   for (const change of changes) {
     const name = nameMapping[change.name] || change.name
-
-    names.push(name)
     const { origin, value } = change
-    const entry = { name, value, origin }
+    const entry = { name, value, origin, seq_id: seqId++ }
 
     if (namesNeedFormatting.has(entry.name)) {
       entry.value = formatMapForTelemetry(entry.value)
@@ -354,21 +353,17 @@ function updateConfig (changes, config) {
     } else if (Array.isArray(entry.value)) {
       entry.value = value.join(',')
     }
-    configuration.push(entry)
-  }
 
-  function isNotModified (entry) {
-    return !names.includes(entry.name)
+    // Use composite key to support multiple origins for same config name
+    configWithOrigin.set(`${name}|${origin}`, entry)
   }
 
-  if (configWithOrigin.length) {
+  if (changed) {
     // update configWithOrigin to contain up-to-date full list of config values for app-extended-heartbeat
-    configWithOrigin = configWithOrigin.filter(isNotModified)
-    configWithOrigin = [...configWithOrigin, ...configuration]
-    const { reqType, payload } = createPayload('app-client-configuration-change', { configuration })
+    const { reqType, payload } = createPayload('app-client-configuration-change', {
+      configuration: [...configWithOrigin.values()]
+    })
     sendData(config, application, host, reqType, payload, updateRetryData)
-  } else {
-    configWithOrigin = configuration
   }
 }
 
@@ -376,12 +371,7 @@ function profilingEnabledToBoolean (profilingEnabled) {
   if (typeof profilingEnabled === 'boolean') {
     return profilingEnabled
   }
-  if (['auto', 'true'].includes(profilingEnabled)) {
-    return true
-  }
-  if (profilingEnabled === 'false') {
-    return false
-  }
+  return profilingEnabled === 'true' || profilingEnabled === 'auto'
 }
 
 module.exports = {

package/packages/dd-trace/src/tracer.js

@@ -46,9 +46,7 @@ class DatadogTracer extends Tracer {
   }
 
   trace (name, options, fn) {
-    options = Object.assign({
-      childOf: this.scope().active()
-    }, options)
+    options = { childOf: this.scope().active(), ...options }
 
     const span = this.startSpan(name, options)
 
@@ -76,9 +74,8 @@ class DatadogTracer extends Tracer {
             throw err
           }
         )
-      } else {
-        span.finish()
       }
+      span.finish()
 
       return result
     } catch (e) {
@@ -110,9 +107,8 @@ class DatadogTracer extends Tracer {
 
           return fn.apply(this, arguments)
         })
-      } else {
-        return tracer.trace(name, optionsObj, () => fn.apply(this, arguments))
       }
+      return tracer.trace(name, optionsObj, () => fn.apply(this, arguments))
     }
   }
 

package/packages/dd-trace/src/util.js

@@ -69,10 +69,6 @@ function calculateDDBasePath (dirname) {
   return dirSteps.slice(0, packagesIndex + 1).join(path.sep) + path.sep
 }
 
-function hasOwn (object, prop) {
-  return Object.prototype.hasOwnProperty.call(object, prop)
-}
-
 function normalizeProfilingEnabledValue (configValue) {
   return isTrue(configValue)
     ? 'true'
@@ -87,6 +83,5 @@ module.exports = {
   isError,
   globMatch,
   ddBasePath: calculateDDBasePath(__dirname),
-  hasOwn,
   normalizeProfilingEnabledValue
 }

package/packages/datadog-core/src/utils/src/get.js

@@ -1,11 +0,0 @@
-'use strict'
-
-module.exports = function get (object, path) {
-  const pathArr = path.split('.')
-  let val = object
-  for (const p of pathArr) {
-    if (val === undefined) return val
-    val = val[p]
-  }
-  return val
-}

package/packages/datadog-core/src/utils/src/has.js

@@ -1,14 +0,0 @@
-'use strict'
-
-module.exports = function has (object, path) {
-  const pathArr = path.split('.')
-  let property = object
-  for (const n of pathArr) {
-    if (property.hasOwnProperty(n)) {
-      property = property[n]
-    } else {
-      return false
-    }
-  }
-  return true
-}

package/packages/dd-trace/src/appsec/iast/analyzers/header-injection-analyzer.js

@@ -1,120 +0,0 @@
-'use strict'
-
-const InjectionAnalyzer = require('./injection-analyzer')
-const { HEADER_INJECTION } = require('../vulnerabilities')
-const { getNodeModulesPaths } = require('../path-line')
-const { HEADER_NAME_VALUE_SEPARATOR } = require('../vulnerabilities-formatter/constants')
-const { getRanges } = require('../taint-tracking/operations')
-const {
-  HTTP_REQUEST_COOKIE_VALUE,
-  HTTP_REQUEST_HEADER_VALUE
-} = require('../taint-tracking/source-types')
-
-const EXCLUDED_PATHS = getNodeModulesPaths('express')
-const EXCLUDED_HEADER_NAMES = new Set([
-  'location',
-  'sec-websocket-location',
-  'sec-websocket-accept',
-  'upgrade',
-  'connection'
-])
-
-class HeaderInjectionAnalyzer extends InjectionAnalyzer {
-  constructor () {
-    super(HEADER_INJECTION)
-  }
-
-  onConfigure () {
-    this.addSub('datadog:http:server:response:set-header:finish', ({ name, value }) => {
-      if (Array.isArray(value)) {
-        for (const headerValue of value) {
-          this.analyze({ name, value: headerValue })
-        }
-      } else {
-        this.analyze({ name, value })
-      }
-    })
-  }
-
-  _isVulnerable ({ name, value }, iastContext) {
-    const lowerCasedHeaderName = name?.trim().toLowerCase()
-
-    if (this.isExcludedHeaderName(lowerCasedHeaderName) || typeof value !== 'string') return
-
-    const ranges = getRanges(iastContext, value)
-    return ranges?.length > 0 && !this.shouldIgnoreHeader(lowerCasedHeaderName, ranges)
-  }
-
-  _getEvidence (headerInfo, iastContext) {
-    const prefix = headerInfo.name + HEADER_NAME_VALUE_SEPARATOR
-    const prefixLength = prefix.length
-
-    const evidence = super._getEvidence(headerInfo.value, iastContext)
-    evidence.value = prefix + evidence.value
-    evidence.ranges = evidence.ranges.map(range => {
-      return {
-        ...range,
-        start: range.start + prefixLength,
-        end: range.end + prefixLength
-      }
-    })
-
-    return evidence
-  }
-
-  isExcludedHeaderName (name) {
-    return EXCLUDED_HEADER_NAMES.has(name)
-  }
-
-  isAllRangesFromHeader (ranges, headerName) {
-    return ranges
-      .every(range =>
-        range.iinfo.type === HTTP_REQUEST_HEADER_VALUE && range.iinfo.parameterName?.toLowerCase() === headerName
-      )
-  }
-
-  isAllRangesFromSource (ranges, source) {
-    return ranges
-      .every(range => range.iinfo.type === source)
-  }
-
-  /**
-   * Exclude access-control-allow-*: when the header starts with access-control-allow- and the
-   * source of the tainted range is a request header
-   */
-  isAccessControlAllowExclusion (name, ranges) {
-    if (name?.startsWith('access-control-allow-')) {
-      return this.isAllRangesFromSource(ranges, HTTP_REQUEST_HEADER_VALUE)
-    }
-
-    return false
-  }
-
-  /** Exclude when the header is reflected from the request */
-  isSameHeaderExclusion (name, ranges) {
-    return ranges.length === 1 && name === ranges[0].iinfo.parameterName?.toLowerCase()
-  }
-
-  shouldIgnoreHeader (headerName, ranges) {
-    switch (headerName) {
-      case 'set-cookie':
-        /** Exclude set-cookie header if the source of all the tainted ranges are cookies */
-        return this.isAllRangesFromSource(ranges, HTTP_REQUEST_COOKIE_VALUE)
-      case 'pragma':
-        /** Ignore pragma headers when the source is the cache control header. */
-        return this.isAllRangesFromHeader(ranges, 'cache-control')
-      case 'transfer-encoding':
-      case 'content-encoding':
-        /** Ignore transfer and content encoding headers when the source is the accept encoding header. */
-        return this.isAllRangesFromHeader(ranges, 'accept-encoding')
-    }
-
-    return this.isAccessControlAllowExclusion(headerName, ranges) || this.isSameHeaderExclusion(headerName, ranges)
-  }
-
-  _getExcludedPaths () {
-    return EXCLUDED_PATHS
-  }
-}
-
-module.exports = new HeaderInjectionAnalyzer()

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/header-sensitive-analyzer.js

@@ -1,20 +0,0 @@
-'use strict'
-
-const { HEADER_NAME_VALUE_SEPARATOR } = require('../../constants')
-
-module.exports = function extractSensitiveRanges (evidence, namePattern, valuePattern) {
-  const evidenceValue = evidence.value
-  const sections = evidenceValue.split(HEADER_NAME_VALUE_SEPARATOR)
-  const headerName = sections[0]
-  const headerValue = sections.slice(1).join(HEADER_NAME_VALUE_SEPARATOR)
-  namePattern.lastIndex = 0
-  valuePattern.lastIndex = 0
-  if (namePattern.test(headerName) || valuePattern.test(headerValue)) {
-    return [{
-      start: headerName.length + HEADER_NAME_VALUE_SEPARATOR.length,
-      end: evidenceValue.length
-    }]
-  }
-
-  return []
-}