dd-trace 3.56.0 → 3.58.0

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js

@@ -6,6 +6,7 @@ const vulnerabilities = require('../../vulnerabilities')
 const { contains, intersects, remove } = require('./range-utils')
 
 const commandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer')
+const hardcodedPasswordAnalyzer = require('./sensitive-analyzers/hardcoded-password-analyzer')
 const headerSensitiveAnalyzer = require('./sensitive-analyzers/header-sensitive-analyzer')
 const jsonSensitiveAnalyzer = require('./sensitive-analyzers/json-sensitive-analyzer')
 const ldapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer')
@@ -31,6 +32,9 @@ class SensitiveHandler {
     this._sensitiveAnalyzers.set(vulnerabilities.HEADER_INJECTION, (evidence) => {
       return headerSensitiveAnalyzer(evidence, this._namePattern, this._valuePattern)
     })
+    this._sensitiveAnalyzers.set(vulnerabilities.HARDCODED_PASSWORD, (evidence) => {
+      return hardcodedPasswordAnalyzer(evidence, this._valuePattern)
+    })
   }
 
   isSensibleName (name) {
@@ -51,7 +55,9 @@ class SensitiveHandler {
     const sensitiveAnalyzer = this._sensitiveAnalyzers.get(vulnerabilityType)
     if (sensitiveAnalyzer) {
       const sensitiveRanges = sensitiveAnalyzer(evidence)
-      return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources)
+      if (evidence.ranges || sensitiveRanges?.length) {
+        return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources)
+      }
     }
     return null
   }
@@ -67,7 +73,7 @@ class SensitiveHandler {
     let nextTaintedIndex = 0
     let sourceIndex
 
-    let nextTainted = ranges.shift()
+    let nextTainted = ranges?.shift()
     let nextSensitive = sensitive.shift()
 
     for (let i = 0; i < value.length; i++) {

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -49,6 +49,10 @@ class VulnerabilityFormatter {
       evidence.ranges = ranges
     }
 
+    if (!evidence.ranges) {
+      return { value: evidence.value }
+    }
+
     evidence.ranges.forEach((range, rangeIndex) => {
       if (fromIndex < range.start) {
         valueParts.push({ value: evidence.value.substring(fromIndex, range.start) })
@@ -65,12 +69,8 @@ class VulnerabilityFormatter {
   }
 
   formatEvidence (type, evidence, sourcesIndexes, sources) {
-    if (!evidence.ranges && !evidence.rangesToApply) {
-      if (typeof evidence.value === 'undefined') {
-        return undefined
-      } else {
-        return { value: evidence.value }
-      }
+    if (typeof evidence.value === 'undefined') {
+      return undefined
     }
 
     return this._redactVulnearbilities

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -8,7 +8,7 @@ const apiSecuritySampler = require('../api_security_sampler')
 
 let rc
 
-function enable (config) {
+function enable (config, appsec) {
   rc = new RemoteConfigManager(config)
   rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true)
   rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
@@ -31,7 +31,7 @@ function enable (config) {
       if (!rcConfig) return
 
       if (activation === Activation.ONECLICK) {
-        enableOrDisableAppsec(action, rcConfig, config)
+        enableOrDisableAppsec(action, rcConfig, config, appsec)
       }
 
       apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
@@ -41,7 +41,7 @@ function enable (config) {
   return rc
 }
 
-function enableOrDisableAppsec (action, rcConfig, config) {
+function enableOrDisableAppsec (action, rcConfig, config, appsec) {
   if (typeof rcConfig.asm?.enabled === 'boolean') {
     let shouldEnable
 
@@ -52,9 +52,9 @@ function enableOrDisableAppsec (action, rcConfig, config) {
     }
 
     if (shouldEnable) {
-      require('..').enable(config)
+      appsec.enable(config)
     } else {
-      require('..').disable()
+      appsec.disable()
     }
   }
 }

package/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js

@@ -0,0 +1,56 @@
+'use strict'
+
+const Writer = require('./writer')
+const {
+  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
+  JEST_WORKER_TRACE_PAYLOAD_CODE,
+  CUCUMBER_WORKER_TRACE_PAYLOAD_CODE
+} = require('../../../plugins/util/test')
+
+function getInterprocessTraceCode () {
+  if (process.env.JEST_WORKER_ID) {
+    return JEST_WORKER_TRACE_PAYLOAD_CODE
+  }
+  if (process.env.CUCUMBER_WORKER_ID) {
+    return CUCUMBER_WORKER_TRACE_PAYLOAD_CODE
+  }
+  return null
+}
+
+// TODO: make it available with cucumber
+function getInterprocessCoverageCode () {
+  if (process.env.JEST_WORKER_ID) {
+    return JEST_WORKER_COVERAGE_PAYLOAD_CODE
+  }
+  return null
+}
+
+/**
+ * Lightweight exporter whose writers only do simple JSON serialization
+ * of trace and coverage payloads, which they send to the test framework's main process.
+ * Currently used by Jest and Cucumber workers.
+ */
+class TestWorkerCiVisibilityExporter {
+  constructor () {
+    const interprocessTraceCode = getInterprocessTraceCode()
+    const interprocessCoverageCode = getInterprocessCoverageCode()
+
+    this._writer = new Writer(interprocessTraceCode)
+    this._coverageWriter = new Writer(interprocessCoverageCode)
+  }
+
+  export (payload) {
+    this._writer.append(payload)
+  }
+
+  exportCoverage (formattedCoverage) {
+    this._coverageWriter.append(formattedCoverage)
+  }
+
+  flush () {
+    this._writer.flush()
+    this._coverageWriter.flush()
+  }
+}
+
+module.exports = TestWorkerCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/{jest-worker → test-worker}/writer.js

@@ -23,11 +23,18 @@ class Writer {
   }
 
   _sendPayload (data) {
+    // ## Jest
     // Only available when `child_process` is used for the jest worker.
     // eslint-disable-next-line
     // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker
     // If worker_threads is used, this will not work
     // TODO: make it compatible with worker_threads
+
+    // ## Cucumber
+    // This reports to the test's main process the same way test data is reported by Cucumber
+    // See cucumber code:
+    // eslint-disable-next-line
+    // https://github.com/cucumber/cucumber-js/blob/5ce371870b677fe3d1a14915dc535688946f734c/src/runtime/parallel/run_worker.ts#L13
     if (process.send) { // it only works if process.send is available
       process.send([this._interprocessCode, data])
     }

package/packages/dd-trace/src/config.js

@@ -446,6 +446,7 @@ class Config {
     this._setValue(defaults, 'appsec.obfuscatorValueRegex', defaultWafObfuscatorValueRegex)
     this._setValue(defaults, 'appsec.rateLimit', 100)
     this._setValue(defaults, 'appsec.rules', undefined)
+    this._setValue(defaults, 'appsec.sca.enabled', null)
     this._setValue(defaults, 'appsec.wafTimeout', 5e3) // µs
     this._setValue(defaults, 'clientIpEnabled', false)
     this._setValue(defaults, 'clientIpHeader', null)
@@ -516,6 +517,7 @@ class Config {
     this._setValue(defaults, 'tracing', true)
     this._setValue(defaults, 'url', undefined)
     this._setValue(defaults, 'version', pkg.version)
+    this._setValue(defaults, 'instrumentation_config_id', undefined)
   }
 
   _applyEnvironment () {
@@ -528,6 +530,7 @@ class Config {
       DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
       DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
       DD_APPSEC_RULES,
+      DD_APPSEC_SCA_ENABLED,
       DD_APPSEC_TRACE_RATE_LIMIT,
       DD_APPSEC_WAF_TIMEOUT,
       DD_DATA_STREAMS_ENABLED,
@@ -547,6 +550,7 @@ class Config {
       DD_IAST_REQUEST_SAMPLING,
       DD_IAST_TELEMETRY_VERBOSITY,
       DD_INSTRUMENTATION_TELEMETRY_ENABLED,
+      DD_INSTRUMENTATION_CONFIG_ID,
       DD_LOGS_INJECTION,
       DD_OPENAI_LOGS_ENABLED,
       DD_OPENAI_SPAN_CHAR_LIMIT,
@@ -615,6 +619,8 @@ class Config {
     this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP)
     this._setValue(env, 'appsec.rateLimit', maybeInt(DD_APPSEC_TRACE_RATE_LIMIT))
     this._setString(env, 'appsec.rules', DD_APPSEC_RULES)
+    // DD_APPSEC_SCA_ENABLED is never used locally, but only sent to the backend
+    this._setBoolean(env, 'appsec.sca.enabled', DD_APPSEC_SCA_ENABLED)
     this._setValue(env, 'appsec.wafTimeout', maybeInt(DD_APPSEC_WAF_TIMEOUT))
     this._setBoolean(env, 'clientIpEnabled', DD_TRACE_CLIENT_IP_ENABLED)
     this._setString(env, 'clientIpHeader', DD_TRACE_CLIENT_IP_HEADER)
@@ -695,6 +701,7 @@ class Config {
       DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs
       !(this._isInServerlessEnvironment() || JEST_WORKER_ID)
     ))
+    this._setString(env, 'instrumentation_config_id', DD_INSTRUMENTATION_CONFIG_ID)
     this._setBoolean(env, 'telemetry.debug', DD_TELEMETRY_DEBUG)
     this._setBoolean(env, 'telemetry.dependencyCollection', DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED)
     this._setValue(env, 'telemetry.heartbeatInterval', maybeInt(Math.floor(DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000)))

package/packages/dd-trace/src/exporter.js

@@ -18,7 +18,8 @@ module.exports = name => {
     case exporters.AGENT_PROXY:
       return require('./ci-visibility/exporters/agent-proxy')
     case exporters.JEST_WORKER:
-      return require('./ci-visibility/exporters/jest-worker')
+    case exporters.CUCUMBER_WORKER:
+      return require('./ci-visibility/exporters/test-worker')
     default:
       return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent')
   }

package/packages/dd-trace/src/plugins/database.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const StoragePlugin = require('./storage')
-const { PEER_SERVICE_KEY } = require('../constants')
+const { PEER_SERVICE_KEY, PEER_SERVICE_SOURCE_KEY } = require('../constants')
 
 class DatabasePlugin extends StoragePlugin {
   static get operation () { return 'query' }
@@ -28,16 +28,31 @@ class DatabasePlugin extends StoragePlugin {
     }
   }
 
-  createDBMPropagationCommentService (serviceName) {
+  createDBMPropagationCommentService (serviceName, span) {
     this.encodingServiceTags('dddbs', 'encodedDddbs', serviceName)
     this.encodingServiceTags('dde', 'encodedDde', this.tracer._env)
     this.encodingServiceTags('ddps', 'encodedDdps', this.tracer._service)
     this.encodingServiceTags('ddpv', 'encodedDdpv', this.tracer._version)
+    if (span.context()._tags['out.host']) {
+      this.encodingServiceTags('ddh', 'encodedDdh', span._spanContext._tags['out.host'])
+    }
+    if (span.context()._tags['db.name']) {
+      this.encodingServiceTags('dddb', 'encodedDddb', span._spanContext._tags['db.name'])
+    }
 
-    const { encodedDddbs, encodedDde, encodedDdps, encodedDdpv } = this.serviceTags
+    const { encodedDddb, encodedDddbs, encodedDde, encodedDdh, encodedDdps, encodedDdpv } = this.serviceTags
 
-    return `dddbs='${encodedDddbs}',dde='${encodedDde}',` +
+    let dbmComment = `dddb='${encodedDddb}',dddbs='${encodedDddbs}',dde='${encodedDde}',ddh='${encodedDdh}',` +
       `ddps='${encodedDdps}',ddpv='${encodedDdpv}'`
+
+    const peerData = this.getPeerService(span.context()._tags)
+    if (peerData !== undefined && peerData[PEER_SERVICE_SOURCE_KEY] === PEER_SERVICE_KEY) {
+      this.encodingServiceTags('ddprs', 'encodedDdprs', peerData[PEER_SERVICE_KEY])
+
+      const { encodedDdprs } = this.serviceTags
+      dbmComment += `,ddprs='${encodedDdprs}'`
+    }
+    return dbmComment
   }
 
   getDbmServiceName (span, tracerService) {
@@ -56,7 +71,7 @@ class DatabasePlugin extends StoragePlugin {
       return query
     }
 
-    const servicePropagation = this.createDBMPropagationCommentService(dbmService)
+    const servicePropagation = this.createDBMPropagationCommentService(dbmService, span)
 
     if (isPreparedStatement || mode === 'service') {
       return `/*${servicePropagation}*/ ${query}`

package/packages/dd-trace/src/plugins/util/test.js

@@ -61,6 +61,8 @@ const CI_APP_ORIGIN = 'ciapp-test'
 const JEST_TEST_RUNNER = 'test.jest.test_runner'
 const JEST_DISPLAY_NAME = 'test.jest.display_name'
 
+const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel'
+
 const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped'
 const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled'
 const TEST_ITR_SKIPPING_TYPE = 'test.itr.tests_skipping.type'
@@ -82,6 +84,9 @@ const TEST_BROWSER_VERSION = 'test.browser.version'
 const JEST_WORKER_TRACE_PAYLOAD_CODE = 60
 const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61
 
+// cucumber worker variables
+const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70
+
 // Early flake detection util strings
 const EFD_STRING = "Retried by Datadog's Early Flake Detection"
 const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
@@ -92,6 +97,7 @@ module.exports = {
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
   JEST_DISPLAY_NAME,
+  CUCUMBER_IS_PARALLEL,
   TEST_TYPE,
   TEST_NAME,
   TEST_SUITE,
@@ -104,6 +110,7 @@ module.exports = {
   LIBRARY_VERSION,
   JEST_WORKER_TRACE_PAYLOAD_CODE,
   JEST_WORKER_COVERAGE_PAYLOAD_CODE,
+  CUCUMBER_WORKER_TRACE_PAYLOAD_CODE,
   TEST_SOURCE_START,
   TEST_SKIPPED_BY_ITR,
   TEST_CONFIGURATION_BROWSER_NAME,

package/packages/dd-trace/src/proxy.js

@@ -15,6 +15,21 @@ const NoopDogStatsDClient = require('./noop/dogstatsd')
 const spanleak = require('./spanleak')
 const { SSITelemetry } = require('./profiling/ssi-telemetry')
 
+class LazyModule {
+  constructor (provider) {
+    this.provider = provider
+  }
+
+  enable (...args) {
+    this.module = this.provider()
+    this.module.enable(...args)
+  }
+
+  disable () {
+    this.module?.disable()
+  }
+}
+
 class Tracer extends NoopProxy {
   constructor () {
     super()
@@ -24,6 +39,12 @@ class Tracer extends NoopProxy {
     this._pluginManager = new PluginManager(this)
     this.dogstatsd = new NoopDogStatsDClient()
     this._tracingInitialized = false
+
+    // these requires must work with esm bundler
+    this._modules = {
+      appsec: new LazyModule(() => require('./appsec')),
+      iast: new LazyModule(() => require('./appsec/iast'))
+    }
   }
 
   init (options) {
@@ -58,7 +79,7 @@ class Tracer extends NoopProxy {
       }
 
       if (config.remoteConfig.enabled && !config.isCiVisibility) {
-        const rc = remoteConfig.enable(config)
+        const rc = remoteConfig.enable(config, this._modules.appsec)
 
         rc.on('APM_TRACING', (action, conf) => {
           if (action === 'unapply') {
@@ -113,9 +134,8 @@ class Tracer extends NoopProxy {
 
   _enableOrDisableTracing (config) {
     if (config.tracing !== false) {
-      // dirty require for now so zero appsec code is executed unless explicitly enabled
       if (config.appsec.enabled) {
-        require('./appsec').enable(config)
+        this._modules.appsec.enable(config)
       }
       if (!this._tracingInitialized) {
         this._tracer = new DatadogTracer(config)
@@ -123,11 +143,11 @@ class Tracer extends NoopProxy {
         this._tracingInitialized = true
       }
       if (config.iast.enabled) {
-        require('./appsec/iast').enable(config, this._tracer)
+        this._modules.iast.enable(config, this._tracer)
       }
     } else if (this._tracingInitialized) {
-      require('./appsec').disable()
-      require('./appsec/iast').disable()
+      this._modules.appsec.disable()
+      this._modules.iast.disable()
     }
 
     if (this._tracingInitialized) {

package/packages/dd-trace/src/telemetry/index.js

@@ -6,7 +6,8 @@ const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
 const { errors } = require('../startup-log')
 const { manager: metricsManager } = require('./metrics')
-const logs = require('./logs')
+const telemetryLogger = require('./logs')
+const logger = require('../log')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -211,7 +212,7 @@ function createPayload (currReqType, currPayload = {}) {
 function heartbeat (config, application, host) {
   heartbeatTimeout = setTimeout(() => {
     metricsManager.send(config, application, host)
-    logs.send(config, application, host)
+    telemetryLogger.send(config, application, host)
 
     const { reqType, payload } = createPayload('app-heartbeat')
     sendData(config, application, host, reqType, payload, updateRetryData)
@@ -235,6 +236,10 @@ function extendedHeartbeat (config) {
 
 function start (aConfig, thePluginManager) {
   if (!aConfig.telemetry.enabled) {
+    if (aConfig.sca?.enabled) {
+      logger.warn('DD_APPSEC_SCA_ENABLED requires enabling telemetry to work.')
+    }
+
     return
   }
   config = aConfig
@@ -245,7 +250,7 @@ function start (aConfig, thePluginManager) {
   integrations = getIntegrations()
 
   dependencies.start(config, application, host, getRetryData, updateRetryData)
-  logs.start(config)
+  telemetryLogger.start(config)
 
   sendData(config, application, host, 'app-started', appStarted(config))
 
@@ -318,6 +323,7 @@ function updateConfig (changes, config) {
 
   for (const change of changes) {
     const name = nameMapping[change.name] || change.name
+
     names.push(name)
     const { origin, value } = change
     const entry = { name, value, origin }

package/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js

@@ -1,33 +0,0 @@
-'use strict'
-
-const Writer = require('./writer')
-const {
-  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
-  JEST_WORKER_TRACE_PAYLOAD_CODE
-} = require('../../../plugins/util/test')
-
-/**
- * Lightweight exporter whose writers only do simple JSON serialization
- * of trace and coverage payloads, which they send to the jest main process.
- */
-class JestWorkerCiVisibilityExporter {
-  constructor () {
-    this._writer = new Writer(JEST_WORKER_TRACE_PAYLOAD_CODE)
-    this._coverageWriter = new Writer(JEST_WORKER_COVERAGE_PAYLOAD_CODE)
-  }
-
-  export (payload) {
-    this._writer.append(payload)
-  }
-
-  exportCoverage (formattedCoverage) {
-    this._coverageWriter.append(formattedCoverage)
-  }
-
-  flush () {
-    this._writer.flush()
-    this._coverageWriter.flush()
-  }
-}
-
-module.exports = JestWorkerCiVisibilityExporter