dd-trace 3.41.0 → 3.43.0

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -9,5 +9,6 @@ module.exports = {
   ASM_USER_BLOCKING: 1n << 7n,
   ASM_CUSTOM_RULES: 1n << 8n,
   ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
-  ASM_TRUSTED_IPS: 1n << 10n
+  ASM_TRUSTED_IPS: 1n << 10n,
+  ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -1,38 +1,59 @@
 'use strict'
 
+const Activation = require('../activation')
+
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
+const apiSecuritySampler = require('../api_security_sampler')
 
 let rc
 
 function enable (config) {
   rc = new RemoteConfigManager(config)
 
-  if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally
-    rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+  const activation = Activation.fromConfig(config)
+
+  if (activation !== Activation.DISABLED) {
+    if (activation === Activation.ONECLICK) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+    }
 
-    rc.on('ASM_FEATURES', (action, conf) => {
-      if (conf && conf.asm && typeof conf.asm.enabled === 'boolean') {
-        let shouldEnable
+    if (config.appsec.apiSecurity?.enabled) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
+    }
 
-        if (action === 'apply' || action === 'modify') {
-          shouldEnable = conf.asm.enabled // take control
-        } else {
-          shouldEnable = config.appsec.enabled // give back control to local config
-        }
+    rc.on('ASM_FEATURES', (action, rcConfig) => {
+      if (!rcConfig) return
 
-        if (shouldEnable) {
-          require('..').enable(config)
-        } else {
-          require('..').disable()
-        }
+      if (activation === Activation.ONECLICK) {
+        enableOrDisableAppsec(action, rcConfig, config)
       }
+
+      apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
     })
   }
 
   return rc
 }
 
+function enableOrDisableAppsec (action, rcConfig, config) {
+  if (typeof rcConfig.asm?.enabled === 'boolean') {
+    let shouldEnable
+
+    if (action === 'apply' || action === 'modify') {
+      shouldEnable = rcConfig.asm.enabled // take control
+    } else {
+      shouldEnable = config.appsec.enabled // give back control to local config
+    }
+
+    if (shouldEnable) {
+      require('..').enable(config)
+    } else {
+      require('..').disable()
+    }
+  }
+}
+
 function enableWafUpdate (appsecConfig) {
   if (rc && appsecConfig && !appsecConfig.customRulesProvided) {
     // dirty require to make startup faster for serverless

package/packages/dd-trace/src/appsec/reporter.js

@@ -10,6 +10,7 @@ const {
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric
 } = require('./telemetry')
+const zlib = require('zlib')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
@@ -140,6 +141,23 @@ function reportAttack (attackData) {
   rootSpan.addTags(newTags)
 }
 
+function reportSchemas (derivatives) {
+  if (!derivatives) return
+
+  const req = storage.getStore()?.req
+  const rootSpan = web.root(req)
+
+  if (!rootSpan) return
+
+  const tags = {}
+  for (const [address, value] of Object.entries(derivatives)) {
+    const gzippedValue = zlib.gzipSync(JSON.stringify(value))
+    tags[address] = gzippedValue.toString('base64')
+  }
+
+  rootSpan.addTags(tags)
+}
+
 function finishRequest (req, res) {
   const rootSpan = web.root(req)
   if (!rootSpan) return
@@ -175,6 +193,7 @@ module.exports = {
   reportMetrics,
   reportAttack,
   reportWafUpdate: incrementWafUpdatesMetric,
+  reportSchemas,
   finishRequest,
   setRateLimit,
   mapHeaderAndTags

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -9,7 +9,7 @@ const { setUserTags } = require('./set_user')
 const log = require('../../log')
 
 function isUserBlocked (user) {
-  const actions = waf.run({ [USER_ID]: user.id })
+  const actions = waf.run({ persistent: { [USER_ID]: user.id } })
 
   if (!actions) return false
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -10,37 +10,50 @@ const preventDuplicateAddresses = new Set([
 ])
 
 class WAFContextWrapper {
-  constructor (ddwafContext, requiredAddresses, wafTimeout, wafVersion, rulesVersion) {
+  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) {
     this.ddwafContext = ddwafContext
-    this.requiredAddresses = requiredAddresses
     this.wafTimeout = wafTimeout
     this.wafVersion = wafVersion
     this.rulesVersion = rulesVersion
     this.addressesToSkip = new Set()
   }
 
-  run (params) {
+  run ({ persistent, ephemeral }) {
+    const payload = {}
+    let payloadHasData = false
     const inputs = {}
-    let someInputAdded = false
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
-    for (const key of Object.keys(params)) {
-      if (this.requiredAddresses.has(key) && !this.addressesToSkip.has(key)) {
-        inputs[key] = params[key]
-        if (preventDuplicateAddresses.has(key)) {
-          newAddressesToSkip.add(key)
+    if (persistent && typeof persistent === 'object') {
+      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      for (const key of Object.keys(persistent)) {
+        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
+        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
+        if (!this.addressesToSkip.has(key)) {
+          inputs[key] = persistent[key]
+          if (preventDuplicateAddresses.has(key)) {
+            newAddressesToSkip.add(key)
+          }
         }
-        someInputAdded = true
       }
     }
 
-    if (!someInputAdded) return
+    if (Object.keys(inputs).length) {
+      payload['persistent'] = inputs
+      payloadHasData = true
+    }
+
+    if (ephemeral && Object.keys(ephemeral).length) {
+      payload['ephemeral'] = ephemeral
+      payloadHasData = true
+    }
+
+    if (!payloadHasData) return
 
     try {
       const start = process.hrtime.bigint()
 
-      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+      const result = this.ddwafContext.run(payload, this.wafTimeout)
 
       const end = process.hrtime.bigint()
 
@@ -63,6 +76,8 @@ class WAFContextWrapper {
         Reporter.reportAttack(JSON.stringify(result.events))
       }
 
+      Reporter.reportSchemas(result.derivatives)
+
       return result.actions
     } catch (err) {
       log.error('Error while running the AppSec WAF')

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -37,7 +37,6 @@ class WAFManager {
     if (!wafContext) {
       wafContext = new WAFContextWrapper(
         this.ddwaf.createContext(),
-        this.ddwaf.requiredAddresses,
         this.wafTimeout,
         this.ddwafVersion,
         this.rulesVersion

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -143,7 +143,23 @@ class CiVisibilityExporter extends AgentInfoExporter {
          * where the tests run in a subprocess, because `getItrConfiguration` is called only once.
          */
         this._itrConfig = itrConfig
-        callback(err, itrConfig)
+
+        if (err) {
+          callback(err, {})
+        } else if (itrConfig?.requireGit) {
+          // If the backend requires git, we'll wait for the upload to finish and request settings again
+          this._gitUploadPromise.then(gitUploadError => {
+            if (gitUploadError) {
+              return callback(gitUploadError, {})
+            }
+            getItrConfigurationRequest(configuration, (err, finalItrConfig) => {
+              this._itrConfig = finalItrConfig
+              callback(err, finalItrConfig)
+            })
+          })
+        } else {
+          callback(null, itrConfig)
+        }
       })
     })
   }

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -10,7 +10,7 @@ const {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload,
+  getCommitsRevList,
   isShallowRepository,
   unshallowRepository
 } = require('../../../plugins/util/git')
@@ -46,11 +46,7 @@ function getCommonRequestOptions (url) {
  * The response are the commits for which the backend already has information
  * This response is used to know which commits can be ignored from there on
  */
-function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
-  const latestCommits = getLatestCommits()
-
-  log.debug(`There were ${latestCommits.length} commits since last month.`)
-
+function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy }, callback) {
   const commonOptions = getCommonRequestOptions(url)
 
   const options = {
@@ -83,13 +79,23 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
       const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
-    let commitsToExclude
+    let alreadySeenCommits
     try {
-      commitsToExclude = validateCommits(JSON.parse(response).data)
+      alreadySeenCommits = validateCommits(JSON.parse(response).data)
     } catch (e) {
       return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }
-    callback(null, commitsToExclude, latestCommits)
+    log.debug(`There are ${alreadySeenCommits.length} commits to exclude.`)
+    const commitsToInclude = latestCommits.filter((commit) => !alreadySeenCommits.includes(commit))
+    log.debug(`There are ${commitsToInclude.length} commits to include.`)
+
+    if (!commitsToInclude.length) {
+      return callback(null, [])
+    }
+
+    const commitsToUpload = getCommitsRevList(alreadySeenCommits, commitsToInclude)
+
+    callback(null, commitsToUpload)
   })
 }
 
@@ -150,6 +156,53 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea
   })
 }
 
+function generateAndUploadPackFiles ({
+  url,
+  isEvpProxy,
+  commitsToUpload,
+  repositoryUrl,
+  headCommit
+}, callback) {
+  log.debug(`There are ${commitsToUpload.length} commits to upload`)
+
+  const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+  log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
+
+  if (!packFilesToUpload.length) {
+    return callback(new Error('Failed to generate packfiles'))
+  }
+
+  let packFileIndex = 0
+  // This uploads packfiles sequentially
+  const uploadPackFileCallback = (err) => {
+    if (err || packFileIndex === packFilesToUpload.length) {
+      return callback(err)
+    }
+    return uploadPackFile(
+      {
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        url,
+        isEvpProxy,
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  }
+
+  uploadPackFile(
+    {
+      packFileToUpload: packFilesToUpload[packFileIndex++],
+      url,
+      isEvpProxy,
+      repositoryUrl,
+      headCommit
+    },
+    uploadPackFileCallback
+  )
+}
+
 /**
  * This function uploads git metadata to CI Visibility's backend.
 */
@@ -165,65 +218,31 @@ function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
     return callback(new Error('Repository URL is empty'))
   }
 
-  if (isShallowRepository()) {
-    log.debug('It is shallow clone, unshallowing...')
-    unshallowRepository()
-  }
+  const latestCommits = getLatestCommits()
+  log.debug(`There were ${latestCommits.length} commits since last month.`)
+  const [headCommit] = latestCommits
 
-  getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, latestCommits) => {
+  const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => {
     if (err) {
       return callback(err)
     }
-    log.debug(`There are ${commitsToExclude.length} commits to exclude.`)
-    const [headCommit] = latestCommits
-    const commitsToInclude = latestCommits.filter((commit) => !commitsToExclude.includes(commit))
-    log.debug(`There are ${commitsToInclude.length} commits to include.`)
-
-    const commitsToUpload = getCommitsToUpload(commitsToExclude, commitsToInclude)
 
     if (!commitsToUpload.length) {
       log.debug('No commits to upload')
       return callback(null)
     }
-    log.debug(`There are ${commitsToUpload.length} commits to upload`)
-
-    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
-
-    log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
-
-    if (!packFilesToUpload.length) {
-      return callback(new Error('Failed to generate packfiles'))
-    }
 
-    let packFileIndex = 0
-    // This uploads packfiles sequentially
-    const uploadPackFileCallback = (err) => {
-      if (err || packFileIndex === packFilesToUpload.length) {
-        return callback(err)
-      }
-      return uploadPackFile(
-        {
-          packFileToUpload: packFilesToUpload[packFileIndex++],
-          url,
-          isEvpProxy,
-          repositoryUrl,
-          headCommit
-        },
-        uploadPackFileCallback
-      )
+    // If it has already unshallowed or the clone is not shallow, we move on
+    if (hasCheckedShallow || !isShallowRepository()) {
+      return generateAndUploadPackFiles({ url, isEvpProxy, commitsToUpload, repositoryUrl, headCommit }, callback)
     }
+    // Otherwise we unshallow and get commits to upload again
+    log.debug('It is shallow clone, unshallowing...')
+    unshallowRepository()
+    getCommitsToUpload({ url, repositoryUrl, latestCommits, isEvpProxy }, getOnFinishGetCommitsToUpload(true))
+  }
 
-    uploadPackFile(
-      {
-        packFileToUpload: packFilesToUpload[packFileIndex++],
-        url,
-        isEvpProxy,
-        repositoryUrl,
-        headCommit
-      },
-      uploadPackFileCallback
-    )
-  })
+  getCommitsToUpload({ url, repositoryUrl, latestCommits, isEvpProxy }, getOnFinishGetCommitsToUpload(false))
 }
 
 module.exports = {

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -15,6 +15,7 @@ function getItrConfiguration ({
   runtimeName,
   runtimeVersion,
   branch,
+  testLevel = 'suite',
   custom
 }, done) {
   const options = {
@@ -23,7 +24,8 @@ function getItrConfiguration ({
     headers: {
       'Content-Type': 'application/json'
     },
-    url
+    url,
+    timeout: 20000
   }
 
   if (isEvpProxy) {
@@ -42,7 +44,7 @@ function getItrConfiguration ({
       id: id().toString(10),
       type: 'ci_app_test_service_libraries_settings',
       attributes: {
-        test_level: 'suite',
+        test_level: testLevel,
         configurations: {
           'os.platform': osPlatform,
           'os.version': osVersion,
@@ -67,25 +69,29 @@ function getItrConfiguration ({
       try {
         const {
           data: {
-            attributes
+            attributes: {
+              code_coverage: isCodeCoverageEnabled,
+              tests_skipping: isSuitesSkippingEnabled,
+              itr_enabled: isItrEnabled,
+              require_git: requireGit
+            }
           }
         } = JSON.parse(res)
 
-        let isCodeCoverageEnabled = attributes.code_coverage
-        let isSuitesSkippingEnabled = attributes.tests_skipping
+        const settings = { isCodeCoverageEnabled, isSuitesSkippingEnabled, isItrEnabled, requireGit }
 
-        log.debug(() => `Remote settings: ${{ isCodeCoverageEnabled, isSuitesSkippingEnabled }}`)
+        log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)
 
         if (process.env.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE) {
-          isCodeCoverageEnabled = true
+          settings.isCodeCoverageEnabled = true
           log.debug(() => 'Dangerously set code coverage to true')
         }
         if (process.env.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING) {
-          isSuitesSkippingEnabled = true
+          settings.isSuitesSkippingEnabled = true
           log.debug(() => 'Dangerously set test skipping to true')
         }
 
-        done(null, { isCodeCoverageEnabled, isSuitesSkippingEnabled })
+        done(null, settings)
       } catch (err) {
         done(err)
       }

package/packages/dd-trace/src/config.js

@@ -399,7 +399,6 @@ class Config {
       appsec.enabled,
       process.env.DD_APPSEC_ENABLED && isTrue(process.env.DD_APPSEC_ENABLED)
     )
-
     const DD_APPSEC_RULES = coalesce(
       appsec.rules,
       process.env.DD_APPSEC_RULES
@@ -436,11 +435,25 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maybeFile(appsec.blockedTemplateJson),
       maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)
     )
+    const DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = coalesce(
+      maybeFile(appsec.blockedTemplateGraphql),
+      maybeFile(process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)
+    )
     const DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = coalesce(
       appsec.eventTracking && appsec.eventTracking.mode,
       process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING,
       'safe'
     ).toLowerCase()
+    const DD_EXPERIMENTAL_API_SECURITY_ENABLED = coalesce(
+      appsec?.apiSecurity?.enabled,
+      isTrue(process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED),
+      false
+    )
+    const DD_API_SECURITY_REQUEST_SAMPLE_RATE = coalesce(
+      appsec?.apiSecurity?.requestSampling,
+      parseFloat(process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE),
+      0.1
+    )
 
     const remoteConfigOptions = options.remoteConfig || {}
     const DD_REMOTE_CONFIGURATION_ENABLED = coalesce(
@@ -465,6 +478,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       DD_IAST_ENABLED
     )
 
+    const DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED,
+      true
+    )
+
     const defaultIastRequestSampling = 30
     const iastRequestSampling = coalesce(
       parseInt(iastOptions?.requestSampling),
@@ -526,6 +544,12 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       true
     )
 
+    // 0: disabled, 1: logging, 2: garbage collection + logging
+    const DD_TRACE_SPAN_LEAK_DEBUG = coalesce(
+      process.env.DD_TRACE_SPAN_LEAK_DEBUG,
+      0
+    )
+
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
     const sampler = {
@@ -609,7 +633,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL,
       debug: isTrue(DD_TELEMETRY_DEBUG),
       logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
-      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED)
+      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED),
+      dependencyCollection: DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED
     }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
@@ -623,11 +648,18 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
       blockedTemplateHtml: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML,
       blockedTemplateJson: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON,
+      blockedTemplateGraphql: DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON,
       eventTracking: {
         enabled: ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING),
         mode: DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING
+      },
+      apiSecurity: {
+        enabled: DD_EXPERIMENTAL_API_SECURITY_ENABLED,
+        // Coerce value between 0 and 1
+        requestSampling: Math.min(1, Math.max(0, DD_API_SECURITY_REQUEST_SAMPLE_RATE))
       }
     }
+
     this.remoteConfig = {
       enabled: DD_REMOTE_CONFIGURATION_ENABLED,
       pollInterval: DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS
@@ -701,6 +733,8 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.isGCPFunction = isGCPFunction
     this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan
 
+    this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG)
+
     tagger.add(this.tags, {
       service: this.service,
       env: this.env,