dd-trace 4.18.0 → 4.22.0

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -9,5 +9,6 @@ module.exports = {
   ASM_USER_BLOCKING: 1n << 7n,
   ASM_CUSTOM_RULES: 1n << 8n,
   ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
-  ASM_TRUSTED_IPS: 1n << 10n
+  ASM_TRUSTED_IPS: 1n << 10n,
+  ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -1,38 +1,59 @@
 'use strict'
 
+const Activation = require('../activation')
+
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
+const apiSecuritySampler = require('../api_security_sampler')
 
 let rc
 
 function enable (config) {
   rc = new RemoteConfigManager(config)
 
-  if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally
-    rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+  const activation = Activation.fromConfig(config)
+
+  if (activation !== Activation.DISABLED) {
+    if (activation === Activation.ONECLICK) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+    }
 
-    rc.on('ASM_FEATURES', (action, conf) => {
-      if (conf && conf.asm && typeof conf.asm.enabled === 'boolean') {
-        let shouldEnable
+    if (config.appsec.apiSecurity?.enabled) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
+    }
 
-        if (action === 'apply' || action === 'modify') {
-          shouldEnable = conf.asm.enabled // take control
-        } else {
-          shouldEnable = config.appsec.enabled // give back control to local config
-        }
+    rc.on('ASM_FEATURES', (action, rcConfig) => {
+      if (!rcConfig) return
 
-        if (shouldEnable) {
-          require('..').enable(config)
-        } else {
-          require('..').disable()
-        }
+      if (activation === Activation.ONECLICK) {
+        enableOrDisableAppsec(action, rcConfig, config)
       }
+
+      apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
     })
   }
 
   return rc
 }
 
+function enableOrDisableAppsec (action, rcConfig, config) {
+  if (typeof rcConfig.asm?.enabled === 'boolean') {
+    let shouldEnable
+
+    if (action === 'apply' || action === 'modify') {
+      shouldEnable = rcConfig.asm.enabled // take control
+    } else {
+      shouldEnable = config.appsec.enabled // give back control to local config
+    }
+
+    if (shouldEnable) {
+      require('..').enable(config)
+    } else {
+      require('..').disable()
+    }
+  }
+}
+
 function enableWafUpdate (appsecConfig) {
   if (rc && appsecConfig && !appsecConfig.customRulesProvided) {
     // dirty require to make startup faster for serverless

package/packages/dd-trace/src/appsec/reporter.js

@@ -3,64 +3,61 @@
 const Limiter = require('../rate_limiter')
 const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
+const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   updateWafRequestsMetricTags,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric
 } = require('./telemetry')
+const zlib = require('zlib')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
 
-// TODO: use precomputed maps instead
-const REQUEST_HEADERS_PASSLIST = [
-  'accept',
-  'accept-encoding',
-  'accept-language',
+const metricsQueue = new Map()
+
+const contentHeaderList = [
   'content-encoding',
   'content-language',
   'content-length',
-  'content-type',
-  'forwarded',
-  'forwarded-for',
+  'content-type'
+]
+
+const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+  'accept',
+  'accept-encoding',
+  'accept-language',
   'host',
-  'true-client-ip',
   'user-agent',
+  'forwarded',
   'via',
-  'x-client-ip',
-  'x-cluster-client-ip',
-  'x-forwarded',
-  'x-forwarded-for',
-  'x-real-ip'
-]
 
-const RESPONSE_HEADERS_PASSLIST = [
-  'content-encoding',
-  'content-language',
-  'content-length',
-  'content-type'
-]
+  ...ipHeaderList,
+  ...contentHeaderList
+], 'http.request.headers.')
 
-const metricsQueue = new Map()
+const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.')
 
-function filterHeaders (headers, passlist, prefix) {
+function mapHeaderAndTags (headerList, tagPrefix) {
+  return new Map(headerList.map(headerName => [headerName, `${tagPrefix}${formatHeaderName(headerName)}`]))
+}
+
+function filterHeaders (headers, map) {
   const result = {}
 
   if (!headers) return result
 
-  for (let i = 0; i < passlist.length; ++i) {
-    const headerName = passlist[i]
-
-    if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = '' + headers[headerName]
+  for (const [headerName, tagName] of map) {
+    const headerValue = headers[headerName]
+    if (headerValue) {
+      result[tagName] = '' + headerValue
     }
   }
 
   return result
 }
 
-// TODO: this can be precomputed at start time
 function formatHeaderName (name) {
   return name
     .trim()
@@ -86,7 +83,7 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
 function reportMetrics (metrics) {
   // TODO: metrics should be incremental, there already is an RFC to report metrics
   const store = storage.getStore()
-  const rootSpan = store && store.req && web.root(store.req)
+  const rootSpan = store?.req && web.root(store.req)
   if (!rootSpan) return
 
   if (metrics.duration) {
@@ -106,13 +103,13 @@ function reportMetrics (metrics) {
 
 function reportAttack (attackData) {
   const store = storage.getStore()
-  const req = store && store.req
+  const req = store?.req
   const rootSpan = web.root(req)
   if (!rootSpan) return
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
+  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
 
   newTags['appsec.event'] = 'true'
 
@@ -144,6 +141,23 @@ function reportAttack (attackData) {
   rootSpan.addTags(newTags)
 }
 
+function reportSchemas (derivatives) {
+  if (!derivatives) return
+
+  const req = storage.getStore()?.req
+  const rootSpan = web.root(req)
+
+  if (!rootSpan) return
+
+  const tags = {}
+  for (const [address, value] of Object.entries(derivatives)) {
+    const gzippedValue = zlib.gzipSync(JSON.stringify(value))
+    tags[address] = gzippedValue.toString('base64')
+  }
+
+  rootSpan.addTags(tags)
+}
+
 function finishRequest (req, res) {
   const rootSpan = web.root(req)
   if (!rootSpan) return
@@ -158,7 +172,7 @@ function finishRequest (req, res) {
 
   if (!rootSpan.context()._tags['appsec.event']) return
 
-  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
+  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP)
 
   if (req.route && typeof req.route.path === 'string') {
     newTags['http.endpoint'] = req.route.path
@@ -179,6 +193,8 @@ module.exports = {
   reportMetrics,
   reportAttack,
   reportWafUpdate: incrementWafUpdatesMetric,
+  reportSchemas,
   finishRequest,
-  setRateLimit
+  setRateLimit,
+  mapHeaderAndTags
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const fs = require('fs')
 const waf = require('./waf')
 const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 const blocking = require('./blocking')
@@ -13,13 +14,15 @@ let appliedExclusions = new Map()
 let appliedCustomRules = new Map()
 let appliedActions = new Map()
 
-function applyRules (rules, config) {
-  defaultRules = rules
+function loadRules (config) {
+  defaultRules = config.rules
+    ? JSON.parse(fs.readFileSync(config.rules))
+    : require('./recommended.json')
 
-  waf.init(rules, config)
+  waf.init(defaultRules, config)
 
-  if (rules.actions) {
-    blocking.updateBlockingConfiguration(rules.actions.find(action => action.id === 'block'))
+  if (defaultRules.actions) {
+    blocking.updateBlockingConfiguration(defaultRules.actions.find(action => action.id === 'block'))
   }
 }
 
@@ -252,7 +255,7 @@ function clearAllRules () {
 }
 
 module.exports = {
-  applyRules,
+  loadRules,
   updateWafFromRC,
   clearAllRules
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -9,7 +9,7 @@ const { setUserTags } = require('./set_user')
 const log = require('../../log')
 
 function isUserBlocked (user) {
-  const actions = waf.run({ [USER_ID]: user.id })
+  const actions = waf.run({ persistent: { [USER_ID]: user.id } })
 
   if (!actions) return false
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -10,37 +10,50 @@ const preventDuplicateAddresses = new Set([
 ])
 
 class WAFContextWrapper {
-  constructor (ddwafContext, requiredAddresses, wafTimeout, wafVersion, rulesVersion) {
+  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) {
     this.ddwafContext = ddwafContext
-    this.requiredAddresses = requiredAddresses
     this.wafTimeout = wafTimeout
     this.wafVersion = wafVersion
     this.rulesVersion = rulesVersion
     this.addressesToSkip = new Set()
   }
 
-  run (params) {
+  run ({ persistent, ephemeral }) {
+    const payload = {}
+    let payloadHasData = false
     const inputs = {}
-    let someInputAdded = false
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
-    for (const key of Object.keys(params)) {
-      if (this.requiredAddresses.has(key) && !this.addressesToSkip.has(key)) {
-        inputs[key] = params[key]
-        if (preventDuplicateAddresses.has(key)) {
-          newAddressesToSkip.add(key)
+    if (persistent && typeof persistent === 'object') {
+      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      for (const key of Object.keys(persistent)) {
+        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
+        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
+        if (!this.addressesToSkip.has(key)) {
+          inputs[key] = persistent[key]
+          if (preventDuplicateAddresses.has(key)) {
+            newAddressesToSkip.add(key)
+          }
         }
-        someInputAdded = true
       }
     }
 
-    if (!someInputAdded) return
+    if (Object.keys(inputs).length) {
+      payload['persistent'] = inputs
+      payloadHasData = true
+    }
+
+    if (ephemeral && Object.keys(ephemeral).length) {
+      payload['ephemeral'] = ephemeral
+      payloadHasData = true
+    }
+
+    if (!payloadHasData) return
 
     try {
       const start = process.hrtime.bigint()
 
-      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+      const result = this.ddwafContext.run(payload, this.wafTimeout)
 
       const end = process.hrtime.bigint()
 
@@ -63,6 +76,8 @@ class WAFContextWrapper {
         Reporter.reportAttack(JSON.stringify(result.events))
       }
 
+      Reporter.reportSchemas(result.derivatives)
+
       return result.actions
     } catch (err) {
       log.error('Error while running the AppSec WAF')

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -37,7 +37,6 @@ class WAFManager {
     if (!wafContext) {
       wafContext = new WAFContextWrapper(
         this.ddwaf.createContext(),
-        this.ddwaf.requiredAddresses,
         this.wafTimeout,
         this.ddwafVersion,
         this.rulesVersion

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -143,7 +143,23 @@ class CiVisibilityExporter extends AgentInfoExporter {
          * where the tests run in a subprocess, because `getItrConfiguration` is called only once.
          */
         this._itrConfig = itrConfig
-        callback(err, itrConfig)
+
+        if (err) {
+          callback(err, {})
+        } else if (itrConfig?.requireGit) {
+          // If the backend requires git, we'll wait for the upload to finish and request settings again
+          this._gitUploadPromise.then(gitUploadError => {
+            if (gitUploadError) {
+              return callback(gitUploadError, {})
+            }
+            getItrConfigurationRequest(configuration, (err, finalItrConfig) => {
+              this._itrConfig = finalItrConfig
+              callback(err, finalItrConfig)
+            })
+          })
+        } else {
+          callback(null, itrConfig)
+        }
       })
     })
   }

package/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js

@@ -10,7 +10,7 @@ const {
   getLatestCommits,
   getRepositoryUrl,
   generatePackFilesForCommits,
-  getCommitsToUpload,
+  getCommitsRevList,
   isShallowRepository,
   unshallowRepository
 } = require('../../../plugins/util/git')
@@ -46,11 +46,7 @@ function getCommonRequestOptions (url) {
  * The response are the commits for which the backend already has information
  * This response is used to know which commits can be ignored from there on
  */
-function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
-  const latestCommits = getLatestCommits()
-
-  log.debug(`There were ${latestCommits.length} commits since last month.`)
-
+function getCommitsToUpload ({ url, repositoryUrl, latestCommits, isEvpProxy }, callback) {
   const commonOptions = getCommonRequestOptions(url)
 
   const options = {
@@ -83,13 +79,23 @@ function getCommitsToExclude ({ url, isEvpProxy, repositoryUrl }, callback) {
       const error = new Error(`Error fetching commits to exclude: ${err.message}`)
       return callback(error)
     }
-    let commitsToExclude
+    let alreadySeenCommits
     try {
-      commitsToExclude = validateCommits(JSON.parse(response).data)
+      alreadySeenCommits = validateCommits(JSON.parse(response).data)
     } catch (e) {
       return callback(new Error(`Can't parse commits to exclude response: ${e.message}`))
     }
-    callback(null, commitsToExclude, latestCommits)
+    log.debug(`There are ${alreadySeenCommits.length} commits to exclude.`)
+    const commitsToInclude = latestCommits.filter((commit) => !alreadySeenCommits.includes(commit))
+    log.debug(`There are ${commitsToInclude.length} commits to include.`)
+
+    if (!commitsToInclude.length) {
+      return callback(null, [])
+    }
+
+    const commitsToUpload = getCommitsRevList(alreadySeenCommits, commitsToInclude)
+
+    callback(null, commitsToUpload)
   })
 }
 
@@ -150,6 +156,53 @@ function uploadPackFile ({ url, isEvpProxy, packFileToUpload, repositoryUrl, hea
   })
 }
 
+function generateAndUploadPackFiles ({
+  url,
+  isEvpProxy,
+  commitsToUpload,
+  repositoryUrl,
+  headCommit
+}, callback) {
+  log.debug(`There are ${commitsToUpload.length} commits to upload`)
+
+  const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
+
+  log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
+
+  if (!packFilesToUpload.length) {
+    return callback(new Error('Failed to generate packfiles'))
+  }
+
+  let packFileIndex = 0
+  // This uploads packfiles sequentially
+  const uploadPackFileCallback = (err) => {
+    if (err || packFileIndex === packFilesToUpload.length) {
+      return callback(err)
+    }
+    return uploadPackFile(
+      {
+        packFileToUpload: packFilesToUpload[packFileIndex++],
+        url,
+        isEvpProxy,
+        repositoryUrl,
+        headCommit
+      },
+      uploadPackFileCallback
+    )
+  }
+
+  uploadPackFile(
+    {
+      packFileToUpload: packFilesToUpload[packFileIndex++],
+      url,
+      isEvpProxy,
+      repositoryUrl,
+      headCommit
+    },
+    uploadPackFileCallback
+  )
+}
+
 /**
  * This function uploads git metadata to CI Visibility's backend.
 */
@@ -165,65 +218,31 @@ function sendGitMetadata (url, isEvpProxy, configRepositoryUrl, callback) {
     return callback(new Error('Repository URL is empty'))
   }
 
-  if (isShallowRepository()) {
-    log.debug('It is shallow clone, unshallowing...')
-    unshallowRepository()
-  }
+  const latestCommits = getLatestCommits()
+  log.debug(`There were ${latestCommits.length} commits since last month.`)
+  const [headCommit] = latestCommits
 
-  getCommitsToExclude({ url, repositoryUrl, isEvpProxy }, (err, commitsToExclude, latestCommits) => {
+  const getOnFinishGetCommitsToUpload = (hasCheckedShallow) => (err, commitsToUpload) => {
     if (err) {
       return callback(err)
     }
-    log.debug(`There are ${commitsToExclude.length} commits to exclude.`)
-    const [headCommit] = latestCommits
-    const commitsToInclude = latestCommits.filter((commit) => !commitsToExclude.includes(commit))
-    log.debug(`There are ${commitsToInclude.length} commits to include.`)
-
-    const commitsToUpload = getCommitsToUpload(commitsToExclude, commitsToInclude)
 
     if (!commitsToUpload.length) {
       log.debug('No commits to upload')
       return callback(null)
     }
-    log.debug(`There are ${commitsToUpload.length} commits to upload`)
-
-    const packFilesToUpload = generatePackFilesForCommits(commitsToUpload)
-
-    log.debug(`Uploading ${packFilesToUpload.length} packfiles.`)
-
-    if (!packFilesToUpload.length) {
-      return callback(new Error('Failed to generate packfiles'))
-    }
 
-    let packFileIndex = 0
-    // This uploads packfiles sequentially
-    const uploadPackFileCallback = (err) => {
-      if (err || packFileIndex === packFilesToUpload.length) {
-        return callback(err)
-      }
-      return uploadPackFile(
-        {
-          packFileToUpload: packFilesToUpload[packFileIndex++],
-          url,
-          isEvpProxy,
-          repositoryUrl,
-          headCommit
-        },
-        uploadPackFileCallback
-      )
+    // If it has already unshallowed or the clone is not shallow, we move on
+    if (hasCheckedShallow || !isShallowRepository()) {
+      return generateAndUploadPackFiles({ url, isEvpProxy, commitsToUpload, repositoryUrl, headCommit }, callback)
     }
+    // Otherwise we unshallow and get commits to upload again
+    log.debug('It is shallow clone, unshallowing...')
+    unshallowRepository()
+    getCommitsToUpload({ url, repositoryUrl, latestCommits, isEvpProxy }, getOnFinishGetCommitsToUpload(true))
+  }
 
-    uploadPackFile(
-      {
-        packFileToUpload: packFilesToUpload[packFileIndex++],
-        url,
-        isEvpProxy,
-        repositoryUrl,
-        headCommit
-      },
-      uploadPackFileCallback
-    )
-  })
+  getCommitsToUpload({ url, repositoryUrl, latestCommits, isEvpProxy }, getOnFinishGetCommitsToUpload(false))
 }
 
 module.exports = {

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -15,6 +15,7 @@ function getItrConfiguration ({
   runtimeName,
   runtimeVersion,
   branch,
+  testLevel = 'suite',
   custom
 }, done) {
   const options = {
@@ -23,7 +24,8 @@ function getItrConfiguration ({
     headers: {
       'Content-Type': 'application/json'
     },
-    url
+    url,
+    timeout: 20000
   }
 
   if (isEvpProxy) {
@@ -42,7 +44,7 @@ function getItrConfiguration ({
       id: id().toString(10),
       type: 'ci_app_test_service_libraries_settings',
       attributes: {
-        test_level: 'suite',
+        test_level: testLevel,
         configurations: {
           'os.platform': osPlatform,
           'os.version': osVersion,
@@ -69,13 +71,27 @@ function getItrConfiguration ({
           data: {
             attributes: {
               code_coverage: isCodeCoverageEnabled,
-              tests_skipping: isSuitesSkippingEnabled
+              tests_skipping: isSuitesSkippingEnabled,
+              itr_enabled: isItrEnabled,
+              require_git: requireGit
             }
           }
         } = JSON.parse(res)
-        const config = { isCodeCoverageEnabled, isSuitesSkippingEnabled }
-        log.debug(() => `Received settings: ${config}`)
-        done(null, config)
+
+        const settings = { isCodeCoverageEnabled, isSuitesSkippingEnabled, isItrEnabled, requireGit }
+
+        log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)
+
+        if (process.env.DD_CIVISIBILITY_DANGEROUSLY_FORCE_COVERAGE) {
+          settings.isCodeCoverageEnabled = true
+          log.debug(() => 'Dangerously set code coverage to true')
+        }
+        if (process.env.DD_CIVISIBILITY_DANGEROUSLY_FORCE_TEST_SKIPPING) {
+          settings.isSuitesSkippingEnabled = true
+          log.debug(() => 'Dangerously set test skipping to true')
+        }
+
+        done(null, settings)
       } catch (err) {
         done(err)
       }