dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -9,5 +9,11 @@ module.exports = {
   ASM_USER_BLOCKING: 1n << 7n,
   ASM_CUSTOM_RULES: 1n << 8n,
   ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
-  ASM_TRUSTED_IPS: 1n << 10n
+  ASM_TRUSTED_IPS: 1n << 10n,
+  ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n,
+  APM_TRACING_SAMPLE_RATE: 1n << 12n,
+  APM_TRACING_LOGS_INJECTION: 1n << 13n,
+  APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n,
+  APM_TRACING_CUSTOM_TAGS: 1n << 15n,
+  APM_TRACING_ENABLED: 1n << 19n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -1,40 +1,66 @@
 'use strict'
 
+const Activation = require('../activation')
+
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
+const apiSecuritySampler = require('../api_security_sampler')
 
 let rc
 
 function enable (config) {
   rc = new RemoteConfigManager(config)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLED, true)
+
+  const activation = Activation.fromConfig(config)
 
-  if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally
-    rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+  if (activation !== Activation.DISABLED) {
+    if (activation === Activation.ONECLICK) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+    }
 
-    rc.on('ASM_FEATURES', (action, conf) => {
-      if (conf && conf.asm && typeof conf.asm.enabled === 'boolean') {
-        let shouldEnable
+    if (config.appsec.apiSecurity?.enabled) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
+    }
 
-        if (action === 'apply' || action === 'modify') {
-          shouldEnable = conf.asm.enabled // take control
-        } else {
-          shouldEnable = config.appsec.enabled // give back control to local config
-        }
+    rc.on('ASM_FEATURES', (action, rcConfig) => {
+      if (!rcConfig) return
 
-        if (shouldEnable) {
-          require('..').enable(config)
-        } else {
-          require('..').disable()
-        }
+      if (activation === Activation.ONECLICK) {
+        enableOrDisableAppsec(action, rcConfig, config)
       }
+
+      apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
     })
   }
 
   return rc
 }
 
+function enableOrDisableAppsec (action, rcConfig, config) {
+  if (typeof rcConfig.asm?.enabled === 'boolean') {
+    let shouldEnable
+
+    if (action === 'apply' || action === 'modify') {
+      shouldEnable = rcConfig.asm.enabled // take control
+    } else {
+      shouldEnable = config.appsec.enabled // give back control to local config
+    }
+
+    if (shouldEnable) {
+      require('..').enable(config)
+    } else {
+      require('..').disable()
+    }
+  }
+}
+
 function enableWafUpdate (appsecConfig) {
-  if (rc && appsecConfig && !appsecConfig.customRulesProvided) {
+  if (rc && appsecConfig && !appsecConfig.rules) {
     // dirty require to make startup faster for serverless
     const RuleManager = require('../rule_manager')
 

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -25,7 +25,8 @@ class RemoteConfigManager extends EventEmitter {
     super()
 
     const pollInterval = Math.floor(config.remoteConfig.pollInterval * 1000)
-    const url = config.url || new URL(format({
+
+    this.url = config.url || new URL(format({
       protocol: 'http:',
       hostname: config.hostname || 'localhost',
       port: config.port
@@ -33,12 +34,6 @@ class RemoteConfigManager extends EventEmitter {
 
     this.scheduler = new Scheduler((cb) => this.poll(cb), pollInterval)
 
-    this.requestOptions = {
-      url,
-      method: 'POST',
-      path: '/v0.7/config'
-    }
-
     this.state = {
       client: {
         state: { // updated by `parseConfig()`
@@ -122,7 +117,13 @@ class RemoteConfigManager extends EventEmitter {
   }
 
   poll (cb) {
-    request(this.getPayload(), this.requestOptions, (err, data, statusCode) => {
+    const options = {
+      url: this.url,
+      method: 'POST',
+      path: '/v0.7/config'
+    }
+
+    request(this.getPayload(), options, (err, data, statusCode) => {
       // 404 means RC is disabled, ignore it
       if (statusCode === 404) return cb()
 

package/packages/dd-trace/src/appsec/reporter.js

@@ -3,64 +3,62 @@
 const Limiter = require('../rate_limiter')
 const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
+const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   updateWafRequestsMetricTags,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric
 } = require('./telemetry')
+const zlib = require('zlib')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
 
-// TODO: use precomputed maps instead
-const REQUEST_HEADERS_PASSLIST = [
-  'accept',
-  'accept-encoding',
-  'accept-language',
+const metricsQueue = new Map()
+
+const contentHeaderList = [
   'content-encoding',
   'content-language',
   'content-length',
-  'content-type',
-  'forwarded',
-  'forwarded-for',
+  'content-type'
+]
+
+const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+  'accept',
+  'accept-encoding',
+  'accept-language',
   'host',
-  'true-client-ip',
+  'forwarded',
   'user-agent',
   'via',
-  'x-client-ip',
-  'x-cluster-client-ip',
-  'x-forwarded',
-  'x-forwarded-for',
-  'x-real-ip'
-]
+  'x-amzn-trace-id',
 
-const RESPONSE_HEADERS_PASSLIST = [
-  'content-encoding',
-  'content-language',
-  'content-length',
-  'content-type'
-]
+  ...ipHeaderList,
+  ...contentHeaderList
+], 'http.request.headers.')
 
-const metricsQueue = new Map()
+const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.')
 
-function filterHeaders (headers, passlist, prefix) {
+function mapHeaderAndTags (headerList, tagPrefix) {
+  return new Map(headerList.map(headerName => [headerName, `${tagPrefix}${formatHeaderName(headerName)}`]))
+}
+
+function filterHeaders (headers, map) {
   const result = {}
 
   if (!headers) return result
 
-  for (let i = 0; i < passlist.length; ++i) {
-    const headerName = passlist[i]
-
-    if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = '' + headers[headerName]
+  for (const [headerName, tagName] of map) {
+    const headerValue = headers[headerName]
+    if (headerValue) {
+      result[tagName] = '' + headerValue
     }
   }
 
   return result
 }
 
-// TODO: this can be precomputed at start time
 function formatHeaderName (name) {
   return name
     .trim()
@@ -86,7 +84,7 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
 function reportMetrics (metrics) {
   // TODO: metrics should be incremental, there already is an RFC to report metrics
   const store = storage.getStore()
-  const rootSpan = store && store.req && web.root(store.req)
+  const rootSpan = store?.req && web.root(store.req)
   if (!rootSpan) return
 
   if (metrics.duration) {
@@ -106,13 +104,13 @@ function reportMetrics (metrics) {
 
 function reportAttack (attackData) {
   const store = storage.getStore()
-  const req = store && store.req
+  const req = store?.req
   const rootSpan = web.root(req)
   if (!rootSpan) return
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
+  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
 
   newTags['appsec.event'] = 'true'
 
@@ -144,6 +142,23 @@ function reportAttack (attackData) {
   rootSpan.addTags(newTags)
 }
 
+function reportSchemas (derivatives) {
+  if (!derivatives) return
+
+  const req = storage.getStore()?.req
+  const rootSpan = web.root(req)
+
+  if (!rootSpan) return
+
+  const tags = {}
+  for (const [address, value] of Object.entries(derivatives)) {
+    const gzippedValue = zlib.gzipSync(JSON.stringify(value))
+    tags[address] = gzippedValue.toString('base64')
+  }
+
+  rootSpan.addTags(tags)
+}
+
 function finishRequest (req, res) {
   const rootSpan = web.root(req)
   if (!rootSpan) return
@@ -158,7 +173,7 @@ function finishRequest (req, res) {
 
   if (!rootSpan.context()._tags['appsec.event']) return
 
-  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
+  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP)
 
   if (req.route && typeof req.route.path === 'string') {
     newTags['http.endpoint'] = req.route.path
@@ -179,6 +194,8 @@ module.exports = {
   reportMetrics,
   reportAttack,
   reportWafUpdate: incrementWafUpdatesMetric,
+  reportSchemas,
   finishRequest,
-  setRateLimit
+  setRateLimit,
+  mapHeaderAndTags
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const fs = require('fs')
 const waf = require('./waf')
 const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 const blocking = require('./blocking')
@@ -13,13 +14,15 @@ let appliedExclusions = new Map()
 let appliedCustomRules = new Map()
 let appliedActions = new Map()
 
-function applyRules (rules, config) {
-  defaultRules = rules
+function loadRules (config) {
+  defaultRules = config.rules
+    ? JSON.parse(fs.readFileSync(config.rules))
+    : require('./recommended.json')
 
-  waf.init(rules, config)
+  waf.init(defaultRules, config)
 
-  if (rules.actions) {
-    blocking.updateBlockingConfiguration(rules.actions.find(action => action.id === 'block'))
+  if (defaultRules.actions) {
+    blocking.updateBlockingConfiguration(defaultRules.actions.find(action => action.id === 'block'))
   }
 }
 
@@ -66,9 +69,9 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) {
         item.apply_error = 'Multiple ruleset received in ASM_DD'
       } else {
         if (file && file.rules && file.rules.length) {
-          const { version, metadata, rules } = file
+          const { version, metadata, rules, processors, scanners } = file
 
-          newRuleset = { version, metadata, rules }
+          newRuleset = { version, metadata, rules, processors, scanners }
           newRulesetId = id
         }
 
@@ -252,7 +255,7 @@ function clearAllRules () {
 }
 
 module.exports = {
-  applyRules,
+  loadRules,
   updateWafFromRC,
   clearAllRules
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -9,7 +9,7 @@ const { setUserTags } = require('./set_user')
 const log = require('../../log')
 
 function isUserBlocked (user) {
-  const actions = waf.run({ [USER_ID]: user.id })
+  const actions = waf.run({ persistent: { [USER_ID]: user.id } })
 
   if (!actions) return false
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -10,37 +10,50 @@ const preventDuplicateAddresses = new Set([
 ])
 
 class WAFContextWrapper {
-  constructor (ddwafContext, requiredAddresses, wafTimeout, wafVersion, rulesVersion) {
+  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) {
     this.ddwafContext = ddwafContext
-    this.requiredAddresses = requiredAddresses
     this.wafTimeout = wafTimeout
     this.wafVersion = wafVersion
     this.rulesVersion = rulesVersion
     this.addressesToSkip = new Set()
   }
 
-  run (params) {
+  run ({ persistent, ephemeral }) {
+    const payload = {}
+    let payloadHasData = false
     const inputs = {}
-    let someInputAdded = false
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
-    for (const key of Object.keys(params)) {
-      if (this.requiredAddresses.has(key) && !this.addressesToSkip.has(key)) {
-        inputs[key] = params[key]
-        if (preventDuplicateAddresses.has(key)) {
-          newAddressesToSkip.add(key)
+    if (persistent && typeof persistent === 'object') {
+      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      for (const key of Object.keys(persistent)) {
+        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
+        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
+        if (!this.addressesToSkip.has(key)) {
+          inputs[key] = persistent[key]
+          if (preventDuplicateAddresses.has(key)) {
+            newAddressesToSkip.add(key)
+          }
         }
-        someInputAdded = true
       }
     }
 
-    if (!someInputAdded) return
+    if (Object.keys(inputs).length) {
+      payload['persistent'] = inputs
+      payloadHasData = true
+    }
+
+    if (ephemeral && Object.keys(ephemeral).length) {
+      payload['ephemeral'] = ephemeral
+      payloadHasData = true
+    }
+
+    if (!payloadHasData) return
 
     try {
       const start = process.hrtime.bigint()
 
-      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+      const result = this.ddwafContext.run(payload, this.wafTimeout)
 
       const end = process.hrtime.bigint()
 
@@ -63,6 +76,8 @@ class WAFContextWrapper {
         Reporter.reportAttack(JSON.stringify(result.events))
       }
 
+      Reporter.reportSchemas(result.derivatives)
+
       return result.actions
     } catch (err) {
       log.error('Error while running the AppSec WAF')

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -37,7 +37,6 @@ class WAFManager {
     if (!wafContext) {
       wafContext = new WAFContextWrapper(
         this.ddwaf.createContext(),
-        this.ddwaf.requiredAddresses,
         this.wafTimeout,
         this.ddwafVersion,
         this.rulesVersion

package/packages/dd-trace/src/ci-visibility/{intelligent-test-runner/get-itr-configuration.js → early-flake-detection/get-known-tests.js}

@@ -1,10 +1,11 @@
 const request = require('../../exporters/common/request')
 const id = require('../../id')
-const log = require('../../log')
 
-function getItrConfiguration ({
+function getKnownTests ({
   url,
   isEvpProxy,
+  evpProxyPrefix,
+  isGzipCompatible,
   env,
   service,
   repositoryUrl,
@@ -14,35 +15,39 @@ function getItrConfiguration ({
   osArchitecture,
   runtimeName,
   runtimeVersion,
-  branch,
   custom
 }, done) {
   const options = {
-    path: '/api/v2/libraries/tests/services/setting',
+    path: '/api/v2/ci/libraries/tests',
     method: 'POST',
     headers: {
       'Content-Type': 'application/json'
     },
+    timeout: 20000,
     url
   }
 
+  if (isGzipCompatible) {
+    options.headers['accept-encoding'] = 'gzip'
+  }
+
   if (isEvpProxy) {
-    options.path = '/evp_proxy/v2/api/v2/libraries/tests/services/setting'
+    options.path = `${evpProxyPrefix}/api/v2/ci/libraries/tests`
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
   } else {
     const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
     if (!apiKey) {
-      return done(new Error('Request to settings endpoint was not done because Datadog API key is not defined.'))
+      return done(new Error('Known tests were not fetched because Datadog API key is not defined.'))
     }
+
     options.headers['dd-api-key'] = apiKey
   }
 
   const data = JSON.stringify({
     data: {
       id: id().toString(10),
-      type: 'ci_app_test_service_libraries_settings',
+      type: 'ci_app_libraries_tests_request',
       attributes: {
-        test_level: 'suite',
         configurations: {
           'os.platform': osPlatform,
           'os.version': osVersion,
@@ -54,8 +59,7 @@ function getItrConfiguration ({
         service,
         env,
         repository_url: repositoryUrl,
-        sha,
-        branch
+        sha
       }
     }
   })
@@ -65,17 +69,8 @@ function getItrConfiguration ({
       done(err)
     } else {
       try {
-        const {
-          data: {
-            attributes: {
-              code_coverage: isCodeCoverageEnabled,
-              tests_skipping: isSuitesSkippingEnabled
-            }
-          }
-        } = JSON.parse(res)
-        const config = { isCodeCoverageEnabled, isSuitesSkippingEnabled }
-        log.debug(() => `Received settings: ${config}`)
-        done(null, config)
+        const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
+        done(null, knownTests)
       } catch (err) {
         done(err)
       }
@@ -83,4 +78,4 @@ function getItrConfiguration ({
   })
 }
 
-module.exports = { getItrConfiguration }
+module.exports = { getKnownTests }

package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -5,10 +5,23 @@ const AgentlessWriter = require('../agentless/writer')
 const CoverageWriter = require('../agentless/coverage-writer')
 const CiVisibilityExporter = require('../ci-visibility-exporter')
 
-const AGENT_EVP_PROXY_PATH = '/evp_proxy/v2'
+const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
+const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
 
-function getIsEvpCompatible (err, agentInfo) {
-  return !err && agentInfo.endpoints.some(url => url.includes(AGENT_EVP_PROXY_PATH))
+function getLatestEvpProxyVersion (err, agentInfo) {
+  if (err) {
+    return 0
+  }
+  return agentInfo.endpoints.reduce((acc, endpoint) => {
+    if (endpoint.includes(AGENT_EVP_PROXY_PATH_PREFIX)) {
+      const version = Number(endpoint.replace(AGENT_EVP_PROXY_PATH_REGEX, '$1'))
+      if (isNaN(version)) {
+        return acc
+      }
+      return version > acc ? version : acc
+    }
+    return acc
+  }, 0)
 }
 
 class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
@@ -25,17 +38,22 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
 
     this.getAgentInfo((err, agentInfo) => {
       this._isInitialized = true
-      const isEvpCompatible = getIsEvpCompatible(err, agentInfo)
+      const latestEvpProxyVersion = getLatestEvpProxyVersion(err, agentInfo)
+      const isEvpCompatible = latestEvpProxyVersion >= 2
+      const isGzipCompatible = latestEvpProxyVersion >= 4
+
+      const evpProxyPrefix = `${AGENT_EVP_PROXY_PATH_PREFIX}${latestEvpProxyVersion}`
       if (isEvpCompatible) {
         this._isUsingEvpProxy = true
+        this.evpProxyPrefix = evpProxyPrefix
         this._writer = new AgentlessWriter({
           url: this._url,
           tags,
-          evpProxyPrefix: AGENT_EVP_PROXY_PATH
+          evpProxyPrefix
         })
         this._coverageWriter = new CoverageWriter({
           url: this._url,
-          evpProxyPrefix: AGENT_EVP_PROXY_PATH
+          evpProxyPrefix
         })
       } else {
         this._writer = new AgentWriter({
@@ -51,6 +69,7 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
       this._resolveCanUseCiVisProtocol(isEvpCompatible)
       this.exportUncodedTraces()
       this.exportUncodedCoverages()
+      this._isGzipCompatible = isGzipCompatible
     })
   }
 

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -5,6 +5,16 @@ const { safeJSONStringify } = require('../../../exporters/common/util')
 
 const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS,
+  TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+  getErrorTypeFromStatusCode
+} = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
   constructor ({ url, evpProxyPrefix = '' }) {
@@ -34,8 +44,27 @@ class Writer extends BaseWriter {
 
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
-    request(form, options, (err, res) => {
+    const startRequestTime = Date.now()
+
+    incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'code_coverage' })
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'code_coverage' }, form.size())
+
+    request(form, options, (err, res, statusCode) => {
+      distributionMetric(
+        TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+        { endpoint: 'code_coverage' },
+        Date.now() - startRequestTime
+      )
       if (err) {
+        const errorType = getErrorTypeFromStatusCode(statusCode)
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+          { endpoint: 'code_coverage', errorType }
+        )
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+          { endpoint: 'code_coverage' }
+        )
         log.error(err)
         done()
         return

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -21,6 +21,8 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
     this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
 
     this._apiUrl = url || new URL(`https://api.${site}`)
+    // Agentless is always gzip compatible
+    this._isGzipCompatible = true
   }
 
   setUrl (url, coverageUrl = url, apiUrl = url) {