dd-trace 4.18.0 → 4.23.0

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -9,5 +9,10 @@ module.exports = {
   ASM_USER_BLOCKING: 1n << 7n,
   ASM_CUSTOM_RULES: 1n << 8n,
   ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
-  ASM_TRUSTED_IPS: 1n << 10n
+  ASM_TRUSTED_IPS: 1n << 10n,
+  ASM_API_SECURITY_SAMPLE_RATE: 1n << 11n,
+  APM_TRACING_SAMPLE_RATE: 1n << 12n,
+  APM_TRACING_LOGS_INJECTION: 1n << 13n,
+  APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n,
+  APM_TRACING_CUSTOM_TAGS: 1n << 15n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -1,38 +1,63 @@
 'use strict'
 
+const Activation = require('../activation')
+
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
+const apiSecuritySampler = require('../api_security_sampler')
 
 let rc
 
 function enable (config) {
   rc = new RemoteConfigManager(config)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true)
+
+  const activation = Activation.fromConfig(config)
 
-  if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally
-    rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+  if (activation !== Activation.DISABLED) {
+    if (activation === Activation.ONECLICK) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
+    }
 
-    rc.on('ASM_FEATURES', (action, conf) => {
-      if (conf && conf.asm && typeof conf.asm.enabled === 'boolean') {
-        let shouldEnable
+    if (config.appsec.apiSecurity?.enabled) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
+    }
 
-        if (action === 'apply' || action === 'modify') {
-          shouldEnable = conf.asm.enabled // take control
-        } else {
-          shouldEnable = config.appsec.enabled // give back control to local config
-        }
+    rc.on('ASM_FEATURES', (action, rcConfig) => {
+      if (!rcConfig) return
 
-        if (shouldEnable) {
-          require('..').enable(config)
-        } else {
-          require('..').disable()
-        }
+      if (activation === Activation.ONECLICK) {
+        enableOrDisableAppsec(action, rcConfig, config)
       }
+
+      apiSecuritySampler.setRequestSampling(rcConfig.api_security?.request_sample_rate)
     })
   }
 
   return rc
 }
 
+function enableOrDisableAppsec (action, rcConfig, config) {
+  if (typeof rcConfig.asm?.enabled === 'boolean') {
+    let shouldEnable
+
+    if (action === 'apply' || action === 'modify') {
+      shouldEnable = rcConfig.asm.enabled // take control
+    } else {
+      shouldEnable = config.appsec.enabled // give back control to local config
+    }
+
+    if (shouldEnable) {
+      require('..').enable(config)
+    } else {
+      require('..').disable()
+    }
+  }
+}
+
 function enableWafUpdate (appsecConfig) {
   if (rc && appsecConfig && !appsecConfig.customRulesProvided) {
     // dirty require to make startup faster for serverless

package/packages/dd-trace/src/appsec/reporter.js

@@ -3,64 +3,61 @@
 const Limiter = require('../rate_limiter')
 const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
+const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   updateWafRequestsMetricTags,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric
 } = require('./telemetry')
+const zlib = require('zlib')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
 
-// TODO: use precomputed maps instead
-const REQUEST_HEADERS_PASSLIST = [
-  'accept',
-  'accept-encoding',
-  'accept-language',
+const metricsQueue = new Map()
+
+const contentHeaderList = [
   'content-encoding',
   'content-language',
   'content-length',
-  'content-type',
-  'forwarded',
-  'forwarded-for',
+  'content-type'
+]
+
+const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+  'accept',
+  'accept-encoding',
+  'accept-language',
   'host',
-  'true-client-ip',
   'user-agent',
+  'forwarded',
   'via',
-  'x-client-ip',
-  'x-cluster-client-ip',
-  'x-forwarded',
-  'x-forwarded-for',
-  'x-real-ip'
-]
 
-const RESPONSE_HEADERS_PASSLIST = [
-  'content-encoding',
-  'content-language',
-  'content-length',
-  'content-type'
-]
+  ...ipHeaderList,
+  ...contentHeaderList
+], 'http.request.headers.')
 
-const metricsQueue = new Map()
+const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.')
 
-function filterHeaders (headers, passlist, prefix) {
+function mapHeaderAndTags (headerList, tagPrefix) {
+  return new Map(headerList.map(headerName => [headerName, `${tagPrefix}${formatHeaderName(headerName)}`]))
+}
+
+function filterHeaders (headers, map) {
   const result = {}
 
   if (!headers) return result
 
-  for (let i = 0; i < passlist.length; ++i) {
-    const headerName = passlist[i]
-
-    if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = '' + headers[headerName]
+  for (const [headerName, tagName] of map) {
+    const headerValue = headers[headerName]
+    if (headerValue) {
+      result[tagName] = '' + headerValue
     }
   }
 
   return result
 }
 
-// TODO: this can be precomputed at start time
 function formatHeaderName (name) {
   return name
     .trim()
@@ -86,7 +83,7 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
 function reportMetrics (metrics) {
   // TODO: metrics should be incremental, there already is an RFC to report metrics
   const store = storage.getStore()
-  const rootSpan = store && store.req && web.root(store.req)
+  const rootSpan = store?.req && web.root(store.req)
   if (!rootSpan) return
 
   if (metrics.duration) {
@@ -106,13 +103,13 @@ function reportMetrics (metrics) {
 
 function reportAttack (attackData) {
   const store = storage.getStore()
-  const req = store && store.req
+  const req = store?.req
   const rootSpan = web.root(req)
   if (!rootSpan) return
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
+  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
 
   newTags['appsec.event'] = 'true'
 
@@ -144,6 +141,23 @@ function reportAttack (attackData) {
   rootSpan.addTags(newTags)
 }
 
+function reportSchemas (derivatives) {
+  if (!derivatives) return
+
+  const req = storage.getStore()?.req
+  const rootSpan = web.root(req)
+
+  if (!rootSpan) return
+
+  const tags = {}
+  for (const [address, value] of Object.entries(derivatives)) {
+    const gzippedValue = zlib.gzipSync(JSON.stringify(value))
+    tags[address] = gzippedValue.toString('base64')
+  }
+
+  rootSpan.addTags(tags)
+}
+
 function finishRequest (req, res) {
   const rootSpan = web.root(req)
   if (!rootSpan) return
@@ -158,7 +172,7 @@ function finishRequest (req, res) {
 
   if (!rootSpan.context()._tags['appsec.event']) return
 
-  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
+  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP)
 
   if (req.route && typeof req.route.path === 'string') {
     newTags['http.endpoint'] = req.route.path
@@ -179,6 +193,8 @@ module.exports = {
   reportMetrics,
   reportAttack,
   reportWafUpdate: incrementWafUpdatesMetric,
+  reportSchemas,
   finishRequest,
-  setRateLimit
+  setRateLimit,
+  mapHeaderAndTags
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const fs = require('fs')
 const waf = require('./waf')
 const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 const blocking = require('./blocking')
@@ -13,13 +14,15 @@ let appliedExclusions = new Map()
 let appliedCustomRules = new Map()
 let appliedActions = new Map()
 
-function applyRules (rules, config) {
-  defaultRules = rules
+function loadRules (config) {
+  defaultRules = config.rules
+    ? JSON.parse(fs.readFileSync(config.rules))
+    : require('./recommended.json')
 
-  waf.init(rules, config)
+  waf.init(defaultRules, config)
 
-  if (rules.actions) {
-    blocking.updateBlockingConfiguration(rules.actions.find(action => action.id === 'block'))
+  if (defaultRules.actions) {
+    blocking.updateBlockingConfiguration(defaultRules.actions.find(action => action.id === 'block'))
   }
 }
 
@@ -252,7 +255,7 @@ function clearAllRules () {
 }
 
 module.exports = {
-  applyRules,
+  loadRules,
   updateWafFromRC,
   clearAllRules
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -9,7 +9,7 @@ const { setUserTags } = require('./set_user')
 const log = require('../../log')
 
 function isUserBlocked (user) {
-  const actions = waf.run({ [USER_ID]: user.id })
+  const actions = waf.run({ persistent: { [USER_ID]: user.id } })
 
   if (!actions) return false
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -10,37 +10,50 @@ const preventDuplicateAddresses = new Set([
 ])
 
 class WAFContextWrapper {
-  constructor (ddwafContext, requiredAddresses, wafTimeout, wafVersion, rulesVersion) {
+  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) {
     this.ddwafContext = ddwafContext
-    this.requiredAddresses = requiredAddresses
     this.wafTimeout = wafTimeout
     this.wafVersion = wafVersion
     this.rulesVersion = rulesVersion
     this.addressesToSkip = new Set()
   }
 
-  run (params) {
+  run ({ persistent, ephemeral }) {
+    const payload = {}
+    let payloadHasData = false
     const inputs = {}
-    let someInputAdded = false
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
-    for (const key of Object.keys(params)) {
-      if (this.requiredAddresses.has(key) && !this.addressesToSkip.has(key)) {
-        inputs[key] = params[key]
-        if (preventDuplicateAddresses.has(key)) {
-          newAddressesToSkip.add(key)
+    if (persistent && typeof persistent === 'object') {
+      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      for (const key of Object.keys(persistent)) {
+        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
+        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
+        if (!this.addressesToSkip.has(key)) {
+          inputs[key] = persistent[key]
+          if (preventDuplicateAddresses.has(key)) {
+            newAddressesToSkip.add(key)
+          }
         }
-        someInputAdded = true
       }
     }
 
-    if (!someInputAdded) return
+    if (Object.keys(inputs).length) {
+      payload['persistent'] = inputs
+      payloadHasData = true
+    }
+
+    if (ephemeral && Object.keys(ephemeral).length) {
+      payload['ephemeral'] = ephemeral
+      payloadHasData = true
+    }
+
+    if (!payloadHasData) return
 
     try {
       const start = process.hrtime.bigint()
 
-      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+      const result = this.ddwafContext.run(payload, this.wafTimeout)
 
       const end = process.hrtime.bigint()
 
@@ -63,6 +76,8 @@ class WAFContextWrapper {
         Reporter.reportAttack(JSON.stringify(result.events))
       }
 
+      Reporter.reportSchemas(result.derivatives)
+
       return result.actions
     } catch (err) {
       log.error('Error while running the AppSec WAF')

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -37,7 +37,6 @@ class WAFManager {
     if (!wafContext) {
       wafContext = new WAFContextWrapper(
         this.ddwaf.createContext(),
-        this.ddwaf.requiredAddresses,
         this.wafTimeout,
         this.ddwafVersion,
         this.rulesVersion

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -5,6 +5,16 @@ const { safeJSONStringify } = require('../../../exporters/common/util')
 
 const { CoverageCIVisibilityEncoder } = require('../../../encode/coverage-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS,
+  TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+  getErrorTypeFromStatusCode
+} = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
   constructor ({ url, evpProxyPrefix = '' }) {
@@ -34,8 +44,27 @@ class Writer extends BaseWriter {
 
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
-    request(form, options, (err, res) => {
+    const startRequestTime = Date.now()
+
+    incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'code_coverage' })
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'code_coverage' }, form.size())
+
+    request(form, options, (err, res, statusCode) => {
+      distributionMetric(
+        TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+        { endpoint: 'code_coverage' },
+        Date.now() - startRequestTime
+      )
       if (err) {
+        const errorType = getErrorTypeFromStatusCode(statusCode)
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+          { endpoint: 'code_coverage', errorType }
+        )
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+          { endpoint: 'code_coverage' }
+        )
         log.error(err)
         done()
         return

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -5,6 +5,16 @@ const log = require('../../../log')
 
 const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
 const BaseWriter = require('../../../exporters/common/writer')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS,
+  TELEMETRY_ENDPOINT_PAYLOAD_BYTES,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+  TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+  getErrorTypeFromStatusCode
+} = require('../../../ci-visibility/telemetry')
 
 class Writer extends BaseWriter {
   constructor ({ url, tags, evpProxyPrefix = '' }) {
@@ -35,8 +45,27 @@ class Writer extends BaseWriter {
 
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
-    request(data, options, (err, res) => {
+    const startRequestTime = Date.now()
+
+    incrementCountMetric(TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS, { endpoint: 'test_cycle' })
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_BYTES, { endpoint: 'test_cycle' }, data.length)
+
+    request(data, options, (err, res, statusCode) => {
+      distributionMetric(
+        TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_MS,
+        { endpoint: 'test_cycle' },
+        Date.now() - startRequestTime
+      )
       if (err) {
+        const errorType = getErrorTypeFromStatusCode(statusCode)
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_REQUESTS_ERRORS,
+          { endpoint: 'test_cycle', errorType }
+        )
+        incrementCountMetric(
+          TELEMETRY_ENDPOINT_PAYLOAD_DROPPED,
+          { endpoint: 'test_cycle' }
+        )
         log.error(err)
         done()
         return

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -143,7 +143,23 @@ class CiVisibilityExporter extends AgentInfoExporter {
          * where the tests run in a subprocess, because `getItrConfiguration` is called only once.
          */
         this._itrConfig = itrConfig
-        callback(err, itrConfig)
+
+        if (err) {
+          callback(err, {})
+        } else if (itrConfig?.requireGit) {
+          // If the backend requires git, we'll wait for the upload to finish and request settings again
+          this._gitUploadPromise.then(gitUploadError => {
+            if (gitUploadError) {
+              return callback(gitUploadError, {})
+            }
+            getItrConfigurationRequest(configuration, (err, finalItrConfig) => {
+              this._itrConfig = finalItrConfig
+              callback(err, finalItrConfig)
+            })
+          })
+        } else {
+          callback(null, itrConfig)
+        }
       })
     })
   }