dd-trace 4.14.0 → 4.16.0

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -8,5 +8,6 @@ module.exports = {
   ASM_REQUEST_BLOCKING: 1n << 5n,
   ASM_USER_BLOCKING: 1n << 7n,
   ASM_CUSTOM_RULES: 1n << 8n,
-  ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n
+  ASM_CUSTOM_BLOCKING_RESPONSE: 1n << 9n,
+  ASM_TRUSTED_IPS: 1n << 10n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -46,6 +46,7 @@ function enableWafUpdate (appsecConfig) {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true)
 
     rc.on('ASM_DATA', noop)
     rc.on('ASM_DD', noop)
@@ -66,6 +67,7 @@ function disableWafUpdate () {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false)
 
     rc.off('ASM_DATA', noop)
     rc.off('ASM_DD', noop)

package/packages/dd-trace/src/appsec/reporter.js

@@ -3,6 +3,12 @@
 const Limiter = require('../rate_limiter')
 const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
+const {
+  incrementWafInitMetric,
+  updateWafRequestsMetricTags,
+  incrementWafUpdatesMetric,
+  incrementWafRequestsMetric
+} = require('./telemetry')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
@@ -63,6 +69,20 @@ function formatHeaderName (name) {
     .toLowerCase()
 }
 
+function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
+  metricsQueue.set('_dd.appsec.waf.version', wafVersion)
+
+  metricsQueue.set('_dd.appsec.event_rules.loaded', diagnosticsRules.loaded?.length || 0)
+  metricsQueue.set('_dd.appsec.event_rules.error_count', diagnosticsRules.failed?.length || 0)
+  if (diagnosticsRules.failed?.length) {
+    metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors))
+  }
+
+  metricsQueue.set('manual.keep', 'true')
+
+  incrementWafInitMetric(wafVersion, rulesVersion)
+}
+
 function reportMetrics (metrics) {
   // TODO: metrics should be incremental, there already is an RFC to report metrics
   const store = storage.getStore()
@@ -80,6 +100,8 @@ function reportMetrics (metrics) {
   if (metrics.rulesVersion) {
     rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
   }
+
+  updateWafRequestsMetricTags(metrics, store.req)
 }
 
 function reportAttack (attackData) {
@@ -132,6 +154,8 @@ function finishRequest (req, res) {
     metricsQueue.clear()
   }
 
+  incrementWafRequestsMetric(req)
+
   if (!rootSpan.context()._tags['appsec.event']) return
 
   const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
@@ -151,8 +175,10 @@ module.exports = {
   metricsQueue,
   filterHeaders,
   formatHeaderName,
+  reportWafInit,
   reportMetrics,
   reportAttack,
+  reportWafUpdate: incrementWafUpdatesMetric,
   finishRequest,
   setRateLimit
 }

package/packages/dd-trace/src/appsec/telemetry.js

@@ -0,0 +1,132 @@
+'use strict'
+
+const telemetryMetrics = require('../telemetry/metrics')
+
+const appsecMetrics = telemetryMetrics.manager.namespace('appsec')
+
+const DD_TELEMETRY_WAF_RESULT_TAGS = Symbol('_dd.appsec.telemetry.waf.result.tags')
+
+const tags = {
+  REQUEST_BLOCKED: 'request_blocked',
+  RULE_TRIGGERED: 'rule_triggered',
+  WAF_TIMEOUT: 'waf_timeout',
+  WAF_VERSION: 'waf_version',
+  EVENT_RULES_VERSION: 'event_rules_version'
+}
+
+const metricsStoreMap = new WeakMap()
+
+let enabled = false
+
+function enable (telemetryConfig) {
+  enabled = telemetryConfig?.enabled && telemetryConfig.metrics
+}
+
+function disable () {
+  enabled = false
+}
+
+function getStore (req) {
+  let store = metricsStoreMap.get(req)
+  if (!store) {
+    store = {}
+    metricsStoreMap.set(req, store)
+  }
+  return store
+}
+
+function getVersionsTags (wafVersion, rulesVersion) {
+  return {
+    [tags.WAF_VERSION]: wafVersion,
+    [tags.EVENT_RULES_VERSION]: rulesVersion
+  }
+}
+
+function trackWafDurations (metrics, versionsTags) {
+  if (metrics.duration) {
+    appsecMetrics.distribution('waf.duration', versionsTags).track(metrics.duration)
+  }
+  if (metrics.durationExt) {
+    appsecMetrics.distribution('waf.duration_ext', versionsTags).track(metrics.durationExt)
+  }
+}
+
+function getOrCreateMetricTags (req, versionsTags) {
+  const store = getStore(req)
+
+  let metricTags = store[DD_TELEMETRY_WAF_RESULT_TAGS]
+  if (!metricTags) {
+    metricTags = {
+      [tags.REQUEST_BLOCKED]: false,
+      [tags.RULE_TRIGGERED]: false,
+      [tags.WAF_TIMEOUT]: false,
+
+      ...versionsTags
+    }
+    store[DD_TELEMETRY_WAF_RESULT_TAGS] = metricTags
+  }
+  return metricTags
+}
+
+function updateWafRequestsMetricTags (metrics, req) {
+  if (!req || !enabled) return
+
+  const versionsTags = getVersionsTags(metrics.wafVersion, metrics.rulesVersion)
+
+  trackWafDurations(metrics, versionsTags)
+
+  const metricTags = getOrCreateMetricTags(req, versionsTags)
+
+  const { blockTriggered, ruleTriggered, wafTimeout } = metrics
+
+  if (blockTriggered) {
+    metricTags[tags.REQUEST_BLOCKED] = blockTriggered
+  }
+  if (ruleTriggered) {
+    metricTags[tags.RULE_TRIGGERED] = ruleTriggered
+  }
+  if (wafTimeout) {
+    metricTags[tags.WAF_TIMEOUT] = wafTimeout
+  }
+
+  return metricTags
+}
+
+function incrementWafInitMetric (wafVersion, rulesVersion) {
+  if (!enabled) return
+
+  const versionsTags = getVersionsTags(wafVersion, rulesVersion)
+
+  appsecMetrics.count('waf.init', versionsTags).inc()
+}
+
+function incrementWafUpdatesMetric (wafVersion, rulesVersion) {
+  if (!enabled) return
+
+  const versionsTags = getVersionsTags(wafVersion, rulesVersion)
+
+  appsecMetrics.count('waf.updates', versionsTags).inc()
+}
+
+function incrementWafRequestsMetric (req) {
+  if (!req || !enabled) return
+
+  const store = getStore(req)
+
+  const metricTags = store[DD_TELEMETRY_WAF_RESULT_TAGS]
+  if (metricTags) {
+    appsecMetrics.count('waf.requests', metricTags).inc()
+  }
+
+  metricsStoreMap.delete(req)
+}
+
+module.exports = {
+  enable,
+  disable,
+
+  updateWafRequestsMetricTags,
+  incrementWafInitMetric,
+  incrementWafUpdatesMetric,
+  incrementWafRequestsMetric
+}

package/packages/dd-trace/src/appsec/waf/index.js

@@ -39,7 +39,7 @@ function update (newRules) {
   if (!waf.wafManager) throw new Error('Cannot update disabled WAF')
 
   try {
-    waf.wafManager.ddwaf.update(newRules)
+    waf.wafManager.update(newRules)
   } catch (err) {
     log.error('Could not apply rules from remote config')
     throw err

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -4,11 +4,12 @@ const log = require('../../log')
 const Reporter = require('../reporter')
 
 class WAFContextWrapper {
-  constructor (ddwafContext, requiredAddresses, wafTimeout, rulesInfo) {
+  constructor (ddwafContext, requiredAddresses, wafTimeout, wafVersion, rulesVersion) {
     this.ddwafContext = ddwafContext
     this.requiredAddresses = requiredAddresses
     this.wafTimeout = wafTimeout
-    this.rulesInfo = rulesInfo
+    this.wafVersion = wafVersion
+    this.rulesVersion = rulesVersion
   }
 
   run (params) {
@@ -32,14 +33,21 @@ class WAFContextWrapper {
 
       const end = process.hrtime.bigint()
 
+      const ruleTriggered = !!result.events?.length
+      const blockTriggered = result.actions?.includes('block')
+
       Reporter.reportMetrics({
         duration: result.totalRuntime / 1e3,
         durationExt: parseInt(end - start) / 1e3,
-        rulesVersion: this.rulesInfo.version
+        rulesVersion: this.rulesVersion,
+        ruleTriggered,
+        blockTriggered,
+        wafVersion: this.wafVersion,
+        wafTimeout: result.timeout
       })
 
-      if (result.data && result.data !== '[]') {
-        Reporter.reportAttack(result.data)
+      if (ruleTriggered) {
+        Reporter.reportAttack(JSON.stringify(result.events))
       }
 
       return result.actions

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -11,7 +11,10 @@ class WAFManager {
     this.config = config
     this.wafTimeout = config.wafTimeout
     this.ddwaf = this._loadDDWAF(rules)
-    this._reportMetrics()
+    this.ddwafVersion = this.ddwaf.constructor.version()
+    this.rulesVersion = this.ddwaf.diagnostics.ruleset_version
+
+    Reporter.reportWafInit(this.ddwafVersion, this.rulesVersion, this.ddwaf.diagnostics.rules)
   }
 
   _loadDDWAF (rules) {
@@ -28,18 +31,6 @@ class WAFManager {
     }
   }
 
-  _reportMetrics () {
-    Reporter.metricsQueue.set('_dd.appsec.waf.version', this.ddwaf.constructor.version())
-
-    const { loaded, failed, errors } = this.ddwaf.rulesInfo
-
-    Reporter.metricsQueue.set('_dd.appsec.event_rules.loaded', loaded)
-    Reporter.metricsQueue.set('_dd.appsec.event_rules.error_count', failed)
-    if (failed) Reporter.metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(errors))
-
-    Reporter.metricsQueue.set('manual.keep', 'true')
-  }
-
   getWAFContext (req) {
     let wafContext = contexts.get(req)
 
@@ -48,7 +39,8 @@ class WAFManager {
         this.ddwaf.createContext(),
         this.ddwaf.requiredAddresses,
         this.wafTimeout,
-        this.ddwaf.rulesInfo
+        this.ddwafVersion,
+        this.rulesVersion
       )
       contexts.set(req, wafContext)
     }
@@ -56,6 +48,12 @@ class WAFManager {
     return wafContext
   }
 
+  update (newRules) {
+    this.ddwaf.update(newRules)
+
+    Reporter.reportWafUpdate(this.ddwafVersion, this.rulesVersion)
+  }
+
   destroy () {
     if (this.ddwaf) {
       this.ddwaf.dispose()

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-itr-configuration.js

@@ -28,25 +28,12 @@ function getItrConfiguration ({
   if (isEvpProxy) {
     options.path = '/evp_proxy/v2/api/v2/libraries/tests/services/setting'
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
-    options.headers['X-Datadog-NeedsAppKey'] = 'true'
   } else {
     const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    const appKey = process.env.DATADOG_APP_KEY ||
-      process.env.DD_APP_KEY ||
-      process.env.DATADOG_APPLICATION_KEY ||
-      process.env.DD_APPLICATION_KEY
-
-    const messagePrefix = 'Request to settings endpoint was not done because Datadog'
-
-    if (!appKey) {
-      return done(new Error(`${messagePrefix} application key is not defined.`))
-    }
     if (!apiKey) {
-      return done(new Error(`${messagePrefix} API key is not defined.`))
+      return done(new Error('Request to settings endpoint was not done because Datadog API key is not defined.'))
     }
-
     options.headers['dd-api-key'] = apiKey
-    options.headers['dd-application-key'] = appKey
   }
 
   const data = JSON.stringify({

package/packages/dd-trace/src/ci-visibility/intelligent-test-runner/get-skippable-suites.js

@@ -28,25 +28,13 @@ function getSkippableSuites ({
   if (isEvpProxy) {
     options.path = '/evp_proxy/v2/api/v2/ci/tests/skippable'
     options.headers['X-Datadog-EVP-Subdomain'] = 'api'
-    options.headers['X-Datadog-NeedsAppKey'] = 'true'
   } else {
     const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY
-    const appKey = process.env.DATADOG_APP_KEY ||
-      process.env.DD_APP_KEY ||
-      process.env.DATADOG_APPLICATION_KEY ||
-      process.env.DD_APPLICATION_KEY
-
-    const messagePrefix = 'Skippable suites were not fetched because Datadog'
-
-    if (!appKey) {
-      return done(new Error(`${messagePrefix} application key is not defined.`))
-    }
     if (!apiKey) {
-      return done(new Error(`${messagePrefix} API key is not defined.`))
+      return done(new Error('Skippable suites were not fetched because Datadog API key is not defined.'))
     }
 
     options.headers['dd-api-key'] = apiKey
-    options.headers['dd-application-key'] = appKey
   }
 
   const data = JSON.stringify({

package/packages/dd-trace/src/datastreams/processor.js

@@ -66,7 +66,9 @@ class DataStreamsProcessor {
     port,
     url,
     env,
-    tags
+    tags,
+    version,
+    service
   } = {}) {
     this.writer = new DataStreamsWriter({
       hostname,
@@ -79,7 +81,8 @@ class DataStreamsProcessor {
     this.enabled = dsmEnabled
     this.env = env
     this.tags = tags || {}
-    this.service = this.tags.service || 'unnamed-nodejs-service'
+    this.service = service || 'unnamed-nodejs-service'
+    this.version = version || ''
     this.sequence = 0
 
     if (this.enabled) {
@@ -96,6 +99,7 @@ class DataStreamsProcessor {
       Service: this.service,
       Stats: serialized,
       TracerVersion: pkg.version,
+      Version: this.version,
       Lang: 'javascript'
     }
     this.writer.flush(payload)

package/packages/dd-trace/src/format.js

@@ -13,7 +13,7 @@ const SPAN_SAMPLING_MECHANISM = constants.SPAN_SAMPLING_MECHANISM
 const SPAN_SAMPLING_RULE_RATE = constants.SPAN_SAMPLING_RULE_RATE
 const SPAN_SAMPLING_MAX_PER_SECOND = constants.SPAN_SAMPLING_MAX_PER_SECOND
 const SAMPLING_MECHANISM_SPAN = constants.SAMPLING_MECHANISM_SPAN
-const MEASURED = tags.MEASURED
+const { MEASURED, BASE_SERVICE } = tags
 const ORIGIN_KEY = constants.ORIGIN_KEY
 const HOSTNAME_KEY = constants.HOSTNAME_KEY
 const TOP_LEVEL_KEY = constants.TOP_LEVEL_KEY
@@ -73,6 +73,11 @@ function extractTags (trace, span) {
     addTag({}, trace.metrics, MEASURED, 1)
   }
 
+  const tracerService = span.tracer()._service.toLowerCase()
+  if (tags['service.name']?.toLowerCase() !== tracerService) {
+    span.setTag(BASE_SERVICE, tracerService)
+  }
+
   for (const tag in tags) {
     switch (tag) {
       case 'service.name':

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -343,10 +343,10 @@ class TextMapPropagator {
               spanContext._trace.origin = value
               break
             case 't.dm': {
-              const mechanism = -Math.abs(parseInt(value, 10))
+              const mechanism = Math.abs(parseInt(value, 10))
               if (Number.isInteger(mechanism)) {
                 spanContext._sampling.mechanism = mechanism
-                spanContext._trace.tags['_dd.p.dm'] = String(mechanism)
+                spanContext._trace.tags['_dd.p.dm'] = `-${mechanism}`
               }
               break
             }

package/packages/dd-trace/src/opentracing/tracer.js

@@ -26,8 +26,6 @@ class DatadogTracer {
     this._version = config.version
     this._env = config.env
     this._tags = config.tags
-    this._computePeerService = config.spanComputePeerService
-    this._peerServiceMapping = config.peerServiceMapping
     this._logInjection = config.logInjection
     this._debug = config.debug
     this._prioritySampler = new PrioritySampler(config.env, config.sampler)

package/packages/dd-trace/src/plugin_manager.js

@@ -72,11 +72,10 @@ module.exports = class PluginManager {
     const Plugin = pluginClasses[name]
 
     if (!Plugin) return
+    if (!this._tracerConfig) return // TODO: don't wait for tracer to be initialized
     if (!this._pluginsByName[name]) {
       this._pluginsByName[name] = new Plugin(this._tracer, this._tracerConfig)
     }
-    if (!this._tracerConfig) return // TODO: don't wait for tracer to be initialized
-
     const pluginConfig = this._configsByName[name] || {
       enabled: this._tracerConfig.plugins !== false
     }

package/packages/dd-trace/src/plugins/database.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const StoragePlugin = require('./storage')
+const { PEER_SERVICE_KEY } = require('../constants')
 
 class DatabasePlugin extends StoragePlugin {
   static get operation () { return 'query' }
@@ -38,20 +39,29 @@ class DatabasePlugin extends StoragePlugin {
     `ddps='${encodedDdps}',ddpv='${encodedDdpv}'`
   }
 
-  injectDbmQuery (query, serviceName, isPreparedStatement = false) {
+  getDbmServiceName (span, tracerService) {
+    if (this._tracerConfig.spanComputePeerService) {
+      const peerData = this.getPeerService(span.context()._tags)
+      return this.getPeerServiceRemap(peerData)[PEER_SERVICE_KEY] || tracerService
+    }
+    return tracerService
+  }
+
+  injectDbmQuery (span, query, serviceName, isPreparedStatement = false) {
     const mode = this.config.dbmPropagationMode
+    const dbmService = this.getDbmServiceName(span, serviceName)
 
     if (mode === 'disabled') {
       return query
     }
 
-    const servicePropagation = this.createDBMPropagationCommentService(serviceName)
+    const servicePropagation = this.createDBMPropagationCommentService(dbmService)
 
     if (isPreparedStatement || mode === 'service') {
       return `/*${servicePropagation}*/ ${query}`
     } else if (mode === 'full') {
-      this.activeSpan.setTag('_dd.dbm_trace_injected', 'true')
-      const traceparent = this.activeSpan._spanContext.toTraceparent()
+      span.setTag('_dd.dbm_trace_injected', 'true')
+      const traceparent = span._spanContext.toTraceparent()
       return `/*${servicePropagation},traceparent='${traceparent}'*/ ${query}`
     }
   }

package/packages/dd-trace/src/plugins/index.js

@@ -64,6 +64,7 @@ module.exports = {
   get 'pg' () { return require('../../../datadog-plugin-pg/src') },
   get 'pino' () { return require('../../../datadog-plugin-pino/src') },
   get 'pino-pretty' () { return require('../../../datadog-plugin-pino/src') },
+  get 'playwright' () { return require('../../../datadog-plugin-playwright/src') },
   get 'redis' () { return require('../../../datadog-plugin-redis/src') },
   get 'restify' () { return require('../../../datadog-plugin-restify/src') },
   get 'rhea' () { return require('../../../datadog-plugin-rhea/src') },

package/packages/dd-trace/src/plugins/outbound.js

@@ -64,10 +64,11 @@ class OutboundPlugin extends TracingPlugin {
      * peer service and add the value we overrode.
      */
     const peerService = peerData[PEER_SERVICE_KEY]
-    if (peerService && this.tracer._peerServiceMapping[peerService]) {
+    const mappedService = this._tracerConfig.peerServiceMapping?.[peerService]
+    if (peerService && mappedService) {
       return {
         ...peerData,
-        [PEER_SERVICE_KEY]: this.tracer._peerServiceMapping[peerService],
+        [PEER_SERVICE_KEY]: mappedService,
         [PEER_SERVICE_REMAP_KEY]: peerService
       }
     }
@@ -80,7 +81,7 @@ class OutboundPlugin extends TracingPlugin {
   }
 
   tagPeerService (span) {
-    if (this.tracer._computePeerService) {
+    if (this._tracerConfig.spanComputePeerService) {
       const peerData = this.getPeerService(span.context()._tags)
       if (peerData !== undefined) {
         span.addTags(this.getPeerServiceRemap(peerData))

package/packages/dd-trace/src/plugins/tracing.js

@@ -55,7 +55,7 @@ class TracingPlugin extends Plugin {
   start () {} // implemented by individual plugins
 
   finish () {
-    this.activeSpan.finish()
+    this.activeSpan?.finish()
   }
 
   error (error) {

package/packages/dd-trace/src/telemetry/index.js

@@ -57,11 +57,20 @@ function appStarted () {
   return {
     integrations: getIntegrations(),
     dependencies: [],
-    configuration: flatten(config),
+    configuration: flatten(formatConfig(config)),
     additional_payload: []
   }
 }
 
+function formatConfig (config) {
+  // format peerServiceMapping from an object to a string map in order for
+  // telemetry intake to accept the configuration
+  config.peerServiceMapping = config.peerServiceMapping
+    ? Object.entries(config.peerServiceMapping).map(([key, value]) => `${key}:${value}`).join(',')
+    : ''
+  return config
+}
+
 function onBeforeExit () {
   process.removeListener('beforeExit', onBeforeExit)
   sendData(config, application, host, 'app-closing')

package/packages/dd-trace/src/util.js

@@ -66,7 +66,7 @@ function globMatch (pattern, subject) {
 function calculateDDBasePath (dirname) {
   const dirSteps = dirname.split(path.sep)
   const packagesIndex = dirSteps.lastIndexOf('packages')
-  return dirSteps.slice(0, packagesIndex).join(path.sep) + path.sep
+  return dirSteps.slice(0, packagesIndex + 1).join(path.sep) + path.sep
 }
 
 module.exports = {