dd-trace 2.5.0 → 2.6.0

package/packages/dd-trace/src/appsec/reporter.js

@@ -35,6 +35,8 @@ const RESPONSE_HEADERS_PASSLIST = [
   'content-type'
 ]
 
+const metricsQueue = new Map()
+
 function resolveHTTPRequest (context) {
   if (!context) return {}
 
@@ -82,6 +84,20 @@ function formatHeaderName (name) {
     .toLowerCase()
 }
 
+function reportMetrics (metrics, store) {
+  const req = store && store.get('req')
+  const topSpan = web.root(req)
+  if (!topSpan) return false
+
+  if (metrics.duration) {
+    topSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
+  }
+
+  if (metrics.rulesVersion) {
+    topSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
+  }
+}
+
 function reportAttack (attackData, store) {
   const req = store && store.get('req')
   const topSpan = web.root(req)
@@ -129,9 +145,17 @@ function reportAttack (attackData, store) {
   topSpan.addTags(newTags)
 }
 
-function finishAttacks (req, context) {
+function finishRequest (req, context) {
   const topSpan = web.root(req)
-  if (!topSpan || !context) return false
+  if (!topSpan) return false
+
+  if (metricsQueue.size) {
+    topSpan.addTags(Object.fromEntries(metricsQueue))
+
+    metricsQueue.clear()
+  }
+
+  if (!context || !topSpan.context()._tags['appsec.event']) return false
 
   const resolvedResponse = resolveHTTPResponse(context)
 
@@ -149,11 +173,13 @@ function setRateLimit (rateLimit) {
 }
 
 module.exports = {
+  metricsQueue,
   resolveHTTPRequest,
   resolveHTTPResponse,
   filterHeaders,
   formatHeaderName,
+  reportMetrics,
   reportAttack,
-  finishAttacks,
+  finishRequest,
   setRateLimit
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,11 +4,11 @@ const callbacks = require('./callbacks')
 
 const appliedCallbacks = new Map()
 
-function applyRules (rules) {
+function applyRules (rules, config) {
   if (appliedCallbacks.has(rules)) return
 
   // for now there is only WAF
-  const callback = new callbacks.DDWAF(rules)
+  const callback = new callbacks.DDWAF(rules, config)
 
   appliedCallbacks.set(rules, callback)
 }

package/packages/dd-trace/src/config.js

@@ -150,10 +150,29 @@ class Config {
       path.join(__dirname, 'appsec', 'recommended.json')
     )
     const DD_APPSEC_TRACE_RATE_LIMIT = coalesce(
-      appsec.rateLimit,
-      process.env.DD_APPSEC_TRACE_RATE_LIMIT,
+      parseInt(appsec.rateLimit),
+      parseInt(process.env.DD_APPSEC_TRACE_RATE_LIMIT),
       100
     )
+    const DD_APPSEC_WAF_TIMEOUT = coalesce(
+      parseInt(appsec.wafTimeout),
+      parseInt(process.env.DD_APPSEC_WAF_TIMEOUT),
+      5e3 // µs
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = coalesce(
+      appsec.obfuscatorKeyRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|se\
+cret)|sign(?:ed|ature)|bearer|authorization`
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = coalesce(
+      appsec.obfuscatorValueRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\
+ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\
+\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\
+|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
+    )
 
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
@@ -223,7 +242,10 @@ class Config {
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
       rules: DD_APPSEC_RULES,
-      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT
+      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
+      wafTimeout: DD_APPSEC_WAF_TIMEOUT,
+      obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
 
     tagger.add(this.tags, {

package/packages/dd-trace/src/format.js

@@ -22,7 +22,6 @@ const map = {
 function format (span) {
   const formatted = formatSpan(span)
 
-  extractError(formatted, span)
   extractRootTags(formatted, span)
   extractChunkTags(formatted, span)
   extractTags(formatted, span)
@@ -74,8 +73,8 @@ function extractTags (trace, span) {
         addTag({}, trace.metrics, tag, tags[tag] === undefined || tags[tag] ? 1 : 0)
         break
       case 'error':
-        if (tags[tag] && (context._name !== 'fs.operation')) {
-          trace.error = 1
+        if (context._name !== 'fs.operation') {
+          extractError(trace, tags[tag])
         }
         break
       case 'error.type':
@@ -84,6 +83,8 @@ function extractTags (trace, span) {
         // HACK: remove when implemented in the backend
         if (context._name !== 'fs.operation') {
           trace.error = 1
+        } else {
+          break
         }
       default: // eslint-disable-line no-fallthrough
         addTag(trace.meta, trace.metrics, tag, tags[tag])
@@ -122,8 +123,11 @@ function extractChunkTags (trace, span) {
   }
 }
 
-function extractError (trace, span) {
-  const error = span.context()._tags['error']
+function extractError (trace, error) {
+  if (!error) return
+
+  trace.error = 1
+
   if (isError(error)) {
     addTag(trace.meta, trace.metrics, 'error.msg', error.message)
     addTag(trace.meta, trace.metrics, 'error.type', error.name)

package/packages/dd-trace/src/plugins/util/test.js

@@ -19,6 +19,10 @@ const {
 } = require('./tags')
 const id = require('../../id')
 
+const { SPAN_TYPE, RESOURCE_NAME, SAMPLING_PRIORITY } = require('../../../../../ext/tags')
+const { SAMPLING_RULE_DECISION } = require('../../constants')
+const { AUTO_KEEP } = require('../../../../../ext/priority')
+
 const TEST_FRAMEWORK = 'test.framework'
 const TEST_FRAMEWORK_VERSION = 'test.framework_version'
 const TEST_TYPE = 'test.type'
@@ -60,7 +64,8 @@ module.exports = {
   getTestParentSpan,
   getTestSuitePath,
   getCodeOwnersFileEntries,
-  getCodeOwnersForFilename
+  getCodeOwnersForFilename,
+  getTestCommonTags
 }
 
 function getTestEnvironmentMetadata (testFramework, config) {
@@ -134,6 +139,20 @@ function getTestParentSpan (tracer) {
     'x-datadog-parent-id': '0000000000000000'
   })
 }
+
+function getTestCommonTags (name, suite, version) {
+  return {
+    [SPAN_TYPE]: 'test',
+    [TEST_TYPE]: 'test',
+    [SAMPLING_RULE_DECISION]: 1,
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_NAME]: name,
+    [TEST_SUITE]: suite,
+    [RESOURCE_NAME]: `${suite}.${name}`,
+    [TEST_FRAMEWORK_VERSION]: version
+  }
+}
+
 /**
  * We want to make sure that test suites are reported the same way for
  * every OS, so we replace `path.sep` by `/`

package/packages/dd-trace/src/plugins/util/web.js

@@ -69,6 +69,17 @@ const web = {
     context.span = span
     context.res = res
 
+    if (!config.filter(req.url)) {
+      span.setTag(MANUAL_DROP, true)
+      span.context()._trace.isRecording = false
+    }
+
+    if (config.service) {
+      span.setTag(SERVICE_NAME, config.service)
+    }
+
+    analyticsSampler.sample(span, config.measured, true)
+
     return span
   },
   wrap (req) {
@@ -83,16 +94,6 @@ const web = {
   instrument (tracer, config, req, res, name, callback) {
     const span = this.startSpan(tracer, config, req, res, name)
 
-    if (!config.filter(req.url)) {
-      span.setTag(MANUAL_DROP, true)
-    }
-
-    if (config.service) {
-      span.setTag(SERVICE_NAME, config.service)
-    }
-
-    analyticsSampler.sample(span, config.measured, true)
-
     this.wrap(req)
 
     return callback && tracer.scope().activate(span, () => callback(span))

package/packages/dd-trace/src/profiling/profilers/cpu.js

@@ -3,7 +3,7 @@
 class NativeCpuProfiler {
   constructor (options = {}) {
     this.type = 'wall'
-    this._samplingInterval = options.samplingInterval || 10 * 1000
+    this._samplingInterval = options.samplingInterval || 1e6 / 99 // 99hz
     this._mapper = undefined
     this._pprof = undefined
   }

package/packages/dd-trace/src/span_processor.js

@@ -32,7 +32,10 @@ class SpanProcessor {
         }
       }
 
-      this._exporter.export(formatted)
+      if (formatted.length !== 0 && trace.isRecording !== false) {
+        this._exporter.export(formatted)
+      }
+
       this._erase(trace, active)
     }
   }

package/scripts/install_plugin_modules.js

@@ -186,6 +186,7 @@ const requirePackageJson = require('${requirePackageJsonPath}')
 
 module.exports = {
   get (id) { return require(id || '${name}') },
+  getPath (id) { return require.resolve(id || '${name}' ) },
   version () { return requirePackageJson('${name}', module).version }
 }
 `

package/packages/datadog-plugin-jest/src/jest-environment.js

@@ -1,272 +0,0 @@
-const { promisify } = require('util')
-
-const { RESOURCE_NAME } = require('../../../ext/tags')
-const {
-  TEST_NAME,
-  TEST_SUITE,
-  TEST_STATUS,
-  TEST_FRAMEWORK_VERSION,
-  JEST_TEST_RUNNER,
-  ERROR_MESSAGE,
-  ERROR_TYPE,
-  TEST_PARAMETERS,
-  CI_APP_ORIGIN,
-  getTestEnvironmentMetadata,
-  getTestParametersString,
-  finishAllTraceSpans,
-  getTestSuitePath
-} = require('../../dd-trace/src/plugins/util/test')
-const {
-  getFormattedJestTestParameters,
-  getTestSpanTags,
-  setSuppressedErrors
-} = require('./util')
-
-const originals = new WeakMap()
-
-function getVmContext (environment) {
-  if (typeof environment.getVmContext === 'function') {
-    return environment.getVmContext()
-  }
-  return null
-}
-
-function wrapEnvironment (BaseEnvironment) {
-  return class DatadogJestEnvironment extends BaseEnvironment {
-    constructor (config, context) {
-      super(config, context)
-      this.testSuite = getTestSuitePath(context.testPath, config.rootDir)
-      this.testSpansByTestName = {}
-      this.originalTestFnByTestName = {}
-    }
-  }
-}
-
-function createWrapTeardown (tracer, instrumenter) {
-  return function wrapTeardown (teardown) {
-    return async function teardownWithTrace () {
-      instrumenter.unwrap(this.global.test, 'each')
-      nameToParams = {}
-      // for jest-jasmine2
-      if (this.global.jasmine) {
-        instrumenter.unwrap(this.global.jasmine.Spec.prototype, 'onException')
-        instrumenter.unwrap(this.global, 'it')
-        instrumenter.unwrap(this.global, 'fit')
-        instrumenter.unwrap(this.global, 'xit')
-      }
-
-      instrumenter.unwrap(this.global.test, 'each')
-
-      return teardown.apply(this, arguments).finally(() => {
-        return new Promise(resolve => tracer._exporter._writer.flush(resolve))
-      })
-    }
-  }
-}
-
-let nameToParams = {}
-
-const isTimeout = (event) => {
-  return event.error &&
-  typeof event.error === 'string' &&
-  event.error.startsWith('Exceeded timeout')
-}
-
-function createHandleTestEvent (tracer, testEnvironmentMetadata, instrumenter) {
-  return async function handleTestEventWithTrace (event) {
-    if (event.name === 'test_retry') {
-      let testName = event.test && event.test.name
-      const context = getVmContext(this)
-      if (context) {
-        const { currentTestName } = context.expect.getState()
-        testName = currentTestName
-      }
-      // If it's a retry, we restore the original test function so that it is not wrapped again
-      if (this.originalTestFnByTestName[testName]) {
-        event.test.fn = this.originalTestFnByTestName[testName]
-      }
-      return
-    }
-    if (event.name === 'test_fn_failure') {
-      if (!isTimeout(event)) {
-        return
-      }
-      const context = getVmContext(this)
-      if (context) {
-        const { currentTestName } = context.expect.getState()
-        const testSpan = this.testSpansByTestName[`${currentTestName}_${event.test.invocations}`]
-        if (testSpan) {
-          testSpan.setTag(ERROR_TYPE, 'Timeout')
-          testSpan.setTag(ERROR_MESSAGE, event.error)
-          testSpan.setTag(TEST_STATUS, 'fail')
-        }
-      }
-      return
-    }
-    if (event.name === 'setup') {
-      instrumenter.wrap(this.global.test, 'each', function (original) {
-        return function () {
-          const testParameters = getFormattedJestTestParameters(arguments)
-          const eachBind = original.apply(this, arguments)
-          return function () {
-            const [testName] = arguments
-            nameToParams[testName] = testParameters
-            return eachBind.apply(this, arguments)
-          }
-        }
-      })
-      return
-    }
-
-    if (event.name !== 'test_skip' &&
-      event.name !== 'test_todo' &&
-      event.name !== 'test_start' &&
-      event.name !== 'hook_failure') {
-      return
-    }
-    // for hook_failure events the test entry might not be defined, because the hook
-    // is not necessarily associated to a test:
-    if (!event.test) {
-      return
-    }
-
-    const { childOf, commonSpanTags } = getTestSpanTags(tracer, testEnvironmentMetadata)
-
-    let testName = event.test.name
-    const context = getVmContext(this)
-
-    if (context) {
-      const { currentTestName } = context.expect.getState()
-      testName = currentTestName
-    }
-    const spanTags = {
-      ...commonSpanTags,
-      [TEST_NAME]: testName,
-      [TEST_SUITE]: this.testSuite,
-      [TEST_FRAMEWORK_VERSION]: tracer._version,
-      [JEST_TEST_RUNNER]: 'jest-circus'
-    }
-
-    const testParametersString = getTestParametersString(nameToParams, event.test.name)
-    if (testParametersString) {
-      spanTags[TEST_PARAMETERS] = testParametersString
-    }
-
-    const resource = `${this.testSuite}.${testName}`
-    if (event.name === 'test_skip' || event.name === 'test_todo') {
-      const testSpan = tracer.startSpan(
-        'jest.test',
-        {
-          childOf,
-          tags: {
-            ...spanTags,
-            [RESOURCE_NAME]: resource,
-            [TEST_STATUS]: 'skip'
-          }
-        }
-      )
-      testSpan.context()._trace.origin = CI_APP_ORIGIN
-      testSpan.finish()
-      return
-    }
-    if (event.name === 'hook_failure') {
-      const testSpan = tracer.startSpan(
-        'jest.test',
-        {
-          childOf,
-          tags: {
-            ...spanTags,
-            [RESOURCE_NAME]: resource,
-            [TEST_STATUS]: 'fail'
-          }
-        }
-      )
-      testSpan.context()._trace.origin = CI_APP_ORIGIN
-      if (event.test.errors && event.test.errors.length) {
-        const error = new Error(event.test.errors[0][0])
-        error.stack = event.test.errors[0][1].stack
-        testSpan.setTag('error', error)
-      }
-      testSpan.finish()
-      return
-    }
-    // event.name === test_start at this point
-    const environment = this
-    environment.originalTestFnByTestName[testName] = event.test.fn
-
-    let specFunction = event.test.fn
-    if (specFunction.length) {
-      specFunction = promisify(specFunction)
-    }
-    event.test.fn = tracer.wrap(
-      'jest.test',
-      {
-        type: 'test',
-        childOf,
-        resource,
-        tags: spanTags
-      },
-      async () => {
-        let result
-        const testSpan = tracer.scope().active()
-        environment.testSpansByTestName[`${testName}_${event.test.invocations}`] = testSpan
-        testSpan.context()._trace.origin = CI_APP_ORIGIN
-        try {
-          result = await specFunction()
-          // it may have been set already if the test timed out
-          let suppressedErrors = []
-          const context = getVmContext(environment)
-          if (context) {
-            suppressedErrors = context.expect.getState().suppressedErrors
-          }
-          setSuppressedErrors(suppressedErrors, testSpan)
-          if (!testSpan._spanContext._tags[TEST_STATUS]) {
-            testSpan.setTag(TEST_STATUS, 'pass')
-          }
-        } catch (error) {
-          testSpan.setTag(TEST_STATUS, 'fail')
-          testSpan.setTag('error', error)
-          throw error
-        } finally {
-          finishAllTraceSpans(testSpan)
-        }
-        return result
-      }
-    )
-  }
-}
-
-function patch (Environment, tracer, config) {
-  const testEnvironmentMetadata = getTestEnvironmentMetadata('jest', config)
-  const proto = Environment.prototype
-
-  this.wrap(proto, 'teardown', createWrapTeardown(tracer, this))
-
-  const newHandleTestEvent = createHandleTestEvent(tracer, testEnvironmentMetadata, this)
-  originals.set(newHandleTestEvent, proto.handleTestEvent)
-  proto.handleTestEvent = newHandleTestEvent
-
-  return wrapEnvironment(Environment)
-}
-
-function unpatch (Environment) {
-  const proto = Environment.prototype
-
-  this.unwrap(Environment.prototype, 'teardown')
-  proto.handleTestEvent = originals.get(proto.handleTestEvent)
-}
-
-module.exports = [
-  {
-    name: 'jest-environment-node',
-    versions: ['>=24.8.0'],
-    patch,
-    unpatch
-  },
-  {
-    name: 'jest-environment-jsdom',
-    versions: ['>=24.8.0'],
-    patch,
-    unpatch
-  }
-]