dd-trace 2.4.2 → 2.6.0

package/packages/datadog-plugin-jest/src/util.js

@@ -1,8 +1,3 @@
-const { SAMPLING_RULE_DECISION } = require('../../dd-trace/src/constants')
-const { SAMPLING_PRIORITY, SPAN_TYPE } = require('../../../ext/tags')
-const { AUTO_KEEP } = require('../../../ext/priority')
-const { TEST_TYPE, TEST_STATUS, getTestParentSpan } = require('../../dd-trace/src/plugins/util/test')
-
 /**
  * There are two ways to call `test.each` in `jest`:
  * 1. With an array of arrays: https://jestjs.io/docs/api#1-testeachtablename-fn-timeout
@@ -38,27 +33,4 @@ function getFormattedJestTestParameters (testParameters) {
   return formattedParameters
 }
 
-function getTestSpanTags (tracer, testEnvironmentMetadata) {
-  const childOf = getTestParentSpan(tracer)
-
-  const commonSpanTags = {
-    [TEST_TYPE]: 'test',
-    [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP,
-    [SPAN_TYPE]: 'test',
-    ...testEnvironmentMetadata
-  }
-  return {
-    childOf,
-    commonSpanTags
-  }
-}
-
-function setSuppressedErrors (suppressedErrors, testSpan) {
-  if (suppressedErrors && suppressedErrors.length) {
-    testSpan.setTag('error', suppressedErrors[0])
-    testSpan.setTag(TEST_STATUS, 'fail')
-  }
-}
-
-module.exports = { getFormattedJestTestParameters, getTestSpanTags, setSuppressedErrors }
+module.exports = { getFormattedJestTestParameters }

package/packages/datadog-plugin-mocha/src/index.js

@@ -5,21 +5,19 @@ const { storage } = require('../../datadog-core')
 
 const {
   CI_APP_ORIGIN,
-  TEST_TYPE,
-  TEST_NAME,
+  TEST_CODE_OWNERS,
   TEST_SUITE,
-  TEST_FRAMEWORK_VERSION,
   TEST_STATUS,
   TEST_PARAMETERS,
   finishAllTraceSpans,
   getTestEnvironmentMetadata,
   getTestSuitePath,
   getTestParentSpan,
-  getTestParametersString
+  getTestParametersString,
+  getCodeOwnersFileEntries,
+  getCodeOwnersForFilename,
+  getTestCommonTags
 } = require('../../dd-trace/src/plugins/util/test')
-const { SPAN_TYPE, RESOURCE_NAME, SAMPLING_PRIORITY } = require('../../../ext/tags')
-const { SAMPLING_RULE_DECISION } = require('../../dd-trace/src/constants')
-const { AUTO_KEEP } = require('../../../ext/priority')
 
 const skippedTests = new WeakSet()
 
@@ -30,16 +28,11 @@ function getTestSpanMetadata (tracer, test, sourceRoot) {
   const fullTestName = test.fullTitle()
   const testSuite = getTestSuitePath(testSuiteAbsolutePath, sourceRoot)
 
+  const commonTags = getTestCommonTags(fullTestName, testSuite, tracer._version)
+
   return {
     childOf,
-    [SPAN_TYPE]: 'test',
-    [TEST_TYPE]: 'test',
-    [TEST_NAME]: fullTestName,
-    [TEST_SUITE]: testSuite,
-    [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP,
-    [TEST_FRAMEWORK_VERSION]: tracer._version,
-    [RESOURCE_NAME]: `${testSuite}.${fullTestName}`
+    ...commonTags
   }
 }
 
@@ -54,6 +47,7 @@ class MochaPlugin extends Plugin {
     this._testNameToParams = {}
     this.testEnvironmentMetadata = getTestEnvironmentMetadata('mocha', this.config)
     this.sourceRoot = process.cwd()
+    this.codeOwnersEntries = getCodeOwnersFileEntries(this.sourceRoot)
 
     this.addSub('ci:mocha:test:start', (test) => {
       const store = storage.getStore()
@@ -139,6 +133,11 @@ class MochaPlugin extends Plugin {
     if (testParametersString) {
       testSpanMetadata[TEST_PARAMETERS] = testParametersString
     }
+    const codeOwners = getCodeOwnersForFilename(testSpanMetadata[TEST_SUITE], this.codeOwnersEntries)
+
+    if (codeOwners) {
+      testSpanMetadata[TEST_CODE_OWNERS] = codeOwners
+    }
 
     const testSpan = this.tracer
       .startSpan('mocha.test', {

package/packages/dd-trace/lib/version.js

@@ -1 +1 @@
-module.exports = '2.4.2'
+module.exports = '2.6.0'

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -7,16 +7,14 @@ const Reporter = require('../reporter')
 
 const validAddressSet = new Set(Object.values(addresses))
 
-const DEFAULT_MAX_BUDGET = 5e3 // µs
-
 // TODO: put reusable code in a base class
 class WAFCallback {
-  static loadDDWAF (rules) {
+  static loadDDWAF (rules, config) {
     try {
       // require in `try/catch` because this can throw at require time
       const { DDWAF } = require('@datadog/native-appsec')
 
-      return new DDWAF(rules)
+      return new DDWAF(rules, config)
     } catch (err) {
       log.error('AppSec could not load native package. In-app WAF features will not be available.')
 
@@ -24,8 +22,24 @@ class WAFCallback {
     }
   }
 
-  constructor (rules) {
-    this.ddwaf = WAFCallback.loadDDWAF(rules)
+  constructor (rules, config) {
+    const { wafTimeout, obfuscatorKeyRegex, obfuscatorValueRegex } = config
+
+    this.ddwaf = WAFCallback.loadDDWAF(rules, { obfuscatorKeyRegex, obfuscatorValueRegex })
+
+    this.wafTimeout = wafTimeout
+
+    const version = this.ddwaf.constructor.version()
+
+    Reporter.metricsQueue.set('_dd.appsec.waf.version', `${version.major}.${version.minor}.${version.patch}`)
+
+    const { loaded, failed } = this.ddwaf.rulesInfo
+
+    Reporter.metricsQueue.set('_dd.appsec.event_rules.loaded', loaded)
+    Reporter.metricsQueue.set('_dd.appsec.event_rules.error_count', failed)
+
+    Reporter.metricsQueue.set('manual.keep', true)
+
     this.wafContextCache = new WeakMap()
 
     // closures are faster than binds
@@ -70,7 +84,7 @@ class WAFCallback {
       }
     }
 
-    if (!wafContext) {
+    if (!wafContext || wafContext.disposed) {
       wafContext = this.ddwaf.createContext()
     }
 
@@ -81,23 +95,26 @@ class WAFCallback {
 
     try {
       // TODO: possible optimizaion: only send params that haven't already been sent to this wafContext
-      const result = wafContext.run(params, DEFAULT_MAX_BUDGET)
+      const result = wafContext.run(params, this.wafTimeout)
 
       return this.applyResult(result, store)
     } catch (err) {
       log.error('Error while running the AppSec WAF')
       log.error(err)
+    } finally {
+      wafContext.dispose()
     }
   }
 
   applyResult (result, store) {
+    Reporter.reportMetrics({
+      duration: result.totalRuntime,
+      rulesVersion: this.ddwaf.rulesInfo.version
+    }, store)
+
     if (result.data && result.data !== '[]') {
       Reporter.reportAttack(result.data, store)
     }
-
-    // TODO: use these values later for budget management
-    // result.perfData
-    // result.perfTotalRuntime
   }
 
   clear () {

package/packages/dd-trace/src/appsec/index.js

@@ -16,7 +16,7 @@ function enable (config) {
     let rules = fs.readFileSync(config.appsec.rules)
     rules = JSON.parse(rules)
 
-    RuleManager.applyRules(rules)
+    RuleManager.applyRules(rules, config.appsec)
   } catch (err) {
     log.error('Unable to start AppSec')
     log.error(err)
@@ -94,12 +94,16 @@ function incomingHttpEndTranslator (data) {
   }
 
   if (data.req.cookies && typeof data.req.cookies === 'object') {
-    payload[addresses.HTTP_INCOMING_COOKIES] = data.req.cookies
+    payload[addresses.HTTP_INCOMING_COOKIES] = {}
+
+    for (const k of Object.keys(data.req.cookies)) {
+      payload[addresses.HTTP_INCOMING_COOKIES][k] = [ data.req.cookies[k] ]
+    }
   }
 
   Gateway.propagate(payload, context)
 
-  Reporter.finishAttacks(data.req, context)
+  Reporter.finishRequest(data.req, context)
 }
 
 function disable () {

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.3.0"
+    "rules_version": "1.3.1"
   },
   "rules": [
     {
@@ -3040,7 +3040,9 @@
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "keys_only"
+      ]
     },
     {
       "id": "crs-942-360",
@@ -4097,15 +4099,23 @@
           "parameters": {
             "inputs": [
               {
-                "address": "server.request.headers.no_cookies"
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
               }
             ],
-            "regex": "\\$(eq|ne|lte?|gte?|n?in)\\b"
+            "regex": "^\\$(eq|ne|(l|g)te?|n?in|not|(n|x|)or|and|regex|where|expr|exists)$"
           },
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "keys_only"
+      ]
     },
     {
       "id": "sqr-000-008",

package/packages/dd-trace/src/appsec/reporter.js

@@ -35,6 +35,8 @@ const RESPONSE_HEADERS_PASSLIST = [
   'content-type'
 ]
 
+const metricsQueue = new Map()
+
 function resolveHTTPRequest (context) {
   if (!context) return {}
 
@@ -82,6 +84,20 @@ function formatHeaderName (name) {
     .toLowerCase()
 }
 
+function reportMetrics (metrics, store) {
+  const req = store && store.get('req')
+  const topSpan = web.root(req)
+  if (!topSpan) return false
+
+  if (metrics.duration) {
+    topSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
+  }
+
+  if (metrics.rulesVersion) {
+    topSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
+  }
+}
+
 function reportAttack (attackData, store) {
   const req = store && store.get('req')
   const topSpan = web.root(req)
@@ -129,9 +145,17 @@ function reportAttack (attackData, store) {
   topSpan.addTags(newTags)
 }
 
-function finishAttacks (req, context) {
+function finishRequest (req, context) {
   const topSpan = web.root(req)
-  if (!topSpan || !context) return false
+  if (!topSpan) return false
+
+  if (metricsQueue.size) {
+    topSpan.addTags(Object.fromEntries(metricsQueue))
+
+    metricsQueue.clear()
+  }
+
+  if (!context || !topSpan.context()._tags['appsec.event']) return false
 
   const resolvedResponse = resolveHTTPResponse(context)
 
@@ -149,11 +173,13 @@ function setRateLimit (rateLimit) {
 }
 
 module.exports = {
+  metricsQueue,
   resolveHTTPRequest,
   resolveHTTPResponse,
   filterHeaders,
   formatHeaderName,
+  reportMetrics,
   reportAttack,
-  finishAttacks,
+  finishRequest,
   setRateLimit
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,11 +4,11 @@ const callbacks = require('./callbacks')
 
 const appliedCallbacks = new Map()
 
-function applyRules (rules) {
+function applyRules (rules, config) {
   if (appliedCallbacks.has(rules)) return
 
   // for now there is only WAF
-  const callback = new callbacks.DDWAF(rules)
+  const callback = new callbacks.DDWAF(rules, config)
 
   appliedCallbacks.set(rules, callback)
 }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const URL = require('url').URL
+const Writer = require('./writer')
+const Scheduler = require('../../../exporters/scheduler')
+
+class AgentlessCiVisibilityExporter {
+  constructor (config) {
+    const { flushInterval, tags, site, url } = config
+    this._url = url || new URL(`https://citestcycle-intake.${site}`)
+    this._writer = new Writer({ url: this._url, tags })
+
+    if (flushInterval > 0) {
+      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+    }
+    this._scheduler && this._scheduler.start()
+  }
+
+  export (trace) {
+    this._writer.append(trace)
+
+    if (!this._scheduler) {
+      this._writer.flush()
+    }
+  }
+
+  flush () {
+    this._writer.flush()
+  }
+}
+
+module.exports = AgentlessCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -0,0 +1,51 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+class Writer extends BaseWriter {
+  constructor ({ url, tags }) {
+    super(...arguments)
+    const { 'runtime-id': runtimeId, env, service } = tags
+    this._url = url
+    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+  }
+
+  _sendPayload (data, _, done) {
+    makeRequest(data, this._url, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+function makeRequest (data, url, cb) {
+  const options = {
+    path: '/api/v2/citestcycle',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/msgpack',
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000
+  }
+
+  options.protocol = url.protocol
+  options.hostname = url.hostname
+  options.port = url.port
+
+  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
+
+  request(data, options, false, (err, res) => {
+    cb(err, res)
+  })
+}
+
+module.exports = Writer

package/packages/dd-trace/src/config.js

@@ -66,6 +66,7 @@ class Config {
       process.env.DD_TRACE_URL,
       null
     )
+    const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -90,6 +91,10 @@ class Config {
       process.env.DD_TRACE_STARTUP_LOGS,
       false
     )
+    const DD_TRACE_TELEMETRY_ENABLED = coalesce(
+      process.env.DD_TRACE_TELEMETRY_ENABLED,
+      true
+    )
     const DD_TRACE_DEBUG = coalesce(
       process.env.DD_TRACE_DEBUG,
       false
@@ -145,10 +150,29 @@ class Config {
       path.join(__dirname, 'appsec', 'recommended.json')
     )
     const DD_APPSEC_TRACE_RATE_LIMIT = coalesce(
-      appsec.rateLimit,
-      process.env.DD_APPSEC_TRACE_RATE_LIMIT,
+      parseInt(appsec.rateLimit),
+      parseInt(process.env.DD_APPSEC_TRACE_RATE_LIMIT),
       100
     )
+    const DD_APPSEC_WAF_TIMEOUT = coalesce(
+      parseInt(appsec.wafTimeout),
+      parseInt(process.env.DD_APPSEC_WAF_TIMEOUT),
+      5e3 // µs
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = coalesce(
+      appsec.obfuscatorKeyRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|se\
+cret)|sign(?:ed|ature)|bearer|authorization`
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = coalesce(
+      appsec.obfuscatorValueRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\
+ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\
+\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\
+|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
+    )
 
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
@@ -171,7 +195,8 @@ class Config {
     this.debug = isTrue(DD_TRACE_DEBUG)
     this.logInjection = isTrue(DD_LOGS_INJECTION)
     this.env = DD_ENV
-    this.url = getAgentUrl(DD_TRACE_AGENT_URL, options)
+    this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL)
+      : getAgentUrl(DD_TRACE_AGENT_URL, options)
     this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com')
     this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname)
     this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port))
@@ -212,11 +237,15 @@ class Config {
     }
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
+    this.telemetryEnabled = isTrue(DD_TRACE_TELEMETRY_ENABLED)
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
       rules: DD_APPSEC_RULES,
-      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT
+      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
+      wafTimeout: DD_APPSEC_WAF_TIMEOUT,
+      obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
 
     tagger.add(this.tags, {

package/packages/dd-trace/src/encode/0.4.js

@@ -196,7 +196,6 @@ class AgentEncoder {
 
     for (const key of keys) {
       if (typeof value[key] !== 'string' && typeof value[key] !== 'number') return
-
       length++
 
       this._encodeString(bytes, key)