dd-trace 2.31.0 → 2.33.0

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const log = require('../../log')
+const Reporter = require('../reporter')
+
+class WAFContextWrapper {
+  constructor (ddwafContext, requiredAddresses, wafTimeout, rulesInfo) {
+    this.ddwafContext = ddwafContext
+    this.requiredAddresses = requiredAddresses
+    this.wafTimeout = wafTimeout
+    this.rulesInfo = rulesInfo
+  }
+
+  run (params) {
+    const inputs = {}
+    let someInputAdded = false
+
+    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
+    for (const key of Object.keys(params)) {
+      if (this.requiredAddresses.has(key)) {
+        inputs[key] = params[key]
+        someInputAdded = true
+      }
+    }
+
+    if (!someInputAdded) return
+
+    try {
+      const start = process.hrtime.bigint()
+
+      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+
+      const end = process.hrtime.bigint()
+
+      Reporter.reportMetrics({
+        duration: result.totalRuntime / 1e3,
+        durationExt: parseInt(end - start) / 1e3,
+        rulesVersion: this.rulesInfo.version
+      })
+
+      if (result.data && result.data !== '[]') {
+        Reporter.reportAttack(result.data)
+      }
+
+      return result.actions
+    } catch (err) {
+      log.error('Error while running the AppSec WAF')
+      log.error(err)
+    }
+  }
+
+  dispose () {
+    this.ddwafContext.dispose()
+  }
+}
+
+module.exports = WAFContextWrapper

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const log = require('../../log')
+const Reporter = require('../reporter')
+const WAFContextWrapper = require('./waf_context_wrapper')
+
+const contexts = new WeakMap()
+
+class WAFManager {
+  constructor (rules, config) {
+    this.config = config
+    this.wafTimeout = config.wafTimeout
+    this.ddwaf = this._loadDDWAF(rules)
+    this._reportMetrics()
+  }
+
+  _loadDDWAF (rules) {
+    try {
+      // require in `try/catch` because this can throw at require time
+      const { DDWAF } = require('@datadog/native-appsec')
+
+      const { obfuscatorKeyRegex, obfuscatorValueRegex } = this.config
+      return new DDWAF(rules, { obfuscatorKeyRegex, obfuscatorValueRegex })
+    } catch (err) {
+      log.error('AppSec could not load native package. In-app WAF features will not be available.')
+
+      throw err
+    }
+  }
+
+  _reportMetrics () {
+    Reporter.metricsQueue.set('_dd.appsec.waf.version', this.ddwaf.constructor.version())
+
+    const { loaded, failed, errors } = this.ddwaf.rulesInfo
+
+    Reporter.metricsQueue.set('_dd.appsec.event_rules.loaded', loaded)
+    Reporter.metricsQueue.set('_dd.appsec.event_rules.error_count', failed)
+    if (failed) Reporter.metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(errors))
+
+    Reporter.metricsQueue.set('manual.keep', 'true')
+  }
+
+  getWAFContext (req) {
+    let wafContext = contexts.get(req)
+
+    if (!wafContext) {
+      wafContext = new WAFContextWrapper(
+        this.ddwaf.createContext(),
+        this.ddwaf.requiredAddresses,
+        this.wafTimeout,
+        this.ddwaf.rulesInfo
+      )
+      contexts.set(req, wafContext)
+    }
+
+    return wafContext
+  }
+
+  destroy () {
+    if (this.ddwaf) {
+      this.ddwaf.dispose()
+    }
+  }
+}
+
+module.exports = WAFManager

package/packages/dd-trace/src/encode/0.4.js

@@ -3,6 +3,8 @@
 const { truncateSpan, normalizeSpan } = require('./tags-processors')
 const Chunk = require('./chunk')
 const log = require('../log')
+const { isTrue } = require('../util')
+const coalesce = require('koalas')
 
 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB
 
@@ -24,6 +26,10 @@ class AgentEncoder {
     this._stringBytes = new Chunk()
     this._writer = writer
     this._reset()
+    this._debugEncoding = isTrue(coalesce(
+      process.env.DD_TRACE_ENCODING_DEBUG,
+      false
+    ))
   }
 
   count () {
@@ -40,11 +46,13 @@ class AgentEncoder {
 
     const end = bytes.length
 
-    log.debug(() => {
-      const hex = bytes.buffer.subarray(start, end).toString('hex').match(/../g).join(' ')
+    if (this._debugEncoding) {
+      log.debug(() => {
+        const hex = bytes.buffer.subarray(start, end).toString('hex').match(/../g).join(' ')
 
-      return `Adding encoded trace to buffer: ${hex}`
-    })
+        return `Adding encoded trace to buffer: ${hex}`
+      })
+    }
 
     // we can go over the soft limit since the agent has a 50MB hard limit
     if (this._traceBytes.length > this._limit || this._stringBytes.length > this._limit) {

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -129,12 +129,17 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   _encodeEventContent (bytes, content) {
     const keysLength = Object.keys(content).length
 
+    let totalKeysLength = keysLength
     if (content.meta.test_session_id) {
-      this._encodeMapPrefix(bytes, keysLength + 3)
-    } else {
-      this._encodeMapPrefix(bytes, keysLength)
+      totalKeysLength = totalKeysLength + 1
     }
-
+    if (content.meta.test_module_id) {
+      totalKeysLength = totalKeysLength + 1
+    }
+    if (content.meta.test_suite_id) {
+      totalKeysLength = totalKeysLength + 1
+    }
+    this._encodeMapPrefix(bytes, totalKeysLength)
     if (content.type) {
       this._encodeString(bytes, 'type')
       this._encodeString(bytes, content.type)
@@ -170,15 +175,20 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       this._encodeString(bytes, 'test_session_id')
       this._encodeId(bytes, id(content.meta.test_session_id, 10))
       delete content.meta.test_session_id
+    }
 
+    if (content.meta.test_module_id) {
       this._encodeString(bytes, 'test_module_id')
       this._encodeId(bytes, id(content.meta.test_module_id, 10))
       delete content.meta.test_module_id
+    }
 
+    if (content.meta.test_suite_id) {
       this._encodeString(bytes, 'test_suite_id')
       this._encodeId(bytes, id(content.meta.test_suite_id, 10))
       delete content.meta.test_suite_id
     }
+
     this._encodeString(bytes, 'meta')
     this._encodeMap(bytes, content.meta)
     this._encodeString(bytes, 'metrics')

package/packages/dd-trace/src/plugins/util/test.js

@@ -50,12 +50,10 @@ const CI_APP_ORIGIN = 'ciapp-test'
 const JEST_TEST_RUNNER = 'test.jest.test_runner'
 
 const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped'
-const TEST_SESSION_ITR_SKIPPING_ENABLED = 'test_session.itr.tests_skipping.enabled'
-const TEST_SESSION_CODE_COVERAGE_ENABLED = 'test_session.code_coverage.enabled'
-const TEST_MODULE_ITR_SKIPPING_ENABLED = 'test_module.itr.tests_skipping.enabled'
-const TEST_MODULE_CODE_COVERAGE_ENABLED = 'test_module.code_coverage.enabled'
+const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled'
+const TEST_CODE_COVERAGE_ENABLED = 'test.code_coverage.enabled'
 
-const TEST_CODE_COVERAGE_LINES_TOTAL = 'test.codecov_lines_total'
+const TEST_CODE_COVERAGE_LINES_PCT = 'test.code_coverage.lines_pct'
 
 // jest worker variables
 const JEST_WORKER_TRACE_PAYLOAD_CODE = 60
@@ -97,17 +95,16 @@ module.exports = {
   TEST_SUITE_ID,
   TEST_ITR_TESTS_SKIPPED,
   TEST_MODULE,
-  TEST_SESSION_ITR_SKIPPING_ENABLED,
-  TEST_SESSION_CODE_COVERAGE_ENABLED,
-  TEST_MODULE_ITR_SKIPPING_ENABLED,
-  TEST_MODULE_CODE_COVERAGE_ENABLED,
-  TEST_CODE_COVERAGE_LINES_TOTAL,
+  TEST_ITR_SKIPPING_ENABLED,
+  TEST_CODE_COVERAGE_ENABLED,
+  TEST_CODE_COVERAGE_LINES_PCT,
   addIntelligentTestRunnerSpanTags,
   getCoveredFilenamesFromCoverage,
   resetCoverage,
   mergeCoverage,
   fromCoverageMapToCoverage,
-  getTestLineStart
+  getTestLineStart,
+  getCallSites
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -316,17 +313,17 @@ function addIntelligentTestRunnerSpanTags (
   { isSuitesSkipped, isSuitesSkippingEnabled, isCodeCoverageEnabled, testCodeCoverageLinesTotal }
 ) {
   testSessionSpan.setTag(TEST_ITR_TESTS_SKIPPED, isSuitesSkipped ? 'true' : 'false')
-  testSessionSpan.setTag(TEST_SESSION_ITR_SKIPPING_ENABLED, isSuitesSkippingEnabled ? 'true' : 'false')
-  testSessionSpan.setTag(TEST_SESSION_CODE_COVERAGE_ENABLED, isCodeCoverageEnabled ? 'true' : 'false')
+  testSessionSpan.setTag(TEST_ITR_SKIPPING_ENABLED, isSuitesSkippingEnabled ? 'true' : 'false')
+  testSessionSpan.setTag(TEST_CODE_COVERAGE_ENABLED, isCodeCoverageEnabled ? 'true' : 'false')
 
   testModuleSpan.setTag(TEST_ITR_TESTS_SKIPPED, isSuitesSkipped ? 'true' : 'false')
-  testModuleSpan.setTag(TEST_MODULE_ITR_SKIPPING_ENABLED, isSuitesSkippingEnabled ? 'true' : 'false')
-  testModuleSpan.setTag(TEST_MODULE_CODE_COVERAGE_ENABLED, isCodeCoverageEnabled ? 'true' : 'false')
+  testModuleSpan.setTag(TEST_ITR_SKIPPING_ENABLED, isSuitesSkippingEnabled ? 'true' : 'false')
+  testModuleSpan.setTag(TEST_CODE_COVERAGE_ENABLED, isCodeCoverageEnabled ? 'true' : 'false')
 
   // If suites have been skipped we don't want to report the total coverage, as it will be wrong
   if (testCodeCoverageLinesTotal !== undefined && !isSuitesSkipped) {
-    testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_TOTAL, testCodeCoverageLinesTotal)
-    testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_TOTAL, testCodeCoverageLinesTotal)
+    testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
+    testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
   }
 }
 
@@ -399,3 +396,23 @@ function getTestLineStart (err, testSuitePath) {
     return null
   }
 }
+
+// From https://github.com/felixge/node-stack-trace/blob/ba06dcdb50d465cd440d84a563836e293b360427/index.js#L1
+function getCallSites () {
+  const oldLimit = Error.stackTraceLimit
+  Error.stackTraceLimit = Infinity
+
+  const dummy = {}
+
+  const v8Handler = Error.prepareStackTrace
+  Error.prepareStackTrace = function (_, v8StackTrace) {
+    return v8StackTrace
+  }
+  Error.captureStackTrace(dummy)
+
+  const v8StackTrace = dummy.stack
+  Error.prepareStackTrace = v8Handler
+  Error.stackTraceLimit = oldLimit
+
+  return v8StackTrace
+}

package/packages/dd-trace/src/telemetry/send-data.js

@@ -1,5 +1,17 @@
 const request = require('../exporters/common/request')
 let seqId = 0
+
+function getPayload (payload) {
+  // Some telemetry endpoints payloads accept collections of elements such as the 'logs' endpoint.
+  // 'logs' request type payload is meant to send library logs to Datadog’s backend.
+  if (Array.isArray(payload)) {
+    return payload
+  } else {
+    const { logger, tags, serviceMapping, ...trimmedPayload } = payload
+    return trimmedPayload
+  }
+}
+
 function sendData (config, application, host, reqType, payload = {}) {
   const {
     hostname,
@@ -7,8 +19,6 @@ function sendData (config, application, host, reqType, payload = {}) {
     url
   } = config
 
-  const { logger, tags, serviceMapping, ...trimmedPayload } = payload
-
   const options = {
     url,
     hostname,
@@ -27,7 +37,7 @@ function sendData (config, application, host, reqType, payload = {}) {
     tracer_time: Math.floor(Date.now() / 1000),
     runtime_id: config.tags['runtime-id'],
     seq_id: ++seqId,
-    payload: trimmedPayload,
+    payload: getPayload(payload),
     application,
     host
   })

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -1,137 +0,0 @@
-'use strict'
-
-const log = require('../../log')
-const addresses = require('../addresses')
-const Gateway = require('../gateway/engine')
-const Reporter = require('../reporter')
-
-const validAddressSet = new Set(Object.values(addresses))
-
-// TODO: put reusable code in a base class
-class WAFCallback {
-  static loadDDWAF (rules, config) {
-    try {
-      // require in `try/catch` because this can throw at require time
-      const { DDWAF } = require('@datadog/native-appsec')
-
-      return new DDWAF(rules, config)
-    } catch (err) {
-      log.error('AppSec could not load native package. In-app WAF features will not be available.')
-
-      throw err
-    }
-  }
-
-  constructor (rules, config) {
-    const { wafTimeout, obfuscatorKeyRegex, obfuscatorValueRegex } = config
-
-    this.ddwaf = WAFCallback.loadDDWAF(rules, { obfuscatorKeyRegex, obfuscatorValueRegex })
-
-    this.wafTimeout = wafTimeout
-
-    Reporter.metricsQueue.set('_dd.appsec.waf.version', this.ddwaf.constructor.version())
-
-    const { loaded, failed, errors } = this.ddwaf.rulesInfo
-
-    Reporter.metricsQueue.set('_dd.appsec.event_rules.loaded', loaded)
-    Reporter.metricsQueue.set('_dd.appsec.event_rules.error_count', failed)
-    if (failed) Reporter.metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(errors))
-
-    Reporter.metricsQueue.set('manual.keep', 'true')
-
-    this.wafContextCache = new WeakMap()
-
-    // closures are faster than binds
-    const self = this
-    const method = (params, store) => {
-      return self.action(params, store)
-    }
-
-    // might be its own class with more info later
-    const callback = { method }
-
-    const subscribedAddresses = new Set()
-
-    for (const rule of rules.rules) {
-      for (const condition of rule.conditions) {
-        for (const input of condition.parameters.inputs) {
-          const address = input.address.split(':', 2)[0]
-
-          if (!validAddressSet.has(address) || subscribedAddresses.has(address)) continue
-
-          subscribedAddresses.add(address)
-
-          Gateway.manager.addSubscription({ addresses: [address], callback })
-        }
-      }
-    }
-  }
-
-  action (params, store) {
-    let wafContext
-
-    if (store) {
-      const key = store.get('context')
-
-      if (key) {
-        if (this.wafContextCache.has(key)) {
-          wafContext = this.wafContextCache.get(key)
-        } else {
-          wafContext = this.ddwaf.createContext()
-          this.wafContextCache.set(key, wafContext)
-        }
-      }
-    }
-
-    if (!wafContext || wafContext.disposed) {
-      wafContext = this.ddwaf.createContext()
-    }
-
-    // cast status code to string
-    if (params[addresses.HTTP_INCOMING_RESPONSE_CODE]) {
-      params[addresses.HTTP_INCOMING_RESPONSE_CODE] = params[addresses.HTTP_INCOMING_RESPONSE_CODE] + ''
-    }
-
-    try {
-      // TODO: possible optimizaion: only send params that haven't already been sent to this wafContext
-      const start = process.hrtime.bigint()
-
-      const result = wafContext.run(params, this.wafTimeout)
-
-      result.durationExt = parseInt(process.hrtime.bigint() - start)
-
-      return this.applyResult(result, store)
-    } catch (err) {
-      log.error('Error while running the AppSec WAF')
-      log.error(err)
-    } finally {
-      wafContext.dispose()
-    }
-  }
-
-  applyResult (result, store) {
-    Reporter.reportMetrics({
-      duration: result.totalRuntime / 1e3,
-      durationExt: result.durationExt / 1e3,
-      rulesVersion: this.ddwaf.rulesInfo.version
-    }, store)
-
-    if (result.data && result.data !== '[]') {
-      Reporter.reportAttack(result.data, store)
-    }
-
-    return result.actions
-  }
-
-  updateRuleData (ruleData) {
-    this.ddwaf.updateRuleData(ruleData)
-  }
-
-  clear () {
-    this.ddwaf.dispose()
-
-    this.wafContextCache = new WeakMap()
-  }
-}
-
-module.exports = WAFCallback

package/packages/dd-trace/src/appsec/callbacks/index.js

@@ -1,7 +0,0 @@
-'use strict'
-
-// lazy loading
-// TODO: cache the returned value
-module.exports = {
-  get DDWAF () { return require('./ddwaf') }
-}

package/packages/dd-trace/src/appsec/gateway/als.js

@@ -1,6 +0,0 @@
-'use strict'
-
-// TODO: use datadog-core storage instead
-const { AsyncLocalStorage } = require('async_hooks')
-
-module.exports = new AsyncLocalStorage()

package/packages/dd-trace/src/appsec/gateway/engine/engine.js

@@ -1,140 +0,0 @@
-'use strict'
-
-const Runner = require('./runner')
-
-const MAX_CONTEXT_SIZE = 1024
-
-class SubscriptionManager {
-  constructor () {
-    this.addressToSubscriptions = new Map()
-    this.addresses = new Set()
-    this.subscriptions = new Set()
-  }
-
-  clear () {
-    this.addressToSubscriptions = new Map()
-    this.addresses = new Set()
-    this.subscriptions = new Set()
-  }
-
-  addSubscription (subscription) {
-    if (!subscription.addresses.length || this.subscriptions.has(subscription)) return
-
-    for (let i = 0; i < subscription.addresses.length; ++i) {
-      const address = subscription.addresses[i]
-
-      this.addresses.add(address)
-
-      const list = this.addressToSubscriptions.get(address)
-
-      if (list === undefined) {
-        this.addressToSubscriptions.set(address, [subscription])
-      } else {
-        list.push(subscription)
-      }
-    }
-
-    this.subscriptions.add(subscription)
-  }
-
-  matchSubscriptions (newAddresses, allAddresses) {
-    const addresses = new Set()
-    const subscriptions = new Set()
-    const knownSubscriptions = new Set()
-
-    // TODO: possible optimization: collect matchedSubscriptions on the fly in Context#setValue
-    newAddresses.forEach((newAddress) => {
-      const matchedSubscriptions = this.addressToSubscriptions.get(newAddress)
-
-      if (matchedSubscriptions === undefined) return
-
-      for (let j = 0; j < matchedSubscriptions.length; ++j) {
-        const subscription = matchedSubscriptions[j]
-
-        if (knownSubscriptions.has(subscription) === true) continue
-        knownSubscriptions.add(subscription)
-
-        const isFulfilled = subscription.addresses.every(allAddresses.has, allAddresses)
-
-        if (isFulfilled === true) {
-          for (let k = 0; k < subscription.addresses.length; ++k) {
-            addresses.add(subscription.addresses[k])
-          }
-
-          subscriptions.add(subscription)
-        }
-      }
-    })
-
-    return { addresses, subscriptions }
-  }
-
-  dispatch (newAddresses, allAddresses, context) {
-    const matches = this.matchSubscriptions(newAddresses, allAddresses)
-
-    // TODO: possible optimization
-    // check if matches.subscriptions is empty here instead of in runner.js
-
-    const params = {}
-
-    matches.addresses.forEach((address) => {
-      params[address] = context.resolve(address)
-    })
-
-    return Runner.runSubscriptions(matches.subscriptions, params)
-  }
-}
-
-class Context {
-  static setManager (manager) {
-    this.manager = manager
-  }
-
-  constructor () {
-    // TODO: this probably don't need to be a Map()
-    this.store = new Map()
-    this.allAddresses = new Set()
-    this.newAddresses = new Set()
-  }
-
-  clear () {
-    this.store = new Map()
-    this.allAddresses = new Set()
-    this.newAddresses = new Set()
-  }
-
-  setValue (address, value) {
-    if (this.allAddresses.size >= MAX_CONTEXT_SIZE) return this
-
-    // cannot optimize for objects because they're pointers
-    if (typeof value !== 'object') {
-      const oldValue = this.store.get(address)
-      if (oldValue === value) return this
-    }
-
-    this.store.set(address, value)
-    this.allAddresses.add(address)
-    this.newAddresses.add(address)
-
-    return this
-  }
-
-  dispatch () {
-    if (this.newAddresses.size === 0) return []
-
-    const result = Context.manager.dispatch(this.newAddresses, this.allAddresses, this)
-
-    this.newAddresses.clear()
-
-    return result
-  }
-
-  resolve (address) {
-    return this.store.get(address)
-  }
-}
-
-module.exports = {
-  SubscriptionManager,
-  Context
-}

package/packages/dd-trace/src/appsec/gateway/engine/index.js

@@ -1,51 +0,0 @@
-'use strict'
-
-const { SubscriptionManager, Context } = require('./engine')
-const als = require('../als')
-
-const manager = new SubscriptionManager()
-Context.setManager(manager)
-
-function startContext () {
-  const store = new Map()
-
-  store.set('context', new Context())
-
-  als.enterWith(store)
-
-  return store
-}
-
-function getContext () {
-  const store = als.getStore()
-
-  return store && store.get('context')
-}
-
-function needsAddress (address) {
-  return manager.addresses.has(address)
-}
-
-function propagate (data, context = getContext()) {
-  if (!context) return
-
-  const keys = Object.keys(data)
-
-  for (let i = 0; i < keys.length; ++i) {
-    const key = keys[i]
-
-    if (needsAddress(key)) {
-      context.setValue(key, data[key])
-    }
-  }
-
-  return context.dispatch()
-}
-
-module.exports = {
-  manager,
-  startContext,
-  getContext,
-  needsAddress,
-  propagate
-}