dd-trace 1.6.0-beta.0 → 2.0.0-appsec-beta.2

package/LICENSE-3rdparty.csv

@@ -1,9 +1,11 @@
 Component,Origin,License,Copyright
+require,@datadog/native-appsec,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/native-metrics,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/pprof,Apache license 2.0,Copyright 2019 Google Inc.
 require,@datadog/sketches-js,Apache license 2.0,Copyright 2020 Datadog Inc.
 require,@types/node,MIT,Copyright Authors
 require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
+require,diagnostics_channel,MIT,Copyright 2021 Simon D.
 require,form-data,MIT,Copyright 2012 Felix Geisendörfer and contributors
 require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward

package/index.d.ts

@@ -285,12 +285,6 @@ export declare interface TracerOptions {
    */
   runtimeMetrics?: boolean
 
-  /**
-   * Whether to track the scope of async functions. This is needed for async/await to work with non-native promises (thenables). Only disable this if you are sure only native promises are used with async/await, or if you are using Node >=14.5 since the issue has been fixed in that version.
-   * @default true
-   */
-  trackAsyncScope?: boolean
-
   /**
    * Custom function for DNS lookups when sending requests to the agent.
    * @default dns.lookup()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "1.6.0-beta.0",
+  "version": "2.0.0-appsec-beta.2",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -16,8 +16,10 @@
     "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services",
     "tdd": "node scripts/tdd.js",
     "test": "SERVICES=* yarn services && mocha --exit --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'",
-    "test:core": "mocha --exit --expose-gc --file packages/dd-trace/test/setup/core.js \"packages/dd-trace/test/**/*.spec.js\"",
-    "test:core:ci": "nyc --include \"packages/dd-trace/src/**/*.js\" -- npm run test:core --  --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
+    "test:trace:core": "mocha --exit --expose-gc --file packages/dd-trace/test/setup/core.js \"packages/dd-trace/test/**/*.spec.js\"",
+    "test:trace:core:ci": "nyc --include \"packages/dd-trace/src/**/*.js\" -- npm run test:trace:core --  --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
+    "test:core": "mocha --file packages/datadog-core/test/setup.js 'packages/datadog-core/test/**/*.spec.js'",
+    "test:core:ci": "nyc --include 'packages/datadog-core/src/**/*.js' -- npm run test:core -- --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
     "test:plugins": "mocha --exit --file \"packages/dd-trace/test/setup/core.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"",
     "test:plugins:ci": "yarn services && nyc --include \"packages/datadog-plugin-@($(echo $PLUGINS))/src/**/*.js\" -- npm run test:plugins -- --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
     "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js",
@@ -56,11 +58,13 @@
     "node": ">=12"
   },
   "dependencies": {
+    "@datadog/native-appsec": "^0.6.1",
     "@datadog/native-metrics": "^1.0.1",
     "@datadog/pprof": "^0.3.0",
     "@datadog/sketches-js": "^1.0.4",
     "@types/node": "^10.12.18",
     "crypto-randomuuid": "^1.0.0",
+    "diagnostics_channel": "^1.1.0",
     "form-data": "^3.0.0",
     "import-in-the-middle": "^1.1.2",
     "koalas": "^1.0.2",

package/packages/datadog-core/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const LocalStorage = require('./src/storage')
+
+const storage = new LocalStorage()
+
+module.exports = { storage }

package/packages/datadog-core/src/storage/async_hooks.js

@@ -0,0 +1,49 @@
+'use strict'
+
+const { createHook, executionAsyncId } = require('async_hooks')
+const AsyncResourceStorage = require('./async_resource')
+
+class AsyncHooksStorage extends AsyncResourceStorage {
+  constructor () {
+    super()
+
+    this._resources = new Map()
+  }
+
+  disable () {
+    super.disable()
+
+    this._resources.clear()
+  }
+
+  _createHook () {
+    return createHook({
+      init: this._init.bind(this),
+      destroy: this._destroy.bind(this)
+    })
+  }
+
+  _init (asyncId, type, triggerAsyncId, resource) {
+    super._init.apply(this, arguments)
+
+    this._resources.set(asyncId, resource)
+  }
+
+  _destroy (asyncId) {
+    this._resources.delete(asyncId)
+  }
+
+  _executionAsyncResource () {
+    const asyncId = executionAsyncId()
+
+    let resource = this._resources.get(asyncId)
+
+    if (!resource) {
+      this._resources.set(asyncId, resource = {})
+    }
+
+    return resource
+  }
+}
+
+module.exports = AsyncHooksStorage

package/packages/datadog-core/src/storage/async_resource.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { createHook, executionAsyncResource } = require('async_hooks')
+
+class AsyncResourceStorage {
+  constructor () {
+    this._ddResourceStore = Symbol('ddResourceStore')
+    this._enabled = false
+    this._hook = this._createHook()
+  }
+
+  disable () {
+    if (!this._enabled) return
+
+    this._hook.disable()
+    this._enabled = false
+  }
+
+  getStore () {
+    if (!this._enabled) return
+
+    const resource = this._executionAsyncResource()
+
+    return resource[this._ddResourceStore]
+  }
+
+  enterWith (store) {
+    this._enable()
+
+    const resource = this._executionAsyncResource()
+
+    resource[this._ddResourceStore] = store
+  }
+
+  run (store, callback, ...args) {
+    this._enable()
+
+    const resource = this._executionAsyncResource()
+    const oldStore = resource[this._ddResourceStore]
+
+    resource[this._ddResourceStore] = store
+
+    try {
+      return callback(...args)
+    } finally {
+      resource[this._ddResourceStore] = oldStore
+    }
+  }
+
+  _createHook () {
+    return createHook({
+      init: this._init.bind(this)
+    })
+  }
+
+  _enable () {
+    if (this._enabled) return
+
+    this._enabled = true
+    this._hook.enable()
+  }
+
+  _init (asyncId, type, triggerAsyncId, resource) {
+    const currentResource = this._executionAsyncResource()
+
+    if (Object.prototype.hasOwnProperty.call(currentResource, this._ddResourceStore)) {
+      resource[this._ddResourceStore] = currentResource[this._ddResourceStore]
+    }
+  }
+
+  _executionAsyncResource () {
+    return executionAsyncResource()
+  }
+}
+
+module.exports = AsyncResourceStorage

package/packages/datadog-core/src/storage/index.js

@@ -0,0 +1,14 @@
+'use strict'
+
+// TODO: default to AsyncLocalStorage when it supports triggerAsyncResource
+
+const semver = require('semver')
+
+// https://github.com/nodejs/node/pull/33801
+const hasJavaScriptAsyncHooks = semver.satisfies(process.versions.node, '>=14.5 || ^12.19.0')
+
+if (hasJavaScriptAsyncHooks) {
+  module.exports = require('./async_resource')
+} else {
+  module.exports = require('./async_hooks')
+}

package/packages/datadog-plugin-cucumber/src/index.js

@@ -5,6 +5,7 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
   TEST_SKIP_REASON,
   CI_APP_ORIGIN,
   ERROR_MESSAGE,
@@ -52,6 +53,7 @@ function createWrapRun (tracer, testEnvironmentMetadata, sourceRoot, setStatus)
         [TEST_NAME]: testName,
         [TEST_SUITE]: testSuite,
         [SAMPLING_RULE_DECISION]: 1,
+        [TEST_FRAMEWORK_VERSION]: tracer._version,
         ...testEnvironmentMetadata
       }
 

package/packages/datadog-plugin-cypress/src/plugin.js

@@ -3,6 +3,7 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
   getTestEnvironmentMetadata,
   CI_APP_ORIGIN,
   getTestParentSpan
@@ -19,7 +20,7 @@ const CYPRESS_STATUS_TO_TEST_STATUS = {
   skipped: 'skip'
 }
 
-function getTestSpanMetadata (tracer, testName, testSuite) {
+function getTestSpanMetadata (tracer, testName, testSuite, cypressConfig) {
   const childOf = getTestParentSpan(tracer)
 
   return {
@@ -29,7 +30,8 @@ function getTestSpanMetadata (tracer, testName, testSuite) {
     [TEST_NAME]: testName,
     [TEST_SUITE]: testSuite,
     [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_FRAMEWORK_VERSION]: cypressConfig.version
   }
 }
 
@@ -50,7 +52,7 @@ module.exports = (on, config) => {
         childOf,
         resource,
         ...testSpanMetadata
-      } = getTestSpanMetadata(tracer, testName, testSuite)
+      } = getTestSpanMetadata(tracer, testName, testSuite, config)
 
       if (!activeSpan) {
         activeSpan = tracer.startSpan('cypress.test', {

package/packages/datadog-plugin-http/src/server.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const web = require('../../dd-trace/src/plugins/util/web')
-const Scope = require('../../dd-trace/src/scope/base')
+const Scope = require('../../dd-trace/src/scope')
 
 function createWrapEmit (tracer, config) {
   config = web.normalizeConfig(config)

package/packages/datadog-plugin-jest/src/jest-environment.js

@@ -5,6 +5,8 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
+  JEST_TEST_RUNNER,
   ERROR_MESSAGE,
   ERROR_TYPE,
   TEST_PARAMETERS,
@@ -138,7 +140,9 @@ function createHandleTestEvent (tracer, testEnvironmentMetadata, instrumenter) {
     const spanTags = {
       ...commonSpanTags,
       [TEST_NAME]: testName,
-      [TEST_SUITE]: this.testSuite
+      [TEST_SUITE]: this.testSuite,
+      [TEST_FRAMEWORK_VERSION]: tracer._version,
+      [JEST_TEST_RUNNER]: 'jest-circus'
     }
 
     const testParametersString = getTestParametersString(nameToParams, event.test.name)

package/packages/datadog-plugin-jest/src/jest-jasmine2.js

@@ -5,6 +5,8 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
+  JEST_TEST_RUNNER,
   CI_APP_ORIGIN,
   getTestEnvironmentMetadata,
   finishAllTraceSpans,
@@ -29,7 +31,12 @@ function createWrapIt (tracer, globalConfig, globalInput, testEnvironmentMetadat
         {
           type: 'test',
           childOf,
-          tags: { ...commonSpanTags, [TEST_SUITE]: testSuite }
+          tags: {
+            ...commonSpanTags,
+            [TEST_SUITE]: testSuite,
+            [TEST_FRAMEWORK_VERSION]: tracer._version,
+            [JEST_TEST_RUNNER]: 'jest-jasmine2'
+          }
         },
         async (done) => {
           const testSpan = tracer.scope().active()
@@ -124,7 +131,9 @@ function createWrapItSkip (tracer, globalConfig, globalInput, testEnvironmentMet
             [RESOURCE_NAME]: resource,
             [TEST_NAME]: testName,
             [TEST_SUITE]: testSuite,
-            [TEST_STATUS]: 'skip'
+            [TEST_STATUS]: 'skip',
+            [TEST_FRAMEWORK_VERSION]: tracer._version,
+            [JEST_TEST_RUNNER]: 'jest-jasmine2'
           }
         }
       )

package/packages/datadog-plugin-mocha/src/index.js

@@ -9,6 +9,7 @@ const {
   TEST_SUITE,
   TEST_STATUS,
   TEST_PARAMETERS,
+  TEST_FRAMEWORK_VERSION,
   CI_APP_ORIGIN,
   getTestEnvironmentMetadata,
   getTestParametersString,
@@ -31,7 +32,8 @@ function getTestSpanMetadata (tracer, test, sourceRoot) {
     [TEST_NAME]: fullTestName,
     [TEST_SUITE]: testSuite,
     [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_FRAMEWORK_VERSION]: tracer._version
   }
 }
 

package/packages/dd-trace/lib/version.js

@@ -1 +1 @@
-module.exports = '1.6.0-beta.0'
+module.exports = '2.0.0-pre'

package/packages/dd-trace/src/appsec/addresses.js

@@ -0,0 +1,11 @@
+'use strict'
+
+module.exports = {
+  HTTP_INCOMING_URL: 'server.request.uri.raw',
+  HTTP_INCOMING_HEADERS: 'server.request.headers.no_cookies',
+  HTTP_INCOMING_METHOD: 'server.request.method',
+  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
+  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port',
+  HTTP_INCOMING_RESPONSE_CODE: 'server.response.status',
+  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies'
+}

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -0,0 +1,115 @@
+'use strict'
+
+const log = require('../../log')
+const addresses = require('../addresses')
+const Gateway = require('../../gateway/engine')
+const Reporter = require('../reporter')
+
+let warned = false
+
+const validAddressSet = new Set(Object.values(addresses))
+
+const DEFAULT_MAX_BUDGET = 5e3 // µs
+
+// TODO: put reusable code in a base class
+class WAFCallback {
+  static loadDDWAF (rules) {
+    try {
+      // require in `try/catch` because this can throw at require time
+      const { DDWAF } = require('@datadog/native-appsec')
+
+      return new DDWAF(rules)
+    } catch (err) {
+      if (!warned) {
+        log.warn('AppSec could not load native package. In-app WAF features will not be available.')
+        warned = true
+      }
+
+      throw err
+    }
+  }
+
+  constructor (rules) {
+    this.ddwaf = WAFCallback.loadDDWAF(rules)
+    this.wafContextCache = new WeakMap()
+
+    // closures are faster than binds
+    const self = this
+    const method = (params, store) => {
+      return self.action(params, store)
+    }
+
+    // might be its own class with more info later
+    const callback = { method }
+
+    const subscribedAddresses = new Set()
+
+    for (const rule of rules.rules) {
+      for (const condition of rule.conditions) {
+        for (const input of condition.parameters.inputs) {
+          const address = input.address.split(':', 2)[0]
+
+          if (!validAddressSet.has(address) || subscribedAddresses.has(address)) continue
+
+          subscribedAddresses.add(address)
+
+          Gateway.manager.addSubscription({ addresses: [ address ], callback })
+        }
+      }
+    }
+  }
+
+  action (params, store) {
+    let wafContext
+
+    if (store) {
+      const key = store.get('context')
+
+      if (key) {
+        if (this.wafContextCache.has(key)) {
+          wafContext = this.wafContextCache.get(key)
+        } else {
+          wafContext = this.ddwaf.createContext()
+          this.wafContextCache.set(key, wafContext)
+        }
+      }
+    }
+
+    if (!wafContext) {
+      wafContext = this.ddwaf.createContext()
+    }
+
+    // cast status code to string
+    if (params[addresses.HTTP_INCOMING_RESPONSE_CODE]) {
+      params[addresses.HTTP_INCOMING_RESPONSE_CODE] = params[addresses.HTTP_INCOMING_RESPONSE_CODE] + ''
+    }
+
+    try {
+      // TODO: possible optimizaion: only send params that haven't already been sent to this wafContext
+      const result = wafContext.run(params, DEFAULT_MAX_BUDGET)
+
+      return this.applyResult(result, store)
+    } catch (err) {
+      log.warn('Error while running the AppSec WAF')
+    }
+  }
+
+  applyResult (result, store) {
+    if (result.data && result.data !== '[]') {
+      Reporter.reportAttack(result.data, store)
+    }
+
+    // result.perfData
+    // result.perfTotalRuntime
+  }
+
+  clear () {
+    this.ddwaf.dispose()
+
+    this.wafContextCache = new WeakMap()
+
+    Gateway.manager.clear()
+  }
+}
+
+module.exports = WAFCallback

package/packages/dd-trace/src/appsec/callbacks/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+// lazy loading
+// TODO: cache the returned value
+module.exports = {
+  get DDWAF () { return require('./ddwaf') }
+}

package/packages/dd-trace/src/appsec/index.js

@@ -1,8 +1,97 @@
 'use strict'
 
-function enable () {
+const fs = require('fs')
+const log = require('../log')
+const RuleManager = require('./rule_manager')
+const { INCOMING_HTTP_REQUEST_START, INCOMING_HTTP_REQUEST_END } = require('../gateway/channels')
+const Gateway = require('../gateway/engine/index')
+const addresses = require('./addresses')
+const Reporter = require('./reporter')
+
+function enable (config) {
+  try {
+    // TODO: enable dc_blocking: config.appsec.blocking === true
+
+    let rules = fs.readFileSync(config.appsec.rules)
+    rules = JSON.parse(rules)
+
+    RuleManager.applyRules(rules)
+  } catch (err) {
+    log.error(`Unable to apply AppSec rules: ${err}`)
+
+    // abort AppSec start
+    RuleManager.clearAllRules()
+    return
+  }
+
+  INCOMING_HTTP_REQUEST_START.subscribe(incomingHttpStartTranslator)
+  INCOMING_HTTP_REQUEST_END.subscribe(incomingHttpEndTranslator)
+
+  config.tags['_dd.appsec.enabled'] = 1
+  config.tags['_dd.runtime_family'] = 'nodejs'
+
+  // add needed fields for HTTP context reporting
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_URL)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_HEADERS)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_METHOD)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_PORT)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_CODE)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
+}
+
+function incomingHttpStartTranslator (data) {
+  const store = Gateway.startContext()
+
+  store.set('req', data.req)
+  store.set('res', data.res)
+
+  const headers = Object.assign({}, data.req.headers)
+  delete headers.cookie
+
+  const context = store.get('context')
+
+  Gateway.propagate({
+    [addresses.HTTP_INCOMING_URL]: data.req.url,
+    [addresses.HTTP_INCOMING_HEADERS]: headers,
+    [addresses.HTTP_INCOMING_METHOD]: data.req.method,
+    [addresses.HTTP_INCOMING_REMOTE_IP]: data.req.socket.remoteAddress,
+    [addresses.HTTP_INCOMING_REMOTE_PORT]: data.req.socket.remotePort
+  }, context)
+}
+
+function incomingHttpEndTranslator (data) {
+  const context = Gateway.getContext()
+
+  if (!context) return
+
+  // TODO: this doesn't support headers sent with res.writeHead()
+  const headers = Object.assign({}, data.res.getHeaders())
+  delete headers['set-cookie']
+
+  Gateway.propagate({
+    [addresses.HTTP_INCOMING_RESPONSE_CODE]: data.res.statusCode,
+    [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: headers
+  }, context)
+
+  Reporter.finishAttacks(data.req, context)
+}
+
+function disable () {
+  RuleManager.clearAllRules()
+
+  if (INCOMING_HTTP_REQUEST_START.hasSubscribers) INCOMING_HTTP_REQUEST_START.unsubscribe(incomingHttpStartTranslator)
+  if (INCOMING_HTTP_REQUEST_END.hasSubscribers) INCOMING_HTTP_REQUEST_END.unsubscribe(incomingHttpEndTranslator)
+
+  const tags = global._ddtrace._tracer._tags
+
+  delete tags['_dd.appsec.enabled']
+  delete tags['_dd.runtime_family']
 }
 
 module.exports = {
-  enable
+  enable,
+  disable,
+  incomingHttpStartTranslator,
+  incomingHttpEndTranslator
 }