dd-trace 1.5.0 → 2.0.0-appsec-beta.1

package/LICENSE-3rdparty.csv

@@ -1,9 +1,11 @@
 Component,Origin,License,Copyright
+require,@datadog/native-appsec,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/native-metrics,Apache license 2.0,Copyright 2018 Datadog Inc.
 require,@datadog/pprof,Apache license 2.0,Copyright 2019 Google Inc.
 require,@datadog/sketches-js,Apache license 2.0,Copyright 2020 Datadog Inc.
 require,@types/node,MIT,Copyright Authors
 require,crypto-randomuuid,MIT,Copyright 2021 Node.js Foundation and contributors
+require,diagnostics_channel,MIT,Copyright 2021 Simon D.
 require,form-data,MIT,Copyright 2012 Felix Geisendörfer and contributors
 require,import-in-the-middle,Apache license 2.0,Copyright 2021 Datadog Inc.
 require,koalas,MIT,Copyright 2013-2017 Brian Woodward

package/index.d.ts

@@ -285,12 +285,6 @@ export declare interface TracerOptions {
    */
   runtimeMetrics?: boolean
 
-  /**
-   * Whether to track the scope of async functions. This is needed for async/await to work with non-native promises (thenables). Only disable this if you are sure only native promises are used with async/await, or if you are using Node >=14.5 since the issue has been fixed in that version.
-   * @default true
-   */
-  trackAsyncScope?: boolean
-
   /**
    * Custom function for DNS lookups when sending requests to the agent.
    * @default dns.lookup()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "1.5.0",
+  "version": "2.0.0-appsec-beta.1",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -16,8 +16,10 @@
     "services": "node ./scripts/install_plugin_modules && node packages/dd-trace/test/setup/services",
     "tdd": "node scripts/tdd.js",
     "test": "SERVICES=* yarn services && mocha --exit --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'",
-    "test:core": "mocha --exit --expose-gc --file packages/dd-trace/test/setup/core.js \"packages/dd-trace/test/**/*.spec.js\"",
-    "test:core:ci": "nyc --include \"packages/dd-trace/src/**/*.js\" -- npm run test:core --  --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
+    "test:trace:core": "mocha --exit --expose-gc --file packages/dd-trace/test/setup/core.js \"packages/dd-trace/test/**/*.spec.js\"",
+    "test:trace:core:ci": "nyc --include \"packages/dd-trace/src/**/*.js\" -- npm run test:trace:core --  --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
+    "test:core": "mocha --file packages/datadog-core/test/setup.js 'packages/datadog-core/test/**/*.spec.js'",
+    "test:core:ci": "nyc --include 'packages/datadog-core/src/**/*.js' -- npm run test:core -- --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
     "test:plugins": "mocha --exit --file \"packages/dd-trace/test/setup/core.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\"",
     "test:plugins:ci": "yarn services && nyc --include \"packages/datadog-plugin-@($(echo $PLUGINS))/src/**/*.js\" -- npm run test:plugins -- --reporter mocha-multi-reporters --reporter-options configFile=mocha-reporter-config.json",
     "test:plugins:upstream": "node ./packages/dd-trace/test/plugins/suite.js",
@@ -56,13 +58,15 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@datadog/native-metrics": "^1.0.0",
-    "@datadog/pprof": "^0.1.3",
+    "@datadog/native-appsec": "^0.6.0",
+    "@datadog/native-metrics": "^1.0.1",
+    "@datadog/pprof": "^0.3.0",
     "@datadog/sketches-js": "^1.0.4",
     "@types/node": "^10.12.18",
     "crypto-randomuuid": "^1.0.0",
+    "diagnostics_channel": "^1.1.0",
     "form-data": "^3.0.0",
-    "import-in-the-middle": "^1.1.1",
+    "import-in-the-middle": "^1.1.2",
     "koalas": "^1.0.2",
     "limiter": "^1.1.4",
     "lodash.kebabcase": "^4.1.1",

package/packages/datadog-core/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const LocalStorage = require('./src/storage')
+
+const storage = new LocalStorage()
+
+module.exports = { storage }

package/packages/datadog-core/src/storage/async_hooks.js

@@ -0,0 +1,49 @@
+'use strict'
+
+const { createHook, executionAsyncId } = require('async_hooks')
+const AsyncResourceStorage = require('./async_resource')
+
+class AsyncHooksStorage extends AsyncResourceStorage {
+  constructor () {
+    super()
+
+    this._resources = new Map()
+  }
+
+  disable () {
+    super.disable()
+
+    this._resources.clear()
+  }
+
+  _createHook () {
+    return createHook({
+      init: this._init.bind(this),
+      destroy: this._destroy.bind(this)
+    })
+  }
+
+  _init (asyncId, type, triggerAsyncId, resource) {
+    super._init.apply(this, arguments)
+
+    this._resources.set(asyncId, resource)
+  }
+
+  _destroy (asyncId) {
+    this._resources.delete(asyncId)
+  }
+
+  _executionAsyncResource () {
+    const asyncId = executionAsyncId()
+
+    let resource = this._resources.get(asyncId)
+
+    if (!resource) {
+      this._resources.set(asyncId, resource = {})
+    }
+
+    return resource
+  }
+}
+
+module.exports = AsyncHooksStorage

package/packages/datadog-core/src/storage/async_resource.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { createHook, executionAsyncResource } = require('async_hooks')
+
+class AsyncResourceStorage {
+  constructor () {
+    this._ddResourceStore = Symbol('ddResourceStore')
+    this._enabled = false
+    this._hook = this._createHook()
+  }
+
+  disable () {
+    if (!this._enabled) return
+
+    this._hook.disable()
+    this._enabled = false
+  }
+
+  getStore () {
+    if (!this._enabled) return
+
+    const resource = this._executionAsyncResource()
+
+    return resource[this._ddResourceStore]
+  }
+
+  enterWith (store) {
+    this._enable()
+
+    const resource = this._executionAsyncResource()
+
+    resource[this._ddResourceStore] = store
+  }
+
+  run (store, callback, ...args) {
+    this._enable()
+
+    const resource = this._executionAsyncResource()
+    const oldStore = resource[this._ddResourceStore]
+
+    resource[this._ddResourceStore] = store
+
+    try {
+      return callback(...args)
+    } finally {
+      resource[this._ddResourceStore] = oldStore
+    }
+  }
+
+  _createHook () {
+    return createHook({
+      init: this._init.bind(this)
+    })
+  }
+
+  _enable () {
+    if (this._enabled) return
+
+    this._enabled = true
+    this._hook.enable()
+  }
+
+  _init (asyncId, type, triggerAsyncId, resource) {
+    const currentResource = this._executionAsyncResource()
+
+    if (Object.prototype.hasOwnProperty.call(currentResource, this._ddResourceStore)) {
+      resource[this._ddResourceStore] = currentResource[this._ddResourceStore]
+    }
+  }
+
+  _executionAsyncResource () {
+    return executionAsyncResource()
+  }
+}
+
+module.exports = AsyncResourceStorage

package/packages/datadog-core/src/storage/index.js

@@ -0,0 +1,14 @@
+'use strict'
+
+// TODO: default to AsyncLocalStorage when it supports triggerAsyncResource
+
+const semver = require('semver')
+
+// https://github.com/nodejs/node/pull/33801
+const hasJavaScriptAsyncHooks = semver.satisfies(process.versions.node, '>=14.5 || ^12.19.0')
+
+if (hasJavaScriptAsyncHooks) {
+  module.exports = require('./async_resource')
+} else {
+  module.exports = require('./async_hooks')
+}

package/packages/datadog-plugin-cucumber/src/index.js

@@ -5,6 +5,7 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
   TEST_SKIP_REASON,
   CI_APP_ORIGIN,
   ERROR_MESSAGE,
@@ -52,6 +53,7 @@ function createWrapRun (tracer, testEnvironmentMetadata, sourceRoot, setStatus)
         [TEST_NAME]: testName,
         [TEST_SUITE]: testSuite,
         [SAMPLING_RULE_DECISION]: 1,
+        [TEST_FRAMEWORK_VERSION]: tracer._version,
         ...testEnvironmentMetadata
       }
 

package/packages/datadog-plugin-cypress/src/plugin.js

@@ -3,6 +3,7 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
   getTestEnvironmentMetadata,
   CI_APP_ORIGIN,
   getTestParentSpan
@@ -19,7 +20,7 @@ const CYPRESS_STATUS_TO_TEST_STATUS = {
   skipped: 'skip'
 }
 
-function getTestSpanMetadata (tracer, testName, testSuite) {
+function getTestSpanMetadata (tracer, testName, testSuite, cypressConfig) {
   const childOf = getTestParentSpan(tracer)
 
   return {
@@ -29,7 +30,8 @@ function getTestSpanMetadata (tracer, testName, testSuite) {
     [TEST_NAME]: testName,
     [TEST_SUITE]: testSuite,
     [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_FRAMEWORK_VERSION]: cypressConfig.version
   }
 }
 
@@ -50,7 +52,7 @@ module.exports = (on, config) => {
         childOf,
         resource,
         ...testSpanMetadata
-      } = getTestSpanMetadata(tracer, testName, testSuite)
+      } = getTestSpanMetadata(tracer, testName, testSuite, config)
 
       if (!activeSpan) {
         activeSpan = tracer.startSpan('cypress.test', {

package/packages/datadog-plugin-graphql/src/index.js

@@ -9,20 +9,17 @@ let tools
 function createWrapExecute (tracer, config, defaultFieldResolver) {
   return function wrapExecute (execute) {
     return function executeWithTrace () {
-      const args = normalizeArgs(arguments)
+      const args = normalizeArgs(arguments, tracer, config, defaultFieldResolver)
       const schema = args.schema
       const document = args.document
       const source = document && document._datadog_source
-      const fieldResolver = args.fieldResolver || defaultFieldResolver
-      const contextValue = args.contextValue = args.contextValue || {}
+      const contextValue = args.contextValue
       const operation = getOperation(document, args.operationName)
 
       if (contextValue._datadog_graphql) {
         return execute.apply(this, arguments)
       }
 
-      args.fieldResolver = wrapResolve(fieldResolver, tracer, config)
-
       if (schema) {
         wrapFields(schema._queryType, tracer, config)
         wrapFields(schema._mutationType, tracer, config)
@@ -32,7 +29,7 @@ function createWrapExecute (tracer, config, defaultFieldResolver) {
 
       contextValue._datadog_graphql = { source, span, fields: {} }
 
-      return call(execute, span, this, [args], (err, res) => {
+      return call(execute, span, this, arguments, (err, res) => {
         finishResolvers(contextValue, config)
 
         setError(span, err || (res && res.errors && res.errors[0]))
@@ -212,10 +209,19 @@ function getField (contextValue, path) {
   return contextValue._datadog_graphql.fields[path.join('.')]
 }
 
-function normalizeArgs (args) {
-  if (args.length === 1) {
-    return args[0]
-  }
+function normalizeArgs (args, tracer, config, defaultFieldResolver) {
+  if (args.length !== 1) return normalizePositional(args, tracer, config, defaultFieldResolver)
+
+  args[0].contextValue = args[0].contextValue || {}
+  args[0].fieldResolver = wrapResolve(args[0].fieldResolver || defaultFieldResolver, tracer, config)
+
+  return args[0]
+}
+
+function normalizePositional (args, tracer, config, defaultFieldResolver) {
+  args[3] = args[3] || {} // contextValue
+  args[6] = wrapResolve(args[6] || defaultFieldResolver, tracer, config) // fieldResolver
+  args.length = Math.max(args.length, 7)
 
   return {
     schema: args[0],

package/packages/datadog-plugin-grpc/src/client.js

@@ -48,7 +48,7 @@ function createWrapMakeClientConstructor (tracer, config) {
 function wrapPackageDefinition (tracer, config, def) {
   for (const name in def) {
     if (def[name].format) continue
-    if (def[name].service) {
+    if (def[name].service && def[name].prototype) {
       wrapClientConstructor(tracer, config, def[name], def[name].service)
     } else {
       wrapPackageDefinition(tracer, config, def[name])

package/packages/datadog-plugin-http/src/server.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const web = require('../../dd-trace/src/plugins/util/web')
-const Scope = require('../../dd-trace/src/scope/base')
+const Scope = require('../../dd-trace/src/scope')
 
 function createWrapEmit (tracer, config) {
   config = web.normalizeConfig(config)

package/packages/datadog-plugin-jest/src/jest-environment.js

@@ -5,6 +5,8 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
+  JEST_TEST_RUNNER,
   ERROR_MESSAGE,
   ERROR_TYPE,
   TEST_PARAMETERS,
@@ -138,7 +140,9 @@ function createHandleTestEvent (tracer, testEnvironmentMetadata, instrumenter) {
     const spanTags = {
       ...commonSpanTags,
       [TEST_NAME]: testName,
-      [TEST_SUITE]: this.testSuite
+      [TEST_SUITE]: this.testSuite,
+      [TEST_FRAMEWORK_VERSION]: tracer._version,
+      [JEST_TEST_RUNNER]: 'jest-circus'
     }
 
     const testParametersString = getTestParametersString(nameToParams, event.test.name)

package/packages/datadog-plugin-jest/src/jest-jasmine2.js

@@ -5,6 +5,8 @@ const {
   TEST_NAME,
   TEST_SUITE,
   TEST_STATUS,
+  TEST_FRAMEWORK_VERSION,
+  JEST_TEST_RUNNER,
   CI_APP_ORIGIN,
   getTestEnvironmentMetadata,
   finishAllTraceSpans,
@@ -29,7 +31,12 @@ function createWrapIt (tracer, globalConfig, globalInput, testEnvironmentMetadat
         {
           type: 'test',
           childOf,
-          tags: { ...commonSpanTags, [TEST_SUITE]: testSuite }
+          tags: {
+            ...commonSpanTags,
+            [TEST_SUITE]: testSuite,
+            [TEST_FRAMEWORK_VERSION]: tracer._version,
+            [JEST_TEST_RUNNER]: 'jest-jasmine2'
+          }
         },
         async (done) => {
           const testSpan = tracer.scope().active()
@@ -124,7 +131,9 @@ function createWrapItSkip (tracer, globalConfig, globalInput, testEnvironmentMet
             [RESOURCE_NAME]: resource,
             [TEST_NAME]: testName,
             [TEST_SUITE]: testSuite,
-            [TEST_STATUS]: 'skip'
+            [TEST_STATUS]: 'skip',
+            [TEST_FRAMEWORK_VERSION]: tracer._version,
+            [JEST_TEST_RUNNER]: 'jest-jasmine2'
           }
         }
       )

package/packages/datadog-plugin-mocha/src/index.js

@@ -9,6 +9,7 @@ const {
   TEST_SUITE,
   TEST_STATUS,
   TEST_PARAMETERS,
+  TEST_FRAMEWORK_VERSION,
   CI_APP_ORIGIN,
   getTestEnvironmentMetadata,
   getTestParametersString,
@@ -31,13 +32,20 @@ function getTestSpanMetadata (tracer, test, sourceRoot) {
     [TEST_NAME]: fullTestName,
     [TEST_SUITE]: testSuite,
     [SAMPLING_RULE_DECISION]: 1,
-    [SAMPLING_PRIORITY]: AUTO_KEEP
+    [SAMPLING_PRIORITY]: AUTO_KEEP,
+    [TEST_FRAMEWORK_VERSION]: tracer._version
   }
 }
 
 function createWrapRunTest (tracer, testEnvironmentMetadata, sourceRoot) {
   return function wrapRunTest (runTest) {
     return async function runTestWithTrace () {
+      // `runTest` is rerun when retries are configured through `this.retries` and the test fails.
+      // This clause prevents rewrapping `this.test.fn` when it has already been wrapped.
+      if (this.test._currentRetry !== undefined && this.test._currentRetry !== 0) {
+        return runTest.apply(this, arguments)
+      }
+
       let specFunction = this.test.fn
       if (specFunction.length) {
         specFunction = promisify(specFunction)

package/packages/dd-trace/lib/version.js

@@ -1 +1 @@
-module.exports = '1.5.0'
+module.exports = '2.0.0-pre'

package/packages/dd-trace/src/appsec/addresses.js

@@ -0,0 +1,11 @@
+'use strict'
+
+module.exports = {
+  HTTP_INCOMING_URL: 'server.request.uri.raw',
+  HTTP_INCOMING_HEADERS: 'server.request.headers.no_cookies',
+  HTTP_INCOMING_METHOD: 'server.request.method',
+  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
+  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port',
+  HTTP_INCOMING_RESPONSE_CODE: 'server.response.status',
+  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies'
+}

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -0,0 +1,126 @@
+'use strict'
+
+const log = require('../../log')
+const addresses = require('../addresses')
+const Gateway = require('../../gateway/engine')
+const Reporter = require('../reporter')
+
+let warned = false
+
+const validAddressSet = new Set(Object.values(addresses))
+
+const DEFAULT_MAX_BUDGET = 5e3 // µs
+
+// TODO: put reusable code in a base class
+class WAFCallback {
+  static loadDDWAF (rules) {
+    try {
+      // require in `try/catch` because this can throw at require time
+      const { DDWAF } = require('@datadog/native-appsec')
+
+      return new DDWAF(rules)
+    } catch (err) {
+      if (!warned) {
+        log.warn('AppSec could not load native package. In-app WAF features will not be available.')
+        warned = true
+      }
+
+      throw err
+    }
+  }
+
+  constructor (rules) {
+    this.ddwaf = WAFCallback.loadDDWAF(rules)
+    this.wafContextCache = new WeakMap()
+
+    // closures are faster than binds
+    const self = this
+    const method = (params, store) => {
+      return self.action(params, store)
+    }
+
+    // might be its own class with more info later
+    const callback = { method }
+
+    const subscribedAddresses = new Set()
+
+    for (const rule of rules.rules) {
+      for (const condition of rule.conditions) {
+        for (const input of condition.parameters.inputs) {
+          const address = input.address.split(':', 2)[0]
+
+          if (!validAddressSet.has(address) || subscribedAddresses.has(address)) continue
+
+          subscribedAddresses.add(address)
+
+          Gateway.manager.addSubscription({ addresses: [ address ], callback })
+        }
+      }
+    }
+  }
+
+  action (params, store) {
+    let wafContext
+
+    if (store) {
+      const key = store.get('context')
+
+      if (key) {
+        if (this.wafContextCache.has(key)) {
+          wafContext = this.wafContextCache.get(key)
+        } else {
+          wafContext = this.ddwaf.createContext()
+          this.wafContextCache.set(key, wafContext)
+        }
+      }
+    }
+
+    if (!wafContext) {
+      wafContext = this.ddwaf.createContext()
+    }
+
+    // TODO: if status code in params, convert it to string
+
+    try {
+      // TODO: possible optimizaion: only send params that haven't already been sent to this wafContext
+      const result = wafContext.run(params, DEFAULT_MAX_BUDGET)
+
+      return this.applyResult(result)
+    } catch (err) {
+      log.warn('Error while running the AppSec WAF')
+    }
+  }
+
+  applyResult (result) {
+    if (result.action) {
+      const data = JSON.parse(result.data)
+
+      for (let i = 0; i < data.length; ++i) {
+        const point = data[i]
+        const ruleMatch = point.rule_matches[0]
+
+        ruleMatch.highlight = []
+
+        for (const param of ruleMatch.parameters) {
+          ruleMatch.highlight = ruleMatch.highlight.concat(param.highlight)
+          delete param.highlight
+        }
+
+        Reporter.reportAttack(point.rule, ruleMatch)
+      }
+    }
+
+    // result.perfData
+    // result.perfTotalRuntime
+  }
+
+  clear () {
+    this.ddwaf.dispose()
+
+    this.wafContextCache = new WeakMap()
+
+    Gateway.manager.clear()
+  }
+}
+
+module.exports = WAFCallback

package/packages/dd-trace/src/appsec/callbacks/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+// lazy loading
+// TODO: cache the returned value
+module.exports = {
+  get DDWAF () { return require('./ddwaf') }
+}