dd-trace 2.0.0-appsec-beta.3 → 2.0.0

package/packages/datadog-plugin-redis/src/index.js

@@ -1,7 +1,25 @@
 'use strict'
 
+// TODO: always use uppercase for command names
+
 const tx = require('../../dd-trace/src/plugins/util/redis')
 
+function createWrapAddCommand (tracer, config) {
+  return function wrapAddCommand (addCommand) {
+    return function addCommandWithTrace (command) {
+      const name = command[0]
+      const args = command.slice(1)
+
+      if (!config.filter(name)) return addCommand.apply(this, arguments)
+
+      const scope = tracer.scope()
+      const span = startSpan(tracer, config, this, name, args)
+
+      return tx.wrap(span, scope.bind(addCommand, span).apply(this, arguments))
+    }
+  }
+}
+
 function createWrapInternalSendCommand (tracer, config) {
   return function wrapInternalSendCommand (internalSendCommand) {
     return function internalSendCommandWithTrace (options) {
@@ -49,9 +67,21 @@ function startSpan (tracer, config, client, command, args) {
 }
 
 module.exports = [
+  {
+    name: '@node-redis/client',
+    versions: ['>=1'],
+    file: 'dist/lib/client/commands-queue.js',
+    patch (redis, tracer, config) {
+      config = tx.normalizeConfig(config)
+      this.wrap(redis.default.prototype, 'addCommand', createWrapAddCommand(tracer, config))
+    },
+    unpatch (redis) {
+      this.unwrap(redis.default.prototype, 'addCommand')
+    }
+  },
   {
     name: 'redis',
-    versions: ['>=2.6'],
+    versions: ['>=2.6 <4'],
     patch (redis, tracer, config) {
       config = tx.normalizeConfig(config)
       this.wrap(redis.RedisClient.prototype, 'internal_send_command', createWrapInternalSendCommand(tracer, config))

package/packages/datadog-plugin-router/src/index.js

@@ -5,6 +5,9 @@ const pathToRegExp = require('path-to-regexp')
 const shimmer = require('../../datadog-shimmer')
 const web = require('../../dd-trace/src/plugins/util/web')
 
+// TODO: clean this up to not use web util internals
+// TODO: stop checking for fast star and fast slash
+
 const regexpCache = Object.create(null)
 
 function createWrapHandle (tracer, config) {
@@ -12,6 +15,19 @@ function createWrapHandle (tracer, config) {
     return function handleWithTrace (req, res, done) {
       web.patch(req)
 
+      if (!req._datadog.router) {
+        const context = {
+          route: '',
+          stack: []
+        }
+
+        web.beforeEnd(req, () => {
+          req._datadog.paths = [context.route]
+        })
+
+        req._datadog.router = context
+      }
+
       return handle.apply(this, arguments)
     }
   }
@@ -82,8 +98,8 @@ function wrapNext (layer, req, next) {
   const originalNext = next
 
   return function (error) {
-    if (!error && layer.path && !isFastStar(layer) && !isFastSlash(layer)) {
-      web.exitRoute(req)
+    if (layer.path && !isFastStar(layer) && !isFastSlash(layer)) {
+      req._datadog.router.stack.pop()
     }
 
     web.finish(req, error)
@@ -99,7 +115,16 @@ function callHandle (layer, handle, req, args) {
     // Try to guess which path actually matched
     for (let i = 0; i < matchers.length; i++) {
       if (matchers[i].test(layer)) {
-        web.enterRoute(req, matchers[i].path)
+        const context = req._datadog.router
+
+        context.stack.push(matchers[i].path)
+
+        const route = context.stack.join('')
+
+        // Longer route is more likely to be the actual route handler route.
+        if (route.length > context.route.length) {
+          context.route = route
+        }
 
         break
       }

package/packages/dd-trace/lib/version.js

@@ -1 +1 @@
-module.exports = '2.0.0-pre'
+module.exports = '2.0.0'

package/packages/dd-trace/src/appsec/addresses.js

@@ -1,11 +1,18 @@
 'use strict'
 
 module.exports = {
-  HTTP_INCOMING_URL: 'server.request.uri.raw',
+  HTTP_INCOMING_BODY: 'server.request.body',
+  HTTP_INCOMING_QUERY: 'server.request.query',
   HTTP_INCOMING_HEADERS: 'server.request.headers.no_cookies',
+  // TODO: 'server.request.trailers',
+  HTTP_INCOMING_URL: 'server.request.uri.raw',
   HTTP_INCOMING_METHOD: 'server.request.method',
-  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
-  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port',
+  HTTP_INCOMING_ENDPOINT: 'server.request.framework_endpoint',
+  HTTP_INCOMING_PARAMS: 'server.request.path_params',
+  HTTP_INCOMING_COOKIES: 'server.request.cookies',
   HTTP_INCOMING_RESPONSE_CODE: 'server.response.status',
-  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies'
+  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies',
+  // TODO: 'server.response.trailers',
+  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
+  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port'
 }

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -2,11 +2,9 @@
 
 const log = require('../../log')
 const addresses = require('../addresses')
-const Gateway = require('../../gateway/engine')
+const Gateway = require('../gateway/engine')
 const Reporter = require('../reporter')
 
-let warned = false
-
 const validAddressSet = new Set(Object.values(addresses))
 
 const DEFAULT_MAX_BUDGET = 5e3 // µs
@@ -20,10 +18,7 @@ class WAFCallback {
 
       return new DDWAF(rules)
     } catch (err) {
-      if (!warned) {
-        log.warn('AppSec could not load native package. In-app WAF features will not be available.')
-        warned = true
-      }
+      log.error('AppSec could not load native package. In-app WAF features will not be available.')
 
       throw err
     }
@@ -90,7 +85,8 @@ class WAFCallback {
 
       return this.applyResult(result, store)
     } catch (err) {
-      log.warn('Error while running the AppSec WAF')
+      log.error('Error while running the AppSec WAF')
+      log.error(err)
     }
   }
 
@@ -99,6 +95,7 @@ class WAFCallback {
       Reporter.reportAttack(result.data, store)
     }
 
+    // TODO: use these values later for budget management
     // result.perfData
     // result.perfTotalRuntime
   }

package/packages/dd-trace/src/{gateway → appsec/gateway}/als.js

@@ -1,5 +1,6 @@
 'use strict'
 
+// TODO: use datadog-core storage instead
 const { AsyncLocalStorage } = require('async_hooks')
 
 module.exports = new AsyncLocalStorage()

package/packages/dd-trace/src/appsec/gateway/channels.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const dc = require('diagnostics_channel')
+
+// TODO: use TBD naming convention
+//       or directly use http plugin's channels
+//       when it gets converted to new plugin system
+module.exports = {
+  incomingHttpRequestStart: dc.channel('dd-trace:incomingHttpRequestStart'),
+  incomingHttpRequestEnd: dc.channel('dd-trace:incomingHttpRequestEnd')
+}

package/packages/dd-trace/src/{gateway → appsec/gateway}/engine/engine.js

@@ -43,10 +43,10 @@ class SubscriptionManager {
     const knownSubscriptions = new Set()
 
     // TODO: possible optimization: collect matchedSubscriptions on the fly in Context#setValue
-    for (let i = 0; i < newAddresses.length; ++i) {
-      const matchedSubscriptions = this.addressToSubscriptions.get(newAddresses[i])
+    newAddresses.forEach((newAddress) => {
+      const matchedSubscriptions = this.addressToSubscriptions.get(newAddress)
 
-      if (matchedSubscriptions === undefined) continue
+      if (matchedSubscriptions === undefined) return
 
       for (let j = 0; j < matchedSubscriptions.length; ++j) {
         const subscription = matchedSubscriptions[j]
@@ -64,24 +64,24 @@ class SubscriptionManager {
           subscriptions.add(subscription)
         }
       }
-    }
+    })
 
     return { addresses, subscriptions }
   }
 
   dispatch (newAddresses, allAddresses, context) {
-    const { addresses, subscriptions } = this.matchSubscriptions(newAddresses, allAddresses)
+    const matches = this.matchSubscriptions(newAddresses, allAddresses)
 
     // TODO: possible optimization
-    // if(!subscriptions.size) return []
+    // check if matches.subscriptions is empty here instead of in runner.js
 
     const params = {}
 
-    addresses.forEach((address) => {
+    matches.addresses.forEach((address) => {
       params[address] = context.resolve(address)
     })
 
-    return Runner.runSubscriptions(subscriptions, params)
+    return Runner.runSubscriptions(matches.subscriptions, params)
   }
 }
 
@@ -91,50 +91,40 @@ class Context {
   }
 
   constructor () {
+    // TODO: this probably don't need to be a Map()
     this.store = new Map()
     this.allAddresses = new Set()
-    this.newAddresses = [] // TODO: maybe this should be a Set
+    this.newAddresses = new Set()
   }
 
   clear () {
     this.store = new Map()
     this.allAddresses = new Set()
-    this.newAddresses = []
+    this.newAddresses = new Set()
   }
 
   setValue (address, value) {
     if (this.allAddresses.size >= MAX_CONTEXT_SIZE) return this
 
-    const oldValue = this.store.get(address)
-    if (oldValue === value) return this
-
-    this.store.set(address, value)
-
-    if (!this.newAddresses.includes(address)) {
-      this.allAddresses.add(address)
-      this.newAddresses.push(address)
+    // cannot optimize for objects because they're pointers
+    if (typeof value !== 'object') {
+      const oldValue = this.store.get(address)
+      if (oldValue === value) return this
     }
 
-    return this
-  }
-
-  setMultipleValues (params) {
-    const addresses = Object.keys(params)
-
-    for (let i = 0; i < addresses.length; ++i) {
-      const address = addresses[i]
-      this.setValue(address, params[address])
-    }
+    this.store.set(address, value)
+    this.allAddresses.add(address)
+    this.newAddresses.add(address)
 
     return this
   }
 
   dispatch () {
-    if (this.newAddresses.length === 0) return []
+    if (this.newAddresses.size === 0) return []
 
     const result = Context.manager.dispatch(this.newAddresses, this.allAddresses, this)
 
-    this.newAddresses = []
+    this.newAddresses.clear()
 
     return result
   }

package/packages/dd-trace/src/{gateway → appsec/gateway}/engine/runner.js

@@ -12,6 +12,8 @@ function runSubscriptions (subscriptions, params) {
 
   const store = als.getStore()
 
+  // TODO: possible optimization
+  // can we deduplicate those before ?
   const executedCallbacks = new Set()
 
   for (const subscription of subscriptions) {

package/packages/dd-trace/src/appsec/index.js

@@ -3,8 +3,8 @@
 const fs = require('fs')
 const log = require('../log')
 const RuleManager = require('./rule_manager')
-const { INCOMING_HTTP_REQUEST_START, INCOMING_HTTP_REQUEST_END } = require('../gateway/channels')
-const Gateway = require('../gateway/engine/index')
+const { incomingHttpRequestStart, incomingHttpRequestEnd } = require('./gateway/channels')
+const Gateway = require('./gateway/engine')
 const addresses = require('./addresses')
 const Reporter = require('./reporter')
 
@@ -17,47 +17,40 @@ function enable (config) {
 
     RuleManager.applyRules(rules)
   } catch (err) {
-    log.error(`Unable to apply AppSec rules: ${err}`)
+    log.error('Unable to start AppSec')
+    log.error(err)
 
     // abort AppSec start
     RuleManager.clearAllRules()
     return
   }
 
-  INCOMING_HTTP_REQUEST_START.subscribe(incomingHttpStartTranslator)
-  INCOMING_HTTP_REQUEST_END.subscribe(incomingHttpEndTranslator)
+  Reporter.setRateLimit(config.appsec.rateLimit)
 
-  config.tags['_dd.appsec.enabled'] = 1
-  config.tags['_dd.runtime_family'] = 'nodejs'
+  incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
+  incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
 
-  // add needed fields for HTTP context reporting
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_URL)
+  // add fields needed for HTTP context reporting
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_HEADERS)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_METHOD)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_PORT)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_CODE)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_ENDPOINT)
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
 }
 
 function incomingHttpStartTranslator (data) {
+  // TODO: get span from datadog-core storage instead
+  const topSpan = data.req._datadog && data.req._datadog.span
+  if (topSpan) {
+    topSpan.addTags({
+      '_dd.appsec.enabled': 1,
+      '_dd.runtime_family': 'nodejs'
+    })
+  }
+
   const store = Gateway.startContext()
 
   store.set('req', data.req)
   store.set('res', data.res)
-
-  const headers = Object.assign({}, data.req.headers)
-  delete headers.cookie
-
-  const context = store.get('context')
-
-  Gateway.propagate({
-    [addresses.HTTP_INCOMING_URL]: data.req.url,
-    [addresses.HTTP_INCOMING_HEADERS]: headers,
-    [addresses.HTTP_INCOMING_METHOD]: data.req.method,
-    [addresses.HTTP_INCOMING_REMOTE_IP]: data.req.socket.remoteAddress,
-    [addresses.HTTP_INCOMING_REMOTE_PORT]: data.req.socket.remotePort
-  }, context)
 }
 
 function incomingHttpEndTranslator (data) {
@@ -65,14 +58,41 @@ function incomingHttpEndTranslator (data) {
 
   if (!context) return
 
+  const requestHeaders = Object.assign({}, data.req.headers)
+  delete requestHeaders.cookie
+
   // TODO: this doesn't support headers sent with res.writeHead()
-  const headers = Object.assign({}, data.res.getHeaders())
-  delete headers['set-cookie']
+  const responseHeaders = Object.assign({}, data.res.getHeaders())
+  delete responseHeaders['set-cookie']
 
-  Gateway.propagate({
+  const payload = {
+    [addresses.HTTP_INCOMING_URL]: data.req.url,
+    [addresses.HTTP_INCOMING_HEADERS]: requestHeaders,
+    [addresses.HTTP_INCOMING_METHOD]: data.req.method,
+    [addresses.HTTP_INCOMING_REMOTE_IP]: data.req.socket.remoteAddress,
+    [addresses.HTTP_INCOMING_REMOTE_PORT]: data.req.socket.remotePort,
     [addresses.HTTP_INCOMING_RESPONSE_CODE]: data.res.statusCode,
-    [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: headers
-  }, context)
+    [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: responseHeaders
+  }
+
+  // TODO: temporary express instrumentation, will use express plugin later
+  if (data.req.query && typeof data.req.query === 'object') {
+    payload[addresses.HTTP_INCOMING_QUERY] = data.req.query
+  }
+
+  if (data.req.route && typeof data.req.route.path === 'string') {
+    payload[addresses.HTTP_INCOMING_ENDPOINT] = data.req.route.path
+  }
+
+  if (data.req.params && typeof data.req.params === 'object') {
+    payload[addresses.HTTP_INCOMING_PARAMS] = data.req.params
+  }
+
+  if (data.req.cookies && typeof data.req.cookies === 'object') {
+    payload[addresses.HTTP_INCOMING_COOKIES] = data.req.cookies
+  }
+
+  Gateway.propagate(payload, context)
 
   Reporter.finishAttacks(data.req, context)
 }
@@ -80,13 +100,9 @@ function incomingHttpEndTranslator (data) {
 function disable () {
   RuleManager.clearAllRules()
 
-  if (INCOMING_HTTP_REQUEST_START.hasSubscribers) INCOMING_HTTP_REQUEST_START.unsubscribe(incomingHttpStartTranslator)
-  if (INCOMING_HTTP_REQUEST_END.hasSubscribers) INCOMING_HTTP_REQUEST_END.unsubscribe(incomingHttpEndTranslator)
-
-  const tags = global._ddtrace._tracer._tags
-
-  delete tags['_dd.appsec.enabled']
-  delete tags['_dd.runtime_family']
+  // Channel#unsubscribe() is undefined for non active channels
+  if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
+  if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
 }
 
 module.exports = {