dd-trace 2.0.0-appsec-beta.4 → 2.0.1

package/packages/datadog-plugin-pino/src/index.js

@@ -100,7 +100,7 @@ module.exports = [
   },
   {
     name: 'pino',
-    versions: ['>=5.14.0'],
+    versions: ['>=5.14.0 <6.8.0'],
     patch (pino, tracer, config) {
       if (!tracer._logInjection) return
 
@@ -112,6 +112,30 @@ module.exports = [
       return this.unwrapExport(pino)
     }
   },
+  {
+    name: 'pino',
+    versions: ['>=6.8.0'],
+    patch (pino, tracer, config) {
+      if (!tracer._logInjection) return
+
+      const mixinSym = pino.symbols.mixinSym
+
+      const wrapped = this.wrapExport(pino, createWrapPino(tracer, config, mixinSym, createWrapMixin)(pino))
+
+      wrapped.pino = wrapped
+      wrapped.default = wrapped
+
+      return wrapped
+    },
+    unpatch (pino) {
+      const unwrapped = this.unwrapExport(pino)
+
+      unwrapped.pino = unwrapped
+      unwrapped.default = unwrapped
+
+      return unwrapped
+    }
+  },
   {
     name: 'pino-pretty',
     versions: ['>=3'], // will only work starting from pino@5.0.0 as previous versions are not using pino-pretty

package/packages/datadog-plugin-router/src/index.js

@@ -5,6 +5,9 @@ const pathToRegExp = require('path-to-regexp')
 const shimmer = require('../../datadog-shimmer')
 const web = require('../../dd-trace/src/plugins/util/web')
 
+// TODO: clean this up to not use web util internals
+// TODO: stop checking for fast star and fast slash
+
 const regexpCache = Object.create(null)
 
 function createWrapHandle (tracer, config) {
@@ -12,6 +15,19 @@ function createWrapHandle (tracer, config) {
     return function handleWithTrace (req, res, done) {
       web.patch(req)
 
+      if (!req._datadog.router) {
+        const context = {
+          route: '',
+          stack: []
+        }
+
+        web.beforeEnd(req, () => {
+          req._datadog.paths = [context.route]
+        })
+
+        req._datadog.router = context
+      }
+
       return handle.apply(this, arguments)
     }
   }
@@ -82,8 +98,8 @@ function wrapNext (layer, req, next) {
   const originalNext = next
 
   return function (error) {
-    if (!error && layer.path && !isFastStar(layer) && !isFastSlash(layer)) {
-      web.exitRoute(req)
+    if (layer.path && !isFastStar(layer) && !isFastSlash(layer)) {
+      req._datadog.router.stack.pop()
     }
 
     web.finish(req, error)
@@ -99,7 +115,16 @@ function callHandle (layer, handle, req, args) {
     // Try to guess which path actually matched
     for (let i = 0; i < matchers.length; i++) {
       if (matchers[i].test(layer)) {
-        web.enterRoute(req, matchers[i].path)
+        const context = req._datadog.router
+
+        context.stack.push(matchers[i].path)
+
+        const route = context.stack.join('')
+
+        // Longer route is more likely to be the actual route handler route.
+        if (route.length > context.route.length) {
+          context.route = route
+        }
 
         break
       }

package/packages/datadog-plugin-winston/src/index.js

@@ -2,18 +2,37 @@
 
 const { LOG } = require('../../../ext/formats')
 
+function chunkProxy (chunk, holder) {
+  return new Proxy(chunk, {
+    get (target, p, receiver) {
+      switch (p) {
+        case Symbol.toStringTag:
+          return Object.prototype.toString.call(target).slice(8, -1)
+        case 'dd':
+          return holder.dd
+        default:
+          return Reflect.get(target, p, receiver)
+      }
+    },
+    ownKeys (target) {
+      return ['dd', ...Reflect.ownKeys(target)]
+    },
+    getOwnPropertyDescriptor (target, p) {
+      return Reflect.getOwnPropertyDescriptor(p === 'dd' ? holder : target, p)
+    }
+  })
+}
+
 function createWrapWrite (tracer, config) {
   return function wrapWrite (write) {
     return function writeWithTrace (chunk, encoding, callback) {
       const span = tracer.scope().active()
 
-      tracer.inject(span, LOG, chunk)
+      const holder = {}
+      tracer.inject(span, LOG, holder)
+      arguments[0] = chunkProxy(chunk, holder)
 
-      const result = write.apply(this, arguments)
-
-      delete chunk.dd
-
-      return result
+      return write.apply(this, arguments)
     }
   }
 }
@@ -42,15 +61,14 @@ function createWrapLog (tracer, config) {
     return function logWithTrace (level, msg, meta, callback) {
       const span = tracer.scope().active()
 
-      meta = arguments[2] = meta || {}
+      meta = meta || {}
 
-      tracer.inject(span, LOG, meta)
+      const holder = {}
+      tracer.inject(span, LOG, holder)
 
-      const result = log.apply(this, arguments)
+      arguments[2] = chunkProxy(meta, holder)
 
-      delete meta.dd
-
-      return result
+      return log.apply(this, arguments)
     }
   }
 }

package/packages/dd-trace/lib/version.js

@@ -1 +1 @@
-module.exports = '2.0.0-pre'
+module.exports = '2.0.1'

package/packages/dd-trace/src/appsec/addresses.js

@@ -1,11 +1,18 @@
 'use strict'
 
 module.exports = {
-  HTTP_INCOMING_URL: 'server.request.uri.raw',
+  HTTP_INCOMING_BODY: 'server.request.body',
+  HTTP_INCOMING_QUERY: 'server.request.query',
   HTTP_INCOMING_HEADERS: 'server.request.headers.no_cookies',
+  // TODO: 'server.request.trailers',
+  HTTP_INCOMING_URL: 'server.request.uri.raw',
   HTTP_INCOMING_METHOD: 'server.request.method',
-  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
-  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port',
+  HTTP_INCOMING_ENDPOINT: 'server.request.framework_endpoint',
+  HTTP_INCOMING_PARAMS: 'server.request.path_params',
+  HTTP_INCOMING_COOKIES: 'server.request.cookies',
   HTTP_INCOMING_RESPONSE_CODE: 'server.response.status',
-  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies'
+  HTTP_INCOMING_RESPONSE_HEADERS: 'server.response.headers.no_cookies',
+  // TODO: 'server.response.trailers',
+  HTTP_INCOMING_REMOTE_IP: 'server.request.client_ip',
+  HTTP_INCOMING_REMOTE_PORT: 'server.request.client_port'
 }

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -5,8 +5,6 @@ const addresses = require('../addresses')
 const Gateway = require('../gateway/engine')
 const Reporter = require('../reporter')
 
-let warned = false
-
 const validAddressSet = new Set(Object.values(addresses))
 
 const DEFAULT_MAX_BUDGET = 5e3 // µs
@@ -20,10 +18,7 @@ class WAFCallback {
 
       return new DDWAF(rules)
     } catch (err) {
-      if (!warned) {
-        log.warn('AppSec could not load native package. In-app WAF features will not be available.')
-        warned = true
-      }
+      log.error('AppSec could not load native package. In-app WAF features will not be available.')
 
       throw err
     }
@@ -90,7 +85,8 @@ class WAFCallback {
 
       return this.applyResult(result, store)
     } catch (err) {
-      log.warn('Error while running the AppSec WAF')
+      log.error('Error while running the AppSec WAF')
+      log.error(err)
     }
   }
 
@@ -99,6 +95,7 @@ class WAFCallback {
       Reporter.reportAttack(result.data, store)
     }
 
+    // TODO: use these values later for budget management
     // result.perfData
     // result.perfTotalRuntime
   }

package/packages/dd-trace/src/appsec/gateway/als.js

@@ -1,5 +1,6 @@
 'use strict'
 
+// TODO: use datadog-core storage instead
 const { AsyncLocalStorage } = require('async_hooks')
 
 module.exports = new AsyncLocalStorage()

package/packages/dd-trace/src/appsec/gateway/channels.js

@@ -2,6 +2,9 @@
 
 const dc = require('diagnostics_channel')
 
+// TODO: use TBD naming convention
+//       or directly use http plugin's channels
+//       when it gets converted to new plugin system
 module.exports = {
   incomingHttpRequestStart: dc.channel('dd-trace:incomingHttpRequestStart'),
   incomingHttpRequestEnd: dc.channel('dd-trace:incomingHttpRequestEnd')

package/packages/dd-trace/src/appsec/gateway/engine/engine.js

@@ -43,10 +43,10 @@ class SubscriptionManager {
     const knownSubscriptions = new Set()
 
     // TODO: possible optimization: collect matchedSubscriptions on the fly in Context#setValue
-    for (let i = 0; i < newAddresses.length; ++i) {
-      const matchedSubscriptions = this.addressToSubscriptions.get(newAddresses[i])
+    newAddresses.forEach((newAddress) => {
+      const matchedSubscriptions = this.addressToSubscriptions.get(newAddress)
 
-      if (matchedSubscriptions === undefined) continue
+      if (matchedSubscriptions === undefined) return
 
       for (let j = 0; j < matchedSubscriptions.length; ++j) {
         const subscription = matchedSubscriptions[j]
@@ -64,24 +64,24 @@ class SubscriptionManager {
           subscriptions.add(subscription)
         }
       }
-    }
+    })
 
     return { addresses, subscriptions }
   }
 
   dispatch (newAddresses, allAddresses, context) {
-    const { addresses, subscriptions } = this.matchSubscriptions(newAddresses, allAddresses)
+    const matches = this.matchSubscriptions(newAddresses, allAddresses)
 
     // TODO: possible optimization
-    // if(!subscriptions.size) return []
+    // check if matches.subscriptions is empty here instead of in runner.js
 
     const params = {}
 
-    addresses.forEach((address) => {
+    matches.addresses.forEach((address) => {
       params[address] = context.resolve(address)
     })
 
-    return Runner.runSubscriptions(subscriptions, params)
+    return Runner.runSubscriptions(matches.subscriptions, params)
   }
 }
 
@@ -91,50 +91,40 @@ class Context {
   }
 
   constructor () {
+    // TODO: this probably don't need to be a Map()
     this.store = new Map()
     this.allAddresses = new Set()
-    this.newAddresses = [] // TODO: maybe this should be a Set
+    this.newAddresses = new Set()
   }
 
   clear () {
     this.store = new Map()
     this.allAddresses = new Set()
-    this.newAddresses = []
+    this.newAddresses = new Set()
   }
 
   setValue (address, value) {
     if (this.allAddresses.size >= MAX_CONTEXT_SIZE) return this
 
-    const oldValue = this.store.get(address)
-    if (oldValue === value) return this
-
-    this.store.set(address, value)
-
-    if (!this.newAddresses.includes(address)) {
-      this.allAddresses.add(address)
-      this.newAddresses.push(address)
+    // cannot optimize for objects because they're pointers
+    if (typeof value !== 'object') {
+      const oldValue = this.store.get(address)
+      if (oldValue === value) return this
     }
 
-    return this
-  }
-
-  setMultipleValues (params) {
-    const addresses = Object.keys(params)
-
-    for (let i = 0; i < addresses.length; ++i) {
-      const address = addresses[i]
-      this.setValue(address, params[address])
-    }
+    this.store.set(address, value)
+    this.allAddresses.add(address)
+    this.newAddresses.add(address)
 
     return this
   }
 
   dispatch () {
-    if (this.newAddresses.length === 0) return []
+    if (this.newAddresses.size === 0) return []
 
     const result = Context.manager.dispatch(this.newAddresses, this.allAddresses, this)
 
-    this.newAddresses = []
+    this.newAddresses.clear()
 
     return result
   }

package/packages/dd-trace/src/appsec/gateway/engine/runner.js

@@ -12,6 +12,8 @@ function runSubscriptions (subscriptions, params) {
 
   const store = als.getStore()
 
+  // TODO: possible optimization
+  // can we deduplicate those before ?
   const executedCallbacks = new Set()
 
   for (const subscription of subscriptions) {

package/packages/dd-trace/src/appsec/index.js

@@ -17,27 +17,28 @@ function enable (config) {
 
     RuleManager.applyRules(rules)
   } catch (err) {
-    log.error(`Unable to apply AppSec rules: ${err}`)
+    log.error('Unable to start AppSec')
+    log.error(err)
 
     // abort AppSec start
     RuleManager.clearAllRules()
     return
   }
 
+  Reporter.setRateLimit(config.appsec.rateLimit)
+
   incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
   incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
 
-  // add needed fields for HTTP context reporting
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_URL)
+  // add fields needed for HTTP context reporting
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_HEADERS)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_METHOD)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_PORT)
-  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_CODE)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_ENDPOINT)
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
+  Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
 }
 
 function incomingHttpStartTranslator (data) {
+  // TODO: get span from datadog-core storage instead
   const topSpan = data.req._datadog && data.req._datadog.span
   if (topSpan) {
     topSpan.addTags({
@@ -50,19 +51,6 @@ function incomingHttpStartTranslator (data) {
 
   store.set('req', data.req)
   store.set('res', data.res)
-
-  const headers = Object.assign({}, data.req.headers)
-  delete headers.cookie
-
-  const context = store.get('context')
-
-  Gateway.propagate({
-    [addresses.HTTP_INCOMING_URL]: data.req.url,
-    [addresses.HTTP_INCOMING_HEADERS]: headers,
-    [addresses.HTTP_INCOMING_METHOD]: data.req.method,
-    [addresses.HTTP_INCOMING_REMOTE_IP]: data.req.socket.remoteAddress,
-    [addresses.HTTP_INCOMING_REMOTE_PORT]: data.req.socket.remotePort
-  }, context)
 }
 
 function incomingHttpEndTranslator (data) {
@@ -70,14 +58,41 @@ function incomingHttpEndTranslator (data) {
 
   if (!context) return
 
+  const requestHeaders = Object.assign({}, data.req.headers)
+  delete requestHeaders.cookie
+
   // TODO: this doesn't support headers sent with res.writeHead()
-  const headers = Object.assign({}, data.res.getHeaders())
-  delete headers['set-cookie']
+  const responseHeaders = Object.assign({}, data.res.getHeaders())
+  delete responseHeaders['set-cookie']
 
-  Gateway.propagate({
+  const payload = {
+    [addresses.HTTP_INCOMING_URL]: data.req.url,
+    [addresses.HTTP_INCOMING_HEADERS]: requestHeaders,
+    [addresses.HTTP_INCOMING_METHOD]: data.req.method,
+    [addresses.HTTP_INCOMING_REMOTE_IP]: data.req.socket.remoteAddress,
+    [addresses.HTTP_INCOMING_REMOTE_PORT]: data.req.socket.remotePort,
     [addresses.HTTP_INCOMING_RESPONSE_CODE]: data.res.statusCode,
-    [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: headers
-  }, context)
+    [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: responseHeaders
+  }
+
+  // TODO: temporary express instrumentation, will use express plugin later
+  if (data.req.query && typeof data.req.query === 'object') {
+    payload[addresses.HTTP_INCOMING_QUERY] = data.req.query
+  }
+
+  if (data.req.route && typeof data.req.route.path === 'string') {
+    payload[addresses.HTTP_INCOMING_ENDPOINT] = data.req.route.path
+  }
+
+  if (data.req.params && typeof data.req.params === 'object') {
+    payload[addresses.HTTP_INCOMING_PARAMS] = data.req.params
+  }
+
+  if (data.req.cookies && typeof data.req.cookies === 'object') {
+    payload[addresses.HTTP_INCOMING_COOKIES] = data.req.cookies
+  }
+
+  Gateway.propagate(payload, context)
 
   Reporter.finishAttacks(data.req, context)
 }
@@ -85,6 +100,7 @@ function incomingHttpEndTranslator (data) {
 function disable () {
   RuleManager.clearAllRules()
 
+  // Channel#unsubscribe() is undefined for non active channels
   if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
   if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
 }