dd-trace 2.0.0-appsec-beta.3 → 2.0.0

package/packages/dd-trace/src/plugins/util/web.js

@@ -8,7 +8,7 @@ const tags = require('../../../../../ext/tags')
 const types = require('../../../../../ext/types')
 const kinds = require('../../../../../ext/kinds')
 const urlFilter = require('./urlfilter')
-const { INCOMING_HTTP_REQUEST_START, INCOMING_HTTP_REQUEST_END } = require('../../gateway/channels')
+const { incomingHttpRequestEnd } = require('../../appsec/gateway/channels')
 
 const WEB = types.WEB
 const SERVER = kinds.SERVER
@@ -23,6 +23,7 @@ const HTTP_STATUS_CODE = tags.HTTP_STATUS_CODE
 const HTTP_ROUTE = tags.HTTP_ROUTE
 const HTTP_REQUEST_HEADERS = tags.HTTP_REQUEST_HEADERS
 const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS
+const MANUAL_DROP = tags.MANUAL_DROP
 
 const HTTP2_HEADER_AUTHORITY = ':authority'
 const HTTP2_HEADER_SCHEME = ':scheme'
@@ -54,9 +55,8 @@ const web = {
 
     const span = startSpan(tracer, config, req, res, name)
 
-    // TODO: replace this with a REFERENCE_NOOP after we split http/express/etc
     if (!config.filter(req.url)) {
-      span.context()._traceFlags.sampled = false
+      span.setTag(MANUAL_DROP, true)
     }
 
     if (config.service) {
@@ -72,17 +72,7 @@ const web = {
       req._datadog.instrumented = true
     }
 
-    if (callback) {
-      return tracer.scope().activate(span, () => {
-        if (INCOMING_HTTP_REQUEST_START.hasSubscribers) {
-          INCOMING_HTTP_REQUEST_START.publish({ req, res })
-        }
-
-        callback(span)
-      })
-    } else if (INCOMING_HTTP_REQUEST_START.hasSubscribers) {
-      INCOMING_HTTP_REQUEST_START.publish({ req, res })
-    }
+    return callback && tracer.scope().activate(span, () => callback(span))
   },
 
   // Reactivate the request scope in case it was changed by a middleware.
@@ -277,7 +267,7 @@ function wrapEnd (req) {
 
     finishMiddleware(req, res)
 
-    if (INCOMING_HTTP_REQUEST_END.hasSubscribers) INCOMING_HTTP_REQUEST_END.publish({ req, res })
+    if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.publish({ req, res })
 
     const returnValue = end.apply(res, arguments)
 
@@ -320,7 +310,7 @@ function addAllowHeaders (req, headers) {
   const contextHeaders = [
     'x-datadog-origin',
     'x-datadog-parent-id',
-    'x-datadog-sampled',
+    'x-datadog-sampled', // Deprecated, but still accept it in case it's sent.
     'x-datadog-sampling-priority',
     'x-datadog-trace-id'
   ]

package/packages/dd-trace/src/priority_sampler.js

@@ -1,14 +1,20 @@
 'use strict'
 
+const coalesce = require('koalas')
 const RateLimiter = require('./rate_limiter')
 const Sampler = require('./sampler')
 const ext = require('../../../ext')
 const { setSamplingRules } = require('./startup-log')
 
 const {
+  SAMPLING_MECHANISM_DEFAULT,
+  SAMPLING_MECHANISM_AGENT,
+  SAMPLING_MECHANISM_RULE,
+  SAMPLING_MECHANISM_MANUAL,
   SAMPLING_RULE_DECISION,
   SAMPLING_LIMIT_DECISION,
-  SAMPLING_AGENT_DECISION
+  SAMPLING_AGENT_DECISION,
+  UPSTREAM_SERVICES_KEY
 } = require('./constants')
 
 const SERVICE_NAME = ext.tags.SERVICE_NAME
@@ -21,6 +27,9 @@ const AUTO_KEEP = ext.priority.AUTO_KEEP
 const USER_KEEP = ext.priority.USER_KEEP
 const DEFAULT_KEY = 'service:,env:'
 
+const defaultSampler = new Sampler(AUTO_KEEP)
+const serviceNames = new Map()
+
 class PrioritySampler {
   constructor (env, { sampleRate, rateLimit = 100, rules = [] } = {}) {
     this._env = env
@@ -33,12 +42,8 @@ class PrioritySampler {
   }
 
   isSampled (span) {
-    const context = this._getContext(span)
-    const rule = this._findRule(context)
-
-    return rule
-      ? this._isSampledByRule(context, rule) && this._isSampledByRateLimit(context)
-      : this._isSampledByAgent(context)
+    const priority = this._getPriorityFromAuto(span)
+    return priority === USER_KEEP || priority === AUTO_KEEP
   }
 
   sample (span, auto = true) {
@@ -50,16 +55,18 @@ class PrioritySampler {
     if (context._sampling.priority !== undefined) return
     if (!root) return // noop span
 
-    const tag = this._getPriority(context._tags)
+    const tag = this._getPriorityFromTags(context._tags)
 
     if (this.validate(tag)) {
       context._sampling.priority = tag
+      context._sampling.mechanism = SAMPLING_MECHANISM_MANUAL
+    } else if (auto) {
+      context._sampling.priority = this._getPriorityFromAuto(root)
+    } else {
       return
     }
 
-    if (auto) {
-      context._sampling.priority = this.isSampled(root) ? AUTO_KEEP : AUTO_REJECT
-    }
+    this._addUpstreamService(root)
   }
 
   update (rates) {
@@ -72,7 +79,7 @@ class PrioritySampler {
       samplers[key] = sampler
     }
 
-    samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || new Sampler(AUTO_KEEP)
+    samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || defaultSampler
 
     this._samplers = samplers
   }
@@ -93,7 +100,16 @@ class PrioritySampler {
     return typeof span.context === 'function' ? span.context() : span
   }
 
-  _getPriority (tags) {
+  _getPriorityFromAuto (span) {
+    const context = this._getContext(span)
+    const rule = this._findRule(context)
+
+    return rule
+      ? this._getPriorityByRule(context, rule)
+      : this._getPriorityByAgent(context)
+  }
+
+  _getPriorityFromTags (tags) {
     if (tags.hasOwnProperty(MANUAL_KEEP) && tags[MANUAL_KEEP] !== false) {
       return USER_KEEP
     } else if (tags.hasOwnProperty(MANUAL_DROP) && tags[MANUAL_DROP] !== false) {
@@ -109,10 +125,11 @@ class PrioritySampler {
     }
   }
 
-  _isSampledByRule (context, rule) {
+  _getPriorityByRule (context, rule) {
     context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate
+    context._sampling.mechanism = SAMPLING_MECHANISM_RULE
 
-    return rule.sampler.isSampled(context)
+    return rule.sampler.isSampled(context) && this._isSampledByRateLimit(context) ? USER_KEEP : USER_REJECT
   }
 
   _isSampledByRateLimit (context) {
@@ -123,13 +140,48 @@ class PrioritySampler {
     return allowed
   }
 
-  _isSampledByAgent (context) {
+  _getPriorityByAgent (context) {
     const key = `service:${context._tags[SERVICE_NAME]},env:${this._env}`
     const sampler = this._samplers[key] || this._samplers[DEFAULT_KEY]
 
     context._trace[SAMPLING_AGENT_DECISION] = sampler.rate()
 
-    return sampler.isSampled(context)
+    if (sampler === defaultSampler) {
+      context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+    } else {
+      context._sampling.mechanism = SAMPLING_MECHANISM_AGENT
+    }
+
+    return sampler.isSampled(context) ? AUTO_KEEP : AUTO_REJECT
+  }
+
+  _addUpstreamService (span) {
+    const context = span.context()
+    const trace = context._trace
+    const service = this._toBase64(context._tags['service.name'])
+    const priority = context._sampling.priority
+    const mechanism = context._sampling.mechanism
+    const rate = Math.ceil(coalesce(
+      context._trace[SAMPLING_RULE_DECISION],
+      context._trace[SAMPLING_AGENT_DECISION]
+    ) * 10000) / 10000
+    const group = `${service}|${priority}|${mechanism}|${rate}`
+    const groups = trace.tags[UPSTREAM_SERVICES_KEY]
+      ? `${trace.tags[UPSTREAM_SERVICES_KEY]};${group}`
+      : group
+
+    trace.tags[UPSTREAM_SERVICES_KEY] = groups
+  }
+
+  _toBase64 (serviceName) {
+    let encoded = serviceNames.get(serviceName)
+
+    if (!encoded) {
+      encoded = Buffer.from(serviceName).toString('base64')
+      serviceNames.set(serviceName, encoded)
+    }
+
+    return encoded
   }
 
   _normalizeRules (rules, sampleRate) {
@@ -151,9 +203,9 @@ class PrioritySampler {
     const service = context._tags['service.name']
 
     if (rule.name instanceof RegExp && !rule.name.test(name)) return false
-    if (rule.name && rule.name !== name) return false
+    if (typeof rule.name === 'string' && rule.name !== name) return false
     if (rule.service instanceof RegExp && !rule.service.test(service)) return false
-    if (rule.service && rule.service !== service) return false
+    if (typeof rule.service === 'string' && rule.service !== service) return false
 
     return true
   }

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -40,7 +40,7 @@ function computeRetries (uploadTimeout) {
     tries++
     uploadTimeout /= 2
   }
-  return [tries, uploadTimeout]
+  return [tries, Math.floor(uploadTimeout)]
 }
 
 class AgentExporter {
@@ -55,7 +55,6 @@ class AgentExporter {
   }
 
   export ({ profiles, start, end, tags }) {
-    const form = new FormData()
     const types = Object.keys(profiles)
 
     const fields = [
@@ -75,10 +74,6 @@ class AgentExporter {
       ...Object.entries(tags).map(([key, value]) => ['tags[]', `${key}:${value}`])
     ]
 
-    for (const [key, value] of fields) {
-      form.append(key, value)
-    }
-
     this._logger.debug(() => {
       const body = fields.map(([key, value]) => `  ${key}: ${value}`).join('\n')
       return `Building agent export report: ${'\n' + body}`
@@ -93,36 +88,14 @@ class AgentExporter {
         return `Adding ${type} profile to agent export: ` + bytes
       })
 
-      form.append(`types[${index}]`, type)
-      form.append(`data[${index}]`, buffer, {
+      fields.push([`types[${index}]`, type])
+      fields.push([`data[${index}]`, buffer, {
         filename: `${type}.pb.gz`,
         contentType: 'application/octet-stream',
         knownLength: buffer.length
-      })
-    }
-
-    const options = {
-      method: 'POST',
-      path: '/profiling/v1/input',
-      headers: form.getHeaders()
-    }
-
-    if (containerId) {
-      options.headers['Datadog-Container-ID'] = containerId
+      }])
     }
 
-    if (this._url.protocol === 'unix:') {
-      options.socketPath = this._url.pathname
-    } else {
-      options.protocol = this._url.protocol
-      options.hostname = this._url.hostname
-      options.port = this._url.port
-    }
-
-    this._logger.debug(() => {
-      return `Submitting agent report to: ${JSON.stringify(options)}`
-    })
-
     return new Promise((resolve, reject) => {
       const operation = retry.operation({
         randomize: true,
@@ -131,8 +104,36 @@ class AgentExporter {
       })
 
       operation.attempt((attempt) => {
-        const timeout = Math.pow(this._backoffTime, attempt)
-        sendRequest({ ...options, timeout }, form, (err, response) => {
+        const form = new FormData()
+
+        for (const [key, value, options] of fields) {
+          form.append(key, value, options)
+        }
+
+        const options = {
+          method: 'POST',
+          path: '/profiling/v1/input',
+          headers: form.getHeaders(),
+          timeout: this._backoffTime * Math.pow(2, attempt)
+        }
+
+        if (containerId) {
+          options.headers['Datadog-Container-ID'] = containerId
+        }
+
+        if (this._url.protocol === 'unix:') {
+          options.socketPath = this._url.pathname
+        } else {
+          options.protocol = this._url.protocol
+          options.hostname = this._url.hostname
+          options.port = this._url.port
+        }
+
+        this._logger.debug(() => {
+          return `Submitting profiler agent report attempt #${attempt} to: ${JSON.stringify(options)}`
+        })
+
+        sendRequest(options, form, (err, response) => {
           if (operation.retry(err)) {
             this._logger.error(`Error from the agent: ${err.message}`)
             return
@@ -159,4 +160,4 @@ class AgentExporter {
   }
 }
 
-module.exports = { AgentExporter }
+module.exports = { AgentExporter, computeRetries }

package/packages/dd-trace/src/proxy.js

@@ -5,8 +5,10 @@ const NoopTracer = require('./noop/tracer')
 const DatadogTracer = require('./tracer')
 const Config = require('./config')
 const Instrumenter = require('./instrumenter')
+const PluginManager = require('./plugin_manager')
 const metrics = require('./metrics')
 const log = require('./log')
+const { isFalse } = require('./util')
 const { setStartupLogInstrumenter } = require('./startup-log')
 
 const noop = new NoopTracer()
@@ -14,8 +16,11 @@ const noop = new NoopTracer()
 class Tracer extends BaseTracer {
   constructor () {
     super()
+
+    this._initialized = false
     this._tracer = noop
     this._instrumenter = new Instrumenter(this)
+    this._pluginManager = new PluginManager(this)
     this._deprecate = method => log.deprecate(`tracer.${method}`, [
       `tracer.${method}() is deprecated.`,
       'Please use tracer.startSpan() and tracer.scope() instead.',
@@ -24,47 +29,51 @@ class Tracer extends BaseTracer {
   }
 
   init (options) {
-    if (this._tracer === noop) {
-      try {
-        const config = new Config(options)
-
-        log.use(config.logger)
-        log.toggle(config.debug, config.logLevel, this)
-
-        if (config.hasOwnProperty('profiling') && config.profiling.enabled) {
-          // do not stop tracer initialization if the profiler fails to be imported
-          try {
-            const profiler = require('./profiler')
-            profiler.start(config)
-          } catch (e) {
-            log.error(e)
-          }
-        }
+    if (isFalse(process.env.DD_TRACE_ENABLED) || this._initialized) return this
+
+    this._initialized = true
 
-        if (config.enabled) {
-          if (config.runtimeMetrics) {
-            metrics.start(config)
-          }
+    try {
+      const config = new Config(options) // TODO: support dynamic config
 
-          // dirty require for now so zero appsec code is executed unless explicitly enabled
-          if (config.appsec.enabled) {
-            require('./appsec').enable(config)
-          }
+      log.use(config.logger)
+      log.toggle(config.debug, config.logLevel, this)
 
-          this._tracer = new DatadogTracer(config)
-          this._instrumenter.enable(config)
-          setStartupLogInstrumenter(this._instrumenter)
+      if (config.profiling.enabled) {
+        // do not stop tracer initialization if the profiler fails to be imported
+        try {
+          const profiler = require('./profiler')
+          profiler.start(config)
+        } catch (e) {
+          log.error(e)
         }
-      } catch (e) {
-        log.error(e)
       }
+
+      if (config.runtimeMetrics) {
+        metrics.start(config)
+      }
+
+      if (config.tracing) {
+        // dirty require for now so zero appsec code is executed unless explicitly enabled
+        if (config.appsec.enabled) {
+          require('./appsec').enable(config)
+        }
+
+        this._tracer = new DatadogTracer(config)
+        this._instrumenter.enable(config)
+        this._pluginManager.configure(config)
+        setStartupLogInstrumenter(this._instrumenter)
+      }
+    } catch (e) {
+      log.error(e)
     }
 
     return this
   }
 
   use () {
-    this._instrumenter.use.apply(this._instrumenter, arguments)
+    this._instrumenter.use(...arguments)
+    this._pluginManager.configurePlugin(...arguments)
     return this
   }
 
@@ -111,11 +120,6 @@ class Tracer extends BaseTracer {
     return this._tracer.extract.apply(this._tracer, arguments)
   }
 
-  scopeManager () {
-    this._deprecate('scopeManager')
-    return this._tracer.scopeManager.apply(this._tracer, arguments)
-  }
-
   scope () {
     return this._tracer.scope.apply(this._tracer, arguments)
   }

package/packages/dd-trace/src/ritm.js

@@ -4,32 +4,56 @@ const path = require('path')
 const Module = require('module')
 const parse = require('module-details-from-path')
 
-module.exports = function hook (modules, onrequire) {
-  if (!hook.orig) {
-    hook.orig = Module.prototype.require
+const origRequire = Module.prototype.require
 
-    Module.prototype.require = function (request) {
-      return hook.require.apply(this, arguments)
-    }
+// derived from require-in-the-middle@3 with tweaks
+
+module.exports = Hook
+
+Hook.reset = function () {
+  Module.prototype.require = origRequire
+}
+
+function Hook (modules, options, onrequire) {
+  if (!(this instanceof Hook)) return new Hook(modules, options, onrequire)
+  if (typeof modules === 'function') {
+    onrequire = modules
+    modules = null
+    options = {}
+  } else if (typeof options === 'function') {
+    onrequire = options
+    options = {}
   }
 
-  hook.cache = {}
+  options = options || {}
+
+  this.cache = {}
+  this._unhooked = false
+  this._origRequire = Module.prototype.require
 
+  const self = this
   const patching = {}
 
-  hook.require = function (request) {
+  this._require = Module.prototype.require = function (request) {
+    if (self._unhooked) {
+      // if the patched require function could not be removed because
+      // someone else patched it after it was patched here, we just
+      // abort and pass the request onwards to the original require
+      return self._origRequire.apply(this, arguments)
+    }
+
     const filename = Module._resolveFilename(request, this)
     const core = filename.indexOf(path.sep) === -1
     let name, basedir
 
     // return known patched modules immediately
-    if (hook.cache.hasOwnProperty(filename)) {
+    if (self.cache.hasOwnProperty(filename)) {
       // require.cache was potentially altered externally
-      if (require.cache[filename] && require.cache[filename].exports !== hook.cache[filename].original) {
+      if (require.cache[filename] && require.cache[filename].exports !== self.cache[filename].original) {
         return require.cache[filename].exports
       }
 
-      return hook.cache[filename].exports
+      return self.cache[filename].exports
     }
 
     // Check if this module has a patcher in-progress already.
@@ -39,7 +63,7 @@ module.exports = function hook (modules, onrequire) {
       patching[filename] = true
     }
 
-    const exports = hook.orig.apply(this, arguments)
+    const exports = self._origRequire.apply(this, arguments)
 
     // If it's already patched, just return it as-is.
     if (patched) return exports
@@ -75,10 +99,10 @@ module.exports = function hook (modules, onrequire) {
 
     // ensure that the cache entry is assigned a value before calling
     // onrequire, in case calling onrequire requires the same module.
-    hook.cache[filename] = { exports }
-    hook.cache[filename].original = exports
-    hook.cache[filename].exports = onrequire(exports, name, basedir)
+    self.cache[filename] = { exports }
+    self.cache[filename].original = exports
+    self.cache[filename].exports = onrequire(exports, name, basedir)
 
-    return hook.cache[filename].exports
+    return self.cache[filename].exports
   }
 }

package/packages/dd-trace/src/span_processor.js

@@ -16,13 +16,6 @@ class SpanProcessor {
 
     if (trace.started.length === trace.finished.length) {
       this._prioritySampler.sample(spanContext)
-
-      if (spanContext._traceFlags.sampled === false) {
-        log.debug(() => `Dropping trace due to user configured filtering: ${trace.started}`)
-        this._erase(trace)
-        return
-      }
-
       const formattedSpans = trace.finished.map(format)
       this._exporter.export(formattedSpans)
       this._erase(trace)

package/packages/dd-trace/src/tracer.js

@@ -2,8 +2,8 @@
 
 const Tracer = require('./opentracing/tracer')
 const tags = require('../../../ext/tags')
-const ScopeManager = require('./scope/noop/scope_manager')
 const Scope = require('./scope')
+const { storage } = require('../../datadog-core')
 const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
 
@@ -16,7 +16,6 @@ class DatadogTracer extends Tracer {
   constructor (config) {
     super(config)
 
-    this._scopeManager = new ScopeManager()
     this._scope = new Scope()
     setStartupLogConfig(config)
   }
@@ -68,6 +67,10 @@ class DatadogTracer extends Tracer {
     const tracer = this
 
     return function () {
+      const store = storage.getStore()
+
+      if (store && store.noop) return fn.apply(this, arguments)
+
       let optionsObj = options
       if (typeof optionsObj === 'function' && typeof fn === 'function') {
         optionsObj = optionsObj.apply(this, arguments)
@@ -100,10 +103,6 @@ class DatadogTracer extends Tracer {
     this._exporter.setUrl(url)
   }
 
-  scopeManager () {
-    return this._scopeManager
-  }
-
   scope () {
     return this._scope
   }

package/scripts/install_plugin_modules.js

@@ -4,9 +4,11 @@ const fs = require('fs')
 const path = require('path')
 const crypto = require('crypto')
 const semver = require('semver')
+const proxyquire = require('proxyquire')
 const exec = require('./helpers/exec')
 const childProcess = require('child_process')
 const plugins = require('../packages/dd-trace/src/plugins')
+const Plugin = require('../packages/dd-trace/src/plugins/plugin')
 const externals = require('../packages/dd-trace/test/plugins/externals')
 
 const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json')
@@ -23,6 +25,13 @@ Object.keys(externals).forEach(external => externals[external].forEach(thing =>
   }
 }))
 
+fs.readdirSync(path.join(__dirname, '../packages/datadog-instrumentations/src'))
+  .filter(file => file.endsWith('js'))
+  .forEach(file => {
+    file = file.replace('.js', '')
+    plugins[file] = { name: file, prototype: Object.create(Plugin.prototype) }
+  })
+
 run()
 
 async function run () {
@@ -42,7 +51,27 @@ async function assertVersions () {
   }
 
   const internals = names
-    .map(key => plugins[key])
+    .map(key => {
+      const plugin = plugins[key]
+      if (plugin.prototype instanceof Plugin) {
+        const instrumentations = []
+        const instrument = {
+          addHook (instrumentation) {
+            instrumentations.push(instrumentation)
+          }
+        }
+        const instPath = path.join(
+          __dirname,
+          `../packages/datadog-instrumentations/src/${plugin.name}.js`
+        )
+        proxyquire.noPreserveCache()(instPath, {
+          './helpers/instrument': instrument
+        })
+        return instrumentations
+      } else {
+        return plugin
+      }
+    })
     .reduce((prev, next) => prev.concat(next), [])
 
   for (const inst of internals) {