dd-trace 2.0.0-appsec-beta.3 → 2.0.0

package/packages/dd-trace/src/appsec/reporter.js

@@ -1,6 +1,10 @@
 'use strict'
 
 const addresses = require('./addresses')
+const Limiter = require('../rate_limiter')
+
+// default limiter, configurable with setRateLimit()
+let limiter = new Limiter(100)
 
 const REQUEST_HEADERS_PASSLIST = [
   'accept',
@@ -36,7 +40,6 @@ function resolveHTTPRequest (context) {
   const headers = context.resolve(addresses.HTTP_INCOMING_HEADERS)
 
   return {
-    // route: context.resolve(addresses.HTTP_INCOMING_ROUTE),
     remote_ip: context.resolve(addresses.HTTP_INCOMING_REMOTE_IP),
     headers: filterHeaders(headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
   }
@@ -48,6 +51,7 @@ function resolveHTTPResponse (context) {
   const headers = context.resolve(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
 
   return {
+    endpoint: context.resolve(addresses.HTTP_INCOMING_ENDPOINT),
     headers: filterHeaders(headers, RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
   }
 }
@@ -61,16 +65,16 @@ function filterHeaders (headers, passlist, prefix) {
     const headerName = passlist[i]
 
     if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = headers[headerName].toString()
+      result[`${prefix}${formatHeaderName(headerName)}`] = headers[headerName] + ''
     }
   }
 
   return result
 }
 
+// TODO: this can be precomputed at start time
 function formatHeaderName (name) {
   return name
-    .toString()
     .trim()
     .slice(0, 200)
     .replace(/[^a-zA-Z0-9_\-:/]/g, '_')
@@ -85,10 +89,14 @@ function reportAttack (attackData, store) {
   const currentTags = topSpan.context()._tags
 
   const newTags = {
-    'appsec.event': true,
-    'manual.keep': undefined
+    'appsec.event': 'true'
+  }
+
+  if (limiter.isAllowed()) {
+    newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp
   }
 
+  // TODO: maybe add this to format.js later (to take decision as late as possible)
   if (!currentTags['_dd.origin']) {
     newTags['_dd.origin'] = 'appsec'
   }
@@ -115,8 +123,6 @@ function reportAttack (attackData, store) {
     }
 
     newTags['network.client.ip'] = resolvedRequest.remote_ip
-
-    // newTags['http.endpoint'] = resolvedRequest.route
   }
 
   topSpan.addTags(newTags)
@@ -126,9 +132,19 @@ function finishAttacks (req, context) {
   const topSpan = req && req._datadog && req._datadog.span
   if (!topSpan || !context) return false
 
-  const resolvedReponse = resolveHTTPResponse(context)
+  const resolvedResponse = resolveHTTPResponse(context)
+
+  const newTags = resolvedResponse.headers
+
+  if (resolvedResponse.endpoint) {
+    newTags['http.endpoint'] = resolvedResponse.endpoint
+  }
+
+  topSpan.addTags(newTags)
+}
 
-  topSpan.addTags(resolvedReponse.headers)
+function setRateLimit (rateLimit) {
+  limiter = new Limiter(rateLimit)
 }
 
 module.exports = {
@@ -137,5 +153,6 @@ module.exports = {
   filterHeaders,
   formatHeaderName,
   reportAttack,
-  finishAttacks
+  finishAttacks,
+  setRateLimit
 }

package/packages/dd-trace/src/config.js

@@ -24,9 +24,12 @@ class Config {
     tagger.add(this.tags, process.env.DD_TRACE_GLOBAL_TAGS)
     tagger.add(this.tags, options.tags)
 
-    // Temporary disabled
+    const DD_TRACING_ENABLED = coalesce(
+      process.env.DD_TRACING_ENABLED,
+      true
+    )
     const DD_PROFILING_ENABLED = coalesce(
-      options.profiling,
+      options.profiling, // TODO: remove when enabled by default
       process.env.DD_EXPERIMENTAL_PROFILING_ENABLED,
       process.env.DD_PROFILING_ENABLED,
       false
@@ -42,7 +45,7 @@ class Config {
       false
     )
     const DD_RUNTIME_METRICS_ENABLED = coalesce(
-      options.runtimeMetrics,
+      options.runtimeMetrics, // TODO: remove when enabled by default
       process.env.DD_RUNTIME_METRICS_ENABLED,
       false
     )
@@ -85,15 +88,9 @@ class Config {
     const DD_TRACE_STARTUP_LOGS = coalesce(
       options.startupLogs,
       process.env.DD_TRACE_STARTUP_LOGS,
-      true
-    )
-    const DD_TRACE_ENABLED = coalesce(
-      options.enabled,
-      process.env.DD_TRACE_ENABLED,
-      true
+      false
     )
     const DD_TRACE_DEBUG = coalesce(
-      options.debug,
       process.env.DD_TRACE_DEBUG,
       false
     )
@@ -121,39 +118,46 @@ class Config {
       process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED,
       false
     )
-    const DD_TRACE_INTERNAL_ERRORS_ENABLED = coalesce(
-      options.experimental && options.experimental.internalErrors,
-      process.env.DD_TRACE_EXPERIMENTAL_INTERNAL_ERRORS_ENABLED,
-      false
-    )
-    // TODO(simon-id): add documentation for appsec config when we release it in public beta
+
+    let appsec = options.appsec || (options.experimental && options.experimental.appsec)
+
     const DD_APPSEC_ENABLED = coalesce(
-      options.experimental &&
-        options.experimental.appsec &&
-        (options.experimental.appsec === true || options.experimental.appsec.enabled === true),
-      process.env.DD_EXPERIMENTAL_APPSEC_ENABLED,
+      appsec && (appsec === true || appsec.enabled === true), // TODO: remove when enabled by default
       process.env.DD_APPSEC_ENABLED,
       false
     )
+
+    appsec = appsec || {}
+
     const DD_APPSEC_RULES = coalesce(
-      options.experimental && options.experimental.appsec && options.experimental.appsec.rules,
+      appsec.rules,
       process.env.DD_APPSEC_RULES,
       path.join(__dirname, 'appsec', 'recommended.json')
     )
+    const DD_APPSEC_TRACE_RATE_LIMIT = coalesce(
+      appsec.rateLimit,
+      process.env.DD_APPSEC_TRACE_RATE_LIMIT,
+      100
+    )
 
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
 
     Object.assign(sampler, {
-      sampleRate: coalesce(ingestion.sampleRate, sampler.sampleRate, process.env.DD_TRACE_SAMPLE_RATE),
+      sampleRate: coalesce(
+        options.sampleRate,
+        ingestion.sampleRate,
+        sampler.sampleRate,
+        process.env.DD_TRACE_SAMPLE_RATE
+      ),
       rateLimit: coalesce(ingestion.rateLimit, sampler.rateLimit, process.env.DD_TRACE_RATE_LIMIT)
     })
 
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
     const defaultFlushInterval = inAWSLambda ? 0 : 2000
 
-    this.enabled = isTrue(DD_TRACE_ENABLED)
+    this.tracing = !isFalse(DD_TRACING_ENABLED)
     this.debug = isTrue(DD_TRACE_DEBUG)
     this.logInjection = isTrue(DD_LOGS_INJECTION)
     this.env = DD_ENV
@@ -162,7 +166,7 @@ class Config {
     this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname)
     this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port))
     this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval)
-    this.sampleRate = coalesce(Math.min(Math.max(options.sampleRate, 0), 1), 1)
+    this.sampleRate = coalesce(Math.min(Math.max(sampler.sampleRate, 0), 1), 1)
     this.logger = options.logger
     this.plugins = !!coalesce(options.plugins, true)
     this.service = DD_SERVICE
@@ -180,8 +184,7 @@ class Config {
       runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED),
       exporter: DD_TRACE_EXPORTER,
       enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED),
-      sampler,
-      internalErrors: isTrue(DD_TRACE_INTERNAL_ERRORS_ENABLED)
+      sampler
     }
     this.reportHostname = isTrue(coalesce(options.reportHostname, process.env.DD_TRACE_REPORT_HOSTNAME, false))
     this.scope = process.env.DD_TRACE_SCOPE
@@ -200,7 +203,8 @@ class Config {
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
-      rules: DD_APPSEC_RULES
+      rules: DD_APPSEC_RULES,
+      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT
     }
 
     tagger.add(this.tags, {

package/packages/dd-trace/src/constants.js

@@ -1,14 +1,17 @@
 'use strict'
 
 module.exports = {
-  SAMPLE_RATE_METRIC_KEY: '_sample_rate',
   SAMPLING_PRIORITY_KEY: '_sampling_priority_v1',
   ANALYTICS_KEY: '_dd1.sr.eausr',
   ORIGIN_KEY: '_dd.origin',
   HOSTNAME_KEY: '_dd.hostname',
-  REFERENCE_NOOP: 'noop',
   SAMPLING_RULE_DECISION: '_dd.rule_psr',
   SAMPLING_LIMIT_DECISION: '_dd.limit_psr',
   SAMPLING_AGENT_DECISION: '_dd.agent_psr',
-  DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent'
+  SAMPLING_MECHANISM_DEFAULT: 0,
+  SAMPLING_MECHANISM_AGENT: 1,
+  SAMPLING_MECHANISM_RULE: 3,
+  SAMPLING_MECHANISM_MANUAL: 4,
+  DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent',
+  UPSTREAM_SERVICES_KEY: '_dd.p.upstream_services'
 }

package/packages/dd-trace/src/exporters/agent/request.js

@@ -4,12 +4,17 @@ const http = require('http')
 const https = require('https')
 const docker = require('./docker')
 const log = require('../../log')
+const { storage } = require('../../../../datadog-core')
 
 const httpAgent = new http.Agent({ keepAlive: true })
 const httpsAgent = new https.Agent({ keepAlive: true })
 const containerId = docker.id()
 
 function retriableRequest (options, callback, client, data) {
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+
   const req = client.request(options, res => {
     let data = ''
 
@@ -29,6 +34,9 @@ function retriableRequest (options, callback, client, data) {
   })
   req.setTimeout(options.timeout, req.abort)
   data.forEach(buffer => req.write(buffer))
+
+  storage.enterWith(store)
+
   return req
 }
 

package/packages/dd-trace/src/format.js

@@ -2,7 +2,6 @@
 
 const constants = require('./constants')
 const tags = require('../../../ext/tags')
-const log = require('./log')
 const id = require('./id')
 const { isError } = require('./util')
 
@@ -25,6 +24,7 @@ function format (span) {
 
   extractError(formatted, span)
   extractRootTags(formatted, span)
+  extractChunkTags(formatted, span)
   extractTags(formatted, span)
 
   return formatted
@@ -37,8 +37,8 @@ function formatSpan (span) {
     trace_id: spanContext._traceId,
     span_id: spanContext._spanId,
     parent_id: spanContext._parentId || id('0'),
-    name: serialize(spanContext._name),
-    resource: serialize(spanContext._name),
+    name: String(spanContext._name),
+    resource: String(spanContext._name),
     error: 0,
     meta: {},
     metrics: {},
@@ -53,7 +53,6 @@ function extractTags (trace, span) {
   const tags = context._tags
   const hostname = context._hostname
   const priority = context._sampling.priority
-  const internalErrors = span.tracer()._internalErrors
 
   if (tags['span.kind'] && tags['span.kind'] !== 'internal') {
     addTag({}, trace.metrics, MEASURED, 1)
@@ -75,7 +74,7 @@ function extractTags (trace, span) {
         addTag({}, trace.metrics, tag, tags[tag] === undefined || tags[tag] ? 1 : 0)
         break
       case 'error':
-        if (tags[tag] && (context._name !== 'fs.operation' || internalErrors)) {
+        if (tags[tag] && (context._name !== 'fs.operation')) {
           trace.error = 1
         }
         break
@@ -83,7 +82,7 @@ function extractTags (trace, span) {
       case 'error.msg':
       case 'error.stack':
         // HACK: remove when implemented in the backend
-        if (context._name !== 'fs.operation' || internalErrors) {
+        if (context._name !== 'fs.operation') {
           trace.error = 1
         }
       default: // eslint-disable-line no-fallthrough
@@ -112,6 +111,17 @@ function extractRootTags (trace, span) {
   addTag({}, trace.metrics, SAMPLING_AGENT_DECISION, context._trace[SAMPLING_AGENT_DECISION])
 }
 
+function extractChunkTags (trace, span) {
+  const context = span.context()
+  const isLocalRoot = span === context._trace.started[0]
+
+  if (!isLocalRoot) return
+
+  for (const key in context._trace.tags) {
+    addTag(trace.meta, trace.metrics, key, context._trace.tags[key])
+  }
+}
+
 function extractError (trace, span) {
   const error = span.context()._tags['error']
   if (isError(error)) {
@@ -121,7 +131,7 @@ function extractError (trace, span) {
   }
 }
 
-function addTag (meta, metrics, key, value, seen) {
+function addTag (meta, metrics, key, value, nested) {
   switch (typeof value) {
     case 'string':
       if (!value) break
@@ -131,6 +141,9 @@ function addTag (meta, metrics, key, value, seen) {
       if (isNaN(value)) break
       metrics[key] = value
       break
+    case 'boolean':
+      metrics[key] = value ? 1 : 0
+      break
     case 'undefined':
       break
     case 'object':
@@ -139,41 +152,15 @@ function addTag (meta, metrics, key, value, seen) {
       // Special case for Node.js Buffer and URL
       if (isNodeBuffer(value) || isUrl(value)) {
         metrics[key] = value.toString()
-        break
-      }
+      } else if (!Array.isArray(value) && !nested) {
+        for (const prop in value) {
+          if (!value.hasOwnProperty(prop)) continue
 
-      if (!Array.isArray(value)) {
-        addObjectTag(meta, metrics, key, value, seen)
-        break
+          addTag(meta, metrics, `${key}.${prop}`, value[prop], true)
+        }
       }
 
-    default: // eslint-disable-line no-fallthrough
-      addTag(meta, metrics, key, serialize(value))
-  }
-}
-
-function addObjectTag (meta, metrics, key, value, seen) {
-  seen = seen || []
-
-  if (~seen.indexOf(value)) {
-    meta[key] = '[Circular]'
-    return
-  }
-
-  seen.push(value)
-
-  for (const prop in value) {
-    addTag(meta, metrics, `${key}.${prop}`, value[prop], seen)
-  }
-
-  seen.pop()
-}
-
-function serialize (obj) {
-  try {
-    return obj && typeof obj.toString !== 'function' ? JSON.stringify(obj) : String(obj)
-  } catch (e) {
-    log.error(e)
+      break
   }
 }
 

package/packages/dd-trace/src/instrumenter.js

@@ -6,6 +6,7 @@ const metrics = require('./metrics')
 const Loader = require('./loader')
 const { isTrue } = require('./util')
 const plugins = require('./plugins')
+const Plugin = require('./plugins/plugin')
 
 const disabledPlugins = process.env.DD_TRACE_DISABLED_PLUGINS
 
@@ -42,6 +43,9 @@ class Instrumenter {
   }
 
   use (name, config) {
+    if (typeof name !== 'string') return
+    const plugin = plugins[name.toLowerCase()]
+    if (plugin && plugin.prototype instanceof Plugin) return
     if (typeof config === 'boolean') {
       config = { enabled: config }
     }
@@ -49,7 +53,7 @@ class Instrumenter {
     config = getConfig(name, config)
 
     try {
-      this._set(plugins[name.toLowerCase()], { name, config })
+      this._set(plugin, { name, config })
     } catch (e) {
       log.debug(`Could not find a plugin named "${name}".`)
     }
@@ -69,6 +73,7 @@ class Instrumenter {
       Object.keys(plugins)
         .filter(name => !this._plugins.has(plugins[name]))
         .forEach(name => {
+          if (plugins[name].prototype instanceof Plugin) return
           const pluginConfig = {}
           if (serviceMapping && serviceMapping[name]) {
             pluginConfig.service = serviceMapping[name]

package/packages/dd-trace/src/log.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const NoopSpan = require('./noop/span')
+const { storage } = require('../../datadog-core')
 
 const _default = {
   debug: msg => console.debug(msg), /* eslint-disable-line no-console */
@@ -45,11 +45,11 @@ function processMsg (msg) {
 }
 
 function withNoop (fn) {
-  if (!log._tracer) {
-    fn()
-  } else {
-    log._tracer.scope().activate(log._noopSpan(), fn)
-  }
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+  fn()
+  storage.enterWith(store)
 }
 
 const log = {
@@ -73,18 +73,9 @@ const log = {
     return this
   },
 
-  _noopSpan () {
-    if (!this.__noopSpan) {
-      this.__noopSpan = new NoopSpan(this._tracer)
-    }
-    return this.__noopSpan
-  },
-
   reset () {
     this._logger = _default
     this._enabled = false
-    delete this._tracer
-    delete this.__noopSpan
     this._deprecate = memoize((code, message) => {
       withNoop(() => this._logger.error(message))
       return this

package/packages/dd-trace/src/noop/span_context.js

@@ -9,7 +9,6 @@ class NoopSpanContext extends DatadogSpanContext {
   constructor (props) {
     super(props)
 
-    this._traceFlags.sampled = false
     this._sampling.priority = USER_REJECT
   }
 }

package/packages/dd-trace/src/noop/tracer.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const Tracer = require('opentracing').Tracer
-const ScopeManager = require('../scope/noop/scope_manager')
 const Scope = require('../noop/scope')
 const Span = require('./span')
 
@@ -9,7 +8,6 @@ class NoopTracer extends Tracer {
   constructor (config) {
     super(config)
 
-    this._scopeManager = new ScopeManager()
     this._scope = new Scope()
     this._span = new Span(this)
   }
@@ -22,10 +20,6 @@ class NoopTracer extends Tracer {
     return fn
   }
 
-  scopeManager () {
-    return this._scopeManager
-  }
-
   scope () {
     return this._scope
   }