dd-trace 2.0.0-appsec-beta.4 → 2.0.1

package/packages/dd-trace/src/appsec/reporter.js

@@ -1,6 +1,10 @@
 'use strict'
 
 const addresses = require('./addresses')
+const Limiter = require('../rate_limiter')
+
+// default limiter, configurable with setRateLimit()
+let limiter = new Limiter(100)
 
 const REQUEST_HEADERS_PASSLIST = [
   'accept',
@@ -36,7 +40,6 @@ function resolveHTTPRequest (context) {
   const headers = context.resolve(addresses.HTTP_INCOMING_HEADERS)
 
   return {
-    // route: context.resolve(addresses.HTTP_INCOMING_ROUTE),
     remote_ip: context.resolve(addresses.HTTP_INCOMING_REMOTE_IP),
     headers: filterHeaders(headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
   }
@@ -48,6 +51,7 @@ function resolveHTTPResponse (context) {
   const headers = context.resolve(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
 
   return {
+    endpoint: context.resolve(addresses.HTTP_INCOMING_ENDPOINT),
     headers: filterHeaders(headers, RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
   }
 }
@@ -61,16 +65,16 @@ function filterHeaders (headers, passlist, prefix) {
     const headerName = passlist[i]
 
     if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = headers[headerName].toString()
+      result[`${prefix}${formatHeaderName(headerName)}`] = headers[headerName] + ''
     }
   }
 
   return result
 }
 
+// TODO: this can be precomputed at start time
 function formatHeaderName (name) {
   return name
-    .toString()
     .trim()
     .slice(0, 200)
     .replace(/[^a-zA-Z0-9_\-:/]/g, '_')
@@ -85,10 +89,14 @@ function reportAttack (attackData, store) {
   const currentTags = topSpan.context()._tags
 
   const newTags = {
-    'appsec.event': true,
-    'manual.keep': undefined
+    'appsec.event': 'true'
+  }
+
+  if (limiter.isAllowed()) {
+    newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp
   }
 
+  // TODO: maybe add this to format.js later (to take decision as late as possible)
   if (!currentTags['_dd.origin']) {
     newTags['_dd.origin'] = 'appsec'
   }
@@ -115,8 +123,6 @@ function reportAttack (attackData, store) {
     }
 
     newTags['network.client.ip'] = resolvedRequest.remote_ip
-
-    // newTags['http.endpoint'] = resolvedRequest.route
   }
 
   topSpan.addTags(newTags)
@@ -126,9 +132,19 @@ function finishAttacks (req, context) {
   const topSpan = req && req._datadog && req._datadog.span
   if (!topSpan || !context) return false
 
-  const resolvedReponse = resolveHTTPResponse(context)
+  const resolvedResponse = resolveHTTPResponse(context)
+
+  const newTags = resolvedResponse.headers
+
+  if (resolvedResponse.endpoint) {
+    newTags['http.endpoint'] = resolvedResponse.endpoint
+  }
+
+  topSpan.addTags(newTags)
+}
 
-  topSpan.addTags(resolvedReponse.headers)
+function setRateLimit (rateLimit) {
+  limiter = new Limiter(rateLimit)
 }
 
 module.exports = {
@@ -137,5 +153,6 @@ module.exports = {
   filterHeaders,
   formatHeaderName,
   reportAttack,
-  finishAttacks
+  finishAttacks,
+  setRateLimit
 }

package/packages/dd-trace/src/config.js

@@ -24,9 +24,12 @@ class Config {
     tagger.add(this.tags, process.env.DD_TRACE_GLOBAL_TAGS)
     tagger.add(this.tags, options.tags)
 
-    // Temporary disabled
+    const DD_TRACING_ENABLED = coalesce(
+      process.env.DD_TRACING_ENABLED,
+      true
+    )
     const DD_PROFILING_ENABLED = coalesce(
-      options.profiling,
+      options.profiling, // TODO: remove when enabled by default
       process.env.DD_EXPERIMENTAL_PROFILING_ENABLED,
       process.env.DD_PROFILING_ENABLED,
       false
@@ -42,7 +45,7 @@ class Config {
       false
     )
     const DD_RUNTIME_METRICS_ENABLED = coalesce(
-      options.runtimeMetrics,
+      options.runtimeMetrics, // TODO: remove when enabled by default
       process.env.DD_RUNTIME_METRICS_ENABLED,
       false
     )
@@ -85,15 +88,9 @@ class Config {
     const DD_TRACE_STARTUP_LOGS = coalesce(
       options.startupLogs,
       process.env.DD_TRACE_STARTUP_LOGS,
-      true
-    )
-    const DD_TRACE_ENABLED = coalesce(
-      options.enabled,
-      process.env.DD_TRACE_ENABLED,
-      true
+      false
     )
     const DD_TRACE_DEBUG = coalesce(
-      options.debug,
       process.env.DD_TRACE_DEBUG,
       false
     )
@@ -121,39 +118,46 @@ class Config {
       process.env.DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED,
       false
     )
-    const DD_TRACE_INTERNAL_ERRORS_ENABLED = coalesce(
-      options.experimental && options.experimental.internalErrors,
-      process.env.DD_TRACE_EXPERIMENTAL_INTERNAL_ERRORS_ENABLED,
-      false
-    )
-    // TODO(simon-id): add documentation for appsec config when we release it in public beta
+
+    let appsec = options.appsec || (options.experimental && options.experimental.appsec)
+
     const DD_APPSEC_ENABLED = coalesce(
-      options.experimental &&
-        options.experimental.appsec &&
-        (options.experimental.appsec === true || options.experimental.appsec.enabled === true),
-      process.env.DD_EXPERIMENTAL_APPSEC_ENABLED,
+      appsec && (appsec === true || appsec.enabled === true), // TODO: remove when enabled by default
       process.env.DD_APPSEC_ENABLED,
       false
     )
+
+    appsec = appsec || {}
+
     const DD_APPSEC_RULES = coalesce(
-      options.experimental && options.experimental.appsec && options.experimental.appsec.rules,
+      appsec.rules,
       process.env.DD_APPSEC_RULES,
       path.join(__dirname, 'appsec', 'recommended.json')
     )
+    const DD_APPSEC_TRACE_RATE_LIMIT = coalesce(
+      appsec.rateLimit,
+      process.env.DD_APPSEC_TRACE_RATE_LIMIT,
+      100
+    )
 
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
 
     Object.assign(sampler, {
-      sampleRate: coalesce(ingestion.sampleRate, sampler.sampleRate, process.env.DD_TRACE_SAMPLE_RATE),
+      sampleRate: coalesce(
+        options.sampleRate,
+        ingestion.sampleRate,
+        sampler.sampleRate,
+        process.env.DD_TRACE_SAMPLE_RATE
+      ),
       rateLimit: coalesce(ingestion.rateLimit, sampler.rateLimit, process.env.DD_TRACE_RATE_LIMIT)
     })
 
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
     const defaultFlushInterval = inAWSLambda ? 0 : 2000
 
-    this.enabled = isTrue(DD_TRACE_ENABLED)
+    this.tracing = !isFalse(DD_TRACING_ENABLED)
     this.debug = isTrue(DD_TRACE_DEBUG)
     this.logInjection = isTrue(DD_LOGS_INJECTION)
     this.env = DD_ENV
@@ -162,7 +166,7 @@ class Config {
     this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname)
     this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port))
     this.flushInterval = coalesce(parseInt(options.flushInterval, 10), defaultFlushInterval)
-    this.sampleRate = coalesce(Math.min(Math.max(options.sampleRate, 0), 1), 1)
+    this.sampleRate = coalesce(Math.min(Math.max(sampler.sampleRate, 0), 1), 1)
     this.logger = options.logger
     this.plugins = !!coalesce(options.plugins, true)
     this.service = DD_SERVICE
@@ -180,8 +184,7 @@ class Config {
       runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED),
       exporter: DD_TRACE_EXPORTER,
       enableGetRumData: isTrue(DD_TRACE_GET_RUM_DATA_ENABLED),
-      sampler,
-      internalErrors: isTrue(DD_TRACE_INTERNAL_ERRORS_ENABLED)
+      sampler
     }
     this.reportHostname = isTrue(coalesce(options.reportHostname, process.env.DD_TRACE_REPORT_HOSTNAME, false))
     this.scope = process.env.DD_TRACE_SCOPE
@@ -200,7 +203,8 @@ class Config {
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
-      rules: DD_APPSEC_RULES
+      rules: DD_APPSEC_RULES,
+      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT
     }
 
     tagger.add(this.tags, {

package/packages/dd-trace/src/constants.js

@@ -1,12 +1,10 @@
 'use strict'
 
 module.exports = {
-  SAMPLE_RATE_METRIC_KEY: '_sample_rate',
   SAMPLING_PRIORITY_KEY: '_sampling_priority_v1',
   ANALYTICS_KEY: '_dd1.sr.eausr',
   ORIGIN_KEY: '_dd.origin',
   HOSTNAME_KEY: '_dd.hostname',
-  REFERENCE_NOOP: 'noop',
   SAMPLING_RULE_DECISION: '_dd.rule_psr',
   SAMPLING_LIMIT_DECISION: '_dd.limit_psr',
   SAMPLING_AGENT_DECISION: '_dd.agent_psr',

package/packages/dd-trace/src/exporters/agent/request.js

@@ -4,12 +4,17 @@ const http = require('http')
 const https = require('https')
 const docker = require('./docker')
 const log = require('../../log')
+const { storage } = require('../../../../datadog-core')
 
 const httpAgent = new http.Agent({ keepAlive: true })
 const httpsAgent = new https.Agent({ keepAlive: true })
 const containerId = docker.id()
 
 function retriableRequest (options, callback, client, data) {
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+
   const req = client.request(options, res => {
     let data = ''
 
@@ -29,6 +34,9 @@ function retriableRequest (options, callback, client, data) {
   })
   req.setTimeout(options.timeout, req.abort)
   data.forEach(buffer => req.write(buffer))
+
+  storage.enterWith(store)
+
   return req
 }
 

package/packages/dd-trace/src/format.js

@@ -2,7 +2,6 @@
 
 const constants = require('./constants')
 const tags = require('../../../ext/tags')
-const log = require('./log')
 const id = require('./id')
 const { isError } = require('./util')
 
@@ -38,8 +37,8 @@ function formatSpan (span) {
     trace_id: spanContext._traceId,
     span_id: spanContext._spanId,
     parent_id: spanContext._parentId || id('0'),
-    name: serialize(spanContext._name),
-    resource: serialize(spanContext._name),
+    name: String(spanContext._name),
+    resource: String(spanContext._name),
     error: 0,
     meta: {},
     metrics: {},
@@ -54,7 +53,6 @@ function extractTags (trace, span) {
   const tags = context._tags
   const hostname = context._hostname
   const priority = context._sampling.priority
-  const internalErrors = span.tracer()._internalErrors
 
   if (tags['span.kind'] && tags['span.kind'] !== 'internal') {
     addTag({}, trace.metrics, MEASURED, 1)
@@ -76,7 +74,7 @@ function extractTags (trace, span) {
         addTag({}, trace.metrics, tag, tags[tag] === undefined || tags[tag] ? 1 : 0)
         break
       case 'error':
-        if (tags[tag] && (context._name !== 'fs.operation' || internalErrors)) {
+        if (tags[tag] && (context._name !== 'fs.operation')) {
           trace.error = 1
         }
         break
@@ -84,7 +82,7 @@ function extractTags (trace, span) {
       case 'error.msg':
       case 'error.stack':
         // HACK: remove when implemented in the backend
-        if (context._name !== 'fs.operation' || internalErrors) {
+        if (context._name !== 'fs.operation') {
           trace.error = 1
         }
       default: // eslint-disable-line no-fallthrough
@@ -133,7 +131,7 @@ function extractError (trace, span) {
   }
 }
 
-function addTag (meta, metrics, key, value, seen) {
+function addTag (meta, metrics, key, value, nested) {
   switch (typeof value) {
     case 'string':
       if (!value) break
@@ -143,6 +141,9 @@ function addTag (meta, metrics, key, value, seen) {
       if (isNaN(value)) break
       metrics[key] = value
       break
+    case 'boolean':
+      metrics[key] = value ? 1 : 0
+      break
     case 'undefined':
       break
     case 'object':
@@ -151,41 +152,15 @@ function addTag (meta, metrics, key, value, seen) {
       // Special case for Node.js Buffer and URL
       if (isNodeBuffer(value) || isUrl(value)) {
         metrics[key] = value.toString()
-        break
-      }
+      } else if (!Array.isArray(value) && !nested) {
+        for (const prop in value) {
+          if (!value.hasOwnProperty(prop)) continue
 
-      if (!Array.isArray(value)) {
-        addObjectTag(meta, metrics, key, value, seen)
-        break
+          addTag(meta, metrics, `${key}.${prop}`, value[prop], true)
+        }
       }
 
-    default: // eslint-disable-line no-fallthrough
-      addTag(meta, metrics, key, serialize(value))
-  }
-}
-
-function addObjectTag (meta, metrics, key, value, seen) {
-  seen = seen || []
-
-  if (~seen.indexOf(value)) {
-    meta[key] = '[Circular]'
-    return
-  }
-
-  seen.push(value)
-
-  for (const prop in value) {
-    addTag(meta, metrics, `${key}.${prop}`, value[prop], seen)
-  }
-
-  seen.pop()
-}
-
-function serialize (obj) {
-  try {
-    return obj && typeof obj.toString !== 'function' ? JSON.stringify(obj) : String(obj)
-  } catch (e) {
-    log.error(e)
+      break
   }
 }
 

package/packages/dd-trace/src/log.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const NoopSpan = require('./noop/span')
+const { storage } = require('../../datadog-core')
 
 const _default = {
   debug: msg => console.debug(msg), /* eslint-disable-line no-console */
@@ -45,11 +45,11 @@ function processMsg (msg) {
 }
 
 function withNoop (fn) {
-  if (!log._tracer) {
-    fn()
-  } else {
-    log._tracer.scope().activate(log._noopSpan(), fn)
-  }
+  const store = storage.getStore()
+
+  storage.enterWith({ noop: true })
+  fn()
+  storage.enterWith(store)
 }
 
 const log = {
@@ -73,18 +73,9 @@ const log = {
     return this
   },
 
-  _noopSpan () {
-    if (!this.__noopSpan) {
-      this.__noopSpan = new NoopSpan(this._tracer)
-    }
-    return this.__noopSpan
-  },
-
   reset () {
     this._logger = _default
     this._enabled = false
-    delete this._tracer
-    delete this.__noopSpan
     this._deprecate = memoize((code, message) => {
       withNoop(() => this._logger.error(message))
       return this

package/packages/dd-trace/src/noop/span_context.js

@@ -9,7 +9,6 @@ class NoopSpanContext extends DatadogSpanContext {
   constructor (props) {
     super(props)
 
-    this._traceFlags.sampled = false
     this._sampling.priority = USER_REJECT
   }
 }

package/packages/dd-trace/src/noop/tracer.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const Tracer = require('opentracing').Tracer
-const ScopeManager = require('../scope/noop/scope_manager')
 const Scope = require('../noop/scope')
 const Span = require('./span')
 
@@ -9,7 +8,6 @@ class NoopTracer extends Tracer {
   constructor (config) {
     super(config)
 
-    this._scopeManager = new ScopeManager()
     this._scope = new Scope()
     this._span = new Span(this)
   }
@@ -22,10 +20,6 @@ class NoopTracer extends Tracer {
     return fn
   }
 
-  scopeManager () {
-    return this._scopeManager
-  }
-
   scope () {
     return this._scope
   }

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -3,14 +3,14 @@
 const pick = require('lodash.pick')
 const id = require('../../id')
 const DatadogSpanContext = require('../span_context')
-const NoopSpanContext = require('../../noop/span_context')
 const log = require('../../log')
 
+const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority')
+
 const traceKey = 'x-datadog-trace-id'
 const spanKey = 'x-datadog-parent-id'
 const originKey = 'x-datadog-origin'
 const samplingKey = 'x-datadog-sampling-priority'
-const sampleKey = 'x-datadog-sampled'
 const tagsKey = 'x-datadog-tags'
 const baggagePrefix = 'ot-baggage-'
 const b3TraceKey = 'x-b3-traceid'
@@ -36,7 +36,6 @@ class TextMapPropagator {
   inject (spanContext, carrier) {
     carrier[traceKey] = spanContext.toTraceId()
     carrier[spanKey] = spanContext.toSpanId()
-    carrier[sampleKey] = spanContext._traceFlags.sampled ? '1' : '0'
 
     this._injectOrigin(spanContext, carrier)
     this._injectSamplingPriority(spanContext, carrier)
@@ -52,11 +51,6 @@ class TextMapPropagator {
 
     if (!spanContext) return spanContext
 
-    this._extractOrigin(carrier, spanContext)
-    this._extractBaggageItems(carrier, spanContext)
-    this._extractSamplingPriority(carrier, spanContext)
-    this._extractTags(carrier, spanContext)
-
     log.debug(() => `Extract from carrier: ${JSON.stringify(pick(carrier, logKeys))}.`)
 
     return spanContext
@@ -108,9 +102,9 @@ class TextMapPropagator {
 
     carrier[b3TraceKey] = spanContext._traceId.toString('hex')
     carrier[b3SpanKey] = spanContext._spanId.toString('hex')
-    carrier[b3SampledKey] = spanContext._traceFlags.sampled ? '1' : '0'
+    carrier[b3SampledKey] = spanContext._sampling.priority >= AUTO_KEEP ? '1' : '0'
 
-    if (spanContext._traceFlags.debug) {
+    if (spanContext._sampling.priority > AUTO_KEEP) {
       carrier[b3FlagsKey] = '1'
     }
 
@@ -120,25 +114,20 @@ class TextMapPropagator {
   }
 
   _extractSpanContext (carrier) {
-    const context = this._extractContext(carrier)
-
-    if (!context) return null
-
-    if (context.traceFlags.sampled !== false) {
-      return new DatadogSpanContext(context)
-    } else {
-      return new NoopSpanContext(context)
-    }
-  }
-
-  _extractContext (carrier) {
     return this._extractDatadogContext(carrier) || this._extractB3Context(carrier) || this._extractSqsdContext(carrier)
   }
 
   _extractDatadogContext (carrier) {
-    const sampled = this._isSampled(carrier[sampleKey])
+    const spanContext = this._extractGenericContext(carrier, traceKey, spanKey, 10)
 
-    return this._extractGenericContext(carrier, traceKey, spanKey, { sampled }, 10)
+    if (spanContext) {
+      this._extractOrigin(carrier, spanContext)
+      this._extractBaggageItems(carrier, spanContext)
+      this._extractSamplingPriority(carrier, spanContext)
+      this._extractTags(carrier, spanContext)
+    }
+
+    return spanContext
   }
 
   _extractB3Context (carrier) {
@@ -146,9 +135,23 @@ class TextMapPropagator {
 
     const b3 = this._extractB3Headers(carrier)
     const debug = b3[b3FlagsKey] === '1'
-    const sampled = this._isSampled(b3[b3SampledKey], debug)
+    const priority = this._getPriority(b3[b3SampledKey], debug)
+    const spanContext = this._extractGenericContext(b3, b3TraceKey, b3SpanKey)
+
+    if (priority !== undefined) {
+      if (!spanContext) {
+        // B3 can force a sampling decision without providing IDs
+        return new DatadogSpanContext({
+          traceId: id(),
+          spanId: null,
+          sampling: { priority }
+        })
+      }
+
+      spanContext._sampling.priority = priority
+    }
 
-    return this._extractGenericContext(b3, b3TraceKey, b3SpanKey, { sampled, debug })
+    return spanContext
   }
 
   _extractSqsdContext (carrier) {
@@ -165,19 +168,12 @@ class TextMapPropagator {
     return this._extractDatadogContext(parsed)
   }
 
-  _extractGenericContext (carrier, traceKey, spanKey, traceFlags, radix) {
+  _extractGenericContext (carrier, traceKey, spanKey, radix) {
     if (carrier[traceKey] && carrier[spanKey]) {
-      return {
+      return new DatadogSpanContext({
         traceId: id(carrier[traceKey], radix),
-        spanId: id(carrier[spanKey], radix),
-        traceFlags
-      }
-    } else if (typeof traceFlags.sampled === 'boolean') {
-      return {
-        traceId: id(),
-        spanId: null,
-        traceFlags
-      }
+        spanId: id(carrier[spanKey], radix)
+      })
     }
 
     return null
@@ -225,12 +221,15 @@ class TextMapPropagator {
     } else {
       const b3 = {
         [b3TraceKey]: parts[0],
-        [b3SpanKey]: parts[1],
-        [b3SampledKey]: parts[2] !== '0' ? '1' : '0'
+        [b3SpanKey]: parts[1]
       }
 
-      if (parts[2] === 'd') {
-        b3[b3FlagsKey] = '1'
+      if (parts[2]) {
+        b3[b3SampledKey] = parts[2] !== '0' ? '1' : '0'
+
+        if (parts[2] === 'd') {
+          b3[b3FlagsKey] = '1'
+        }
       }
 
       return b3
@@ -275,14 +274,14 @@ class TextMapPropagator {
     }
   }
 
-  _isSampled (sampled, debug) {
-    if (debug || sampled === '1') {
-      return true
+  _getPriority (sampled, debug) {
+    if (debug) {
+      return USER_KEEP
+    } else if (sampled === '1') {
+      return AUTO_KEEP
     } else if (sampled === '0') {
-      return false
+      return AUTO_REJECT
     }
-
-    return null
   }
 }
 

package/packages/dd-trace/src/opentracing/span.js

@@ -4,29 +4,24 @@ const opentracing = require('opentracing')
 const now = require('performance-now')
 const Span = opentracing.Span
 const SpanContext = require('./span_context')
-const constants = require('../constants')
 const id = require('../id')
 const tagger = require('../tagger')
 const log = require('../log')
 const { storage } = require('../../../datadog-core')
 
-const SAMPLE_RATE_METRIC_KEY = constants.SAMPLE_RATE_METRIC_KEY
 const { DD_TRACE_EXPERIMENTAL_STATE_TRACKING } = process.env
 
 class DatadogSpan extends Span {
-  constructor (tracer, processor, sampler, prioritySampler, fields, debug) {
+  constructor (tracer, processor, prioritySampler, fields, debug) {
     super()
 
     const operationName = fields.operationName
     const parent = fields.parent || null
-    const tags = Object.assign({
-      [SAMPLE_RATE_METRIC_KEY]: sampler.rate()
-    }, fields.tags)
+    const tags = Object.assign({}, fields.tags)
     const hostname = fields.hostname
 
     this._parentTracer = tracer
     this._debug = debug
-    this._sampler = sampler
     this._processor = processor
     this._prioritySampler = prioritySampler
     this._store = storage.getStore()

package/packages/dd-trace/src/opentracing/span_context.js

@@ -16,9 +16,6 @@ class DatadogSpanContext extends SpanContext {
     this._tags = props.tags || {}
     this._sampling = props.sampling || {}
     this._baggageItems = props.baggageItems || {}
-    this._traceFlags = props.traceFlags || {}
-    this._traceFlags.sampled = this._traceFlags.sampled !== false
-    this._traceFlags.debug = this._traceFlags.debug === true
     this._noop = props.noop || null
     this._trace = props.trace || {
       started: [],