dd-trace 4.38.1 → 4.40.0

package/packages/dd-trace/src/flare/file.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const { Writable } = require('stream')
+
+const INITIAL_SIZE = 64 * 1024
+
+class FlareFile extends Writable {
+  constructor () {
+    super()
+
+    this.length = 0
+
+    this._buffer = Buffer.alloc(INITIAL_SIZE)
+  }
+
+  get data () {
+    return this._buffer.subarray(0, this.length)
+  }
+
+  _write (chunk, encoding, callback) {
+    const length = Buffer.byteLength(chunk)
+
+    this._reserve(length)
+
+    if (Buffer.isBuffer(chunk)) {
+      this.length += chunk.copy(this._buffer, this.length)
+    } else {
+      this.length += this._buffer.write(chunk, encoding)
+    }
+
+    callback()
+  }
+
+  _reserve (length) {
+    while (this.length + length > this._buffer.length) {
+      const buffer = Buffer.alloc(this.length * 2)
+
+      this._buffer.copy(buffer)
+      this._buffer = buffer
+    }
+  }
+}
+
+module.exports = FlareFile

package/packages/dd-trace/src/flare/index.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const log = require('../log')
+const startupLog = require('../startup-log')
+const FlareFile = require('./file')
+const { LogChannel } = require('../log/channels')
+const request = require('../exporters/common/request')
+const FormData = require('../exporters/common/form-data')
+
+const MAX_LOG_SIZE = 12 * 1024 * 1024 // 12MB soft limit
+const TIMEOUT = 20 * 1000 * 60
+
+let logChannel = null
+let tracerLogs = null
+let timer
+let tracerConfig = null
+
+const logger = {
+  debug: (msg) => recordLog(msg),
+  info: (msg) => recordLog(msg),
+  warn: (msg) => recordLog(msg),
+  error: (err) => recordLog(err.stack)
+}
+
+const flare = {
+  enable (tracerConfig_) {
+    tracerConfig = tracerConfig_
+  },
+
+  disable () {
+    tracerConfig = null
+
+    flare.cleanup()
+  },
+
+  prepare (logLevel) {
+    if (!tracerConfig) return
+
+    logChannel?.unsubscribe(logger)
+    logChannel = new LogChannel(logLevel)
+    logChannel.subscribe(logger)
+    tracerLogs = tracerLogs || new FlareFile()
+    timer = timer || setTimeout(flare.cleanup, TIMEOUT)
+  },
+
+  send (task) {
+    if (!tracerConfig) return
+
+    const tracerInfo = new FlareFile()
+
+    tracerInfo.write(JSON.stringify(startupLog.tracerInfo(), null, 2))
+
+    flare._sendFile(task, tracerInfo, 'tracer_info.txt')
+    flare._sendFile(task, tracerLogs, 'tracer_logs.txt')
+
+    flare.cleanup()
+  },
+
+  cleanup () {
+    logChannel?.unsubscribe(logger)
+    timer = clearTimeout(timer)
+    logChannel = null
+    tracerLogs = null
+  },
+
+  _sendFile (task, file, filename) {
+    if (!file) return
+
+    const form = new FormData()
+
+    form.append('case_id', task.case_id)
+    form.append('hostname', task.hostname)
+    form.append('email', task.user_handle)
+    form.append('source', 'tracer_nodejs')
+    form.append('flare_file', file.data, { filename })
+
+    request(form, {
+      url: tracerConfig.url,
+      hostname: tracerConfig.hostname,
+      port: tracerConfig.port,
+      method: 'POST',
+      path: '/tracer_flare/v1',
+      headers: form.getHeaders()
+    }, (err) => {
+      if (err) {
+        log.error(err)
+      }
+    })
+  }
+}
+
+function recordLog (msg) {
+  if (tracerLogs.length > MAX_LOG_SIZE) return
+
+  tracerLogs.write(`${msg}\n`) // TODO: gzip
+}
+
+module.exports = flare

package/packages/dd-trace/src/log/channels.js

@@ -3,44 +3,69 @@
 const { channel } = require('dc-polyfill')
 
 const Level = {
-  Debug: 'debug',
-  Info: 'info',
-  Warn: 'warn',
-  Error: 'error'
+  trace: 20,
+  debug: 20,
+  info: 30,
+  warn: 40,
+  error: 50,
+  critical: 50,
+  off: 100
 }
 
-const defaultLevel = Level.Debug
+const debugChannel = channel('datadog:log:debug')
+const infoChannel = channel('datadog:log:info')
+const warnChannel = channel('datadog:log:warn')
+const errorChannel = channel('datadog:log:error')
 
-// based on: https://github.com/trentm/node-bunyan#levels
-const logChannels = {
-  [Level.Debug]: createLogChannel(Level.Debug, 20),
-  [Level.Info]: createLogChannel(Level.Info, 30),
-  [Level.Warn]: createLogChannel(Level.Warn, 40),
-  [Level.Error]: createLogChannel(Level.Error, 50)
-}
+const defaultLevel = Level.debug
 
-function createLogChannel (name, logLevel) {
-  const logChannel = channel(`datadog:log:${name}`)
-  logChannel.logLevel = logLevel
-  return logChannel
+function getChannelLogLevel (level) {
+  return level && typeof level === 'string'
+    ? Level[level.toLowerCase().trim()] || defaultLevel
+    : defaultLevel
 }
 
-function getChannelLogLevel (level) {
-  let logChannel
-  if (level && typeof level === 'string') {
-    logChannel = logChannels[level.toLowerCase().trim()] || logChannels[defaultLevel]
-  } else {
-    logChannel = logChannels[defaultLevel]
+class LogChannel {
+  constructor (level) {
+    this._level = getChannelLogLevel(level)
+  }
+
+  subscribe (logger) {
+    if (Level.debug >= this._level) {
+      debugChannel.subscribe(logger.debug)
+    }
+    if (Level.info >= this._level) {
+      infoChannel.subscribe(logger.info)
+    }
+    if (Level.warn >= this._level) {
+      warnChannel.subscribe(logger.warn)
+    }
+    if (Level.error >= this._level) {
+      errorChannel.subscribe(logger.error)
+    }
+  }
+
+  unsubscribe (logger) {
+    if (debugChannel.hasSubscribers) {
+      debugChannel.unsubscribe(logger.debug)
+    }
+    if (infoChannel.hasSubscribers) {
+      infoChannel.unsubscribe(logger.info)
+    }
+    if (warnChannel.hasSubscribers) {
+      warnChannel.unsubscribe(logger.warn)
+    }
+    if (errorChannel.hasSubscribers) {
+      errorChannel.unsubscribe(logger.error)
+    }
   }
-  return logChannel.logLevel
 }
 
 module.exports = {
-  Level,
-  getChannelLogLevel,
+  LogChannel,
 
-  debugChannel: logChannels[Level.Debug],
-  infoChannel: logChannels[Level.Info],
-  warnChannel: logChannels[Level.Warn],
-  errorChannel: logChannels[Level.Error]
+  debugChannel,
+  infoChannel,
+  warnChannel,
+  errorChannel
 }

package/packages/dd-trace/src/log/writer.js

@@ -1,8 +1,7 @@
 'use strict'
 
 const { storage } = require('../../../datadog-core')
-const { getChannelLogLevel, debugChannel, infoChannel, warnChannel, errorChannel } = require('./channels')
-
+const { LogChannel } = require('./channels')
 const defaultLogger = {
   debug: msg => console.debug(msg), /* eslint-disable-line no-console */
   info: msg => console.info(msg), /* eslint-disable-line no-console */
@@ -12,7 +11,7 @@ const defaultLogger = {
 
 let enabled = false
 let logger = defaultLogger
-let logLevel = getChannelLogLevel()
+let logChannel = new LogChannel()
 
 function withNoop (fn) {
   const store = storage.getStore()
@@ -23,45 +22,21 @@ function withNoop (fn) {
 }
 
 function unsubscribeAll () {
-  if (debugChannel.hasSubscribers) {
-    debugChannel.unsubscribe(onDebug)
-  }
-  if (infoChannel.hasSubscribers) {
-    infoChannel.unsubscribe(onInfo)
-  }
-  if (warnChannel.hasSubscribers) {
-    warnChannel.unsubscribe(onWarn)
-  }
-  if (errorChannel.hasSubscribers) {
-    errorChannel.unsubscribe(onError)
-  }
+  logChannel.unsubscribe({ debug, info, warn, error })
 }
 
-function toggleSubscription (enable) {
+function toggleSubscription (enable, level) {
   unsubscribeAll()
 
   if (enable) {
-    if (debugChannel.logLevel >= logLevel) {
-      debugChannel.subscribe(onDebug)
-    }
-    if (infoChannel.logLevel >= logLevel) {
-      infoChannel.subscribe(onInfo)
-    }
-    if (warnChannel.logLevel >= logLevel) {
-      warnChannel.subscribe(onWarn)
-    }
-    if (errorChannel.logLevel >= logLevel) {
-      errorChannel.subscribe(onError)
-    }
+    logChannel = new LogChannel(level)
+    logChannel.subscribe({ debug, info, warn, error })
   }
 }
 
 function toggle (enable, level) {
-  if (level !== undefined) {
-    logLevel = getChannelLogLevel(level)
-  }
   enabled = enable
-  toggleSubscription(enabled)
+  toggleSubscription(enabled, level)
 }
 
 function use (newLogger) {
@@ -73,26 +48,9 @@ function use (newLogger) {
 function reset () {
   logger = defaultLogger
   enabled = false
-  logLevel = getChannelLogLevel()
   toggleSubscription(false)
 }
 
-function onError (err) {
-  if (enabled) error(err)
-}
-
-function onWarn (message) {
-  if (enabled) warn(message)
-}
-
-function onInfo (message) {
-  if (enabled) info(message)
-}
-
-function onDebug (message) {
-  if (enabled) debug(message)
-}
-
 function error (err) {
   if (typeof err !== 'object' || !err) {
     err = String(err)

package/packages/dd-trace/src/opentelemetry/span.js

@@ -179,11 +179,19 @@ class Span {
   }
 
   setAttribute (key, value) {
+    if (key === 'http.response.status_code') {
+      this._ddSpan.setTag('http.status_code', value.toString())
+    }
+
     this._ddSpan.setTag(key, value)
     return this
   }
 
   setAttributes (attributes) {
+    if ('http.response.status_code' in attributes) {
+      attributes['http.status_code'] = attributes['http.response.status_code'].toString()
+    }
+
     this._ddSpan.addTags(attributes)
     return this
   }

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -5,6 +5,7 @@ const id = require('../../id')
 const DatadogSpanContext = require('../span_context')
 const log = require('../../log')
 const TraceState = require('./tracestate')
+const tags = require('../../../../../ext/tags')
 
 const { AUTO_KEEP, AUTO_REJECT, USER_KEEP } = require('../../../../../ext/priority')
 
@@ -39,6 +40,7 @@ const tracestateTagKeyFilter = /[^\x21-\x2b\x2d-\x3c\x3e-\x7e]/g
 // Tag values in tracestate replace ',', '~' and ';' with '_'
 const tracestateTagValueFilter = /[^\x20-\x2b\x2d-\x3a\x3c-\x7d]/g
 const invalidSegment = /^0+$/
+const zeroTraceId = '0000000000000000'
 
 class TextMapPropagator {
   constructor (config) {
@@ -175,9 +177,9 @@ class TextMapPropagator {
         // SpanContext was created by a ddtrace span.
         // Last datadog span id should be set to the current span.
         state.set('p', spanContext._spanId)
-      } else if (spanContext._trace.tags['_dd.parent_id']) {
+      } else if (spanContext._trace.tags[tags.DD_PARENT_ID]) {
         // Propagate the last Datadog span id set on the remote span.
-        state.set('p', spanContext._trace.tags['_dd.parent_id'])
+        state.set('p', spanContext._trace.tags[tags.DD_PARENT_ID])
       }
       state.set('s', priority)
       if (mechanism) {
@@ -214,9 +216,56 @@ class TextMapPropagator {
     return this._config.tracePropagationStyle[mode].includes(name)
   }
 
+  _hasTraceIdConflict (w3cSpanContext, firstSpanContext) {
+    return w3cSpanContext !== null &&
+           firstSpanContext.toTraceId(true) === w3cSpanContext.toTraceId(true) &&
+           firstSpanContext.toSpanId() !== w3cSpanContext.toSpanId()
+  }
+
+  _hasParentIdInTags (spanContext) {
+    return tags.DD_PARENT_ID in spanContext._trace.tags &&
+      spanContext._trace.tags[tags.DD_PARENT_ID] !== zeroTraceId
+  }
+
+  _updateParentIdFromDdHeaders (carrier, firstSpanContext) {
+    const ddCtx = this._extractDatadogContext(carrier)
+    if (ddCtx !== null) {
+      firstSpanContext._trace.tags[tags.DD_PARENT_ID] = ddCtx._spanId.toString().padStart(16, '0')
+    }
+  }
+
+  _resolveTraceContextConflicts (w3cSpanContext, firstSpanContext, carrier) {
+    if (!this._hasTraceIdConflict(w3cSpanContext, firstSpanContext)) {
+      return firstSpanContext
+    }
+    if (this._hasParentIdInTags(w3cSpanContext)) {
+      // tracecontext headers contain a p value, ensure this value is sent to backend
+      firstSpanContext._trace.tags[tags.DD_PARENT_ID] = w3cSpanContext._trace.tags[tags.DD_PARENT_ID]
+    } else {
+      // if p value is not present in tracestate, use the parent id from the datadog headers
+      this._updateParentIdFromDdHeaders(carrier, firstSpanContext)
+    }
+    // the span_id in tracecontext takes precedence over the first extracted propagation style
+    firstSpanContext._spanId = w3cSpanContext._spanId
+    return firstSpanContext
+  }
+
   _extractSpanContext (carrier) {
+    let spanContext = null
     for (const extractor of this._config.tracePropagationStyle.extract) {
-      let spanContext = null
+      // add logic to ensure tracecontext headers takes precedence over other extracted headers
+      if (spanContext !== null) {
+        if (this._config.tracePropagationExtractFirst) {
+          return spanContext
+        }
+        if (extractor !== 'tracecontext') {
+          continue
+        }
+        spanContext = this._resolveTraceContextConflicts(
+          this._extractTraceparentContext(carrier), spanContext, carrier)
+        break
+      }
+
       switch (extractor) {
         case 'datadog':
           spanContext = this._extractDatadogContext(carrier)
@@ -238,13 +287,9 @@ class TextMapPropagator {
         default:
           log.warn(`Unknown propagation style: ${extractor}`)
       }
-
-      if (spanContext !== null) {
-        return spanContext
-      }
     }
 
-    return this._extractSqsdContext(carrier)
+    return spanContext || this._extractSqsdContext(carrier)
   }
 
   _extractDatadogContext (carrier) {
@@ -354,7 +399,7 @@ class TextMapPropagator {
         for (const [key, value] of state.entries()) {
           switch (key) {
             case 'p': {
-              spanContext._trace.tags['_dd.parent_id'] = value
+              spanContext._trace.tags[tags.DD_PARENT_ID] = value
               break
             }
             case 's': {
@@ -387,8 +432,8 @@ class TextMapPropagator {
         }
       })
 
-      if (!spanContext._trace.tags['_dd.parent_id']) {
-        spanContext._trace.tags['_dd.parent_id'] = '0000000000000000'
+      if (!spanContext._trace.tags[tags.DD_PARENT_ID]) {
+        spanContext._trace.tags[tags.DD_PARENT_ID] = zeroTraceId
       }
 
       this._extractBaggageItems(carrier, spanContext)
@@ -531,7 +576,7 @@ class TextMapPropagator {
 
     const tid = traceId.substring(0, 16)
 
-    if (tid === '0000000000000000') return
+    if (tid === zeroTraceId) return
 
     spanContext._trace.tags['_dd.p.tid'] = tid
   }

package/packages/dd-trace/src/plugins/index.js

@@ -54,6 +54,7 @@ module.exports = {
   get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') },
   get mocha () { return require('../../../datadog-plugin-mocha/src') },
   get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') },
+  get workerpool () { return require('../../../datadog-plugin-mocha/src') },
   get moleculer () { return require('../../../datadog-plugin-moleculer/src') },
   get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') },
   get 'mongodb-core' () { return require('../../../datadog-plugin-mongodb-core/src') },

package/packages/dd-trace/src/plugins/util/ip_extractor.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const BlockList = require('./ip_blocklist')
+const { BlockList } = require('net')
 const net = require('net')
 
 const ipHeaderList = [

package/packages/dd-trace/src/plugins/util/test.js

@@ -62,6 +62,7 @@ const JEST_TEST_RUNNER = 'test.jest.test_runner'
 const JEST_DISPLAY_NAME = 'test.jest.display_name'
 
 const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel'
+const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel'
 
 const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped'
 const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled'
@@ -87,6 +88,9 @@ const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61
 // cucumber worker variables
 const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70
 
+// mocha worker variables
+const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80
+
 // Early flake detection util strings
 const EFD_STRING = "Retried by Datadog's Early Flake Detection"
 const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
@@ -98,6 +102,7 @@ module.exports = {
   JEST_TEST_RUNNER,
   JEST_DISPLAY_NAME,
   CUCUMBER_IS_PARALLEL,
+  MOCHA_IS_PARALLEL,
   TEST_TYPE,
   TEST_NAME,
   TEST_SUITE,
@@ -111,6 +116,7 @@ module.exports = {
   JEST_WORKER_TRACE_PAYLOAD_CODE,
   JEST_WORKER_COVERAGE_PAYLOAD_CODE,
   CUCUMBER_WORKER_TRACE_PAYLOAD_CODE,
+  MOCHA_WORKER_TRACE_PAYLOAD_CODE,
   TEST_SOURCE_START,
   TEST_SKIPPED_BY_ITR,
   TEST_CONFIGURATION_BROWSER_NAME,

package/packages/dd-trace/src/priority_sampler.js

@@ -10,6 +10,8 @@ const {
   SAMPLING_MECHANISM_AGENT,
   SAMPLING_MECHANISM_RULE,
   SAMPLING_MECHANISM_MANUAL,
+  SAMPLING_MECHANISM_REMOTE_USER,
+  SAMPLING_MECHANISM_REMOTE_DYNAMIC,
   SAMPLING_RULE_DECISION,
   SAMPLING_LIMIT_DECISION,
   SAMPLING_AGENT_DECISION,
@@ -41,9 +43,9 @@ class PrioritySampler {
     this.update({})
   }
 
-  configure (env, { sampleRate, rateLimit = 100, rules = [] } = {}) {
+  configure (env, { sampleRate, provenance = undefined, rateLimit = 100, rules = [] } = {}) {
     this._env = env
-    this._rules = this._normalizeRules(rules, sampleRate, rateLimit)
+    this._rules = this._normalizeRules(rules, sampleRate, rateLimit, provenance)
     this._limiter = new RateLimiter(rateLimit)
 
     setSamplingRules(this._rules)
@@ -137,6 +139,8 @@ class PrioritySampler {
   _getPriorityByRule (context, rule) {
     context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate
     context._sampling.mechanism = SAMPLING_MECHANISM_RULE
+    if (rule.provenance === 'customer') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_USER
+    if (rule.provenance === 'dynamic') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_DYNAMIC
 
     return rule.sample() && this._isSampledByRateLimit(context)
       ? USER_KEEP
@@ -181,11 +185,11 @@ class PrioritySampler {
     }
   }
 
-  _normalizeRules (rules, sampleRate, rateLimit) {
+  _normalizeRules (rules, sampleRate, rateLimit, provenance) {
     rules = [].concat(rules || [])
 
     return rules
-      .concat({ sampleRate, maxPerSecond: rateLimit })
+      .concat({ sampleRate, maxPerSecond: rateLimit, provenance })
       .map(rule => ({ ...rule, sampleRate: parseFloat(rule.sampleRate) }))
       .filter(rule => !isNaN(rule.sampleRate))
       .map(SamplingRule.from)

package/packages/dd-trace/src/profiler.js

@@ -9,7 +9,7 @@ process.once('beforeExit', () => { profiler.stop() })
 module.exports = {
   start: config => {
     const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA } = config
-    const { enabled, sourceMap, exporters } = config.profiling
+    const { enabled, sourceMap, exporters, heuristicsEnabled } = config.profiling
     const logger = {
       debug: (message) => log.debug(message),
       info: (message) => log.info(message),
@@ -19,6 +19,7 @@ module.exports = {
 
     return profiler.start({
       enabled,
+      heuristicsEnabled,
       service,
       version,
       env,

package/packages/dd-trace/src/profiling/config.js

@@ -65,6 +65,7 @@ class Config {
       DD_PROFILING_PPROF_PREFIX, '')
 
     this.enabled = enabled
+    this.heuristicsEnabled = options.heuristicsEnabled
     this.service = service
     this.env = env
     this.host = host

package/packages/dd-trace/src/profiling/profiler.js

@@ -55,7 +55,7 @@ class Profiler extends EventEmitter {
     if (this._enabled) return true
 
     const config = this._config = new Config(options)
-    if (!config.enabled) return false
+    if (!config.enabled && !config.heuristicsEnabled) return false
 
     this._logger = config.logger
     this._enabled = true