dd-trace 2.27.0 → 2.28.0

package/packages/dd-trace/src/metrics.js

@@ -8,6 +8,7 @@ const os = require('os')
 const Client = require('./dogstatsd')
 const log = require('./log')
 const Histogram = require('./histogram')
+const { performance } = require('perf_hooks')
 
 const INTERVAL = 10 * 1000
 
@@ -20,6 +21,7 @@ let cpuUsage
 let gauges
 let counters
 let histograms
+let elu
 
 reset()
 
@@ -259,6 +261,21 @@ function captureHistograms () {
   })
 }
 
+/**
+ * Gathers and reports Event Loop Utilization (ELU) since last run
+ *
+ * ELU is a measure of how busy the event loop is, like running JavaScript or
+ * waiting on *Sync functions. The value is between 0 (idle) and 1 (exhausted).
+ *
+ * performance.eventLoopUtilization available in Node.js >= v14.10, >= v12.19, >= v16
+ */
+const captureELU = ('eventLoopUtilization' in performance) ? () => {
+  // if elu is undefined (first run) the measurement is from start of process
+  elu = performance.eventLoopUtilization(elu)
+
+  client.gauge('runtime.node.event_loop.utilization', elu.utilization)
+} : () => {}
+
 function captureCommonMetrics () {
   captureMemoryUsage()
   captureProcess()
@@ -266,6 +283,7 @@ function captureCommonMetrics () {
   captureGauges()
   captureCounters()
   captureHistograms()
+  captureELU()
 }
 
 function captureNativeMetrics () {

package/packages/dd-trace/src/noop/proxy.js

@@ -81,8 +81,8 @@ class Tracer {
     return this._tracer.getRumData.apply(this._tracer, arguments)
   }
 
-  setUser () {
-    this._tracer.setUser.apply(this._tracer, arguments)
+  setUser (user) {
+    this.appsec.setUser(user)
     return this
   }
 }

package/packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -302,6 +302,7 @@ class TextMapPropagator {
     const matches = headerValue.trim().match(traceparentExpr)
     if (matches.length) {
       const [ version, traceId, spanId, flags, tail ] = matches.slice(1)
+      const traceparent = { version }
       const tracestate = TraceState.fromString(carrier.tracestate)
       if (invalidSegment.test(traceId)) return null
       if (invalidSegment.test(spanId)) return null
@@ -316,6 +317,7 @@ class TextMapPropagator {
         traceId: id(traceId, 16),
         spanId: id(spanId, 16),
         sampling: { priority: parseInt(flags, 10) & 1 ? 1 : 0 },
+        traceparent,
         tracestate
       })
 

package/packages/dd-trace/src/opentracing/span_context.js

@@ -12,8 +12,9 @@ class DatadogSpanContext {
     this._name = props.name
     this._isFinished = props.isFinished || false
     this._tags = props.tags || {}
-    this._sampling = props.sampling || {}
+    this._sampling = Object.assign({}, props.sampling)
     this._baggageItems = props.baggageItems || {}
+    this._traceparent = props.traceparent
     this._tracestate = props.tracestate
     this._noop = props.noop || null
     this._trace = props.trace || {
@@ -32,10 +33,11 @@ class DatadogSpanContext {
   }
 
   toTraceparent () {
-    const sampling = this._sampling.priority >= AUTO_KEEP ? '01' : '00'
+    const flags = this._sampling.priority >= AUTO_KEEP ? '01' : '00'
     const traceId = this._traceId.toString(16).padStart(32, '0')
     const spanId = this._spanId.toString(16).padStart(16, '0')
-    return `01-${traceId}-${spanId}-${sampling}`
+    const version = (this._traceparent && this._traceparent.version) || '00'
+    return `${version}-${traceId}-${spanId}-${flags}`
   }
 }
 

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -128,10 +128,13 @@ module.exports = class CiPlugin extends Plugin {
       const suiteTags = {
         [TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(),
         [TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(),
-        [TEST_MODULE_ID]: testSuiteSpan.context()._parentId.toString(10),
         [TEST_COMMAND]: testSuiteSpan.context()._tags[TEST_COMMAND],
         [TEST_BUNDLE]: testSuiteSpan.context()._tags[TEST_COMMAND]
       }
+      if (testSuiteSpan.context()._parentId) {
+        suiteTags[TEST_MODULE_ID] = testSuiteSpan.context()._parentId.toString(10)
+      }
+
       testTags = {
         ...testTags,
         ...suiteTags

package/packages/dd-trace/src/plugins/util/exec.js

@@ -1,8 +1,8 @@
-const { execSync } = require('child_process')
+const cp = require('child_process')
 
 const sanitizedExec = (cmd, options = {}) => {
   try {
-    return execSync(cmd, options).toString().replace(/(\r\n|\n|\r)/gm, '')
+    return cp.execSync(cmd, options).toString().replace(/(\r\n|\n|\r)/gm, '')
   } catch (e) {
     return ''
   }

package/packages/dd-trace/src/plugins/util/git.js

@@ -21,6 +21,19 @@ const {
 
 const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
 
+function isShallowRepository () {
+  return sanitizedExec('git rev-parse --is-shallow-repository', { stdio: 'pipe' }) === 'true'
+}
+
+function unshallowRepository () {
+  try {
+    execSync('git config remote.origin.partialclonefilter "blob:none"', { stdio: 'pipe' })
+    execSync('git fetch --shallow-since="1 month ago" --update-shallow --refetch', { stdio: 'pipe' })
+  } catch (err) {
+    log.error(err)
+  }
+}
+
 function getRepositoryUrl () {
   return sanitizedExec('git config --get remote.origin.url', { stdio: 'pipe' })
 }
@@ -146,5 +159,7 @@ module.exports = {
   getRepositoryUrl,
   generatePackFilesForCommits,
   getCommitsToUpload,
-  GIT_REV_LIST_MAX_BUFFER
+  GIT_REV_LIST_MAX_BUFFER,
+  isShallowRepository,
+  unshallowRepository
 }

package/packages/dd-trace/src/profiler.js

@@ -3,6 +3,9 @@
 const log = require('./log')
 const { profiler } = require('./profiling')
 
+// Stop profiler upon exit in order to collect and export the current profile
+process.once('beforeExit', () => { profiler.stop() })
+
 module.exports = {
   start: config => {
     const { service, version, env, url, hostname, port, tags } = config

package/packages/dd-trace/src/profiling/config.js

@@ -23,7 +23,9 @@ const {
   DD_AGENT_HOST,
   DD_TRACE_AGENT_PORT,
   DD_PROFILING_UPLOAD_TIMEOUT,
-  DD_PROFILING_SOURCE_MAP
+  DD_PROFILING_SOURCE_MAP,
+  DD_PROFILING_UPLOAD_PERIOD,
+  DD_PROFILING_PPROF_PREFIX
 } = process.env
 
 class Config {
@@ -35,13 +37,15 @@ class Config {
     const version = coalesce(options.version, DD_VERSION)
     const functionname = process.env.AWS_LAMBDA_FUNCTION_NAME
     // Must be longer than one minute so pad with five seconds
-    const flushInterval = coalesce(options.interval, 65 * 1000)
+    const flushInterval = coalesce(options.interval, Number(DD_PROFILING_UPLOAD_PERIOD) * 1000, 65 * 1000)
     const uploadTimeout = coalesce(options.uploadTimeout,
-      DD_PROFILING_UPLOAD_TIMEOUT, 60 * 1000)
+      Number(DD_PROFILING_UPLOAD_TIMEOUT), 60 * 1000)
     const sourceMap = coalesce(options.sourceMap,
       DD_PROFILING_SOURCE_MAP, true)
     const endpointCollection = coalesce(options.endpointCollection,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, false)
+    const pprofPrefix = coalesce(options.pprofPrefix,
+      DD_PROFILING_PPROF_PREFIX)
 
     this.enabled = String(enabled) !== 'false'
     this.service = service
@@ -60,6 +64,7 @@ class Config {
     this.uploadTimeout = uploadTimeout
     this.sourceMap = sourceMap
     this.endpointCollection = endpointCollection
+    this.pprofPrefix = pprofPrefix
 
     const hostname = coalesce(options.hostname, DD_AGENT_HOST) || 'localhost'
     const port = coalesce(options.port, DD_TRACE_AGENT_PORT) || 8126

package/packages/dd-trace/src/profiling/exporters/file.js

@@ -4,11 +4,22 @@ const fs = require('fs')
 const { promisify } = require('util')
 const writeFile = promisify(fs.writeFile)
 
+function formatDateTime (t) {
+  const pad = (n) => String(n).padStart(2, '0')
+  return `${t.getUTCFullYear()}${pad(t.getUTCMonth() + 1)}${pad(t.getUTCDate())}` +
+         `T${pad(t.getUTCHours())}${pad(t.getUTCMinutes())}${pad(t.getUTCSeconds())}Z`
+}
+
 class FileExporter {
-  export ({ profiles }) {
+  constructor ({ pprofPrefix } = {}) {
+    this._pprofPrefix = pprofPrefix || ''
+  }
+
+  export ({ profiles, end }) {
     const types = Object.keys(profiles)
+    const dateStr = formatDateTime(end)
     const tasks = types.map(type => {
-      return writeFile(`${type}.pb.gz`, profiles[type])
+      return writeFile(`${this._pprofPrefix}${type}_${dateStr}.pprof`, profiles[type])
     })
 
     return Promise.all(tasks)

package/packages/dd-trace/src/profiling/profiler.js

@@ -56,7 +56,7 @@ class Profiler extends EventEmitter {
       this._capture(this._timeoutInterval)
     } catch (e) {
       this._logger.error(e)
-      this.stop()
+      this._stop()
     }
   }
 
@@ -64,7 +64,16 @@ class Profiler extends EventEmitter {
     this._timeoutInterval = this._config.flushInterval
   }
 
-  stop () {
+  async stop () {
+    if (!this._enabled) return
+
+    // collect and export current profiles
+    // once collect returns, profilers can be safely stopped
+    this._collect()
+    this._stop()
+  }
+
+  _stop () {
     if (!this._enabled) return
 
     this._enabled = false
@@ -92,16 +101,24 @@ class Profiler extends EventEmitter {
   }
 
   async _collect () {
+    if (!this._enabled) return
+
     const start = this._lastStart
     const end = new Date()
-    const profiles = {}
+    const profiles = []
+    const encodedProfiles = {}
 
     try {
+      // collect profiles synchronously so that profilers can be safely stopped asynchronously
       for (const profiler of this._config.profilers) {
         const profile = profiler.profile()
         if (!profile) continue
+        profiles.push({ profiler, profile })
+      }
 
-        profiles[profiler.type] = await profiler.encode(profile)
+      // encode and export asynchronously
+      for (const { profiler, profile } of profiles) {
+        encodedProfiles[profiler.type] = await profiler.encode(profile)
         this._logger.debug(() => {
           const profileJson = JSON.stringify(profile, (key, value) => {
             return typeof value === 'bigint' ? value.toString() : value
@@ -111,11 +128,11 @@ class Profiler extends EventEmitter {
       }
 
       this._capture(this._timeoutInterval)
-      await this._submit(profiles, start, end)
+      await this._submit(encodedProfiles, start, end)
       this._logger.debug('Submitted profiles')
     } catch (err) {
       this._logger.error(err)
-      this.stop()
+      this._stop()
     }
   }
 

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -35,6 +35,7 @@ class NativeWallProfiler {
   stop () {
     if (!this._stop) return
     this._stop()
+    this._stop = undefined
   }
 
   _record () {

package/packages/dd-trace/src/proxy.js

@@ -51,7 +51,7 @@ class Tracer extends NoopProxy {
         }
 
         this._tracer = new DatadogTracer(config)
-        this.appsec = new AppsecSdk(this._tracer)
+        this.appsec = new AppsecSdk(this._tracer, config)
 
         if (config.iast.enabled) {
           require('./appsec/iast').enable(config, this._tracer)

package/packages/dd-trace/src/span_processor.js

@@ -17,7 +17,7 @@ class SpanProcessor {
     this._killAll = false
 
     this._stats = new SpanStatsProcessor(config)
-    this._spanSampler = new SpanSampler(config)
+    this._spanSampler = new SpanSampler(config.sampler)
   }
 
   process (span) {

package/packages/dd-trace/src/span_sampler.js

@@ -2,76 +2,92 @@
 const { globMatch } = require('../src/util')
 const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority
 const RateLimiter = require('./rate_limiter')
+const Sampler = require('./sampler')
 
-class SpanSampler {
-  constructor ({ spanSamplingRules = [] }) {
-    this._rules = spanSamplingRules
-    this._limiters = {}
+class SpanSamplingRule {
+  constructor ({ service, name, sampleRate = 1.0, maxPerSecond } = {}) {
+    this.service = service
+    this.name = name
+
+    this._sampler = new Sampler(sampleRate)
+    this._limiter = undefined
+
+    if (Number.isFinite(maxPerSecond)) {
+      this._limiter = new RateLimiter(maxPerSecond)
+    }
   }
 
-  sample (spanContext) {
-    const decision = spanContext._sampling.priority
-    if (decision === USER_KEEP || decision === AUTO_KEEP) return
+  get sampleRate () {
+    return this._sampler.rate()
+  }
 
-    const { started } = spanContext._trace
-    for (const span of started) {
-      const service = span.tracer()._service
-      const name = span._name
-      const rule = findRule(this._rules, service, name)
-      if (!rule) continue
+  get maxPerSecond () {
+    return this._limiter && this._limiter._rateLimit
+  }
 
-      const sampleRate = getSampleRate(rule.sampleRate)
-      const maxPerSecond = getMaxPerSecond(rule.maxPerSecond)
-      const sampled = sample(sampleRate)
-      if (!sampled) continue
+  static from (config) {
+    return new SpanSamplingRule(config)
+  }
 
-      const key = `${service}:${name}`
-      const limiter = getLimiter(this._limiters, key, maxPerSecond)
-      if (limiter.isAllowed()) {
-        span.context()._sampling.spanSampling = {
-          sampleRate,
-          maxPerSecond
-        }
-      }
+  match (service, name) {
+    if (this.service && !globMatch(this.service, service)) {
+      return false
     }
-  }
-}
 
-function findRule (rules, service, name) {
-  for (const rule of rules) {
-    const servicePattern = getService(rule.service)
-    const namePattern = getName(rule.name)
-    if (globMatch(servicePattern, service) && globMatch(namePattern, name)) {
-      return rule
+    if (this.name && !globMatch(this.name, name)) {
+      return false
     }
+
+    return true
   }
-}
 
-function getLimiter (list, key, maxPerSecond) {
-  if (typeof list[key] === 'undefined') {
-    list[key] = new RateLimiter(maxPerSecond)
+  sample () {
+    if (!this._sampler.isSampled()) {
+      return false
+    }
+
+    if (this._limiter) {
+      return this._limiter.isAllowed()
+    }
+
+    return true
   }
-  return list[key]
 }
 
-function sample (sampleRate) {
-  return Math.random() < sampleRate
-}
+class SpanSampler {
+  constructor ({ spanSamplingRules = [] } = {}) {
+    this._rules = spanSamplingRules.map(SpanSamplingRule.from)
+  }
 
-function getService (service) {
-  return service || '*'
-}
+  findRule (service, name) {
+    for (const rule of this._rules) {
+      if (rule.match(service, name)) {
+        return rule
+      }
+    }
+  }
 
-function getName (name) {
-  return name || '*'
-}
+  sample (spanContext) {
+    const decision = spanContext._sampling.priority
+    if (decision === USER_KEEP || decision === AUTO_KEEP) return
 
-function getSampleRate (sampleRate) {
-  return sampleRate || 1.0
-}
+    const { started } = spanContext._trace
+    for (const span of started) {
+      const tags = span.context()._tags || {}
+      const name = span._name
+      const service = tags.service ||
+        tags['service.name'] ||
+        span.tracer()._service
 
-function getMaxPerSecond (maxPerSecond) {
-  return maxPerSecond || Infinity
+      const rule = this.findRule(service, name)
+      if (rule && rule.sample()) {
+        span.context()._sampling.spanSampling = {
+          sampleRate: rule.sampleRate,
+          maxPerSecond: rule.maxPerSecond
+        }
+      }
+    }
+  }
 }
 
 module.exports = SpanSampler

package/packages/dd-trace/src/startup-log.js

@@ -1,14 +1,11 @@
 'use strict'
 
-const mainLogger = require('./log')
+const { info, warn } = require('./log/writer')
 
 const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
 
-const logger = Object.create(mainLogger)
-logger.toggle(true)
-
 let config
 let pluginManager
 let samplingRules = []
@@ -89,9 +86,9 @@ function startupLog ({ agentError } = {}) {
   // out.service_mapping
   // out.service_mapping_error
 
-  logger.info('DATADOG TRACER CONFIGURATION - ' + out)
+  info('DATADOG TRACER CONFIGURATION - ' + out)
   if (agentError) {
-    logger.warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+    warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
   }
 
   config = undefined

package/packages/dd-trace/src/tracer.js

@@ -127,22 +127,6 @@ class DatadogTracer extends Tracer {
 <meta name="dd-trace-id" content="${traceId}" />\
 <meta name="dd-trace-time" content="${traceTime}" />`
   }
-
-  setUser (user) {
-    if (!user || !user.id) return this
-
-    const span = this.scope().active()
-    if (!span) return this
-
-    const rootSpan = span._spanContext._trace.started[0]
-    if (!rootSpan) return this
-
-    for (const k of Object.keys(user)) {
-      rootSpan.setTag(`usr.${k}`, '' + user[k])
-    }
-
-    return this
-  }
 }
 
 function addError (span, error) {