dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -2,21 +2,19 @@
 
 const { storage } = require('../../../../datadog-core')
 
-const dc = require('../../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../../../ext/tags')
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
+const { END_TIMESTAMP_LABEL, getNonJSThreadsLabels, getThreadLabels } = require('./shared')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
+const spanFinishCh = dc.channel('dd-trace:span:finish')
 const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers')
 
-const threadName = (function () {
-  const { isMainThread, threadId } = require('node:worker_threads')
-  const name = isMainThread ? 'Main' : `Worker #${threadId}`
-  return `${name} Event Loop`
-})()
+const MemoizedWebTags = Symbol('NativeWallProfiler.MemoizedWebTags')
 
 let kSampleCount
 
@@ -29,30 +27,6 @@ function getStartedSpans (context) {
   return context._trace.started
 }
 
-function generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, timestamp }) {
-  const labels = { 'thread name': threadName }
-  if (spanId) {
-    labels['span id'] = spanId
-  }
-  if (rootSpanId) {
-    labels['local root span id'] = rootSpanId
-  }
-  if (webTags && Object.keys(webTags).length !== 0) {
-    labels['trace endpoint'] = endpointNameFromTags(webTags)
-  } else if (endpoint) {
-    // fallback to endpoint computed when sample was taken
-    labels['trace endpoint'] = endpoint
-  }
-  // Incoming timestamps are in microseconds, we emit nanos.
-  labels['end_timestamp_ns'] = timestamp * 1000n
-
-  return labels
-}
-
-function getSpanContextTags (span) {
-  return span.context()._tags
-}
-
 function isWebServerSpan (tags) {
   return tags[SPAN_TYPE] === WEB
 }
@@ -64,6 +38,38 @@ function endpointNameFromTags (tags) {
   ].filter(v => v).join(' ')
 }
 
+function getWebTags (startedSpans, i, span) {
+  // Are web tags for this span already memoized?
+  const memoizedWebTags = span[MemoizedWebTags]
+  if (memoizedWebTags !== undefined) {
+    return memoizedWebTags
+  }
+  // No, we'll have to memoize a new value
+  function memoize (tags) {
+    span[MemoizedWebTags] = tags
+    return tags
+  }
+  // Is this span itself a web span?
+  const context = span.context()
+  const tags = context._tags
+  if (isWebServerSpan(tags)) {
+    return memoize(tags)
+  }
+  // It isn't. Get parent's web tags (memoize them too recursively.)
+  // There might be several webspans, for example with next.js, http plugin creates the first span
+  // and then next.js plugin creates a child span, and this child span has the correct endpoint
+  // information. That's why we always use the tags of the closest ancestor web span.
+  const parentId = context._parentId
+  while (--i >= 0) {
+    const ispan = startedSpans[i]
+    if (ispan.context()._spanId === parentId) {
+      return memoize(getWebTags(startedSpans, i, ispan))
+    }
+  }
+  // Local root span with no web span
+  return memoize(null)
+}
+
 class NativeWallProfiler {
   constructor (options = {}) {
     this.type = 'wall'
@@ -71,13 +77,29 @@ class NativeWallProfiler {
     this._flushIntervalMillis = options.flushInterval || 60 * 1e3 // 60 seconds
     this._codeHotspotsEnabled = !!options.codeHotspotsEnabled
     this._endpointCollectionEnabled = !!options.endpointCollectionEnabled
-    this._withContexts = this._codeHotspotsEnabled || this._endpointCollectionEnabled
+    this._timelineEnabled = !!options.timelineEnabled
+    this._cpuProfilingEnabled = !!options.cpuProfilingEnabled
+    // We need to capture span data into the sample context for either code hotspots
+    // or endpoint collection.
+    this._captureSpanData = this._codeHotspotsEnabled || this._endpointCollectionEnabled
+    // We need to run the pprof wall profiler with sample contexts if we're either
+    // capturing span data or timeline is enabled (so we need sample timestamps, and for now
+    // timestamps require the sample contexts feature in the pprof wall profiler), or
+    // cpu profiling is enabled.
+    this._withContexts = this._captureSpanData || this._timelineEnabled || this._cpuProfilingEnabled
     this._v8ProfilerBugWorkaroundEnabled = !!options.v8ProfilerBugWorkaroundEnabled
     this._mapper = undefined
     this._pprof = undefined
 
-    // Bind to this so the same value can be used to unsubscribe later
-    this._enter = this._enter.bind(this)
+    // Bind these to this so they can be used as callbacks
+    if (this._withContexts) {
+      if (this._captureSpanData) {
+        this._enter = this._enter.bind(this)
+        this._spanFinished = this._spanFinished.bind(this)
+      }
+    }
+    this._generateLabels = this._generateLabels.bind(this)
+
     this._logger = options.logger
     this._started = false
   }
@@ -111,19 +133,25 @@ class NativeWallProfiler {
       sourceMapper: this._mapper,
       withContexts: this._withContexts,
       lineNumbers: false,
-      workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled
+      workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled,
+      collectCpuTime: this._cpuProfilingEnabled
     })
 
     if (this._withContexts) {
-      this._profilerState = this._pprof.time.getState()
       this._currentContext = {}
       this._pprof.time.setContext(this._currentContext)
-      this._lastSpan = undefined
-      this._lastStartedSpans = undefined
-      this._lastSampleCount = 0
 
-      beforeCh.subscribe(this._enter)
-      enterCh.subscribe(this._enter)
+      if (this._captureSpanData) {
+        this._profilerState = this._pprof.time.getState()
+        this._lastSpan = undefined
+        this._lastStartedSpans = undefined
+        this._lastWebTags = undefined
+        this._lastSampleCount = 0
+
+        beforeCh.subscribe(this._enter)
+        enterCh.subscribe(this._enter)
+        spanFinishCh.subscribe(this._spanFinished)
+      }
     }
 
     this._started = true
@@ -144,11 +172,17 @@ class NativeWallProfiler {
 
     const span = getActiveSpan()
     if (span) {
+      const context = span.context()
       this._lastSpan = span
-      this._lastStartedSpans = getStartedSpans(span.context())
+      const startedSpans = getStartedSpans(context)
+      this._lastStartedSpans = startedSpans
+      if (this._endpointCollectionEnabled) {
+        this._lastWebTags = getWebTags(startedSpans, startedSpans.length, span)
+      }
     } else {
       this._lastStartedSpans = undefined
       this._lastSpan = undefined
+      this._lastWebTags = undefined
     }
   }
 
@@ -163,21 +197,17 @@ class NativeWallProfiler {
         context.rootSpanId = rootSpan.context().toSpanId()
       }
     }
-    if (this._endpointCollectionEnabled) {
-      const startedSpans = this._lastStartedSpans
-      // Find the first webspan starting from the end:
-      // There might be several webspans, for example with next.js, http plugin creates a first span
-      // and then next.js plugin creates a child span, and this child span haves the correct endpoint information.
-      for (let i = startedSpans.length - 1; i >= 0; i--) {
-        const tags = getSpanContextTags(startedSpans[i])
-        if (isWebServerSpan(tags)) {
-          context.webTags = tags
-          // endpoint may not be determined yet, but keep it as fallback
-          // if tags are not available anymore during serialization
-          context.endpoint = endpointNameFromTags(tags)
-          break
-        }
-      }
+    if (this._lastWebTags) {
+      context.webTags = this._lastWebTags
+      // endpoint may not be determined yet, but keep it as fallback
+      // if tags are not available anymore during serialization
+      context.endpoint = endpointNameFromTags(this._lastWebTags)
+    }
+  }
+
+  _spanFinished (span) {
+    if (span[MemoizedWebTags]) {
+      span[MemoizedWebTags] = undefined
     }
   }
 
@@ -193,23 +223,75 @@ class NativeWallProfiler {
 
   _stop (restart) {
     if (!this._started) return
-    if (this._withContexts) {
+
+    if (this._captureSpanData) {
       // update last sample context if needed
       this._enter()
       this._lastSampleCount = 0
     }
-    const profile = this._pprof.time.stop(restart, this._withContexts ? generateLabels : undefined)
+    const profile = this._pprof.time.stop(restart, this._generateLabels)
+
     if (restart) {
       const v8BugDetected = this._pprof.time.v8ProfilerStuckEventLoopDetected()
       if (v8BugDetected !== 0) {
         this._reportV8bug(v8BugDetected === 1)
       }
+    } else {
+      if (this._captureSpanData) {
+        beforeCh.unsubscribe(this._enter)
+        enterCh.unsubscribe(this._enter)
+        spanFinishCh.unsubscribe(this._spanFinished)
+        this._profilerState = undefined
+        this._lastSpan = undefined
+        this._lastStartedSpans = undefined
+        this._lastWebTags = undefined
+      }
+      this._started = false
     }
+
     return profile
   }
 
-  profile () {
-    return this._stop(true)
+  _generateLabels ({ node, context }) {
+    // check for special node that represents CPU time all non-JS threads.
+    // In that case only return a special thread name label since we cannot associate any timestamp/span/endpoint to it.
+    if (node.name === this._pprof.time.constants.NON_JS_THREADS_FUNCTION_NAME) {
+      return getNonJSThreadsLabels()
+    }
+
+    if (context == null) {
+      // generateLabels is also called for samples without context.
+      // In that case just return thread labels.
+      return getThreadLabels()
+    }
+
+    const labels = { ...getThreadLabels() }
+
+    const { context: { spanId, rootSpanId, webTags, endpoint }, timestamp } = context
+
+    if (this._timelineEnabled) {
+      // Incoming timestamps are in microseconds, we emit nanos.
+      labels[END_TIMESTAMP_LABEL] = timestamp * 1000n
+    }
+
+    if (spanId) {
+      labels['span id'] = spanId
+    }
+    if (rootSpanId) {
+      labels['local root span id'] = rootSpanId
+    }
+    if (webTags && Object.keys(webTags).length !== 0) {
+      labels['trace endpoint'] = endpointNameFromTags(webTags)
+    } else if (endpoint) {
+      // fallback to endpoint computed when sample was taken
+      labels['trace endpoint'] = endpoint
+    }
+
+    return labels
+  }
+
+  profile (restart) {
+    return this._stop(restart)
   }
 
   encode (profile) {
@@ -217,17 +299,11 @@ class NativeWallProfiler {
   }
 
   stop () {
-    if (!this._started) return
-
-    const profile = this._stop(false)
-    if (this._withContexts) {
-      beforeCh.unsubscribe(this._enter)
-      enterCh.unsubscribe(this._enter)
-      this._profilerState = undefined
-    }
+    this._stop(false)
+  }
 
-    this._started = false
-    return profile
+  isStarted () {
+    return this._started
   }
 }
 

package/packages/dd-trace/src/proxy.js

@@ -10,6 +10,7 @@ const PluginManager = require('./plugin_manager')
 const remoteConfig = require('./appsec/remote_config')
 const AppsecSdk = require('./appsec/sdk')
 const dogstatsd = require('./dogstatsd')
+const spanleak = require('./spanleak')
 
 class Tracer extends NoopProxy {
   constructor () {
@@ -18,6 +19,7 @@ class Tracer extends NoopProxy {
     this._initialized = false
     this._pluginManager = new PluginManager(this)
     this.dogstatsd = new dogstatsd.NoopDogStatsDClient()
+    this._tracingInitialized = false
   }
 
   init (options) {
@@ -27,6 +29,7 @@ class Tracer extends NoopProxy {
 
     try {
       const config = new Config(options) // TODO: support dynamic code config
+      telemetry.start(config, this._pluginManager)
 
       if (config.dogstatsd) {
         // Custom Metrics
@@ -35,6 +38,19 @@ class Tracer extends NoopProxy {
         setInterval(() => {
           this.dogstatsd.flush()
         }, 10 * 1000).unref()
+
+        process.once('beforeExit', () => {
+          this.dogstatsd.flush()
+        })
+      }
+
+      if (config.spanLeakDebug > 0) {
+        if (config.spanLeakDebug === spanleak.MODES.LOG) {
+          spanleak.enableLogging()
+        } else if (config.spanLeakDebug === spanleak.MODES.GC_AND_LOG) {
+          spanleak.enableGarbageCollection()
+        }
+        spanleak.startScrubber()
       }
 
       if (config.remoteConfig.enabled && !config.isCiVisibility) {
@@ -46,11 +62,7 @@ class Tracer extends NoopProxy {
           } else {
             config.configure(conf.lib_config, true)
           }
-
-          if (config.tracing) {
-            this._tracer.configure(config)
-            this._pluginManager.configure(config)
-          }
+          this._enableOrDisableTracing(config)
         })
       }
 
@@ -62,35 +74,22 @@ class Tracer extends NoopProxy {
         // do not stop tracer initialization if the profiler fails to be imported
         try {
           const profiler = require('./profiler')
-          profiler.start(config)
+          this._profilerStarted = profiler.start(config)
         } catch (e) {
           log.error(e)
         }
       }
+      if (!this._profilerStarted) {
+        this._profilerStarted = Promise.resolve(false)
+      }
 
       if (config.runtimeMetrics) {
         runtimeMetrics.start(config)
       }
 
-      if (config.tracing) {
-        // TODO: This should probably not require tracing to be enabled.
-        telemetry.start(config, this._pluginManager)
-
-        // dirty require for now so zero appsec code is executed unless explicitly enabled
-        if (config.appsec.enabled) {
-          require('./appsec').enable(config)
-        }
-
-        this._tracer = new DatadogTracer(config)
-        this.appsec = new AppsecSdk(this._tracer, config)
-
-        if (config.iast.enabled) {
-          require('./appsec/iast').enable(config, this._tracer)
-        }
-
-        this._pluginManager.configure(config)
-        setStartupLogPluginManager(this._pluginManager)
+      this._enableOrDisableTracing(config)
 
+      if (config.tracing) {
         if (config.isManualApiEnabled) {
           const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin')
           this._testApiManualPlugin = new TestApiManualPlugin(this)
@@ -104,6 +103,39 @@ class Tracer extends NoopProxy {
     return this
   }
 
+  _enableOrDisableTracing (config) {
+    if (config.tracing !== false) {
+      // dirty require for now so zero appsec code is executed unless explicitly enabled
+      if (config.appsec.enabled) {
+        require('./appsec').enable(config)
+      }
+      if (!this._tracingInitialized) {
+        this._tracer = new DatadogTracer(config)
+        this.appsec = new AppsecSdk(this._tracer, config)
+        this._tracingInitialized = true
+      }
+      if (config.iast.enabled) {
+        require('./appsec/iast').enable(config, this._tracer)
+      }
+    } else if (this._tracingInitialized) {
+      require('./appsec').disable()
+      require('./appsec/iast').disable()
+    }
+
+    if (this._tracingInitialized) {
+      this._tracer.configure(config)
+      this._pluginManager.configure(config)
+      setStartupLogPluginManager(this._pluginManager)
+    }
+  }
+
+  profilerStarted () {
+    if (!this._profilerStarted) {
+      throw new Error('profilerStarted() must be called after init()')
+    }
+    return this._profilerStarted
+  }
+
   use () {
     this._pluginManager.configurePlugin(...arguments)
     return this

package/packages/dd-trace/src/ritm.js

@@ -3,7 +3,7 @@
 const path = require('path')
 const Module = require('module')
 const parse = require('module-details-from-path')
-const dc = require('../../diagnostics_channel')
+const dc = require('dc-polyfill')
 
 const origRequire = Module.prototype.require
 

package/packages/dd-trace/src/sampling_rule.js

@@ -0,0 +1,130 @@
+'use strict'
+
+const { globMatch } = require('../src/util')
+const RateLimiter = require('./rate_limiter')
+const Sampler = require('./sampler')
+
+class AlwaysMatcher {
+  match () {
+    return true
+  }
+}
+
+class GlobMatcher {
+  constructor (pattern, locator) {
+    this.pattern = pattern
+    this.locator = locator
+  }
+
+  match (span) {
+    const subject = this.locator(span)
+    if (!subject) return false
+    return globMatch(this.pattern, subject)
+  }
+}
+
+class RegExpMatcher {
+  constructor (pattern, locator) {
+    this.pattern = pattern
+    this.locator = locator
+  }
+
+  match (span) {
+    const subject = this.locator(span)
+    if (!subject) return false
+    return this.pattern.test(subject)
+  }
+}
+
+function matcher (pattern, locator) {
+  if (pattern instanceof RegExp) {
+    return new RegExpMatcher(pattern, locator)
+  }
+
+  if (typeof pattern === 'string' && pattern !== '*') {
+    return new GlobMatcher(pattern, locator)
+  }
+
+  return new AlwaysMatcher()
+}
+
+function makeTagLocator (tag) {
+  return (span) => span.context()._tags[tag]
+}
+
+function nameLocator (span) {
+  return span.context()._name
+}
+
+function serviceLocator (span) {
+  const { _tags: tags } = span.context()
+  return tags.service ||
+    tags['service.name'] ||
+    span.tracer()._service
+}
+
+class SamplingRule {
+  constructor ({ name, service, resource, tags, sampleRate = 1.0, maxPerSecond } = {}) {
+    this.matchers = []
+
+    if (name) {
+      this.matchers.push(matcher(name, nameLocator))
+    }
+    if (service) {
+      this.matchers.push(matcher(service, serviceLocator))
+    }
+    if (resource) {
+      this.matchers.push(matcher(resource, makeTagLocator('resource.name')))
+    }
+    for (const [key, value] of Object.entries(tags || {})) {
+      this.matchers.push(matcher(value, makeTagLocator(key)))
+    }
+
+    this._sampler = new Sampler(sampleRate)
+    this._limiter = undefined
+
+    if (Number.isFinite(maxPerSecond)) {
+      this._limiter = new RateLimiter(maxPerSecond)
+    }
+  }
+
+  static from (config) {
+    return new SamplingRule(config)
+  }
+
+  get sampleRate () {
+    return this._sampler.rate()
+  }
+
+  get effectiveRate () {
+    return this._limiter && this._limiter.effectiveRate()
+  }
+
+  get maxPerSecond () {
+    return this._limiter && this._limiter._rateLimit
+  }
+
+  match (span) {
+    for (const matcher of this.matchers) {
+      if (!matcher.match(span)) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  sample () {
+    if (!this._sampler.isSampled()) {
+      return false
+    }
+
+    if (this._limiter) {
+      return this._limiter.isAllowed()
+    }
+
+    return true
+  }
+}
+
+module.exports = SamplingRule

package/packages/dd-trace/src/service-naming/schemas/v0/storage.js

@@ -37,6 +37,11 @@ const redisConfig = {
 
 const storage = {
   client: {
+    aerospike: {
+      opName: () => 'aerospike.command',
+      serviceName: ({ tracerService, pluginConfig }) =>
+        pluginConfig.service || `${tracerService}-aerospike`
+    },
     'cassandra-driver': {
       opName: () => 'cassandra.query',
       serviceName: ({ tracerService, pluginConfig, system }) =>

package/packages/dd-trace/src/service-naming/schemas/v1/storage.js

@@ -22,6 +22,10 @@ function withFunction ({ tracerService, pluginConfig, params }) {
 
 const storage = {
   client: {
+    aerospike: {
+      opName: () => 'aerospike.command',
+      serviceName: configWithFallback
+    },
     'cassandra-driver': {
       opName: () => 'cassandra.query',
       serviceName: configWithFallback

package/packages/dd-trace/src/span_processor.js

@@ -27,10 +27,14 @@ class SpanProcessor {
     const active = []
     const formatted = []
     const trace = spanContext._trace
-    const { flushMinSpans } = this._config
+    const { flushMinSpans, tracing } = this._config
     const { started, finished } = trace
 
     if (trace.record === false) return
+    if (tracing === false) {
+      this._erase(trace, active)
+      return
+    }
     if (started.length === finished.length || finished.length >= flushMinSpans) {
       this._prioritySampler.sample(spanContext)
       this._spanSampler.sample(spanContext)
@@ -138,6 +142,10 @@ class SpanProcessor {
       }
     }
 
+    for (const span of trace.finished) {
+      span.context()._tags = {}
+    }
+
     trace.started = active
     trace.finished = []
   }