dd-trace 5.61.0 → 5.62.0

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -3,9 +3,12 @@
 const { performance, constants, PerformanceObserver } = require('perf_hooks')
 const { END_TIMESTAMP_LABEL, SPAN_ID_LABEL, LOCAL_ROOT_SPAN_ID_LABEL, encodeProfileAsync } = require('./shared')
 const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
-
+const PoissonProcessSamplingFilter = require('./poisson')
+const { availableParallelism, effectiveLibuvThreadCount } = require('../libuv-size')
 // perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file.
 const MS_TO_NS = 1_000_000
+// The number of sampling intervals that need to pass before we reset the Poisson process sampling instant.
+const POISSON_RESET_FACTOR = 2
 
 // While this is an "events profiler", meaning it emits a pprof file based on events observed as
 // perf_hooks events, the emitted pprof file uses the type "timeline".
@@ -38,6 +41,24 @@ function labelFromStrStr (stringTable, keyStr, valStr) {
   return labelFromStr(stringTable, stringTable.dedup(keyStr), valStr)
 }
 
+function getSamplingIntervalMillis (options) {
+  return (options.samplingInterval || 1e3 / 99) // 99Hz
+}
+
+function getMaxSamples (options) {
+  const cpuSamplingInterval = getSamplingIntervalMillis(options)
+  const flushInterval = options.flushInterval || 65 * 1e3 // 60 seconds
+  const maxCpuSamples = flushInterval / cpuSamplingInterval
+
+  // The lesser of max parallelism and libuv thread pool size, plus one so we can detect
+  // oversubscription on libuv thread pool, plus another one for GC.
+  const factor = Math.max(1, Math.min(availableParallelism(), effectiveLibuvThreadCount)) + 2
+
+  // Let's not go overboard with too large limit and cap it at 100k. With current defaults, the
+  // value will be 65000/10.1*(4+2) = 38613.
+  return Math.min(100_000, Math.floor(maxCpuSamples * factor))
+}
+
 class GCDecorator {
   constructor (stringTable) {
     this.stringTable = stringTable
@@ -181,12 +202,15 @@ const decoratorTypes = {
 
 // Translates performance entries into pprof samples.
 class EventSerializer {
-  constructor () {
+  #sampleCount = 0
+
+  constructor (maxSamples) {
     this.stringTable = new StringTable()
     this.samples = []
     this.locations = []
     this.functions = []
     this.decorators = {}
+    this.maxSamples = maxSamples
 
     // A synthetic single-frame location to serve as the location for timeline
     // samples. We need these as the profiling backend (mimicking official pprof
@@ -204,6 +228,31 @@ class EventSerializer {
   }
 
   addEvent (item) {
+    if (this.samples.length < this.maxSamples) {
+      const sample = this.#createSample(item)
+      if (sample !== undefined) {
+        this.samples.push(sample)
+        this.#sampleCount++
+      }
+    } else {
+      this.#sampleCount++
+      // Reservoir sampling
+      const replacementIndex = Math.floor(Math.random() * this.#sampleCount)
+      if (replacementIndex < this.maxSamples) {
+        const sample = this.#createSample(item)
+        if (sample === undefined) {
+          this.#sampleCount-- // unlikely
+        } else {
+          // This will cause the samples to no longer be sorted in their array
+          // by their end time. This is fine as the backend has no ordering
+          // expectations.
+          this.samples[replacementIndex] = sample
+        }
+      }
+    }
+  }
+
+  #createSample (item) {
     const { entryType, startTime, duration, _ddSpanId, _ddRootSpanId } = item
     let decorator = this.decorators[entryType]
     if (!decorator) {
@@ -236,7 +285,7 @@ class EventSerializer {
       label
     }
     decorator.decorateSample(sampleInput, item)
-    this.samples.push(new Sample(sampleInput))
+    return new Sample(sampleInput)
   }
 
   createProfile (startDate, endDate) {
@@ -324,50 +373,13 @@ class DatadogInstrumentationEventSource {
   }
 }
 
-class CompositeEventSource {
-  constructor (sources) {
-    this.sources = sources
-  }
-
-  start () {
-    this.sources.forEach(s => s.start())
-  }
-
-  stop () {
-    this.sources.forEach(s => s.stop())
-  }
-}
-
-function createPossionProcessSamplingFilter (samplingIntervalMillis) {
-  let nextSamplingInstant = performance.now()
-  let currentSamplingInstant = 0
-  setNextSamplingInstant()
-
-  return event => {
-    const endTime = event.startTime + event.duration
-    while (endTime >= nextSamplingInstant) {
-      setNextSamplingInstant()
-    }
-    // An event is sampled if it started before, and ended on or after a sampling instant. The above
-    // while loop will ensure that the ending invariant is always true for the current sampling
-    // instant so we don't have to test for it below. Across calls, the invariant also holds as long
-    // as the events arrive in endTime order. This is true for events coming from
-    // DatadogInstrumentationEventSource; they will be ordered by endTime by virtue of this method
-    // being invoked synchronously with the plugins' finish() handler which evaluates
-    // performance.now(). OTOH, events coming from NodeAPIEventSource (GC in typical setup) might be
-    // somewhat delayed as they are queued by Node, so they can arrive out of order with regard to
-    // events coming from the non-queued source. By omitting the endTime check, we will pass through
-    // some short events that started and ended before the current sampling instant. OTOH, if we
-    // were to check for this.currentSamplingInstant <= endTime, we would discard some long events
-    // that also ended before the current sampling instant. We'd rather err on the side of including
-    // some short events than excluding some long events.
-    return event.startTime < currentSamplingInstant
-  }
-
-  function setNextSamplingInstant () {
-    currentSamplingInstant = nextSamplingInstant
-    nextSamplingInstant -= Math.log(1 - Math.random()) * samplingIntervalMillis
-  }
+function createPoissonProcessSamplingFilter (samplingIntervalMillis) {
+  const poissonFilter = new PoissonProcessSamplingFilter({
+    samplingInterval: samplingIntervalMillis,
+    resetInterval: samplingIntervalMillis * POISSON_RESET_FACTOR,
+    now: performance.now.bind(performance)
+  })
+  return poissonFilter.filter.bind(poissonFilter)
 }
 
 /**
@@ -376,45 +388,51 @@ function createPossionProcessSamplingFilter (samplingIntervalMillis) {
  */
 class EventsProfiler {
   type = 'events'
-  eventSerializer = new EventSerializer()
+  #maxSamples
+  #eventSerializer
+  #eventSources
 
   constructor (options = {}) {
-    const eventHandler = event => this.eventSerializer.addEvent(event)
+    this.#maxSamples = getMaxSamples(options)
+    this.#eventSerializer = new EventSerializer(this.#maxSamples)
+
+    const eventHandler = event => this.#eventSerializer.addEvent(event)
     const eventFilter = options.timelineSamplingEnabled
-      // options.samplingInterval comes in microseconds, we need millis
-      ? createPossionProcessSamplingFilter((options.samplingInterval ?? 1e6 / 99) / 1000)
-      : _ => true
+      ? createPoissonProcessSamplingFilter(getSamplingIntervalMillis(options))
+      : () => true
     const filteringEventHandler = event => {
       if (eventFilter(event)) {
         eventHandler(event)
       }
     }
 
-    this.eventSource = options.codeHotspotsEnabled
+    this.#eventSources = options.codeHotspotsEnabled
       // Use Datadog instrumentation to collect events with span IDs. Still use
       // Node API for GC events.
-      ? new CompositeEventSource([
-        new DatadogInstrumentationEventSource(eventHandler, eventFilter),
-        new NodeApiEventSource(filteringEventHandler, ['gc'])
-      ])
+      ? [
+          new DatadogInstrumentationEventSource(eventHandler, eventFilter),
+          new NodeApiEventSource(filteringEventHandler, ['gc']),
+        ]
       // Use Node API instrumentation to collect events without span IDs
-      : new NodeApiEventSource(filteringEventHandler)
+      : [
+          new NodeApiEventSource(filteringEventHandler)
+        ]
   }
 
   start () {
-    this.eventSource.start()
+    this.#eventSources.forEach(s => s.start())
   }
 
   stop () {
-    this.eventSource.stop()
+    this.#eventSources.forEach(s => s.stop())
   }
 
   profile (restart, startDate, endDate) {
     if (!restart) {
       this.stop()
     }
-    const thatEventSerializer = this.eventSerializer
-    this.eventSerializer = new EventSerializer()
+    const thatEventSerializer = this.#eventSerializer
+    this.#eventSerializer = new EventSerializer(this.#maxSamples)
     return () => thatEventSerializer.createProfile(startDate, endDate)
   }
 

package/packages/dd-trace/src/profiling/profilers/poisson.js

@@ -0,0 +1,105 @@
+'use strict'
+
+class PoissonProcessSamplingFilter {
+  #currentSamplingInstant = 0
+  #nextSamplingInstant
+  #samplingInterval
+  #resetInterval
+  #now
+  #lastNow = Number.NEGATIVE_INFINITY
+  #samplingInstantCount = 0
+
+  constructor ({ samplingInterval, now, resetInterval }) {
+    if (samplingInterval <= 0) {
+      throw new RangeError(`samplingInterval (${samplingInterval}) must be greater than 0`)
+    }
+    if (resetInterval < samplingInterval) {
+      throw new RangeError(
+        `resetInterval (${resetInterval}) must be greater than samplingInterval (${samplingInterval})`
+      )
+    }
+    if (typeof now !== 'function') {
+      throw new TypeError('now must be a function')
+    }
+    this.#samplingInterval = samplingInterval
+    this.#resetInterval = resetInterval
+    this.#now = now
+    this.#nextSamplingInstant = this.#callNow()
+    this.#setNextSamplingInstant()
+  }
+
+  get currentSamplingInstant () {
+    return this.#currentSamplingInstant
+  }
+
+  get nextSamplingInstant () {
+    return this.#nextSamplingInstant
+  }
+
+  get samplingInstantCount () {
+    return this.#samplingInstantCount
+  }
+
+  #callNow () {
+    const nowValue = this.#now()
+    if (typeof nowValue !== 'number') {
+      throw new TypeError('now() must return a number')
+    }
+    if (nowValue < this.#lastNow) {
+      throw new RangeError('now() must return a value greater than or equal to the last returned value')
+    }
+    this.#lastNow = nowValue
+    return nowValue
+  }
+
+  filter (event) {
+    const endTime = event.startTime + event.duration
+    // We're using the end times of events as an approximation of current time as events are
+    // expected to be reported close to where they ended. If the end time (and thus, presumably, the
+    // current time) is past the next sampling instant, we make it the current sampling instant and
+    // compute the next sampling instant in its future.
+    if (endTime >= this.#nextSamplingInstant) {
+      // All observed events are supposed to have happened in the past. For purposes of advancing
+      // the next sampling instant, we cap endTime to now(). This protects us from advancing it far
+      // into future if we receive an event with erroneously long duration, which would also take
+      // many iterations of the below "while" loop.
+      const cappedEndTime = Math.min(endTime, this.#callNow())
+
+      // If nextSamplingInstant is far in cappedEndTime's past, first advance it close to it. This
+      // can happen if we didn't receive any events for a while. Since a Poisson process has no
+      // memory, we can reset it anytime. This will ensure that the "while" loop below runs at most
+      // few iterations.
+      const earliestContinuousPast = cappedEndTime - this.#resetInterval
+      if (this.#nextSamplingInstant < earliestContinuousPast) {
+        this.#nextSamplingInstant = earliestContinuousPast
+      }
+
+      // Advance the next sampling instant until it is in cappedEndTime's future.
+      while (cappedEndTime >= this.#nextSamplingInstant) {
+        this.#setNextSamplingInstant()
+      }
+    }
+    // An event is sampled if it started before, and ended on or after a sampling instant. The above
+    // while loop will ensure that the ending invariant is always true for the current sampling
+    // instant so we don't have to test for it below. Across calls, the invariant also holds as long
+    // as the events arrive in endTime order. This is true for events coming from
+    // DatadogInstrumentationEventSource; they will be ordered by endTime by virtue of this method
+    // being invoked synchronously with the plugins' finish() handler which evaluates
+    // performance.now(). OTOH, events coming from NodeAPIEventSource (GC in typical setup) might be
+    // somewhat delayed as they are queued by Node, so they can arrive out of order with regard to
+    // events coming from the non-queued source. By omitting the endTime check, we will pass through
+    // some short events that started and ended before the current sampling instant. OTOH, if we
+    // were to check for this.currentSamplingInstant <= endTime, we would discard some long events
+    // that also ended before the current sampling instant. We'd rather err on the side of including
+    // some short events than excluding some long events.
+    return event.startTime < this.#currentSamplingInstant
+  }
+
+  #setNextSamplingInstant () {
+    this.#currentSamplingInstant = this.#nextSamplingInstant
+    this.#nextSamplingInstant -= Math.log(1 - Math.random()) * this.#samplingInterval
+    this.#samplingInstantCount++
+  }
+}
+
+module.exports = PoissonProcessSamplingFilter

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -75,7 +75,7 @@ class NativeWallProfiler {
   _started = false
 
   constructor (options = {}) {
-    this._samplingIntervalMicros = options.samplingInterval || 1e6 / 99 // 99hz
+    this._samplingIntervalMicros = (options.samplingInterval || 1e3 / 99) * 1000 // 99hz
     this._flushIntervalMillis = options.flushInterval || 60 * 1e3 // 60 seconds
     this._codeHotspotsEnabled = !!options.codeHotspotsEnabled
     this._endpointCollectionEnabled = !!options.endpointCollectionEnabled

package/packages/dd-trace/src/remote_config/manager.js

@@ -22,7 +22,7 @@ const kSupportsAckCallback = Symbol('kSupportsAckCallback')
 // There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest
 // with their own separated Client.ClientState.
 class RemoteConfigManager extends EventEmitter {
-  static get kPreUpdate () { return kPreUpdate }
+  static kPreUpdate = kPreUpdate
 
   constructor (config) {
     super()