@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -0,0 +1,466 @@
+'use strict'
+
+const { performance, constants, PerformanceObserver } = require('perf_hooks')
+const {
+  Function,
+  Label,
+  Line,
+  Location,
+  Profile,
+  Sample,
+  StringTable,
+  ValueType,
+} = require('../../../../../vendor/dist/pprof-format')
+const { availableParallelism, effectiveLibuvThreadCount } = require('../libuv-size')
+const { END_TIMESTAMP_LABEL, SPAN_ID_LABEL, LOCAL_ROOT_SPAN_ID_LABEL, encodeProfileAsync } = require('./shared')
+const PoissonProcessSamplingFilter = require('./poisson')
+// perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file.
+const MS_TO_NS = 1_000_000
+// The number of sampling intervals that need to pass before we reset the Poisson process sampling instant.
+const POISSON_RESET_FACTOR = 2
+
+// While this is an "events profiler", meaning it emits a pprof file based on events observed as
+// perf_hooks events, the emitted pprof file uses the type "timeline".
+const pprofValueType = 'timeline'
+const pprofValueUnit = 'nanoseconds'
+
+const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS))
+
+function labelFromStr (stringTable, key, valStr) {
+  return new Label({ key, str: stringTable.dedup(safeToString(valStr)) })
+}
+
+// We don't want to invoke toString for objects and functions, rather we'll
+// provide dummy values. These values are not meant to emulate built-in toString
+// behavior.
+function safeToString (val) {
+  switch (typeof val) {
+    case 'string':
+      return val
+    case 'object':
+      return '[object]'
+    case 'function':
+      return '[function]'
+    default:
+      return String(val)
+  }
+}
+
+function labelFromStrStr (stringTable, keyStr, valStr) {
+  return labelFromStr(stringTable, stringTable.dedup(keyStr), valStr)
+}
+
+function getMaxSamples (options) {
+  const flushInterval = options.flushInterval || 65 * 1e3 // 65 seconds
+  const maxCpuSamples = flushInterval / options.samplingInterval
+
+  // The lesser of max parallelism and libuv thread pool size, plus one so we can detect
+  // oversubscription on libuv thread pool, plus another one for GC.
+  const factor = Math.max(1, Math.min(availableParallelism(), effectiveLibuvThreadCount)) + 2
+
+  // Let's not go overboard with too large limit and cap it at 100k. With current defaults, the
+  // value will be 65000/10.1*(4+2) = 38613.
+  return Math.min(100_000, Math.floor(maxCpuSamples * factor))
+}
+
+class GCDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.reasonLabelKey = stringTable.dedup('gc reason')
+    this.kindLabels = []
+    this.reasonLabels = []
+    this.flagObj = {}
+
+    const kindLabelKey = stringTable.dedup('gc type')
+
+    // Create labels for all GC performance flags and kinds of GC
+    for (const [key, value] of Object.entries(constants)) {
+      if (key.startsWith('NODE_PERFORMANCE_GC_FLAGS_')) {
+        this.flagObj[key.slice(26).toLowerCase()] = value
+      } else if (key.startsWith('NODE_PERFORMANCE_GC_')) {
+        // It's a constant for a kind of GC
+        const kind = key.slice(20).toLowerCase()
+        this.kindLabels[value] = labelFromStr(stringTable, kindLabelKey, kind)
+      }
+    }
+  }
+
+  decorateSample (sampleInput, item) {
+    const { kind, flags } = item.detail
+    sampleInput.label.push(this.kindLabels[kind])
+    const reasonLabel = this.getReasonLabel(flags)
+    if (reasonLabel) {
+      sampleInput.label.push(reasonLabel)
+    }
+  }
+
+  getReasonLabel (flags) {
+    if (flags === 0) {
+      return null
+    }
+    let reasonLabel = this.reasonLabels[flags]
+    if (!reasonLabel) {
+      const reasons = []
+      for (const [key, value] of Object.entries(this.flagObj)) {
+        if (value & flags) {
+          reasons.push(key)
+        }
+      }
+      const reasonStr = reasons.join(',')
+      reasonLabel = labelFromStr(this.stringTable, this.reasonLabelKey, reasonStr)
+      this.reasonLabels[flags] = reasonLabel
+    }
+    return reasonLabel
+  }
+}
+
+class DNSDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.addressLabelKey = stringTable.dedup('address')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, item.name)
+    const detail = item.detail
+    switch (op) {
+      case 'lookup':
+        addLabel(this.hostLabelKey, detail.hostname)
+        break
+      case 'lookupService':
+        addLabel(this.addressLabelKey, detail.host)
+        labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+        break
+      case 'getHostByAddr':
+        addLabel(this.addressLabelKey, detail.host)
+        break
+      default:
+        if (op.startsWith('query')) {
+          addLabel(this.hostLabelKey, detail.host)
+        }
+    }
+  }
+}
+
+class NetDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, op)
+    if (op === 'connect') {
+      const detail = item.detail
+      addLabel(this.hostLabelKey, detail.host)
+      labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+    }
+  }
+}
+
+class FilesystemDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    for (const [key, value] of Object.entries(item.detail)) {
+      switch (typeof value) {
+        case 'string':
+          labels.push(labelFromStrStr(stringTable, key, value))
+          break
+        case 'number':
+          labels.push(new Label({ key: stringTable.dedup(key), num: value }))
+      }
+    }
+  }
+}
+
+// Keys correspond to PerformanceEntry.entryType, values are constructor
+// functions for type-specific decorators.
+const decoratorTypes = {
+  fs: FilesystemDecorator,
+  dns: DNSDecorator,
+  gc: GCDecorator,
+  net: NetDecorator,
+}
+
+// Translates performance entries into pprof samples.
+class EventSerializer {
+  #sampleCount = 0
+
+  constructor (maxSamples) {
+    this.stringTable = new StringTable()
+    this.samples = []
+    this.locations = []
+    this.functions = []
+    this.decorators = {}
+    this.maxSamples = maxSamples
+
+    // A synthetic single-frame location to serve as the location for timeline
+    // samples. We need these as the profiling backend (mimicking official pprof
+    // tool's behavior) ignores these.
+    const fn = new Function({ id: this.functions.length + 1, name: this.stringTable.dedup('') })
+    this.functions.push(fn)
+    const line = new Line({ functionId: fn.id })
+    const location = new Location({ id: this.locations.length + 1, line: [line] })
+    this.locations.push(location)
+    this.locationId = [location.id]
+
+    this.timestampLabelKey = this.stringTable.dedup(END_TIMESTAMP_LABEL)
+    this.spanIdKey = this.stringTable.dedup(SPAN_ID_LABEL)
+    this.rootSpanIdKey = this.stringTable.dedup(LOCAL_ROOT_SPAN_ID_LABEL)
+  }
+
+  addEvent (item) {
+    if (this.samples.length < this.maxSamples) {
+      const sample = this.#createSample(item)
+      if (sample !== undefined) {
+        this.samples.push(sample)
+        this.#sampleCount++
+      }
+    } else {
+      this.#sampleCount++
+      // Reservoir sampling
+      const replacementIndex = Math.floor(Math.random() * this.#sampleCount)
+      if (replacementIndex < this.maxSamples) {
+        const sample = this.#createSample(item)
+        if (sample === undefined) {
+          this.#sampleCount-- // unlikely
+        } else {
+          // This will cause the samples to no longer be sorted in their array
+          // by their end time. This is fine as the backend has no ordering
+          // expectations.
+          this.samples[replacementIndex] = sample
+        }
+      }
+    }
+  }
+
+  #createSample (item) {
+    const { entryType, startTime, duration, _ddSpanId, _ddRootSpanId } = item
+    let decorator = this.decorators[entryType]
+    if (!decorator) {
+      const DecoratorCtor = decoratorTypes[entryType]
+      if (DecoratorCtor) {
+        decorator = new DecoratorCtor(this.stringTable)
+        decorator.eventTypeLabel = labelFromStrStr(this.stringTable, 'event', entryType)
+        this.decorators[entryType] = decorator
+      } else {
+        // Shouldn't happen but it's better to not rely on observer only getting
+        // requested event types.
+        return
+      }
+    }
+    const endTime = startTime + duration
+    const label = [
+      decorator.eventTypeLabel,
+      new Label({ key: this.timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) }),
+    ]
+    if (_ddSpanId) {
+      label.push(
+        new Label({ key: this.spanIdKey, num: _ddSpanId }))
+    }
+    if (_ddRootSpanId) {
+      label.push(new Label({ key: this.rootSpanIdKey, num: _ddRootSpanId }))
+    }
+
+    const sampleInput = {
+      value: [Math.round(duration * MS_TO_NS)],
+      locationId: this.locationId,
+      label,
+    }
+    decorator.decorateSample(sampleInput, item)
+    return new Sample(sampleInput)
+  }
+
+  createProfile (startDate, endDate) {
+    const timeValueType = new ValueType({
+      type: this.stringTable.dedup(pprofValueType),
+      unit: this.stringTable.dedup(pprofValueUnit),
+    })
+
+    return new Profile({
+      sampleType: [timeValueType],
+      timeNanos: endDate.getTime() * MS_TO_NS,
+      periodType: timeValueType,
+      period: 1,
+      durationNanos: (endDate.getTime() - startDate.getTime()) * MS_TO_NS,
+      sample: this.samples,
+      location: this.locations,
+      function: this.functions,
+      stringTable: this.stringTable,
+    })
+  }
+}
+
+function add (items) {
+  for (const item of items.getEntries()) {
+    this.eventHandler(item)
+  }
+}
+
+/**
+ * Class that sources timeline events through Node.js performance measurement APIs.
+ */
+class NodeApiEventSource {
+  constructor (eventHandler, entryTypes) {
+    this.eventHandler = eventHandler
+    this.observer = undefined
+    this.entryTypes = entryTypes || Object.keys(decoratorTypes)
+  }
+
+  start () {
+    // if already started, do nothing
+    if (this.observer) return
+
+    this.observer = new PerformanceObserver(add.bind(this))
+    this.observer.observe({ entryTypes: this.entryTypes })
+  }
+
+  stop () {
+    if (this.observer) {
+      this.observer.disconnect()
+      this.observer = undefined
+    }
+  }
+}
+
+class DatadogInstrumentationEventSource {
+  constructor (eventHandler, eventFilter) {
+    // List all entries explicitly for bundlers to pick up the require calls correctly.
+    const plugins = [
+      require('./event_plugins/dns_lookup'),
+      require('./event_plugins/dns_lookupservice'),
+      require('./event_plugins/dns_resolve'),
+      require('./event_plugins/dns_reverse'),
+      require('./event_plugins/fs'),
+      require('./event_plugins/net'),
+    ]
+    this.plugins = plugins.map((Plugin) => {
+      return new Plugin(eventHandler, eventFilter)
+    })
+
+    this.started = false
+  }
+
+  start () {
+    if (!this.started) {
+      for (const plugin of this.plugins) {
+        plugin.configure({ enabled: true })
+      }
+      this.started = true
+    }
+  }
+
+  stop () {
+    if (this.started) {
+      for (const plugin of this.plugins) {
+        plugin.configure({ enabled: false })
+      }
+      this.started = false
+    }
+  }
+}
+
+function createPoissonProcessSamplingFilter (samplingIntervalMillis) {
+  const poissonFilter = new PoissonProcessSamplingFilter({
+    samplingInterval: samplingIntervalMillis,
+    resetInterval: samplingIntervalMillis * POISSON_RESET_FACTOR,
+    now: performance.now.bind(performance),
+  })
+  return poissonFilter.filter.bind(poissonFilter)
+}
+
+/**
+ * This class generates pprof files with timeline events. It combines an event
+ * source with a sampling event filter and an event serializer.
+ */
+class EventsProfiler {
+  #maxSamples = 0
+  #timelineSamplingEnabled = false
+  #eventSerializer
+  #eventSources
+
+  get type () { return 'events' }
+
+  constructor (options = {}) {
+    this.#maxSamples = getMaxSamples(options)
+    this.#timelineSamplingEnabled = !!options.timelineSamplingEnabled
+    this.#eventSerializer = new EventSerializer(this.#maxSamples)
+
+    const eventHandler = event => this.#eventSerializer.addEvent(event)
+    const eventFilter = this.#timelineSamplingEnabled
+      ? createPoissonProcessSamplingFilter(options.samplingInterval)
+      : () => true
+    const filteringEventHandler = event => {
+      if (eventFilter(event)) {
+        eventHandler(event)
+      }
+    }
+
+    this.#eventSources = options.codeHotspotsEnabled
+      // Use Datadog instrumentation to collect events with span IDs. Still use
+      // Node API for GC events.
+      ? [
+          new DatadogInstrumentationEventSource(eventHandler, eventFilter),
+          new NodeApiEventSource(filteringEventHandler, ['gc']),
+        ]
+      // Use Node API instrumentation to collect events without span IDs
+      : [
+          new NodeApiEventSource(filteringEventHandler),
+        ]
+  }
+
+  start () {
+    for (const source of this.#eventSources) {
+      source.start()
+    }
+  }
+
+  stop () {
+    for (const source of this.#eventSources) {
+      source.stop()
+    }
+  }
+
+  profile (restart, startDate, endDate) {
+    if (!restart) {
+      this.stop()
+    }
+    const thatEventSerializer = this.#eventSerializer
+    this.#eventSerializer = new EventSerializer(this.#maxSamples)
+    return () => thatEventSerializer.createProfile(startDate, endDate)
+  }
+
+  getInfo () {
+    return {
+      maxSamples: this.#maxSamples,
+    }
+  }
+
+  encode (profile) {
+    return encodeProfileAsync(profile())
+  }
+}
+
+module.exports = EventsProfiler

package/packages/dd-trace/src/profiling/profilers/poisson.js

@@ -0,0 +1,105 @@
+'use strict'
+
+class PoissonProcessSamplingFilter {
+  #currentSamplingInstant = 0
+  #nextSamplingInstant
+  #samplingInterval
+  #resetInterval
+  #now
+  #lastNow = Number.NEGATIVE_INFINITY
+  #samplingInstantCount = 0
+
+  constructor ({ samplingInterval, now, resetInterval }) {
+    if (samplingInterval <= 0) {
+      throw new RangeError(`samplingInterval (${samplingInterval}) must be greater than 0`)
+    }
+    if (resetInterval < samplingInterval) {
+      throw new RangeError(
+        `resetInterval (${resetInterval}) must be greater than samplingInterval (${samplingInterval})`
+      )
+    }
+    if (typeof now !== 'function') {
+      throw new TypeError('now must be a function')
+    }
+    this.#samplingInterval = samplingInterval
+    this.#resetInterval = resetInterval
+    this.#now = now
+    this.#nextSamplingInstant = this.#callNow()
+    this.#setNextSamplingInstant()
+  }
+
+  get currentSamplingInstant () {
+    return this.#currentSamplingInstant
+  }
+
+  get nextSamplingInstant () {
+    return this.#nextSamplingInstant
+  }
+
+  get samplingInstantCount () {
+    return this.#samplingInstantCount
+  }
+
+  #callNow () {
+    const nowValue = this.#now()
+    if (typeof nowValue !== 'number') {
+      throw new TypeError('now() must return a number')
+    }
+    if (nowValue < this.#lastNow) {
+      throw new RangeError('now() must return a value greater than or equal to the last returned value')
+    }
+    this.#lastNow = nowValue
+    return nowValue
+  }
+
+  filter (event) {
+    const endTime = event.startTime + event.duration
+    // We're using the end times of events as an approximation of current time as events are
+    // expected to be reported close to where they ended. If the end time (and thus, presumably, the
+    // current time) is past the next sampling instant, we make it the current sampling instant and
+    // compute the next sampling instant in its future.
+    if (endTime >= this.#nextSamplingInstant) {
+      // All observed events are supposed to have happened in the past. For purposes of advancing
+      // the next sampling instant, we cap endTime to now(). This protects us from advancing it far
+      // into future if we receive an event with erroneously long duration, which would also take
+      // many iterations of the below "while" loop.
+      const cappedEndTime = Math.min(endTime, this.#callNow())
+
+      // If nextSamplingInstant is far in cappedEndTime's past, first advance it close to it. This
+      // can happen if we didn't receive any events for a while. Since a Poisson process has no
+      // memory, we can reset it anytime. This will ensure that the "while" loop below runs at most
+      // few iterations.
+      const earliestContinuousPast = cappedEndTime - this.#resetInterval
+      if (this.#nextSamplingInstant < earliestContinuousPast) {
+        this.#nextSamplingInstant = earliestContinuousPast
+      }
+
+      // Advance the next sampling instant until it is in cappedEndTime's future.
+      while (cappedEndTime >= this.#nextSamplingInstant) {
+        this.#setNextSamplingInstant()
+      }
+    }
+    // An event is sampled if it started before, and ended on or after a sampling instant. The above
+    // while loop will ensure that the ending invariant is always true for the current sampling
+    // instant so we don't have to test for it below. Across calls, the invariant also holds as long
+    // as the events arrive in endTime order. This is true for events coming from
+    // DatadogInstrumentationEventSource; they will be ordered by endTime by virtue of this method
+    // being invoked synchronously with the plugins' finish() handler which evaluates
+    // performance.now(). OTOH, events coming from NodeAPIEventSource (GC in typical setup) might be
+    // somewhat delayed as they are queued by Node, so they can arrive out of order with regard to
+    // events coming from the non-queued source. By omitting the endTime check, we will pass through
+    // some short events that started and ended before the current sampling instant. OTOH, if we
+    // were to check for this.currentSamplingInstant <= endTime, we would discard some long events
+    // that also ended before the current sampling instant. We'd rather err on the side of including
+    // some short events than excluding some long events.
+    return event.startTime < this.#currentSamplingInstant
+  }
+
+  #setNextSamplingInstant () {
+    this.#currentSamplingInstant = this.#nextSamplingInstant
+    this.#nextSamplingInstant -= Math.log(1 - Math.random()) * this.#samplingInterval
+    this.#samplingInstantCount++
+  }
+}
+
+module.exports = PoissonProcessSamplingFilter

package/packages/dd-trace/src/profiling/profilers/shared.js

@@ -0,0 +1,55 @@
+'use strict'
+
+const { isMainThread, threadId } = require('worker_threads')
+
+const END_TIMESTAMP_LABEL = 'end_timestamp_ns'
+const THREAD_NAME_LABEL = 'thread name'
+const OS_THREAD_ID_LABEL = 'os thread id'
+const THREAD_ID_LABEL = 'thread id'
+const SPAN_ID_LABEL = 'span id'
+const LOCAL_ROOT_SPAN_ID_LABEL = 'local root span id'
+
+const threadNamePrefix = isMainThread ? 'Main' : `Worker #${threadId}`
+const eventLoopThreadName = `${threadNamePrefix} Event Loop`
+
+function getThreadLabels () {
+  const pprof = require('@datadog/pprof')
+  const nativeThreadId = pprof.getNativeThreadId()
+  return {
+    [THREAD_NAME_LABEL]: eventLoopThreadName,
+    [THREAD_ID_LABEL]: `${threadId}`,
+    [OS_THREAD_ID_LABEL]: `${nativeThreadId}`,
+  }
+}
+
+function cacheThreadLabels () {
+  let labels
+  return () => {
+    if (!labels) {
+      labels = getThreadLabels()
+    }
+    return labels
+  }
+}
+
+function getNonJSThreadsLabels () {
+  return { [THREAD_NAME_LABEL]: 'Non-JS threads', [THREAD_ID_LABEL]: 'NA', [OS_THREAD_ID_LABEL]: 'NA' }
+}
+
+function encodeProfileAsync (profile) {
+  return profile.encodeAsync().then(Buffer.from)
+}
+
+module.exports = {
+  END_TIMESTAMP_LABEL,
+  THREAD_NAME_LABEL,
+  THREAD_ID_LABEL,
+  OS_THREAD_ID_LABEL,
+  SPAN_ID_LABEL,
+  LOCAL_ROOT_SPAN_ID_LABEL,
+  threadNamePrefix,
+  eventLoopThreadName,
+  getNonJSThreadsLabels,
+  getThreadLabels: cacheThreadLabels(),
+  encodeProfileAsync,
+}

package/packages/dd-trace/src/profiling/profilers/space.js

@@ -0,0 +1,77 @@
+'use strict'
+
+const { oomExportStrategies } = require('../constants')
+const { encodeProfileAsync, getThreadLabels } = require('./shared')
+
+function strategiesToCallbackMode (strategies, callbackMode) {
+  return strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0
+}
+
+const STACK_DEPTH = 64
+
+class NativeSpaceProfiler {
+  #mapper
+  #oomMonitoring
+  #pprof
+  #samplingInterval = 512 * 1024
+  #started = false
+
+  constructor (options = {}) {
+    // TODO: Remove default value. It is only used in testing.
+    this.#samplingInterval = options.heapSamplingInterval || 512 * 1024
+    this.#oomMonitoring = options.oomMonitoring || {}
+  }
+
+  get type () {
+    return 'space'
+  }
+
+  start ({ mapper, nearOOMCallback } = {}) {
+    if (this.#started) return
+
+    this.#mapper = mapper
+    this.#pprof = require('@datadog/pprof')
+    this.#pprof.heap.start(this.#samplingInterval, STACK_DEPTH)
+    if (this.#oomMonitoring.enabled) {
+      const strategies = this.#oomMonitoring.exportStrategies
+      this.#pprof.heap.monitorOutOfMemory(
+        this.#oomMonitoring.heapLimitExtensionSize,
+        this.#oomMonitoring.maxHeapExtensionCount,
+        strategies.includes(oomExportStrategies.LOGS),
+        strategies.includes(oomExportStrategies.PROCESS) ? this.#oomMonitoring.exportCommand : [],
+        (profile) => nearOOMCallback(this.type, this.#pprof.encodeSync(profile), this.getInfo()),
+        strategiesToCallbackMode(strategies, this.#pprof.heap.CallbackMode)
+      )
+    }
+
+    this.#started = true
+  }
+
+  profile (restart) {
+    const profile = this.#pprof.heap.profile(undefined, this.#mapper, getThreadLabels)
+    if (!restart) {
+      this.stop()
+    }
+    return profile
+  }
+
+  getInfo () {
+    return {}
+  }
+
+  encode (profile) {
+    return encodeProfileAsync(profile)
+  }
+
+  stop () {
+    if (!this.#started) return
+    this.#pprof.heap.stop()
+    this.#started = false
+  }
+
+  isStarted () {
+    return this.#started
+  }
+}
+
+module.exports = NativeSpaceProfiler