@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/ritm.js

@@ -0,0 +1,211 @@
+'use strict'
+
+const path = require('path')
+const fs = require('fs')
+const Module = require('module')
+
+const dc = require('dc-polyfill')
+
+const parse = require('../../../vendor/dist/module-details-from-path')
+const { isRelativeRequire } = require('../../datadog-instrumentations/src/helpers/shared-utils')
+const { getEnvironmentVariable, getValueFromEnvSources } = require('./config/helper')
+
+const origRequire = Module.prototype.require
+// derived from require-in-the-middle@3 with tweaks
+
+module.exports = Hook
+
+let moduleHooks = Object.create(null)
+let cache = Object.create(null)
+let patching = Object.create(null)
+let patchedRequire = null
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+const moduleLoadEndChannel = dc.channel('dd-trace:moduleLoadEnd')
+
+function Hook (modules, options, onrequire) {
+  if (!(this instanceof Hook)) return new Hook(modules, options, onrequire)
+  if (typeof modules === 'function') {
+    onrequire = modules
+    modules = null
+    options = {}
+  } else if (typeof options === 'function') {
+    onrequire = options
+    options = {}
+  }
+
+  modules = modules || []
+  options = options || {}
+
+  this.modules = modules
+  this.options = options
+  this.onrequire = onrequire
+
+  if (Array.isArray(modules)) {
+    for (const mod of modules) {
+      const hooks = moduleHooks[mod]
+
+      if (hooks) {
+        hooks.push(onrequire)
+      } else {
+        moduleHooks[mod] = [onrequire]
+      }
+    }
+  }
+
+  if (patchedRequire) return
+
+  const _origRequire = Module.prototype.require
+  patchedRequire = Module.prototype.require = function (request) {
+    /*
+    If resolving the filename for a `require(...)` fails, defer to the wrapped
+    require implementation rather than failing right away. This allows a
+    possibly monkey patched `require` to work.
+    */
+    let filename
+    try {
+      // @ts-expect-error Module._resolveFilename is not typed
+      filename = Module._resolveFilename(request, this)
+    } catch {
+      return _origRequire.apply(this, arguments)
+    }
+    const core = !filename.includes(path.sep)
+    let name, basedir, hooks
+    // return known patched modules immediately
+    if (cache[filename]) {
+      const externalCacheEntry = require.cache[filename]
+      // require.cache was potentially altered externally
+      if (externalCacheEntry && externalCacheEntry.exports !== cache[filename].original) {
+        return externalCacheEntry.exports
+      }
+
+      return cache[filename].exports
+    }
+
+    // Check if this module has a patcher in-progress already.
+    // Otherwise, mark this module as patching in-progress.
+    const patched = patching[filename]
+    if (patched) {
+      // If it's already patched, just return it as-is.
+      return origRequire.apply(this, arguments)
+    }
+    patching[filename] = true
+
+    const payload = {
+      filename,
+      request,
+    }
+
+    if (moduleLoadStartChannel.hasSubscribers) {
+      moduleLoadStartChannel.publish(payload)
+    }
+    let exports = origRequire.apply(this, arguments)
+    payload.module = exports
+    if (moduleLoadEndChannel.hasSubscribers) {
+      moduleLoadEndChannel.publish(payload)
+      exports = payload.module
+    }
+
+    // The module has already been loaded,
+    // so the patching mark can be cleaned up.
+    delete patching[filename]
+
+    if (core) {
+      hooks = moduleHooks[filename]
+      if (!hooks) return exports // abort if module name isn't on whitelist
+      name = filename
+    } else {
+      const inAWSLambda = getEnvironmentVariable('AWS_LAMBDA_FUNCTION_NAME') !== undefined
+      const hasLambdaHandler = getValueFromEnvSources('DD_LAMBDA_HANDLER') !== undefined
+      const segments = filename.split(path.sep)
+      const filenameFromNodeModule = segments.includes('node_modules')
+      // decide how to assign the stat
+      // first case will only happen when patching an AWS Lambda Handler
+      const stat = inAWSLambda && hasLambdaHandler && !filenameFromNodeModule ? { name: filename } : parse(filename)
+
+      if (stat) {
+        name = stat.name
+        basedir = stat.basedir
+
+        hooks = moduleHooks[name]
+        if (!hooks) return exports // abort if module name isn't on whitelist
+
+        // @ts-expect-error Module._resolveLookupPaths is not typed
+        const paths = Module._resolveLookupPaths(name, this, true)
+        if (!paths) {
+          // abort if _resolveLookupPaths return null
+          return exports
+        }
+
+        let res
+        try {
+          // @ts-expect-error Module._findPath is not typed
+          res = Module._findPath(name, [basedir, ...paths])
+        } catch {
+          // case where the file specified in package.json "main" doesn't exist
+          // in this case, the file is treated as module-internal
+        }
+
+        if (!res || res !== filename) {
+          // this is a module-internal file
+          // use the module-relative path to the file, prefixed by original module name
+          name = name + path.sep + path.relative(basedir, filename)
+        }
+      } else {
+        if (isRelativeRequire(request) && moduleHooks[request]) {
+          hooks = moduleHooks[request]
+          name = request
+          basedir = findProjectRoot(filename)
+        }
+
+        if (!hooks) return exports
+      }
+    }
+
+    // ensure that the cache entry is assigned a value before calling
+    // onrequire, in case calling onrequire requires the same module.
+    cache[filename] = { exports }
+    cache[filename].original = exports
+
+    for (const hook of hooks) {
+      cache[filename].exports = hook(cache[filename].exports, name, basedir)
+    }
+
+    return cache[filename].exports
+  }
+}
+
+Hook.reset = function () {
+  Module.prototype.require = origRequire
+  patchedRequire = null
+  patching = Object.create(null)
+  cache = Object.create(null)
+  moduleHooks = Object.create(null)
+}
+
+function findProjectRoot (startDir) {
+  let dir = startDir
+
+  while (!fs.existsSync(path.join(dir, 'package.json'))) {
+    const parent = path.dirname(dir)
+    if (parent === dir) break
+    dir = parent
+  }
+
+  return dir
+}
+
+Hook.prototype.unhook = function () {
+  for (const mod of this.modules) {
+    const hooks = (moduleHooks[mod] || []).filter(hook => hook !== this.onrequire)
+
+    if (hooks.length > 0) {
+      moduleHooks[mod] = hooks
+    } else {
+      delete moduleHooks[mod]
+    }
+  }
+
+  if (Object.keys(moduleHooks).length === 0) {
+    Hook.reset()
+  }
+}

package/packages/dd-trace/src/runtime_metrics/index.js

@@ -0,0 +1,34 @@
+'use strict'
+
+let runtimeMetrics
+
+const noop = runtimeMetrics = {
+  stop () {},
+  track () {},
+  boolean () {},
+  histogram () {},
+  count () {},
+  gauge () {},
+  increment () {},
+  decrement () {},
+}
+
+module.exports = {
+  start (config) {
+    if (!config?.runtimeMetrics.enabled) return
+
+    runtimeMetrics = require('./runtime_metrics')
+
+    Object.setPrototypeOf(module.exports, runtimeMetrics)
+
+    runtimeMetrics.start(config)
+  },
+
+  stop () {
+    runtimeMetrics.stop()
+
+    Object.setPrototypeOf(module.exports, runtimeMetrics = noop)
+  },
+}
+
+Object.setPrototypeOf(module.exports, noop)

package/packages/dd-trace/src/runtime_metrics/runtime_metrics.js

@@ -0,0 +1,373 @@
+'use strict'
+
+// TODO: capture every second and flush every 10 seconds
+
+const v8 = require('v8')
+const os = require('os')
+const process = require('process')
+const { performance, PerformanceObserver, monitorEventLoopDelay } = require('perf_hooks')
+const { DogStatsDClient, MetricsAggregationClient } = require('../dogstatsd')
+const log = require('../log')
+const { getValueFromEnvSources } = require('../config/helper')
+
+const { NODE_MAJOR } = require('../../../../version')
+const processTags = require('../process-tags')
+// TODO: This environment variable may not be changed, since the agent expects a flush every ten seconds.
+// It is only a variable for testing. Think about alternatives.
+const DD_RUNTIME_METRICS_FLUSH_INTERVAL = getValueFromEnvSources('DD_RUNTIME_METRICS_FLUSH_INTERVAL') ?? '10000'
+const INTERVAL = Number.parseInt(DD_RUNTIME_METRICS_FLUSH_INTERVAL, 10)
+
+const eventLoopDelayResolution = 4
+
+let nativeMetrics = null
+let gcObserver = null
+let interval = null
+let client = null
+let lastTime = 0
+let lastCpuUsage = null
+let eventLoopDelayObserver = null
+
+// !!!!!!!!!!!
+//  IMPORTANT
+// !!!!!!!!!!!
+//
+// ALL metrics that relate to time are handled in nanoseconds in the backend.
+// https://github.com/DataDog/dogweb/blob/prod/integration/node/node_metadata.csv
+
+module.exports = {
+  start (config) {
+    this.stop()
+    const clientConfig = DogStatsDClient.generateClientConfig(config)
+
+    if (config.propagateProcessTags?.enabled) {
+      for (const tag of processTags.tagsArray) {
+        clientConfig.tags.push(tag)
+      }
+    }
+
+    const trackEventLoop = config.runtimeMetrics.eventLoop !== false
+    const trackGc = config.runtimeMetrics.gc !== false
+
+    if (trackGc) {
+      startGCObserver()
+    }
+
+    // Using no-gc prevents the native gc metrics from being tracked. Not
+    // passing any options means all metrics are tracked.
+    // TODO: This is a workaround. We should find a better solution.
+    const watchers = trackEventLoop ? ['loop'] : ['no-gc']
+
+    try {
+      nativeMetrics = require('@datadog/native-metrics')
+      nativeMetrics.start(...watchers)
+    } catch (error) {
+      log.error('Error starting native metrics', error)
+      nativeMetrics = null
+    }
+
+    client = new MetricsAggregationClient(new DogStatsDClient(clientConfig))
+
+    lastTime = performance.now()
+
+    if (nativeMetrics) {
+      interval = setInterval(() => {
+        captureNativeMetrics(trackEventLoop, trackGc)
+        captureCommonMetrics(trackEventLoop)
+        client.flush()
+      }, INTERVAL)
+    } else {
+      lastCpuUsage = process.cpuUsage()
+
+      if (trackEventLoop) {
+        eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+        eventLoopDelayObserver.enable()
+      }
+
+      interval = setInterval(() => {
+        captureCpuUsage()
+        captureCommonMetrics(trackEventLoop)
+        captureHeapSpace()
+        if (trackEventLoop) {
+          // Experimental: The Node.js implementation deviates from the native metrics.
+          // We normalize the metrics to the same format but the Node.js values
+          // are that way lower than they should be, while they are still nearer
+          // to the native ones that way.
+          // We use these only as fallback values.
+          captureEventLoopDelay()
+        }
+        client.flush()
+      }, INTERVAL)
+    }
+
+    interval.unref()
+  },
+
+  stop () {
+    nativeMetrics?.stop()
+    nativeMetrics = null
+
+    clearInterval(interval)
+    interval = null
+
+    client = null
+    lastCpuUsage = null
+
+    gcObserver?.disconnect()
+    gcObserver = null
+
+    eventLoopDelayObserver?.disable()
+    eventLoopDelayObserver = null
+  },
+
+  track (span) {
+    if (nativeMetrics) {
+      const handle = nativeMetrics.track(span)
+
+      return {
+        finish: () => nativeMetrics.finish(handle),
+      }
+    }
+
+    return { finish: () => {} }
+  },
+
+  boolean (name, value, tag) {
+    client?.boolean(name, value, tag)
+  },
+
+  histogram (name, value, tag) {
+    client?.histogram(name, value, tag)
+  },
+
+  count (name, count, tag, monotonic = false) {
+    client?.count(name, count, tag, monotonic)
+  },
+
+  gauge (name, value, tag) {
+    client?.gauge(name, value, tag)
+  },
+
+  increment (name, tag, monotonic) {
+    this.count(name, 1, tag, monotonic)
+  },
+
+  decrement (name, tag) {
+    this.count(name, -1, tag)
+  },
+}
+
+function captureCpuUsage () {
+  const currentCpuUsage = process.cpuUsage()
+  const elapsedUsageUser = currentCpuUsage.user - lastCpuUsage.user
+  const elapsedUsageSystem = currentCpuUsage.system - lastCpuUsage.system
+
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  const userPercent = elapsedUsageUser / elapsedUsDividedBy100
+  const systemPercent = elapsedUsageSystem / elapsedUsDividedBy100
+  const totalPercent = userPercent + systemPercent
+
+  lastTime = currentTime
+  lastCpuUsage = currentCpuUsage
+
+  client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
+  client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
+  client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
+}
+
+function captureMemoryUsage () {
+  const stats = process.memoryUsage()
+
+  client.gauge('runtime.node.mem.heap_total', stats.heapTotal)
+  client.gauge('runtime.node.mem.heap_used', stats.heapUsed)
+  client.gauge('runtime.node.mem.rss', stats.rss)
+  client.gauge('runtime.node.mem.total', os.totalmem())
+  client.gauge('runtime.node.mem.free', os.freemem())
+  client.gauge('runtime.node.mem.external', stats.external)
+  // TODO: Add arrayBuffers to the metrics. That also requires the
+  // node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.mem.arrayBuffers', stats.arrayBuffers)
+}
+
+function captureUptime () {
+  // WARNING: lastTime must be updated in the same interval before this function is called!
+  // This is a faster `process.uptime()`.
+  client.gauge('runtime.node.process.uptime', Math.round((lastTime + 499) / 1000))
+}
+
+function captureEventLoopDelay () {
+  eventLoopDelayObserver.disable()
+
+  if (eventLoopDelayObserver.count !== 0) {
+    const minimum = eventLoopDelayResolution * 1e6
+    const avg = Math.max(eventLoopDelayObserver.mean - minimum, 0)
+    const sum = Math.round(avg * eventLoopDelayObserver.count)
+
+    if (sum !== 0) {
+      // Normalize the metrics to the same format as the native metrics.
+      const stats = {
+        min: Math.max(eventLoopDelayObserver.min - minimum, 0),
+        max: Math.max(eventLoopDelayObserver.max - minimum, 0),
+        sum,
+        total: sum,
+        avg,
+        count: eventLoopDelayObserver.count,
+        p95: Math.max(eventLoopDelayObserver.percentile(95) - minimum, 0),
+      }
+
+      histogram('runtime.node.event_loop.delay', stats)
+    }
+  }
+  eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+  eventLoopDelayObserver.enable()
+}
+
+function captureHeapStats () {
+  const stats = v8.getHeapStatistics()
+
+  client.gauge('runtime.node.heap.total_heap_size', stats.total_heap_size)
+  client.gauge('runtime.node.heap.total_heap_size_executable', stats.total_heap_size_executable)
+  client.gauge('runtime.node.heap.total_physical_size', stats.total_physical_size)
+  client.gauge('runtime.node.heap.total_available_size', stats.total_available_size)
+  client.gauge('runtime.node.heap.heap_size_limit', stats.heap_size_limit)
+  client.gauge('runtime.node.heap.malloced_memory', stats.malloced_memory)
+  client.gauge('runtime.node.heap.peak_malloced_memory', stats.peak_malloced_memory)
+  // TODO: Add number_of_native_contexts and number_of_detached_contexts to the
+  // metrics. Those metrics allow to identify memory leaks. Adding them also
+  // requires the node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.heap.number_of_native_contexts', stats.number_of_native_contexts)
+  // client.gauge('runtime.node.heap.number_of_detached_contexts', stats.number_of_detached_contexts)
+}
+
+function captureHeapSpace () {
+  const stats = v8.getHeapSpaceStatistics()
+
+  for (let i = 0, l = stats.length; i < l; i++) {
+    const tags = [`heap_space:${stats[i].space_name}`]
+
+    client.gauge('runtime.node.heap.size.by.space', stats[i].space_size, tags)
+    client.gauge('runtime.node.heap.used_size.by.space', stats[i].space_used_size, tags)
+    client.gauge('runtime.node.heap.available_size.by.space', stats[i].space_available_size, tags)
+    client.gauge('runtime.node.heap.physical_size.by.space', stats[i].physical_space_size, tags)
+  }
+}
+
+/**
+ * Gathers and reports Event Loop Utilization (ELU) since last run, or from the
+ * start of the process on first run.
+ *
+ * ELU is a measure of how busy the event loop is, like running JavaScript or
+ * waiting on *Sync functions. The value is between 0 (idle) and 1 (exhausted).
+ */
+let lastElu = { idle: 0, active: 0 }
+function captureELU () {
+  const elu = performance.eventLoopUtilization()
+
+  const idle = elu.idle - lastElu.idle
+  const active = elu.active - lastElu.active
+  const utilization = active / (idle + active)
+
+  lastElu = elu
+
+  client.gauge('runtime.node.event_loop.utilization', utilization)
+}
+
+function captureCommonMetrics (trackEventLoop) {
+  captureMemoryUsage()
+  captureUptime()
+  captureHeapStats()
+  if (trackEventLoop) {
+    captureELU()
+  }
+}
+
+function captureNativeMetrics (trackEventLoop, trackGc) {
+  const stats = nativeMetrics.stats()
+  const spaces = stats.heap.spaces
+
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  lastTime = currentTime
+
+  const userPercent = stats.cpu.user / elapsedUsDividedBy100
+  const systemPercent = stats.cpu.system / elapsedUsDividedBy100
+  const totalPercent = userPercent + systemPercent
+
+  client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
+  client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
+  client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
+
+  if (trackEventLoop) {
+    histogram('runtime.node.event_loop.delay', stats.eventLoop)
+  }
+
+  if (trackGc) {
+    for (const [type, value] of Object.entries(stats.gc)) {
+      if (type === 'all') {
+        histogram('runtime.node.gc.pause', value)
+      } else {
+        histogram('runtime.node.gc.pause.by.type', value, `gc_type:${type}`)
+      }
+    }
+  }
+
+  for (let i = 0, l = spaces.length; i < l; i++) {
+    const tag = `heap_space:${spaces[i].space_name}`
+
+    client.gauge('runtime.node.heap.size.by.space', spaces[i].space_size, tag)
+    client.gauge('runtime.node.heap.used_size.by.space', spaces[i].space_used_size, tag)
+    client.gauge('runtime.node.heap.available_size.by.space', spaces[i].space_available_size, tag)
+    client.gauge('runtime.node.heap.physical_size.by.space', spaces[i].physical_space_size, tag)
+  }
+}
+
+function histogram (name, stats, tag) {
+  if (stats.count === 0) {
+    return
+  }
+  client.gauge(`${name}.min`, stats.min, tag)
+  client.gauge(`${name}.max`, stats.max, tag)
+  client.increment(`${name}.sum`, stats.sum, tag)
+  client.increment(`${name}.total`, stats.sum, tag)
+  client.gauge(`${name}.avg`, stats.avg, tag)
+  client.increment(`${name}.count`, stats.count, tag)
+  if (stats.median !== undefined) {
+    // TODO: Consider adding the median to the Node.js histogram/adding stddev to native metrics.
+    client.gauge(`${name}.median`, stats.median, tag)
+  }
+  client.gauge(`${name}.95percentile`, stats.p95, tag)
+}
+
+function startGCObserver () {
+  if (gcObserver) return
+
+  gcObserver = new PerformanceObserver(list => {
+    for (const entry of list.getEntries()) {
+      // @ts-expect-error - entry.detail?.kind and entry.kind are not typed
+      const type = gcType(entry.detail?.kind || entry.kind)
+      const duration = entry.duration * 1_000_000
+
+      // These are individual metrics for each type of GC.
+      client.histogram('runtime.node.gc.pause.by.type', duration, `gc_type:${type}`)
+      client.histogram('runtime.node.gc.pause', duration)
+    }
+  })
+
+  gcObserver.observe({ type: 'gc' })
+}
+
+const minorGCType = NODE_MAJOR >= 22 ? 'minor_mark_sweep' : 'minor_mark_compact'
+
+function gcType (kind) {
+  switch (kind) {
+    case 1: return 'scavenge'
+    case 2: return minorGCType
+    case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
+    case 8: return 'incremental_marking'
+    case 16: return 'process_weak_callbacks'
+    case 31: return 'all'
+    default: return 'unknown'
+  }
+}

package/packages/dd-trace/src/sampler.js

@@ -0,0 +1,63 @@
+'use strict'
+
+// Maximum trace ID value is the maximum value for a 64-bit unsigned integer.
+// Javascript cannot handle such large numbers, we will loose precision but it's fine
+// as it is cast into a float64 when computing the threshold
+const MAX_TRACE_ID = 2 ** 64 - 1
+
+const UINT64_MODULO = 2n ** 64n
+
+// Knuth's factor for the sampling algorithm
+const SAMPLING_KNUTH_FACTOR = 1_111_111_111_111_111_111n
+
+/**
+ * `Sampler` determines whether or not to sample a trace/span based on the trace ID.
+ *
+ * This class uses a deterministic sampling algorithm that is consistent across all languages.
+ */
+class Sampler {
+  #threshold = 0n
+
+  /**
+   * @param {number} rate
+   */
+  constructor (rate) {
+    // TODO: Should this be moved up to the calling parts?
+    rate = Math.min(Math.max(rate, 0), 1)
+    this._rate = rate
+    this.#threshold = BigInt(Math.floor(rate * MAX_TRACE_ID))
+  }
+
+  /**
+   * @returns {number}
+   */
+  rate () {
+    return this._rate
+  }
+
+  get threshold () {
+    return this.#threshold
+  }
+
+  /**
+   * Determines whether a trace/span should be sampled based on the configured sampling rate.
+   *
+   * @param {Span|SpanContext} span - The span or span context to evaluate.
+   * @returns {boolean} `true` if the trace/span should be sampled, otherwise `false`.
+   */
+  isSampled (span) {
+    if (this._rate === 1) {
+      return true
+    }
+
+    if (this._rate === 0) {
+      return false
+    }
+
+    span = typeof span.context === 'function' ? span.context() : span
+
+    return (span._traceId.toBigInt() * SAMPLING_KNUTH_FACTOR) % UINT64_MODULO <= this.#threshold
+  }
+}
+
+module.exports = Sampler