dd-trace 5.64.0 → 5.66.0

package/packages/dd-trace/src/runtime_metrics/runtime_metrics.js

@@ -4,72 +4,90 @@
 
 const v8 = require('v8')
 const os = require('os')
+const process = require('process')
 const { DogStatsDClient, MetricsAggregationClient } = require('../dogstatsd')
 const log = require('../log')
-const { performance, PerformanceObserver } = require('perf_hooks')
+const { performance, PerformanceObserver, monitorEventLoopDelay } = require('perf_hooks')
 const { getEnvironmentVariable } = require('../config-helper')
 
-const { NODE_MAJOR, NODE_MINOR } = require('../../../../version')
+const { NODE_MAJOR } = require('../../../../version')
+// TODO: This environment variable may not be changed, since the agent expects a flush every ten seconds.
+// It is only a variable for testing. Think about alternatives.
 const DD_RUNTIME_METRICS_FLUSH_INTERVAL = getEnvironmentVariable('DD_RUNTIME_METRICS_FLUSH_INTERVAL') ?? '10000'
 const INTERVAL = Number.parseInt(DD_RUNTIME_METRICS_FLUSH_INTERVAL, 10)
 
-// Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug.
-// See: https://github.com/nodejs/node/issues/39548
-const hasGCObserver = NODE_MAJOR >= 18 || (NODE_MAJOR === 16 && NODE_MINOR >= 7)
+const eventLoopDelayResolution = 4
 
 let nativeMetrics = null
 let gcObserver = null
-
-let interval
-let client
-let time
-let cpuUsage
-let elu
-
-reset()
-
-const runtimeMetrics = module.exports = {
+let interval = null
+let client = null
+let lastTime = 0n
+let lastCpuUsage = null
+let eventLoopDelayObserver = null
+
+// !!!!!!!!!!!
+//  IMPORTANT
+// !!!!!!!!!!!
+//
+// ALL metrics that relate to time are handled in nanoseconds in the backend.
+// https://github.com/DataDog/dogweb/blob/prod/integration/node/node_metadata.csv
+
+module.exports = {
   start (config) {
+    this.stop()
     const clientConfig = DogStatsDClient.generateClientConfig(config)
-    const watchers = []
 
-    if (config.runtimeMetrics.gc !== false) {
-      if (hasGCObserver) {
-        startGCObserver()
-      } else {
-        watchers.push('gc')
-      }
-    }
+    const trackEventLoop = config.runtimeMetrics.eventLoop !== false
+    const trackGc = config.runtimeMetrics.gc !== false
 
-    if (config.runtimeMetrics.eventLoop !== false) {
-      watchers.push('loop')
+    if (trackGc) {
+      startGCObserver()
     }
 
+    // Using no-gc prevents the native gc metrics from being tracked. Not
+    // passing any options means all metrics are tracked.
+    // TODO: This is a workaround. We should find a better solution.
+    const watchers = trackEventLoop ? ['loop'] : ['no-gc']
+
     try {
       nativeMetrics = require('@datadog/native-metrics')
       nativeMetrics.start(...watchers)
-    } catch (e) {
-      log.error('Error starting native metrics', e)
+    } catch (error) {
+      log.error('Error starting native metrics', error)
       nativeMetrics = null
     }
 
     client = new MetricsAggregationClient(new DogStatsDClient(clientConfig))
 
-    time = process.hrtime()
+    lastTime = performance.now()
 
     if (nativeMetrics) {
       interval = setInterval(() => {
-        captureCommonMetrics()
-        captureNativeMetrics()
+        captureNativeMetrics(trackEventLoop, trackGc)
+        captureCommonMetrics(trackEventLoop)
         client.flush()
       }, INTERVAL)
     } else {
-      cpuUsage = process.cpuUsage()
+      lastCpuUsage = process.cpuUsage()
+
+      if (trackEventLoop) {
+        eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+        eventLoopDelayObserver.enable()
+      }
 
       interval = setInterval(() => {
-        captureCommonMetrics()
         captureCpuUsage()
+        captureCommonMetrics(trackEventLoop)
         captureHeapSpace()
+        if (trackEventLoop) {
+          // Experimental: The Node.js implementation deviates from the native metrics.
+          // We normalize the metrics to the same format but the Node.js values
+          // are that way lower than they should be, while they are still nearer
+          // to the native ones that way.
+          // We use these only as fallback values.
+          captureEventLoopDelay()
+        }
         client.flush()
       }, INTERVAL)
     }
@@ -78,12 +96,21 @@ const runtimeMetrics = module.exports = {
   },
 
   stop () {
-    if (nativeMetrics) {
-      nativeMetrics.stop()
-    }
+    nativeMetrics?.stop()
+    nativeMetrics = null
 
     clearInterval(interval)
-    reset()
+    interval = null
+
+    client = null
+    lastTime = 0n
+    lastCpuUsage = null
+
+    gcObserver?.disconnect()
+    gcObserver = null
+
+    eventLoopDelayObserver?.disable()
+    eventLoopDelayObserver = null
   },
 
   track (span) {
@@ -99,19 +126,19 @@ const runtimeMetrics = module.exports = {
   },
 
   boolean (name, value, tag) {
-    client && client.boolean(name, value, tag)
+    client?.boolean(name, value, tag)
   },
 
   histogram (name, value, tag) {
-    client && client.histogram(name, value, tag)
+    client?.histogram(name, value, tag)
   },
 
   count (name, count, tag, monotonic = false) {
-    client && client.count(name, count, tag, monotonic)
+    client?.count(name, count, tag, monotonic)
   },
 
   gauge (name, value, tag) {
-    client && client.gauge(name, value, tag)
+    client?.gauge(name, value, tag)
   },
 
   increment (name, tag, monotonic) {
@@ -123,30 +150,20 @@ const runtimeMetrics = module.exports = {
   }
 }
 
-function reset () {
-  interval = null
-  client = null
-  time = null
-  cpuUsage = null
-  nativeMetrics = null
-  gcObserver && gcObserver.disconnect()
-  gcObserver = null
-}
-
 function captureCpuUsage () {
-  if (!process.cpuUsage) return
-
-  const elapsedTime = process.hrtime(time)
-  const elapsedUsage = process.cpuUsage(cpuUsage)
-
-  time = process.hrtime()
-  cpuUsage = process.cpuUsage()
-
-  const elapsedMs = elapsedTime[0] * 1000 + elapsedTime[1] / 1_000_000
-  const userPercent = 100 * elapsedUsage.user / 1000 / elapsedMs
-  const systemPercent = 100 * elapsedUsage.system / 1000 / elapsedMs
+  const currentCpuUsage = process.cpuUsage()
+  const elapsedUsageUser = currentCpuUsage.user - lastCpuUsage.user
+  const elapsedUsageSystem = currentCpuUsage.system - lastCpuUsage.system
+
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  const userPercent = elapsedUsageUser / elapsedUsDividedBy100
+  const systemPercent = elapsedUsageSystem / elapsedUsDividedBy100
   const totalPercent = userPercent + systemPercent
 
+  lastTime = currentTime
+  lastCpuUsage = currentCpuUsage
+
   client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
   client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
   client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
@@ -160,12 +177,44 @@ function captureMemoryUsage () {
   client.gauge('runtime.node.mem.rss', stats.rss)
   client.gauge('runtime.node.mem.total', os.totalmem())
   client.gauge('runtime.node.mem.free', os.freemem())
+  client.gauge('runtime.node.mem.external', stats.external)
+  // TODO: Add arrayBuffers to the metrics. That also requires the
+  // node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.mem.arrayBuffers', stats.arrayBuffers)
+}
 
-  stats.external && client.gauge('runtime.node.mem.external', stats.external)
+function captureUptime () {
+  // WARNING: lastTime must be updated in the same interval before this function is called!
+  // This is a faster `process.uptime()`.
+  client.gauge('runtime.node.process.uptime', Math.round((lastTime + 499) / 1000))
 }
 
-function captureProcess () {
-  client.gauge('runtime.node.process.uptime', Math.round(process.uptime()))
+function captureEventLoopDelay () {
+  eventLoopDelayObserver.disable()
+
+  if (eventLoopDelayObserver.count !== 0) {
+    const minimum = eventLoopDelayResolution * 1e6
+    const avg = Math.max(eventLoopDelayObserver.mean - minimum, 0)
+    const sum = Math.round(avg * eventLoopDelayObserver.count)
+
+    if (sum !== 0) {
+      // Normalize the metrics to the same format as the native metrics.
+      const stats = {
+        min: Math.max(eventLoopDelayObserver.min - minimum, 0),
+        max: Math.max(eventLoopDelayObserver.max - minimum, 0),
+        sum,
+        total: sum,
+        avg,
+        count: eventLoopDelayObserver.count,
+        p95: Math.max(eventLoopDelayObserver.percentile(95) - minimum, 0)
+      }
+
+      histogram('runtime.node.event_loop.delay', stats)
+    }
+  }
+  eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+  eventLoopDelayObserver.enable()
 }
 
 function captureHeapStats () {
@@ -176,14 +225,17 @@ function captureHeapStats () {
   client.gauge('runtime.node.heap.total_physical_size', stats.total_physical_size)
   client.gauge('runtime.node.heap.total_available_size', stats.total_available_size)
   client.gauge('runtime.node.heap.heap_size_limit', stats.heap_size_limit)
-
-  stats.malloced_memory && client.gauge('runtime.node.heap.malloced_memory', stats.malloced_memory)
-  stats.peak_malloced_memory && client.gauge('runtime.node.heap.peak_malloced_memory', stats.peak_malloced_memory)
+  client.gauge('runtime.node.heap.malloced_memory', stats.malloced_memory)
+  client.gauge('runtime.node.heap.peak_malloced_memory', stats.peak_malloced_memory)
+  // TODO: Add number_of_native_contexts and number_of_detached_contexts to the
+  // metrics. Those metrics allow to identify memory leaks. Adding them also
+  // requires the node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.heap.number_of_native_contexts', stats.number_of_native_contexts)
+  // client.gauge('runtime.node.heap.number_of_detached_contexts', stats.number_of_detached_contexts)
 }
 
 function captureHeapSpace () {
-  if (!v8.getHeapSpaceStatistics) return
-
   const stats = v8.getHeapSpaceStatistics()
 
   for (let i = 0, l = stats.length; i < l; i++) {
@@ -197,55 +249,63 @@ function captureHeapSpace () {
 }
 
 /**
- * Gathers and reports Event Loop Utilization (ELU) since last run
+ * Gathers and reports Event Loop Utilization (ELU) since last run, or from the
+ * start of the process on first run.
  *
  * ELU is a measure of how busy the event loop is, like running JavaScript or
  * waiting on *Sync functions. The value is between 0 (idle) and 1 (exhausted).
- *
- * performance.eventLoopUtilization available in Node.js >= v14.10, >= v12.19, >= v16
  */
-let captureELU = () => {}
-if ('eventLoopUtilization' in performance) {
-  captureELU = () => {
-    // if elu is undefined (first run) the measurement is from start of process
-    elu = performance.eventLoopUtilization(elu)
+let lastElu = { idle: 0, active: 0 }
+function captureELU () {
+  const elu = performance.eventLoopUtilization()
 
-    client.gauge('runtime.node.event_loop.utilization', elu.utilization)
-  }
+  const idle = elu.idle - lastElu.idle
+  const active = elu.active - lastElu.active
+  const utilization = active / (idle + active)
+
+  lastElu = elu
+
+  client.gauge('runtime.node.event_loop.utilization', utilization)
 }
 
-function captureCommonMetrics () {
+function captureCommonMetrics (trackEventLoop) {
   captureMemoryUsage()
-  captureProcess()
+  captureUptime()
   captureHeapStats()
-  captureELU()
+  if (trackEventLoop) {
+    captureELU()
+  }
 }
 
-function captureNativeMetrics () {
+function captureNativeMetrics (trackEventLoop, trackGc) {
   const stats = nativeMetrics.stats()
   const spaces = stats.heap.spaces
-  const elapsedTime = process.hrtime(time)
 
-  time = process.hrtime()
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  lastTime = currentTime
 
-  const elapsedUs = elapsedTime[0] * 1e6 + elapsedTime[1] / 1e3
-  const userPercent = 100 * stats.cpu.user / elapsedUs
-  const systemPercent = 100 * stats.cpu.system / elapsedUs
+  const userPercent = stats.cpu.user / elapsedUsDividedBy100
+  const systemPercent = stats.cpu.system / elapsedUsDividedBy100
   const totalPercent = userPercent + systemPercent
 
   client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
   client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
   client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
 
-  histogram('runtime.node.event_loop.delay', stats.eventLoop)
+  if (trackEventLoop) {
+    histogram('runtime.node.event_loop.delay', stats.eventLoop)
+  }
 
-  Object.keys(stats.gc).forEach(type => {
-    if (type === 'all') {
-      histogram('runtime.node.gc.pause', stats.gc[type])
-    } else {
-      histogram('runtime.node.gc.pause.by.type', stats.gc[type], `gc_type:${type}`)
+  if (trackGc) {
+    for (const [type, value] of Object.entries(stats.gc)) {
+      if (type === 'all') {
+        histogram('runtime.node.gc.pause', value)
+      } else {
+        histogram('runtime.node.gc.pause.by.type', value, `gc_type:${type}`)
+      }
     }
-  })
+  }
 
   for (let i = 0, l = spaces.length; i < l; i++) {
     const tag = `heap_space:${spaces[i].space_name}`
@@ -258,13 +318,19 @@ function captureNativeMetrics () {
 }
 
 function histogram (name, stats, tag) {
+  if (stats.count === 0) {
+    return
+  }
   client.gauge(`${name}.min`, stats.min, tag)
   client.gauge(`${name}.max`, stats.max, tag)
   client.increment(`${name}.sum`, stats.sum, tag)
   client.increment(`${name}.total`, stats.sum, tag)
   client.gauge(`${name}.avg`, stats.avg, tag)
   client.increment(`${name}.count`, stats.count, tag)
-  client.gauge(`${name}.median`, stats.median, tag)
+  if (stats.median !== undefined) {
+    // TODO: Consider adding the median to the Node.js histogram/adding stddev to native metrics.
+    client.gauge(`${name}.median`, stats.median, tag)
+  }
   client.gauge(`${name}.95percentile`, stats.p95, tag)
 }
 
@@ -276,42 +342,25 @@ function startGCObserver () {
       const type = gcType(entry.detail?.kind || entry.kind)
       const duration = entry.duration * 1_000_000
 
-      runtimeMetrics.histogram('runtime.node.gc.pause.by.type', duration, `gc_type:${type}`)
-      runtimeMetrics.histogram('runtime.node.gc.pause', duration)
+      // These are individual metrics for each type of GC.
+      client.histogram('runtime.node.gc.pause.by.type', duration, `gc_type:${type}`)
+      client.histogram('runtime.node.gc.pause', duration)
     }
   })
 
   gcObserver.observe({ type: 'gc' })
 }
 
+const minorGCType = NODE_MAJOR >= 22 ? 'minor_mark_sweep' : 'minor_mark_compact'
+
 function gcType (kind) {
-  if (NODE_MAJOR >= 22) {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'minor_mark_sweep'
-      case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
-      case 8: return 'incremental_marking'
-      case 16: return 'process_weak_callbacks'
-      case 31: return 'all'
-    }
-  } else if (NODE_MAJOR >= 18) {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'minor_mark_compact'
-      case 4: return 'mark_sweep_compact'
-      case 8: return 'incremental_marking'
-      case 16: return 'process_weak_callbacks'
-      case 31: return 'all'
-    }
-  } else {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'mark_sweep_compact'
-      case 4: return 'incremental_marking'
-      case 8: return 'process_weak_callbacks'
-      case 15: return 'all'
-    }
+  switch (kind) {
+    case 1: return 'scavenge'
+    case 2: return minorGCType
+    case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
+    case 8: return 'incremental_marking'
+    case 16: return 'process_weak_callbacks'
+    case 31: return 'all'
+    default: return 'unknown'
   }
-
-  return 'unknown'
 }

package/packages/dd-trace/src/service-naming/schemas/v0/index.js

@@ -6,5 +6,6 @@ const storage = require('./storage')
 const graphql = require('./graphql')
 const web = require('./web')
 const serverless = require('./serverless')
+const websocket = require('./websocket')
 
-module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless })
+module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless, websocket })

package/packages/dd-trace/src/service-naming/schemas/v0/websocket.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const websocket = {
+  request: {
+    ws: {
+      opName: () => 'web.request',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  producer: {
+    ws: {
+      opName: () => 'websocket.send',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  consumer: {
+    ws: {
+      opName: () => 'websocket.receive',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  close: {
+    ws: {
+      opName: () => 'websocket.close',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  }
+}
+
+module.exports = websocket

package/packages/dd-trace/src/service-naming/schemas/v1/index.js

@@ -6,5 +6,6 @@ const storage = require('./storage')
 const graphql = require('./graphql')
 const web = require('./web')
 const serverless = require('./serverless')
+const websocket = require('./websocket')
 
-module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless })
+module.exports = new SchemaDefinition({ messaging, storage, web, graphql, serverless, websocket })

package/packages/dd-trace/src/service-naming/schemas/v1/websocket.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const websocket = {
+  request: {
+    ws: {
+      opName: () => 'web.request',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  producer: {
+    ws: {
+      opName: () => 'websocket.send',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  consumer: {
+    ws: {
+      opName: () => 'websocket.receive',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  },
+  close: {
+    ws: {
+      opName: () => 'websocket.close',
+      serviceName: ({ pluginConfig, tracerService }) => pluginConfig.service || tracerService
+    }
+  }
+}
+
+module.exports = websocket

package/packages/dd-trace/src/supported-configurations.json

@@ -31,7 +31,6 @@
     "DD_APPSEC_STACK_TRACE_ENABLED": ["A"],
     "DD_APPSEC_TRACE_RATE_LIMIT": ["A"],
     "DD_APPSEC_WAF_TIMEOUT": ["A"],
-    "DD_AZURE_APP_SERVICES": ["A"],
     "DD_CIVISIBILITY_AGENTLESS_ENABLED": ["A"],
     "DD_CIVISIBILITY_AGENTLESS_URL": ["A"],
     "DD_CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER": ["A"],
@@ -415,6 +414,9 @@
     "DD_TRACE_VITEST_ENABLED": ["A"],
     "DD_TRACE_VITEST_RUNNER_ENABLED": ["A"],
     "DD_TRACE_VM_ENABLED": ["A"],
+    "DD_TRACE_WEBSOCKET_MESSAGES_ENABLED":["A"],
+    "DD_TRACE_WEBSOCKET_MESSAGES_INHERIT_SAMPLING": ["A"],
+    "DD_TRACE_WEBSOCKET_MESSAGES_SEPARATE_TRACES":["A"],
     "DD_TRACE_WHEN_ENABLED": ["A"],
     "DD_TRACE_WINSTON_ENABLED": ["A"],
     "DD_TRACE_WORKERPOOL_ENABLED": ["A"],