dd-trace 5.28.0 → 5.29.1

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -254,10 +254,10 @@ class NodeApiEventSource {
 }
 
 class DatadogInstrumentationEventSource {
-  constructor (eventHandler) {
+  constructor (eventHandler, eventFilter) {
     this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'net'].map(m => {
       const Plugin = require(`./event_plugins/${m}`)
-      return new Plugin(eventHandler)
+      return new Plugin(eventHandler, eventFilter)
     })
 
     this.started = false
@@ -292,29 +292,68 @@ class CompositeEventSource {
   }
 }
 
+function createPossionProcessSamplingFilter (samplingIntervalMillis) {
+  let nextSamplingInstant = performance.now()
+  let currentSamplingInstant = 0
+  setNextSamplingInstant()
+
+  return event => {
+    const endTime = event.startTime + event.duration
+    while (endTime >= nextSamplingInstant) {
+      setNextSamplingInstant()
+    }
+    // An event is sampled if it started before, and ended on or after a sampling instant. The above
+    // while loop will ensure that the ending invariant is always true for the current sampling
+    // instant so we don't have to test for it below. Across calls, the invariant also holds as long
+    // as the events arrive in endTime order. This is true for events coming from
+    // DatadogInstrumentationEventSource; they will be ordered by endTime by virtue of this method
+    // being invoked synchronously with the plugins' finish() handler which evaluates
+    // performance.now(). OTOH, events coming from NodeAPIEventSource (GC in typical setup) might be
+    // somewhat delayed as they are queued by Node, so they can arrive out of order with regard to
+    // events coming from the non-queued source. By omitting the endTime check, we will pass through
+    // some short events that started and ended before the current sampling instant. OTOH, if we
+    // were to check for this.currentSamplingInstant <= endTime, we would discard some long events
+    // that also ended before the current sampling instant. We'd rather err on the side of including
+    // some short events than excluding some long events.
+    return event.startTime < currentSamplingInstant
+  }
+
+  function setNextSamplingInstant () {
+    currentSamplingInstant = nextSamplingInstant
+    nextSamplingInstant -= Math.log(1 - Math.random()) * samplingIntervalMillis
+  }
+}
+
 /**
  * This class generates pprof files with timeline events. It combines an event
- * source with an event serializer.
+ * source with a sampling event filter and an event serializer.
  */
 class EventsProfiler {
   constructor (options = {}) {
     this.type = 'events'
     this.eventSerializer = new EventSerializer()
 
-    const eventHandler = event => {
-      this.eventSerializer.addEvent(event)
+    const eventHandler = event => this.eventSerializer.addEvent(event)
+    const eventFilter = options.timelineSamplingEnabled
+      // options.samplingInterval comes in microseconds, we need millis
+      ? createPossionProcessSamplingFilter((options.samplingInterval ?? 1e6 / 99) / 1000)
+      : _ => true
+    const filteringEventHandler = event => {
+      if (eventFilter(event)) {
+        eventHandler(event)
+      }
     }
 
     if (options.codeHotspotsEnabled) {
       // Use Datadog instrumentation to collect events with span IDs. Still use
       // Node API for GC events.
       this.eventSource = new CompositeEventSource([
-        new DatadogInstrumentationEventSource(eventHandler),
-        new NodeApiEventSource(eventHandler, ['gc'])
+        new DatadogInstrumentationEventSource(eventHandler, eventFilter),
+        new NodeApiEventSource(filteringEventHandler, ['gc'])
       ])
     } else {
       // Use Node API instrumentation to collect events without span IDs
-      this.eventSource = new NodeApiEventSource(eventHandler)
+      this.eventSource = new NodeApiEventSource(filteringEventHandler)
     }
   }
 

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -3,8 +3,6 @@
 const { storage } = require('../../../../datadog-core')
 
 const dc = require('dc-polyfill')
-const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../../../ext/tags')
-const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
 const {
@@ -15,6 +13,8 @@ const {
   getThreadLabels
 } = require('./shared')
 
+const { isWebServerSpan, endpointNameFromTags, getStartedSpans } = require('../webspan-utils')
+
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
 const spanFinishCh = dc.channel('dd-trace:span:finish')
@@ -29,21 +29,6 @@ function getActiveSpan () {
   return store && store.span
 }
 
-function getStartedSpans (context) {
-  return context._trace.started
-}
-
-function isWebServerSpan (tags) {
-  return tags[SPAN_TYPE] === WEB
-}
-
-function endpointNameFromTags (tags) {
-  return tags[RESOURCE_NAME] || [
-    tags[HTTP_METHOD],
-    tags[HTTP_ROUTE]
-  ].filter(v => v).join(' ')
-}
-
 let channelsActivated = false
 function ensureChannelsActivated () {
   if (channelsActivated) return

package/packages/dd-trace/src/profiling/webspan-utils.js

@@ -0,0 +1,23 @@
+const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../../ext/tags')
+const { WEB } = require('../../../../ext/types')
+
+function isWebServerSpan (tags) {
+  return tags[SPAN_TYPE] === WEB
+}
+
+function endpointNameFromTags (tags) {
+  return tags[RESOURCE_NAME] || [
+    tags[HTTP_METHOD],
+    tags[HTTP_ROUTE]
+  ].filter(v => v).join(' ')
+}
+
+function getStartedSpans (context) {
+  return context._trace.started
+}
+
+module.exports = {
+  isWebServerSpan,
+  endpointNameFromTags,
+  getStartedSpans
+}

package/packages/dd-trace/src/proxy.js

@@ -181,8 +181,13 @@ class Tracer extends NoopProxy {
           )
         }
       }
+
+      if (config.isTestDynamicInstrumentationEnabled) {
+        const testVisibilityDynamicInstrumentation = require('./ci-visibility/dynamic-instrumentation')
+        testVisibilityDynamicInstrumentation.start()
+      }
     } catch (e) {
-      log.error(e)
+      log.error('Error initialising tracer', e)
     }
 
     return this
@@ -193,7 +198,7 @@ class Tracer extends NoopProxy {
     try {
       return require('./profiler').start(config)
     } catch (e) {
-      log.error(e)
+      log.error('Error starting profiler', e)
     }
   }
 

package/packages/dd-trace/src/runtime_metrics.js

@@ -7,11 +7,19 @@ const os = require('os')
 const { DogStatsDClient } = require('./dogstatsd')
 const log = require('./log')
 const Histogram = require('./histogram')
-const { performance } = require('perf_hooks')
+const { performance, PerformanceObserver } = require('perf_hooks')
 
+const { NODE_MAJOR, NODE_MINOR } = require('../../../version')
 const INTERVAL = 10 * 1000
 
+// Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug.
+// See: https://github.com/nodejs/node/issues/39548
+const hasGCObserver = NODE_MAJOR >= 18 || (NODE_MAJOR === 16 && NODE_MINOR >= 7)
+const hasGCProfiler = NODE_MAJOR >= 20 || (NODE_MAJOR === 18 && NODE_MINOR >= 15)
+
 let nativeMetrics = null
+let gcObserver = null
+let gcProfiler = null
 
 let interval
 let client
@@ -24,15 +32,20 @@ let elu
 
 reset()
 
-module.exports = {
+const runtimeMetrics = module.exports = {
   start (config) {
     const clientConfig = DogStatsDClient.generateClientConfig(config)
 
     try {
       nativeMetrics = require('@datadog/native-metrics')
-      nativeMetrics.start()
+
+      if (hasGCObserver) {
+        nativeMetrics.start('loop') // Only add event loop watcher and not GC.
+      } else {
+        nativeMetrics.start()
+      }
     } catch (e) {
-      log.error(e)
+      log.error('Error starting native metrics', e)
       nativeMetrics = null
     }
 
@@ -40,6 +53,9 @@ module.exports = {
 
     time = process.hrtime()
 
+    startGCObserver()
+    startGCProfiler()
+
     if (nativeMetrics) {
       interval = setInterval(() => {
         captureCommonMetrics()
@@ -138,6 +154,10 @@ function reset () {
   counters = {}
   histograms = {}
   nativeMetrics = null
+  gcObserver && gcObserver.disconnect()
+  gcObserver = null
+  gcProfiler && gcProfiler.stop()
+  gcProfiler = null
 }
 
 function captureCpuUsage () {
@@ -202,6 +222,29 @@ function captureHeapSpace () {
     client.gauge('runtime.node.heap.physical_size.by.space', stats[i].physical_space_size, tags)
   }
 }
+function captureGCMetrics () {
+  if (!gcProfiler) return
+
+  const profile = gcProfiler.stop()
+  const pauseAll = new Histogram()
+  const pause = {}
+
+  for (const stat of profile.statistics) {
+    const type = stat.gcType.replace(/([a-z])([A-Z])/g, '$1_$2').toLowerCase()
+
+    pause[type] = pause[type] || new Histogram()
+    pause[type].record(stat.cost)
+    pauseAll.record(stat.cost)
+  }
+
+  histogram('runtime.node.gc.pause', pauseAll)
+
+  for (const type in pause) {
+    histogram('runtime.node.gc.pause.by.type', pause[type], [`gc_type:${type}`])
+  }
+
+  gcProfiler.start()
+}
 
 function captureGauges () {
   Object.keys(gauges).forEach(name => {
@@ -256,6 +299,7 @@ function captureCommonMetrics () {
   captureCounters()
   captureHistograms()
   captureELU()
+  captureGCMetrics()
 }
 
 function captureNativeMetrics () {
@@ -297,6 +341,11 @@ function captureNativeMetrics () {
 function histogram (name, stats, tags) {
   tags = [].concat(tags)
 
+  // Stats can contain garbage data when a value was never recorded.
+  if (stats.count === 0) {
+    stats = { max: 0, min: 0, sum: 0, avg: 0, median: 0, p95: 0, count: 0 }
+  }
+
   client.gauge(`${name}.min`, stats.min, tags)
   client.gauge(`${name}.max`, stats.max, tags)
   client.increment(`${name}.sum`, stats.sum, tags)
@@ -306,3 +355,57 @@ function histogram (name, stats, tags) {
   client.gauge(`${name}.median`, stats.median, tags)
   client.gauge(`${name}.95percentile`, stats.p95, tags)
 }
+
+function startGCObserver () {
+  if (gcObserver || hasGCProfiler || !hasGCObserver) return
+
+  gcObserver = new PerformanceObserver(list => {
+    for (const entry of list.getEntries()) {
+      const type = gcType(entry.detail?.kind || entry.kind)
+
+      runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`)
+      runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration)
+    }
+  })
+
+  gcObserver.observe({ type: 'gc' })
+}
+
+function startGCProfiler () {
+  if (gcProfiler || !hasGCProfiler) return
+
+  gcProfiler = new v8.GCProfiler()
+  gcProfiler.start()
+}
+
+function gcType (kind) {
+  if (NODE_MAJOR >= 22) {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'minor_mark_sweep'
+      case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
+      case 8: return 'incremental_marking'
+      case 16: return 'process_weak_callbacks'
+      case 31: return 'all'
+    }
+  } else if (NODE_MAJOR >= 18) {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'minor_mark_compact'
+      case 4: return 'mark_sweep_compact'
+      case 8: return 'incremental_marking'
+      case 16: return 'process_weak_callbacks'
+      case 31: return 'all'
+    }
+  } else {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'mark_sweep_compact'
+      case 4: return 'incremental_marking'
+      case 8: return 'process_weak_callbacks'
+      case 15: return 'all'
+    }
+  }
+
+  return 'unknown'
+}

package/packages/dd-trace/src/serverless.js

@@ -23,7 +23,7 @@ function maybeStartServerlessMiniAgent (config) {
   try {
     require('child_process').spawn(rustBinaryPath, { stdio: 'inherit' })
   } catch (err) {
-    log.error(`Error spawning mini agent process: ${err}`)
+    log.error('Error spawning mini agent process: %s', err.message)
   }
 }
 

package/packages/dd-trace/src/span_processor.js

@@ -87,22 +87,22 @@ class SpanProcessor {
         const id = context.toSpanId()
 
         if (finished.has(span)) {
-          log.error(`Span was already finished in the same trace: ${span}`)
+          log.error('Span was already finished in the same trace: %s', span)
         } else {
           finished.add(span)
 
           if (finishedIds.has(id)) {
-            log.error(`Another span with the same ID was already finished in the same trace: ${span}`)
+            log.error('Another span with the same ID was already finished in the same trace: %s', span)
           } else {
             finishedIds.add(id)
           }
 
           if (context._trace !== trace) {
-            log.error(`A span was finished in the wrong trace: ${span}.`)
+            log.error('A span was finished in the wrong trace: %s', span)
           }
 
           if (finishedSpans.has(span)) {
-            log.error(`Span was already finished in a different trace: ${span}`)
+            log.error('Span was already finished in a different trace: %s', span)
           } else {
             finishedSpans.add(span)
           }
@@ -114,35 +114,35 @@ class SpanProcessor {
         const id = context.toSpanId()
 
         if (started.has(span)) {
-          log.error(`Span was already started in the same trace: ${span}`)
+          log.error('Span was already started in the same trace: %s', span)
         } else {
           started.add(span)
 
           if (startedIds.has(id)) {
-            log.error(`Another span with the same ID was already started in the same trace: ${span}`)
+            log.error('Another span with the same ID was already started in the same trace: %s', span)
           } else {
             startedIds.add(id)
           }
 
           if (context._trace !== trace) {
-            log.error(`A span was started in the wrong trace: ${span}.`)
+            log.error('A span was started in the wrong trace: %s', span)
           }
 
           if (startedSpans.has(span)) {
-            log.error(`Span was already started in a different trace: ${span}`)
+            log.error('Span was already started in a different trace: %s', span)
           } else {
             startedSpans.add(span)
           }
         }
 
         if (!finished.has(span)) {
-          log.error(`Span started in one trace but was finished in another trace: ${span}`)
+          log.error('Span started in one trace but was finished in another trace: %s', span)
         }
       }
 
       for (const span of trace.finished) {
         if (!started.has(span)) {
-          log.error(`Span finished in one trace but was started in another trace: ${span}`)
+          log.error('Span finished in one trace but was started in another trace: %s', span)
         }
       }
     }

package/packages/dd-trace/src/tagger.js

@@ -44,7 +44,7 @@ function add (carrier, keyValuePairs, parseOtelTags = false) {
       Object.assign(carrier, keyValuePairs)
     }
   } catch (e) {
-    log.error(e)
+    log.error('Error adding tags', e)
   }
 }
 

package/packages/dd-trace/src/telemetry/index.js

@@ -137,6 +137,7 @@ function appClosing () {
   sendData(config, application, host, reqType, payload)
   // We flush before shutting down.
   metricsManager.send(config, application, host)
+  telemetryLogger.send(config, application, host)
 }
 
 function onBeforeExit () {

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -40,6 +40,7 @@ function onErrorLog (msg) {
 
   const telLog = {
     level: 'ERROR',
+    count: 1,
 
     // existing log.error(err) without message will be reported as 'Generic Error'
     message: message ?? 'Generic Error'
@@ -47,8 +48,7 @@ function onErrorLog (msg) {
 
   if (cause) {
     telLog.stack_trace = cause.stack
-    const errorType = cause.name ?? 'Error'
-    telLog.message = `${errorType}: ${telLog.message}`
+    telLog.errorType = cause.constructor.name
   }
 
   onLog(telLog)

package/packages/dd-trace/src/telemetry/logs/log-collector.js

@@ -3,7 +3,7 @@
 const log = require('../../log')
 const { calculateDDBasePath } = require('../../util')
 
-const logs = new Map()
+const logs = new Map() // hash -> log
 
 // NOTE: Is this a reasonable number?
 let maxEntries = 10000
@@ -47,8 +47,14 @@ function sanitize (logEntry) {
     .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath))
     .map(line => line.replace(ddBasePath, ''))
 
+  if (!isDDCode && logEntry.errorType && stackLines.length) {
+    stackLines = [`${logEntry.errorType}: redacted`, ...stackLines]
+  }
+
+  delete logEntry.errorType
+
   logEntry.stack_trace = stackLines.join(EOL)
-  if (logEntry.stack_trace === '' && !logEntry.message) {
+  if (logEntry.stack_trace === '' && (!logEntry.message || logEntry.message === 'Generic Error')) {
     // If entire stack was removed and there is no message we'd rather not log it at all.
     return null
   }
@@ -75,6 +81,8 @@ const logCollector = {
       if (!logs.has(hash)) {
         logs.set(hash, logEntry)
         return true
+      } else {
+        logs.get(hash).count++
       }
     } catch (e) {
       log.error('Unable to add log to logCollector: %s', e.message)

package/packages/dd-trace/src/telemetry/send-data.js

@@ -57,7 +57,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
     try {
       url = url || new URL(getAgentlessTelemetryEndpoint(config.site))
     } catch (err) {
-      log.error(err)
+      log.error('Telemetry endpoint url is invalid', err)
       // No point to do the request if the URL is invalid
       return cb(err, { payload, reqType })
     }
@@ -100,7 +100,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
         path: '/api/v2/apmtelemetry'
       }
       if (backendUrl) {
-        request(data, backendOptions, (error) => { log.error(error) })
+        request(data, backendOptions, (error) => { log.error('Error sending telemetry data', error) })
       } else {
         log.error('Invalid Telemetry URL')
       }

package/packages/dd-trace/src/util.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const crypto = require('crypto')
 const path = require('path')
 
 function isTrue (str) {
@@ -25,6 +24,8 @@ function isError (value) {
 
 // Matches a glob pattern to a given subject string
 function globMatch (pattern, subject) {
+  if (typeof pattern === 'string') pattern = pattern.toLowerCase()
+  if (typeof subject === 'string') subject = subject.toLowerCase()
   let px = 0 // [p]attern inde[x]
   let sx = 0 // [s]ubject inde[x]
   let nextPx = 0
@@ -64,6 +65,8 @@ function globMatch (pattern, subject) {
   return true
 }
 
+// TODO: this adds stack traces relative to packages/
+// shouldn't paths be relative to the root of dd-trace?
 function calculateDDBasePath (dirname) {
   const dirSteps = dirname.split(path.sep)
   const packagesIndex = dirSteps.lastIndexOf('packages')
@@ -74,25 +77,11 @@ function hasOwn (object, prop) {
   return Object.prototype.hasOwnProperty.call(object, prop)
 }
 
-/**
- * Generates a unique hash from an array of strings by joining them with | before hashing.
- * Used to uniquely identify AWS requests for span pointers.
- * @param {string[]} components - Array of strings to hash
- * @returns {string} A 32-character hash uniquely identifying the components
- */
-function generatePointerHash (components) {
-  // If passing S3's ETag as a component, make sure any quotes have already been removed!
-  const dataToHash = components.join('|')
-  const hash = crypto.createHash('sha256').update(dataToHash).digest('hex')
-  return hash.substring(0, 32)
-}
-
 module.exports = {
   isTrue,
   isFalse,
   isError,
   globMatch,
   calculateDDBasePath,
-  hasOwn,
-  generatePointerHash
+  hasOwn
 }

package/packages/datadog-instrumentations/src/qs.js

@@ -1,24 +0,0 @@
-'use strict'
-
-const { addHook, channel } = require('./helpers/instrument')
-const shimmer = require('../../datadog-shimmer')
-
-const qsParseCh = channel('datadog:qs:parse:finish')
-
-function wrapParse (originalParse) {
-  return function () {
-    const qsParsedObj = originalParse.apply(this, arguments)
-    if (qsParseCh.hasSubscribers && qsParsedObj) {
-      qsParseCh.publish({ qs: qsParsedObj })
-    }
-    return qsParsedObj
-  }
-}
-
-addHook({
-  name: 'qs',
-  versions: ['>=1']
-}, qs => {
-  shimmer.wrap(qs, 'parse', wrapParse)
-  return qs
-})