dd-trace 4.20.0 → 4.22.0

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -1,5 +1,5 @@
 const { performance, constants, PerformanceObserver } = require('node:perf_hooks')
-const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared')
+const { END_TIMESTAMP_LABEL } = require('./shared')
 const semver = require('semver')
 const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
 const pprof = require('@datadog/pprof/')
@@ -14,7 +14,137 @@ const MS_TO_NS = 1000000
 // perf_hooks events, the emitted pprof file uses the type "timeline".
 const pprofValueType = 'timeline'
 const pprofValueUnit = 'nanoseconds'
-const threadName = `${threadNamePrefix} GC`
+
+function labelFromStr (stringTable, key, valStr) {
+  return new Label({ key, str: stringTable.dedup(valStr) })
+}
+
+function labelFromStrStr (stringTable, keyStr, valStr) {
+  return labelFromStr(stringTable, stringTable.dedup(keyStr), valStr)
+}
+
+class GCDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.reasonLabelKey = stringTable.dedup('gc reason')
+    this.kindLabels = []
+    this.reasonLabels = []
+    this.flagObj = {}
+
+    const kindLabelKey = stringTable.dedup('gc type')
+
+    // Create labels for all GC performance flags and kinds of GC
+    for (const [key, value] of Object.entries(constants)) {
+      if (key.startsWith('NODE_PERFORMANCE_GC_FLAGS_')) {
+        this.flagObj[key.substring(26).toLowerCase()] = value
+      } else if (key.startsWith('NODE_PERFORMANCE_GC_')) {
+        // It's a constant for a kind of GC
+        const kind = key.substring(20).toLowerCase()
+        this.kindLabels[value] = labelFromStr(stringTable, kindLabelKey, kind)
+      }
+    }
+  }
+
+  decorateSample (sampleInput, item) {
+    const { kind, flags } = node16 ? item.detail : item
+    sampleInput.label.push(this.kindLabels[kind])
+    const reasonLabel = this.getReasonLabel(flags)
+    if (reasonLabel) {
+      sampleInput.label.push(reasonLabel)
+    }
+  }
+
+  getReasonLabel (flags) {
+    if (flags === 0) {
+      return null
+    }
+    let reasonLabel = this.reasonLabels[flags]
+    if (!reasonLabel) {
+      const reasons = []
+      for (const [key, value] of Object.entries(this.flagObj)) {
+        if (value & flags) {
+          reasons.push(key)
+        }
+      }
+      const reasonStr = reasons.join(',')
+      reasonLabel = labelFromStr(this.stringTable, this.reasonLabelKey, reasonStr)
+      this.reasonLabels[flags] = reasonLabel
+    }
+    return reasonLabel
+  }
+}
+
+class DNSDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.addressLabelKey = stringTable.dedup('address')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, item.name)
+    const detail = item.detail
+    switch (op) {
+      case 'lookup':
+        addLabel(this.hostLabelKey, detail.hostname)
+        break
+      case 'lookupService':
+        addLabel(this.addressLabelKey, detail.host)
+        labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+        break
+      case 'getHostByAddr':
+        addLabel(this.addressLabelKey, detail.host)
+        break
+      default:
+        if (op.startsWith('query')) {
+          addLabel(this.hostLabelKey, detail.host)
+        }
+    }
+  }
+}
+
+class NetDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, op)
+    if (op === 'connect') {
+      const detail = item.detail
+      addLabel(this.hostLabelKey, detail.host)
+      labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+    }
+  }
+}
+
+// Keys correspond to PerformanceEntry.entryType, values are constructor
+// functions for type-specific decorators.
+const decoratorTypes = {
+  gc: GCDecorator
+}
+// Needs at least node 16 for DNS and Net
+if (node16) {
+  decoratorTypes.dns = DNSDecorator
+  decoratorTypes.net = NetDecorator
+}
 
 /**
  * This class generates pprof files with timeline events sourced from Node.js
@@ -35,8 +165,7 @@ class EventsProfiler {
     if (!this._observer) {
       this._observer = new PerformanceObserver(add.bind(this))
     }
-    // Currently only support GC
-    this._observer.observe({ entryTypes: ['gc'] })
+    this._observer.observe({ entryTypes: Object.keys(decoratorTypes) })
   }
 
   stop () {
@@ -45,100 +174,69 @@ class EventsProfiler {
     }
   }
 
-  profile () {
+  profile (startDate, endDate) {
     if (this.entries.length === 0) {
       // No events in the period; don't produce a profile
       return null
     }
 
     const stringTable = new StringTable()
-    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP)
-    const kindLabelKey = stringTable.dedup('gc type')
-    const reasonLabelKey = stringTable.dedup('gc reason')
-    const kindLabels = []
-    const reasonLabels = []
     const locations = []
     const functions = []
-    const locationsPerKind = []
-    const flagObj = {}
-
-    function labelFromStr (key, valStr) {
-      return new Label({ key, str: stringTable.dedup(valStr) })
-    }
-
-    function labelFromStrStr (keyStr, valStr) {
-      return labelFromStr(stringTable.dedup(keyStr), valStr)
-    }
-
-    // Create labels for all GC performance flags and kinds of GC
-    for (const [key, value] of Object.entries(constants)) {
-      if (key.startsWith('NODE_PERFORMANCE_GC_FLAGS_')) {
-        flagObj[key.substring(26).toLowerCase()] = value
-      } else if (key.startsWith('NODE_PERFORMANCE_GC_')) {
-        // It's a constant for a kind of GC
-        const kind = key.substring(20).toLowerCase()
-        kindLabels[value] = labelFromStr(kindLabelKey, kind)
-        // Construct a single-frame "location" too
-        const fn = new Function({ id: functions.length + 1, name: stringTable.dedup(`${kind} GC`) })
-        functions.push(fn)
-        const line = new Line({ functionId: fn.id })
-        const location = new Location({ id: locations.length + 1, line: [line] })
-        locations.push(location)
-        locationsPerKind[value] = [location.id]
-      }
-    }
 
-    const gcEventLabel = labelFromStrStr('event', 'gc')
-    const threadLabel = labelFromStrStr(THREAD_NAME, threadName)
+    // A synthetic single-frame location to serve as the location for timeline
+    // samples. We need these as the profiling backend (mimicking official pprof
+    // tool's behavior) ignores these.
+    const locationId = (() => {
+      const fn = new Function({ id: functions.length + 1, name: stringTable.dedup('') })
+      functions.push(fn)
+      const line = new Line({ functionId: fn.id })
+      const location = new Location({ id: locations.length + 1, line: [line] })
+      locations.push(location)
+      return [location.id]
+    })()
 
-    function getReasonLabel (flags) {
-      if (flags === 0) {
-        return null
-      }
-      let reasonLabel = reasonLabels[flags]
-      if (!reasonLabel) {
-        const reasons = []
-        for (const [key, value] of Object.entries(flagObj)) {
-          if (value & flags) {
-            reasons.push(key)
-          }
-        }
-        const reasonStr = reasons.join(',')
-        reasonLabel = labelFromStr(reasonLabelKey, reasonStr)
-        reasonLabels[flags] = reasonLabel
-      }
-      return reasonLabel
+    const decorators = {}
+    for (const [eventType, DecoratorCtor] of Object.entries(decoratorTypes)) {
+      const decorator = new DecoratorCtor(stringTable)
+      decorator.eventTypeLabel = labelFromStrStr(stringTable, 'event', eventType)
+      decorators[eventType] = decorator
     }
+    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP_LABEL)
 
-    let durationFrom = Number.POSITIVE_INFINITY
-    let durationTo = 0
     const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS))
-
+    const lateEntries = []
+    const perfEndDate = endDate.getTime() - performance.timeOrigin
     const samples = this.entries.map((item) => {
+      const decorator = decorators[item.entryType]
+      if (!decorator) {
+        // Shouldn't happen but it's better to not rely on observer only getting
+        // requested event types.
+        return null
+      }
       const { startTime, duration } = item
-      const { kind, flags } = node16 ? item.detail : item
-      const endTime = startTime + duration
-      if (durationFrom > startTime) durationFrom = startTime
-      if (durationTo < endTime) durationTo = endTime
-      const labels = [
-        gcEventLabel,
-        threadLabel,
-        new Label({ key: timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) }),
-        kindLabels[kind]
-      ]
-      const reasonLabel = getReasonLabel(flags)
-      if (reasonLabel) {
-        labels.push(reasonLabel)
+      if (startTime >= perfEndDate) {
+        // An event past the current recording end date; save it for the next
+        // profile. Not supposed to happen as long as there's no async activity
+        // between capture of the endDate value in profiler.js _collect() and
+        // here, but better be safe than sorry.
+        lateEntries.push(item)
+        return null
       }
-      const sample = new Sample({
+      const endTime = startTime + duration
+      const sampleInput = {
         value: [Math.round(duration * MS_TO_NS)],
-        label: labels,
-        locationId: locationsPerKind[kind]
-      })
-      return sample
-    })
+        locationId,
+        label: [
+          decorator.eventTypeLabel,
+          new Label({ key: timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) })
+        ]
+      }
+      decorator.decorateSample(sampleInput, item)
+      return new Sample(sampleInput)
+    }).filter(v => v)
 
-    this.entries = []
+    this.entries = lateEntries
 
     const timeValueType = new ValueType({
       type: stringTable.dedup(pprofValueType),
@@ -147,10 +245,10 @@ class EventsProfiler {
 
     return new Profile({
       sampleType: [timeValueType],
-      timeNanos: dateOffset + BigInt(Math.round(durationFrom * MS_TO_NS)),
+      timeNanos: endDate.getTime() * MS_TO_NS,
       periodType: timeValueType,
-      period: this._flushIntervalNanos,
-      durationNanos: Math.max(0, Math.round((durationTo - durationFrom) * MS_TO_NS)),
+      period: 1,
+      durationNanos: (endDate.getTime() - startDate.getTime()) * MS_TO_NS,
       sample: samples,
       location: locations,
       function: functions,

package/packages/dd-trace/src/profiling/profilers/shared.js

@@ -2,8 +2,38 @@
 
 const { isMainThread, threadId } = require('node:worker_threads')
 
+const END_TIMESTAMP_LABEL = 'end_timestamp_ns'
+const THREAD_NAME_LABEL = 'thread name'
+const OS_THREAD_ID_LABEL = 'os thread id'
+const THREAD_ID_LABEL = 'thread id'
+const threadNamePrefix = isMainThread ? 'Main' : `Worker #${threadId}`
+const eventLoopThreadName = `${threadNamePrefix} Event Loop`
+
+function getThreadLabels () {
+  const pprof = require('@datadog/pprof')
+  const nativeThreadId = pprof.getNativeThreadId()
+  return {
+    [THREAD_NAME_LABEL]: eventLoopThreadName,
+    [THREAD_ID_LABEL]: `${threadId}`,
+    [OS_THREAD_ID_LABEL]: `${nativeThreadId}`
+  }
+}
+
+function cacheThreadLabels () {
+  let labels
+  return () => {
+    if (!labels) {
+      labels = getThreadLabels()
+    }
+    return labels
+  }
+}
+
 module.exports = {
-  END_TIMESTAMP: 'end_timestamp_ns',
-  THREAD_NAME: 'thread name',
-  threadNamePrefix: isMainThread ? 'Main' : `Worker #${threadId}`
+  END_TIMESTAMP_LABEL,
+  THREAD_NAME_LABEL,
+  THREAD_ID_LABEL,
+  threadNamePrefix,
+  eventLoopThreadName,
+  getThreadLabels: cacheThreadLabels()
 }

package/packages/dd-trace/src/profiling/profilers/space.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { oomExportStrategies } = require('../constants')
+const { getThreadLabels } = require('./shared')
 
 function strategiesToCallbackMode (strategies, callbackMode) {
   return strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0
@@ -33,7 +34,7 @@ class NativeSpaceProfiler {
   }
 
   profile () {
-    return this._pprof.heap.profile(undefined, this._mapper)
+    return this._pprof.heap.profile(undefined, this._mapper, getThreadLabels)
   }
 
   encode (profile) {

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -7,13 +7,12 @@ const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
-const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared')
+const { END_TIMESTAMP_LABEL, getThreadLabels } = require('./shared')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
 const spanFinishCh = dc.channel('dd-trace:span:finish')
 const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers')
-const threadName = `${threadNamePrefix} Event Loop`
 
 const MemoizedWebTags = Symbol('NativeWallProfiler.MemoizedWebTags')
 
@@ -96,12 +95,9 @@ class NativeWallProfiler {
         this._enter = this._enter.bind(this)
         this._spanFinished = this._spanFinished.bind(this)
       }
-      this._generateLabels = this._generateLabels.bind(this)
-    } else {
-      // Explicitly assigning, to express the intent that this is meant to be
-      // undefined when passed to pprof.time.stop() when not using sample contexts.
-      this._generateLabels = undefined
     }
+    this._generateLabels = this._generateLabels.bind(this)
+
     this._logger = options.logger
     this._started = false
   }
@@ -239,12 +235,21 @@ class NativeWallProfiler {
     return profile
   }
 
-  _generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, timestamp }) {
-    const labels = this._timelineEnabled ? {
-      [THREAD_NAME]: threadName,
+  _generateLabels (context) {
+    if (context == null) {
+      // generateLabels is also called for samples without context.
+      // In that case just return thread labels.
+      return getThreadLabels()
+    }
+
+    const labels = { ...getThreadLabels() }
+
+    const { context: { spanId, rootSpanId, webTags, endpoint }, timestamp } = context
+
+    if (this._timelineEnabled) {
       // Incoming timestamps are in microseconds, we emit nanos.
-      [END_TIMESTAMP]: timestamp * 1000n
-    } : {}
+      labels[END_TIMESTAMP_LABEL] = timestamp * 1000n
+    }
 
     if (spanId) {
       labels['span id'] = spanId

package/packages/dd-trace/src/proxy.js

@@ -10,6 +10,7 @@ const PluginManager = require('./plugin_manager')
 const remoteConfig = require('./appsec/remote_config')
 const AppsecSdk = require('./appsec/sdk')
 const dogstatsd = require('./dogstatsd')
+const spanleak = require('./spanleak')
 
 class Tracer extends NoopProxy {
   constructor () {
@@ -35,6 +36,19 @@ class Tracer extends NoopProxy {
         setInterval(() => {
           this.dogstatsd.flush()
         }, 10 * 1000).unref()
+
+        process.once('beforeExit', () => {
+          this.dogstatsd.flush()
+        })
+      }
+
+      if (config.spanLeakDebug > 0) {
+        if (config.spanLeakDebug === spanleak.MODES.LOG) {
+          spanleak.enableLogging()
+        } else if (config.spanLeakDebug === spanleak.MODES.GC_AND_LOG) {
+          spanleak.enableGarbageCollection()
+        }
+        spanleak.startScrubber()
       }
 
       if (config.remoteConfig.enabled && !config.isCiVisibility) {
@@ -62,11 +76,14 @@ class Tracer extends NoopProxy {
         // do not stop tracer initialization if the profiler fails to be imported
         try {
           const profiler = require('./profiler')
-          profiler.start(config)
+          this._profilerStarted = profiler.start(config)
         } catch (e) {
           log.error(e)
         }
       }
+      if (!this._profilerStarted) {
+        this._profilerStarted = Promise.resolve(false)
+      }
 
       if (config.runtimeMetrics) {
         runtimeMetrics.start(config)
@@ -104,6 +121,13 @@ class Tracer extends NoopProxy {
     return this
   }
 
+  profilerStarted () {
+    if (!this._profilerStarted) {
+      throw new Error('profilerStarted() must be called after init()')
+    }
+    return this._profilerStarted
+  }
+
   use () {
     this._pluginManager.configurePlugin(...arguments)
     return this

package/packages/dd-trace/src/service-naming/schemas/v0/storage.js

@@ -37,6 +37,11 @@ const redisConfig = {
 
 const storage = {
   client: {
+    aerospike: {
+      opName: () => 'aerospike.command',
+      serviceName: ({ tracerService, pluginConfig }) =>
+        pluginConfig.service || `${tracerService}-aerospike`
+    },
     'cassandra-driver': {
       opName: () => 'cassandra.query',
       serviceName: ({ tracerService, pluginConfig, system }) =>

package/packages/dd-trace/src/service-naming/schemas/v1/storage.js

@@ -22,6 +22,10 @@ function withFunction ({ tracerService, pluginConfig, params }) {
 
 const storage = {
   client: {
+    aerospike: {
+      opName: () => 'aerospike.command',
+      serviceName: configWithFallback
+    },
     'cassandra-driver': {
       opName: () => 'cassandra.query',
       serviceName: configWithFallback

package/packages/dd-trace/src/spanleak.js

@@ -0,0 +1,98 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+const SortedSet = require('tlhunter-sorted-set')
+
+const INTERVAL = 1000 // look for expired spans every 1s
+const LIFETIME = 60 * 1000 // all spans have a max lifetime of 1m
+
+const MODES = {
+  DISABLED: 0,
+  // METRICS_ONLY
+  LOG: 1,
+  GC_AND_LOG: 2
+  // GC
+}
+
+module.exports.MODES = MODES
+
+const spans = new SortedSet()
+
+// TODO: should these also be delivered as runtime metrics?
+
+// const registry = new FinalizationRegistry(name => {
+//   spans.del(span) // there is no span
+// })
+
+let interval
+let mode = MODES.DISABLED
+
+module.exports.disable = function () {
+  mode = MODES.DISABLED
+}
+
+module.exports.enableLogging = function () {
+  mode = MODES.LOG
+}
+
+module.exports.enableGarbageCollection = function () {
+  mode = MODES.GC_AND_LOG
+}
+
+module.exports.startScrubber = function () {
+  if (!isEnabled()) return
+
+  interval = setInterval(() => {
+    const now = Date.now()
+    const expired = spans.rangeByScore(0, now)
+
+    if (!expired.length) return
+
+    const gc = isGarbageCollecting()
+
+    const expirationsByType = Object.create(null) // { [spanType]: count }
+
+    for (const wrapped of expired) {
+      spans.del(wrapped)
+      const span = wrapped.deref()
+
+      if (!span) continue // span has already been garbage collected
+
+      // TODO: Should we also do things like record the route to help users debug leaks?
+      if (!expirationsByType[span._name]) expirationsByType[span._name] = 0
+      expirationsByType[span._name]++
+
+      if (!gc) continue // everything after this point is related to manual GC
+
+      // TODO: what else can we do to alleviate memory usage
+      span.context()._tags = Object.create(null)
+    }
+
+    console.log('expired spans:' +
+      Object.keys(expirationsByType).reduce((a, c) => `${a} ${c}: ${expirationsByType[c]}`, ''))
+  }, INTERVAL)
+}
+
+module.exports.stopScrubber = function () {
+  clearInterval(interval)
+}
+
+module.exports.addSpan = function (span) {
+  if (!isEnabled()) return
+
+  const now = Date.now()
+  const expiration = now + LIFETIME
+  // eslint-disable-next-line no-undef
+  const wrapped = new WeakRef(span)
+  spans.add(wrapped, expiration)
+  // registry.register(span, span._name)
+}
+
+function isEnabled () {
+  return mode > MODES.DISABLED
+}
+
+function isGarbageCollecting () {
+  return mode >= MODES.GC_AND_LOG
+}

package/packages/dd-trace/src/startup-log.js

@@ -6,6 +6,7 @@ const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
 
+const errors = {}
 let config
 let pluginManager
 let samplingRules = []
@@ -89,6 +90,10 @@ function startupLog ({ agentError } = {}) {
   info('DATADOG TRACER CONFIGURATION - ' + out)
   if (agentError) {
     warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+    errors.agentError = {
+      code: agentError.code ? agentError.code : '',
+      message: `Agent Error:${agentError.message}`
+    }
   }
 
   config = undefined
@@ -112,5 +117,6 @@ module.exports = {
   startupLog,
   setStartupLogConfig,
   setStartupLogPluginManager,
-  setSamplingRules
+  setSamplingRules,
+  errors
 }