dd-trace 3.38.1 → 3.40.0

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -2,15 +2,20 @@
 
 const { storage } = require('../../../../datadog-core')
 
-const dc = require('../../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../../../ext/tags')
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
+const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
+const spanFinishCh = dc.channel('dd-trace:span:finish')
 const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers')
+const threadName = `${threadNamePrefix} Event Loop`
+
+const CachedWebTags = Symbol('NativeWallProfiler.CachedWebTags')
 
 let kSampleCount
 
@@ -24,7 +29,11 @@ function getStartedSpans (context) {
 }
 
 function generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, timestamp }) {
-  const labels = {}
+  const labels = {
+    [THREAD_NAME]: threadName,
+    // Incoming timestamps are in microseconds, we emit nanos.
+    [END_TIMESTAMP]: timestamp * 1000n
+  }
   if (spanId) {
     labels['span id'] = spanId
   }
@@ -37,16 +46,10 @@ function generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, t
     // fallback to endpoint computed when sample was taken
     labels['trace endpoint'] = endpoint
   }
-  // Incoming timestamps are in microseconds, we emit nanos.
-  labels['end_timestamp_ns'] = timestamp * 1000n
 
   return labels
 }
 
-function getSpanContextTags (span) {
-  return span.context()._tags
-}
-
 function isWebServerSpan (tags) {
   return tags[SPAN_TYPE] === WEB
 }
@@ -58,29 +61,6 @@ function endpointNameFromTags (tags) {
   ].filter(v => v).join(' ')
 }
 
-function updateContext (context, span, startedSpans, endpointCollectionEnabled) {
-  context.spanId = span.context().toSpanId()
-  const rootSpan = startedSpans[0]
-  if (rootSpan) {
-    context.rootSpanId = rootSpan.context().toSpanId()
-    if (endpointCollectionEnabled) {
-      // Find the first webspan starting from the end:
-      // There might be several webspans, for example with next.js, http plugin creates a first span
-      // and then next.js plugin creates a child span, and this child span haves the correct endpoint information.
-      for (let i = startedSpans.length - 1; i >= 0; i--) {
-        const tags = getSpanContextTags(startedSpans[i])
-        if (isWebServerSpan(tags)) {
-          context.webTags = tags
-          // endpoint may not be determined yet, but keep it as fallback
-          // if tags are not available anymore during serialization
-          context.endpoint = endpointNameFromTags(tags)
-          break
-        }
-      }
-    }
-  }
-}
-
 class NativeWallProfiler {
   constructor (options = {}) {
     this.type = 'wall'
@@ -88,12 +68,14 @@ class NativeWallProfiler {
     this._flushIntervalMillis = options.flushInterval || 60 * 1e3 // 60 seconds
     this._codeHotspotsEnabled = !!options.codeHotspotsEnabled
     this._endpointCollectionEnabled = !!options.endpointCollectionEnabled
+    this._withContexts = this._codeHotspotsEnabled || this._endpointCollectionEnabled
     this._v8ProfilerBugWorkaroundEnabled = !!options.v8ProfilerBugWorkaroundEnabled
     this._mapper = undefined
     this._pprof = undefined
 
     // Bind to this so the same value can be used to unsubscribe later
     this._enter = this._enter.bind(this)
+    this._spanFinished = this._spanFinished.bind(this)
     this._logger = options.logger
     this._started = false
   }
@@ -109,12 +91,6 @@ class NativeWallProfiler {
   start ({ mapper } = {}) {
     if (this._started) return
 
-    if (this._codeHotspotsEnabled && !this._emittedFFMessage && this._logger) {
-      this._logger.debug(
-        `Wall profiler: Enable trace_show_breakdown_profiling_for_node feature flag to see code hotspots.`)
-      this._emittedFFMessage = true
-    }
-
     this._mapper = mapper
     this._pprof = require('@datadog/pprof')
     kSampleCount = this._pprof.time.constants.kSampleCount
@@ -131,21 +107,23 @@ class NativeWallProfiler {
       intervalMicros: this._samplingIntervalMicros,
       durationMillis: this._flushIntervalMillis,
       sourceMapper: this._mapper,
-      withContexts: this._codeHotspotsEnabled,
+      withContexts: this._withContexts,
       lineNumbers: false,
       workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled
     })
 
-    if (this._codeHotspotsEnabled) {
+    if (this._withContexts) {
       this._profilerState = this._pprof.time.getState()
       this._currentContext = {}
       this._pprof.time.setContext(this._currentContext)
       this._lastSpan = undefined
       this._lastStartedSpans = undefined
+      this._lastWebTags = undefined
       this._lastSampleCount = 0
 
       beforeCh.subscribe(this._enter)
       enterCh.subscribe(this._enter)
+      spanFinishCh.subscribe(this._spanFinished)
     }
 
     this._started = true
@@ -161,18 +139,73 @@ class NativeWallProfiler {
       this._currentContext = {}
       this._pprof.time.setContext(this._currentContext)
 
-      if (this._lastSpan) {
-        updateContext(context, this._lastSpan, this._lastStartedSpans, this._endpointCollectionEnabled)
-      }
+      this._updateContext(context)
     }
 
     const span = getActiveSpan()
     if (span) {
+      const context = span.context()
       this._lastSpan = span
-      this._lastStartedSpans = getStartedSpans(span.context())
+      const startedSpans = getStartedSpans(context)
+      this._lastStartedSpans = startedSpans
+      if (this._endpointCollectionEnabled) {
+        const cachedWebTags = span[CachedWebTags]
+        if (cachedWebTags === undefined) {
+          let found = false
+          // Find the first webspan starting from the end:
+          // There might be several webspans, for example with next.js, http plugin creates a first span
+          // and then next.js plugin creates a child span, and this child span has the correct endpoint information.
+          let nextSpanId = context._spanId
+          for (let i = startedSpans.length - 1; i >= 0; i--) {
+            const nextContext = startedSpans[i].context()
+            if (nextContext._spanId === nextSpanId) {
+              const tags = nextContext._tags
+              if (isWebServerSpan(tags)) {
+                this._lastWebTags = tags
+                span[CachedWebTags] = tags
+                found = true
+                break
+              }
+              nextSpanId = nextContext._parentId
+            }
+          }
+          if (!found) {
+            this._lastWebTags = undefined
+            span[CachedWebTags] = null // cache negative lookup result
+          }
+        } else {
+          this._lastWebTags = cachedWebTags
+        }
+      }
     } else {
       this._lastStartedSpans = undefined
       this._lastSpan = undefined
+      this._lastWebTags = undefined
+    }
+  }
+
+  _updateContext (context) {
+    if (!this._lastSpan) {
+      return
+    }
+    if (this._codeHotspotsEnabled) {
+      context.spanId = this._lastSpan.context().toSpanId()
+      const rootSpan = this._lastStartedSpans[0]
+      if (rootSpan) {
+        context.rootSpanId = rootSpan.context().toSpanId()
+      }
+    }
+    if (this._lastWebTags) {
+      context.webTags = this._lastWebTags
+      // endpoint may not be determined yet, but keep it as fallback
+      // if tags are not available anymore during serialization
+      context.endpoint = endpointNameFromTags(this._lastWebTags)
+    }
+  }
+
+  _spanFinished (span) {
+    if (span[CachedWebTags]) {
+      span[CachedWebTags] = undefined
     }
   }
 
@@ -188,12 +221,12 @@ class NativeWallProfiler {
 
   _stop (restart) {
     if (!this._started) return
-    if (this._codeHotspotsEnabled) {
+    if (this._withContexts) {
       // update last sample context if needed
       this._enter()
       this._lastSampleCount = 0
     }
-    const profile = this._pprof.time.stop(restart, this._codeHotspotsEnabled ? generateLabels : undefined)
+    const profile = this._pprof.time.stop(restart, this._withContexts ? generateLabels : undefined)
     if (restart) {
       const v8BugDetected = this._pprof.time.v8ProfilerStuckEventLoopDetected()
       if (v8BugDetected !== 0) {
@@ -215,10 +248,14 @@ class NativeWallProfiler {
     if (!this._started) return
 
     const profile = this._stop(false)
-    if (this._codeHotspotsEnabled) {
+    if (this._withContexts) {
       beforeCh.unsubscribe(this._enter)
-      enterCh.subscribe(this._enter)
+      enterCh.unsubscribe(this._enter)
+      spanFinishCh.unsubscribe(this._spanFinished)
       this._profilerState = undefined
+      this._lastSpan = undefined
+      this._lastStartedSpans = undefined
+      this._lastWebTags = undefined
     }
 
     this._started = false

package/packages/dd-trace/src/ritm.js

@@ -3,7 +3,7 @@
 const path = require('path')
 const Module = require('module')
 const parse = require('module-details-from-path')
-const dc = require('../../diagnostics_channel')
+const dc = require('dc-polyfill')
 
 const origRequire = Module.prototype.require
 

package/packages/dd-trace/src/span_processor.js

@@ -138,6 +138,10 @@ class SpanProcessor {
       }
     }
 
+    for (const span of trace.finished) {
+      span.context()._tags = {}
+    }
+
     trace.started = active
     trace.finished = []
   }

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -4,7 +4,7 @@ const path = require('path')
 const parse = require('module-details-from-path')
 const requirePackageJson = require('../require-package-json')
 const { sendData } = require('./send-data')
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const { fileURLToPath } = require('url')
 
 const savedDependenciesToSend = new Set()

package/packages/dd-trace/src/telemetry/index.js

@@ -1,12 +1,13 @@
 'use strict'
 
 const tracerVersion = require('../../../../package.json').version
-const dc = require('../../../diagnostics_channel')
+const dc = require('dc-polyfill')
 const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
 
 const { manager: metricsManager } = require('./metrics')
+const logs = require('./logs')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -139,10 +140,13 @@ function start (aConfig, thePluginManager) {
   heartbeatInterval = config.telemetry.heartbeatInterval
 
   dependencies.start(config, application, host)
+  logs.start(config)
+
   sendData(config, application, host, 'app-started', appStarted())
   heartbeat(config, application, host)
   interval = setInterval(() => {
     metricsManager.send(config, application, host)
+    logs.send(config, application, host)
   }, heartbeatInterval)
   interval.unref()
   process.on('beforeExit', onBeforeExit)

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const logCollector = require('./log-collector')
+const { sendData } = require('../send-data')
+
+const telemetryLog = dc.channel('datadog:telemetry:log')
+
+let enabled = false
+
+/**
+ * Telemetry logs api defines only ERROR, WARN and DEBUG levels:
+ * - WARN level is enabled by default
+ * - DEBUG level will be possible to activate with an env var or telemetry config property
+ */
+function isLevelEnabled (level) {
+  return isValidLevel(level)
+}
+
+function isValidLevel (level) {
+  switch (level) {
+    case 'ERROR':
+    case 'WARN':
+      return true
+    default:
+      return false
+  }
+}
+
+function onLog (log) {
+  if (isLevelEnabled(log?.level?.toUpperCase())) {
+    logCollector.add(log)
+  }
+}
+
+function start (config) {
+  if (!config.telemetry.logCollection || enabled) return
+
+  enabled = true
+
+  telemetryLog.subscribe(onLog)
+}
+
+function stop () {
+  enabled = false
+
+  if (telemetryLog.hasSubscribers) {
+    telemetryLog.unsubscribe(onLog)
+  }
+}
+
+function send (config, application, host) {
+  if (!enabled) return
+
+  const logs = logCollector.drain()
+  if (logs) {
+    sendData(config, application, host, 'logs', logs)
+  }
+}
+
+module.exports = {
+  start,
+  stop,
+  send
+}

package/packages/dd-trace/src/{appsec/iast/telemetry/log → telemetry/logs}/log-collector.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const log = require('../../../../log')
+const log = require('../../log')
 
 const logs = new Map()
 
@@ -18,37 +18,21 @@ function hashCode (hashSource) {
 }
 
 function createHash (logEntry) {
-  if (!logEntry) return 0
-
   const prime = 31
   let result = ((!logEntry.level) ? 0 : hashCode(logEntry.level))
   result = (((prime * result) | 0) + ((!logEntry.message) ? 0 : hashCode(logEntry.message))) | 0
-
-  // NOTE: tags are not used at the moment
-  // result = (((prime * result) | 0) + ((!logEntry.tags) ? 0 : hashCode(logEntry.tags))) | 0
   result = (((prime * result) | 0) + ((!logEntry.stack_trace) ? 0 : hashCode(logEntry.stack_trace))) | 0
   return result
 }
 
-function newLogEntry (message, level, tags) {
-  return {
-    message,
-    level,
-    tags
-  }
-}
-
 function isValid (logEntry) {
-  return logEntry && logEntry.level && logEntry.message
+  return logEntry?.level && logEntry.message
 }
 
 const logCollector = {
   add (logEntry) {
     try {
-      if (!isValid(logEntry)) {
-        log.info('IAST log collector discarding invalid log')
-        return
-      }
+      if (!isValid(logEntry)) return false
 
       // NOTE: should errors have higher priority? and discard log entries with lower priority?
       if (logs.size >= maxEntries) {
@@ -70,11 +54,13 @@ const logCollector = {
   drain () {
     if (logs.size === 0) return
 
-    const drained = []
-    drained.push(...logs.values())
+    const drained = [...logs.values()]
 
     if (overflowedCount > 0) {
-      drained.push(newLogEntry(`Omitted ${overflowedCount} entries due to overflowing`, 'ERROR'))
+      drained.push({
+        message: `Omitted ${overflowedCount} entries due to overflowing`,
+        level: 'ERROR'
+      })
     }
 
     this.reset()
@@ -85,6 +71,7 @@ const logCollector = {
   reset (max) {
     logs.clear()
     overflowedCount = 0
+
     if (max) {
       maxEntries = max
     }

package/packages/dd-trace/src/tracer.js

@@ -31,8 +31,10 @@ class DatadogTracer extends Tracer {
 
   // todo[piochelepiotr] These two methods are not related to the tracer, but to data streams monitoring.
   // They should be moved outside of the tracer in the future.
-  setCheckpoint (edgeTags) {
-    const ctx = this._dataStreamsProcessor.setCheckpoint(edgeTags, DataStreamsContext.getDataStreamsContext())
+  setCheckpoint (edgeTags, span, payloadSize = 0) {
+    const ctx = this._dataStreamsProcessor.setCheckpoint(
+      edgeTags, span, DataStreamsContext.getDataStreamsContext(), payloadSize
+    )
     DataStreamsContext.setDataStreamsContext(ctx)
     return ctx
   }

package/packages/dd-trace/src/appsec/iast/telemetry/log/index.js

@@ -1,87 +0,0 @@
-'use strict'
-
-const dc = require('../../../../../../diagnostics_channel')
-const logCollector = require('./log-collector')
-const { sendData } = require('../../../../telemetry/send-data')
-const log = require('../../../../log')
-
-const telemetryStartChannel = dc.channel('datadog:telemetry:start')
-const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
-
-let config, application, host, interval
-
-function publish (log) {
-  if (log && isLevelEnabled(log.level)) {
-    logCollector.add(log)
-  }
-}
-
-function sendLogs () {
-  try {
-    const logs = logCollector.drain()
-    if (logs) {
-      sendData(config, application, host, 'logs', logs)
-    }
-  } catch (e) {
-    log.error(e)
-  }
-}
-
-function isLevelEnabled (level) {
-  return isLogCollectionEnabled(config) && level !== 'DEBUG'
-}
-
-function isLogCollectionEnabled (config) {
-  return config && config.telemetry && config.telemetry.logCollection
-}
-
-function onTelemetryStart (msg) {
-  if (!msg || !isLogCollectionEnabled(msg.config)) {
-    log.info('IAST telemetry logs start event received but log collection is not enabled or configuration is incorrect')
-    return false
-  }
-
-  log.info('IAST telemetry logs starting')
-
-  config = msg.config
-  application = msg.application
-  host = msg.host
-
-  if (msg.heartbeatInterval) {
-    interval = setInterval(sendLogs, msg.heartbeatInterval)
-    interval.unref()
-  }
-
-  return true
-}
-
-function onTelemetryStop () {
-  stop()
-}
-
-function start () {
-  telemetryStartChannel.subscribe(onTelemetryStart)
-  telemetryStopChannel.subscribe(onTelemetryStop)
-}
-
-function stop () {
-  if (!isLogCollectionEnabled(config)) return
-
-  log.info('IAST telemetry logs stopping')
-
-  config = null
-  application = null
-  host = null
-
-  if (telemetryStartChannel.hasSubscribers) {
-    telemetryStartChannel.unsubscribe(onTelemetryStart)
-  }
-
-  if (telemetryStopChannel.hasSubscribers) {
-    telemetryStopChannel.unsubscribe(onTelemetryStop)
-  }
-
-  clearInterval(interval)
-}
-
-module.exports = { start, stop, publish, isLevelEnabled }

package/packages/diagnostics_channel/index.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('./src')

package/packages/diagnostics_channel/src/index.js

@@ -1,121 +0,0 @@
-'use strict'
-
-const {
-  Channel,
-  channel
-} = require('diagnostics_channel') // eslint-disable-line n/no-restricted-require
-
-const [major, minor] = process.versions.node.split('.')
-const channels = new WeakSet()
-
-// Our own DC with a limited subset of functionality stable across Node versions.
-// TODO: Move the rest of the polyfill here.
-// TODO: Switch to using global subscribe/unsubscribe/hasSubscribers.
-const dc = { channel }
-
-// Prevent going to 0 subscribers to avoid bug in Node.
-// See https://github.com/nodejs/node/pull/47520
-if (major === '19' && minor === '9') {
-  dc.channel = function () {
-    const ch = channel.apply(this, arguments)
-
-    if (!channels.has(ch)) {
-      const subscribe = ch.subscribe
-      const unsubscribe = ch.unsubscribe
-
-      ch.subscribe = function () {
-        delete ch.subscribe
-        delete ch.unsubscribe
-
-        const result = subscribe.apply(this, arguments)
-
-        this.subscribe(() => {}) // Keep it active forever.
-
-        return result
-      }
-
-      if (ch.unsubscribe === Channel.prototype.unsubscribe) {
-        // Needed because another subscriber could have subscribed to something
-        // that we unsubscribe to before the library is loaded.
-        ch.unsubscribe = function () {
-          delete ch.subscribe
-          delete ch.unsubscribe
-
-          this.subscribe(() => {}) // Keep it active forever.
-
-          return unsubscribe.apply(this, arguments)
-        }
-      }
-
-      channels.add(ch)
-    }
-
-    return ch
-  }
-}
-
-if (!Channel.prototype.runStores) {
-  const ActiveChannelPrototype = getActiveChannelPrototype()
-
-  Channel.prototype.bindStore = ActiveChannelPrototype.bindStore = function (store, transform) {
-    if (!this._stores) {
-      this._stores = new Map()
-    }
-    this._stores.set(store, transform)
-  }
-
-  Channel.prototype.unbindStore = ActiveChannelPrototype.unbindStore = function (store) {
-    if (!this._stores) return
-    this._stores.delete(store)
-  }
-
-  Channel.prototype.runStores = ActiveChannelPrototype.runStores = function (data, fn, thisArg, ...args) {
-    if (!this._stores) return Reflect.apply(fn, thisArg, args)
-
-    let run = () => {
-      this.publish(data)
-      return Reflect.apply(fn, thisArg, args)
-    }
-
-    for (const entry of this._stores.entries()) {
-      const store = entry[0]
-      const transform = entry[1]
-      run = wrapStoreRun(store, data, run, transform)
-    }
-
-    return run()
-  }
-}
-
-function defaultTransform (data) {
-  return data
-}
-
-function wrapStoreRun (store, data, next, transform = defaultTransform) {
-  return () => {
-    let context
-    try {
-      context = transform(data)
-    } catch (err) {
-      process.nextTick(() => {
-        throw err
-      })
-      return next()
-    }
-
-    return store.run(context, next)
-  }
-}
-
-function getActiveChannelPrototype () {
-  const dummyChannel = channel('foo')
-  const listener = () => {}
-
-  dummyChannel.subscribe(listener)
-  const ActiveChannelPrototype = Object.getPrototypeOf(dummyChannel)
-  dummyChannel.unsubscribe(listener)
-
-  return ActiveChannelPrototype
-}
-
-module.exports = dc