@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/profiling/profiler.js

@@ -0,0 +1,430 @@
+'use strict'
+
+const { EventEmitter } = require('events')
+const { promisify } = require('util')
+const zlib = require('zlib')
+const dc = require('dc-polyfill')
+const crashtracker = require('../crashtracking')
+const log = require('../log')
+const { Config } = require('./config')
+const { snapshotKinds } = require('./constants')
+const { threadNamePrefix } = require('./profilers/shared')
+const { isWebServerSpan, endpointNameFromTags, getStartedSpans } = require('./webspan-utils')
+
+const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
+const spanFinishedChannel = dc.channel('dd-trace:span:finish')
+
+function logError (logger, ...args) {
+  if (logger) {
+    logger.error(...args)
+  }
+}
+
+function findWebSpan (startedSpans, spanId) {
+  for (let i = startedSpans.length; --i >= 0;) {
+    const ispan = startedSpans[i]
+    const context = ispan.context()
+    if (context._spanId === spanId) {
+      if (isWebServerSpan(context._tags)) {
+        return true
+      }
+      spanId = context._parentId
+    }
+  }
+  return false
+}
+
+function processInfo (infos, info, type) {
+  if (Object.keys(info).length > 0) {
+    infos[type] = info
+  }
+}
+
+class Profiler extends EventEmitter {
+  #compressionFn
+  #compressionOptions
+  #config
+  #enabled = false
+  #endpointCounts = new Map()
+  #lastStart
+  #logger
+  #profileSeq = 0
+  #spanFinishListener
+  #sourceMapCount = 0
+  #timer
+
+  constructor () {
+    super()
+    this._timeoutInterval = undefined
+  }
+
+  get serverless () { return false }
+
+  get flushInterval () {
+    return this.#config?.flushInterval
+  }
+
+  start (config) {
+    const {
+      service,
+      version,
+      env,
+      url,
+      hostname,
+      port,
+      tags,
+      repositoryUrl,
+      commitSHA,
+      injectionEnabled,
+      reportHostname,
+    } = config
+    const { enabled, sourceMap, exporters } = config.profiling
+    const { heartbeatInterval } = config.telemetry
+
+    // TODO: Unify with main logger and rewrite template strings to use printf formatting.
+    const logger = {
+      debug (message) { log.debug(message) },
+      info (message) { log.info(message) },
+      warn (message) { log.warn(message) },
+      error (...args) { log.error(...args) },
+    }
+
+    const libraryInjected = injectionEnabled.length > 0
+    let activation
+    if (enabled === 'auto') {
+      activation = 'auto'
+    } else if (enabled === 'true') {
+      activation = 'manual'
+    } // else activation = undefined
+
+    const options = {
+      service,
+      version,
+      env,
+      logger,
+      sourceMap,
+      exporters,
+      url,
+      hostname,
+      port,
+      tags,
+      repositoryUrl,
+      commitSHA,
+      libraryInjected,
+      activation,
+      heartbeatInterval,
+      reportHostname,
+    }
+
+    return this._start(options).catch((err) => {
+      logError(logger, 'Error starting profiler. For troubleshooting tips, see ' +
+        '<https://dtdg.co/nodejs-profiler-troubleshooting>', err)
+      return false
+    })
+  }
+
+  get enabled () {
+    return this.#enabled
+  }
+
+  #logError (err) {
+    logError(this.#logger, err)
+  }
+
+  async _start (options) {
+    if (this.enabled) return true
+
+    const config = this.#config = new Config(options)
+
+    this.#logger = config.logger
+    this.#enabled = true
+    this._setInterval()
+
+    // Log errors if the source map finder fails, but don't prevent the rest
+    // of the profiler from running without source maps.
+    let mapper
+    try {
+      const { setLogger, SourceMapper } = require('@datadog/pprof')
+      setLogger(config.logger)
+
+      if (config.sourceMap) {
+        mapper = await SourceMapper.create([process.cwd()], config.debugSourceMaps)
+        this.#sourceMapCount = mapper.infoMap.size
+        if (config.debugSourceMaps) {
+          this.#logger.debug(() => {
+            return this.#sourceMapCount === 0
+              ? 'Found no source maps'
+              : `Found source maps for following files: [${[...mapper.infoMap.keys()].join(', ')}]`
+          })
+        }
+      }
+
+      const clevel = config.uploadCompression.level
+      switch (config.uploadCompression.method) {
+        case 'gzip':
+          this.#compressionFn = promisify(zlib.gzip)
+          if (clevel !== undefined) {
+            this.#compressionOptions = {
+              level: clevel,
+            }
+          }
+          break
+        case 'zstd':
+          if (typeof zlib.zstdCompress === 'function') { // eslint-disable-line n/no-unsupported-features/node-builtins
+            // eslint-disable-next-line n/no-unsupported-features/node-builtins
+            this.#compressionFn = promisify(zlib.zstdCompress)
+            if (clevel !== undefined) {
+              this.#compressionOptions = {
+                params: {
+                  // eslint-disable-next-line n/no-unsupported-features/node-builtins
+                  [zlib.constants.ZSTD_c_compressionLevel]: clevel,
+                },
+              }
+            }
+          } else {
+            const zstdCompress = require('@datadog/libdatadog').load('datadog-js-zstd').zstd_compress
+            const level = clevel ?? 0 // 0 is zstd default compression level
+            this.#compressionFn = (buffer) => Promise.resolve(Buffer.from(zstdCompress(buffer, level)))
+          }
+          break
+      }
+    } catch (err) {
+      this.#logError(err)
+    }
+
+    try {
+      const start = new Date()
+      const nearOOMCallback = this.#nearOOMExport.bind(this)
+      for (const profiler of config.profilers) {
+        // TODO: move this out of Profiler when restoring sourcemap support
+        profiler.start({
+          mapper,
+          nearOOMCallback,
+        })
+        this.#logger.debug(`Started ${profiler.type} profiler in ${threadNamePrefix} thread`)
+      }
+
+      if (config.endpointCollectionEnabled) {
+        this.#spanFinishListener = this.#onSpanFinish.bind(this)
+        spanFinishedChannel.subscribe(this.#spanFinishListener)
+      }
+
+      this._capture(this._timeoutInterval, start)
+      return true
+    } catch (e) {
+      this.#logError(e)
+      this.#stop()
+      return false
+    }
+  }
+
+  #nearOOMExport (profileType, encodedProfile, info) {
+    const start = this.#lastStart
+    const end = new Date()
+    const infos = this.#createInitialInfos()
+    processInfo(infos, info, profileType)
+    this.#submit({
+      [profileType]: encodedProfile,
+    }, infos, start, end, snapshotKinds.ON_OUT_OF_MEMORY)
+  }
+
+  _setInterval () {
+    this._timeoutInterval = this.#config.flushInterval
+  }
+
+  stop () {
+    if (!this.enabled) return
+
+    // collect and export current profiles
+    // once collect returns, profilers can be safely stopped
+    this._collect(snapshotKinds.ON_SHUTDOWN, false)
+    this.#stop()
+  }
+
+  #stop () {
+    if (!this.enabled) return
+
+    this.#enabled = false
+
+    if (this.#spanFinishListener !== undefined) {
+      spanFinishedChannel.unsubscribe(this.#spanFinishListener)
+      this.#spanFinishListener = undefined
+    }
+
+    for (const profiler of this.#config.profilers) {
+      profiler.stop()
+      this.#logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`)
+    }
+
+    clearTimeout(this.#timer)
+    this.#timer = undefined
+  }
+
+  _capture (timeout, start) {
+    if (!this.enabled) return
+    this.#lastStart = start
+    if (!this.#timer || timeout !== this._timeoutInterval) {
+      this.#timer = setTimeout(() => this._collect(snapshotKinds.PERIODIC), timeout)
+      this.#timer.unref()
+    } else {
+      this.#timer.refresh()
+    }
+  }
+
+  #onSpanFinish (span) {
+    const context = span.context()
+    const tags = context._tags
+    if (!isWebServerSpan(tags)) return
+
+    const endpointName = endpointNameFromTags(tags)
+    if (!endpointName) return
+
+    // Make sure this is the outermost web span, just in case so we don't overcount
+    if (findWebSpan(getStartedSpans(context), context._parentId)) return
+
+    let counter = this.#endpointCounts.get(endpointName)
+    if (counter === undefined) {
+      counter = { count: 1 }
+      this.#endpointCounts.set(endpointName, counter)
+    } else {
+      counter.count++
+    }
+  }
+
+  #createInitialInfos () {
+    return {
+      serverless: this.serverless,
+      settings: this.#config.systemInfoReport,
+      sourceMapCount: this.#sourceMapCount,
+    }
+  }
+
+  async _collect (snapshotKind, restart = true) {
+    if (!this.enabled) return
+
+    try {
+      if (this.#config.profilers.length === 0) {
+        throw new Error('No profile types configured.')
+      }
+
+      const startDate = this.#lastStart
+      const endDate = new Date()
+      const profiles = []
+
+      crashtracker.withProfilerSerializing(() => {
+        // collect profiles synchronously so that profilers can be safely stopped asynchronously
+        for (const profiler of this.#config.profilers) {
+          const info = profiler.getInfo()
+          const profile = profiler.profile(restart, startDate, endDate)
+          if (!restart) {
+            this.#logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`)
+          }
+          if (!profile) continue
+          profiles.push({ profiler, profile, info })
+        }
+      })
+
+      if (restart) {
+        this._capture(this._timeoutInterval, endDate)
+      }
+
+      let hasEncoded = false
+
+      const encodedProfiles = {}
+      const infos = this.#createInitialInfos()
+
+      // encode and export asynchronously
+      await Promise.all(profiles.map(async ({ profiler, profile, info }) => {
+        try {
+          const encoded = await profiler.encode(profile)
+          const compressed = encoded instanceof Buffer && this.#compressionFn !== undefined
+            ? await this.#compressionFn(encoded, this.#compressionOptions)
+            : encoded
+          encodedProfiles[profiler.type] = compressed
+          processInfo(infos, info, profiler.type)
+          this.#logger.debug(() => {
+            const profileJson = JSON.stringify(profile, (_, value) => {
+              return typeof value === 'bigint' ? value.toString() : value
+            })
+            return `Collected ${profiler.type} profile: ` + profileJson
+          })
+          hasEncoded = true
+        } catch (err) {
+          // If encoding one of the profile types fails, we should still try to
+          // encode and submit the other profile types.
+          this.#logError(err)
+        }
+      }))
+
+      if (hasEncoded) {
+        await this.#submit(encodedProfiles, infos, startDate, endDate, snapshotKind)
+        profileSubmittedChannel.publish()
+        this.#logger.debug('Submitted profiles')
+      }
+    } catch (err) {
+      this.#logError(err)
+      this.#stop()
+    }
+  }
+
+  #submit (profiles, infos, start, end, snapshotKind) {
+    const { tags } = this.#config
+
+    // Flatten endpoint counts
+    const endpointCounts = {}
+    for (const [endpoint, { count }] of this.#endpointCounts) {
+      endpointCounts[endpoint] = count
+    }
+    this.#endpointCounts.clear()
+
+    tags.snapshot = snapshotKind
+    tags.profile_seq = this.#profileSeq++
+    const exportSpec = { profiles, infos, start, end, tags, endpointCounts }
+    const tasks = this.#config.exporters.map(exporter =>
+      exporter.export(exportSpec).catch(err => {
+        if (this.#logger) {
+          this.#logger.warn(err)
+        }
+      })
+    )
+
+    return Promise.all(tasks)
+  }
+}
+
+class ServerlessProfiler extends Profiler {
+  #profiledIntervals = 0
+  #interval = 1 // seconds
+  #flushAfterIntervals
+
+  constructor () {
+    super()
+    this.#profiledIntervals = 0
+    this.#interval = 1
+    this.#flushAfterIntervals = undefined
+  }
+
+  get serverless () { return true }
+
+  get profiledIntervals () {
+    return this.#profiledIntervals
+  }
+
+  _setInterval () {
+    this._timeoutInterval = this.#interval * 1000
+    this.#flushAfterIntervals = this.flushInterval / 1000
+  }
+
+  async _collect (snapshotKind, restart = true) {
+    if (this.#profiledIntervals >= this.#flushAfterIntervals || !restart) {
+      this.#profiledIntervals = 0
+      await super._collect(snapshotKind, restart)
+    } else {
+      this.#profiledIntervals += 1
+      this._capture(this._timeoutInterval, new Date())
+      // Don't submit profile until 65 (flushAfterIntervals) intervals have elapsed
+    }
+  }
+}
+
+module.exports = { Profiler, ServerlessProfiler }

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const EventPlugin = require('./event')
+
+class DNSPlugin extends EventPlugin {
+  static id = 'dns'
+
+  static entryType = 'dns'
+}
+
+module.exports = DNSPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const DNSPlugin = require('./dns')
+
+class DNSLookupPlugin extends DNSPlugin {
+  static operation = 'lookup'
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookup'
+    event.detail = { hostname: startEvent.args[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const DNSPlugin = require('./dns')
+
+class DNSLookupServicePlugin extends DNSPlugin {
+  static operation = 'lookup_service'
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookupService'
+    event.detail = { host: startEvent.args[0], port: startEvent.args[1] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupServicePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const DNSPlugin = require('./dns')
+
+const queryNames = new Map()
+
+class DNSResolvePlugin extends DNSPlugin {
+  static operation = 'resolve'
+
+  extendEvent (event, startEvent) {
+    const rrtype = startEvent.args[1]
+    let name = queryNames.get(rrtype)
+    if (!name) {
+      name = `query${rrtype}`
+      queryNames.set(rrtype, name)
+    }
+    event.name = name
+    event.detail = { host: startEvent.args[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSResolvePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const DNSPlugin = require('./dns')
+
+class DNSReversePlugin extends DNSPlugin {
+  static operation = 'reverse'
+
+  extendEvent (event, startEvent) {
+    event.name = 'getHostByAddr'
+    event.detail = { host: startEvent.args[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSReversePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/event.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const { performance } = require('perf_hooks')
+const TracingPlugin = require('../../../plugins/tracing')
+
+// We are leveraging the TracingPlugin class for its functionality to bind
+// start/error/finish methods to the appropriate diagnostic channels.
+// TODO: Decouple this from TracingPlugin.
+class EventPlugin extends TracingPlugin {
+  #eventHandler
+  #eventFilter
+  #dataSymbol
+  #entryType
+
+  constructor (eventHandler, eventFilter) {
+    super()
+    this.#eventHandler = eventHandler
+    this.#eventFilter = eventFilter
+    this.#entryType = this.constructor.entryType
+    this.#dataSymbol = Symbol(`dd-trace.profiling.event.${this.#entryType}.${this.constructor.operation}`)
+  }
+
+  start (ctx) {
+    ctx[this.#dataSymbol] = performance.now()
+  }
+
+  error (ctx) {
+    // We don't emit perf events for failed operations
+    ctx[this.#dataSymbol] = undefined
+  }
+
+  finish (ctx) {
+    const startTime = ctx[this.#dataSymbol]
+    if (startTime === undefined) {
+      return
+    }
+    ctx[this.#dataSymbol] = undefined
+
+    if (this.ignoreEvent(ctx)) {
+      return // don't emit perf events for ignored events
+    }
+
+    const duration = performance.now() - startTime
+    const event = {
+      entryType: this.#entryType,
+      startTime,
+      duration,
+    }
+
+    if (!this.#eventFilter(event)) {
+      return
+    }
+
+    const context = (ctx.currentStore?.span || this.activeSpan)?.context()
+    event._ddSpanId = context?.toBigIntSpanId()
+    event._ddRootSpanId = context?._trace.started[0]?.context().toBigIntSpanId() || event._ddSpanId
+
+    this.#eventHandler(this.extendEvent(event, ctx))
+  }
+
+  ignoreEvent () {
+    return false
+  }
+}
+
+module.exports = EventPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/fs.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const EventPlugin = require('./event')
+
+// Values taken from parameter names in datadog-instrumentations/src/fs.js.
+// Known param names that are disallowed because they can be strings and have arbitrary sizes:
+// 'data'
+// Known param names that are disallowed because they are never a string or number:
+// 'buffer', 'buffers', 'listener'
+const allowedParams = new Set([
+  'atime', 'dest',
+  'existingPath', 'fd', 'file',
+  'flag', 'gid', 'len',
+  'length', 'mode', 'mtime',
+  'newPath', 'offset', 'oldPath',
+  'operation', 'options', 'path',
+  'position', 'prefix', 'src',
+  'target', 'type', 'uid',
+])
+
+class FilesystemPlugin extends EventPlugin {
+  static id = 'fs'
+
+  static operation = 'operation'
+
+  static entryType = 'fs'
+
+  ignoreEvent (event) {
+    // Don't care about sync events, they show up in the event loop samples anyway
+    return event.operation?.endsWith('Sync')
+  }
+
+  extendEvent (event, detail) {
+    const d = { ...detail }
+    for (const [k, v] of Object.entries(d)) {
+      if (!(allowedParams.has(k) && (typeof v === 'string' || typeof v === 'number'))) {
+        delete d[k]
+      }
+    }
+    event.detail = d
+
+    return event
+  }
+}
+module.exports = FilesystemPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/net.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const EventPlugin = require('./event')
+
+class NetPlugin extends EventPlugin {
+  static id = 'net'
+
+  static operation = 'tcp'
+
+  static entryType = 'net'
+
+  extendEvent (event, { options }) {
+    event.name = 'connect'
+    event.detail = options
+
+    return event
+  }
+}
+
+module.exports = NetPlugin