dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -9,6 +9,8 @@ const docker = require('../../exporters/common/docker')
 const FormData = require('../../exporters/common/form-data')
 const { storage } = require('../../../../datadog-core')
 const version = require('../../../../../package.json').version
+const os = require('os')
+const perf = require('perf_hooks').performance
 
 const containerId = docker.id()
 
@@ -50,7 +52,7 @@ function computeRetries (uploadTimeout) {
 }
 
 class AgentExporter {
-  constructor ({ url, logger, uploadTimeout } = {}) {
+  constructor ({ url, logger, uploadTimeout, env, host, service, version } = {}) {
     this._url = url
     this._logger = logger
 
@@ -58,47 +60,87 @@ class AgentExporter {
 
     this._backoffTime = backoffTime
     this._backoffTries = backoffTries
+    this._env = env
+    this._host = host
+    this._service = service
+    this._appVersion = version
   }
 
   export ({ profiles, start, end, tags }) {
-    const types = Object.keys(profiles)
-
-    const fields = [
-      ['recording-start', start.toISOString()],
-      ['recording-end', end.toISOString()],
-      ['language', 'javascript'],
-      ['runtime', 'nodejs'],
-      ['runtime_version', process.version],
-      ['profiler_version', version],
-      ['format', 'pprof'],
-
-      ['tags[]', 'language:javascript'],
-      ['tags[]', 'runtime:nodejs'],
-      ['tags[]', `runtime_version:${process.version}`],
-      ['tags[]', `profiler_version:${version}`],
-      ['tags[]', 'format:pprof'],
-      ...Object.entries(tags).map(([key, value]) => ['tags[]', `${key}:${value}`])
-    ]
+    const fields = []
 
-    this._logger.debug(() => {
-      const body = fields.map(([key, value]) => `  ${key}: ${value}`).join('\n')
-      return `Building agent export report: ${'\n' + body}`
+    function typeToFile (type) {
+      return `${type}.pprof`
+    }
+
+    const event = JSON.stringify({
+      attachments: Object.keys(profiles).map(typeToFile),
+      start: start.toISOString(),
+      end: end.toISOString(),
+      family: 'node',
+      version: '4',
+      tags_profiler: [
+        'language:javascript',
+        'runtime:nodejs',
+        `runtime_arch:${process.arch}`,
+        `runtime_os:${process.platform}`,
+        `runtime_version:${process.version}`,
+        `process_id:${process.pid}`,
+        `profiler_version:${version}`,
+        'format:pprof',
+        ...Object.entries(tags).map(([key, value]) => `${key}:${value}`)
+      ].join(','),
+      info: {
+        application: {
+          env: this._env,
+          service: this._service,
+          start_time: new Date(perf.nodeTiming.nodeStart + perf.timeOrigin).toISOString(),
+          version: this._appVersion
+        },
+        platform: {
+          hostname: this._host,
+          kernel_name: os.type(),
+          kernel_release: os.release(),
+          kernel_version: os.version()
+        },
+        profiler: {
+          version
+        },
+        runtime: {
+          // Using `nodejs` for consistency with the existing `runtime` tag.
+          // Note that the event `family` property uses `node`, as that's what's
+          // proscribed by the Intake API, but that's an internal enum and is
+          // not customer visible.
+          engine: 'nodejs',
+          // strip off leading 'v'. This makes the format consistent with other
+          // runtimes (e.g. Ruby) but not with the existing `runtime_version` tag.
+          // We'll keep it like this as we want cross-engine consistency. We
+          // also aren't changing the format of the existing tag as we don't want
+          // to break it.
+          version: process.version.substring(1)
+        }
+      }
     })
 
-    for (let index = 0; index < types.length; index++) {
-      const type = types[index]
-      const buffer = profiles[type]
+    fields.push(['event', event, {
+      filename: 'event.json',
+      contentType: 'application/json'
+    }])
+
+    this._logger.debug(() => {
+      return `Building agent export report:\n${event}`
+    })
 
+    for (const [type, buffer] of Object.entries(profiles)) {
       this._logger.debug(() => {
         const bytes = buffer.toString('hex').match(/../g).join(' ')
         return `Adding ${type} profile to agent export: ` + bytes
       })
 
-      fields.push([`types[${index}]`, type])
-      fields.push([`data[${index}]`, buffer, {
-        filename: `${type}.pb.gz`,
-        contentType: 'application/octet-stream',
-        knownLength: buffer.length
+      const filename = typeToFile(type)
+      fields.push([filename, buffer, {
+        filename,
+        contentType: 'application/octet-stream'
       }])
     }
 
@@ -120,7 +162,11 @@ class AgentExporter {
         const options = {
           method: 'POST',
           path: '/profiling/v1/input',
-          headers: form.getHeaders(),
+          headers: {
+            'DD-EVP-ORIGIN': 'dd-trace-js',
+            'DD-EVP-ORIGIN-VERSION': version,
+            ...form.getHeaders()
+          },
           timeout: this._backoffTime * Math.pow(2, attempt)
         }
 

package/packages/dd-trace/src/profiling/exporters/file.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs')
 const { promisify } = require('util')
+const { threadId } = require('worker_threads')
 const writeFile = promisify(fs.writeFile)
 
 function formatDateTime (t) {
@@ -19,7 +20,7 @@ class FileExporter {
     const types = Object.keys(profiles)
     const dateStr = formatDateTime(end)
     const tasks = types.map(type => {
-      return writeFile(`${this._pprofPrefix}${type}_${dateStr}.pprof`, profiles[type])
+      return writeFile(`${this._pprofPrefix}${type}_worker_${threadId}_${dateStr}.pprof`, profiles[type])
     })
 
     return Promise.all(tasks)

package/packages/dd-trace/src/profiling/profiler.js

@@ -3,6 +3,7 @@
 const { EventEmitter } = require('events')
 const { Config } = require('./config')
 const { snapshotKinds } = require('./constants')
+const { threadNamePrefix } = require('./profilers/shared')
 
 function maybeSourceMap (sourceMap, SourceMapper, debug) {
   if (!sourceMap) return
@@ -23,15 +24,19 @@ class Profiler extends EventEmitter {
   }
 
   start (options) {
-    this._start(options).catch((err) => { if (options.logger) options.logger.error(err) })
-    return this
+    return this._start(options).catch((err) => {
+      if (options.logger) {
+        options.logger.error(err)
+      }
+      return false
+    })
   }
 
   async _start (options) {
-    if (this._enabled) return
+    if (this._enabled) return true
 
     const config = this._config = new Config(options)
-    if (!config.enabled) return
+    if (!config.enabled) return false
 
     this._logger = config.logger
     this._enabled = true
@@ -57,19 +62,22 @@ class Profiler extends EventEmitter {
     }
 
     try {
+      const start = new Date()
       for (const profiler of config.profilers) {
         // TODO: move this out of Profiler when restoring sourcemap support
         profiler.start({
           mapper,
           nearOOMCallback: this._nearOOMExport.bind(this)
         })
-        this._logger.debug(`Started ${profiler.type} profiler`)
+        this._logger.debug(`Started ${profiler.type} profiler in ${threadNamePrefix} thread`)
       }
 
-      this._capture(this._timeoutInterval)
+      this._capture(this._timeoutInterval, start)
+      return true
     } catch (e) {
       this._logger.error(e)
       this._stop()
+      return false
     }
   }
 
@@ -90,7 +98,7 @@ class Profiler extends EventEmitter {
 
     // collect and export current profiles
     // once collect returns, profilers can be safely stopped
-    this._collect(snapshotKinds.ON_SHUTDOWN)
+    this._collect(snapshotKinds.ON_SHUTDOWN, false)
     this._stop()
   }
 
@@ -101,18 +109,16 @@ class Profiler extends EventEmitter {
 
     for (const profiler of this._config.profilers) {
       profiler.stop()
-      this._logger.debug(`Stopped ${profiler.type} profiler`)
+      this._logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`)
     }
 
     clearTimeout(this._timer)
     this._timer = undefined
-
-    return this
   }
 
-  _capture (timeout) {
+  _capture (timeout, start) {
     if (!this._enabled) return
-    this._lastStart = new Date()
+    this._lastStart = start
     if (!this._timer || timeout !== this._timeoutInterval) {
       this._timer = setTimeout(() => this._collect(snapshotKinds.PERIODIC), timeout)
       this._timer.unref()
@@ -121,18 +127,21 @@ class Profiler extends EventEmitter {
     }
   }
 
-  async _collect (snapshotKind) {
+  async _collect (snapshotKind, restart = true) {
     if (!this._enabled) return
 
-    const start = this._lastStart
-    const end = new Date()
+    const startDate = this._lastStart
+    const endDate = new Date()
     const profiles = []
     const encodedProfiles = {}
 
     try {
       // collect profiles synchronously so that profilers can be safely stopped asynchronously
       for (const profiler of this._config.profilers) {
-        const profile = profiler.profile()
+        const profile = profiler.profile(restart, startDate, endDate)
+        if (!restart) {
+          this._logger.debug(`Stopped ${profiler.type} profiler in ${threadNamePrefix} thread`)
+        }
         if (!profile) continue
         profiles.push({ profiler, profile })
       }
@@ -148,8 +157,10 @@ class Profiler extends EventEmitter {
         })
       }
 
-      this._capture(this._timeoutInterval)
-      await this._submit(encodedProfiles, start, end, snapshotKind)
+      if (restart) {
+        this._capture(this._timeoutInterval, endDate)
+      }
+      await this._submit(encodedProfiles, startDate, endDate, snapshotKind)
       this._logger.debug('Submitted profiles')
     } catch (err) {
       this._logger.error(err)
@@ -189,13 +200,13 @@ class ServerlessProfiler extends Profiler {
     this._flushAfterIntervals = this._config.flushInterval / 1000
   }
 
-  async _collect (snapshotKind) {
-    if (this._profiledIntervals >= this._flushAfterIntervals) {
+  async _collect (snapshotKind, restart = true) {
+    if (this._profiledIntervals >= this._flushAfterIntervals || !restart) {
       this._profiledIntervals = 0
-      await super._collect(snapshotKind)
+      await super._collect(snapshotKind, restart)
     } else {
       this._profiledIntervals += 1
-      this._capture(this._timeoutInterval)
+      this._capture(this._timeoutInterval, new Date())
       // Don't submit profile until 65 (flushAfterIntervals) intervals have elapsed
     }
   }

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -0,0 +1,270 @@
+const { performance, constants, PerformanceObserver } = require('perf_hooks')
+const { END_TIMESTAMP_LABEL } = require('./shared')
+const semver = require('semver')
+const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
+const pprof = require('@datadog/pprof/')
+
+// Format of perf_hooks events changed with Node 16, we need to be mindful of it.
+const node16 = semver.gte(process.version, '16.0.0')
+
+// perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file.
+const MS_TO_NS = 1000000
+
+// While this is an "events profiler", meaning it emits a pprof file based on events observed as
+// perf_hooks events, the emitted pprof file uses the type "timeline".
+const pprofValueType = 'timeline'
+const pprofValueUnit = 'nanoseconds'
+
+function labelFromStr (stringTable, key, valStr) {
+  return new Label({ key, str: stringTable.dedup(valStr) })
+}
+
+function labelFromStrStr (stringTable, keyStr, valStr) {
+  return labelFromStr(stringTable, stringTable.dedup(keyStr), valStr)
+}
+
+class GCDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.reasonLabelKey = stringTable.dedup('gc reason')
+    this.kindLabels = []
+    this.reasonLabels = []
+    this.flagObj = {}
+
+    const kindLabelKey = stringTable.dedup('gc type')
+
+    // Create labels for all GC performance flags and kinds of GC
+    for (const [key, value] of Object.entries(constants)) {
+      if (key.startsWith('NODE_PERFORMANCE_GC_FLAGS_')) {
+        this.flagObj[key.substring(26).toLowerCase()] = value
+      } else if (key.startsWith('NODE_PERFORMANCE_GC_')) {
+        // It's a constant for a kind of GC
+        const kind = key.substring(20).toLowerCase()
+        this.kindLabels[value] = labelFromStr(stringTable, kindLabelKey, kind)
+      }
+    }
+  }
+
+  decorateSample (sampleInput, item) {
+    const { kind, flags } = node16 ? item.detail : item
+    sampleInput.label.push(this.kindLabels[kind])
+    const reasonLabel = this.getReasonLabel(flags)
+    if (reasonLabel) {
+      sampleInput.label.push(reasonLabel)
+    }
+  }
+
+  getReasonLabel (flags) {
+    if (flags === 0) {
+      return null
+    }
+    let reasonLabel = this.reasonLabels[flags]
+    if (!reasonLabel) {
+      const reasons = []
+      for (const [key, value] of Object.entries(this.flagObj)) {
+        if (value & flags) {
+          reasons.push(key)
+        }
+      }
+      const reasonStr = reasons.join(',')
+      reasonLabel = labelFromStr(this.stringTable, this.reasonLabelKey, reasonStr)
+      this.reasonLabels[flags] = reasonLabel
+    }
+    return reasonLabel
+  }
+}
+
+class DNSDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.addressLabelKey = stringTable.dedup('address')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, item.name)
+    const detail = item.detail
+    switch (op) {
+      case 'lookup':
+        addLabel(this.hostLabelKey, detail.hostname)
+        break
+      case 'lookupService':
+        addLabel(this.addressLabelKey, detail.host)
+        labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+        break
+      case 'getHostByAddr':
+        addLabel(this.addressLabelKey, detail.host)
+        break
+      default:
+        if (op.startsWith('query')) {
+          addLabel(this.hostLabelKey, detail.host)
+        }
+    }
+  }
+}
+
+class NetDecorator {
+  constructor (stringTable) {
+    this.stringTable = stringTable
+    this.operationNameLabelKey = stringTable.dedup('operation')
+    this.hostLabelKey = stringTable.dedup('host')
+    this.portLabelKey = stringTable.dedup('port')
+  }
+
+  decorateSample (sampleInput, item) {
+    const labels = sampleInput.label
+    const stringTable = this.stringTable
+    function addLabel (labelNameKey, labelValue) {
+      labels.push(labelFromStr(stringTable, labelNameKey, labelValue))
+    }
+    const op = item.name
+    addLabel(this.operationNameLabelKey, op)
+    if (op === 'connect') {
+      const detail = item.detail
+      addLabel(this.hostLabelKey, detail.host)
+      labels.push(new Label({ key: this.portLabelKey, num: detail.port }))
+    }
+  }
+}
+
+// Keys correspond to PerformanceEntry.entryType, values are constructor
+// functions for type-specific decorators.
+const decoratorTypes = {
+  gc: GCDecorator
+}
+// Needs at least node 16 for DNS and Net
+if (node16) {
+  decoratorTypes.dns = DNSDecorator
+  decoratorTypes.net = NetDecorator
+}
+
+/**
+ * This class generates pprof files with timeline events sourced from Node.js
+ * performance measurement APIs.
+ */
+class EventsProfiler {
+  constructor (options = {}) {
+    this.type = 'events'
+    this._flushIntervalNanos = (options.flushInterval || 60000) * 1e6 // 60 sec
+    this._observer = undefined
+    this.entries = []
+  }
+
+  start () {
+    // if already started, do nothing
+    if (this._observer) return
+
+    function add (items) {
+      this.entries.push(...items.getEntries())
+    }
+    this._observer = new PerformanceObserver(add.bind(this))
+    this._observer.observe({ entryTypes: Object.keys(decoratorTypes) })
+  }
+
+  stop () {
+    if (this._observer) {
+      this._observer.disconnect()
+      this._observer = undefined
+    }
+  }
+
+  profile (restart, startDate, endDate) {
+    if (this.entries.length === 0) {
+      // No events in the period; don't produce a profile
+      return null
+    }
+
+    const stringTable = new StringTable()
+    const locations = []
+    const functions = []
+
+    // A synthetic single-frame location to serve as the location for timeline
+    // samples. We need these as the profiling backend (mimicking official pprof
+    // tool's behavior) ignores these.
+    const locationId = (() => {
+      const fn = new Function({ id: functions.length + 1, name: stringTable.dedup('') })
+      functions.push(fn)
+      const line = new Line({ functionId: fn.id })
+      const location = new Location({ id: locations.length + 1, line: [line] })
+      locations.push(location)
+      return [location.id]
+    })()
+
+    const decorators = {}
+    for (const [eventType, DecoratorCtor] of Object.entries(decoratorTypes)) {
+      const decorator = new DecoratorCtor(stringTable)
+      decorator.eventTypeLabel = labelFromStrStr(stringTable, 'event', eventType)
+      decorators[eventType] = decorator
+    }
+    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP_LABEL)
+
+    const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS))
+    const lateEntries = []
+    const perfEndDate = endDate.getTime() - performance.timeOrigin
+    const samples = this.entries.map((item) => {
+      const decorator = decorators[item.entryType]
+      if (!decorator) {
+        // Shouldn't happen but it's better to not rely on observer only getting
+        // requested event types.
+        return null
+      }
+      const { startTime, duration } = item
+      if (startTime >= perfEndDate) {
+        // An event past the current recording end date; save it for the next
+        // profile. Not supposed to happen as long as there's no async activity
+        // between capture of the endDate value in profiler.js _collect() and
+        // here, but better be safe than sorry.
+        lateEntries.push(item)
+        return null
+      }
+      const endTime = startTime + duration
+      const sampleInput = {
+        value: [Math.round(duration * MS_TO_NS)],
+        locationId,
+        label: [
+          decorator.eventTypeLabel,
+          new Label({ key: timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) })
+        ]
+      }
+      decorator.decorateSample(sampleInput, item)
+      return new Sample(sampleInput)
+    }).filter(v => v)
+
+    this.entries = lateEntries
+
+    const timeValueType = new ValueType({
+      type: stringTable.dedup(pprofValueType),
+      unit: stringTable.dedup(pprofValueUnit)
+    })
+
+    if (!restart) {
+      this.stop()
+    }
+
+    return new Profile({
+      sampleType: [timeValueType],
+      timeNanos: endDate.getTime() * MS_TO_NS,
+      periodType: timeValueType,
+      period: 1,
+      durationNanos: (endDate.getTime() - startDate.getTime()) * MS_TO_NS,
+      sample: samples,
+      location: locations,
+      function: functions,
+      stringTable: stringTable
+    })
+  }
+
+  encode (profile) {
+    return pprof.encode(profile)
+  }
+}
+
+module.exports = EventsProfiler

package/packages/dd-trace/src/profiling/profilers/shared.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const { isMainThread, threadId } = require('worker_threads')
+
+const END_TIMESTAMP_LABEL = 'end_timestamp_ns'
+const THREAD_NAME_LABEL = 'thread name'
+const OS_THREAD_ID_LABEL = 'os thread id'
+const THREAD_ID_LABEL = 'thread id'
+const threadNamePrefix = isMainThread ? 'Main' : `Worker #${threadId}`
+const eventLoopThreadName = `${threadNamePrefix} Event Loop`
+
+function getThreadLabels () {
+  const pprof = require('@datadog/pprof')
+  const nativeThreadId = pprof.getNativeThreadId()
+  return {
+    [THREAD_NAME_LABEL]: eventLoopThreadName,
+    [THREAD_ID_LABEL]: `${threadId}`,
+    [OS_THREAD_ID_LABEL]: `${nativeThreadId}`
+  }
+}
+
+function cacheThreadLabels () {
+  let labels
+  return () => {
+    if (!labels) {
+      labels = getThreadLabels()
+    }
+    return labels
+  }
+}
+
+function getNonJSThreadsLabels () {
+  return { [THREAD_NAME_LABEL]: 'Non-JS threads', [THREAD_ID_LABEL]: 'NA', [OS_THREAD_ID_LABEL]: 'NA' }
+}
+
+module.exports = {
+  END_TIMESTAMP_LABEL,
+  THREAD_NAME_LABEL,
+  THREAD_ID_LABEL,
+  OS_THREAD_ID_LABEL,
+  threadNamePrefix,
+  eventLoopThreadName,
+  getNonJSThreadsLabels,
+  getThreadLabels: cacheThreadLabels()
+}

package/packages/dd-trace/src/profiling/profilers/space.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { oomExportStrategies } = require('../constants')
+const { getThreadLabels } = require('./shared')
 
 function strategiesToCallbackMode (strategies, callbackMode) {
   return strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0
@@ -13,9 +14,12 @@ class NativeSpaceProfiler {
     this._stackDepth = options.stackDepth || 64
     this._pprof = undefined
     this._oomMonitoring = options.oomMonitoring || {}
+    this._started = false
   }
 
   start ({ mapper, nearOOMCallback } = {}) {
+    if (this._started) return
+
     this._mapper = mapper
     this._pprof = require('@datadog/pprof')
     this._pprof.heap.start(this._samplingInterval, this._stackDepth)
@@ -30,10 +34,16 @@ class NativeSpaceProfiler {
         strategiesToCallbackMode(strategies, this._pprof.heap.CallbackMode)
       )
     }
+
+    this._started = true
   }
 
-  profile () {
-    return this._pprof.heap.profile(undefined, this._mapper)
+  profile (restart) {
+    const profile = this._pprof.heap.profile(undefined, this._mapper, getThreadLabels)
+    if (!restart) {
+      this.stop()
+    }
+    return profile
   }
 
   encode (profile) {
@@ -41,7 +51,13 @@ class NativeSpaceProfiler {
   }
 
   stop () {
+    if (!this._started) return
     this._pprof.heap.stop()
+    this._started = false
+  }
+
+  isStarted () {
+    return this._started
   }
 }