dd-trace 3.14.1 → 3.16.0

package/packages/dd-trace/src/profiling/config.js

@@ -2,48 +2,59 @@
 
 const coalesce = require('koalas')
 const os = require('os')
-const { URL, format } = require('url')
+const path = require('path')
+const { URL, format, pathToFileURL } = require('url')
 const { AgentExporter } = require('./exporters/agent')
 const { FileExporter } = require('./exporters/file')
 const { ConsoleLogger } = require('./loggers/console')
 const CpuProfiler = require('./profilers/cpu')
 const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
+const { oomExportStrategies, snapshotKinds } = require('./constants')
 const { tagger } = require('./tagger')
-
-const {
-  DD_PROFILING_ENABLED,
-  DD_PROFILING_PROFILERS,
-  DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
-  DD_ENV,
-  DD_TAGS,
-  DD_SERVICE,
-  DD_VERSION,
-  DD_TRACE_AGENT_URL,
-  DD_AGENT_HOST,
-  DD_TRACE_AGENT_PORT,
-  DD_PROFILING_UPLOAD_TIMEOUT,
-  DD_PROFILING_SOURCE_MAP
-} = process.env
+const { isTrue } = require('../util')
 
 class Config {
   constructor (options = {}) {
-    const enabled = coalesce(options.enabled, DD_PROFILING_ENABLED, true)
+    const {
+      DD_PROFILING_ENABLED,
+      DD_PROFILING_PROFILERS,
+      DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
+      DD_ENV,
+      DD_TAGS,
+      DD_SERVICE,
+      DD_VERSION,
+      DD_TRACE_AGENT_URL,
+      DD_AGENT_HOST,
+      DD_TRACE_AGENT_PORT,
+      DD_PROFILING_UPLOAD_TIMEOUT,
+      DD_PROFILING_SOURCE_MAP,
+      DD_PROFILING_UPLOAD_PERIOD,
+      DD_PROFILING_PPROF_PREFIX,
+      DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED,
+      DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE,
+      DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT,
+      DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES
+    } = process.env
+
+    const enabled = isTrue(coalesce(options.enabled, DD_PROFILING_ENABLED, true))
     const env = coalesce(options.env, DD_ENV)
     const service = options.service || DD_SERVICE || 'node'
     const host = os.hostname()
     const version = coalesce(options.version, DD_VERSION)
     const functionname = process.env.AWS_LAMBDA_FUNCTION_NAME
     // Must be longer than one minute so pad with five seconds
-    const flushInterval = coalesce(options.interval, 65 * 1000)
+    const flushInterval = coalesce(options.interval, Number(DD_PROFILING_UPLOAD_PERIOD) * 1000, 65 * 1000)
     const uploadTimeout = coalesce(options.uploadTimeout,
-      DD_PROFILING_UPLOAD_TIMEOUT, 60 * 1000)
+      Number(DD_PROFILING_UPLOAD_TIMEOUT), 60 * 1000)
     const sourceMap = coalesce(options.sourceMap,
       DD_PROFILING_SOURCE_MAP, true)
     const endpointCollection = coalesce(options.endpointCollection,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, false)
+    const pprofPrefix = coalesce(options.pprofPrefix,
+      DD_PROFILING_PPROF_PREFIX)
 
-    this.enabled = String(enabled) !== 'false'
+    this.enabled = enabled
     this.service = service
     this.env = env
     this.host = host
@@ -60,6 +71,7 @@ class Config {
     this.uploadTimeout = uploadTimeout
     this.sourceMap = sourceMap
     this.endpointCollection = endpointCollection
+    this.pprofPrefix = pprofPrefix
 
     const hostname = coalesce(options.hostname, DD_AGENT_HOST) || 'localhost'
     const port = coalesce(options.port, DD_TRACE_AGENT_PORT) || 8126
@@ -73,6 +85,23 @@ class Config {
       new AgentExporter(this)
     ], this)
 
+    const oomMonitoringEnabled = isTrue(coalesce(options.oomMonitoring,
+      DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, false))
+    const heapLimitExtensionSize = coalesce(options.oomHeapLimitExtensionSize,
+      Number(DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE), 0)
+    const maxHeapExtensionCount = coalesce(options.oomMaxHeapExtensionCount,
+      Number(DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT), 0)
+    const exportStrategies = ensureOOMExportStrategies(coalesce(options.oomExportStrategies,
+      DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES), this)
+    const exportCommand = oomMonitoringEnabled ? buildExportCommand(this) : undefined
+    this.oomMonitoring = {
+      enabled: oomMonitoringEnabled,
+      heapLimitExtensionSize,
+      maxHeapExtensionCount,
+      exportStrategies,
+      exportCommand
+    }
+
     const profilers = coalesce(options.profilers, DD_PROFILING_PROFILERS, [
       new WallProfiler(this),
       new SpaceProfiler(this)
@@ -84,6 +113,33 @@ class Config {
 
 module.exports = { Config }
 
+function getExportStrategy (name, options) {
+  const strategy = Object.values(oomExportStrategies).find(value => value === name)
+  if (strategy === undefined) {
+    options.logger.error(`Unknown oom export strategy "${name}"`)
+  }
+  return strategy
+}
+
+function ensureOOMExportStrategies (strategies, options) {
+  if (!strategies) {
+    return []
+  }
+
+  if (typeof strategies === 'string') {
+    strategies = strategies.split(',')
+  }
+
+  for (let i = 0; i < strategies.length; i++) {
+    const strategy = strategies[i]
+    if (typeof strategy === 'string') {
+      strategies[i] = getExportStrategy(strategy, options)
+    }
+  }
+
+  return [ ...new Set(strategies) ]
+}
+
 function getExporter (name, options) {
   switch (name) {
     case 'agent':
@@ -149,3 +205,19 @@ function ensureLogger (logger) {
 
   return logger
 }
+
+function buildExportCommand (options) {
+  const tags = [...Object.entries(options.tags),
+    ['snapshot', snapshotKinds.ON_OUT_OF_MEMORY]].map(([key, value]) => `${key}:${value}`).join(',')
+  const urls = []
+  for (const exporter of options.exporters) {
+    if (exporter instanceof AgentExporter) {
+      urls.push(options.url.toString())
+    } else if (exporter instanceof FileExporter) {
+      urls.push(pathToFileURL(options.pprofPrefix).toString())
+    }
+  }
+  return [process.execPath,
+    path.join(__dirname, 'exporter_cli.js'),
+    urls.join(','), tags, 'space']
+}

package/packages/dd-trace/src/profiling/constants.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const snapshotKinds = Object.freeze({
+  PERIODIC: 'periodic',
+  ON_SHUTDOWN: 'on_shutdown',
+  ON_OUT_OF_MEMORY: 'on_oom'
+})
+
+const oomExportStrategies = Object.freeze({
+  PROCESS: 'process',
+  ASYNC_CALLBACK: 'async',
+  INTERRUPT_CALLBACK: 'interrupt',
+  LOGS: 'logs'
+})
+
+module.exports = { snapshotKinds, oomExportStrategies }

package/packages/dd-trace/src/profiling/exporter_cli.js

@@ -0,0 +1,62 @@
+'use strict'
+
+const { AgentExporter } = require('./exporters/agent')
+const { FileExporter } = require('./exporters/file')
+
+const { SourceMapper, heap, encode } = require('@datadog/pprof')
+const { ConsoleLogger } = require('./loggers/console')
+const { tagger } = require('./tagger')
+const fs = require('fs')
+const { fileURLToPath } = require('url')
+
+const logger = new ConsoleLogger()
+const timeoutMs = 10 * 1000
+
+function exporterFromURL (url) {
+  if (url.protocol === 'file:') {
+    return new FileExporter({ pprofPrefix: fileURLToPath(url) })
+  } else {
+    return new AgentExporter({
+      url,
+      logger,
+      uploadTimeout: timeoutMs
+    })
+  }
+}
+
+async function exportProfile (urls, tags, profileType, profile) {
+  let mapper
+  try {
+    mapper = await SourceMapper.create([process.cwd()])
+  } catch (err) {
+    logger.error(err)
+  }
+
+  const encodedProfile = await encode(heap.convertProfile(profile, undefined, mapper))
+  const start = new Date()
+  for (const url of urls) {
+    const exporter = exporterFromURL(url)
+
+    await exporter.export({
+      profiles: {
+        [profileType]: encodedProfile
+      },
+      start,
+      end: start,
+      tags
+    })
+  }
+}
+
+/** Expected command line arguments are:
+* - Comma separated list of URLs (eg. "http://127.0.0.1:8126/,file:///tmp/foo.pprof")
+* - Tags (eg. "service:nodejs_oom_test,version:1.0.0")
+* - Profiletype (eg. space,wall,cpu)
+* - JSON profile filepath
+**/
+const urls = process.argv[2].split(',').map(s => new URL(s))
+const tags = tagger.parse(process.argv[3])
+const profileType = process.argv[4]
+const profile = JSON.parse(fs.readFileSync(process.argv[5]))
+
+exportProfile(urls, tags, profileType, profile)

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -103,7 +103,8 @@ class AgentExporter {
       const operation = retry.operation({
         randomize: true,
         minTimeout: this._backoffTime,
-        retries: this._backoffTries
+        retries: this._backoffTries,
+        unref: true
       })
 
       operation.attempt((attempt) => {

package/packages/dd-trace/src/profiling/exporters/file.js

@@ -4,11 +4,22 @@ const fs = require('fs')
 const { promisify } = require('util')
 const writeFile = promisify(fs.writeFile)
 
+function formatDateTime (t) {
+  const pad = (n) => String(n).padStart(2, '0')
+  return `${t.getUTCFullYear()}${pad(t.getUTCMonth() + 1)}${pad(t.getUTCDate())}` +
+         `T${pad(t.getUTCHours())}${pad(t.getUTCMinutes())}${pad(t.getUTCSeconds())}Z`
+}
+
 class FileExporter {
-  export ({ profiles }) {
+  constructor ({ pprofPrefix } = {}) {
+    this._pprofPrefix = pprofPrefix || ''
+  }
+
+  export ({ profiles, end }) {
     const types = Object.keys(profiles)
+    const dateStr = formatDateTime(end)
     const tasks = types.map(type => {
-      return writeFile(`${type}.pb.gz`, profiles[type])
+      return writeFile(`${this._pprofPrefix}${type}_${dateStr}.pprof`, profiles[type])
     })
 
     return Promise.all(tasks)

package/packages/dd-trace/src/profiling/profiler.js

@@ -2,6 +2,7 @@
 
 const { EventEmitter } = require('events')
 const { Config } = require('./config')
+const { snapshotKinds } = require('./constants')
 
 function maybeSourceMap (sourceMap) {
   if (!sourceMap) return
@@ -49,22 +50,42 @@ class Profiler extends EventEmitter {
     try {
       for (const profiler of config.profilers) {
         // TODO: move this out of Profiler when restoring sourcemap support
-        profiler.start({ mapper })
+        profiler.start({
+          mapper,
+          nearOOMCallback: this._nearOOMExport.bind(this)
+        })
         this._logger.debug(`Started ${profiler.type} profiler`)
       }
 
       this._capture(this._timeoutInterval)
     } catch (e) {
       this._logger.error(e)
-      this.stop()
+      this._stop()
     }
   }
 
+  _nearOOMExport (profileType, encodedProfile) {
+    const start = this._lastStart
+    const end = new Date()
+    this._submit({
+      [profileType]: encodedProfile
+    }, start, end, snapshotKinds.ON_OUT_OF_MEMORY)
+  }
+
   _setInterval () {
     this._timeoutInterval = this._config.flushInterval
   }
 
-  stop () {
+  async stop () {
+    if (!this._enabled) return
+
+    // collect and export current profiles
+    // once collect returns, profilers can be safely stopped
+    this._collect(snapshotKinds.ON_SHUTDOWN)
+    this._stop()
+  }
+
+  _stop () {
     if (!this._enabled) return
 
     this._enabled = false
@@ -84,24 +105,32 @@ class Profiler extends EventEmitter {
     if (!this._enabled) return
     this._lastStart = new Date()
     if (!this._timer || timeout !== this._timeoutInterval) {
-      this._timer = setTimeout(() => this._collect(), timeout)
+      this._timer = setTimeout(() => this._collect(snapshotKinds.PERIODIC), timeout)
       this._timer.unref()
     } else {
       this._timer.refresh()
     }
   }
 
-  async _collect () {
+  async _collect (snapshotKind) {
+    if (!this._enabled) return
+
     const start = this._lastStart
     const end = new Date()
-    const profiles = {}
+    const profiles = []
+    const encodedProfiles = {}
 
     try {
+      // collect profiles synchronously so that profilers can be safely stopped asynchronously
       for (const profiler of this._config.profilers) {
         const profile = profiler.profile()
         if (!profile) continue
+        profiles.push({ profiler, profile })
+      }
 
-        profiles[profiler.type] = await profiler.encode(profile)
+      // encode and export asynchronously
+      for (const { profiler, profile } of profiles) {
+        encodedProfiles[profiler.type] = await profiler.encode(profile)
         this._logger.debug(() => {
           const profileJson = JSON.stringify(profile, (key, value) => {
             return typeof value === 'bigint' ? value.toString() : value
@@ -111,21 +140,22 @@ class Profiler extends EventEmitter {
       }
 
       this._capture(this._timeoutInterval)
-      await this._submit(profiles, start, end)
+      await this._submit(encodedProfiles, start, end, snapshotKind)
       this._logger.debug('Submitted profiles')
     } catch (err) {
       this._logger.error(err)
-      this.stop()
+      this._stop()
     }
   }
 
-  _submit (profiles, start, end) {
+  _submit (profiles, start, end, snapshotKind) {
     if (!Object.keys(profiles).length) {
       return Promise.reject(new Error('No profiles to submit'))
     }
     const { tags } = this._config
     const tasks = []
 
+    tags.snapshot = snapshotKind
     for (const exporter of this._config.exporters) {
       const task = exporter.export({ profiles, start, end, tags })
         .catch(err => this._logger.error(err))
@@ -150,10 +180,10 @@ class ServerlessProfiler extends Profiler {
     this._flushAfterIntervals = this._config.flushInterval / 1000
   }
 
-  async _collect () {
+  async _collect (snapshotKind) {
     if (this._profiledIntervals >= this._flushAfterIntervals) {
       this._profiledIntervals = 0
-      await super._collect()
+      await super._collect(snapshotKind)
     } else {
       this._profiledIntervals += 1
       this._capture(this._timeoutInterval)

package/packages/dd-trace/src/profiling/profilers/space.js

@@ -1,17 +1,37 @@
 'use strict'
 
+const { oomExportStrategies } = require('../constants')
+
+function strategiesToCallbackMode (strategies, callbackMode) {
+  const hasInterrupt = strategies.includes(oomExportStrategies.INTERRUPT_CALLBACK) ? callbackMode.Interrupt : 0
+  const hasCallback = strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0
+  return hasInterrupt | hasCallback
+}
+
 class NativeSpaceProfiler {
   constructor (options = {}) {
     this.type = 'space'
     this._samplingInterval = options.samplingInterval || 512 * 1024
     this._stackDepth = options.stackDepth || 64
     this._pprof = undefined
+    this._oomMonitoring = options.oomMonitoring || {}
   }
 
-  start ({ mapper } = {}) {
+  start ({ mapper, nearOOMCallback } = {}) {
     this._mapper = mapper
     this._pprof = require('@datadog/pprof')
     this._pprof.heap.start(this._samplingInterval, this._stackDepth)
+    if (this._oomMonitoring.enabled) {
+      const strategies = this._oomMonitoring.exportStrategies
+      this._pprof.heap.monitorOutOfMemory(
+        this._oomMonitoring.heapLimitExtensionSize,
+        this._oomMonitoring.maxHeapExtensionCount,
+        strategies.includes(oomExportStrategies.LOGS),
+        strategies.includes(oomExportStrategies.PROCESS) ? this._oomMonitoring.exportCommand : [],
+        (profile) => nearOOMCallback(this.type, this._pprof.encodeSync(profile)),
+        strategiesToCallbackMode(strategies, this._pprof.heap.CallbackMode)
+      )
+    }
   }
 
   profile () {

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -35,6 +35,7 @@ class NativeWallProfiler {
   stop () {
     if (!this._stop) return
     this._stop()
+    this._stop = undefined
   }
 
   _record () {

package/packages/dd-trace/src/proxy.js

@@ -51,7 +51,7 @@ class Tracer extends NoopProxy {
         }
 
         this._tracer = new DatadogTracer(config)
-        this.appsec = new AppsecSdk(this._tracer)
+        this.appsec = new AppsecSdk(this._tracer, config)
 
         if (config.iast.enabled) {
           require('./appsec/iast').enable(config, this._tracer)

package/packages/dd-trace/src/span_processor.js

@@ -17,7 +17,7 @@ class SpanProcessor {
     this._killAll = false
 
     this._stats = new SpanStatsProcessor(config)
-    this._spanSampler = new SpanSampler(config)
+    this._spanSampler = new SpanSampler(config.sampler)
   }
 
   process (span) {

package/packages/dd-trace/src/span_sampler.js

@@ -2,76 +2,93 @@
 const { globMatch } = require('../src/util')
 const { USER_KEEP, AUTO_KEEP } = require('../../../ext').priority
 const RateLimiter = require('./rate_limiter')
+const Sampler = require('./sampler')
 
-class SpanSampler {
-  constructor ({ spanSamplingRules = [] }) {
-    this._rules = spanSamplingRules
-    this._limiters = {}
-  }
+class SpanSamplingRule {
+  constructor ({ service, name, sampleRate = 1.0, maxPerSecond } = {}) {
+    this.service = service
+    this.name = name
 
-  sample (spanContext) {
-    const decision = spanContext._sampling.priority
-    if (decision === USER_KEEP || decision === AUTO_KEEP) return
+    this._sampler = new Sampler(sampleRate)
+    this._limiter = undefined
 
-    const { started } = spanContext._trace
-    for (const span of started) {
-      const service = span.tracer()._service
-      const name = span._name
-      const rule = findRule(this._rules, service, name)
-      if (!rule) continue
-
-      const sampleRate = getSampleRate(rule.sampleRate)
-      const maxPerSecond = getMaxPerSecond(rule.maxPerSecond)
-      const sampled = sample(sampleRate)
-      if (!sampled) continue
-
-      const key = `${service}:${name}`
-      const limiter = getLimiter(this._limiters, key, maxPerSecond)
-      if (limiter.isAllowed()) {
-        span.context()._sampling.spanSampling = {
-          sampleRate,
-          maxPerSecond
-        }
-      }
+    if (Number.isFinite(maxPerSecond)) {
+      this._limiter = new RateLimiter(maxPerSecond)
     }
   }
-}
 
-function findRule (rules, service, name) {
-  for (const rule of rules) {
-    const servicePattern = getService(rule.service)
-    const namePattern = getName(rule.name)
-    if (globMatch(servicePattern, service) && globMatch(namePattern, name)) {
-      return rule
+  get sampleRate () {
+    return this._sampler.rate()
+  }
+
+  get maxPerSecond () {
+    return this._limiter && this._limiter._rateLimit
+  }
+
+  static from (config) {
+    return new SpanSamplingRule(config)
+  }
+
+  match (service, name) {
+    if (this.service && !globMatch(this.service, service)) {
+      return false
     }
+
+    if (this.name && !globMatch(this.name, name)) {
+      return false
+    }
+
+    return true
   }
-}
 
-function getLimiter (list, key, maxPerSecond) {
-  if (typeof list[key] === 'undefined') {
-    list[key] = new RateLimiter(maxPerSecond)
+  sample () {
+    if (!this._sampler.isSampled()) {
+      return false
+    }
+
+    if (this._limiter) {
+      return this._limiter.isAllowed()
+    }
+
+    return true
   }
-  return list[key]
 }
 
-function sample (sampleRate) {
-  return Math.random() < sampleRate
-}
+class SpanSampler {
+  constructor ({ spanSamplingRules = [] } = {}) {
+    this._rules = spanSamplingRules.map(SpanSamplingRule.from)
+  }
 
-function getService (service) {
-  return service || '*'
-}
+  findRule (service, name) {
+    for (const rule of this._rules) {
+      if (rule.match(service, name)) {
+        return rule
+      }
+    }
+  }
 
-function getName (name) {
-  return name || '*'
-}
+  sample (spanContext) {
+    const decision = spanContext._sampling.priority
+    if (decision === USER_KEEP || decision === AUTO_KEEP) return
 
-function getSampleRate (sampleRate) {
-  return sampleRate || 1.0
-}
+    const { started } = spanContext._trace
+    for (const span of started) {
+      const context = span.context()
+      const tags = context._tags || {}
+      const name = context._name
+      const service = tags.service ||
+        tags['service.name'] ||
+        span.tracer()._service
 
-function getMaxPerSecond (maxPerSecond) {
-  return maxPerSecond || Infinity
+      const rule = this.findRule(service, name)
+      if (rule && rule.sample()) {
+        span.context()._sampling.spanSampling = {
+          sampleRate: rule.sampleRate,
+          maxPerSecond: rule.maxPerSecond
+        }
+      }
+    }
+  }
 }
 
 module.exports = SpanSampler

package/packages/dd-trace/src/startup-log.js

@@ -1,14 +1,11 @@
 'use strict'
 
-const mainLogger = require('./log')
+const { info, warn } = require('./log/writer')
 
 const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
 
-const logger = Object.create(mainLogger)
-logger.toggle(true)
-
 let config
 let pluginManager
 let samplingRules = []
@@ -89,9 +86,9 @@ function startupLog ({ agentError } = {}) {
   // out.service_mapping
   // out.service_mapping_error
 
-  logger.info('DATADOG TRACER CONFIGURATION - ' + out)
+  info('DATADOG TRACER CONFIGURATION - ' + out)
   if (agentError) {
-    logger.warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+    warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
   }
 
   config = undefined