dd-trace 3.55.0 → 3.57.0

package/packages/dd-trace/src/profiling/config.js

@@ -21,6 +21,7 @@ class Config {
       DD_AGENT_HOST,
       DD_ENV,
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
+      DD_PROFILING_CPU_ENABLED,
       DD_PROFILING_DEBUG_SOURCE_MAPS,
       DD_PROFILING_ENABLED,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
@@ -165,7 +166,7 @@ class Config {
 
     this.timelineEnabled = isTrue(coalesce(options.timelineEnabled,
       DD_PROFILING_TIMELINE_ENABLED,
-      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, false))
+      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, samplingContextsAvailable))
     logExperimentalVarDeprecation('TIMELINE_ENABLED')
     checkOptionWithSamplingContextAllowed(this.timelineEnabled, 'Timeline view')
 
@@ -176,7 +177,9 @@ class Config {
     checkOptionWithSamplingContextAllowed(this.codeHotspotsEnabled, 'Code hotspots')
 
     this.cpuProfilingEnabled = isTrue(coalesce(options.cpuProfilingEnabled,
-      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, false))
+      DD_PROFILING_CPU_ENABLED,
+      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, samplingContextsAvailable))
+    logExperimentalVarDeprecation('CPU_ENABLED')
     checkOptionWithSamplingContextAllowed(this.cpuProfilingEnabled, 'CPU profiling')
 
     this.profilers = ensureProfilers(profilers, this)
@@ -288,8 +291,9 @@ function ensureProfilers (profilers, options) {
     }
   }
 
-  // Events profiler is a profiler for timeline events
-  if (options.timelineEnabled) {
+  // Events profiler is a profiler that produces timeline events. It is only
+  // added if timeline is enabled and there's a wall profiler.
+  if (options.timelineEnabled && profilers.some(p => p instanceof WallProfiler)) {
     profilers.push(new EventsProfiler(options))
   }
 

package/packages/dd-trace/src/profiling/profiler.js

@@ -4,6 +4,9 @@ const { EventEmitter } = require('events')
 const { Config } = require('./config')
 const { snapshotKinds } = require('./constants')
 const { threadNamePrefix } = require('./profilers/shared')
+const dc = require('dc-polyfill')
+
+const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
 
 function maybeSourceMap (sourceMap, SourceMapper, debug) {
   if (!sourceMap) return
@@ -161,6 +164,7 @@ class Profiler extends EventEmitter {
         this._capture(this._timeoutInterval, endDate)
       }
       await this._submit(encodedProfiles, startDate, endDate, snapshotKind)
+      profileSubmittedChannel.publish()
       this._logger.debug('Submitted profiles')
     } catch (err) {
       this._logger.error(err)

package/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const coalesce = require('koalas')
+const profileSubmittedChannel = dc.channel('datadog:profiling:mock-profile-submitted')
+const { DD_PROFILING_UPLOAD_PERIOD } = process.env
+
+let timerId
+
+module.exports = {
+  start: config => {
+    // Copied from packages/dd-trace/src/profiler.js
+    const flushInterval = coalesce(config.interval, Number(DD_PROFILING_UPLOAD_PERIOD) * 1000, 65 * 1000)
+
+    function scheduleProfileSubmit () {
+      timerId = setTimeout(emitProfileSubmit, flushInterval)
+    }
+
+    function emitProfileSubmit () {
+      profileSubmittedChannel.publish()
+      scheduleProfileSubmit()
+    }
+
+    scheduleProfileSubmit()
+  },
+
+  stop: () => {
+    if (timerId !== undefined) {
+      clearTimeout(timerId)
+      timerId = undefined
+    }
+  }
+}

package/packages/dd-trace/src/profiling/ssi-telemetry.js

@@ -0,0 +1,167 @@
+'use strict'
+
+const telemetryMetrics = require('../telemetry/metrics')
+const profilersNamespace = telemetryMetrics.manager.namespace('profilers')
+const performance = require('perf_hooks').performance
+const dc = require('dc-polyfill')
+const { isTrue, isFalse } = require('../util')
+
+// If the process lived for less than 30 seconds, it's considered short-lived
+const DEFAULT_SHORT_LIVED_THRESHOLD = 30000
+
+const EnablementChoice = {
+  MANUALLY_ENABLED: Symbol('SSITelemetry.EnablementChoice.MANUALLY_ENABLED'),
+  SSI_ENABLED: Symbol('SSITelemetry.EnablementChoice.SSI_ENABLED'),
+  SSI_NOT_ENABLED: Symbol('SSITelemetry.EnablementChoice.SSI_NOT_ENABLED'),
+  DISABLED: Symbol('SSITelemetry.EnablementChoice.MANUALLY_DISABLED')
+}
+Object.freeze(EnablementChoice)
+
+function getEnablementChoiceFromEnv () {
+  const { DD_PROFILING_ENABLED, DD_INJECTION_ENABLED } = process.env
+  if (DD_INJECTION_ENABLED === undefined || isFalse(DD_PROFILING_ENABLED)) {
+    return EnablementChoice.DISABLED
+  } else if (DD_INJECTION_ENABLED.split(',').includes('profiling')) {
+    return EnablementChoice.SSI_ENABLED
+  } else if (isTrue(DD_PROFILING_ENABLED)) {
+    return EnablementChoice.MANUALLY_ENABLED
+  } else {
+    return EnablementChoice.SSI_NOT_ENABLED
+  }
+}
+
+function enablementChoiceToTagValue (enablementChoice) {
+  switch (enablementChoice) {
+    case EnablementChoice.MANUALLY_ENABLED:
+      return 'manually_enabled'
+    case EnablementChoice.SSI_ENABLED:
+      return 'ssi_enabled'
+    case EnablementChoice.SSI_NOT_ENABLED:
+      return 'not_enabled'
+    case EnablementChoice.MANUALLY_DISABLED:
+      // Can't emit this one as a tag
+      throw new Error('Invalid enablement choice')
+  }
+}
+
+/**
+ * This class emits telemetry metrics about the profiler behavior under SSI. It will only emit metrics
+ * when the application closes, and will emit the following metrics:
+ * - `number_of_profiles`: The number of profiles that were submitted
+ * - `number_of_runtime_id`: The number of runtime IDs in the app (always 1 for Node.js)
+ * It will also add tags describing the state of heuristics triggers, the enablement choice, and whether
+ * actual profiles were sent (as opposed to mock profiles). There is a mock profiler that is activated
+ * when the profiler is not enabled, and it will emit mock profile submission events at the same cadence
+ * the profiler would, providing insight into how many profiles would've been emitted if SSI enabled
+ * profiling. Note that telemetry is per tracer instance, and each worker thread will have its own instance.
+ */
+class SSITelemetry {
+  constructor ({
+    enablementChoice = getEnablementChoiceFromEnv(),
+    shortLivedThreshold = DEFAULT_SHORT_LIVED_THRESHOLD
+  } = {}) {
+    if (!Object.values(EnablementChoice).includes(enablementChoice)) {
+      throw new Error('Invalid enablement choice')
+    }
+    if (typeof shortLivedThreshold !== 'number' || shortLivedThreshold <= 0) {
+      throw new Error('Short-lived threshold must be a positive number')
+    }
+    this.enablementChoice = enablementChoice
+    this.shortLivedThreshold = shortLivedThreshold
+
+    this.hasSentProfiles = false
+    this.noSpan = true
+  }
+
+  enabled () {
+    return this.enablementChoice !== EnablementChoice.DISABLED
+  }
+
+  start () {
+    if (this.enabled()) {
+      // Used to determine short-livedness of the process. We could use the process start time as the
+      // reference point, but the tracer initialization point is more relevant, as we couldn't be
+      // collecting profiles earlier anyway. The difference is not particularly significant if the
+      // tracer is initialized early in the process lifetime.
+      this.startTime = performance.now()
+
+      this._onSpanCreated = this._onSpanCreated.bind(this)
+      this._onProfileSubmitted = this._onProfileSubmitted.bind(this)
+      this._onMockProfileSubmitted = this._onMockProfileSubmitted.bind(this)
+      this._onAppClosing = this._onAppClosing.bind(this)
+
+      dc.subscribe('dd-trace:span:start', this._onSpanCreated)
+      dc.subscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted)
+      dc.subscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted)
+      dc.subscribe('datadog:telemetry:app-closing', this._onAppClosing)
+    }
+  }
+
+  _onSpanCreated () {
+    this.noSpan = false
+    dc.unsubscribe('dd-trace:span:start', this._onSpanCreated)
+  }
+
+  _onProfileSubmitted () {
+    this.hasSentProfiles = true
+    this._incProfileCount()
+  }
+
+  _onMockProfileSubmitted () {
+    this._incProfileCount()
+  }
+
+  _incProfileCount () {
+    this._ensureProfileMetrics()
+    this._profileCount.inc()
+  }
+
+  _ensureProfileMetrics () {
+    const decision = []
+    if (this.noSpan) {
+      decision.push('no_span')
+    }
+    if (performance.now() - this.startTime < this.shortLivedThreshold) {
+      decision.push('short_lived')
+    }
+    if (decision.length === 0) {
+      decision.push('triggered')
+    }
+
+    const tags = [
+      'installation:ssi',
+      `enablement_choice:${enablementChoiceToTagValue(this.enablementChoice)}`,
+      `has_sent_profiles:${this.hasSentProfiles}`,
+      `heuristic_hypothetical_decision:${decision.join('_')}`
+    ]
+
+    this._profileCount = profilersNamespace.count('ssi_heuristic.number_of_profiles', tags)
+    this._runtimeIdCount = profilersNamespace.count('ssi_heuristic.number_of_runtime_id', tags)
+
+    if (!this._emittedRuntimeId && decision[0] === 'triggered') {
+      // Tags won't change anymore, so we can emit the runtime ID metric now
+      this._emittedRuntimeId = true
+      this._runtimeIdCount.inc()
+    }
+  }
+
+  _onAppClosing () {
+    this._ensureProfileMetrics()
+    // Last ditch effort to emit a runtime ID count metric
+    if (!this._emittedRuntimeId) {
+      this._emittedRuntimeId = true
+      this._runtimeIdCount.inc()
+    }
+    // So we have the metrics in the final state
+    this._profileCount.inc(0)
+
+    dc.unsubscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted)
+    dc.unsubscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted)
+    dc.unsubscribe('datadog:telemetry:app-closing', this._onAppClosing)
+    if (this.noSpan) {
+      dc.unsubscribe('dd-trace:span:start', this._onSpanCreated)
+    }
+  }
+}
+
+module.exports = { SSITelemetry, EnablementChoice }

package/packages/dd-trace/src/proxy.js

@@ -11,7 +11,24 @@ const PluginManager = require('./plugin_manager')
 const remoteConfig = require('./appsec/remote_config')
 const AppsecSdk = require('./appsec/sdk')
 const dogstatsd = require('./dogstatsd')
+const NoopDogStatsDClient = require('./noop/dogstatsd')
 const spanleak = require('./spanleak')
+const { SSITelemetry } = require('./profiling/ssi-telemetry')
+
+class LazyModule {
+  constructor (provider) {
+    this.provider = provider
+  }
+
+  enable (...args) {
+    this.module = this.provider()
+    this.module.enable(...args)
+  }
+
+  disable () {
+    this.module?.disable()
+  }
+}
 
 class Tracer extends NoopProxy {
   constructor () {
@@ -20,8 +37,14 @@ class Tracer extends NoopProxy {
     this._initialized = false
     this._nomenclature = nomenclature
     this._pluginManager = new PluginManager(this)
-    this.dogstatsd = new dogstatsd.NoopDogStatsDClient()
+    this.dogstatsd = new NoopDogStatsDClient()
     this._tracingInitialized = false
+
+    // these requires must work with esm bundler
+    this._modules = {
+      appsec: new LazyModule(() => require('./appsec')),
+      iast: new LazyModule(() => require('./appsec/iast'))
+    }
   }
 
   init (options) {
@@ -56,7 +79,7 @@ class Tracer extends NoopProxy {
       }
 
       if (config.remoteConfig.enabled && !config.isCiVisibility) {
-        const rc = remoteConfig.enable(config)
+        const rc = remoteConfig.enable(config, this._modules.appsec)
 
         rc.on('APM_TRACING', (action, conf) => {
           if (action === 'unapply') {
@@ -72,6 +95,8 @@ class Tracer extends NoopProxy {
         require('./serverless').maybeStartServerlessMiniAgent(config)
       }
 
+      const ssiTelemetry = new SSITelemetry()
+      ssiTelemetry.start()
       if (config.profiling.enabled) {
         // do not stop tracer initialization if the profiler fails to be imported
         try {
@@ -80,6 +105,8 @@ class Tracer extends NoopProxy {
         } catch (e) {
           log.error(e)
         }
+      } else if (ssiTelemetry.enabled()) {
+        require('./profiling/ssi-telemetry-mock-profiler').start(config)
       }
       if (!this._profilerStarted) {
         this._profilerStarted = Promise.resolve(false)
@@ -107,9 +134,8 @@ class Tracer extends NoopProxy {
 
   _enableOrDisableTracing (config) {
     if (config.tracing !== false) {
-      // dirty require for now so zero appsec code is executed unless explicitly enabled
       if (config.appsec.enabled) {
-        require('./appsec').enable(config)
+        this._modules.appsec.enable(config)
       }
       if (!this._tracingInitialized) {
         this._tracer = new DatadogTracer(config)
@@ -117,11 +143,11 @@ class Tracer extends NoopProxy {
         this._tracingInitialized = true
       }
       if (config.iast.enabled) {
-        require('./appsec/iast').enable(config, this._tracer)
+        this._modules.iast.enable(config, this._tracer)
       }
     } else if (this._tracingInitialized) {
-      require('./appsec').disable()
-      require('./appsec/iast').disable()
+      this._modules.appsec.disable()
+      this._modules.iast.disable()
     }
 
     if (this._tracingInitialized) {

package/packages/dd-trace/src/tagger.js

@@ -2,7 +2,13 @@
 
 const log = require('./log')
 
-function add (carrier, keyValuePairs) {
+const otelTagMap = {
+  'deployment.environment': 'env',
+  'service.name': 'service',
+  'service.version': 'version'
+}
+
+function add (carrier, keyValuePairs, parseOtelTags = false) {
   if (!carrier || !keyValuePairs) return
 
   if (Array.isArray(keyValuePairs)) {
@@ -13,12 +19,16 @@ function add (carrier, keyValuePairs) {
     if (typeof keyValuePairs === 'string') {
       const segments = keyValuePairs.split(',')
       for (const segment of segments) {
-        const separatorIndex = segment.indexOf(':')
+        const separatorIndex = parseOtelTags ? segment.indexOf('=') : segment.indexOf(':')
         if (separatorIndex === -1) continue
 
-        const key = segment.slice(0, separatorIndex)
+        let key = segment.slice(0, separatorIndex)
         const value = segment.slice(separatorIndex + 1)
 
+        if (parseOtelTags && key in otelTagMap) {
+          key = otelTagMap[key]
+        }
+
         carrier[key.trim()] = value.trim()
       }
     } else {

package/packages/dd-trace/src/telemetry/index.js

@@ -10,6 +10,7 @@ const logs = require('./logs')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
+const telemetryAppClosingChannel = dc.channel('datadog:telemetry:app-closing')
 
 let config
 let pluginManager
@@ -129,12 +130,12 @@ function appClosing () {
   if (!config?.telemetry?.enabled) {
     return
   }
+  // Give chance to listeners to update metrics before shutting down.
+  telemetryAppClosingChannel.publish()
   const { reqType, payload } = createPayload('app-closing')
   sendData(config, application, host, reqType, payload)
-  // we flush before shutting down. Only in CI Visibility
-  if (config.isCiVisibility) {
-    metricsManager.send(config, application, host)
-  }
+  // We flush before shutting down.
+  metricsManager.send(config, application, host)
 }
 
 function onBeforeExit () {

package/packages/dd-trace/src/telemetry/metrics.js

@@ -75,8 +75,8 @@ class CountMetric extends Metric {
     return this.track(value)
   }
 
-  dec (value = -1) {
-    return this.track(value)
+  dec (value = 1) {
+    return this.track(-value)
   }
 
   track (value = 1) {