dd-trace 4.33.0 → 4.35.0

package/packages/dd-trace/src/plugins/util/git.js

@@ -28,7 +28,7 @@ const {
 const { filterSensitiveInfoFromRepository } = require('./url')
 const { storage } = require('../../../../datadog-core')
 
-const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
+const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB
 
 function sanitizedExec (
   cmd,
@@ -53,11 +53,15 @@ function sanitizedExec (
       distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime)
     }
     return result
-  } catch (e) {
+  } catch (err) {
     if (errorMetric) {
-      incrementCountMetric(errorMetric.name, { ...errorMetric.tags, exitCode: e.status })
+      incrementCountMetric(errorMetric.name, {
+        ...errorMetric.tags,
+        errorType: err.code,
+        exitCode: err.status || err.errno
+      })
     }
-    log.error(e)
+    log.error(err)
     return ''
   } finally {
     storage.enterWith(store)
@@ -129,7 +133,10 @@ function unshallowRepository () {
   } catch (err) {
     // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail.
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+    )
     const upstreamRemote = sanitizedExec('git', ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'])
     try {
       cp.execFileSync('git', [
@@ -139,7 +146,10 @@ function unshallowRepository () {
     } catch (err) {
       // If the CI is working on a detached HEAD or branch tracking hasn’t been set up, the above command will fail.
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'unshallow', exitCode: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+      )
       // We use sanitizedExec here because if this last option fails, we'll give up.
       sanitizedExec(
         'git',
@@ -175,13 +185,16 @@ function getLatestCommits () {
     return result
   } catch (err) {
     log.error(`Get latest commits failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_local_commits', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_local_commits', errorType: err.status }
+    )
     return []
   }
 }
 
 function getCommitsRevList (commitsToExclude, commitsToInclude) {
-  let result = []
+  let result = null
 
   const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
 
@@ -205,7 +218,10 @@ function getCommitsRevList (commitsToExclude, commitsToInclude) {
       .filter(commit => commit)
   } catch (err) {
     log.error(`Get commits to upload failed: ${err.message}`)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'get_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_objects', errorType: err.code, exitCode: err.status || err.errno } // err.status might be null
+    )
   }
   distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime)
   return result
@@ -245,7 +261,10 @@ function generatePackFilesForCommits (commitsToUpload) {
     result = execGitPackObjects(temporaryPath)
   } catch (err) {
     log.error(err)
-    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+    )
     /**
      * The generation of pack files in the temporary folder (from `os.tmpdir()`)
      * sometimes fails in certain CI setups with the error message
@@ -262,7 +281,10 @@ function generatePackFilesForCommits (commitsToUpload) {
       result = execGitPackObjects(cwdPath)
     } catch (err) {
       log.error(err)
-      incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'pack_objects', errorType: err.status })
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+      )
     }
   }
   distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime)

package/packages/dd-trace/src/plugins/util/test.js

@@ -53,7 +53,8 @@ const TEST_CONFIGURATION_BROWSER_NAME = 'test.configuration.browser_name'
 // Early flake detection
 const TEST_IS_NEW = 'test.is_new'
 const TEST_IS_RETRY = 'test.is_retry'
-const TEST_EARLY_FLAKE_IS_ENABLED = 'test.early_flake.is_enabled'
+const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled'
+const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason'
 
 const CI_APP_ORIGIN = 'ciapp-test'
 
@@ -71,6 +72,12 @@ const ITR_CORRELATION_ID = 'itr_correlation_id'
 
 const TEST_CODE_COVERAGE_LINES_PCT = 'test.code_coverage.lines_pct'
 
+// selenium tags
+const TEST_BROWSER_DRIVER = 'test.browser.driver'
+const TEST_BROWSER_DRIVER_VERSION = 'test.browser.driver_version'
+const TEST_BROWSER_NAME = 'test.browser.name'
+const TEST_BROWSER_VERSION = 'test.browser.version'
+
 // jest worker variables
 const JEST_WORKER_TRACE_PAYLOAD_CODE = 60
 const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61
@@ -102,7 +109,8 @@ module.exports = {
   TEST_CONFIGURATION_BROWSER_NAME,
   TEST_IS_NEW,
   TEST_IS_RETRY,
-  TEST_EARLY_FLAKE_IS_ENABLED,
+  TEST_EARLY_FLAKE_ENABLED,
+  TEST_EARLY_FLAKE_ABORT_REASON,
   getTestEnvironmentMetadata,
   getTestParametersString,
   finishAllTraceSpans,
@@ -141,7 +149,12 @@ module.exports = {
   EFD_STRING,
   EFD_TEST_NAME_REGEX,
   removeEfdStringFromTestName,
-  addEfdStringToTestName
+  addEfdStringToTestName,
+  getIsFaultyEarlyFlakeDetection,
+  TEST_BROWSER_DRIVER,
+  TEST_BROWSER_DRIVER_VERSION,
+  TEST_BROWSER_NAME,
+  TEST_BROWSER_VERSION
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -571,3 +584,21 @@ function addEfdStringToTestName (testName, numAttempt) {
 function removeEfdStringFromTestName (testName) {
   return testName.replace(EFD_TEST_NAME_REGEX, '')
 }
+
+function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faultyThresholdPercentage) {
+  let newSuites = 0
+  for (const suite of projectSuites) {
+    if (!testsBySuiteName[suite]) {
+      newSuites++
+    }
+  }
+  const newSuitesPercentage = (newSuites / projectSuites.length) * 100
+
+  // The faulty threshold represents a percentage, but we also want to consider
+  // smaller projects, where big variations in the % are more likely.
+  // This is why we also check the absolute number of new suites.
+  return (
+    newSuites > faultyThresholdPercentage &&
+    newSuitesPercentage > faultyThresholdPercentage
+  )
+}

package/packages/dd-trace/src/profiling/config.js

@@ -21,6 +21,7 @@ class Config {
       DD_AGENT_HOST,
       DD_ENV,
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
+      DD_PROFILING_CPU_ENABLED,
       DD_PROFILING_DEBUG_SOURCE_MAPS,
       DD_PROFILING_ENABLED,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
@@ -165,7 +166,7 @@ class Config {
 
     this.timelineEnabled = isTrue(coalesce(options.timelineEnabled,
       DD_PROFILING_TIMELINE_ENABLED,
-      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, false))
+      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, samplingContextsAvailable))
     logExperimentalVarDeprecation('TIMELINE_ENABLED')
     checkOptionWithSamplingContextAllowed(this.timelineEnabled, 'Timeline view')
 
@@ -176,7 +177,9 @@ class Config {
     checkOptionWithSamplingContextAllowed(this.codeHotspotsEnabled, 'Code hotspots')
 
     this.cpuProfilingEnabled = isTrue(coalesce(options.cpuProfilingEnabled,
-      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, false))
+      DD_PROFILING_CPU_ENABLED,
+      DD_PROFILING_EXPERIMENTAL_CPU_ENABLED, samplingContextsAvailable))
+    logExperimentalVarDeprecation('CPU_ENABLED')
     checkOptionWithSamplingContextAllowed(this.cpuProfilingEnabled, 'CPU profiling')
 
     this.profilers = ensureProfilers(profilers, this)
@@ -288,8 +291,9 @@ function ensureProfilers (profilers, options) {
     }
   }
 
-  // Events profiler is a profiler for timeline events
-  if (options.timelineEnabled) {
+  // Events profiler is a profiler that produces timeline events. It is only
+  // added if timeline is enabled and there's a wall profiler.
+  if (options.timelineEnabled && profilers.some(p => p instanceof WallProfiler)) {
     profilers.push(new EventsProfiler(options))
   }
 

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -10,6 +10,7 @@ const FormData = require('../../exporters/common/form-data')
 const { storage } = require('../../../../datadog-core')
 const version = require('../../../../../package.json').version
 const os = require('os')
+const { urlToHttpOptions } = require('url')
 const perf = require('perf_hooks').performance
 
 const containerId = docker.id()
@@ -177,9 +178,10 @@ class AgentExporter {
         if (this._url.protocol === 'unix:') {
           options.socketPath = this._url.pathname
         } else {
-          options.protocol = this._url.protocol
-          options.hostname = this._url.hostname
-          options.port = this._url.port
+          const httpOptions = urlToHttpOptions(this._url)
+          options.protocol = httpOptions.protocol
+          options.hostname = httpOptions.hostname
+          options.port = httpOptions.port
         }
 
         this._logger.debug(() => {

package/packages/dd-trace/src/profiling/profiler.js

@@ -4,6 +4,9 @@ const { EventEmitter } = require('events')
 const { Config } = require('./config')
 const { snapshotKinds } = require('./constants')
 const { threadNamePrefix } = require('./profilers/shared')
+const dc = require('dc-polyfill')
+
+const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
 
 function maybeSourceMap (sourceMap, SourceMapper, debug) {
   if (!sourceMap) return
@@ -161,6 +164,7 @@ class Profiler extends EventEmitter {
         this._capture(this._timeoutInterval, endDate)
       }
       await this._submit(encodedProfiles, startDate, endDate, snapshotKind)
+      profileSubmittedChannel.publish()
       this._logger.debug('Submitted profiles')
     } catch (err) {
       this._logger.error(err)

package/packages/dd-trace/src/profiling/ssi-telemetry-mock-profiler.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const coalesce = require('koalas')
+const profileSubmittedChannel = dc.channel('datadog:profiling:mock-profile-submitted')
+const { DD_PROFILING_UPLOAD_PERIOD } = process.env
+
+let timerId
+
+module.exports = {
+  start: config => {
+    // Copied from packages/dd-trace/src/profiler.js
+    const flushInterval = coalesce(config.interval, Number(DD_PROFILING_UPLOAD_PERIOD) * 1000, 65 * 1000)
+
+    function scheduleProfileSubmit () {
+      timerId = setTimeout(emitProfileSubmit, flushInterval)
+    }
+
+    function emitProfileSubmit () {
+      profileSubmittedChannel.publish()
+      scheduleProfileSubmit()
+    }
+
+    scheduleProfileSubmit()
+  },
+
+  stop: () => {
+    if (timerId !== undefined) {
+      clearTimeout(timerId)
+      timerId = undefined
+    }
+  }
+}

package/packages/dd-trace/src/profiling/ssi-telemetry.js

@@ -0,0 +1,167 @@
+'use strict'
+
+const telemetryMetrics = require('../telemetry/metrics')
+const profilersNamespace = telemetryMetrics.manager.namespace('profilers')
+const performance = require('perf_hooks').performance
+const dc = require('dc-polyfill')
+const { isTrue, isFalse } = require('../util')
+
+// If the process lived for less than 30 seconds, it's considered short-lived
+const DEFAULT_SHORT_LIVED_THRESHOLD = 30000
+
+const EnablementChoice = {
+  MANUALLY_ENABLED: Symbol('SSITelemetry.EnablementChoice.MANUALLY_ENABLED'),
+  SSI_ENABLED: Symbol('SSITelemetry.EnablementChoice.SSI_ENABLED'),
+  SSI_NOT_ENABLED: Symbol('SSITelemetry.EnablementChoice.SSI_NOT_ENABLED'),
+  DISABLED: Symbol('SSITelemetry.EnablementChoice.MANUALLY_DISABLED')
+}
+Object.freeze(EnablementChoice)
+
+function getEnablementChoiceFromEnv () {
+  const { DD_PROFILING_ENABLED, DD_INJECTION_ENABLED } = process.env
+  if (DD_INJECTION_ENABLED === undefined || isFalse(DD_PROFILING_ENABLED)) {
+    return EnablementChoice.DISABLED
+  } else if (DD_INJECTION_ENABLED.split(',').includes('profiling')) {
+    return EnablementChoice.SSI_ENABLED
+  } else if (isTrue(DD_PROFILING_ENABLED)) {
+    return EnablementChoice.MANUALLY_ENABLED
+  } else {
+    return EnablementChoice.SSI_NOT_ENABLED
+  }
+}
+
+function enablementChoiceToTagValue (enablementChoice) {
+  switch (enablementChoice) {
+    case EnablementChoice.MANUALLY_ENABLED:
+      return 'manually_enabled'
+    case EnablementChoice.SSI_ENABLED:
+      return 'ssi_enabled'
+    case EnablementChoice.SSI_NOT_ENABLED:
+      return 'not_enabled'
+    case EnablementChoice.MANUALLY_DISABLED:
+      // Can't emit this one as a tag
+      throw new Error('Invalid enablement choice')
+  }
+}
+
+/**
+ * This class emits telemetry metrics about the profiler behavior under SSI. It will only emit metrics
+ * when the application closes, and will emit the following metrics:
+ * - `number_of_profiles`: The number of profiles that were submitted
+ * - `number_of_runtime_id`: The number of runtime IDs in the app (always 1 for Node.js)
+ * It will also add tags describing the state of heuristics triggers, the enablement choice, and whether
+ * actual profiles were sent (as opposed to mock profiles). There is a mock profiler that is activated
+ * when the profiler is not enabled, and it will emit mock profile submission events at the same cadence
+ * the profiler would, providing insight into how many profiles would've been emitted if SSI enabled
+ * profiling. Note that telemetry is per tracer instance, and each worker thread will have its own instance.
+ */
+class SSITelemetry {
+  constructor ({
+    enablementChoice = getEnablementChoiceFromEnv(),
+    shortLivedThreshold = DEFAULT_SHORT_LIVED_THRESHOLD
+  } = {}) {
+    if (!Object.values(EnablementChoice).includes(enablementChoice)) {
+      throw new Error('Invalid enablement choice')
+    }
+    if (typeof shortLivedThreshold !== 'number' || shortLivedThreshold <= 0) {
+      throw new Error('Short-lived threshold must be a positive number')
+    }
+    this.enablementChoice = enablementChoice
+    this.shortLivedThreshold = shortLivedThreshold
+
+    this.hasSentProfiles = false
+    this.noSpan = true
+  }
+
+  enabled () {
+    return this.enablementChoice !== EnablementChoice.DISABLED
+  }
+
+  start () {
+    if (this.enabled()) {
+      // Used to determine short-livedness of the process. We could use the process start time as the
+      // reference point, but the tracer initialization point is more relevant, as we couldn't be
+      // collecting profiles earlier anyway. The difference is not particularly significant if the
+      // tracer is initialized early in the process lifetime.
+      this.startTime = performance.now()
+
+      this._onSpanCreated = this._onSpanCreated.bind(this)
+      this._onProfileSubmitted = this._onProfileSubmitted.bind(this)
+      this._onMockProfileSubmitted = this._onMockProfileSubmitted.bind(this)
+      this._onAppClosing = this._onAppClosing.bind(this)
+
+      dc.subscribe('dd-trace:span:start', this._onSpanCreated)
+      dc.subscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted)
+      dc.subscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted)
+      dc.subscribe('datadog:telemetry:app-closing', this._onAppClosing)
+    }
+  }
+
+  _onSpanCreated () {
+    this.noSpan = false
+    dc.unsubscribe('dd-trace:span:start', this._onSpanCreated)
+  }
+
+  _onProfileSubmitted () {
+    this.hasSentProfiles = true
+    this._incProfileCount()
+  }
+
+  _onMockProfileSubmitted () {
+    this._incProfileCount()
+  }
+
+  _incProfileCount () {
+    this._ensureProfileMetrics()
+    this._profileCount.inc()
+  }
+
+  _ensureProfileMetrics () {
+    const decision = []
+    if (this.noSpan) {
+      decision.push('no_span')
+    }
+    if (performance.now() - this.startTime < this.shortLivedThreshold) {
+      decision.push('short_lived')
+    }
+    if (decision.length === 0) {
+      decision.push('triggered')
+    }
+
+    const tags = [
+      'installation:ssi',
+      `enablement_choice:${enablementChoiceToTagValue(this.enablementChoice)}`,
+      `has_sent_profiles:${this.hasSentProfiles}`,
+      `heuristic_hypothetical_decision:${decision.join('_')}`
+    ]
+
+    this._profileCount = profilersNamespace.count('ssi_heuristic.number_of_profiles', tags)
+    this._runtimeIdCount = profilersNamespace.count('ssi_heuristic.number_of_runtime_id', tags)
+
+    if (!this._emittedRuntimeId && decision[0] === 'triggered') {
+      // Tags won't change anymore, so we can emit the runtime ID metric now
+      this._emittedRuntimeId = true
+      this._runtimeIdCount.inc()
+    }
+  }
+
+  _onAppClosing () {
+    this._ensureProfileMetrics()
+    // Last ditch effort to emit a runtime ID count metric
+    if (!this._emittedRuntimeId) {
+      this._emittedRuntimeId = true
+      this._runtimeIdCount.inc()
+    }
+    // So we have the metrics in the final state
+    this._profileCount.inc(0)
+
+    dc.unsubscribe('datadog:profiling:profile-submitted', this._onProfileSubmitted)
+    dc.unsubscribe('datadog:profiling:mock-profile-submitted', this._onMockProfileSubmitted)
+    dc.unsubscribe('datadog:telemetry:app-closing', this._onAppClosing)
+    if (this.noSpan) {
+      dc.unsubscribe('dd-trace:span:start', this._onSpanCreated)
+    }
+  }
+}
+
+module.exports = { SSITelemetry, EnablementChoice }

package/packages/dd-trace/src/proxy.js

@@ -11,7 +11,9 @@ const PluginManager = require('./plugin_manager')
 const remoteConfig = require('./appsec/remote_config')
 const AppsecSdk = require('./appsec/sdk')
 const dogstatsd = require('./dogstatsd')
+const NoopDogStatsDClient = require('./noop/dogstatsd')
 const spanleak = require('./spanleak')
+const { SSITelemetry } = require('./profiling/ssi-telemetry')
 
 class Tracer extends NoopProxy {
   constructor () {
@@ -20,7 +22,7 @@ class Tracer extends NoopProxy {
     this._initialized = false
     this._nomenclature = nomenclature
     this._pluginManager = new PluginManager(this)
-    this.dogstatsd = new dogstatsd.NoopDogStatsDClient()
+    this.dogstatsd = new NoopDogStatsDClient()
     this._tracingInitialized = false
   }
 
@@ -72,6 +74,8 @@ class Tracer extends NoopProxy {
         require('./serverless').maybeStartServerlessMiniAgent(config)
       }
 
+      const ssiTelemetry = new SSITelemetry()
+      ssiTelemetry.start()
       if (config.profiling.enabled) {
         // do not stop tracer initialization if the profiler fails to be imported
         try {
@@ -80,6 +84,8 @@ class Tracer extends NoopProxy {
         } catch (e) {
           log.error(e)
         }
+      } else if (ssiTelemetry.enabled()) {
+        require('./profiling/ssi-telemetry-mock-profiler').start(config)
       }
       if (!this._profilerStarted) {
         this._profilerStarted = Promise.resolve(false)

package/packages/dd-trace/src/tagger.js

@@ -2,7 +2,13 @@
 
 const log = require('./log')
 
-function add (carrier, keyValuePairs) {
+const otelTagMap = {
+  'deployment.environment': 'env',
+  'service.name': 'service',
+  'service.version': 'version'
+}
+
+function add (carrier, keyValuePairs, parseOtelTags = false) {
   if (!carrier || !keyValuePairs) return
 
   if (Array.isArray(keyValuePairs)) {
@@ -13,12 +19,16 @@ function add (carrier, keyValuePairs) {
     if (typeof keyValuePairs === 'string') {
       const segments = keyValuePairs.split(',')
       for (const segment of segments) {
-        const separatorIndex = segment.indexOf(':')
+        const separatorIndex = parseOtelTags ? segment.indexOf('=') : segment.indexOf(':')
         if (separatorIndex === -1) continue
 
-        const key = segment.slice(0, separatorIndex)
+        let key = segment.slice(0, separatorIndex)
         const value = segment.slice(separatorIndex + 1)
 
+        if (parseOtelTags && key in otelTagMap) {
+          key = otelTagMap[key]
+        }
+
         carrier[key.trim()] = value.trim()
       }
     } else {

package/packages/dd-trace/src/telemetry/index.js

@@ -10,6 +10,7 @@ const logs = require('./logs')
 
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
+const telemetryAppClosingChannel = dc.channel('datadog:telemetry:app-closing')
 
 let config
 let pluginManager
@@ -129,12 +130,12 @@ function appClosing () {
   if (!config?.telemetry?.enabled) {
     return
   }
+  // Give chance to listeners to update metrics before shutting down.
+  telemetryAppClosingChannel.publish()
   const { reqType, payload } = createPayload('app-closing')
   sendData(config, application, host, reqType, payload)
-  // we flush before shutting down. Only in CI Visibility
-  if (config.isCiVisibility) {
-    metricsManager.send(config, application, host)
-  }
+  // We flush before shutting down.
+  metricsManager.send(config, application, host)
 }
 
 function onBeforeExit () {

package/packages/dd-trace/src/telemetry/metrics.js

@@ -75,8 +75,8 @@ class CountMetric extends Metric {
     return this.track(value)
   }
 
-  dec (value = -1) {
-    return this.track(value)
+  dec (value = 1) {
+    return this.track(-value)
   }
 
   track (value = 1) {