dd-trace 3.38.1 → 3.40.0

package/packages/dd-trace/src/plugins/util/ci.js

@@ -1,5 +1,3 @@
-const URL = require('url').URL
-
 const {
   GIT_BRANCH,
   GIT_COMMIT_SHA,
@@ -24,6 +22,7 @@ const {
   CI_NODE_LABELS,
   CI_NODE_NAME
 } = require('./tags')
+const { filterSensitiveInfoFromRepository } = require('./url')
 
 // Receives a string with the form 'John Doe <john.doe@gmail.com>'
 // and returns { name: 'John Doe', email: 'john.doe@gmail.com' }
@@ -67,20 +66,6 @@ function normalizeRef (ref) {
   return ref.replace(/origin\/|refs\/heads\/|tags\//gm, '')
 }
 
-function filterSensitiveInfoFromRepository (repositoryUrl) {
-  if (repositoryUrl.startsWith('git@')) {
-    return repositoryUrl
-  }
-
-  try {
-    const { protocol, hostname, pathname } = new URL(repositoryUrl)
-
-    return `${protocol}//${hostname}${pathname}`
-  } catch (e) {
-    return ''
-  }
-}
-
 function resolveTilde (filePath) {
   if (!filePath || typeof filePath !== 'string') {
     return ''
@@ -271,20 +256,22 @@ module.exports = {
       const ref = GITHUB_HEAD_REF || GITHUB_REF || ''
       const refKey = ref.includes('tags/') ? GIT_TAG : GIT_BRANCH
 
+      // Both pipeline URL and job URL include GITHUB_SERVER_URL, which can include user credentials,
+      // so we pass them through `filterSensitiveInfoFromRepository`.
       tags = {
         [CI_PIPELINE_ID]: GITHUB_RUN_ID,
         [CI_PIPELINE_NAME]: GITHUB_WORKFLOW,
         [CI_PIPELINE_NUMBER]: GITHUB_RUN_NUMBER,
-        [CI_PIPELINE_URL]: pipelineURL,
+        [CI_PIPELINE_URL]: filterSensitiveInfoFromRepository(pipelineURL),
         [CI_PROVIDER_NAME]: 'github',
         [GIT_COMMIT_SHA]: GITHUB_SHA,
         [GIT_REPOSITORY_URL]: repositoryURL,
-        [CI_JOB_URL]: jobUrl,
+        [CI_JOB_URL]: filterSensitiveInfoFromRepository(jobUrl),
         [CI_JOB_NAME]: GITHUB_JOB,
         [CI_WORKSPACE_PATH]: GITHUB_WORKSPACE,
         [refKey]: ref,
         [CI_ENV_VARS]: JSON.stringify({
-          GITHUB_SERVER_URL,
+          GITHUB_SERVER_URL: filterSensitiveInfoFromRepository(GITHUB_SERVER_URL),
           GITHUB_REPOSITORY,
           GITHUB_RUN_ID,
           GITHUB_RUN_ATTEMPT

package/packages/dd-trace/src/plugins/util/git.js

@@ -19,6 +19,7 @@ const {
   GIT_COMMIT_AUTHOR_NAME,
   CI_WORKSPACE_PATH
 } = require('./tags')
+const { filterSensitiveInfoFromRepository } = require('./url')
 
 const GIT_REV_LIST_MAX_BUFFER = 8 * 1024 * 1024 // 8MB
 
@@ -214,7 +215,7 @@ function getGitMetadata (ciMetadata) {
 
   return {
     [GIT_REPOSITORY_URL]:
-      repositoryUrl || sanitizedExec('git', ['ls-remote', '--get-url']),
+      filterSensitiveInfoFromRepository(repositoryUrl || sanitizedExec('git', ['ls-remote', '--get-url'])),
     [GIT_COMMIT_MESSAGE]:
       commitMessage || sanitizedExec('git', ['show', '-s', '--format=%s']),
     [GIT_COMMIT_AUTHOR_DATE]: authorDate,

package/packages/dd-trace/src/plugins/util/ip_extractor.js

@@ -48,8 +48,8 @@ function extractIp (config, req) {
 
   let firstPrivateIp
   if (headers) {
-    for (let i = 0; i < ipHeaderList.length; i++) {
-      const firstIp = findFirstIp(headers[ipHeaderList[i]])
+    for (const ipHeaderName of ipHeaderList) {
+      const firstIp = findFirstIp(headers[ipHeaderName])
 
       if (firstIp.public) {
         return firstIp.public
@@ -59,7 +59,7 @@ function extractIp (config, req) {
     }
   }
 
-  return firstPrivateIp || (req.socket && req.socket.remoteAddress)
+  return firstPrivateIp || req.socket?.remoteAddress
 }
 
 function findFirstIp (str) {
@@ -68,8 +68,8 @@ function findFirstIp (str) {
 
   const splitted = str.split(',')
 
-  for (let i = 0; i < splitted.length; i++) {
-    const chunk = splitted[i].trim()
+  for (const part of splitted) {
+    const chunk = part.trim()
 
     // TODO: strip port and interface data ?
 
@@ -90,5 +90,6 @@ function findFirstIp (str) {
 }
 
 module.exports = {
-  extractIp
+  extractIp,
+  ipHeaderList
 }

package/packages/dd-trace/src/plugins/util/test.js

@@ -118,7 +118,8 @@ module.exports = {
   fromCoverageMapToCoverage,
   getTestLineStart,
   getCallSites,
-  removeInvalidMetadata
+  removeInvalidMetadata,
+  parseAnnotations
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -492,3 +493,30 @@ function getCallSites () {
 
   return v8StackTrace
 }
+
+/**
+ * Gets an object of test tags from an Playwright annotations array.
+ * @param {Object[]} annotations - Annotations from a Playwright test.
+ * @param {string} annotations[].type - Type of annotation. A string of the shape DD_TAGS[$tag_name].
+ * @param {string} annotations[].description - Value of the tag.
+ */
+function parseAnnotations (annotations) {
+  return annotations.reduce((tags, annotation) => {
+    if (!annotation?.type) {
+      return tags
+    }
+    const { type, description } = annotation
+    if (type.startsWith('DD_TAGS')) {
+      const regex = /\[(.*?)\]/
+      const match = regex.exec(type)
+      let tagValue = ''
+      if (match) {
+        tagValue = match[1]
+      }
+      if (tagValue) {
+        tags[tagValue] = description
+      }
+    }
+    return tags
+  }, {})
+}

package/packages/dd-trace/src/plugins/util/url.js

@@ -0,0 +1,26 @@
+const { URL } = require('url')
+
+function filterSensitiveInfoFromRepository (repositoryUrl) {
+  if (!repositoryUrl) {
+    return ''
+  }
+  if (repositoryUrl.startsWith('git@')) {
+    return repositoryUrl
+  }
+
+  // Remove the username from ssh URLs
+  if (repositoryUrl.startsWith('ssh://')) {
+    const sshRegex = /^(ssh:\/\/)[^@/]*@/
+    return repositoryUrl.replace(sshRegex, '$1')
+  }
+
+  try {
+    const { protocol, host, pathname } = new URL(repositoryUrl)
+
+    return `${protocol}//${host}${pathname === '/' ? '' : pathname}`
+  } catch (e) {
+    return ''
+  }
+}
+
+module.exports = { filterSensitiveInfoFromRepository }

package/packages/dd-trace/src/plugins/util/user-provided-git.js

@@ -13,7 +13,7 @@ const {
 } = require('./tags')
 
 const { normalizeRef } = require('./ci')
-const { URL } = require('url')
+const { filterSensitiveInfoFromRepository } = require('./url')
 
 function removeEmptyValues (tags) {
   return Object.keys(tags).reduce((filteredTags, tag) => {
@@ -27,19 +27,6 @@ function removeEmptyValues (tags) {
   }, {})
 }
 
-function filterSensitiveInfoFromRepository (repositoryUrl) {
-  try {
-    if (repositoryUrl.startsWith('git@')) {
-      return repositoryUrl
-    }
-    const { protocol, hostname, pathname } = new URL(repositoryUrl)
-
-    return `${protocol}//${hostname}${pathname}`
-  } catch (e) {
-    return repositoryUrl
-  }
-}
-
 // The regex is extracted from
 // https://github.com/jonschlinkert/is-git-url/blob/396965ffabf2f46656c8af4c47bef1d69f09292e/index.js#L9C15-L9C87
 function validateGitRepositoryUrl (repoUrl) {

package/packages/dd-trace/src/profiling/config.js

@@ -9,6 +9,7 @@ const { FileExporter } = require('./exporters/file')
 const { ConsoleLogger } = require('./loggers/console')
 const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
+const EventsProfiler = require('./profilers/events')
 const { oomExportStrategies, snapshotKinds } = require('./constants')
 const { tagger } = require('./tagger')
 const { isFalse, isTrue } = require('../util')
@@ -37,6 +38,7 @@ class Config {
       DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE,
       DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT,
       DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES,
+      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED,
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
       DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED,
@@ -126,26 +128,17 @@ class Config {
 
     const profilers = options.profilers
       ? options.profilers
-      : getProfilers({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED, DD_PROFILING_PROFILERS })
-
-    function getCodeHotspotsOptionsOr (defvalue) {
-      return coalesce(options.codeHotspotsEnabled,
-        DD_PROFILING_CODEHOTSPOTS_ENABLED,
-        DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, defvalue)
-    }
-    this.codeHotspotsEnabled = isTrue(getCodeHotspotsOptionsOr(false))
+      : getProfilers({
+        DD_PROFILING_HEAP_ENABLED,
+        DD_PROFILING_WALLTIME_ENABLED,
+        DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED,
+        DD_PROFILING_PROFILERS
+      })
+
+    this.codeHotspotsEnabled = isTrue(coalesce(options.codeHotspotsEnabled,
+      DD_PROFILING_CODEHOTSPOTS_ENABLED,
+      DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, false))
     logExperimentalVarDeprecation('CODEHOTSPOTS_ENABLED')
-    if (this.endpointCollectionEnabled && !this.codeHotspotsEnabled) {
-      if (getCodeHotspotsOptionsOr(undefined) !== undefined) {
-        this.logger.warn(
-          'Endpoint collection is enabled, but Code Hotspots are disabled. ' +
-          'Enable Code Hotspots too for endpoint collection to work.')
-        this.endpointCollectionEnabled = false
-      } else {
-        this.logger.info('Code Hotspots are implicitly enabled by endpoint collection.')
-        this.codeHotspotsEnabled = true
-      }
-    }
 
     this.profilers = ensureProfilers(profilers, this)
   }
@@ -153,7 +146,10 @@ class Config {
 
 module.exports = { Config }
 
-function getProfilers ({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED, DD_PROFILING_PROFILERS }) {
+function getProfilers ({
+  DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED,
+  DD_PROFILING_PROFILERS, DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED
+}) {
   // First consider "legacy" DD_PROFILING_PROFILERS env variable, defaulting to wall + space
   // Use a Set to avoid duplicates
   const profilers = new Set(coalesce(DD_PROFILING_PROFILERS, 'wall,space').split(','))
@@ -176,6 +172,11 @@ function getProfilers ({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLE
     }
   }
 
+  // Events profiler is a profiler for timeline events that goes with the wall
+  // profiler
+  if (profilers.has('wall') && DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED) {
+    profilers.add('events')
+  }
   return [...profilers]
 }
 
@@ -237,6 +238,8 @@ function getProfiler (name, options) {
       return new WallProfiler(options)
     case 'space':
       return new SpaceProfiler(options)
+    case 'events':
+      return new EventsProfiler(options)
     default:
       options.logger.error(`Unknown profiler "${name}"`)
   }

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -0,0 +1,161 @@
+const { performance, constants, PerformanceObserver } = require('node:perf_hooks')
+const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared')
+const semver = require('semver')
+const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
+const pprof = require('@datadog/pprof/')
+
+// Format of perf_hooks events changed with Node 16, we need to be mindful of it.
+const node16 = semver.gte(process.version, '16.0.0')
+
+// perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file.
+const MS_TO_NS = 1000000
+
+// While this is an "events profiler", meaning it emits a pprof file based on events observed as
+// perf_hooks events, the emitted pprof file uses the type "timeline".
+const pprofValueType = 'timeline'
+const pprofValueUnit = 'nanoseconds'
+const threadName = `${threadNamePrefix} GC`
+
+/**
+ * This class generates pprof files with timeline events sourced from Node.js
+ * performance measurement APIs.
+ */
+class EventsProfiler {
+  constructor (options = {}) {
+    this.type = 'events'
+    this._flushIntervalNanos = (options.flushInterval || 60000) * 1e6 // 60 sec
+    this._observer = undefined
+    this.entries = []
+  }
+
+  start () {
+    function add (items) {
+      this.entries.push(...items.getEntries())
+    }
+    if (!this._observer) {
+      this._observer = new PerformanceObserver(add.bind(this))
+    }
+    // Currently only support GC
+    this._observer.observe({ type: 'gc' })
+  }
+
+  stop () {
+    if (this._observer) {
+      this._observer.disconnect()
+    }
+  }
+
+  profile () {
+    const stringTable = new StringTable()
+    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP)
+    const kindLabelKey = stringTable.dedup('gc type')
+    const reasonLabelKey = stringTable.dedup('gc reason')
+    const kindLabels = []
+    const reasonLabels = []
+    const locations = []
+    const functions = []
+    const locationsPerKind = []
+    const flagObj = {}
+
+    function labelFromStr (key, valStr) {
+      return new Label({ key, str: stringTable.dedup(valStr) })
+    }
+
+    function labelFromStrStr (keyStr, valStr) {
+      return labelFromStr(stringTable.dedup(keyStr), valStr)
+    }
+
+    // Create labels for all GC performance flags and kinds of GC
+    for (const [key, value] of Object.entries(constants)) {
+      if (key.startsWith('NODE_PERFORMANCE_GC_FLAGS_')) {
+        flagObj[key.substring(26).toLowerCase()] = value
+      } else if (key.startsWith('NODE_PERFORMANCE_GC_')) {
+        // It's a constant for a kind of GC
+        const kind = key.substring(20).toLowerCase()
+        kindLabels[value] = labelFromStr(kindLabelKey, kind)
+        // Construct a single-frame "location" too
+        const fn = new Function({ id: functions.length + 1, name: stringTable.dedup(`${kind} GC`) })
+        functions.push(fn)
+        const line = new Line({ functionId: fn.id })
+        const location = new Location({ id: locations.length + 1, line: [line] })
+        locations.push(location)
+        locationsPerKind[value] = [location.id]
+      }
+    }
+
+    const gcEventLabel = labelFromStrStr('event', 'gc')
+    const threadLabel = labelFromStrStr(THREAD_NAME, threadName)
+
+    function getReasonLabel (flags) {
+      if (flags === 0) {
+        return null
+      }
+      let reasonLabel = reasonLabels[flags]
+      if (!reasonLabel) {
+        const reasons = []
+        for (const [key, value] of Object.entries(flagObj)) {
+          if (value & flags) {
+            reasons.push(key)
+          }
+        }
+        const reasonStr = reasons.join(',')
+        reasonLabel = labelFromStr(reasonLabelKey, reasonStr)
+        reasonLabels[flags] = reasonLabel
+      }
+      return reasonLabel
+    }
+
+    let durationFrom = Number.POSITIVE_INFINITY
+    let durationTo = 0
+    const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS))
+
+    const samples = this.entries.map((item) => {
+      const { startTime, duration } = item
+      const { kind, flags } = node16 ? item.detail : item
+      const endTime = startTime + duration
+      if (durationFrom > startTime) durationFrom = startTime
+      if (durationTo < endTime) durationTo = endTime
+      const labels = [
+        gcEventLabel,
+        threadLabel,
+        new Label({ key: timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) }),
+        kindLabels[kind]
+      ]
+      const reasonLabel = getReasonLabel(flags)
+      if (reasonLabel) {
+        labels.push(reasonLabel)
+      }
+      const sample = new Sample({
+        value: [Math.round(duration * MS_TO_NS)],
+        label: labels,
+        locationId: locationsPerKind[kind]
+      })
+      return sample
+    })
+
+    this.entries = []
+
+    const timeValueType = new ValueType({
+      type: stringTable.dedup(pprofValueType),
+      unit: stringTable.dedup(pprofValueUnit)
+    })
+
+    return new Profile({
+      sampleType: [timeValueType],
+      timeNanos: dateOffset + BigInt(Math.round(durationFrom * MS_TO_NS)),
+      periodType: timeValueType,
+      period: this._flushIntervalNanos,
+      durationNanos: Math.max(0, Math.round((durationTo - durationFrom) * MS_TO_NS)),
+      sample: samples,
+      location: locations,
+      function: functions,
+      stringTable: stringTable
+    })
+  }
+
+  encode (profile) {
+    return pprof.encode(profile)
+  }
+}
+
+module.exports = EventsProfiler

package/packages/dd-trace/src/profiling/profilers/shared.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { isMainThread, threadId } = require('node:worker_threads')
+
+module.exports = {
+  END_TIMESTAMP: 'end_timestamp_ns',
+  THREAD_NAME: 'thread name',
+  threadNamePrefix: isMainThread ? 'Main' : `Worker #${threadId}`
+}