dd-trace 4.46.0 → 4.47.0

package/packages/dd-trace/src/payload-tagging/tagging.js

@@ -0,0 +1,83 @@
+const { PAYLOAD_TAGGING_MAX_TAGS } = require('../constants')
+
+const redactedKeys = [
+  'authorization', 'x-authorization', 'password', 'token'
+]
+const truncated = 'truncated'
+const redacted = 'redacted'
+
+function escapeKey (key) {
+  return key.replaceAll('.', '\\.')
+}
+
+/**
+   * Compute normalized payload tags from any given object.
+   *
+   * @param {object} object
+   * @param {import('./mask').Mask} mask
+   * @param {number} maxDepth
+   * @param {string} prefix
+   * @returns
+   */
+function tagsFromObject (object, opts) {
+  const { maxDepth, prefix } = opts
+
+  let tagCount = 0
+  let abort = false
+  const result = {}
+
+  function tagRec (prefix, object, depth = 0) {
+    // Off by one: _dd.payload_tags_trimmed counts as 1 tag
+    if (abort) { return }
+
+    if (tagCount >= PAYLOAD_TAGGING_MAX_TAGS - 1) {
+      abort = true
+      result['_dd.payload_tags_incomplete'] = true
+      return
+    }
+
+    if (depth >= maxDepth && typeof object === 'object') {
+      tagCount += 1
+      result[prefix] = truncated
+      return
+    }
+
+    if (object === undefined) {
+      tagCount += 1
+      result[prefix] = 'undefined'
+      return
+    }
+
+    if (object === null) {
+      tagCount += 1
+      result[prefix] = 'null'
+      return
+    }
+
+    if (['number', 'boolean'].includes(typeof object) || Buffer.isBuffer(object)) {
+      tagCount += 1
+      result[prefix] = object.toString().substring(0, 5000)
+      return
+    }
+
+    if (typeof object === 'string') {
+      tagCount += 1
+      result[prefix] = object.substring(0, 5000)
+    }
+
+    if (typeof object === 'object') {
+      for (const [key, value] of Object.entries(object)) {
+        if (redactedKeys.includes(key.toLowerCase())) {
+          tagCount += 1
+          result[`${prefix}.${escapeKey(key)}`] = redacted
+        } else {
+          tagRec(`${prefix}.${escapeKey(key)}`, value, depth + 1)
+        }
+      }
+    }
+  }
+  tagRec(prefix, object)
+  return result
+}
+
+module.exports = { tagsFromObject }

package/packages/dd-trace/src/plugin_manager.js

@@ -136,10 +136,19 @@ module.exports = class PluginManager {
       dbmPropagationMode,
       dsmEnabled,
       clientIpEnabled,
-      memcachedCommandEnabled
+      memcachedCommandEnabled,
+      ciVisibilityTestSessionName
     } = this._tracerConfig
 
-    const sharedConfig = {}
+    const sharedConfig = {
+      dbmPropagationMode,
+      dsmEnabled,
+      memcachedCommandEnabled,
+      site,
+      url,
+      headers: headerTags || [],
+      ciVisibilityTestSessionName
+    }
 
     if (logInjection !== undefined) {
       sharedConfig.logInjection = logInjection
@@ -149,10 +158,6 @@ module.exports = class PluginManager {
       sharedConfig.queryStringObfuscation = queryStringObfuscation
     }
 
-    sharedConfig.dbmPropagationMode = dbmPropagationMode
-    sharedConfig.dsmEnabled = dsmEnabled
-    sharedConfig.memcachedCommandEnabled = memcachedCommandEnabled
-
     if (serviceMapping && serviceMapping[name]) {
       sharedConfig.service = serviceMapping[name]
     }
@@ -161,10 +166,6 @@ module.exports = class PluginManager {
       sharedConfig.clientIpEnabled = clientIpEnabled
     }
 
-    sharedConfig.site = site
-    sharedConfig.url = url
-    sharedConfig.headers = headerTags || []
-
     return sharedConfig
   }
 }

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -1,5 +1,6 @@
 const {
   getTestEnvironmentMetadata,
+  getTestSessionName,
   getCodeOwnersFileEntries,
   getTestParentSpan,
   getTestCommonTags,
@@ -13,11 +14,14 @@ const {
   TEST_SESSION_ID,
   TEST_COMMAND,
   TEST_MODULE,
+  TEST_SESSION_NAME,
   getTestSuiteCommonTags,
   TEST_STATUS,
   TEST_SKIPPED_BY_ITR,
   ITR_CORRELATION_ID,
-  TEST_SOURCE_FILE
+  TEST_SOURCE_FILE,
+  TEST_LEVEL_EVENT_TYPES,
+  TEST_SUITE
 } = require('./util/test')
 const Plugin = require('./plugin')
 const { COMPONENT } = require('../constants')
@@ -75,6 +79,19 @@ module.exports = class CiPlugin extends Plugin {
       // only for playwright
       this.rootDir = rootDir
 
+      const testSessionName = getTestSessionName(this.config, this.command, this.testEnvironmentMetadata)
+
+      const metadataTags = {}
+      for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
+        metadataTags[testLevel] = {
+          [TEST_SESSION_NAME]: testSessionName
+        }
+      }
+      // tracer might not be initialized correctly
+      if (this.tracer._exporter.setMetadataTags) {
+        this.tracer._exporter.setMetadataTags(metadataTags)
+      }
+
       this.testSessionSpan = this.tracer.startSpan(`${this.constructor.id}.test_session`, {
         childOf,
         tags: {
@@ -97,6 +114,7 @@ module.exports = class CiPlugin extends Plugin {
       if (this.constructor.id === 'vitest') {
         process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId()
         process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId()
+        process.env.DD_CIVISIBILITY_TEST_COMMAND = this.command
       }
 
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module')
@@ -194,6 +212,19 @@ module.exports = class CiPlugin extends Plugin {
     }
   }
 
+  getCodeOwners (tags) {
+    const {
+      [TEST_SOURCE_FILE]: testSourceFile,
+      [TEST_SUITE]: testSuite
+    } = tags
+    // We'll try with the test source file if available (it could be different from the test suite)
+    let codeOwners = getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries)
+    if (!codeOwners) {
+      codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries)
+    }
+    return codeOwners
+  }
+
   startTestSpan (testName, testSuite, testSuiteSpan, extraTags = {}) {
     const childOf = getTestParentSpan(this.tracer)
 
@@ -208,13 +239,7 @@ module.exports = class CiPlugin extends Plugin {
       ...extraTags
     }
 
-    const { [TEST_SOURCE_FILE]: testSourceFile } = extraTags
-    // We'll try with the test source file if available (it could be different from the test suite)
-    let codeOwners = getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries)
-    if (!codeOwners) {
-      codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries)
-    }
-
+    const codeOwners = this.getCodeOwners(testTags)
     if (codeOwners) {
       testTags[TEST_CODE_OWNERS] = codeOwners
     }

package/packages/dd-trace/src/plugins/util/env.js

@@ -5,6 +5,7 @@ const OS_VERSION = 'os.version'
 const OS_ARCHITECTURE = 'os.architecture'
 const RUNTIME_NAME = 'runtime.name'
 const RUNTIME_VERSION = 'runtime.version'
+const DD_HOST_CPU_COUNT = '_dd.host.vcpu_count'
 
 function getRuntimeAndOSMetadata () {
   return {
@@ -12,7 +13,8 @@ function getRuntimeAndOSMetadata () {
     [OS_ARCHITECTURE]: process.arch,
     [OS_PLATFORM]: process.platform,
     [RUNTIME_NAME]: 'node',
-    [OS_VERSION]: os.release()
+    [OS_VERSION]: os.release(),
+    [DD_HOST_CPU_COUNT]: os.cpus().length
   }
 }
 
@@ -22,5 +24,6 @@ module.exports = {
   OS_VERSION,
   OS_ARCHITECTURE,
   RUNTIME_NAME,
-  RUNTIME_VERSION
+  RUNTIME_VERSION,
+  DD_HOST_CPU_COUNT
 }

package/packages/dd-trace/src/plugins/util/test.js

@@ -19,7 +19,8 @@ const {
   GIT_COMMIT_AUTHOR_NAME,
   GIT_COMMIT_MESSAGE,
   CI_WORKSPACE_PATH,
-  CI_PIPELINE_URL
+  CI_PIPELINE_URL,
+  CI_JOB_NAME
 } = require('./tags')
 const id = require('../../id')
 
@@ -28,6 +29,9 @@ const { SAMPLING_RULE_DECISION } = require('../../constants')
 const { AUTO_KEEP } = require('../../../../../ext/priority')
 const { version: ddTraceVersion } = require('../../../../../package.json')
 
+// session tags
+const TEST_SESSION_NAME = 'test_session.name'
+
 const TEST_FRAMEWORK = 'test.framework'
 const TEST_FRAMEWORK_VERSION = 'test.framework_version'
 const TEST_TYPE = 'test.type'
@@ -95,8 +99,16 @@ const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80
 const EFD_STRING = "Retried by Datadog's Early Flake Detection"
 const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
 
+const TEST_LEVEL_EVENT_TYPES = [
+  'test',
+  'test_suite_end',
+  'test_module_end',
+  'test_session_end'
+]
+
 module.exports = {
   TEST_CODE_OWNERS,
+  TEST_SESSION_NAME,
   TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
@@ -167,7 +179,9 @@ module.exports = {
   TEST_BROWSER_DRIVER,
   TEST_BROWSER_DRIVER_VERSION,
   TEST_BROWSER_NAME,
-  TEST_BROWSER_VERSION
+  TEST_BROWSER_VERSION,
+  getTestSessionName,
+  TEST_LEVEL_EVENT_TYPES
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -615,3 +629,13 @@ function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faulty
     newSuitesPercentage > faultyThresholdPercentage
   )
 }
+
+function getTestSessionName (config, testCommand, envTags) {
+  if (config.ciVisibilityTestSessionName) {
+    return config.ciVisibilityTestSessionName
+  }
+  if (envTags[CI_JOB_NAME]) {
+    return `${envTags[CI_JOB_NAME]}-${testCommand}`
+  }
+  return testCommand
+}

package/packages/dd-trace/src/profiling/config.js

@@ -97,6 +97,11 @@ class Config {
     const samplingContextsAvailable = process.platform !== 'win32'
     function checkOptionAllowed (option, description, condition) {
       if (option && !condition) {
+        // injection hardening: all of these can only happen if user explicitly
+        // sets an environment variable to its non-default value on the platform.
+        // In practical terms, it'd require someone explicitly turning on OOM
+        // monitoring, code hotspots, endpoint profiling, or CPU profiling on
+        // Windows, where it is not supported.
         throw new Error(`${description} not supported on ${process.platform}.`)
       }
     }

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -199,7 +199,7 @@ class AgentExporter {
             this._logger.error(`Error from the agent: ${err.message}`)
             return
           } else if (err) {
-            reject(new Error('Profiler agent export back-off period expired'))
+            reject(err)
             return
           }
 

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js

@@ -0,0 +1,13 @@
+const EventPlugin = require('./event')
+
+class DNSPlugin extends EventPlugin {
+  static get id () {
+    return 'dns'
+  }
+
+  static get entryType () {
+    return 'dns'
+  }
+}
+
+module.exports = DNSPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSLookupPlugin extends DNSPlugin {
+  static get operation () {
+    return 'lookup'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookup'
+    event.detail = { hostname: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSLookupServicePlugin extends DNSPlugin {
+  static get operation () {
+    return 'lookup_service'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookupService'
+    event.detail = { host: startEvent[0], port: startEvent[1] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupServicePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js

@@ -0,0 +1,24 @@
+const DNSPlugin = require('./dns')
+
+const queryNames = new Map()
+
+class DNSResolvePlugin extends DNSPlugin {
+  static get operation () {
+    return 'resolve'
+  }
+
+  extendEvent (event, startEvent) {
+    const rrtype = startEvent[1]
+    let name = queryNames.get(rrtype)
+    if (!name) {
+      name = `query${rrtype}`
+      queryNames.set(rrtype, name)
+    }
+    event.name = name
+    event.detail = { host: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSResolvePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSReversePlugin extends DNSPlugin {
+  static get operation () {
+    return 'reverse'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'getHostByAddr'
+    event.detail = { host: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSReversePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/event.js

@@ -0,0 +1,48 @@
+const { AsyncLocalStorage } = require('async_hooks')
+const TracingPlugin = require('../../../plugins/tracing')
+const { performance } = require('perf_hooks')
+
+// We are leveraging the TracingPlugin class for its functionality to bind
+// start/error/finish methods to the appropriate diagnostic channels.
+class EventPlugin extends TracingPlugin {
+  constructor (eventHandler) {
+    super()
+    this.eventHandler = eventHandler
+    this.store = new AsyncLocalStorage()
+    this.entryType = this.constructor.entryType
+  }
+
+  start (startEvent) {
+    this.store.enterWith({
+      startEvent,
+      startTime: performance.now()
+    })
+  }
+
+  error () {
+    this.store.getStore().error = true
+  }
+
+  finish () {
+    const { startEvent, startTime, error } = this.store.getStore()
+    if (error) {
+      return // don't emit perf events for failed operations
+    }
+    const duration = performance.now() - startTime
+
+    const context = this.activeSpan?.context()
+    const _ddSpanId = context?.toSpanId()
+    const _ddRootSpanId = context?._trace.started[0]?.context().toSpanId() || _ddSpanId
+
+    const event = {
+      entryType: this.entryType,
+      startTime,
+      duration,
+      _ddSpanId,
+      _ddRootSpanId
+    }
+    this.eventHandler(this.extendEvent(event, startEvent))
+  }
+}
+
+module.exports = EventPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/net.js

@@ -0,0 +1,24 @@
+const EventPlugin = require('./event')
+
+class NetPlugin extends EventPlugin {
+  static get id () {
+    return 'net'
+  }
+
+  static get operation () {
+    return 'tcp'
+  }
+
+  static get entryType () {
+    return 'net'
+  }
+
+  extendEvent (event, { options }) {
+    event.name = 'connect'
+    event.detail = options
+
+    return event
+  }
+}
+
+module.exports = NetPlugin

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -1,12 +1,8 @@
 const { performance, constants, PerformanceObserver } = require('perf_hooks')
-const { END_TIMESTAMP_LABEL } = require('./shared')
-const semver = require('semver')
+const { END_TIMESTAMP_LABEL, SPAN_ID_LABEL, LOCAL_ROOT_SPAN_ID_LABEL } = require('./shared')
 const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
 const pprof = require('@datadog/pprof/')
 
-// Format of perf_hooks events changed with Node 16, we need to be mindful of it.
-const node16 = semver.gte(process.version, '16.0.0')
-
 // perf_hooks uses millis, with fractional part representing nanos. We emit nanos into the pprof file.
 const MS_TO_NS = 1000000
 
@@ -48,7 +44,7 @@ class GCDecorator {
   }
 
   decorateSample (sampleInput, item) {
-    const { kind, flags } = node16 ? item.detail : item
+    const { kind, flags } = item.detail
     sampleInput.label.push(this.kindLabels[kind])
     const reasonLabel = this.getReasonLabel(flags)
     if (reasonLabel) {
@@ -140,12 +136,9 @@ class NetDecorator {
 // Keys correspond to PerformanceEntry.entryType, values are constructor
 // functions for type-specific decorators.
 const decoratorTypes = {
-  gc: GCDecorator
-}
-// Needs at least node 16 for DNS and Net
-if (node16) {
-  decoratorTypes.dns = DNSDecorator
-  decoratorTypes.net = NetDecorator
+  gc: GCDecorator,
+  dns: DNSDecorator,
+  net: NetDecorator
 }
 
 // Translates performance entries into pprof samples.
@@ -168,10 +161,12 @@ class EventSerializer {
     this.locationId = [location.id]
 
     this.timestampLabelKey = this.stringTable.dedup(END_TIMESTAMP_LABEL)
+    this.spanIdKey = this.stringTable.dedup(SPAN_ID_LABEL)
+    this.rootSpanIdKey = this.stringTable.dedup(LOCAL_ROOT_SPAN_ID_LABEL)
   }
 
   addEvent (item) {
-    const { entryType, startTime, duration } = item
+    const { entryType, startTime, duration, _ddSpanId, _ddRootSpanId } = item
     let decorator = this.decorators[entryType]
     if (!decorator) {
       const DecoratorCtor = decoratorTypes[entryType]
@@ -186,13 +181,21 @@ class EventSerializer {
       }
     }
     const endTime = startTime + duration
+    const label = [
+      decorator.eventTypeLabel,
+      new Label({ key: this.timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) })
+    ]
+    if (_ddSpanId) {
+      label.push(labelFromStr(this.stringTable, this.spanIdKey, _ddSpanId))
+    }
+    if (_ddRootSpanId) {
+      label.push(labelFromStr(this.stringTable, this.rootSpanIdKey, _ddRootSpanId))
+    }
+
     const sampleInput = {
       value: [Math.round(duration * MS_TO_NS)],
       locationId: this.locationId,
-      label: [
-        decorator.eventTypeLabel,
-        new Label({ key: this.timestampLabelKey, num: dateOffset + BigInt(Math.round(endTime * MS_TO_NS)) })
-      ]
+      label
     }
     decorator.decorateSample(sampleInput, item)
     this.samples.push(new Sample(sampleInput))
@@ -219,36 +222,109 @@ class EventSerializer {
 }
 
 /**
- * This class generates pprof files with timeline events sourced from Node.js
- * performance measurement APIs.
+ * Class that sources timeline events through Node.js performance measurement APIs.
  */
-class EventsProfiler {
-  constructor (options = {}) {
-    this.type = 'events'
-    this._flushIntervalNanos = (options.flushInterval || 60000) * 1e6 // 60 sec
-    this._observer = undefined
-    this.eventSerializer = new EventSerializer()
+class NodeApiEventSource {
+  constructor (eventHandler, entryTypes) {
+    this.eventHandler = eventHandler
+    this.observer = undefined
+    this.entryTypes = entryTypes || Object.keys(decoratorTypes)
   }
 
   start () {
     // if already started, do nothing
-    if (this._observer) return
+    if (this.observer) return
 
     function add (items) {
       for (const item of items.getEntries()) {
-        this.eventSerializer.addEvent(item)
+        this.eventHandler(item)
       }
     }
-    this._observer = new PerformanceObserver(add.bind(this))
-    this._observer.observe({ entryTypes: Object.keys(decoratorTypes) })
+
+    this.observer = new PerformanceObserver(add.bind(this))
+    this.observer.observe({ entryTypes: this.entryTypes })
+  }
+
+  stop () {
+    if (this.observer) {
+      this.observer.disconnect()
+      this.observer = undefined
+    }
+  }
+}
+
+class DatadogInstrumentationEventSource {
+  constructor (eventHandler) {
+    this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'net'].map(m => {
+      const Plugin = require(`./event_plugins/${m}`)
+      return new Plugin(eventHandler)
+    })
+
+    this.started = false
+  }
+
+  start () {
+    if (!this.started) {
+      this.plugins.forEach(p => p.configure({ enabled: true }))
+      this.started = true
+    }
   }
 
   stop () {
-    if (this._observer) {
-      this._observer.disconnect()
-      this._observer = undefined
+    if (this.started) {
+      this.plugins.forEach(p => p.configure({ enabled: false }))
+      this.started = false
     }
   }
+}
+
+class CompositeEventSource {
+  constructor (sources) {
+    this.sources = sources
+  }
+
+  start () {
+    this.sources.forEach(s => s.start())
+  }
+
+  stop () {
+    this.sources.forEach(s => s.stop())
+  }
+}
+
+/**
+ * This class generates pprof files with timeline events. It combines an event
+ * source with an event serializer.
+ */
+class EventsProfiler {
+  constructor (options = {}) {
+    this.type = 'events'
+    this.eventSerializer = new EventSerializer()
+
+    const eventHandler = event => {
+      this.eventSerializer.addEvent(event)
+    }
+
+    if (options.codeHotspotsEnabled) {
+      // Use Datadog instrumentation to collect events with span IDs. Still use
+      // Node API for GC events.
+      this.eventSource = new CompositeEventSource([
+        new DatadogInstrumentationEventSource(eventHandler),
+        new NodeApiEventSource(eventHandler, ['gc'])
+      ])
+    } else {
+      // Use Node API instrumentation to collect events without span IDs
+      this.eventSource = new NodeApiEventSource(eventHandler)
+    }
+  }
+
+  start () {
+    this.eventSource.start()
+  }
+
+  stop () {
+    this.eventSource.stop()
+  }
 
   profile (restart, startDate, endDate) {
     if (!restart) {