dd-trace 4.45.0 → 4.47.0

package/packages/dd-trace/src/payload-tagging/config/index.js

@@ -0,0 +1,30 @@
+const aws = require('./aws.json')
+const sdks = { aws }
+
+function getSDKRules (sdk, requestInput, responseInput) {
+  return Object.fromEntries(
+    Object.entries(sdk).map(([service, serviceRules]) => {
+      return [
+        service,
+        {
+          request: serviceRules.request.concat(requestInput || []),
+          response: serviceRules.response.concat(responseInput || []),
+          expand: serviceRules.expand || []
+        }
+      ]
+    })
+  )
+}
+
+function appendRules (requestInput, responseInput) {
+  return Object.fromEntries(
+    Object.entries(sdks).map(([name, sdk]) => {
+      return [
+        name,
+        getSDKRules(sdk, requestInput, responseInput)
+      ]
+    })
+  )
+}
+
+module.exports = { appendRules }

package/packages/dd-trace/src/payload-tagging/index.js

@@ -0,0 +1,93 @@
+const rfdc = require('rfdc')({ proto: false, circles: false })
+
+const {
+  PAYLOAD_TAG_REQUEST_PREFIX,
+  PAYLOAD_TAG_RESPONSE_PREFIX
+} = require('../constants')
+
+const jsonpath = require('jsonpath-plus').JSONPath
+
+const { tagsFromObject } = require('./tagging')
+
+/**
+ * Given an identified value, attempt to parse it as JSON if relevant
+ *
+ * @param {any} value
+ * @returns {any} the parsed object if parsing was successful, the input if not
+ */
+function maybeJSONParseValue (value) {
+  if (typeof value !== 'string' || value[0] !== '{') {
+    return value
+  }
+
+  try {
+    return JSON.parse(value)
+  } catch (e) {
+    return value
+  }
+}
+
+/**
+ * Apply expansion to all expansion JSONPath queries
+ *
+ * @param {Object} object
+ * @param {[String]} expansionRules list of JSONPath queries
+ */
+function expand (object, expansionRules) {
+  for (const rule of expansionRules) {
+    jsonpath(rule, object, (value, _type, desc) => {
+      desc.parent[desc.parentProperty] = maybeJSONParseValue(value)
+    })
+  }
+}
+
+/**
+ * Apply redaction to all redaction JSONPath queries
+ *
+ * @param {Object} object
+ * @param {[String]} redactionRules
+ */
+function redact (object, redactionRules) {
+  for (const rule of redactionRules) {
+    jsonpath(rule, object, (_value, _type, desc) => {
+      desc.parent[desc.parentProperty] = 'redacted'
+    })
+  }
+}
+
+/**
+ * Generate a map of tag names to tag values by performing:
+ * 1. Attempting to parse identified fields as JSON
+ * 2. Redacting fields identified by redaction rules
+ * 3. Flattening the resulting object, producing as many tag name/tag value pairs
+ *    as there are leaf values in the object
+ * This function performs side-effects on a _copy_ of the input object.
+ *
+ * @param {Object} config sdk configuration for the service
+ * @param {[String]} config.expand expansion rules for the service
+ * @param {[String]} config.request redaction rules for the request
+ * @param {[String]} config.response redaction rules for the response
+ * @param {Object} object the input object to generate tags from
+ * @param {Object} opts tag generation options
+ * @param {String} opts.prefix prefix for all generated tags
+ * @param {number} opts.maxDepth maximum depth to traverse the object
+ * @returns
+ */
+function computeTags (config, object, opts) {
+  const payload = rfdc(object)
+  const redactionRules = opts.prefix === PAYLOAD_TAG_REQUEST_PREFIX ? config.request : config.response
+  const expansionRules = config.expand
+  expand(payload, expansionRules)
+  redact(payload, redactionRules)
+  return tagsFromObject(payload, opts)
+}
+
+function tagsFromRequest (config, object, opts) {
+  return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_REQUEST_PREFIX })
+}
+
+function tagsFromResponse (config, object, opts) {
+  return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_RESPONSE_PREFIX })
+}
+
+module.exports = { computeTags, tagsFromRequest, tagsFromResponse }

package/packages/dd-trace/src/payload-tagging/tagging.js

@@ -0,0 +1,83 @@
+const { PAYLOAD_TAGGING_MAX_TAGS } = require('../constants')
+
+const redactedKeys = [
+  'authorization', 'x-authorization', 'password', 'token'
+]
+const truncated = 'truncated'
+const redacted = 'redacted'
+
+function escapeKey (key) {
+  return key.replaceAll('.', '\\.')
+}
+
+/**
+   * Compute normalized payload tags from any given object.
+   *
+   * @param {object} object
+   * @param {import('./mask').Mask} mask
+   * @param {number} maxDepth
+   * @param {string} prefix
+   * @returns
+   */
+function tagsFromObject (object, opts) {
+  const { maxDepth, prefix } = opts
+
+  let tagCount = 0
+  let abort = false
+  const result = {}
+
+  function tagRec (prefix, object, depth = 0) {
+    // Off by one: _dd.payload_tags_trimmed counts as 1 tag
+    if (abort) { return }
+
+    if (tagCount >= PAYLOAD_TAGGING_MAX_TAGS - 1) {
+      abort = true
+      result['_dd.payload_tags_incomplete'] = true
+      return
+    }
+
+    if (depth >= maxDepth && typeof object === 'object') {
+      tagCount += 1
+      result[prefix] = truncated
+      return
+    }
+
+    if (object === undefined) {
+      tagCount += 1
+      result[prefix] = 'undefined'
+      return
+    }
+
+    if (object === null) {
+      tagCount += 1
+      result[prefix] = 'null'
+      return
+    }
+
+    if (['number', 'boolean'].includes(typeof object) || Buffer.isBuffer(object)) {
+      tagCount += 1
+      result[prefix] = object.toString().substring(0, 5000)
+      return
+    }
+
+    if (typeof object === 'string') {
+      tagCount += 1
+      result[prefix] = object.substring(0, 5000)
+    }
+
+    if (typeof object === 'object') {
+      for (const [key, value] of Object.entries(object)) {
+        if (redactedKeys.includes(key.toLowerCase())) {
+          tagCount += 1
+          result[`${prefix}.${escapeKey(key)}`] = redacted
+        } else {
+          tagRec(`${prefix}.${escapeKey(key)}`, value, depth + 1)
+        }
+      }
+    }
+  }
+  tagRec(prefix, object)
+  return result
+}
+
+module.exports = { tagsFromObject }

package/packages/dd-trace/src/plugin_manager.js

@@ -136,10 +136,19 @@ module.exports = class PluginManager {
       dbmPropagationMode,
       dsmEnabled,
       clientIpEnabled,
-      memcachedCommandEnabled
+      memcachedCommandEnabled,
+      ciVisibilityTestSessionName
     } = this._tracerConfig
 
-    const sharedConfig = {}
+    const sharedConfig = {
+      dbmPropagationMode,
+      dsmEnabled,
+      memcachedCommandEnabled,
+      site,
+      url,
+      headers: headerTags || [],
+      ciVisibilityTestSessionName
+    }
 
     if (logInjection !== undefined) {
       sharedConfig.logInjection = logInjection
@@ -149,10 +158,6 @@ module.exports = class PluginManager {
       sharedConfig.queryStringObfuscation = queryStringObfuscation
     }
 
-    sharedConfig.dbmPropagationMode = dbmPropagationMode
-    sharedConfig.dsmEnabled = dsmEnabled
-    sharedConfig.memcachedCommandEnabled = memcachedCommandEnabled
-
     if (serviceMapping && serviceMapping[name]) {
       sharedConfig.service = serviceMapping[name]
     }
@@ -161,10 +166,6 @@ module.exports = class PluginManager {
       sharedConfig.clientIpEnabled = clientIpEnabled
     }
 
-    sharedConfig.site = site
-    sharedConfig.url = url
-    sharedConfig.headers = headerTags || []
-
     return sharedConfig
   }
 }

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -1,5 +1,6 @@
 const {
   getTestEnvironmentMetadata,
+  getTestSessionName,
   getCodeOwnersFileEntries,
   getTestParentSpan,
   getTestCommonTags,
@@ -13,11 +14,14 @@ const {
   TEST_SESSION_ID,
   TEST_COMMAND,
   TEST_MODULE,
+  TEST_SESSION_NAME,
   getTestSuiteCommonTags,
   TEST_STATUS,
   TEST_SKIPPED_BY_ITR,
   ITR_CORRELATION_ID,
-  TEST_SOURCE_FILE
+  TEST_SOURCE_FILE,
+  TEST_LEVEL_EVENT_TYPES,
+  TEST_SUITE
 } = require('./util/test')
 const Plugin = require('./plugin')
 const { COMPONENT } = require('../constants')
@@ -75,6 +79,19 @@ module.exports = class CiPlugin extends Plugin {
       // only for playwright
       this.rootDir = rootDir
 
+      const testSessionName = getTestSessionName(this.config, this.command, this.testEnvironmentMetadata)
+
+      const metadataTags = {}
+      for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
+        metadataTags[testLevel] = {
+          [TEST_SESSION_NAME]: testSessionName
+        }
+      }
+      // tracer might not be initialized correctly
+      if (this.tracer._exporter.setMetadataTags) {
+        this.tracer._exporter.setMetadataTags(metadataTags)
+      }
+
       this.testSessionSpan = this.tracer.startSpan(`${this.constructor.id}.test_session`, {
         childOf,
         tags: {
@@ -97,6 +114,7 @@ module.exports = class CiPlugin extends Plugin {
       if (this.constructor.id === 'vitest') {
         process.env.DD_CIVISIBILITY_TEST_SESSION_ID = this.testSessionSpan.context().toTraceId()
         process.env.DD_CIVISIBILITY_TEST_MODULE_ID = this.testModuleSpan.context().toSpanId()
+        process.env.DD_CIVISIBILITY_TEST_COMMAND = this.command
       }
 
       this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'module')
@@ -194,6 +212,19 @@ module.exports = class CiPlugin extends Plugin {
     }
   }
 
+  getCodeOwners (tags) {
+    const {
+      [TEST_SOURCE_FILE]: testSourceFile,
+      [TEST_SUITE]: testSuite
+    } = tags
+    // We'll try with the test source file if available (it could be different from the test suite)
+    let codeOwners = getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries)
+    if (!codeOwners) {
+      codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries)
+    }
+    return codeOwners
+  }
+
   startTestSpan (testName, testSuite, testSuiteSpan, extraTags = {}) {
     const childOf = getTestParentSpan(this.tracer)
 
@@ -208,13 +239,7 @@ module.exports = class CiPlugin extends Plugin {
       ...extraTags
     }
 
-    const { [TEST_SOURCE_FILE]: testSourceFile } = extraTags
-    // We'll try with the test source file if available (it could be different from the test suite)
-    let codeOwners = getCodeOwnersForFilename(testSourceFile, this.codeOwnersEntries)
-    if (!codeOwners) {
-      codeOwners = getCodeOwnersForFilename(testSuite, this.codeOwnersEntries)
-    }
-
+    const codeOwners = this.getCodeOwners(testTags)
     if (codeOwners) {
       testTags[TEST_CODE_OWNERS] = codeOwners
     }

package/packages/dd-trace/src/plugins/util/env.js

@@ -5,6 +5,7 @@ const OS_VERSION = 'os.version'
 const OS_ARCHITECTURE = 'os.architecture'
 const RUNTIME_NAME = 'runtime.name'
 const RUNTIME_VERSION = 'runtime.version'
+const DD_HOST_CPU_COUNT = '_dd.host.vcpu_count'
 
 function getRuntimeAndOSMetadata () {
   return {
@@ -12,7 +13,8 @@ function getRuntimeAndOSMetadata () {
     [OS_ARCHITECTURE]: process.arch,
     [OS_PLATFORM]: process.platform,
     [RUNTIME_NAME]: 'node',
-    [OS_VERSION]: os.release()
+    [OS_VERSION]: os.release(),
+    [DD_HOST_CPU_COUNT]: os.cpus().length
   }
 }
 
@@ -22,5 +24,6 @@ module.exports = {
   OS_VERSION,
   OS_ARCHITECTURE,
   RUNTIME_NAME,
-  RUNTIME_VERSION
+  RUNTIME_VERSION,
+  DD_HOST_CPU_COUNT
 }

package/packages/dd-trace/src/plugins/util/test.js

@@ -19,7 +19,8 @@ const {
   GIT_COMMIT_AUTHOR_NAME,
   GIT_COMMIT_MESSAGE,
   CI_WORKSPACE_PATH,
-  CI_PIPELINE_URL
+  CI_PIPELINE_URL,
+  CI_JOB_NAME
 } = require('./tags')
 const id = require('../../id')
 
@@ -28,6 +29,9 @@ const { SAMPLING_RULE_DECISION } = require('../../constants')
 const { AUTO_KEEP } = require('../../../../../ext/priority')
 const { version: ddTraceVersion } = require('../../../../../package.json')
 
+// session tags
+const TEST_SESSION_NAME = 'test_session.name'
+
 const TEST_FRAMEWORK = 'test.framework'
 const TEST_FRAMEWORK_VERSION = 'test.framework_version'
 const TEST_TYPE = 'test.type'
@@ -95,11 +99,16 @@ const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80
 const EFD_STRING = "Retried by Datadog's Early Flake Detection"
 const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
 
-// Flaky test retries
-const NUM_FAILED_TEST_RETRIES = 5
+const TEST_LEVEL_EVENT_TYPES = [
+  'test',
+  'test_suite_end',
+  'test_module_end',
+  'test_session_end'
+]
 
 module.exports = {
   TEST_CODE_OWNERS,
+  TEST_SESSION_NAME,
   TEST_FRAMEWORK,
   TEST_FRAMEWORK_VERSION,
   JEST_TEST_RUNNER,
@@ -171,7 +180,8 @@ module.exports = {
   TEST_BROWSER_DRIVER_VERSION,
   TEST_BROWSER_NAME,
   TEST_BROWSER_VERSION,
-  NUM_FAILED_TEST_RETRIES
+  getTestSessionName,
+  TEST_LEVEL_EVENT_TYPES
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -619,3 +629,13 @@ function getIsFaultyEarlyFlakeDetection (projectSuites, testsBySuiteName, faulty
     newSuitesPercentage > faultyThresholdPercentage
   )
 }
+
+function getTestSessionName (config, testCommand, envTags) {
+  if (config.ciVisibilityTestSessionName) {
+    return config.ciVisibilityTestSessionName
+  }
+  if (envTags[CI_JOB_NAME]) {
+    return `${envTags[CI_JOB_NAME]}-${testCommand}`
+  }
+  return testCommand
+}

package/packages/dd-trace/src/profiler.js

@@ -8,8 +8,8 @@ process.once('beforeExit', () => { profiler.stop() })
 
 module.exports = {
   start: config => {
-    const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA } = config
-    const { enabled, sourceMap, exporters, heuristicsEnabled } = config.profiling
+    const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA, injectionEnabled } = config
+    const { enabled, sourceMap, exporters } = config.profiling
     const logger = {
       debug: (message) => log.debug(message),
       info: (message) => log.info(message),
@@ -17,9 +17,17 @@ module.exports = {
       error: (message) => log.error(message)
     }
 
+    const libraryInjected = injectionEnabled.length > 0
+    let activation
+    if (enabled === 'auto') {
+      activation = 'auto'
+    } else if (enabled === 'true') {
+      activation = 'manual'
+    } else if (injectionEnabled.includes('profiler')) {
+      activation = 'injection'
+    } // else activation = undefined
+
     return profiler.start({
-      enabled,
-      heuristicsEnabled,
       service,
       version,
       env,
@@ -31,7 +39,9 @@ module.exports = {
       port,
       tags,
       repositoryUrl,
-      commitSHA
+      commitSHA,
+      libraryInjected,
+      activation
     })
   },
 

package/packages/dd-trace/src/profiling/config.js

@@ -23,7 +23,6 @@ class Config {
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
       DD_PROFILING_CPU_ENABLED,
       DD_PROFILING_DEBUG_SOURCE_MAPS,
-      DD_PROFILING_ENABLED,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED,
       DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED,
       DD_PROFILING_EXPERIMENTAL_CPU_ENABLED,
@@ -49,7 +48,6 @@ class Config {
       DD_VERSION
     } = process.env
 
-    const enabled = isTrue(coalesce(options.enabled, DD_PROFILING_ENABLED, true))
     const env = coalesce(options.env, DD_ENV)
     const service = options.service || DD_SERVICE || 'node'
     const host = os.hostname()
@@ -64,8 +62,6 @@ class Config {
     const pprofPrefix = coalesce(options.pprofPrefix,
       DD_PROFILING_PPROF_PREFIX, '')
 
-    this.enabled = enabled
-    this.heuristicsEnabled = options.heuristicsEnabled
     this.service = service
     this.env = env
     this.host = host
@@ -101,6 +97,11 @@ class Config {
     const samplingContextsAvailable = process.platform !== 'win32'
     function checkOptionAllowed (option, description, condition) {
       if (option && !condition) {
+        // injection hardening: all of these can only happen if user explicitly
+        // sets an environment variable to its non-default value on the platform.
+        // In practical terms, it'd require someone explicitly turning on OOM
+        // monitoring, code hotspots, endpoint profiling, or CPU profiling on
+        // Windows, where it is not supported.
         throw new Error(`${description} not supported on ${process.platform}.`)
       }
     }
@@ -129,6 +130,8 @@ class Config {
       port
     })))
 
+    this.libraryInjected = options.libraryInjected
+    this.activation = options.activation
     this.exporters = ensureExporters(options.exporters || [
       new AgentExporter(this)
     ], this)

package/packages/dd-trace/src/profiling/exporter_cli.js

@@ -16,10 +16,22 @@ function exporterFromURL (url) {
   if (url.protocol === 'file:') {
     return new FileExporter({ pprofPrefix: fileURLToPath(url) })
   } else {
+    const injectionEnabled = (process.env.DD_INJECTION_ENABLED || '').split(',')
+    const libraryInjected = injectionEnabled.length > 0
+    const profilingEnabled = (process.env.DD_PROFILING_ENABLED || '').toLowerCase()
+    const activation = ['true', '1'].includes(profilingEnabled)
+      ? 'manual'
+      : profilingEnabled === 'auto'
+        ? 'auto'
+        : injectionEnabled.includes('profiling')
+          ? 'injection'
+          : 'unknown'
     return new AgentExporter({
       url,
       logger,
-      uploadTimeout: timeoutMs
+      uploadTimeout: timeoutMs,
+      libraryInjected,
+      activation
     })
   }
 }

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -53,7 +53,7 @@ function computeRetries (uploadTimeout) {
 }
 
 class AgentExporter {
-  constructor ({ url, logger, uploadTimeout, env, host, service, version } = {}) {
+  constructor ({ url, logger, uploadTimeout, env, host, service, version, libraryInjected, activation } = {}) {
     this._url = url
     this._logger = logger
 
@@ -65,6 +65,8 @@ class AgentExporter {
     this._host = host
     this._service = service
     this._appVersion = version
+    this._libraryInjected = !!libraryInjected
+    this._activation = activation || 'unknown'
   }
 
   export ({ profiles, start, end, tags }) {
@@ -105,6 +107,10 @@ class AgentExporter {
           kernel_version: os.version()
         },
         profiler: {
+          activation: this._activation,
+          ssi: {
+            mechanism: this._libraryInjected ? 'injected_agent' : 'none'
+          },
           version
         },
         runtime: {
@@ -193,7 +199,7 @@ class AgentExporter {
             this._logger.error(`Error from the agent: ${err.message}`)
             return
           } else if (err) {
-            reject(new Error('Profiler agent export back-off period expired'))
+            reject(err)
             return
           }
 

package/packages/dd-trace/src/profiling/profiler.js

@@ -5,7 +5,6 @@ const { Config } = require('./config')
 const { snapshotKinds } = require('./constants')
 const { threadNamePrefix } = require('./profilers/shared')
 const dc = require('dc-polyfill')
-const telemetryLog = dc.channel('datadog:telemetry:log')
 
 const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
 
@@ -20,13 +19,6 @@ function logError (logger, err) {
   if (logger) {
     logger.error(err)
   }
-  if (telemetryLog.hasSubscribers) {
-    telemetryLog.publish({
-      message: err.message,
-      level: 'ERROR',
-      stack_trace: err.stack
-    })
-  }
 }
 
 class Profiler extends EventEmitter {
@@ -55,7 +47,6 @@ class Profiler extends EventEmitter {
     if (this._enabled) return true
 
     const config = this._config = new Config(options)
-    if (!config.enabled && !config.heuristicsEnabled) return false
 
     this._logger = config.logger
     this._enabled = true

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns.js

@@ -0,0 +1,13 @@
+const EventPlugin = require('./event')
+
+class DNSPlugin extends EventPlugin {
+  static get id () {
+    return 'dns'
+  }
+
+  static get entryType () {
+    return 'dns'
+  }
+}
+
+module.exports = DNSPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookup.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSLookupPlugin extends DNSPlugin {
+  static get operation () {
+    return 'lookup'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookup'
+    event.detail = { hostname: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupPlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_lookupservice.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSLookupServicePlugin extends DNSPlugin {
+  static get operation () {
+    return 'lookup_service'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'lookupService'
+    event.detail = { host: startEvent[0], port: startEvent[1] }
+
+    return event
+  }
+}
+
+module.exports = DNSLookupServicePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_resolve.js

@@ -0,0 +1,24 @@
+const DNSPlugin = require('./dns')
+
+const queryNames = new Map()
+
+class DNSResolvePlugin extends DNSPlugin {
+  static get operation () {
+    return 'resolve'
+  }
+
+  extendEvent (event, startEvent) {
+    const rrtype = startEvent[1]
+    let name = queryNames.get(rrtype)
+    if (!name) {
+      name = `query${rrtype}`
+      queryNames.set(rrtype, name)
+    }
+    event.name = name
+    event.detail = { host: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSResolvePlugin

package/packages/dd-trace/src/profiling/profilers/event_plugins/dns_reverse.js

@@ -0,0 +1,16 @@
+const DNSPlugin = require('./dns')
+
+class DNSReversePlugin extends DNSPlugin {
+  static get operation () {
+    return 'reverse'
+  }
+
+  extendEvent (event, startEvent) {
+    event.name = 'getHostByAddr'
+    event.detail = { host: startEvent[0] }
+
+    return event
+  }
+}
+
+module.exports = DNSReversePlugin