dd-trace 4.51.1 → 4.53.0

package/packages/dd-trace/src/proxy.js

@@ -181,8 +181,13 @@ class Tracer extends NoopProxy {
           )
         }
       }
+
+      if (config.isTestDynamicInstrumentationEnabled) {
+        const testVisibilityDynamicInstrumentation = require('./ci-visibility/dynamic-instrumentation')
+        testVisibilityDynamicInstrumentation.start()
+      }
     } catch (e) {
-      log.error(e)
+      log.error('Error initialising tracer', e)
     }
 
     return this
@@ -193,7 +198,7 @@ class Tracer extends NoopProxy {
     try {
       return require('./profiler').start(config)
     } catch (e) {
-      log.error(e)
+      log.error('Error starting profiler', e)
     }
   }
 

package/packages/dd-trace/src/runtime_metrics.js

@@ -7,11 +7,19 @@ const os = require('os')
 const { DogStatsDClient } = require('./dogstatsd')
 const log = require('./log')
 const Histogram = require('./histogram')
-const { performance } = require('perf_hooks')
+const { performance, PerformanceObserver } = require('perf_hooks')
 
+const { NODE_MAJOR, NODE_MINOR } = require('../../../version')
 const INTERVAL = 10 * 1000
 
+// Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug.
+// See: https://github.com/nodejs/node/issues/39548
+const hasGCObserver = NODE_MAJOR >= 18 || (NODE_MAJOR === 16 && NODE_MINOR >= 7)
+const hasGCProfiler = NODE_MAJOR >= 20 || (NODE_MAJOR === 18 && NODE_MINOR >= 15)
+
 let nativeMetrics = null
+let gcObserver = null
+let gcProfiler = null
 
 let interval
 let client
@@ -24,15 +32,20 @@ let elu
 
 reset()
 
-module.exports = {
+const runtimeMetrics = module.exports = {
   start (config) {
     const clientConfig = DogStatsDClient.generateClientConfig(config)
 
     try {
       nativeMetrics = require('@datadog/native-metrics')
-      nativeMetrics.start()
+
+      if (hasGCObserver) {
+        nativeMetrics.start('loop') // Only add event loop watcher and not GC.
+      } else {
+        nativeMetrics.start()
+      }
     } catch (e) {
-      log.error(e)
+      log.error('Error starting native metrics', e)
       nativeMetrics = null
     }
 
@@ -40,6 +53,9 @@ module.exports = {
 
     time = process.hrtime()
 
+    startGCObserver()
+    startGCProfiler()
+
     if (nativeMetrics) {
       interval = setInterval(() => {
         captureCommonMetrics()
@@ -138,6 +154,10 @@ function reset () {
   counters = {}
   histograms = {}
   nativeMetrics = null
+  gcObserver && gcObserver.disconnect()
+  gcObserver = null
+  gcProfiler && gcProfiler.stop()
+  gcProfiler = null
 }
 
 function captureCpuUsage () {
@@ -202,6 +222,29 @@ function captureHeapSpace () {
     client.gauge('runtime.node.heap.physical_size.by.space', stats[i].physical_space_size, tags)
   }
 }
+function captureGCMetrics () {
+  if (!gcProfiler) return
+
+  const profile = gcProfiler.stop()
+  const pauseAll = new Histogram()
+  const pause = {}
+
+  for (const stat of profile.statistics) {
+    const type = stat.gcType.replace(/([a-z])([A-Z])/g, '$1_$2').toLowerCase()
+
+    pause[type] = pause[type] || new Histogram()
+    pause[type].record(stat.cost)
+    pauseAll.record(stat.cost)
+  }
+
+  histogram('runtime.node.gc.pause', pauseAll)
+
+  for (const type in pause) {
+    histogram('runtime.node.gc.pause.by.type', pause[type], [`gc_type:${type}`])
+  }
+
+  gcProfiler.start()
+}
 
 function captureGauges () {
   Object.keys(gauges).forEach(name => {
@@ -256,6 +299,7 @@ function captureCommonMetrics () {
   captureCounters()
   captureHistograms()
   captureELU()
+  captureGCMetrics()
 }
 
 function captureNativeMetrics () {
@@ -297,6 +341,11 @@ function captureNativeMetrics () {
 function histogram (name, stats, tags) {
   tags = [].concat(tags)
 
+  // Stats can contain garbage data when a value was never recorded.
+  if (stats.count === 0) {
+    stats = { max: 0, min: 0, sum: 0, avg: 0, median: 0, p95: 0, count: 0 }
+  }
+
   client.gauge(`${name}.min`, stats.min, tags)
   client.gauge(`${name}.max`, stats.max, tags)
   client.increment(`${name}.sum`, stats.sum, tags)
@@ -306,3 +355,57 @@ function histogram (name, stats, tags) {
   client.gauge(`${name}.median`, stats.median, tags)
   client.gauge(`${name}.95percentile`, stats.p95, tags)
 }
+
+function startGCObserver () {
+  if (gcObserver || hasGCProfiler || !hasGCObserver) return
+
+  gcObserver = new PerformanceObserver(list => {
+    for (const entry of list.getEntries()) {
+      const type = gcType(entry.detail?.kind || entry.kind)
+
+      runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`)
+      runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration)
+    }
+  })
+
+  gcObserver.observe({ type: 'gc' })
+}
+
+function startGCProfiler () {
+  if (gcProfiler || !hasGCProfiler) return
+
+  gcProfiler = new v8.GCProfiler()
+  gcProfiler.start()
+}
+
+function gcType (kind) {
+  if (NODE_MAJOR >= 22) {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'minor_mark_sweep'
+      case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
+      case 8: return 'incremental_marking'
+      case 16: return 'process_weak_callbacks'
+      case 31: return 'all'
+    }
+  } else if (NODE_MAJOR >= 18) {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'minor_mark_compact'
+      case 4: return 'mark_sweep_compact'
+      case 8: return 'incremental_marking'
+      case 16: return 'process_weak_callbacks'
+      case 31: return 'all'
+    }
+  } else {
+    switch (kind) {
+      case 1: return 'scavenge'
+      case 2: return 'mark_sweep_compact'
+      case 4: return 'incremental_marking'
+      case 8: return 'process_weak_callbacks'
+      case 15: return 'all'
+    }
+  }
+
+  return 'unknown'
+}

package/packages/dd-trace/src/serverless.js

@@ -23,7 +23,7 @@ function maybeStartServerlessMiniAgent (config) {
   try {
     require('child_process').spawn(rustBinaryPath, { stdio: 'inherit' })
   } catch (err) {
-    log.error(`Error spawning mini agent process: ${err}`)
+    log.error('Error spawning mini agent process: %s', err.message)
   }
 }
 

package/packages/dd-trace/src/service-naming/schemas/v0/serverless.js

@@ -3,7 +3,7 @@ const { identityService } = require('../util')
 const serverless = {
   server: {
     'azure-functions': {
-      opName: () => 'azure-functions.invoke',
+      opName: () => 'azure.functions.invoke',
       serviceName: identityService
     }
   }

package/packages/dd-trace/src/service-naming/schemas/v1/serverless.js

@@ -3,7 +3,7 @@ const { identityService } = require('../util')
 const serverless = {
   server: {
     'azure-functions': {
-      opName: () => 'azure-functions.invoke',
+      opName: () => 'azure.functions.invoke',
       serviceName: identityService
     }
   }

package/packages/dd-trace/src/span_processor.js

@@ -87,22 +87,22 @@ class SpanProcessor {
         const id = context.toSpanId()
 
         if (finished.has(span)) {
-          log.error(`Span was already finished in the same trace: ${span}`)
+          log.error('Span was already finished in the same trace: %s', span)
         } else {
           finished.add(span)
 
           if (finishedIds.has(id)) {
-            log.error(`Another span with the same ID was already finished in the same trace: ${span}`)
+            log.error('Another span with the same ID was already finished in the same trace: %s', span)
           } else {
             finishedIds.add(id)
           }
 
           if (context._trace !== trace) {
-            log.error(`A span was finished in the wrong trace: ${span}.`)
+            log.error('A span was finished in the wrong trace: %s', span)
           }
 
           if (finishedSpans.has(span)) {
-            log.error(`Span was already finished in a different trace: ${span}`)
+            log.error('Span was already finished in a different trace: %s', span)
           } else {
             finishedSpans.add(span)
           }
@@ -114,35 +114,35 @@ class SpanProcessor {
         const id = context.toSpanId()
 
         if (started.has(span)) {
-          log.error(`Span was already started in the same trace: ${span}`)
+          log.error('Span was already started in the same trace: %s', span)
         } else {
           started.add(span)
 
           if (startedIds.has(id)) {
-            log.error(`Another span with the same ID was already started in the same trace: ${span}`)
+            log.error('Another span with the same ID was already started in the same trace: %s', span)
           } else {
             startedIds.add(id)
           }
 
           if (context._trace !== trace) {
-            log.error(`A span was started in the wrong trace: ${span}.`)
+            log.error('A span was started in the wrong trace: %s', span)
           }
 
           if (startedSpans.has(span)) {
-            log.error(`Span was already started in a different trace: ${span}`)
+            log.error('Span was already started in a different trace: %s', span)
           } else {
             startedSpans.add(span)
           }
         }
 
         if (!finished.has(span)) {
-          log.error(`Span started in one trace but was finished in another trace: ${span}`)
+          log.error('Span started in one trace but was finished in another trace: %s', span)
         }
       }
 
       for (const span of trace.finished) {
         if (!started.has(span)) {
-          log.error(`Span finished in one trace but was started in another trace: ${span}`)
+          log.error('Span finished in one trace but was started in another trace: %s', span)
         }
       }
     }

package/packages/dd-trace/src/tagger.js

@@ -44,7 +44,7 @@ function add (carrier, keyValuePairs, parseOtelTags = false) {
       Object.assign(carrier, keyValuePairs)
     }
   } catch (e) {
-    log.error(e)
+    log.error('Error adding tags', e)
   }
 }
 

package/packages/dd-trace/src/telemetry/index.js

@@ -137,6 +137,7 @@ function appClosing () {
   sendData(config, application, host, reqType, payload)
   // We flush before shutting down.
   metricsManager.send(config, application, host)
+  telemetryLogger.send(config, application, host)
 }
 
 function onBeforeExit () {

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -40,6 +40,7 @@ function onErrorLog (msg) {
 
   const telLog = {
     level: 'ERROR',
+    count: 1,
 
     // existing log.error(err) without message will be reported as 'Generic Error'
     message: message ?? 'Generic Error'
@@ -47,8 +48,7 @@ function onErrorLog (msg) {
 
   if (cause) {
     telLog.stack_trace = cause.stack
-    const errorType = cause.name ?? 'Error'
-    telLog.message = `${errorType}: ${telLog.message}`
+    telLog.errorType = cause.constructor.name
   }
 
   onLog(telLog)

package/packages/dd-trace/src/telemetry/logs/log-collector.js

@@ -3,7 +3,7 @@
 const log = require('../../log')
 const { calculateDDBasePath } = require('../../util')
 
-const logs = new Map()
+const logs = new Map() // hash -> log
 
 // NOTE: Is this a reasonable number?
 let maxEntries = 10000
@@ -47,8 +47,14 @@ function sanitize (logEntry) {
     .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath))
     .map(line => line.replace(ddBasePath, ''))
 
+  if (!isDDCode && logEntry.errorType && stackLines.length) {
+    stackLines = [`${logEntry.errorType}: redacted`, ...stackLines]
+  }
+
+  delete logEntry.errorType
+
   logEntry.stack_trace = stackLines.join(EOL)
-  if (logEntry.stack_trace === '' && !logEntry.message) {
+  if (logEntry.stack_trace === '' && (!logEntry.message || logEntry.message === 'Generic Error')) {
     // If entire stack was removed and there is no message we'd rather not log it at all.
     return null
   }
@@ -75,6 +81,8 @@ const logCollector = {
       if (!logs.has(hash)) {
         logs.set(hash, logEntry)
         return true
+      } else {
+        logs.get(hash).count++
       }
     } catch (e) {
       log.error('Unable to add log to logCollector: %s', e.message)

package/packages/dd-trace/src/telemetry/send-data.js

@@ -57,7 +57,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
     try {
       url = url || new URL(getAgentlessTelemetryEndpoint(config.site))
     } catch (err) {
-      log.error(err)
+      log.error('Telemetry endpoint url is invalid', err)
       // No point to do the request if the URL is invalid
       return cb(err, { payload, reqType })
     }
@@ -100,7 +100,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
         path: '/api/v2/apmtelemetry'
       }
       if (backendUrl) {
-        request(data, backendOptions, (error) => { log.error(error) })
+        request(data, backendOptions, (error) => { log.error('Error sending telemetry data', error) })
       } else {
         log.error('Invalid Telemetry URL')
       }

package/packages/dd-trace/src/util.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const crypto = require('crypto')
 const path = require('path')
 
 function isTrue (str) {
@@ -25,6 +24,8 @@ function isError (value) {
 
 // Matches a glob pattern to a given subject string
 function globMatch (pattern, subject) {
+  if (typeof pattern === 'string') pattern = pattern.toLowerCase()
+  if (typeof subject === 'string') subject = subject.toLowerCase()
   let px = 0 // [p]attern inde[x]
   let sx = 0 // [s]ubject inde[x]
   let nextPx = 0
@@ -64,6 +65,8 @@ function globMatch (pattern, subject) {
   return true
 }
 
+// TODO: this adds stack traces relative to packages/
+// shouldn't paths be relative to the root of dd-trace?
 function calculateDDBasePath (dirname) {
   const dirSteps = dirname.split(path.sep)
   const packagesIndex = dirSteps.lastIndexOf('packages')
@@ -74,25 +77,11 @@ function hasOwn (object, prop) {
   return Object.prototype.hasOwnProperty.call(object, prop)
 }
 
-/**
- * Generates a unique hash from an array of strings by joining them with | before hashing.
- * Used to uniquely identify AWS requests for span pointers.
- * @param {string[]} components - Array of strings to hash
- * @returns {string} A 32-character hash uniquely identifying the components
- */
-function generatePointerHash (components) {
-  // If passing S3's ETag as a component, make sure any quotes have already been removed!
-  const dataToHash = components.join('|')
-  const hash = crypto.createHash('sha256').update(dataToHash).digest('hex')
-  return hash.substring(0, 32)
-}
-
 module.exports = {
   isTrue,
   isFalse,
   isError,
   globMatch,
   calculateDDBasePath,
-  hasOwn,
-  generatePointerHash
+  hasOwn
 }

package/packages/datadog-instrumentations/src/qs.js

@@ -1,24 +0,0 @@
-'use strict'
-
-const { addHook, channel } = require('./helpers/instrument')
-const shimmer = require('../../datadog-shimmer')
-
-const qsParseCh = channel('datadog:qs:parse:finish')
-
-function wrapParse (originalParse) {
-  return function () {
-    const qsParsedObj = originalParse.apply(this, arguments)
-    if (qsParseCh.hasSubscribers && qsParsedObj) {
-      qsParseCh.publish({ qs: qsParsedObj })
-    }
-    return qsParsedObj
-  }
-}
-
-addHook({
-  name: 'qs',
-  versions: ['>=1']
-}, qs => {
-  shimmer.wrap(qs, 'parse', wrapParse)
-  return qs
-})

package/packages/dd-trace/src/appsec/iast/iast-log.js

@@ -1,86 +0,0 @@
-'use strict'
-
-const dc = require('dc-polyfill')
-const log = require('../../log')
-
-const telemetryLog = dc.channel('datadog:telemetry:log')
-
-function getTelemetryLog (data, level) {
-  try {
-    data = typeof data === 'function' ? data() : data
-
-    let message
-    if (typeof data !== 'object' || !data) {
-      message = String(data)
-    } else {
-      message = String(data.message || data)
-    }
-
-    const logEntry = {
-      message,
-      level
-    }
-    if (data.stack) {
-      logEntry.stack_trace = data.stack
-    }
-    return logEntry
-  } catch (e) {
-    log.error(e)
-  }
-}
-
-const iastLog = {
-  debug (data) {
-    log.debug(data)
-    return this
-  },
-
-  info (data) {
-    log.info(data)
-    return this
-  },
-
-  warn (data) {
-    log.warn(data)
-    return this
-  },
-
-  error (data) {
-    log.error(data)
-    return this
-  },
-
-  publish (data, level) {
-    if (telemetryLog.hasSubscribers) {
-      telemetryLog.publish(getTelemetryLog(data, level))
-    }
-    return this
-  },
-
-  debugAndPublish (data) {
-    this.debug(data)
-    return this.publish(data, 'DEBUG')
-  },
-
-  /**
-   * forward 'INFO' log level to 'DEBUG' telemetry log level
-   * see also {@link ../../telemetry/logs#isLevelEnabled } method
-   */
-  infoAndPublish (data) {
-    this.info(data)
-    return this.publish(data, 'DEBUG')
-  },
-
-  warnAndPublish (data) {
-    this.warn(data)
-    return this.publish(data, 'WARN')
-  },
-
-  errorAndPublish (data) {
-    this.error(data)
-    // publish is done automatically by log.error()
-    return this
-  }
-}
-
-module.exports = iastLog

package/packages/dd-trace/src/appsec/passport.js

@@ -1,110 +0,0 @@
-'use strict'
-
-const log = require('../log')
-const { trackEvent } = require('./sdk/track_event')
-const { setUserTags } = require('./sdk/set_user')
-
-const UUID_PATTERN = '^[0-9A-F]{8}-[0-9A-F]{4}-[1-5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$'
-const regexUsername = new RegExp(UUID_PATTERN, 'i')
-
-const SDK_USER_EVENT_PATTERN = '^_dd\\.appsec\\.events\\.users\\.[\\W\\w+]+\\.sdk$'
-const regexSdkEvent = new RegExp(SDK_USER_EVENT_PATTERN, 'i')
-
-function isSdkCalled (tags) {
-  let called = false
-
-  if (tags !== null && typeof tags === 'object') {
-    called = Object.entries(tags).some(([key, value]) => regexSdkEvent.test(key) && value === 'true')
-  }
-
-  return called
-}
-
-// delete this function later if we know it's always credential.username
-function getLogin (credentials) {
-  const type = credentials && credentials.type
-  let login
-  if (type === 'local' || type === 'http') {
-    login = credentials.username
-  }
-
-  return login
-}
-
-function parseUser (login, passportUser, mode) {
-  const user = {
-    'usr.id': login
-  }
-
-  if (!user['usr.id']) {
-    return user
-  }
-
-  if (passportUser) {
-    // Guess id
-    if (passportUser.id) {
-      user['usr.id'] = passportUser.id
-    } else if (passportUser._id) {
-      user['usr.id'] = passportUser._id
-    }
-
-    if (mode === 'extended') {
-      if (login) {
-        user['usr.login'] = login
-      }
-
-      if (passportUser.email) {
-        user['usr.email'] = passportUser.email
-      }
-
-      // Guess username
-      if (passportUser.username) {
-        user['usr.username'] = passportUser.username
-      } else if (passportUser.name) {
-        user['usr.username'] = passportUser.name
-      }
-    }
-  }
-
-  if (mode === 'safe') {
-    // Remove PII in safe mode
-    if (!regexUsername.test(user['usr.id'])) {
-      user['usr.id'] = ''
-    }
-  }
-
-  return user
-}
-
-function passportTrackEvent (credentials, passportUser, rootSpan, mode) {
-  const tags = rootSpan && rootSpan.context() && rootSpan.context()._tags
-
-  if (isSdkCalled(tags)) {
-    // Don't overwrite tags set by SDK callings
-    return
-  }
-  const user = parseUser(getLogin(credentials), passportUser, mode)
-
-  if (user['usr.id'] === undefined) {
-    log.warn('No user ID found in authentication instrumentation')
-    return
-  }
-
-  if (passportUser) {
-    // If a passportUser object is published then the login succeded
-    const userTags = {}
-    Object.entries(user).forEach(([k, v]) => {
-      const attr = k.split('.', 2)[1]
-      userTags[attr] = v
-    })
-
-    setUserTags(userTags, rootSpan)
-    trackEvent('users.login.success', null, 'passportTrackEvent', rootSpan, mode)
-  } else {
-    trackEvent('users.login.failure', user, 'passportTrackEvent', rootSpan, mode)
-  }
-}
-
-module.exports = {
-  passportTrackEvent
-}