dd-trace 5.63.3 → 5.65.0

package/index.d.ts

@@ -712,6 +712,16 @@ declare namespace tracer {
          * @default true
          */
         enabled?: boolean,
+
+        /** Whether to enable endpoint collection for API Security.
+         * @default true
+         */
+        endpointCollectionEnabled?: boolean,
+
+        /** Maximum number of endpoints that can be serialized per message.
+         * @default 300
+         */
+        endpointCollectionMessageLimit?: number,
       },
       /**
        * Configuration for RASP

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.63.3",
+  "version": "5.65.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",

package/packages/datadog-instrumentations/src/ai.js

@@ -98,7 +98,18 @@ function wrapWithTracer (fn) {
   return function () {
     const options = arguments[0]
 
-    options.experimental_telemetry ??= { isEnabled: true, tracer: noopTracer }
+    const experimentalTelemetry = options.experimental_telemetry
+    if (experimentalTelemetry?.isEnabled === false) {
+      return fn.apply(this, arguments)
+    }
+
+    if (experimentalTelemetry == null) {
+      options.experimental_telemetry = { isEnabled: true, tracer: noopTracer }
+    } else {
+      experimentalTelemetry.isEnabled = true
+      experimentalTelemetry.tracer ??= noopTracer
+    }
+
     wrapTracer(options.experimental_telemetry.tracer)
 
     return fn.apply(this, arguments)

package/packages/datadog-instrumentations/src/playwright.js

@@ -148,8 +148,8 @@ function getPlaywrightConfig (playwrightRunner) {
   }
 }
 
-function getRootDir (playwrightRunner) {
-  const config = getPlaywrightConfig(playwrightRunner)
+function getRootDir (playwrightRunner, configArg) {
+  const config = configArg?.config || getPlaywrightConfig(playwrightRunner)
   if (config.rootDir) {
     return config.rootDir
   }
@@ -162,8 +162,8 @@ function getRootDir (playwrightRunner) {
   return process.cwd()
 }
 
-function getProjectsFromRunner (runner) {
-  const config = getPlaywrightConfig(runner)
+function getProjectsFromRunner (runner, configArg) {
+  const config = configArg?.projects ? configArg : getPlaywrightConfig(runner)
   return config.projects?.map((project) => {
     if (project.project) {
       return project.project
@@ -509,11 +509,12 @@ function dispatcherHookNew (dispatcherExport, runWrapper) {
   return dispatcherExport
 }
 
-function runnerHook (runnerExport, playwrightVersion) {
-  shimmer.wrap(runnerExport.Runner.prototype, 'runAllTests', runAllTests => async function () {
+function runAllTestsWrapper (runAllTests, playwrightVersion) {
+  // Config parameter is only available from >=1.55.0
+  return async function (config) {
     let onDone
 
-    rootDir = getRootDir(this)
+    rootDir = getRootDir(this, config)
 
     const processArgv = process.argv.slice(2).join(' ')
     const command = `playwright ${processArgv}`
@@ -586,7 +587,7 @@ function runnerHook (runnerExport, playwrightVersion) {
       }
     }
 
-    const projects = getProjectsFromRunner(this)
+    const projects = getProjectsFromRunner(this, config)
 
     const shouldSetRetries = isFlakyTestRetriesEnabled &&
       flakyTestRetriesCount > 0 &&
@@ -651,6 +652,23 @@ function runnerHook (runnerExport, playwrightVersion) {
     // TODO: we can trick playwright into thinking the session passed by returning
     // 'passed' here. We might be able to use this for both EFD and Test Management tests.
     return runAllTestsReturn
+  }
+}
+
+function runnerHook (runnerExport, playwrightVersion) {
+  shimmer.wrap(
+    runnerExport.Runner.prototype,
+    'runAllTests',
+    runAllTests => runAllTestsWrapper(runAllTests, playwrightVersion)
+  )
+}
+
+function runnerHookNew (runnerExport, playwrightVersion) {
+  runnerExport = shimmer.wrap(runnerExport, 'runAllTestsWithConfig', function (originalGetter) {
+    const originalFunction = originalGetter.call(this)
+    return function () {
+      return runAllTestsWrapper(originalFunction, playwrightVersion)
+    }
   })
 
   return runnerExport
@@ -694,6 +712,12 @@ addHook({
   versions: ['>=1.38.0']
 }, runnerHook)
 
+addHook({
+  name: 'playwright',
+  file: 'lib/runner/testRunner.js',
+  versions: ['>=1.55.0']
+}, runnerHookNew)
+
 addHook({
   name: 'playwright',
   file: 'lib/runner/dispatcher.js',

package/packages/dd-trace/src/azure_metadata.js

@@ -76,10 +76,11 @@ function buildMetadata () {
 }
 
 function getAzureAppMetadata () {
-  // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for
-  // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services
-  // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21
-  if (getEnvironmentVariable('DD_AZURE_APP_SERVICES') !== undefined) {
+  // WEBSITE_SITE_NAME is the unique name of the website instance within Azure App Services. Its
+  // presence is used to determine if we are running in Azure App Service
+  // See equivalent in dd-trace-dotnet:
+  // https://github.com/DataDog/dd-trace-dotnet/blob/37030168b2996e549ba23231ae732874b53a37e6/tracer/src/Datadog.Trace/Util/EnvironmentHelpers.cs#L99-L155
+  if (getEnvironmentVariable('WEBSITE_SITE_NAME') !== undefined) {
     return buildMetadata()
   }
 }

package/packages/dd-trace/src/config.js

@@ -323,12 +323,6 @@ class Config {
       }
     }
 
-    if (typeof options.runtimeMetrics?.gc === 'boolean') {
-      options.runtimeMetrics.gc = {
-        enabled: options.runtimeMetrics.gc
-      }
-    }
-
     const DD_INSTRUMENTATION_INSTALL_ID = coalesce(
       getEnvironmentVariable('DD_INSTRUMENTATION_INSTALL_ID'),
       null
@@ -493,6 +487,8 @@ class Config {
     defaults.apmTracingEnabled = true
     defaults['appsec.apiSecurity.enabled'] = true
     defaults['appsec.apiSecurity.sampleDelay'] = 30
+    defaults['appsec.apiSecurity.endpointCollectionEnabled'] = true
+    defaults['appsec.apiSecurity.endpointCollectionMessageLimit'] = 300
     defaults['appsec.blockedTemplateGraphql'] = undefined
     defaults['appsec.blockedTemplateHtml'] = undefined
     defaults['appsec.blockedTemplateJson'] = undefined
@@ -690,6 +686,8 @@ class Config {
       DD_AGENT_HOST,
       DD_API_SECURITY_ENABLED,
       DD_API_SECURITY_SAMPLE_DELAY,
+      DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED,
+      DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT,
       DD_APM_TRACING_ENABLED,
       DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE,
       DD_APPSEC_COLLECT_ALL_HEADERS,
@@ -846,6 +844,10 @@ class Config {
     ))
     this._setBoolean(env, 'appsec.apiSecurity.enabled', DD_API_SECURITY_ENABLED && isTrue(DD_API_SECURITY_ENABLED))
     env['appsec.apiSecurity.sampleDelay'] = maybeFloat(DD_API_SECURITY_SAMPLE_DELAY)
+    this._setBoolean(env, 'appsec.apiSecurity.endpointCollectionEnabled',
+      DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED)
+    env['appsec.apiSecurity.endpointCollectionMessageLimit'] =
+      maybeInt(DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT)
     env['appsec.blockedTemplateGraphql'] = maybeFile(DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)
     env['appsec.blockedTemplateHtml'] = maybeFile(DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML)
     this._envUnprocessed['appsec.blockedTemplateHtml'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML
@@ -1073,6 +1075,10 @@ class Config {
       options.experimental?.appsec?.standalone && !options.experimental.appsec.standalone.enabled
     ))
     this._setBoolean(opts, 'appsec.apiSecurity.enabled', options.appsec?.apiSecurity?.enabled)
+    this._setBoolean(opts, 'appsec.apiSecurity.endpointCollectionEnabled',
+      options.appsec?.apiSecurity?.endpointCollectionEnabled)
+    opts['appsec.apiSecurity.endpointCollectionMessageLimit'] =
+      maybeInt(options.appsec?.apiSecurity?.endpointCollectionMessageLimit)
     opts['appsec.blockedTemplateGraphql'] = maybeFile(options.appsec?.blockedTemplateGraphql)
     opts['appsec.blockedTemplateHtml'] = maybeFile(options.appsec?.blockedTemplateHtml)
     this._optsUnprocessed['appsec.blockedTemplateHtml'] = options.appsec?.blockedTemplateHtml
@@ -1190,7 +1196,7 @@ class Config {
     this._setBoolean(opts, 'reportHostname', options.reportHostname)
     this._setBoolean(opts, 'runtimeMetrics.enabled', options.runtimeMetrics?.enabled)
     this._setBoolean(opts, 'runtimeMetrics.eventLoop', options.runtimeMetrics?.eventLoop)
-    this._setBoolean(opts, 'runtimeMetrics.gc', options.runtimeMetrics?.gc?.enabled)
+    this._setBoolean(opts, 'runtimeMetrics.gc', options.runtimeMetrics?.gc)
     this._setBoolean(opts, 'runtimeMetricsRuntimeId', options.runtimeMetricsRuntimeId)
     this._setArray(opts, 'sampler.spanSamplingRules', reformatSpanSamplingRules(options.spanSamplingRules))
     this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate))

package/packages/dd-trace/src/datastreams/pathway.js

@@ -17,7 +17,7 @@ const CONTEXT_PROPAGATION_KEY_BASE64 = 'dd-pathway-ctx-base64'
 const logKeys = [CONTEXT_PROPAGATION_KEY, CONTEXT_PROPAGATION_KEY_BASE64]
 
 function shaHash (checkpointString) {
-  const hash = crypto.createHash('md5').update(checkpointString).digest('hex').slice(0, 16)
+  const hash = crypto.createHash('sha256').update(checkpointString).digest('hex').slice(0, 16)
   return Buffer.from(hash, 'hex')
 }
 

package/packages/dd-trace/src/dogstatsd.js

@@ -156,23 +156,19 @@ class DogStatsDClient {
     return socket
   }
 
-  static generateClientConfig (config = {}) {
+  static generateClientConfig (config) {
     const tags = []
 
     if (config.tags) {
-      Object.keys(config.tags)
-        .filter(key => typeof config.tags[key] === 'string')
-        .filter(key => {
-          // Skip runtime-id unless enabled as cardinality may be too high
-          if (key !== 'runtime-id') return true
-          return config.runtimeMetricsRuntimeId
-        })
-        .forEach(key => {
+      for (const [key, value] of Object.entries(config.tags)) {
+        // Skip runtime-id unless enabled as cardinality may be too high
+        if (typeof value === 'string' && (key !== 'runtime-id' || config.runtimeMetricsRuntimeId)) {
           // https://docs.datadoghq.com/tagging/#defining-tags
-          const value = config.tags[key].replaceAll(/[^a-z0-9_:./-]/ig, '_')
+          const valueStripped = value.replaceAll(/[^a-z0-9_:./-]/ig, '_')
 
-          tags.push(`${key}:${value}`)
-        })
+          tags.push(`${key}:${valueStripped}`)
+        }
+      }
     }
 
     const clientConfig = {
@@ -216,7 +212,7 @@ class MetricsAggregationClient {
     this._histograms = new Map()
   }
 
-  // TODO: Aggerate with a histogram and send the buckets to the client.
+  // TODO: Aggregate with a histogram and send the buckets to the client.
   distribution (name, value, tags) {
     this._client.distribution(name, value, tags)
   }
@@ -352,9 +348,10 @@ class MetricsAggregationClient {
  * @implements {DogStatsD}
  */
 class CustomMetrics {
+  #client
   constructor (config) {
     const clientConfig = DogStatsDClient.generateClientConfig(config)
-    this._client = new MetricsAggregationClient(new DogStatsDClient(clientConfig))
+    this.#client = new MetricsAggregationClient(new DogStatsDClient(clientConfig))
 
     const flush = this.flush.bind(this)
 
@@ -365,27 +362,27 @@ class CustomMetrics {
   }
 
   increment (stat, value = 1, tags) {
-    this._client.increment(stat, value, CustomMetrics.tagTranslator(tags))
+    this.#client.increment(stat, value, CustomMetrics.tagTranslator(tags))
   }
 
   decrement (stat, value = 1, tags) {
-    this._client.decrement(stat, value, CustomMetrics.tagTranslator(tags))
+    this.#client.decrement(stat, value, CustomMetrics.tagTranslator(tags))
   }
 
   gauge (stat, value, tags) {
-    this._client.gauge(stat, value, CustomMetrics.tagTranslator(tags))
+    this.#client.gauge(stat, value, CustomMetrics.tagTranslator(tags))
   }
 
   distribution (stat, value, tags) {
-    this._client.distribution(stat, value, CustomMetrics.tagTranslator(tags))
+    this.#client.distribution(stat, value, CustomMetrics.tagTranslator(tags))
   }
 
   histogram (stat, value, tags) {
-    this._client.histogram(stat, value, CustomMetrics.tagTranslator(tags))
+    this.#client.histogram(stat, value, CustomMetrics.tagTranslator(tags))
   }
 
   flush () {
-    return this._client.flush()
+    return this.#client.flush()
   }
 
   /**

package/packages/dd-trace/src/opentracing/tracer.js

@@ -86,7 +86,7 @@ class DatadogTracer {
     }
 
     try {
-      if (format !== 'text_map_dsm') {
+      if (format !== 'text_map_dsm' && format !== formats.LOG) {
         this._prioritySampler.sample(context)
       }
       this._propagators[format].inject(context, carrier)

package/packages/dd-trace/src/runtime_metrics/runtime_metrics.js

@@ -4,72 +4,90 @@
 
 const v8 = require('v8')
 const os = require('os')
+const process = require('process')
 const { DogStatsDClient, MetricsAggregationClient } = require('../dogstatsd')
 const log = require('../log')
-const { performance, PerformanceObserver } = require('perf_hooks')
+const { performance, PerformanceObserver, monitorEventLoopDelay } = require('perf_hooks')
 const { getEnvironmentVariable } = require('../config-helper')
 
-const { NODE_MAJOR, NODE_MINOR } = require('../../../../version')
+const { NODE_MAJOR } = require('../../../../version')
+// TODO: This environment variable may not be changed, since the agent expects a flush every ten seconds.
+// It is only a variable for testing. Think about alternatives.
 const DD_RUNTIME_METRICS_FLUSH_INTERVAL = getEnvironmentVariable('DD_RUNTIME_METRICS_FLUSH_INTERVAL') ?? '10000'
 const INTERVAL = Number.parseInt(DD_RUNTIME_METRICS_FLUSH_INTERVAL, 10)
 
-// Node >=16 has PerformanceObserver with `gc` type, but <16.7 had a critical bug.
-// See: https://github.com/nodejs/node/issues/39548
-const hasGCObserver = NODE_MAJOR >= 18 || (NODE_MAJOR === 16 && NODE_MINOR >= 7)
+const eventLoopDelayResolution = 4
 
 let nativeMetrics = null
 let gcObserver = null
-
-let interval
-let client
-let time
-let cpuUsage
-let elu
-
-reset()
-
-const runtimeMetrics = module.exports = {
+let interval = null
+let client = null
+let lastTime = 0n
+let lastCpuUsage = null
+let eventLoopDelayObserver = null
+
+// !!!!!!!!!!!
+//  IMPORTANT
+// !!!!!!!!!!!
+//
+// ALL metrics that relate to time are handled in nanoseconds in the backend.
+// https://github.com/DataDog/dogweb/blob/prod/integration/node/node_metadata.csv
+
+module.exports = {
   start (config) {
+    this.stop()
     const clientConfig = DogStatsDClient.generateClientConfig(config)
-    const watchers = []
 
-    if (config.runtimeMetrics.gc !== false) {
-      if (hasGCObserver) {
-        startGCObserver()
-      } else {
-        watchers.push('gc')
-      }
-    }
+    const trackEventLoop = config.runtimeMetrics.eventLoop !== false
+    const trackGc = config.runtimeMetrics.gc !== false
 
-    if (config.runtimeMetrics.eventLoop !== false) {
-      watchers.push('loop')
+    if (trackGc) {
+      startGCObserver()
     }
 
+    // Using no-gc prevents the native gc metrics from being tracked. Not
+    // passing any options means all metrics are tracked.
+    // TODO: This is a workaround. We should find a better solution.
+    const watchers = trackEventLoop ? ['loop'] : ['no-gc']
+
     try {
       nativeMetrics = require('@datadog/native-metrics')
       nativeMetrics.start(...watchers)
-    } catch (e) {
-      log.error('Error starting native metrics', e)
+    } catch (error) {
+      log.error('Error starting native metrics', error)
       nativeMetrics = null
     }
 
     client = new MetricsAggregationClient(new DogStatsDClient(clientConfig))
 
-    time = process.hrtime()
+    lastTime = performance.now()
 
     if (nativeMetrics) {
       interval = setInterval(() => {
-        captureCommonMetrics()
-        captureNativeMetrics()
+        captureNativeMetrics(trackEventLoop, trackGc)
+        captureCommonMetrics(trackEventLoop)
         client.flush()
       }, INTERVAL)
     } else {
-      cpuUsage = process.cpuUsage()
+      lastCpuUsage = process.cpuUsage()
+
+      if (trackEventLoop) {
+        eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+        eventLoopDelayObserver.enable()
+      }
 
       interval = setInterval(() => {
-        captureCommonMetrics()
         captureCpuUsage()
+        captureCommonMetrics(trackEventLoop)
         captureHeapSpace()
+        if (trackEventLoop) {
+          // Experimental: The Node.js implementation deviates from the native metrics.
+          // We normalize the metrics to the same format but the Node.js values
+          // are that way lower than they should be, while they are still nearer
+          // to the native ones that way.
+          // We use these only as fallback values.
+          captureEventLoopDelay()
+        }
         client.flush()
       }, INTERVAL)
     }
@@ -78,12 +96,21 @@ const runtimeMetrics = module.exports = {
   },
 
   stop () {
-    if (nativeMetrics) {
-      nativeMetrics.stop()
-    }
+    nativeMetrics?.stop()
+    nativeMetrics = null
 
     clearInterval(interval)
-    reset()
+    interval = null
+
+    client = null
+    lastTime = 0n
+    lastCpuUsage = null
+
+    gcObserver?.disconnect()
+    gcObserver = null
+
+    eventLoopDelayObserver?.disable()
+    eventLoopDelayObserver = null
   },
 
   track (span) {
@@ -99,19 +126,19 @@ const runtimeMetrics = module.exports = {
   },
 
   boolean (name, value, tag) {
-    client && client.boolean(name, value, tag)
+    client?.boolean(name, value, tag)
   },
 
   histogram (name, value, tag) {
-    client && client.histogram(name, value, tag)
+    client?.histogram(name, value, tag)
   },
 
   count (name, count, tag, monotonic = false) {
-    client && client.count(name, count, tag, monotonic)
+    client?.count(name, count, tag, monotonic)
   },
 
   gauge (name, value, tag) {
-    client && client.gauge(name, value, tag)
+    client?.gauge(name, value, tag)
   },
 
   increment (name, tag, monotonic) {
@@ -123,30 +150,20 @@ const runtimeMetrics = module.exports = {
   }
 }
 
-function reset () {
-  interval = null
-  client = null
-  time = null
-  cpuUsage = null
-  nativeMetrics = null
-  gcObserver && gcObserver.disconnect()
-  gcObserver = null
-}
-
 function captureCpuUsage () {
-  if (!process.cpuUsage) return
-
-  const elapsedTime = process.hrtime(time)
-  const elapsedUsage = process.cpuUsage(cpuUsage)
-
-  time = process.hrtime()
-  cpuUsage = process.cpuUsage()
-
-  const elapsedMs = elapsedTime[0] * 1000 + elapsedTime[1] / 1_000_000
-  const userPercent = 100 * elapsedUsage.user / 1000 / elapsedMs
-  const systemPercent = 100 * elapsedUsage.system / 1000 / elapsedMs
+  const currentCpuUsage = process.cpuUsage()
+  const elapsedUsageUser = currentCpuUsage.user - lastCpuUsage.user
+  const elapsedUsageSystem = currentCpuUsage.system - lastCpuUsage.system
+
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  const userPercent = elapsedUsageUser / elapsedUsDividedBy100
+  const systemPercent = elapsedUsageSystem / elapsedUsDividedBy100
   const totalPercent = userPercent + systemPercent
 
+  lastTime = currentTime
+  lastCpuUsage = currentCpuUsage
+
   client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
   client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
   client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
@@ -160,12 +177,44 @@ function captureMemoryUsage () {
   client.gauge('runtime.node.mem.rss', stats.rss)
   client.gauge('runtime.node.mem.total', os.totalmem())
   client.gauge('runtime.node.mem.free', os.freemem())
+  client.gauge('runtime.node.mem.external', stats.external)
+  // TODO: Add arrayBuffers to the metrics. That also requires the
+  // node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.mem.arrayBuffers', stats.arrayBuffers)
+}
 
-  stats.external && client.gauge('runtime.node.mem.external', stats.external)
+function captureUptime () {
+  // WARNING: lastTime must be updated in the same interval before this function is called!
+  // This is a faster `process.uptime()`.
+  client.gauge('runtime.node.process.uptime', Math.round((lastTime + 499) / 1000))
 }
 
-function captureProcess () {
-  client.gauge('runtime.node.process.uptime', Math.round(process.uptime()))
+function captureEventLoopDelay () {
+  eventLoopDelayObserver.disable()
+
+  if (eventLoopDelayObserver.count !== 0) {
+    const minimum = eventLoopDelayResolution * 1e6
+    const avg = Math.max(eventLoopDelayObserver.mean - minimum, 0)
+    const sum = Math.round(avg * eventLoopDelayObserver.count)
+
+    if (sum !== 0) {
+      // Normalize the metrics to the same format as the native metrics.
+      const stats = {
+        min: Math.max(eventLoopDelayObserver.min - minimum, 0),
+        max: Math.max(eventLoopDelayObserver.max - minimum, 0),
+        sum,
+        total: sum,
+        avg,
+        count: eventLoopDelayObserver.count,
+        p95: Math.max(eventLoopDelayObserver.percentile(95) - minimum, 0)
+      }
+
+      histogram('runtime.node.event_loop.delay', stats)
+    }
+  }
+  eventLoopDelayObserver = monitorEventLoopDelay({ resolution: eventLoopDelayResolution })
+  eventLoopDelayObserver.enable()
 }
 
 function captureHeapStats () {
@@ -176,14 +225,17 @@ function captureHeapStats () {
   client.gauge('runtime.node.heap.total_physical_size', stats.total_physical_size)
   client.gauge('runtime.node.heap.total_available_size', stats.total_available_size)
   client.gauge('runtime.node.heap.heap_size_limit', stats.heap_size_limit)
-
-  stats.malloced_memory && client.gauge('runtime.node.heap.malloced_memory', stats.malloced_memory)
-  stats.peak_malloced_memory && client.gauge('runtime.node.heap.peak_malloced_memory', stats.peak_malloced_memory)
+  client.gauge('runtime.node.heap.malloced_memory', stats.malloced_memory)
+  client.gauge('runtime.node.heap.peak_malloced_memory', stats.peak_malloced_memory)
+  // TODO: Add number_of_native_contexts and number_of_detached_contexts to the
+  // metrics. Those metrics allow to identify memory leaks. Adding them also
+  // requires the node_metadata.csv to be updated for the website.
+  //
+  // client.gauge('runtime.node.heap.number_of_native_contexts', stats.number_of_native_contexts)
+  // client.gauge('runtime.node.heap.number_of_detached_contexts', stats.number_of_detached_contexts)
 }
 
 function captureHeapSpace () {
-  if (!v8.getHeapSpaceStatistics) return
-
   const stats = v8.getHeapSpaceStatistics()
 
   for (let i = 0, l = stats.length; i < l; i++) {
@@ -197,55 +249,63 @@ function captureHeapSpace () {
 }
 
 /**
- * Gathers and reports Event Loop Utilization (ELU) since last run
+ * Gathers and reports Event Loop Utilization (ELU) since last run, or from the
+ * start of the process on first run.
  *
  * ELU is a measure of how busy the event loop is, like running JavaScript or
  * waiting on *Sync functions. The value is between 0 (idle) and 1 (exhausted).
- *
- * performance.eventLoopUtilization available in Node.js >= v14.10, >= v12.19, >= v16
  */
-let captureELU = () => {}
-if ('eventLoopUtilization' in performance) {
-  captureELU = () => {
-    // if elu is undefined (first run) the measurement is from start of process
-    elu = performance.eventLoopUtilization(elu)
+let lastElu = { idle: 0, active: 0 }
+function captureELU () {
+  const elu = performance.eventLoopUtilization()
 
-    client.gauge('runtime.node.event_loop.utilization', elu.utilization)
-  }
+  const idle = elu.idle - lastElu.idle
+  const active = elu.active - lastElu.active
+  const utilization = active / (idle + active)
+
+  lastElu = elu
+
+  client.gauge('runtime.node.event_loop.utilization', utilization)
 }
 
-function captureCommonMetrics () {
+function captureCommonMetrics (trackEventLoop) {
   captureMemoryUsage()
-  captureProcess()
+  captureUptime()
   captureHeapStats()
-  captureELU()
+  if (trackEventLoop) {
+    captureELU()
+  }
 }
 
-function captureNativeMetrics () {
+function captureNativeMetrics (trackEventLoop, trackGc) {
   const stats = nativeMetrics.stats()
   const spaces = stats.heap.spaces
-  const elapsedTime = process.hrtime(time)
 
-  time = process.hrtime()
+  const currentTime = performance.now() // Milliseconds with decimal places
+  const elapsedUsDividedBy100 = (currentTime - lastTime) * 10
+  lastTime = currentTime
 
-  const elapsedUs = elapsedTime[0] * 1e6 + elapsedTime[1] / 1e3
-  const userPercent = 100 * stats.cpu.user / elapsedUs
-  const systemPercent = 100 * stats.cpu.system / elapsedUs
+  const userPercent = stats.cpu.user / elapsedUsDividedBy100
+  const systemPercent = stats.cpu.system / elapsedUsDividedBy100
   const totalPercent = userPercent + systemPercent
 
   client.gauge('runtime.node.cpu.system', systemPercent.toFixed(2))
   client.gauge('runtime.node.cpu.user', userPercent.toFixed(2))
   client.gauge('runtime.node.cpu.total', totalPercent.toFixed(2))
 
-  histogram('runtime.node.event_loop.delay', stats.eventLoop)
+  if (trackEventLoop) {
+    histogram('runtime.node.event_loop.delay', stats.eventLoop)
+  }
 
-  Object.keys(stats.gc).forEach(type => {
-    if (type === 'all') {
-      histogram('runtime.node.gc.pause', stats.gc[type])
-    } else {
-      histogram('runtime.node.gc.pause.by.type', stats.gc[type], `gc_type:${type}`)
+  if (trackGc) {
+    for (const [type, value] of Object.entries(stats.gc)) {
+      if (type === 'all') {
+        histogram('runtime.node.gc.pause', value)
+      } else {
+        histogram('runtime.node.gc.pause.by.type', value, `gc_type:${type}`)
+      }
     }
-  })
+  }
 
   for (let i = 0, l = spaces.length; i < l; i++) {
     const tag = `heap_space:${spaces[i].space_name}`
@@ -258,13 +318,19 @@ function captureNativeMetrics () {
 }
 
 function histogram (name, stats, tag) {
+  if (stats.count === 0) {
+    return
+  }
   client.gauge(`${name}.min`, stats.min, tag)
   client.gauge(`${name}.max`, stats.max, tag)
   client.increment(`${name}.sum`, stats.sum, tag)
   client.increment(`${name}.total`, stats.sum, tag)
   client.gauge(`${name}.avg`, stats.avg, tag)
   client.increment(`${name}.count`, stats.count, tag)
-  client.gauge(`${name}.median`, stats.median, tag)
+  if (stats.median !== undefined) {
+    // TODO: Consider adding the median to the Node.js histogram/adding stddev to native metrics.
+    client.gauge(`${name}.median`, stats.median, tag)
+  }
   client.gauge(`${name}.95percentile`, stats.p95, tag)
 }
 
@@ -274,43 +340,27 @@ function startGCObserver () {
   gcObserver = new PerformanceObserver(list => {
     for (const entry of list.getEntries()) {
       const type = gcType(entry.detail?.kind || entry.kind)
+      const duration = entry.duration * 1_000_000
 
-      runtimeMetrics.histogram('runtime.node.gc.pause.by.type', entry.duration, `gc_type:${type}`)
-      runtimeMetrics.histogram('runtime.node.gc.pause', entry.duration)
+      // These are individual metrics for each type of GC.
+      client.histogram('runtime.node.gc.pause.by.type', duration, `gc_type:${type}`)
+      client.histogram('runtime.node.gc.pause', duration)
     }
   })
 
   gcObserver.observe({ type: 'gc' })
 }
 
+const minorGCType = NODE_MAJOR >= 22 ? 'minor_mark_sweep' : 'minor_mark_compact'
+
 function gcType (kind) {
-  if (NODE_MAJOR >= 22) {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'minor_mark_sweep'
-      case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
-      case 8: return 'incremental_marking'
-      case 16: return 'process_weak_callbacks'
-      case 31: return 'all'
-    }
-  } else if (NODE_MAJOR >= 18) {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'minor_mark_compact'
-      case 4: return 'mark_sweep_compact'
-      case 8: return 'incremental_marking'
-      case 16: return 'process_weak_callbacks'
-      case 31: return 'all'
-    }
-  } else {
-    switch (kind) {
-      case 1: return 'scavenge'
-      case 2: return 'mark_sweep_compact'
-      case 4: return 'incremental_marking'
-      case 8: return 'process_weak_callbacks'
-      case 15: return 'all'
-    }
+  switch (kind) {
+    case 1: return 'scavenge'
+    case 2: return minorGCType
+    case 4: return 'mark_sweep_compact' // Deprecated, might be removed soon.
+    case 8: return 'incremental_marking'
+    case 16: return 'process_weak_callbacks'
+    case 31: return 'all'
+    default: return 'unknown'
   }
-
-  return 'unknown'
 }

package/packages/dd-trace/src/supported-configurations.json

@@ -8,6 +8,8 @@
     "DD_API_KEY": ["A"],
     "DD_API_SECURITY_ENABLED": ["A"],
     "DD_API_SECURITY_SAMPLE_DELAY": ["A"],
+    "DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED": ["A"],
+    "DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT": ["A"],
     "DD_APM_FLUSH_DEADLINE_MILLISECONDS": ["A"],
     "DD_APM_TRACING_ENABLED": ["A"],
     "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": ["A"],
@@ -29,7 +31,6 @@
     "DD_APPSEC_STACK_TRACE_ENABLED": ["A"],
     "DD_APPSEC_TRACE_RATE_LIMIT": ["A"],
     "DD_APPSEC_WAF_TIMEOUT": ["A"],
-    "DD_AZURE_APP_SERVICES": ["A"],
     "DD_CIVISIBILITY_AGENTLESS_ENABLED": ["A"],
     "DD_CIVISIBILITY_AGENTLESS_URL": ["A"],
     "DD_CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER": ["A"],

package/packages/dd-trace/src/telemetry/endpoints.js

@@ -0,0 +1,124 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const { sendData } = require('./send-data')
+
+const fastifyRouteCh = dc.channel('apm:fastify:route:added')
+
+let config
+let application
+let host
+let getRetryData
+let updateRetryData
+
+/**
+ * Keep track of endpoints that still need to be sent.
+ * Map key is `${METHOD} ${PATH}`, value is { method, path }
+ */
+const pendingEndpoints = new Map()
+let flushScheduled = false
+let isFirstPayload = true
+
+function endpointKey (method, path) {
+  return `${method.toUpperCase()} ${path}`
+}
+
+function scheduleFlush () {
+  if (flushScheduled) return
+  flushScheduled = true
+  setImmediate(flushAndSend).unref()
+}
+
+function recordEndpoint (method, path) {
+  const key = endpointKey(method, path)
+  if (pendingEndpoints.has(key)) return
+
+  pendingEndpoints.set(key, { method: method.toUpperCase(), path })
+  scheduleFlush()
+}
+
+function onFastifyRoute (routeData) {
+  const routeOptions = routeData?.routeOptions
+  if (!routeOptions?.path) return
+
+  const methods = Array.isArray(routeOptions.method) ? routeOptions.method : [routeOptions.method]
+
+  for (const method of methods) {
+    recordEndpoint(method, routeOptions.path)
+  }
+}
+
+function buildEndpointObjects (endpoints) {
+  return endpoints.map(({ method, path }) => {
+    return {
+      type: 'REST',
+      method,
+      path,
+      operation_name: 'http.request',
+      resource_name: endpointKey(method, path)
+    }
+  })
+}
+
+function flushAndSend () {
+  flushScheduled = false
+  if (pendingEndpoints.size === 0) return
+
+  const batchEndpoints = []
+  for (const [key, endpoint] of pendingEndpoints) {
+    batchEndpoints.push(endpoint)
+    pendingEndpoints.delete(key)
+    if (batchEndpoints.length >= config.appsec?.apiSecurity?.endpointCollectionMessageLimit) break
+  }
+
+  const payloadObj = {
+    is_first: isFirstPayload,
+    endpoints: buildEndpointObjects(batchEndpoints)
+  }
+
+  let reqType = 'app-endpoints'
+  let payload = payloadObj
+
+  const retryData = getRetryData()
+  if (retryData) {
+    payload = [
+      { request_type: 'app-endpoints', payload: payloadObj },
+      { request_type: retryData.reqType, payload: retryData.payload }
+    ]
+    reqType = 'message-batch'
+  }
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
+
+  if (isFirstPayload) {
+    isFirstPayload = false
+  }
+
+  // If more endpoints accumulated while sending, schedule another flush.
+  if (pendingEndpoints.size) scheduleFlush()
+}
+
+function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDataFunction) {
+  if (!_config?.appsec?.apiSecurity?.endpointCollectionEnabled) return
+
+  config = _config
+  application = _application
+  host = _host
+  getRetryData = getRetryDataFunction
+  updateRetryData = updateRetryDataFunction
+
+  fastifyRouteCh.subscribe(onFastifyRoute)
+}
+
+function stop () {
+  fastifyRouteCh.unsubscribe(onFastifyRoute)
+
+  pendingEndpoints.clear()
+  flushScheduled = false
+  config = application = host = getRetryData = updateRetryData = null
+}
+
+module.exports = {
+  start,
+  stop
+}

package/packages/dd-trace/src/telemetry/telemetry.js

@@ -3,6 +3,7 @@ const tracerVersion = require('../../../../package.json').version
 const dc = require('dc-polyfill')
 const os = require('os')
 const dependencies = require('./dependencies')
+const endpoints = require('./endpoints')
 const { sendData } = require('./send-data')
 const { errors } = require('../startup-log')
 const { manager: metricsManager } = require('./metrics')
@@ -254,6 +255,7 @@ function start (aConfig, thePluginManager) {
 
   dependencies.start(config, application, host, getRetryData, updateRetryData)
   telemetryLogger.start(config)
+  endpoints.start(config, application, host, getRetryData, updateRetryData)
 
   sendData(config, application, host, 'app-started', appStarted(config))
 
@@ -280,6 +282,7 @@ function stop () {
 
   telemetryStopChannel.publish(getTelemetryData())
 
+  endpoints.stop()
   config = undefined
 }