dd-trace 3.42.0 → 3.43.0

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -18,30 +18,42 @@ class WAFContextWrapper {
     this.addressesToSkip = new Set()
   }
 
-  run (params) {
+  run ({ persistent, ephemeral }) {
+    const payload = {}
+    let payloadHasData = false
     const inputs = {}
-    let someInputAdded = false
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    // TODO: possible optimizaion: only send params that haven't already been sent with same value to this wafContext
-    for (const key of Object.keys(params)) {
-      // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
-      // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
-      if (!this.addressesToSkip.has(key)) {
-        inputs[key] = params[key]
-        if (preventDuplicateAddresses.has(key)) {
-          newAddressesToSkip.add(key)
+    if (persistent && typeof persistent === 'object') {
+      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      for (const key of Object.keys(persistent)) {
+        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
+        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
+        if (!this.addressesToSkip.has(key)) {
+          inputs[key] = persistent[key]
+          if (preventDuplicateAddresses.has(key)) {
+            newAddressesToSkip.add(key)
+          }
         }
-        someInputAdded = true
       }
     }
 
-    if (!someInputAdded) return
+    if (Object.keys(inputs).length) {
+      payload['persistent'] = inputs
+      payloadHasData = true
+    }
+
+    if (ephemeral && Object.keys(ephemeral).length) {
+      payload['ephemeral'] = ephemeral
+      payloadHasData = true
+    }
+
+    if (!payloadHasData) return
 
     try {
       const start = process.hrtime.bigint()
 
-      const result = this.ddwafContext.run(inputs, this.wafTimeout)
+      const result = this.ddwafContext.run(payload, this.wafTimeout)
 
       const end = process.hrtime.bigint()
 

package/packages/dd-trace/src/config.js

@@ -435,6 +435,10 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maybeFile(appsec.blockedTemplateJson),
       maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)
     )
+    const DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = coalesce(
+      maybeFile(appsec.blockedTemplateGraphql),
+      maybeFile(process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)
+    )
     const DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = coalesce(
       appsec.eventTracking && appsec.eventTracking.mode,
       process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING,
@@ -644,6 +648,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
       blockedTemplateHtml: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML,
       blockedTemplateJson: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON,
+      blockedTemplateGraphql: DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON,
       eventTracking: {
         enabled: ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING),
         mode: DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING

package/packages/dd-trace/src/plugins/util/user-provided-git.js

@@ -27,10 +27,11 @@ function removeEmptyValues (tags) {
   }, {})
 }
 
-// The regex is extracted from
+// The regex is inspired by
 // https://github.com/jonschlinkert/is-git-url/blob/396965ffabf2f46656c8af4c47bef1d69f09292e/index.js#L9C15-L9C87
+// The `.git` suffix is optional in this version
 function validateGitRepositoryUrl (repoUrl) {
-  return /(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|#[-\d\w._]+?)$/.test(repoUrl)
+  return /(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\/?|#[-\d\w._]+?)$/.test(repoUrl)
 }
 
 function validateGitCommitSha (gitCommitSha) {

package/packages/dd-trace/src/profiling/profiler.js

@@ -61,6 +61,7 @@ class Profiler extends EventEmitter {
     }
 
     try {
+      const start = new Date()
       for (const profiler of config.profilers) {
         // TODO: move this out of Profiler when restoring sourcemap support
         profiler.start({
@@ -70,7 +71,7 @@ class Profiler extends EventEmitter {
         this._logger.debug(`Started ${profiler.type} profiler`)
       }
 
-      this._capture(this._timeoutInterval)
+      this._capture(this._timeoutInterval, start)
       return true
     } catch (e) {
       this._logger.error(e)
@@ -116,9 +117,9 @@ class Profiler extends EventEmitter {
     return this
   }
 
-  _capture (timeout) {
+  _capture (timeout, start) {
     if (!this._enabled) return
-    this._lastStart = new Date()
+    this._lastStart = start
     if (!this._timer || timeout !== this._timeoutInterval) {
       this._timer = setTimeout(() => this._collect(snapshotKinds.PERIODIC), timeout)
       this._timer.unref()
@@ -138,7 +139,7 @@ class Profiler extends EventEmitter {
     try {
       // collect profiles synchronously so that profilers can be safely stopped asynchronously
       for (const profiler of this._config.profilers) {
-        const profile = profiler.profile()
+        const profile = profiler.profile(start, end)
         if (!profile) continue
         profiles.push({ profiler, profile })
       }
@@ -154,7 +155,7 @@ class Profiler extends EventEmitter {
         })
       }
 
-      this._capture(this._timeoutInterval)
+      this._capture(this._timeoutInterval, end)
       await this._submit(encodedProfiles, start, end, snapshotKind)
       this._logger.debug('Submitted profiles')
     } catch (err) {
@@ -201,7 +202,7 @@ class ServerlessProfiler extends Profiler {
       await super._collect(snapshotKind)
     } else {
       this._profiledIntervals += 1
-      this._capture(this._timeoutInterval)
+      this._capture(this._timeoutInterval, new Date())
       // Don't submit profile until 65 (flushAfterIntervals) intervals have elapsed
     }
   }

package/packages/dd-trace/src/profiling/profilers/events.js

@@ -1,5 +1,5 @@
 const { performance, constants, PerformanceObserver } = require('node:perf_hooks')
-const { END_TIMESTAMP } = require('./shared')
+const { END_TIMESTAMP_LABEL } = require('./shared')
 const semver = require('semver')
 const { Function, Label, Line, Location, Profile, Sample, StringTable, ValueType } = require('pprof-format')
 const pprof = require('@datadog/pprof/')
@@ -174,7 +174,7 @@ class EventsProfiler {
     }
   }
 
-  profile () {
+  profile (startDate, endDate) {
     if (this.entries.length === 0) {
       // No events in the period; don't produce a profile
       return null
@@ -202,12 +202,11 @@ class EventsProfiler {
       decorator.eventTypeLabel = labelFromStrStr(stringTable, 'event', eventType)
       decorators[eventType] = decorator
     }
-    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP)
+    const timestampLabelKey = stringTable.dedup(END_TIMESTAMP_LABEL)
 
-    let durationFrom = Number.POSITIVE_INFINITY
-    let durationTo = 0
     const dateOffset = BigInt(Math.round(performance.timeOrigin * MS_TO_NS))
-
+    const lateEntries = []
+    const perfEndDate = endDate.getTime() - performance.timeOrigin
     const samples = this.entries.map((item) => {
       const decorator = decorators[item.entryType]
       if (!decorator) {
@@ -216,9 +215,15 @@ class EventsProfiler {
         return null
       }
       const { startTime, duration } = item
+      if (startTime >= perfEndDate) {
+        // An event past the current recording end date; save it for the next
+        // profile. Not supposed to happen as long as there's no async activity
+        // between capture of the endDate value in profiler.js _collect() and
+        // here, but better be safe than sorry.
+        lateEntries.push(item)
+        return null
+      }
       const endTime = startTime + duration
-      if (durationFrom > startTime) durationFrom = startTime
-      if (durationTo < endTime) durationTo = endTime
       const sampleInput = {
         value: [Math.round(duration * MS_TO_NS)],
         locationId,
@@ -231,7 +236,7 @@ class EventsProfiler {
       return new Sample(sampleInput)
     }).filter(v => v)
 
-    this.entries = []
+    this.entries = lateEntries
 
     const timeValueType = new ValueType({
       type: stringTable.dedup(pprofValueType),
@@ -240,10 +245,10 @@ class EventsProfiler {
 
     return new Profile({
       sampleType: [timeValueType],
-      timeNanos: dateOffset + BigInt(Math.round(durationFrom * MS_TO_NS)),
+      timeNanos: endDate.getTime() * MS_TO_NS,
       periodType: timeValueType,
-      period: this._flushIntervalNanos,
-      durationNanos: Math.max(0, Math.round((durationTo - durationFrom) * MS_TO_NS)),
+      period: 1,
+      durationNanos: (endDate.getTime() - startDate.getTime()) * MS_TO_NS,
       sample: samples,
       location: locations,
       function: functions,

package/packages/dd-trace/src/profiling/profilers/shared.js

@@ -2,8 +2,38 @@
 
 const { isMainThread, threadId } = require('node:worker_threads')
 
+const END_TIMESTAMP_LABEL = 'end_timestamp_ns'
+const THREAD_NAME_LABEL = 'thread name'
+const OS_THREAD_ID_LABEL = 'os thread id'
+const THREAD_ID_LABEL = 'thread id'
+const threadNamePrefix = isMainThread ? 'Main' : `Worker #${threadId}`
+const eventLoopThreadName = `${threadNamePrefix} Event Loop`
+
+function getThreadLabels () {
+  const pprof = require('@datadog/pprof')
+  const nativeThreadId = pprof.getNativeThreadId()
+  return {
+    [THREAD_NAME_LABEL]: eventLoopThreadName,
+    [THREAD_ID_LABEL]: `${threadId}`,
+    [OS_THREAD_ID_LABEL]: `${nativeThreadId}`
+  }
+}
+
+function cacheThreadLabels () {
+  let labels
+  return () => {
+    if (!labels) {
+      labels = getThreadLabels()
+    }
+    return labels
+  }
+}
+
 module.exports = {
-  END_TIMESTAMP: 'end_timestamp_ns',
-  THREAD_NAME: 'thread name',
-  threadNamePrefix: isMainThread ? 'Main' : `Worker #${threadId}`
+  END_TIMESTAMP_LABEL,
+  THREAD_NAME_LABEL,
+  THREAD_ID_LABEL,
+  threadNamePrefix,
+  eventLoopThreadName,
+  getThreadLabels: cacheThreadLabels()
 }

package/packages/dd-trace/src/profiling/profilers/space.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { oomExportStrategies } = require('../constants')
+const { getThreadLabels } = require('./shared')
 
 function strategiesToCallbackMode (strategies, callbackMode) {
   return strategies.includes(oomExportStrategies.ASYNC_CALLBACK) ? callbackMode.Async : 0
@@ -33,7 +34,7 @@ class NativeSpaceProfiler {
   }
 
   profile () {
-    return this._pprof.heap.profile(undefined, this._mapper)
+    return this._pprof.heap.profile(undefined, this._mapper, getThreadLabels)
   }
 
   encode (profile) {

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -7,13 +7,12 @@ const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
-const { END_TIMESTAMP, THREAD_NAME, threadNamePrefix } = require('./shared')
+const { END_TIMESTAMP_LABEL, getThreadLabels } = require('./shared')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
 const spanFinishCh = dc.channel('dd-trace:span:finish')
 const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers')
-const threadName = `${threadNamePrefix} Event Loop`
 
 const MemoizedWebTags = Symbol('NativeWallProfiler.MemoizedWebTags')
 
@@ -96,12 +95,9 @@ class NativeWallProfiler {
         this._enter = this._enter.bind(this)
         this._spanFinished = this._spanFinished.bind(this)
       }
-      this._generateLabels = this._generateLabels.bind(this)
-    } else {
-      // Explicitly assigning, to express the intent that this is meant to be
-      // undefined when passed to pprof.time.stop() when not using sample contexts.
-      this._generateLabels = undefined
     }
+    this._generateLabels = this._generateLabels.bind(this)
+
     this._logger = options.logger
     this._started = false
   }
@@ -239,12 +235,21 @@ class NativeWallProfiler {
     return profile
   }
 
-  _generateLabels ({ context: { spanId, rootSpanId, webTags, endpoint }, timestamp }) {
-    const labels = this._timelineEnabled ? {
-      [THREAD_NAME]: threadName,
+  _generateLabels (context) {
+    if (context == null) {
+      // generateLabels is also called for samples without context.
+      // In that case just return thread labels.
+      return getThreadLabels()
+    }
+
+    const labels = { ...getThreadLabels() }
+
+    const { context: { spanId, rootSpanId, webTags, endpoint }, timestamp } = context
+
+    if (this._timelineEnabled) {
       // Incoming timestamps are in microseconds, we emit nanos.
-      [END_TIMESTAMP]: timestamp * 1000n
-    } : {}
+      labels[END_TIMESTAMP_LABEL] = timestamp * 1000n
+    }
 
     if (spanId) {
       labels['span id'] = spanId

package/packages/dd-trace/src/proxy.js

@@ -36,6 +36,10 @@ class Tracer extends NoopProxy {
         setInterval(() => {
           this.dogstatsd.flush()
         }, 10 * 1000).unref()
+
+        process.once('beforeExit', () => {
+          this.dogstatsd.flush()
+        })
       }
 
       if (config.spanLeakDebug > 0) {