dd-trace 6.20.1 → 6.22.0

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -22,6 +22,7 @@ const { getLocalStateForCallFrame, evaluateCaptureExpressions } = require('./sna
 const send = require('./send')
 const { getStackFromCallFrames } = require('./state')
 const { ackEmitting } = require('./status')
+const WorkerError = require('./worker-error')
 const config = require('./config')
 const log = require('./log')
 
@@ -51,9 +52,10 @@ session.on('Debugger.paused', async ({ params }) => {
 
   if (params.reason !== 'other') {
     // This error should not be caught, and should exit the worker thread, effectively stopping the debugging session
-    throw Object.assign(new Error(`Unexpected Debugger.paused reason: ${params.reason}`), {
-      reason: WORKER_ERROR_REASON.UNEXPECTED_PAUSE_REASON,
-    })
+    throw new WorkerError(
+      `Unexpected Debugger.paused reason: ${params.reason}`,
+      WORKER_ERROR_REASON.UNEXPECTED_PAUSE_REASON
+    )
   }
 
   let maxReferenceDepth = 0

package/packages/dd-trace/src/debugger/devtools_client/remote_config.js

@@ -4,6 +4,7 @@ const { workerData: { probePort } } = require('node:worker_threads')
 const { WORKER_ERROR_REASON } = require('../constants')
 const { addBreakpoint, removeBreakpoint, modifyBreakpoint } = require('./breakpoints')
 const { ackReceived, ackInstalled, ackError } = require('./status')
+const WorkerError = require('./worker-error')
 const log = require('./log')
 
 // Example log line probe with captureSnapshot (simplified):
@@ -74,20 +75,23 @@ async function processMsg (action, probe) {
   if (action !== 'unapply') ackReceived(probe)
 
   if (probe.type !== 'LOG_PROBE') {
-    throw Object.assign(new Error(
-      `Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`
-    ), { reason: WORKER_ERROR_REASON.UNSUPPORTED_PROBE_TYPE })
+    throw new WorkerError(
+      `Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`,
+      WORKER_ERROR_REASON.UNSUPPORTED_PROBE_TYPE
+    )
   }
   if (!probe.where.sourceFile && !probe.where.lines) {
-    throw Object.assign(new Error(
+    throw new WorkerError(
       // eslint-disable-next-line @stylistic/max-len
-      `Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})`
-    ), { reason: WORKER_ERROR_REASON.UNSUPPORTED_INSERTION_POINT })
+      `Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})`,
+      WORKER_ERROR_REASON.UNSUPPORTED_INSERTION_POINT
+    )
   }
   if (probe.captureSnapshot && probe.captureExpressions?.length > 0) {
-    throw Object.assign(new Error(
-      `Cannot set both captureSnapshot and captureExpressions (probe: ${probe.id}, version: ${probe.version})`
-    ), { reason: WORKER_ERROR_REASON.CONFLICTING_CAPTURE_OPTIONS })
+    throw new WorkerError(
+      `Cannot set both captureSnapshot and captureExpressions (probe: ${probe.id}, version: ${probe.version})`,
+      WORKER_ERROR_REASON.CONFLICTING_CAPTURE_OPTIONS
+    )
   }
 
   switch (action) {
@@ -103,8 +107,9 @@ async function processMsg (action, probe) {
       ackInstalled(probe)
       break
     default:
-      throw Object.assign(new Error(
-        `Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}`
-      ), { reason: WORKER_ERROR_REASON.UNKNOWN_REMOTE_CONFIG_ACTION })
+      throw new WorkerError(
+        `Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}`,
+        WORKER_ERROR_REASON.UNKNOWN_REMOTE_CONFIG_ACTION
+      )
   }
 }

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -17,8 +17,12 @@ const buildTags = require('./tags')
 
 module.exports = send
 
-const MAX_LOG_PAYLOAD_SIZE_MB = 1
-const MAX_LOG_PAYLOAD_SIZE_BYTES = MAX_LOG_PAYLOAD_SIZE_MB * 1024 * 1024
+// The backend caps a single debugger event at 1 MiB, but measures the CBOR-encoded event *after* the
+// intake has enriched it with host tags, `ddtags` and other metadata we cannot size from here. Leave
+// the same 4 KiB of headroom as dd-trace-go: crossing the backend cap is worse than pruning, since
+// its truncation destroys the log message before it drops the snapshot.
+const BACKEND_ENRICHMENT_HEADROOM_BYTES = 4 * 1024
+const MAX_LOG_PAYLOAD_SIZE_BYTES = 1024 * 1024 - BACKEND_ENRICHMENT_HEADROOM_BYTES
 
 const ddsource = 'dd_debugger'
 const hostname = getHostname()
@@ -57,6 +61,7 @@ function send (message, logger, dd, snapshot, processTags, eventType, incomplete
     ddsource,
     hostname,
     service,
+    timestamp: snapshot.timestamp,
     message,
     logger,
     dd,

package/packages/dd-trace/src/debugger/devtools_client/status.js

@@ -129,6 +129,7 @@ function statusPayload (probeId, probeVersion, status) {
   return {
     ddsource,
     service,
+    timestamp: Date.now(),
     debugger: {
       diagnostics: { probeId, runtimeId, probeVersion, status },
     },

package/packages/dd-trace/src/debugger/devtools_client/worker-error.js

@@ -0,0 +1,21 @@
+'use strict'
+
+/** @typedef {typeof import('../constants').WORKER_ERROR_REASON} WorkerErrorReasons */
+
+/**
+ * An error carrying a failure reason for telemetry. Its prototype does not survive the worker boundary, so the main
+ * thread must read `reason` instead of checking `instanceof`. `name` is left as `Error` because structured cloning
+ * only keeps built-in error names, and telemetry should report the same name however the error leaves the worker.
+ */
+class WorkerError extends Error {
+  /**
+   * @param {string} message - The error message
+   * @param {WorkerErrorReasons[keyof WorkerErrorReasons]} reason - The failure reason reported in telemetry
+   */
+  constructor (message, reason) {
+    super(message)
+    this.reason = reason
+  }
+}
+
+module.exports = WorkerError

package/packages/dd-trace/src/debugger/guardrail-metrics.js

@@ -9,10 +9,10 @@
  * atomics, and the main thread periodically drains the counters into instrumentation telemetry metrics.
  *
  * The metric names, tags and reason codes are defined by the "Debugger Observability for GA" specification and are
- * shared with the other tracers. All metrics are counts in the `live_debugger` telemetry namespace.
+ * shared with the other tracers. All metrics are counts in the `debugger` telemetry namespace.
  */
 
-const TELEMETRY_NAMESPACE = 'live_debugger'
+const TELEMETRY_NAMESPACE = 'debugger'
 
 const EVENT_TYPE = Object.freeze({
   SNAPSHOT: 0,

package/packages/dd-trace/src/debugger/index.js

@@ -8,6 +8,7 @@ const dc = require('dc-polyfill')
 const log = require('../log')
 const { fetchAgentInfo } = require('../agent/info')
 const telemetryMetrics = require('../telemetry/metrics')
+const { logGenericError } = require('../startup-log')
 const getDebuggerConfig = require('./config')
 const {
   DEBUGGER_DIAGNOSTICS_V1,
@@ -15,6 +16,7 @@ const {
   DEBUGGER_INPUT_V2,
   INSPECT_SEGMENT_GLOBAL_PROPERTY,
   METRICS_FLUSH_INTERVAL_MS,
+  WORKER_ERROR_REASON,
 } = require('./constants')
 const { GuardrailMetrics, TELEMETRY_NAMESPACE } = require('./guardrail-metrics')
 const { PauseDurationHistogram } = require('./pause-duration-histogram')
@@ -33,6 +35,8 @@ const { configureProbeSampler, installProbeSampler, uninstallProbeSampler } = re
 // be lost when the application exits on its own.
 const TELEMETRY_APP_CLOSING_CHANNEL = 'datadog:telemetry:app-closing'
 
+const WORKER_FILENAME = join(__dirname, 'devtools_client', 'index.js')
+
 let worker = null
 let configChannel = null
 let ackId = 0
@@ -139,7 +143,7 @@ function start (config, rcInstance) {
     inputPath = _inputPath
 
     worker = new Worker(
-      join(__dirname, 'devtools_client', 'index.js'),
+      WORKER_FILENAME,
       {
         name: 'dd-debugger',
         execArgv: [], // Avoid worker thread inheriting the `-r` command line argument
@@ -165,7 +169,7 @@ function start (config, rcInstance) {
       )
     })
 
-    worker.on('error', (err) => logWorkerError(err))
+    worker.on('error', onWorkerError)
     worker.on('messageerror', (err) => log.error('[debugger] received "messageerror" from worker', err))
 
     worker.once('exit', (code) => {
@@ -186,6 +190,24 @@ function start (config, rcInstance) {
   })
 }
 
+/**
+ * @param {Error & { code?: unknown, requireStack?: unknown }} error - The error emitted by the worker thread
+ */
+function onWorkerError (error) {
+  // Node.js reports an empty require stack when the module it cannot find is the worker entry point itself, rather
+  // than one of its dependencies. This happens when the dd-trace files are not on disk where dd-trace expects them,
+  // e.g. because dd-trace was bundled into the application, or because some of its files were not deployed.
+  if (error.code === 'MODULE_NOT_FOUND' && Array.isArray(error.requireStack) && error.requireStack.length === 0) {
+    logGenericError(
+      `Dynamic Instrumentation is disabled, because its worker thread could not load ${WORKER_FILENAME}. ` +
+      'Make sure dd-trace is installed as regular files on disk and is not bundled into the application.'
+    )
+    logWorkerError(error, WORKER_ERROR_REASON.WORKER_ENTRY_POINT_NOT_FOUND)
+    return
+  }
+  logWorkerError(error)
+}
+
 /**
  * Failure metadata belongs in the telemetry message; exception messages remain in the redacted cause.
  *

package/packages/dd-trace/src/debugger/probe_sampler.js

@@ -2,6 +2,7 @@
 
 const { types } = require('node:util')
 
+const { storage } = require('../../../datadog-core')
 const { MAX_SNAPSHOTS_PER_SECOND_GLOBALLY } = require('./devtools_client/defaults')
 const { MAX_MESSAGE_LENGTH } = require('./constants')
 const { EVENT_TYPE, SKIPPED_REASON } = require('./guardrail-metrics')
@@ -32,6 +33,20 @@ const ddTraceGlobal = /** @type {Record<symbol, SharedArrayBuffer | object | und
  * @property {{ DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS: number }} dynamicInstrumentation
  */
 
+/**
+ * The trace object shared by all spans of a trace in this process (`DatadogSpanContext#_trace`).
+ *
+ * @typedef {{ started: object[] }} LocalTrace
+ */
+
+/**
+ * The span active in the current async context, as stored in the legacy storage by the tracer.
+ *
+ * @typedef {{ context: () => { _trace: LocalTrace } }} ActiveSpan
+ */
+
+const legacyStorage = storage('legacy')
+
 let evaluationTimeoutNs = 0n
 
 module.exports = {
@@ -63,6 +78,14 @@ function installProbeSampler (guardrailMetrics, config) {
   const buffer = createProbeSamplerBuffer()
 
   const lastCaptureNsByProbeId = new Map()
+  /**
+   * The coordinated sampling decision of each trace in which a snapshot-producing probe was hit: `false` if the trace
+   * was dropped, otherwise the sampling indexes of the probes that already emitted a snapshot in it. Keyed weakly, so
+   * a decision is released together with its trace.
+   *
+   * @type {WeakMap<LocalTrace, Set<number> | false>}
+   */
+  const emittedProbesByTrace = new WeakMap()
   /**
    * Probes that are skipped at entry because a recent evaluation failed or exceeded its time budget. One error result
    * is reported per throttle window, so the probe stays visible without repeatedly paying for the evaluation.
@@ -219,7 +242,10 @@ function installProbeSampler (guardrailMetrics, config) {
   }
 
   /**
-   * Apply the per-probe and global rate limits and store the sampled probe index for the debugger worker.
+   * Decide if a probe hit should be sampled and store the sampled probe index for the debugger worker.
+   *
+   * Snapshot-producing probes hit within an active trace share one sampling decision per trace, so a trace emits the
+   * snapshots of all of its probes or none of them. Other hits are sampled independently.
    *
    * @param {number} probeIndex - The worker-side probe sampling index.
    * @param {string} probeId - The probe id.
@@ -228,29 +254,100 @@ function installProbeSampler (guardrailMetrics, config) {
    * @param {boolean} isSnapshotProducingProbe - Whether this probe counts toward the global snapshot sample limit.
    */
   function sample (probeIndex, probeId, now, nsBetweenSampling, isSnapshotProducingProbe) {
+    if (isSnapshotProducingProbe === true) {
+      const span = /** @type {ActiveSpan | null | undefined} */ (legacyStorage.getStore()?.span)
+      const trace = span?.context()._trace
+      // The span processor empties `started` once all spans of a trace have finished. A probe hit under such a trace
+      // runs in a context that outlived it, e.g. a connection pool or event emitter callback bound to a request that
+      // has since ended. Coordinating that hit would let a decision made for the old request silence the probe, or
+      // limit it to a single snapshot, for as long as the context lives, so it's sampled independently instead.
+      if (trace !== undefined && trace.started.length !== 0) {
+        return sampleInTrace(trace, probeIndex, probeId, now, nsBetweenSampling)
+      }
+    }
+
+    if (isRateLimited(probeId, now, nsBetweenSampling, isSnapshotProducingProbe)) return false
+    return commitSample(probeIndex, probeId, now, isSnapshotProducingProbe)
+  }
+
+  /**
+   * Sample a snapshot-producing probe hit using the sampling decision of its trace. The first such probe hit in the
+   * trace makes the decision for all of them, subject to its per-probe rate limit and the global snapshot rate limit.
+   * If the trace is sampled, each probe then emits once in it without being subject to the rate limits, so the trace
+   * emits a complete set of snapshots. Those snapshots still count toward the rate limits, which keeps the overall
+   * volume close to the limits by making later traces less likely to be sampled.
+   *
+   * @param {LocalTrace} trace - The trace the probe was hit in.
+   * @param {number} probeIndex - The worker-side probe sampling index.
+   * @param {string} probeId - The probe id.
+   * @param {bigint} now - The current time.
+   * @param {bigint} nsBetweenSampling - Minimum nanoseconds between samples for this probe.
+   */
+  function sampleInTrace (trace, probeIndex, probeId, now, nsBetweenSampling) {
+    const emittedProbes = emittedProbesByTrace.get(trace)
+
+    if (emittedProbes === undefined) {
+      if (isRateLimited(probeId, now, nsBetweenSampling, true)) {
+        emittedProbesByTrace.set(trace, false)
+        return false
+      }
+      // If the sampled probe can't be handed over, no decision is made, so the next probe hit in the trace makes it
+      if (!commitSample(probeIndex, probeId, now, true)) return false
+      emittedProbesByTrace.set(trace, new Set([probeIndex]))
+      return true
+    }
+
+    if (emittedProbes === false || emittedProbes.has(probeIndex)) {
+      guardrailMetrics.eventSkipped(SKIPPED_REASON.RATE_LIMIT_PROBE, EVENT_TYPE.SNAPSHOT)
+      return false
+    }
+
+    if (!commitSample(probeIndex, probeId, now, true)) return false
+    emittedProbes.add(probeIndex)
+    return true
+  }
+
+  /**
+   * Check the per-probe and global rate limits, recording the skip if a limit is reached.
+   *
+   * @param {string} probeId - The probe id.
+   * @param {bigint} now - The current time.
+   * @param {bigint} nsBetweenSampling - Minimum nanoseconds between samples for this probe.
+   * @param {boolean} isSnapshotProducingProbe - Whether this probe is subject to the global snapshot sample limit.
+   */
+  function isRateLimited (probeId, now, nsBetweenSampling, isSnapshotProducingProbe) {
     const lastCaptureNs = lastCaptureNsByProbeId.get(probeId)
     if (lastCaptureNs !== undefined && now - lastCaptureNs < nsBetweenSampling) {
       guardrailMetrics.eventSkipped(
         SKIPPED_REASON.RATE_LIMIT_PROBE,
         isSnapshotProducingProbe === true ? EVENT_TYPE.SNAPSHOT : EVENT_TYPE.LOG
       )
-      return false
+      return true
     }
 
-    let shouldResetGlobalSnapshotRateWindow = false
-    if (isSnapshotProducingProbe === true) {
-      if (now - globalSnapshotSamplingRateWindowStart > oneSecondNs) {
-        shouldResetGlobalSnapshotRateWindow = true
-      } else if (snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
-        guardrailMetrics.eventSkipped(SKIPPED_REASON.RATE_LIMIT_GLOBAL, EVENT_TYPE.SNAPSHOT)
-        return false
-      }
+    if (isSnapshotProducingProbe === true &&
+        now - globalSnapshotSamplingRateWindowStart <= oneSecondNs &&
+        snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
+      guardrailMetrics.eventSkipped(SKIPPED_REASON.RATE_LIMIT_GLOBAL, EVENT_TYPE.SNAPSHOT)
+      return true
     }
 
+    return false
+  }
+
+  /**
+   * Store the sampled probe index for the debugger worker and count the sample toward the rate limits.
+   *
+   * @param {number} probeIndex - The worker-side probe sampling index.
+   * @param {string} probeId - The probe id.
+   * @param {bigint} now - The current time.
+   * @param {boolean} isSnapshotProducingProbe - Whether this probe counts toward the global snapshot sample limit.
+   */
+  function commitSample (probeIndex, probeId, now, isSnapshotProducingProbe) {
     if (!storeSampledProbeIndex(probeIndex)) return false
 
     if (isSnapshotProducingProbe === true) {
-      if (shouldResetGlobalSnapshotRateWindow === true) {
+      if (now - globalSnapshotSamplingRateWindowStart > oneSecondNs) {
         snapshotsSampledWithinTheLastSecond = 1
         globalSnapshotSamplingRateWindowStart = now
       } else {

package/packages/dd-trace/src/dogstatsd.js

@@ -533,7 +533,7 @@ class DogStatsDClient {
       for (const [key, value] of Object.entries(config.tags)) {
         // Skip runtime-id unless enabled as cardinality may be too high
         if (typeof value === 'string' && (key !== 'runtime-id' || config.runtimeMetricsRuntimeId)) {
-          // https://docs.datadoghq.com/tagging/#defining-tags
+          // https://docs.datadoghq.com/getting_started/tagging/#define-tags
           const valueStripped = value.replaceAll(/[^a-z0-9_:./-]/ig, '_')
 
           tags.push(`${key}:${valueStripped}`)

package/packages/dd-trace/src/encode/tags-processors.js

@@ -2,7 +2,7 @@
 
 // From agent truncators: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/agent/truncator.go
 
-// Values from: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/truncate.go#L22-L27
+// Values from: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/normalize/truncate.go
 // MAX_RESOURCE_NAME_LENGTH the maximum length a span resource can have
 const MAX_RESOURCE_NAME_LENGTH = 5000
 // MAX_META_KEY_LENGTH the maximum length of metadata key
@@ -15,7 +15,7 @@ const MAX_META_VALUE_LENGTH_TEST_OPTIMIZATION = 5000
 const MAX_METRIC_KEY_LENGTH = MAX_META_KEY_LENGTH
 
 // From agent normalizer:
-// https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/normalize.go
+// https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/normalize/normalize.go
 // DEFAULT_SPAN_NAME is the default name we assign a span if it's missing and we have no reasonable fallback
 const DEFAULT_SPAN_NAME = 'unnamed_operation'
 // DEFAULT_SERVICE_NAME is the default name we assign a service if it's missing and we have no reasonable fallback

package/packages/dd-trace/src/exporters/common/request.js

@@ -6,6 +6,7 @@
 const { Readable } = require('stream')
 const http = require('http')
 const https = require('https')
+const { URL } = require('url')
 const zlib = require('zlib')
 
 const { storage } = require('../../../../datadog-core')
@@ -85,11 +86,7 @@ function request (data, options, callback) {
   const timeout = options.timeout || 2000
   const isSecure = options.protocol === 'https:'
   const client = isSecure ? https : http
-  let dataArray = data
-
-  if (!Array.isArray(data)) {
-    dataArray = [data]
-  }
+  const dataArray = Array.isArray(data) ? data : [data]
   const contentLength = byteLength(dataArray)
   options.headers['Content-Length'] = contentLength
 

package/packages/dd-trace/src/exporters/common/url.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const net = require('node:net')
-const { format } = require('node:url')
+const { format, URL } = require('node:url')
 
 const { urlToHttpOptions } = require('./url-to-http-options-polyfill')
 

package/packages/dd-trace/src/id.js

@@ -2,17 +2,15 @@
 
 const { randomFillSync } = require('crypto')
 
-const { channel } = require('dc-polyfill')
-
 const UINT_MAX = 4_294_967_296
 
 const data = new Uint8Array(8 * 8192)
 const zeroId = new Uint8Array(8)
+// Reused for hex encoding: toString('hex') copies out, so the bytes need not outlive the call.
+const hexScratch = Buffer.alloc(16)
 
 let batch = 0
 
-channel('datadog:identity:update').subscribe(reseed)
-
 // Internal representation of a trace or span ID.
 class Identifier {
   /** @type {number[] | Uint8Array} */
@@ -39,7 +37,7 @@ class Identifier {
    */
   toString (radix = 16) {
     if (radix === 16) {
-      this.#stringHex ??= Buffer.from(this.#buffer).toString('hex')
+      this.#stringHex ??= toHexString(this.#buffer)
       return this.#stringHex
     }
     if (radix === 10) {
@@ -194,6 +192,17 @@ function toNumberString (buffer, radix) {
   return str
 }
 
+// Same string as Buffer.from(buffer).toString('hex'), without allocating a Buffer per call.
+/**
+ * @param {number[] | Uint8Array} buffer
+ */
+function toHexString (buffer) {
+  const length = buffer.length
+  if (length > hexScratch.length) return Buffer.from(buffer).toString('hex')
+  for (let i = 0; i < length; i++) hexScratch[i] = buffer[i]
+  return hexScratch.toString('hex', 0, length)
+}
+
 // Simple pseudo-random 64-bit ID generator.
 /**
  * @returns {number[] | Uint8Array}
@@ -267,3 +276,4 @@ module.exports = function createIdentifier (value, radix) {
 }
 
 module.exports.Identifier = Identifier
+module.exports.reseed = reseed