dd-trace 6.12.0 → 6.14.0

package/packages/dd-trace/src/span_stats.js

@@ -15,6 +15,7 @@ const {
 } = require('../../../ext/tags')
 const { ORIGIN_KEY, TOP_LEVEL_KEY, SVC_SRC_KEY, GRPC_STATUS_NAMES } = require('./constants')
 const id = require('./id')
+const log = require('./log')
 
 const GRPC_STATUS_CODE_MAP = Object.fromEntries(GRPC_STATUS_NAMES.map((name, i) => [name, String(i)]))
 const ZERO_ID = id('0')
@@ -135,19 +136,8 @@ class SpanAggKey {
   }
 
   toString () {
-    return [
-      this.name,
-      this.service,
-      this.resource,
-      this.type,
-      this.statusCode,
-      this.synthetics,
-      this.method,
-      this.endpoint,
-      this.srvSrc,
-      this.spanKind,
-      this.rpcStatusCode,
-    ].join(',')
+    return `${this.name},${this.service},${this.resource},${this.type},${this.statusCode},${this.synthetics},` +
+      `${this.method},${this.endpoint},${this.srvSrc},${this.spanKind},${this.rpcStatusCode}`
   }
 }
 
@@ -200,19 +190,26 @@ class TimeBuckets extends Map {
 }
 
 class SpanStatsProcessor {
-  constructor ({
-    stats: {
-      DD_TRACE_STATS_COMPUTATION_ENABLED: enabled = false,
-      interval = 10,
-    } = {},
-    hostname,
-    port,
-    url,
-    env,
-    tags,
-    version: appVersion,
-    _DD_TRACE_METRICS_OTEL_FLUSH_INTERVAL: flushIntervalMs,
-  } = {}, otlpExporter) {
+  #config
+
+  /**
+   * @param {import('./config/config-base')} config
+   * @param {import('./opentelemetry/metrics/otlp_span_stats_exporter').OtlpStatsExporter} [otlpExporter]
+   */
+  constructor (config, otlpExporter) {
+    const {
+      stats: {
+        DD_TRACE_STATS_COMPUTATION_ENABLED: enabled = false,
+        interval = 10,
+      } = {},
+      hostname,
+      port,
+      url,
+      env,
+      tags,
+      version: appVersion,
+      _DD_TRACE_METRICS_OTEL_FLUSH_INTERVAL: flushIntervalMs,
+    } = config
     if (!otlpExporter) {
       this.exporter = new SpanStatsExporter({ hostname, port, tags, url })
     }
@@ -224,7 +221,7 @@ class SpanStatsProcessor {
     this.enabled = enabled
     this.otlpExporter = otlpExporter || null
     this.env = env
-    this.tags = tags || {}
+    this.#config = config
     this.sequence = 0
     this.version = appVersion
 
@@ -235,6 +232,18 @@ class SpanStatsProcessor {
   }
 
   onInterval () {
+    this.#flush()
+  }
+
+  /**
+   * Drains pending span statistics and waits for their export.
+   * @param {Function} [done]
+   */
+  forceFlush (done) {
+    this.#flush(done)
+  }
+
+  #flush (done) {
     const drained = this.#drainBuckets()
 
     if (this.enabled && !this.otlpExporter) {
@@ -245,13 +254,37 @@ class SpanStatsProcessor {
         Stats: this.#toV06Payload(drained),
         Lang: 'javascript',
         TracerVersion: pkg.version,
-        RuntimeID: this.tags['runtime-id'],
+        RuntimeID: this.#config.tags['runtime-id'],
         Sequence: ++this.sequence,
         ProcessTags: processTags.serialized,
-      })
+      }, done)
     } else if (this.otlpExporter && drained.length > 0) {
-      this.otlpExporter.export(drained, this.bucketSizeNs)
-    }
+      if (typeof this.otlpExporter.flush === 'function' && done) {
+        // Snapshot requests already in flight before starting this boundary
+        // export, so a later invocation cannot extend this lifecycle barrier.
+        let pending = 2
+        const complete = () => {
+          if (--pending === 0) done()
+        }
+        try {
+          this.otlpExporter.flush(complete)
+        } catch (error) {
+          log.error('Failed to flush OTLP span stats:', error)
+          complete()
+        }
+        try {
+          this.otlpExporter.export(drained, this.bucketSizeNs, complete)
+        } catch (error) {
+          log.error('Failed to export OTLP span stats:', error)
+          complete()
+        }
+      } else {
+        this.otlpExporter.export(drained, this.bucketSizeNs, done)
+      }
+    } else if (this.otlpExporter) {
+      if (typeof this.otlpExporter.flush === 'function') this.otlpExporter.flush(done)
+      else done?.()
+    } else done?.()
   }
 
   onSpanFinished (span) {

package/packages/dd-trace/src/startup-log.js

@@ -78,7 +78,7 @@ function configInfo () {
   const url = config.url
   const profilingEnabled = config.profiling.DD_PROFILING_ENABLED
 
-  return {
+  const startupLog = {
     [inspect.custom] () {
       return String(this)
     },
@@ -102,7 +102,6 @@ function configInfo () {
     sample_rate: config.sampler.sampleRate,
     sampling_rules: samplingRules,
     tags: config.tags,
-    ...(config.tags && config.tags.version && { dd_version: config.tags.version }),
     log_injection_enabled: !!config.logInjection,
     runtime_metrics_enabled: !!config.runtimeMetrics,
     profiling_enabled: profilingEnabled === 'true' || profilingEnabled === 'auto',
@@ -112,6 +111,8 @@ function configInfo () {
     otlp_metrics_export_enabled: !!config.DD_METRICS_OTEL_ENABLED,
     otlp_logs_export_enabled: !!config.DD_LOGS_OTEL_ENABLED,
   }
+  if (config.tags?.version) startupLog.dd_version = config.tags.version
+  return startupLog
 }
 
 /**

package/packages/dd-trace/src/telemetry/agentless-url.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { createSiteUrl } = require('../exporters/common/url')
+
+const STAGING_SITE = 'datad0g.com'
+
+let cachedSite
+let cachedUrl
+
+/**
+ * @param {string} site
+ * @returns {URL}
+ */
+function getAgentlessTelemetryUrl (site) {
+  if (site === cachedSite && cachedUrl !== undefined) return cachedUrl
+
+  const intake = site === STAGING_SITE
+    ? 'all-http-intake.logs'
+    : 'instrumentation-telemetry-intake'
+  const url = createSiteUrl(site, intake)
+  if (url === undefined) throw new Error(`Invalid DD_SITE for agentless telemetry: ${site}`)
+
+  cachedSite = site
+  cachedUrl = url
+  return url
+}
+
+module.exports = getAgentlessTelemetryUrl

package/packages/dd-trace/src/telemetry/send-data.js

@@ -2,6 +2,7 @@
 
 const request = require('../exporters/common/request')
 const log = require('../log')
+const getAgentlessTelemetryUrl = require('./agentless-url')
 
 /**
  * @typedef {Record<string, unknown>} TelemetryPayloadObject
@@ -69,6 +70,7 @@ const log = require('../log')
  */
 
 let agentTelemetry = true
+let getTestOptimizationAgent
 
 /**
  * @param {import('../config/config-base')} config
@@ -98,16 +100,6 @@ function getHeaders (config, application, reqType) {
   return headers
 }
 
-/**
- * @param {string | undefined} site
- */
-function getAgentlessTelemetryEndpoint (site) {
-  if (site === 'datad0g.com') { // staging
-    return 'https://all-http-intake.logs.datad0g.com'
-  }
-  return `https://instrumentation-telemetry-intake.${site}`
-}
-
 let seqId = 0
 
 /**
@@ -144,10 +136,18 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
   let url = config.url
 
   const isCiVisibilityAgentlessMode = isCiVisibility && testOptimization.DD_CIVISIBILITY_AGENTLESS_ENABLED
+  const isApmTracingAgentlessMode = config.experimental?.exporter === 'agentless'
+  const isAgentlessMode = isCiVisibilityAgentlessMode || isApmTracingAgentlessMode
+
+  if (isCiVisibility && getTestOptimizationAgent === undefined) {
+    ({ getAgent: getTestOptimizationAgent } = require('../ci-visibility/exporters/agents'))
+  }
 
-  if (isCiVisibilityAgentlessMode) {
+  if (isAgentlessMode) {
     try {
-      url = testOptimization.DD_CIVISIBILITY_AGENTLESS_URL ?? new URL(getAgentlessTelemetryEndpoint(config.site))
+      url = isCiVisibilityAgentlessMode
+        ? testOptimization.DD_CIVISIBILITY_AGENTLESS_URL ?? getAgentlessTelemetryUrl(config.site)
+        : getAgentlessTelemetryUrl(config.site)
     } catch (err) {
       log.error('Telemetry endpoint url is invalid', err)
       // No point to do the request if the URL is invalid
@@ -160,9 +160,10 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
     hostname,
     port,
     method: 'POST',
-    path: isCiVisibilityAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry',
+    path: isAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry',
     headers: getHeaders(config, application, reqType),
   }
+  if (isCiVisibility) options.agent = getTestOptimizationAgent(url)
 
   const data = JSON.stringify({
     api_version: 'v2',
@@ -177,7 +178,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
   })
 
   request(data, options, (error) => {
-    if (error && config.DD_API_KEY && config.site) {
+    if (!isAgentlessMode && error && config.DD_API_KEY && config.site) {
       if (agentTelemetry) {
         log.warn('Agent telemetry failed, started agentless telemetry')
         agentTelemetry = false
@@ -185,7 +186,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
       // figure out which data center to send to
       let backendUrl
       try {
-        backendUrl = new URL(getAgentlessTelemetryEndpoint(config.site))
+        backendUrl = getAgentlessTelemetryUrl(config.site)
       } catch {
         log.error('Invalid Telemetry URL')
         return
@@ -197,6 +198,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
         headers: backendHeader,
         path: '/api/v2/apmtelemetry',
       }
+      if (isCiVisibility) backendOptions.agent = getTestOptimizationAgent(backendUrl)
       request(data, backendOptions, (error) => {
         if (error) {
           log.error('Error sending telemetry data', error)
@@ -204,7 +206,7 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
       })
     }
 
-    if (!error && !agentTelemetry) {
+    if (!isAgentlessMode && !error && !agentTelemetry) {
       agentTelemetry = true
       log.info('Started agent telemetry')
     }

package/packages/dd-trace/src/telemetry/session-propagation.js

@@ -4,13 +4,14 @@ const dc = /** @type {typeof import('diagnostics_channel')} */ (require('dc-poly
 const childProcessChannel = dc.tracingChannel('datadog:child_process:execution')
 
 let subscribed = false
-let runtimeId
+let savedConfig
 
 function isOptionsObject (value) {
   return value != null && typeof value === 'object' && !Array.isArray(value) && value
 }
 
 function getEnvWithRuntimeId (env) {
+  const runtimeId = savedConfig.DD_ROOT_JS_SESSION_ID || savedConfig.tags['runtime-id']
   // eslint-disable-next-line eslint-rules/eslint-process-env
   return { ...(env ?? process.env), DD_ROOT_JS_SESSION_ID: runtimeId }
 }
@@ -43,7 +44,7 @@ function start (config) {
   if (!config.telemetry.DD_INSTRUMENTATION_TELEMETRY_ENABLED || subscribed) return
   subscribed = true
 
-  runtimeId = config.DD_ROOT_JS_SESSION_ID || config.tags['runtime-id']
+  savedConfig = config
 
   childProcessChannel.subscribe(
     /** @type {import('diagnostics_channel').TracingChannelSubscribers<object>} */ ({ start: onChildProcessStart })

package/packages/dd-trace/src/tracer.js

@@ -13,6 +13,7 @@ const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
 const { DataStreamsCheckpointer, DataStreamsManager, DataStreamsProcessor } = require('./datastreams')
 const { IS_SERVERLESS } = require('./serverless')
+const { flushServerlessTelemetry } = require('./flush')
 const log = require('./log')
 // Always-on writer (console.warn), not the channel-gated `log`: these surface regardless of
 // DD_TRACE_DEBUG.
@@ -42,13 +43,10 @@ class DatadogTracer extends Tracer {
 
     if (!IS_SERVERLESS) {
       const storeConfig = require('./tracer_metadata')
-      // Keep a reference to the handle, to keep the memfd alive in memory.
-      // It is read by the service discovery feature.
       const metadata = storeConfig(config)
       if (metadata === undefined) {
         log.warn('Could not store tracer configuration for service discovery')
       }
-      this._inmem_cfg = metadata
     }
   }
 
@@ -150,6 +148,27 @@ class DatadogTracer extends Tracer {
     this._dataStreamsProcessor.setUrl(url)
   }
 
+  /**
+   * Flushes every configured telemetry pipeline for a serverless lifecycle.
+   * @param {Function} [done] Called after every configured export completes
+   * @param {{ timeout?: number }} [options] Bounds this flush operation.
+   */
+  flushAll (done, options) {
+    const traceExporter = this._exporter
+    const spanStats = this._processor?._stats
+    const traceFlusher = typeof traceExporter?.flush === 'function'
+      ? callback => traceExporter.flush(callback)
+      : undefined
+    const spanStatsFlusher = typeof spanStats?.forceFlush === 'function'
+      ? callback => spanStats.forceFlush(callback)
+      : undefined
+
+    flushServerlessTelemetry(done, options, {
+      trace: traceFlusher,
+      spanStats: spanStatsFlusher,
+    })
+  }
+
   scope () {
     return this._scope
   }
@@ -158,8 +177,10 @@ class DatadogTracer extends Tracer {
     if (!this._enableGetRumData) {
       return ''
     }
-    const span = this.scope().active().context()
-    const traceId = span.toTraceId()
+    const span = this.scope().active()
+    if (!span) return ''
+
+    const traceId = span.context().toTraceId()
     const traceTime = Date.now()
     return `\
 <meta name="dd-trace-id" content="${traceId}" />\

package/packages/dd-trace/src/tracer_metadata.js

@@ -2,11 +2,18 @@
 
 const tracerVersion = require('../../../version').VERSION
 
+// Service discovery reads this metadata after tracer initialization; retaining the handle keeps
+// the backing memfd open for the process lifetime.
+let metadataHandle
+
+/**
+ * @param {import('./config/config-base')} config
+ */
 function storeConfig (config) {
   try {
     // Load binding first to not import other modules if it throws
-    const libdatadog = require('@datadog/libdatadog')
-    const processDiscovery = libdatadog.maybeLoad('process-discovery')
+    const libdatadogExtras = require('@datadog/libdatadog-extras')
+    const processDiscovery = libdatadogExtras.maybeLoad('process-discovery')
     if (processDiscovery === undefined) {
       return
     }
@@ -42,7 +49,8 @@ function storeConfig (config) {
       threadlocalMetadata
     )
 
-    return processDiscovery.storeMetadata(metadata)
+    metadataHandle = processDiscovery.storeMetadata(metadata)
+    return metadataHandle
   } catch {
     // Either libdatadog or process-discovery is unavailable.
   }

package/packages/dd-trace/src/web-tags-cache.js

@@ -3,8 +3,9 @@
 // Per-span cache of "which tag bag from the started-spans chain identifies
 // this span (or its nearest web-server ancestor) as a web-server span?"
 // Populated lazily on first `getCachedWebTags(span)`, refreshed
-// automatically when a `dd-trace:span:tags:update` event promotes a
-// previously-empty answer for that span into a real value.
+// automatically when a `dd-trace:span:tags:update` event turns a span into a
+// web-server span after the fact — for that span and for every descendant
+// whose answer the promotion changes.
 //
 // Used by the wall profiler (endpoint-collection label on samples) and by
 // the OTEP-4947 thread-context writer (endpoint attribute in the record);
@@ -13,11 +14,12 @@
 //
 // Consumers that want to react to late web-server-span discovery
 // subscribe to `resolvedCh` — a diagnostics channel we publish on once
-// per span at the moment its cached webTags transitions from undefined
-// to a real value. Doing it via a channel (rather than exposing a
-// stateful "did the transition happen?" query) means each consumer sees
-// every transition exactly once, regardless of subscription order or
-// how many other consumers are attached.
+// per span at the moment its cached webTags changes, i.e. when a promotion
+// gives it an answer it didn't have or replaces the one it had with a
+// nearer one. Doing it via a channel (rather than exposing a stateful "did
+// the transition happen?" query) means each consumer sees every transition
+// exactly once, regardless of subscription order or how many other
+// consumers are attached.
 //
 // `endpointResolvedCh` is the same idea one field over: published once per
 // web-server span at the moment its endpoint name settles (see finalEndpoint in
@@ -63,17 +65,21 @@ function getCache (span) {
 // Returns the web-server tag bag for this span or its nearest web-server
 // ancestor in the started-spans chain, or undefined if none is a
 // web-server span. Lazy: walks the parent chain on the first call, caches
-// the result on the span.
+// the result on the span. Answers only ever change through onTagsUpdate, which
+// rewrites the affected entries in place, so a resolved answer is never
+// revisited here.
 function getCachedWebTags (span) {
   const cached = getCache(span)
   if (cached.resolved) return cached.webTags
   const spanContext = span.context()
   const tags = spanContext.getTags()
+  const parentId = spanContext._parentId
   let webTags
   if (isWebServerSpan(tags)) {
     webTags = tags
-  } else {
-    const parentId = spanContext._parentId
+  // A span with no parent has nothing to inherit from, and looking for one
+  // anyway means scanning the entire started-spans list to conclude that.
+  } else if (parentId != null) {
     const startedSpans = getStartedSpans(spanContext)
     for (let i = startedSpans.length; --i >= 0;) {
       const ispan = startedSpans[i]
@@ -97,11 +103,17 @@ function getCachedWebTags (span) {
 function onTagsUpdate (span) {
   const cached = span[CachedSym]
   if (cached === undefined || !cached.resolved) return
-  const tags = span.context().getTags()
-  if (cached.webTags === undefined) {
-    if (!isWebServerSpan(tags)) return
+  const spanContext = span.context()
+  const tags = spanContext.getTags()
+  // Anything but this span's own bag is an answer that predates it being a
+  // web-server span: either empty, or an outer web-server ancestor's bag that
+  // this span now supersedes for itself and for its descendants. A span whose
+  // cached answer already is its own bag is the overwhelmingly common case here
+  // (every further tag update on a request span), and stops at one comparison.
+  if (cached.webTags !== tags && isWebServerSpan(tags)) {
     cached.webTags = tags
     resolvedCh.publish(span)
+    resolveDescendants(span, spanContext, tags)
   }
   // Endpoint finality is a property of the web-server span itself: for a
   // descendant, cached.webTags is an ancestor's bag rather than these tags, and
@@ -114,6 +126,48 @@ function onTagsUpdate (span) {
   endpointResolvedCh.publish(span)
 }
 
+// A span has just become a web-server span; answers cached for its descendants
+// before that moment found a farther ancestor or nothing at all, and are now
+// wrong. Rewrite them here rather than letting each descendant discover it on
+// its next lookup: a descendant that is already active keeps handing samplers
+// its stale answer for as long as it runs without re-entering storage, which is
+// precisely the uninterrupted synchronous stretch profiling cares about.
+//
+// One forward pass over the trace's started-spans list, which is in creation
+// order, so a span's parent always precedes it: `answers` holds the new answer
+// for each span in the promoted span's subtree, keyed by span id, and a span
+// inherits its parent's unless it is a web-server span itself, in which case its
+// own bag shadows the promotion for its own subtree. Spans outside the subtree
+// are never in `answers`, so they cost one map lookup and nothing else.
+//
+// That same creation order means nothing before the promoted span can be a
+// descendant of it, so the pass starts just past it. Locating it from the end
+// costs one comparison in the common case, where a request span is promoted as
+// it is created and is still the newest entry — leaving nothing to visit and
+// nothing to allocate. A span the list no longer holds (finished, and dropped by
+// a partial flush) ends that scan at -1, which visits the whole list.
+function resolveDescendants (span, spanContext, webTags) {
+  const startedSpans = getStartedSpans(spanContext)
+  let index = startedSpans.length - 1
+  while (index >= 0 && startedSpans[index] !== span) index--
+  index++
+  if (index === startedSpans.length) return
+  const answers = new Map([[spanContext._spanId, webTags]])
+  for (; index < startedSpans.length; index++) {
+    const descendant = startedSpans[index]
+    const context = descendant.context()
+    const inherited = answers.get(context._parentId)
+    if (inherited === undefined) continue
+    const tags = context.getTags()
+    const answer = isWebServerSpan(tags) ? tags : inherited
+    answers.set(context._spanId, answer)
+    const cached = descendant[CachedSym]
+    if (cached === undefined || !cached.resolved || cached.webTags === answer) continue
+    cached.webTags = answer
+    resolvedCh.publish(descendant)
+  }
+}
+
 let activeCount = 0
 
 function activate () {