dd-trace 6.20.0 → 6.21.0

package/packages/dd-trace/src/llmobs/tagger.js

@@ -18,6 +18,7 @@ const {
   OUTPUT_VALUE,
   METADATA,
   COST_TAGS,
+  AGENT_MANIFEST,
   METRICS,
   TOOL_DEFINITIONS,
   PARENT_ID_KEY,
@@ -53,6 +54,7 @@ const {
   TRACE_ID,
   PROPAGATED_TRACE_ID_KEY,
 } = require('./constants/tags')
+const { buildAgentDeclaration } = require('./agent-manifest')
 const { storage } = require('./storage')
 const {
   findGenAIAncestorSpanId,
@@ -187,6 +189,11 @@ class LLMObsTagger {
       this.tagCostTags(span, annotationContext.costTags, 'annotation_context')
     }
 
+    const agentDeclarations = /** @type {import('./agent-manifest').AgentDeclaration[] | undefined} */ (
+      storage.getStore()?.agentDeclarations
+    )
+    if (agentDeclarations) this.#applyAgentDeclarations(span, agentDeclarations)
+
     // apply annotation context name
     const annotationContextName = annotationContext?.name
     if (annotationContextName) this._setTag(span, NAME, annotationContextName)
@@ -204,6 +211,43 @@ class LLMObsTagger {
     }
   }
 
+  /**
+   * Applies the agents declared by the enclosing annotation contexts, outermost first, to every span in the block.
+   * The manifest is emitted only if the span is an agent when it finishes, since some integrations promote a span
+   * to an agent after registration.
+   *
+   * @param {import('../opentracing/span')} span
+   * @param {import('./agent-manifest').AgentDeclaration[]} declarations
+   */
+  #applyAgentDeclarations (span, declarations) {
+    for (const declaration of declarations) {
+      this.#tagAgentManifestFields(span, declaration.manifest)
+    }
+  }
+
+  /**
+   * Tags the agent a caller declared on an agent span, merged onto what earlier annotations declared.
+   *
+   * @param {import('../opentracing/span')} span
+   * @param {unknown} agent
+   */
+  tagAgent (span, agent) {
+    const declaration = buildAgentDeclaration(agent)
+    if (declaration) this.#tagAgentManifestFields(span, declaration.manifest)
+  }
+
+  /**
+   * @param {import('../opentracing/span')} span
+   * @param {import('./agent-manifest').AgentManifestFields} manifest
+   */
+  #tagAgentManifestFields (span, manifest) {
+    // A shallow update, matching dd-trace-py: each declaration is already validated, so a later field replaces an
+    // earlier one whole (`model_settings` included) and an unset or invalid one leaves it in place. Declarations are
+    // never mutated, so the first one is stored without a copy.
+    const existing = registry.get(span)?.[AGENT_MANIFEST]
+    this._setTag(span, AGENT_MANIFEST, existing ? { ...existing, ...manifest } : manifest)
+  }
+
   #tagSamplingDecision (span, parent) {
     const traceTags = span.context()._trace.tags
     const parentTags = registry.get(parent)
@@ -337,7 +381,16 @@ class LLMObsTagger {
     if (currentTags) {
       Object.assign(currentTags, tags)
     } else {
-      this._setTag(span, TAGS, tags)
+      // Copied so a later annotation cannot write into a caller's object shared by an annotation context. The copy
+      // reads caller getters and Proxy traps during span registration, so it must not throw into the application.
+      let copy
+      try {
+        copy = { ...tags }
+      } catch {
+        log.warn('Dropping span tags that could not be read.')
+        return
+      }
+      this._setTag(span, TAGS, copy)
     }
   }
 

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -4,7 +4,7 @@ const { URL, format } = require('node:url')
 const path = require('node:path')
 const request = require('../../exporters/common/request')
 const { getEnvironmentVariable } = require('../../config/helper')
-const { createServerlessDeliveryTracker } = require('../../serverless')
+const { createServerlessDeliveryTracker, IS_STANDARD_AWS_LAMBDA } = require('../../serverless')
 
 const logger = require('../../log')
 
@@ -118,6 +118,8 @@ class BaseLLMObsWriter {
 
     buffer.size += eventSize
     buffer.events.push(event)
+    // The standard Lambda sandbox can freeze before the periodic flush runs.
+    if (IS_STANDARD_AWS_LAMBDA) this.flush()
     return true
   }
 

package/packages/dd-trace/src/llmobs/writers/util.js

@@ -24,7 +24,9 @@ function setAgentStrategy (config, setWritersAgentlessValue) {
     }
 
     const endpoints = agentInfo.endpoints
-    const hasEndpoint = Array.isArray(endpoints) && endpoints.includes(EVP_PROXY_AGENT_BASE_PATH)
+    // The Lambda extension lists full proxy paths (e.g. /evp_proxy/v2/api/v2/llmobs), not the bare prefix.
+    const hasEndpoint = Array.isArray(endpoints) &&
+      endpoints.some(endpoint => typeof endpoint === 'string' && endpoint.startsWith(EVP_PROXY_AGENT_BASE_PATH))
     setWritersAgentlessValue(!hasEndpoint)
   })
 }

package/packages/dd-trace/src/openfeature/agentless_configuration_source.js

@@ -21,6 +21,7 @@ const RETRY_JITTER = 0.2
  * @property {number} pollIntervalMs
  * @property {number} requestTimeoutMs
  * @property {string | undefined} apiKey
+ * @property {string | undefined} env
  */
 
 /**
@@ -106,9 +107,23 @@ class AgentlessConfigurationSource {
    */
   async #pollOnce (abortController) {
     for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+      log.debug('Feature Flags: agentless configuration poll attempt %d/%d', attempt, MAX_ATTEMPTS)
       const response = await this.#request(abortController.signal)
       if (this.#abortController !== abortController) return
 
+      if (response.statusCode === 304) {
+        // 304 means the cached configuration is still valid, not a failure.
+        log.debug('Feature Flags: agentless configuration poll attempt %d/%d returned HTTP 304 (unchanged)',
+          attempt, MAX_ATTEMPTS)
+      } else if (response.statusCode === undefined) {
+        log.debug('Feature Flags: agentless configuration poll attempt %d/%d failed: %s',
+          attempt, MAX_ATTEMPTS, errorMessage(response.error))
+      } else {
+        // Avoids errorMessage(response.error), which embeds the full request URL.
+        log.debug('Feature Flags: agentless configuration poll attempt %d/%d returned HTTP %d',
+          attempt, MAX_ATTEMPTS, response.statusCode)
+      }
+
       const retryable = response.statusCode === undefined || isRetryableStatus(response.statusCode)
       if (!retryable) {
         this.#apply(response)
@@ -205,6 +220,13 @@ class AgentlessConfigurationSource {
     const etag = response.headers?.etag
     const value = Array.isArray(etag) ? etag[0] : etag
     this.#etag = value?.trim() || undefined
+
+    // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+    log.debug(() => {
+      const flagCount = Object.keys(configuration.flags ?? {}).length
+      return 'Feature Flags: agentless configuration applied successfully ' +
+        `(${flagCount} flag(s), env=${this.#config.env ?? 'unset'})`
+    })
   }
 
   /**
@@ -270,10 +292,17 @@ function parseConfiguration (body) {
 }
 
 /**
+ * Formats a request error, including the underlying network error code
+ * (e.g. ENOTFOUND, ECONNREFUSED, ETIMEDOUT) when available, since "request
+ * failed" alone does not tell a customer whether the problem is DNS, a
+ * refused connection, a timeout, or something else.
+ *
  * @param {unknown} error
  */
 function errorMessage (error) {
-  return error instanceof Error ? error.message : String(error ?? 'request was not sent')
+  if (!(error instanceof Error)) return String(error ?? 'request was not sent')
+  const code = /** @type {{ code?: string }} */ (error).code
+  return code ? `${error.message} (${code})` : error.message
 }
 
 /**

package/packages/dd-trace/src/openfeature/configuration_source.js

@@ -23,7 +23,10 @@ function create (config, applyConfiguration) {
     DD_FEATURE_FLAGS_ENABLED: enabled,
   } = config.featureFlags
 
-  if (!enabled || source !== 'agentless') {
+  if (!enabled) return
+
+  if (source !== 'agentless') {
+    log.debug('Feature Flags: configuration source is %s, not agentless; skipping agentless setup', source)
     return
   }
 
@@ -35,11 +38,14 @@ function create (config, applyConfiguration) {
     }
 
     const AgentlessConfigurationSource = require('./agentless_configuration_source')
+    const resolvedEndpoint = endpoint(config, baseUrl)
+    log.debug('Feature Flags: starting agentless configuration source at %s', redactedUrl(resolvedEndpoint))
     return new AgentlessConfigurationSource({
-      endpoint: endpoint(config, baseUrl),
+      endpoint: resolvedEndpoint,
       pollIntervalMs: Math.min(pollIntervalSeconds, MAX_POLL_INTERVAL_SECONDS) * 1000,
       requestTimeoutMs: requestTimeoutSeconds * 1000,
       apiKey: hasCustomEndpoint ? undefined : config.DD_API_KEY,
+      env: config.env,
     }, applyConfiguration)
   } catch (error) {
     log.error('Unable to configure Feature Flagging configuration source', error)
@@ -85,6 +91,15 @@ function endpoint (config, configuredBaseUrl) {
   return url
 }
 
+/**
+ * Drops everything but the origin; a custom base URL may embed a signed token in its path.
+ *
+ * @param {URL} url
+ */
+function redactedUrl (url) {
+  return `${url.protocol}//${url.host}`
+}
+
 module.exports = {
   create,
 }

package/packages/dd-trace/src/openfeature/debug-logging-hook.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const log = require('../log')
+
+/**
+ * Logs full evaluation details under DD_TRACE_DEBUG, matching EvalMetricsHook's use of `finally`.
+ */
+class DebugLoggingHook {
+  /**
+   * @param {{ flagKey: string }} hookContext - Hook context containing the flag key
+   * @param {object} evaluationDetails - Full evaluation details
+   */
+  finally (hookContext, evaluationDetails) {
+    try {
+      log.debug('Feature Flags: evaluated %s: %o', hookContext?.flagKey, evaluationDetails)
+    } catch (error) {
+      try {
+        // Defense in depth; the OpenFeature SDK already guards finally hooks.
+        log.warn('DebugLoggingHook: error in finally hook: %s', error.message)
+      } catch { /* logging failure must not crash evaluation */ }
+    }
+  }
+}
+
+module.exports = DebugLoggingHook

package/packages/dd-trace/src/openfeature/flagging_provider.js

@@ -4,8 +4,10 @@ const { channel } = require('dc-polyfill')
 
 const { DatadogNodeServerProvider } = require('../../../../vendor/dist/@datadog/openfeature-node-server')
 const log = require('../log')
+const { debugChannel } = require('../log/channels')
 const configurationSource = require('./configuration_source')
 const { EXPOSURE_CHANNEL } = require('./constants/constants')
+const DebugLoggingHook = require('./debug-logging-hook')
 const EvalMetricsHook = require('./eval-metrics-hook')
 const SpanEnrichmentHook = require('./span-enrichment-hook')
 const FlagEvalEVPHook = require('./writers/flag-eval-evp-hook')
@@ -36,6 +38,12 @@ class FlaggingProvider extends DatadogNodeServerProvider {
 
     if (config.DD_METRICS_OTEL_ENABLED === true) {
       this.hooks.push(new EvalMetricsHook(config))
+    } else {
+      log.debug('Feature Flags: evaluation metrics disabled; set %s=true to enable', 'DD_METRICS_OTEL_ENABLED')
+    }
+
+    if (config.debug && debugChannel.hasSubscribers) {
+      this.hooks.push(new DebugLoggingHook())
     }
 
     if (config.featureFlags.DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED) {
@@ -64,6 +72,8 @@ class FlaggingProvider extends DatadogNodeServerProvider {
    * @returns {Promise<void>}
    */
   initialize (context) {
+    log.debug('Feature Flags: waiting for provider initialization...')
+
     const promise = super.initialize(context)
 
     // `DatadogNodeServerProvider#initialize` starts a timer that is never unref'd, which would
@@ -72,6 +82,25 @@ class FlaggingProvider extends DatadogNodeServerProvider {
     // TODO: remove once `@datadog/openfeature-node-server` unrefs this timer itself.
     this.initController?.timeoutId?.unref?.()
 
+    // Only observes the outcome for logging; `promise` itself is returned unmodified below.
+    // Guarded so a logger failure can never leave the derived promise unhandled.
+    promise.then(
+      () => {
+        try {
+          log.debug('Feature Flags: provider initialized successfully')
+        } catch { /* logging failure must not crash the process */ }
+      },
+      (error) => {
+        try {
+          // errorWithoutTelemetry avoids inflating telemetry volume for every routine init timeout.
+          log.errorWithoutTelemetry(
+            'Feature Flags: provider failed to initialize: %s',
+            error instanceof Error ? error.message : String(error)
+          )
+        } catch { /* logging failure must not crash the process */ }
+      }
+    )
+
     return promise
   }
 

package/packages/dd-trace/src/openfeature/remote_config.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const log = require('../log')
 const RemoteConfigCapabilities = require('../remote_config/capabilities')
 
 /**
@@ -12,6 +13,8 @@ const RemoteConfigCapabilities = require('../remote_config/capabilities')
 function enable (rc, getOpenfeatureProxy, subscribe) {
   if (!subscribe) return
 
+  log.debug('Feature Flags: starting remote_config configuration source (Agent Remote Configuration)')
+
   rc.updateCapabilities(RemoteConfigCapabilities.FFE_FLAG_CONFIGURATION_RULES, true)
 
   /**
@@ -21,7 +24,13 @@ function enable (rc, getOpenfeatureProxy, subscribe) {
   const updateConfiguration = (action, conf) => {
     if (action === 'apply' || action === 'modify') {
       getOpenfeatureProxy().setConfiguration(conf)
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.debug(() => {
+        const flagCount = Object.keys(conf?.flags ?? {}).length
+        return `Feature Flags: remote_config configuration ${action} applied successfully (${flagCount} flag(s))`
+      })
     } else if (action === 'unapply') {
+      log.debug('Feature Flags: remote_config configuration removed')
       getOpenfeatureProxy().setConfiguration(undefined)
     }
   }

package/packages/dd-trace/src/openfeature/writers/util.js

@@ -16,6 +16,18 @@ const ROUTE_DISCOVERY_COOLDOWN_MS = 60_000
 const REQUIRED_LOCAL_HEADERS = [EVP_ORIGIN_HEADER_NAME, EVP_ORIGIN_VERSION_HEADER_NAME]
 const STOP_NOOP = () => {}
 
+/**
+ * @typedef {(enabled: boolean, route?: import('./flag-evaluations').FlagEvaluationRoute) => void} DeliverySubscriber
+ */
+/**
+ * @typedef {object} DeliveryStrategy
+ * @property {Set<{notify: DeliverySubscriber}>} subscribers
+ * @property {() => void} stop
+ * @property {[boolean, import('./flag-evaluations').FlagEvaluationRoute?]} [result]
+ */
+/** @type {WeakMap<import('../../config/config-base'), DeliveryStrategy>} */
+const deliveryStrategies = new WeakMap()
+
 let missingRouteWarningLogged = false
 
 /**
@@ -167,18 +179,45 @@ function setAgentlessStrategy (config, setWriterEnabledValue) {
 }
 
 /**
- * Applies one event-delivery strategy that can be shared by all Feature Flags writers.
+ * Subscribes to the delivery strategy owned by this tracer configuration.
+ * The provider and exposure module receive the same configuration object from
+ * the tracer. Share their route transitions, but never share credentials across
+ * distinct configurations. The last subscriber owns discovery cleanup.
  *
  * @param {import('../../config/config-base')} config - Tracer configuration object
  * @param {(enabled: boolean, route?: import('./flag-evaluations').FlagEvaluationRoute) => void} setWriterEnabledValue
  *   Callback to set the writer enabled state
  */
 function setEventDeliveryStrategy (config, setWriterEnabledValue) {
-  if (config.featureFlags?.DD_FEATURE_FLAGS_CONFIGURATION_SOURCE === 'agentless') {
-    return setAgentlessStrategy(config, setWriterEnabledValue)
+  let strategy = deliveryStrategies.get(config)
+  const subscriber = { notify: setWriterEnabledValue }
+
+  if (strategy) {
+    strategy.subscribers.add(subscriber)
+    if (strategy.result) setWriterEnabledValue(...strategy.result)
+  } else {
+    strategy = { subscribers: new Set([subscriber]), stop: STOP_NOOP }
+    deliveryStrategies.set(config, strategy)
+    /** @type {DeliverySubscriber} */
+    const notify = (enabled, route) => {
+      // A Remote Config discovery already in flight cannot be canceled. Ignore
+      // its result after shutdown, including when the same config is reused.
+      if (deliveryStrategies.get(config) !== strategy) return
+      strategy.result = route === undefined ? [enabled] : [enabled, route]
+      for (const subscriber of strategy.subscribers) {
+        subscriber.notify(...strategy.result)
+      }
+    }
+    strategy.stop = config.featureFlags?.DD_FEATURE_FLAGS_CONFIGURATION_SOURCE === 'agentless'
+      ? setAgentlessStrategy(config, notify)
+      : setAgentStrategy(config, notify)
   }
 
-  return setAgentStrategy(config, setWriterEnabledValue)
+  return () => {
+    if (!strategy.subscribers.delete(subscriber) || strategy.subscribers.size !== 0) return
+    deliveryStrategies.delete(config)
+    strategy.stop()
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/opentelemetry/resource-attributes.js

@@ -30,7 +30,7 @@ function buildResourceAttributes (config) {
   const resourceAttributes = {
     'service.name': config.service,
     'service.version': config.version,
-    'deployment.environment': config.env,
+    'deployment.environment.name': config.env,
     ...tags,
   }
 

package/packages/dd-trace/src/otel-thread-ctx.js

@@ -32,7 +32,7 @@ const {
   enterCh,
   spanFinishCh,
   getActiveSpan,
-  ensureChannelsActivated,
+  acquireChannels,
 } = require('./storage-channels')
 const {
   finalEndpoint,
@@ -371,7 +371,7 @@ function start () {
   getContext = ns.getContext
   clearContext = ns.clearContext
 
-  ensureChannelsActivated(isACFActive)
+  acquireChannels(false)
   enterCh.subscribe(onEnter)
   spanFinishCh.subscribe(onSpanFinished)
   // Endpoint updates come from the shared web-tags cache's transition channels

package/packages/dd-trace/src/plugins/util/url.js

@@ -2,6 +2,7 @@
 
 const { URL } = require('url')
 
+const { defaults } = require('../../config/defaults')
 const log = require('../../log')
 
 const HTTP2_HEADER_AUTHORITY = ':authority'
@@ -55,12 +56,15 @@ function obfuscateQs (config, url) {
 
   let qs = url.slice(i + 1)
 
-  qs = qs.replace(queryStringObfuscation, '<redacted>')
+  // Group 1 of the default regex is the delimiter in front of a JWT. A user-provided regex may capture the secret.
+  const replacement = queryStringObfuscation === defaultQsObfuscator ? '$1<redacted>' : '<redacted>'
+  qs = qs.replace(queryStringObfuscation, replacement)
 
   return `${path}?${qs}`
 }
 
 const qsObfuscatorCache = new Map()
+let defaultQsObfuscator
 
 /**
  * Compile the configured query-string obfuscator (a regex string, or a boolean)
@@ -85,6 +89,7 @@ function getQsObfuscator (config) {
     } else if (obfuscator !== '.*') { // '.*' optimizes to a full redact (true)
       try {
         compiled = new RegExp(obfuscator, 'gi')
+        if (obfuscator === defaults.DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP) defaultQsObfuscator = compiled
       } catch (err) {
         log.error('Error getting qs obfuscator', err)
       }

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -10,7 +10,8 @@ const {
   beforeCh,
   spanFinishCh,
   getActiveSpan,
-  ensureChannelsActivated,
+  acquireChannels,
+  releaseChannels,
 } = require('../../storage-channels')
 const webTagsCache = require('../../web-tags-cache')
 
@@ -128,14 +129,6 @@ class NativeWallProfiler {
     this.#pprof = require('@datadog/pprof')
     kSampleCount = this.#pprof.time.constants.kSampleCount
 
-    // pprof otherwise crashes in worker threads
-    if (!process._startProfilerIdleNotifier) {
-      process._startProfilerIdleNotifier = () => {}
-    }
-    if (!process._stopProfilerIdleNotifier) {
-      process._stopProfilerIdleNotifier = () => {}
-    }
-
     this.#pprof.time.start({
       collectCpuTime: this.#cpuProfilingEnabled,
       columnNumbers: 'emit',
@@ -157,7 +150,7 @@ class NativeWallProfiler {
         this.#profilerState = this.#pprof.time.getState()
         this.#lastSampleCount = 0
 
-        ensureChannelsActivated(this.#asyncContextFrameEnabled)
+        acquireChannels(!this.#asyncContextFrameEnabled)
 
         if (this.#asyncContextFrameEnabled) {
           this.#setupTelemetryMetrics()
@@ -346,6 +339,7 @@ class NativeWallProfiler {
         }
         enterCh.unsubscribe(this.#boundEnter)
         spanFinishCh.unsubscribe(this.#boundSpanFinished)
+        releaseChannels(!this.#asyncContextFrameEnabled)
         if (this.#endpointCollectionEnabled) {
           webTagsCache.resolvedCh.unsubscribe(this.#boundSpanTagsUpdated)
           webTagsCache.deactivate()

package/packages/dd-trace/src/proxy.js

@@ -6,6 +6,7 @@ const NoopProxy = require('./noop/proxy')
 const DatadogTracer = require('./tracer')
 const getConfig = require('./config')
 const { getEnvironmentVariable } = require('./config/helper')
+const { reseed } = require('./id')
 const runtimeMetrics = require('./runtime_metrics')
 const log = require('./log')
 const { setStartupLogPluginManager, startupLog } = require('./startup-log')
@@ -353,6 +354,7 @@ class Tracer extends NoopProxy {
       if (request.method === 'POST' && request.url === '/aws/lambda-microvms/runtime/v1/run') {
         ch.unsubscribe(onHttpRequest)
         drainUuidPool()
+        reseed()
         channel('datadog:identity:update').publish(config)
         if (this._tracingInitialized) {
           const storeTracerMetadata = require('./tracer_metadata')

package/packages/dd-trace/src/serverless.js

@@ -3,6 +3,10 @@
 const { getEnvironmentVariable, getValueFromEnvSources } = require('./config/helper')
 
 const IS_AWS_LAMBDA_MICROVM = getEnvironmentVariable('AWS_LAMBDA_MICROVM_IMAGE_ARN') !== undefined
+// Standard Lambda, whose sandbox freezes between invocations. Excludes Lambda Lite (native-http),
+// Managed Instances (lambda-managed-instances) and MicroVMs.
+const IS_STANDARD_AWS_LAMBDA = !IS_AWS_LAMBDA_MICROVM && ['on-demand', 'provisioned-concurrency', 'snap-start']
+  .includes(getEnvironmentVariable('AWS_LAMBDA_INITIALIZATION_TYPE'))
 const isVercelAtStartup = getEnvironmentVariable('VERCEL') === '1'
 
 function getIsGCPFunction () {
@@ -104,6 +108,7 @@ module.exports = {
   getIsFlexConsumptionAzureFunction,
   initializeServerlessTelemetry,
   IS_AWS_LAMBDA_MICROVM,
+  IS_STANDARD_AWS_LAMBDA,
   // true only for a Node that bundles its own OpenSSL, whose CSPRNG keeps the snapshot's DRBG
   // state across a MicroVM clone resume
   NODE_BUNDLES_OPENSSL: process.config.variables.node_shared_openssl === false,