dd-trace 6.10.0 → 6.12.0

package/packages/dd-trace/src/openfeature/writers/base.js

@@ -11,14 +11,52 @@ const log = require('../../log')
  * @property {number} [timeout] - Request timeout in milliseconds
  * @property {object} config - Tracer configuration object
  * @property {string} endpoint - API endpoint path
- * @property {URL} [agentUrl] - Base URL for the agent
+ * @property {URL} [agentUrl] - Initial delivery URL
  * @property {number} [payloadSizeLimit] - Maximum payload size in bytes
  * @property {number} [eventSizeLimit] - Maximum individual event size in bytes
  * @property {object} [headers] - Additional HTTP headers
  */
 
 /**
- * BaseFFEWriter is the base class for sending Feature Flagging & Exposure Events payloads to the Datadog Agent.
+ * @typedef {object} WriterRoute
+ * @property {URL} url - Route base URL
+ * @property {string} endpoint - Route endpoint
+ * @property {object} headers - Route-specific headers
+ * @property {import('node:https').Agent} [agent] - Optional HTTPS proxy agent
+ */
+
+/**
+ * @typedef {object} ActiveWriterRoute
+ * @property {URL} url - Route base URL
+ * @property {string} endpoint - Route endpoint
+ * @property {object} requestOptions - HTTP request options
+ */
+
+/**
+ * Tests whether a local route definitively rejected an event batch.
+ *
+ * @param {Error | null} error - Request error
+ * @param {number | undefined} statusCode - HTTP response status
+ * @returns {boolean} Whether direct retry is safe
+ */
+function isDefinitiveRejection (error, statusCode) {
+  return error?.code === 'EAI_AGAIN' || error?.code === 'ECONNREFUSED' ||
+    error?.code === 'ENOENT' || error?.code === 'ENOTFOUND' ||
+    statusCode === 403 || statusCode === 404 || statusCode === 405
+}
+
+/**
+ * Tests whether a local route can have accepted an event batch before failing.
+ *
+ * @param {Error | null} error - Request error
+ * @returns {boolean} Whether the delivery result is ambiguous
+ */
+function isAmbiguousNetworkFailure (error) {
+  return error?.code === 'ECONNRESET' || error?.code === 'EPIPE' || error?.code === 'ETIMEDOUT'
+}
+
+/**
+ * Base writer for Feature Flagging and Experimentation event delivery.
  * @class BaseFFEWriter
  */
 class BaseFFEWriter {
@@ -40,6 +78,7 @@ class BaseFFEWriter {
     this._payloadSizeLimit = payloadSizeLimit
     this._eventSizeLimit = eventSizeLimit
     this._headers = headers || {}
+    this._fallbackRoute = undefined
 
     this._requestOptions = {
       headers: {
@@ -47,6 +86,7 @@ class BaseFFEWriter {
         'Content-Type': 'application/json',
       },
       method: 'POST',
+      retry: true,
       timeout: this._timeout,
       url: this._baseUrl,
       path: this._endpoint,
@@ -115,15 +155,8 @@ class BaseFFEWriter {
     // eslint-disable-next-line eslint-rules/eslint-log-printf-style
     log.debug(() => `${this.constructor.name} flushing payload: ${safeJSONStringify(payload)}`)
 
-    request(payload, this._requestOptions, (err, resp, code) => {
-      if (err) {
-        log.error('Failed to send events to %s%s', this._baseUrl.href, this._endpoint, err)
-      } else if (code >= 200 && code < 300) {
-        log.debug('Successfully sent %d events', events.length)
-      } else {
-        log.warn('Events request returned status %d', code)
-      }
-    })
+    const route = this.#createActiveRoute()
+    this.#sendRequest(payload, events.length, route, this._fallbackRoute)
   }
 
   /**
@@ -161,6 +194,115 @@ class BaseFFEWriter {
   _encode (payload) {
     return JSON.stringify(payload)
   }
+
+  /**
+   * Applies the active route and an optional direct fallback route.
+   *
+   * @param {WriterRoute} route - Active route
+   * @param {WriterRoute} [fallbackRoute] - Direct fallback route
+   * @returns {void}
+   */
+  _setRoutes (route, fallbackRoute) {
+    this.#activateRoute(this.#createRoute(route))
+    this._fallbackRoute = fallbackRoute ? this.#createRoute(fallbackRoute) : undefined
+  }
+
+  /**
+   * Creates request state for a configured writer route.
+   *
+   * @param {WriterRoute} route - Configured route
+   * @returns {ActiveWriterRoute} Active route state
+   */
+  #createRoute (route) {
+    return {
+      url: route.url,
+      endpoint: route.endpoint,
+      requestOptions: {
+        ...(route.agent && { agent: route.agent }),
+        headers: {
+          ...route.headers,
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+        retry: true,
+        timeout: this._timeout,
+        url: route.url,
+        path: route.endpoint,
+      },
+    }
+  }
+
+  /**
+   * Captures the current route for one event batch.
+   *
+   * @returns {ActiveWriterRoute} Active route state
+   */
+  #createActiveRoute () {
+    return {
+      url: this._baseUrl,
+      endpoint: this._endpoint,
+      requestOptions: this._requestOptions,
+    }
+  }
+
+  /**
+   * Makes a route active for future event batches.
+   *
+   * @param {ActiveWriterRoute} route - Route state
+   * @returns {void}
+   */
+  #activateRoute (route) {
+    this._baseUrl = route.url
+    this._endpoint = route.endpoint
+    this._requestOptions = route.requestOptions
+  }
+
+  /**
+   * Sends an encoded batch and retries it through direct intake after a local route failure.
+   *
+   * @param {string} payload - Encoded event batch
+   * @param {number} eventCount - Event count
+   * @param {ActiveWriterRoute} route - Selected route
+   * @param {ActiveWriterRoute} [fallbackRoute] - Direct fallback route
+   * @returns {void}
+   */
+  #sendRequest (payload, eventCount, route, fallbackRoute) {
+    request(payload, route.requestOptions, (error, response, statusCode) => {
+      if (fallbackRoute && isDefinitiveRejection(error, statusCode)) {
+        log.debug(
+          '%s switching from %s%s to direct intake after definitive rejection',
+          this.constructor.name,
+          route.url.href,
+          route.endpoint
+        )
+        this.#activateRoute(fallbackRoute)
+        this._fallbackRoute = undefined
+        this.#sendRequest(payload, eventCount, fallbackRoute)
+        return
+      }
+
+      if (fallbackRoute && isAmbiguousNetworkFailure(error)) {
+        log.debug(
+          '%s retrying through direct intake and switching future batches from %s%s after ambiguous failure',
+          this.constructor.name,
+          route.url.href,
+          route.endpoint
+        )
+        this.#activateRoute(fallbackRoute)
+        this._fallbackRoute = undefined
+        this.#sendRequest(payload, eventCount, fallbackRoute)
+        return
+      }
+
+      if (error) {
+        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
+      } else if (statusCode >= 200 && statusCode < 300) {
+        log.debug('Successfully sent %d events', eventCount)
+      } else {
+        log.warn('Events request returned status %d', statusCode)
+      }
+    })
+  }
 }
 
 module.exports = BaseFFEWriter

package/packages/dd-trace/src/openfeature/writers/exposures.js

@@ -2,12 +2,15 @@
 
 const {
   EXPOSURES_ENDPOINT,
-  EVP_PROXY_AGENT_BASE_PATH,
-  EVP_SUBDOMAIN_HEADER_NAME,
-  EVP_SUBDOMAIN_VALUE,
   EVP_PAYLOAD_SIZE_LIMIT,
   EVP_EVENT_SIZE_LIMIT,
 } = require('../constants/constants')
+const {
+  EVP_EVENT_PLATFORM_SUBDOMAIN,
+  EVP_PROXY_PATH_V2,
+  EVP_SUBDOMAIN_HEADER_NAME,
+} = require('../../evp_proxy/constants')
+const { joinEVPProxyPath } = require('../../evp_proxy/path')
 const log = require('../../log')
 const BaseFFEWriter = require('./base')
 
@@ -16,6 +19,15 @@ const BaseFFEWriter = require('./base')
 // drop emits a warning and `droppedEventCount` accumulates the cumulative loss.
 const PENDING_MAX_EVENTS = 1000
 
+/**
+ * @typedef {object} ExposureRoute
+ * @property {URL} url - Route base URL
+ * @property {string} basePath - EVP base path
+ * @property {object} [headers] - Route-specific headers
+ * @property {import('node:https').Agent} [agent] - Optional HTTPS proxy agent
+ * @property {ExposureRoute} [fallback] - Optional direct fallback route
+ */
+
 /**
  * @typedef {object} ExposureEvent
  * @property {number} timestamp - Unix timestamp in milliseconds
@@ -45,10 +57,10 @@ const PENDING_MAX_EVENTS = 1000
  */
 
 /**
- * ExposuresWriter is responsible for sending exposure events to the Datadog Agent.
+ * Sends exposure events through the selected local EVP proxy or direct intake route.
  */
 class ExposuresWriter extends BaseFFEWriter {
-  // Disabled until the agent strategy probe resolves.
+  // Disabled until route selection resolves.
   #enabled = false
 
   /** @type {ExposureEvent[]} */
@@ -61,22 +73,27 @@ class ExposuresWriter extends BaseFFEWriter {
 
   /**
    * @param {import('../../config/config-base')} config - Tracer configuration object
+   * @param {ExposureRoute} [route] - Caller-supplied route
    */
-  constructor (config) {
-    const basePath = EVP_PROXY_AGENT_BASE_PATH.replace(/\/$/, '')
-    const endpoint = EXPOSURES_ENDPOINT.replace(/^\/+/, '')
-    const fullEndpoint = `${basePath}/${endpoint}`
+  constructor (config, route) {
+    route ??= { url: config.url, basePath: EVP_PROXY_PATH_V2 }
+    const headers = route.headers ?? {
+      [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+    }
 
     super({
       config,
-      endpoint: fullEndpoint,
+      agentUrl: route.url,
+      endpoint: joinEVPProxyPath(route.basePath, EXPOSURES_ENDPOINT),
       payloadSizeLimit: EVP_PAYLOAD_SIZE_LIMIT,
       eventSizeLimit: EVP_EVENT_SIZE_LIMIT,
-      headers: {
-        [EVP_SUBDOMAIN_HEADER_NAME]: EVP_SUBDOMAIN_VALUE,
-      },
+      headers,
     })
 
+    if (route.agent || route.fallback) {
+      this.#setRoute({ ...route, headers })
+    }
+
     /** @type {ExposureContext} */
     const context = {
       service: config.service,
@@ -95,8 +112,14 @@ class ExposuresWriter extends BaseFFEWriter {
 
   /**
    * @param {boolean} enabled - Whether to enable the writer
+   * @param {ExposureRoute} [route] - Selected EVP route
+   * @returns {void}
    */
-  setEnabled (enabled) {
+  setEnabled (enabled, route) {
+    if (route) {
+      this.#setRoute(route)
+    }
+
     this.#enabled = enabled
 
     if (enabled && this.#pendingEvents.length > 0) {
@@ -106,6 +129,31 @@ class ExposuresWriter extends BaseFFEWriter {
     }
   }
 
+  /**
+   * Applies caller-supplied route data without performing discovery.
+   *
+   * @param {ExposureRoute} route - Selected EVP route
+   * @returns {void}
+   */
+  #setRoute (route) {
+    const fallbackRoute = route.fallback && {
+      url: route.fallback.url,
+      endpoint: joinEVPProxyPath(route.fallback.basePath, EXPOSURES_ENDPOINT),
+      headers: route.fallback.headers ?? {},
+      agent: route.fallback.agent,
+    }
+    const headers = route.headers ?? {
+      [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+    }
+
+    this._setRoutes({
+      url: route.url,
+      endpoint: joinEVPProxyPath(route.basePath, EXPOSURES_ENDPOINT),
+      headers,
+      agent: route.agent,
+    }, fallbackRoute)
+  }
+
   /**
    * Appends exposure event(s) to the buffer
    * @param {ExposureEvent|ExposureEvent[]} events - Exposure event(s) to append

package/packages/dd-trace/src/openfeature/writers/util.js

@@ -1,30 +1,51 @@
 'use strict'
 
+const {
+  EVP_EVENT_PLATFORM_SUBDOMAIN,
+  EVP_PROXY_PATH_V2,
+  EVP_PROXY_PATH_V4,
+  EVP_SUBDOMAIN_HEADER_NAME,
+} = require('../../evp_proxy/constants')
+const { createDirectEVPRoute } = require('../../evp_proxy/direct')
+const { discoverEVPProxy } = require('../../evp_proxy/discovery')
 const logger = require('../../log')
-const { EVP_PROXY_AGENT_BASE_PATH } = require('../constants/constants')
-const { fetchAgentInfo } = require('../../agent/info')
+
+let missingRouteWarningLogged = false
 
 /**
- * Determines if the agent supports EVP proxy and sets the writer enabled state accordingly
+ * Logs the unavailable exposure-delivery warning once.
+ *
+ * @returns {void}
+ */
+function warnExposureDeliveryUnavailable () {
+  if (missingRouteWarningLogged) return
+  missingRouteWarningLogged = true
+  logger.warn(
+    'Feature Flags exposure delivery is disabled because no compatible local EVP route or direct intake ' +
+    'credentials are available.'
+  )
+}
+
+/**
+ * Preserves Agent exposure delivery for the Remote Configuration source.
+ *
  * @param {import('../../config')} config - Tracer configuration object
  * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
  */
 function setAgentStrategy (config, setWriterEnabledValue) {
-  fetchAgentInfo(config.url, (err, agentInfo) => {
-    if (err) {
-      logger.debug('FFE Writer disabled - error getting agent info:', err.message)
+  discoverEVPProxy(config.url, {
+    supportedPaths: [EVP_PROXY_PATH_V2],
+  }, (error, route) => {
+    if (error) {
+      logger.debug('FFE Writer disabled - error getting agent info: %s', error.message)
       setWriterEnabledValue(false)
       return
     }
 
-    const endpoints = agentInfo.endpoints
-    const normalizedPath = EVP_PROXY_AGENT_BASE_PATH.replace(/\/$/, '')
-    const hasEndpoint = Array.isArray(endpoints) &&
-      endpoints.includes(normalizedPath) || endpoints.includes(normalizedPath + '/')
-
-    if (hasEndpoint) {
+    if (route) {
       logger.debug('FFE Writer enabled - agent has EVP proxy support')
-      setWriterEnabledValue(true)
+      setWriterEnabledValue(true, route)
     } else {
       logger.debug('FFE Writer disabled - agent does not have EVP proxy support')
       setWriterEnabledValue(false)
@@ -32,6 +53,67 @@ function setAgentStrategy (config, setWriterEnabledValue) {
   })
 }
 
-module.exports = {
-  setAgentStrategy,
+/**
+ * Selects a local serverless receiver or authenticated direct intake.
+ *
+ * Local discovery is optional for delivery. A missing listener, discovery
+ * error, or incompatible receiver selects direct intake when credentials exist.
+ *
+ * @param {import('../../config')} config - Tracer configuration object
+ * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
+ */
+function setAgentlessStrategy (config, setWriterEnabledValue) {
+  const directRoute = createDirectEVPRoute(config, EVP_EVENT_PLATFORM_SUBDOMAIN)
+
+  discoverEVPProxy(config.url, {
+    supportedPaths: [EVP_PROXY_PATH_V4, EVP_PROXY_PATH_V2],
+  }, (error, localRoute) => {
+    if (localRoute) {
+      const route = {
+        ...localRoute,
+        headers: {
+          [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+        },
+        ...(directRoute && { fallback: directRoute }),
+      }
+      logger.debug('FFE Writer enabled with local EVP route %s', route.basePath)
+      setWriterEnabledValue(true, route)
+      return
+    }
+
+    if (directRoute) {
+      if (error) {
+        logger.debug('FFE Writer using direct EVP intake after local discovery failed: %s', error.message)
+      } else {
+        logger.debug('FFE Writer using direct EVP intake because no compatible local route was advertised')
+      }
+      setWriterEnabledValue(true, directRoute)
+      return
+    }
+
+    if (error) {
+      logger.debug('FFE Writer disabled - error getting local receiver info: %s', error.message)
+    }
+    warnExposureDeliveryUnavailable()
+    setWriterEnabledValue(false)
+  })
 }
+
+/**
+ * Applies the exposure-delivery strategy for the configured Feature Flags source.
+ *
+ * @param {import('../../config')} config - Tracer configuration object
+ * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
+ */
+function setExposureDeliveryStrategy (config, setWriterEnabledValue) {
+  if (config.featureFlags?.DD_FEATURE_FLAGS_CONFIGURATION_SOURCE === 'agentless') {
+    setAgentlessStrategy(config, setWriterEnabledValue)
+    return
+  }
+
+  setAgentStrategy(config, setWriterEnabledValue)
+}
+
+module.exports = { setExposureDeliveryStrategy }

package/packages/dd-trace/src/opentelemetry/context_manager.js

@@ -21,14 +21,14 @@ class ContextManager {
     const storedSpan = store ? trace.getSpan(store) : null
 
     // Convert DD baggage to OTel format
-    const baggages = getAllBaggageItems()
-    const hasBaggage = Object.keys(baggages).length > 0
+    let entries
+    for (const [key, value] of Object.entries(getAllBaggageItems())) {
+      entries ??= {}
+      entries[key] = { value }
+    }
+
     let otelBaggages
-    if (hasBaggage) {
-      const entries = {}
-      for (const [key, value] of Object.entries(baggages)) {
-        entries[key] = { value }
-      }
+    if (entries !== undefined) {
       otelBaggages = propagation.createBaggage(entries)
     }
 

package/packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js

@@ -42,7 +42,8 @@ class BatchLogRecordProcessor {
    */
   onEmit (logRecord, instrumentationScope) {
     // Store the log record (already enriched by Logger.emit)
-    this.#logRecords.push({ ...logRecord, instrumentationScope })
+    logRecord.instrumentationScope = instrumentationScope
+    this.#logRecords.push(logRecord)
 
     if (this.#logRecords.length >= this.#maxExportBatchSize) {
       this.#export()

package/packages/dd-trace/src/opentelemetry/logs/logger.js

@@ -1,8 +1,65 @@
 'use strict'
 
 const { context } = require('@opentelemetry/api')
-const { sanitizeAttributes } = require('../../../../../vendor/dist/@opentelemetry/core')
+const {
+  millisToHrTime,
+  sanitizeAttributes,
+  timeInputToHrTime,
+} = require('../../../../../vendor/dist/@opentelemetry/core')
 const { VERSION: packageVersion } = require('../../../../../version')
+const log = require('../../log')
+
+const MAX_DATE_MILLISECONDS = 8.64e15
+const NANOSECONDS_PER_SECOND = 1e9
+const MAX_UINT64_SECONDS = 18_446_744_073
+const MAX_UINT64_NANOSECONDS = 709_551_615
+let invalidTimestampWarningLogged = false
+
+function isValidHrTime (hrTime) {
+  return Array.isArray(hrTime) &&
+    hrTime.length === 2 &&
+    Number.isInteger(hrTime[0]) &&
+    Number.isInteger(hrTime[1]) &&
+    hrTime[0] >= 0 &&
+    hrTime[1] >= 0 &&
+    hrTime[1] < NANOSECONDS_PER_SECOND &&
+    (hrTime[0] < MAX_UINT64_SECONDS ||
+      (hrTime[0] === MAX_UINT64_SECONDS && hrTime[1] <= MAX_UINT64_NANOSECONDS))
+}
+
+function toHrTime (timestamp) {
+  if (typeof timestamp === 'number') {
+    if (!Number.isFinite(timestamp)) return
+
+    // Older versions documented numeric timestamps as Unix nanoseconds.
+    if (Math.abs(timestamp) > MAX_DATE_MILLISECONDS) {
+      const seconds = Math.trunc(timestamp / NANOSECONDS_PER_SECOND)
+      const hrTime = [seconds, timestamp - seconds * NANOSECONDS_PER_SECOND]
+      return isValidHrTime(hrTime) ? hrTime : undefined
+    }
+  }
+
+  try {
+    const hrTime = timeInputToHrTime(timestamp)
+    return isValidHrTime(hrTime) ? hrTime : undefined
+  } catch {
+    // Invalid timestamps fall back to the current time in normalizeTimestamp.
+  }
+}
+
+function normalizeTimestamp (timestamp) {
+  if (timestamp !== undefined) {
+    const hrTime = toHrTime(timestamp)
+    if (hrTime) return hrTime
+
+    if (!invalidTimestampWarningLogged) {
+      invalidTimestampWarningLogged = true
+      log.warn('Invalid OpenTelemetry log timestamp; using the current time instead')
+    }
+  }
+
+  return millisToHrTime(Date.now())
+}
 
 /**
  * @typedef {import('@opentelemetry/api-logs').LogRecord} LogRecord
@@ -62,21 +119,20 @@ class Logger {
       return
     }
 
-    if (logRecord.attributes) {
-      logRecord.attributes = sanitizeAttributes(logRecord.attributes)
+    const record = {
+      ...logRecord,
+      timestamp: normalizeTimestamp(logRecord.timestamp),
+      context: logRecord.context || context.active(),
     }
 
-    // Note: timestamp is in nanoseconds (as defined by OpenTelemetry LogRecord API)
-    if (!logRecord.timestamp) {
-      logRecord.timestamp = Number(process.hrtime.bigint())
+    if (logRecord.observedTimestamp !== undefined) {
+      record.observedTimestamp = normalizeTimestamp(logRecord.observedTimestamp)
     }
-
-    if (!logRecord.context) {
-      // Store span context in the log record context for trace correlation
-      logRecord.context = context.active()
+    if (logRecord.attributes) {
+      record.attributes = sanitizeAttributes(logRecord.attributes)
     }
 
-    this.loggerProvider.processor.onEmit(logRecord, this.#instrumentationScope)
+    this.loggerProvider.processor.onEmit(record, this.#instrumentationScope)
   }
 }
 

package/packages/dd-trace/src/opentelemetry/logs/otlp_transformer.js

@@ -38,6 +38,17 @@ const SEVERITY_MAP = {
   [SeverityNumber.FATAL4]: 'SEVERITY_NUMBER_FATAL4',
 }
 
+const NANOSECONDS_PER_SECOND = 1_000_000_000n
+
+/**
+ * Converts an OpenTelemetry HrTime to an exact OTLP uint64 decimal string.
+ * @param {import('@opentelemetry/api').HrTime} hrTime
+ * @returns {string}
+ */
+function encodeHrTime (hrTime) {
+  return (BigInt(hrTime[0]) * NANOSECONDS_PER_SECOND + BigInt(hrTime[1])).toString()
+}
+
 /**
  * OtlpTransformer transforms log records to OTLP format.
  *
@@ -138,13 +149,13 @@ class OtlpTransformer extends OtlpTransformerBase {
     const spanContext = this.#extractSpanContext(logRecord.context)
 
     const result = {
-      timeUnixNano: logRecord.timestamp,
+      timeUnixNano: encodeHrTime(logRecord.timestamp),
       body: this.#transformBody(logRecord.body),
     }
 
     // Add optional fields only if they are set
-    if (logRecord.observedTimestamp) {
-      result.observedTimeUnixNano = logRecord.observedTimestamp
+    if (logRecord.observedTimestamp !== undefined) {
+      result.observedTimeUnixNano = encodeHrTime(logRecord.observedTimestamp)
     }
 
     if (logRecord.severityNumber !== undefined) {