dd-trace 6.11.0 → 6.13.0

package/packages/dd-trace/src/openfeature/writers/exposures.js

@@ -2,12 +2,15 @@
 
 const {
   EXPOSURES_ENDPOINT,
-  EVP_PROXY_AGENT_BASE_PATH,
-  EVP_SUBDOMAIN_HEADER_NAME,
-  EVP_SUBDOMAIN_VALUE,
   EVP_PAYLOAD_SIZE_LIMIT,
   EVP_EVENT_SIZE_LIMIT,
 } = require('../constants/constants')
+const {
+  EVP_EVENT_PLATFORM_SUBDOMAIN,
+  EVP_PROXY_PATH_V2,
+  EVP_SUBDOMAIN_HEADER_NAME,
+} = require('../../evp_proxy/constants')
+const { joinEVPProxyPath } = require('../../evp_proxy/path')
 const log = require('../../log')
 const BaseFFEWriter = require('./base')
 
@@ -16,6 +19,15 @@ const BaseFFEWriter = require('./base')
 // drop emits a warning and `droppedEventCount` accumulates the cumulative loss.
 const PENDING_MAX_EVENTS = 1000
 
+/**
+ * @typedef {object} ExposureRoute
+ * @property {URL} url - Route base URL
+ * @property {string} basePath - EVP base path
+ * @property {object} [headers] - Route-specific headers
+ * @property {import('node:https').Agent} [agent] - Optional HTTPS proxy agent
+ * @property {ExposureRoute} [fallback] - Optional direct fallback route
+ */
+
 /**
  * @typedef {object} ExposureEvent
  * @property {number} timestamp - Unix timestamp in milliseconds
@@ -25,6 +37,7 @@ const PENDING_MAX_EVENTS = 1000
  * @property {string} flag.key - Flag key
  * @property {object} variant - Variant information
  * @property {string} variant.key - Variant key
+ * @property {number} [serial_id] - Serial id of the split the subject landed in
  * @property {object} subject - Subject (user/entity) information
  * @property {string} subject.id - Subject identifier
  * @property {string} [subject.type] - Subject type
@@ -45,10 +58,10 @@ const PENDING_MAX_EVENTS = 1000
  */
 
 /**
- * ExposuresWriter is responsible for sending exposure events to the Datadog Agent.
+ * Sends exposure events through the selected local EVP proxy or direct intake route.
  */
 class ExposuresWriter extends BaseFFEWriter {
-  // Disabled until the agent strategy probe resolves.
+  // Disabled until route selection resolves.
   #enabled = false
 
   /** @type {ExposureEvent[]} */
@@ -61,22 +74,27 @@ class ExposuresWriter extends BaseFFEWriter {
 
   /**
    * @param {import('../../config/config-base')} config - Tracer configuration object
+   * @param {ExposureRoute} [route] - Caller-supplied route
    */
-  constructor (config) {
-    const basePath = EVP_PROXY_AGENT_BASE_PATH.replace(/\/$/, '')
-    const endpoint = EXPOSURES_ENDPOINT.replace(/^\/+/, '')
-    const fullEndpoint = `${basePath}/${endpoint}`
+  constructor (config, route) {
+    route ??= { url: config.url, basePath: EVP_PROXY_PATH_V2 }
+    const headers = route.headers ?? {
+      [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+    }
 
     super({
       config,
-      endpoint: fullEndpoint,
+      agentUrl: route.url,
+      endpoint: joinEVPProxyPath(route.basePath, EXPOSURES_ENDPOINT),
       payloadSizeLimit: EVP_PAYLOAD_SIZE_LIMIT,
       eventSizeLimit: EVP_EVENT_SIZE_LIMIT,
-      headers: {
-        [EVP_SUBDOMAIN_HEADER_NAME]: EVP_SUBDOMAIN_VALUE,
-      },
+      headers,
     })
 
+    if (route.agent || route.fallback) {
+      this.#setRoute({ ...route, headers })
+    }
+
     /** @type {ExposureContext} */
     const context = {
       service: config.service,
@@ -95,8 +113,14 @@ class ExposuresWriter extends BaseFFEWriter {
 
   /**
    * @param {boolean} enabled - Whether to enable the writer
+   * @param {ExposureRoute} [route] - Selected EVP route
+   * @returns {void}
    */
-  setEnabled (enabled) {
+  setEnabled (enabled, route) {
+    if (route) {
+      this.#setRoute(route)
+    }
+
     this.#enabled = enabled
 
     if (enabled && this.#pendingEvents.length > 0) {
@@ -106,6 +130,31 @@ class ExposuresWriter extends BaseFFEWriter {
     }
   }
 
+  /**
+   * Applies caller-supplied route data without performing discovery.
+   *
+   * @param {ExposureRoute} route - Selected EVP route
+   * @returns {void}
+   */
+  #setRoute (route) {
+    const fallbackRoute = route.fallback && {
+      url: route.fallback.url,
+      endpoint: joinEVPProxyPath(route.fallback.basePath, EXPOSURES_ENDPOINT),
+      headers: route.fallback.headers ?? {},
+      agent: route.fallback.agent,
+    }
+    const headers = route.headers ?? {
+      [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+    }
+
+    this._setRoutes({
+      url: route.url,
+      endpoint: joinEVPProxyPath(route.basePath, EXPOSURES_ENDPOINT),
+      headers,
+      agent: route.agent,
+    }, fallbackRoute)
+  }
+
   /**
    * Appends exposure event(s) to the buffer
    * @param {ExposureEvent|ExposureEvent[]} events - Exposure event(s) to append
@@ -156,7 +205,7 @@ class ExposuresWriter extends BaseFFEWriter {
   makePayload (events) {
     const formattedEvents = events.map(event => {
       /** @type {ExposureEvent} */
-      return {
+      const formatted = {
         timestamp: event.timestamp || Date.now(),
         allocation: {
           key: event.allocation?.key || event['allocation.key'],
@@ -173,6 +222,12 @@ class ExposuresWriter extends BaseFFEWriter {
           attributes: event.subject?.attributes,
         },
       }
+
+      if (typeof event.serial_id === 'number') {
+        formatted.serial_id = event.serial_id
+      }
+
+      return formatted
     })
 
     return {

package/packages/dd-trace/src/openfeature/writers/util.js

@@ -1,30 +1,51 @@
 'use strict'
 
+const {
+  EVP_EVENT_PLATFORM_SUBDOMAIN,
+  EVP_PROXY_PATH_V2,
+  EVP_PROXY_PATH_V4,
+  EVP_SUBDOMAIN_HEADER_NAME,
+} = require('../../evp_proxy/constants')
+const { createDirectEVPRoute } = require('../../evp_proxy/direct')
+const { discoverEVPProxy } = require('../../evp_proxy/discovery')
 const logger = require('../../log')
-const { EVP_PROXY_AGENT_BASE_PATH } = require('../constants/constants')
-const { fetchAgentInfo } = require('../../agent/info')
+
+let missingRouteWarningLogged = false
 
 /**
- * Determines if the agent supports EVP proxy and sets the writer enabled state accordingly
+ * Logs the unavailable exposure-delivery warning once.
+ *
+ * @returns {void}
+ */
+function warnExposureDeliveryUnavailable () {
+  if (missingRouteWarningLogged) return
+  missingRouteWarningLogged = true
+  logger.warn(
+    'Feature Flags exposure delivery is disabled because no compatible local EVP route or direct intake ' +
+    'credentials are available.'
+  )
+}
+
+/**
+ * Preserves Agent exposure delivery for the Remote Configuration source.
+ *
  * @param {import('../../config')} config - Tracer configuration object
  * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
  */
 function setAgentStrategy (config, setWriterEnabledValue) {
-  fetchAgentInfo(config.url, (err, agentInfo) => {
-    if (err) {
-      logger.debug('FFE Writer disabled - error getting agent info:', err.message)
+  discoverEVPProxy(config.url, {
+    supportedPaths: [EVP_PROXY_PATH_V2],
+  }, (error, route) => {
+    if (error) {
+      logger.debug('FFE Writer disabled - error getting agent info: %s', error.message)
       setWriterEnabledValue(false)
       return
     }
 
-    const endpoints = agentInfo.endpoints
-    const normalizedPath = EVP_PROXY_AGENT_BASE_PATH.replace(/\/$/, '')
-    const hasEndpoint = Array.isArray(endpoints) &&
-      endpoints.includes(normalizedPath) || endpoints.includes(normalizedPath + '/')
-
-    if (hasEndpoint) {
+    if (route) {
       logger.debug('FFE Writer enabled - agent has EVP proxy support')
-      setWriterEnabledValue(true)
+      setWriterEnabledValue(true, route)
     } else {
       logger.debug('FFE Writer disabled - agent does not have EVP proxy support')
       setWriterEnabledValue(false)
@@ -32,6 +53,67 @@ function setAgentStrategy (config, setWriterEnabledValue) {
   })
 }
 
-module.exports = {
-  setAgentStrategy,
+/**
+ * Selects a local serverless receiver or authenticated direct intake.
+ *
+ * Local discovery is optional for delivery. A missing listener, discovery
+ * error, or incompatible receiver selects direct intake when credentials exist.
+ *
+ * @param {import('../../config')} config - Tracer configuration object
+ * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
+ */
+function setAgentlessStrategy (config, setWriterEnabledValue) {
+  const directRoute = createDirectEVPRoute(config, EVP_EVENT_PLATFORM_SUBDOMAIN)
+
+  discoverEVPProxy(config.url, {
+    supportedPaths: [EVP_PROXY_PATH_V4, EVP_PROXY_PATH_V2],
+  }, (error, localRoute) => {
+    if (localRoute) {
+      const route = {
+        ...localRoute,
+        headers: {
+          [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
+        },
+        ...(directRoute && { fallback: directRoute }),
+      }
+      logger.debug('FFE Writer enabled with local EVP route %s', route.basePath)
+      setWriterEnabledValue(true, route)
+      return
+    }
+
+    if (directRoute) {
+      if (error) {
+        logger.debug('FFE Writer using direct EVP intake after local discovery failed: %s', error.message)
+      } else {
+        logger.debug('FFE Writer using direct EVP intake because no compatible local route was advertised')
+      }
+      setWriterEnabledValue(true, directRoute)
+      return
+    }
+
+    if (error) {
+      logger.debug('FFE Writer disabled - error getting local receiver info: %s', error.message)
+    }
+    warnExposureDeliveryUnavailable()
+    setWriterEnabledValue(false)
+  })
 }
+
+/**
+ * Applies the exposure-delivery strategy for the configured Feature Flags source.
+ *
+ * @param {import('../../config')} config - Tracer configuration object
+ * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
+ * @returns {void}
+ */
+function setExposureDeliveryStrategy (config, setWriterEnabledValue) {
+  if (config.featureFlags?.DD_FEATURE_FLAGS_CONFIGURATION_SOURCE === 'agentless') {
+    setAgentlessStrategy(config, setWriterEnabledValue)
+    return
+  }
+
+  setAgentStrategy(config, setWriterEnabledValue)
+}
+
+module.exports = { setExposureDeliveryStrategy }

package/packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js

@@ -1,5 +1,8 @@
 'use strict'
 
+const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
+
 /**
  * @typedef {import('@opentelemetry/api-logs').LogRecord} LogRecord
  * @typedef {import('@opentelemetry/core').InstrumentationScope} InstrumentationScope
@@ -54,10 +57,48 @@ class BatchLogRecordProcessor {
 
   /**
    * Forces an immediate flush of all pending log records.
-   * @returns {undefined} Promise that resolves when flush is complete
+   * @param {Function} [done] Called after all pending log exports complete
    */
-  forceFlush () {
-    this.#export()
+  forceFlush (done) {
+    this.#clearTimer()
+
+    const deliveryTracker = createServerlessDeliveryTracker()
+    if (!deliveryTracker) {
+      // Normal processes preserve the existing fire-and-forget batch flush.
+      this.#export()
+      done?.()
+      return
+    }
+
+    // Flush only records present at this boundary. New records belong to the
+    // later request that produced them and must not extend this lifecycle flush.
+    const logRecords = this.#logRecords
+    this.#logRecords = []
+
+    // Join exports already active at this boundary before draining this snapshot.
+    if (typeof this.exporter.flush === 'function') {
+      deliveryTracker.track(complete => this.exporter.flush(complete))
+    }
+
+    deliveryTracker.track(complete => {
+      const flushNext = () => {
+        if (logRecords.length === 0) {
+          complete()
+          return
+        }
+
+        // Drain the boundary snapshot one batch at a time.
+        const batch = logRecords.splice(0, this.#maxExportBatchSize)
+        try {
+          this.exporter.export(batch, flushNext)
+        } catch (error) {
+          log.error('Error exporting OTLP logs:', error)
+          complete()
+        }
+      }
+      flushNext()
+    })
+    deliveryTracker.waitForIdle(done)
   }
 
   /**
@@ -79,6 +120,7 @@ class BatchLogRecordProcessor {
    * @private
    */
   #export () {
+    if (this.#logRecords.length === 0) return
     const logRecords = this.#logRecords.slice(0, this.#maxExportBatchSize)
     this.#logRecords = this.#logRecords.slice(this.#maxExportBatchSize)
 

package/packages/dd-trace/src/opentelemetry/logs/index.js

@@ -27,6 +27,7 @@ const os = require('os')
  * @package
  */
 
+const { registerTelemetryFlusher } = require('../../flush')
 const LoggerProvider = require('./logger_provider')
 const BatchLogRecordProcessor = require('./batch_log_processor')
 const OtlpHttpLogExporter = require('./otlp_http_log_exporter')
@@ -77,8 +78,10 @@ function initializeOpenTelemetryLogs (config) {
   // Create logger provider with processor for Datadog Agent export
   const loggerProvider = new LoggerProvider({ processor })
 
-  // Register the logger provider globally with OpenTelemetry API
+  // Expose this provider to application calls through the OpenTelemetry Logs API.
   loggerProvider.register()
+  // Include final log batches in lifecycle retention with trace delivery.
+  registerTelemetryFlusher(done => loggerProvider.forceFlush(done))
 }
 
 module.exports = {

package/packages/dd-trace/src/opentelemetry/logs/logger_provider.js

@@ -84,12 +84,15 @@ class LoggerProvider {
 
   /**
    * Forces a flush of all pending log records.
-   * @returns {undefined} Promise that resolves when flush is n ssue cncomplete
+   * @param {Function} [done] Called after all pending log exports complete
    */
-  forceFlush () {
-    if (!this.isShutdown) {
-      return this.processor?.forceFlush()
+  forceFlush (done) {
+    if (this.isShutdown || !this.processor) {
+      done?.()
+      return
     }
+
+    this.processor.forceFlush(done)
   }
 
   /**

package/packages/dd-trace/src/opentelemetry/metrics/index.js

@@ -6,12 +6,12 @@ const { metrics } = require('@opentelemetry/api')
 
 const { VERSION } = require('../../../../../version')
 const processTags = require('../../process-tags')
+const { registerTelemetryFlusher } = require('../../flush')
 const MeterProvider = require('./meter_provider')
 const PeriodicMetricReader = require('./periodic_metric_reader')
 const OtlpHttpMetricExporter = require('./otlp_http_metric_exporter')
 
 const RESERVED_TRACER_TAGS = new Set(['service', 'env', 'version', 'runtime_id', 'runtime-id'])
-
 /**
  * @typedef {import('../../config')} Config
  */
@@ -78,6 +78,8 @@ function initializeOpenTelemetryMetrics (config) {
 
   const meterProvider = new MeterProvider({ reader })
   metrics.setGlobalMeterProvider(meterProvider)
+  // Include the final metric collection and export in lifecycle retention.
+  registerTelemetryFlusher(done => meterProvider.forceFlush(done))
 }
 
 /**

package/packages/dd-trace/src/opentelemetry/metrics/meter_provider.js

@@ -49,6 +49,14 @@ class MeterProvider {
     }
     return meter
   }
+
+  /**
+   * @param {Function} [done] Called after the metric export completes
+   */
+  forceFlush (done) {
+    if (this.reader) this.reader.forceFlush(done)
+    else done?.()
+  }
 }
 
 module.exports = MeterProvider

package/packages/dd-trace/src/opentelemetry/metrics/otlp_http_metric_exporter.js

@@ -34,10 +34,11 @@ class OtlpHttpMetricExporter extends OtlpHttpExporterBase {
    *
    * @param {Map<string, AggregatedMetric>} metrics - Map of metric data to export
    *
-   * @returns {void}
+   * @param {Function} [done] Called after the HTTP export completes
    */
-  export (metrics) {
+  export (metrics, done) {
     if (metrics.size === 0) {
+      done?.({ code: 0 })
       return
     }
 
@@ -56,6 +57,7 @@ class OtlpHttpMetricExporter extends OtlpHttpExporterBase {
       if (result.code === 0) {
         this.recordTelemetry('otel.metrics_export_successes', 1, additionalTags)
       }
+      done?.(result)
     })
   }
 }

package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_exporter.js

@@ -22,14 +22,16 @@ class OtlpStatsExporter extends OtlpHttpExporterBase {
   /**
    * @param {Array<{timeNs: number, bucket: import('../../span_stats').SpanBuckets}>} drained
    * @param {number} bucketSizeNs
+   * @param {Function} [done] Called after the HTTP export completes
    */
-  export (drained, bucketSizeNs) {
-    if (drained.length === 0) return
+  export (drained, bucketSizeNs, done) {
+    if (drained.length === 0) return done?.()
     const payload = this.#transformer.transform(drained, bucketSizeNs)
     this.sendPayload(payload, (result) => {
       if (result.code !== 0) {
         log.error('Failed to export span stats: %s', result.error?.message)
       }
+      done?.()
     })
   }
 }

package/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js

@@ -197,14 +197,33 @@ class PeriodicMetricReader {
 
   /**
    * Forces an immediate collection and export of all metrics.
-   * @returns {void}
+   * @param {Function} [done] Called after the metric export completes
    */
-  forceFlush () {
+  forceFlush (done) {
     if (this.#isShutdown) {
       log.warn('PeriodicMetricReader is shutdown. %d measurement(s) were dropped', this.#droppedCount)
+      done?.()
       return
     }
-    this.#collectAndExport()
+    let pending = 2
+    const complete = () => {
+      if (--pending === 0) done?.()
+    }
+
+    // Snapshot requests already active before starting this flush's export.
+    try {
+      if (typeof this.exporter.flush === 'function') this.exporter.flush(complete)
+      else complete()
+    } catch (error) {
+      log.error('Error flushing OTLP metrics:', error)
+      complete()
+    }
+    try {
+      this.#collectAndExport(complete)
+    } catch (error) {
+      log.error('Error exporting OTLP metrics:', error)
+      complete()
+    }
   }
 
   /**
@@ -250,7 +269,8 @@ class PeriodicMetricReader {
    *
    * @param {Function} [callback] - Called after export completes
    */
-  #collectAndExport (callback = () => {}) {
+  #collectAndExport (callback) {
+    // Observable instruments must be collected even without synchronous measurements.
     // Atomically drain measurements for export. New measurements can be recorded
     // during export without interfering with this batch.
     const allMeasurements = this.#measurements
@@ -292,7 +312,7 @@ class PeriodicMetricReader {
     }
 
     if (allMeasurements.length === 0) {
-      callback()
+      callback?.()
       return
     }
 

package/packages/dd-trace/src/opentelemetry/otlp/otlp_http_exporter_base.js

@@ -5,6 +5,7 @@ const https = require('node:https')
 const { URL } = require('node:url')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 const telemetryMetrics = require('../../telemetry/metrics')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
@@ -20,6 +21,7 @@ const legacyStorage = storage('legacy')
  */
 class OtlpHttpExporterBase {
   #transport = https
+  #serverlessDeliveryTracker
 
   /**
    * Creates a new OtlpHttpExporterBase instance.
@@ -32,6 +34,7 @@ class OtlpHttpExporterBase {
    * @param {string} signalType - Signal type for error messages (e.g., 'logs', 'metrics')
    */
   constructor (url, headers, timeout, protocol, signalType) {
+    this.#serverlessDeliveryTracker = createServerlessDeliveryTracker()
     this.protocol = protocol
     this.signalType = signalType
 
@@ -80,6 +83,13 @@ class OtlpHttpExporterBase {
    * @protected
    */
   sendPayload (payload, resultCallback) {
+    if (this.#serverlessDeliveryTracker) {
+      return this.#serverlessDeliveryTracker.track(done => this.#sendPayload(payload, resultCallback, done))
+    }
+    this.#sendPayload(payload, resultCallback)
+  }
+
+  #sendPayload (payload, resultCallback, done) {
     const options = {
       ...this.options,
       headers: {
@@ -88,39 +98,65 @@ class OtlpHttpExporterBase {
       },
     }
 
-    legacyStorage.run({ noop: true }, () => {
-      const req = this.#transport.request(options, (res) => {
-        let data = ''
+    let completed = false
+    const complete = result => {
+      if (completed) return
+      completed = true
+      resultCallback(result)
+      done?.()
+    }
 
-        res.on('data', (chunk) => {
-          data += chunk
+    try {
+      legacyStorage.run({ noop: true }, () => {
+        const req = this.#transport.request(options, (res) => {
+          let data = ''
+
+          res.on('data', (chunk) => {
+            data += chunk
+          })
+
+          res.once('error', (error) => {
+            complete({ code: 1, error })
+          })
+
+          res.once('end', () => {
+            // @ts-expect-error - res.statusCode can be undefined
+            if (res.statusCode >= 200 && res.statusCode < 300) {
+              complete({ code: 0 })
+            } else {
+              const error = new Error(`HTTP ${res.statusCode}: ${data}`)
+              complete({ code: 1, error })
+            }
+          })
         })
 
-        res.once('end', () => {
-          // @ts-expect-error - res.statusCode can be undefined
-          if (res.statusCode >= 200 && res.statusCode < 300) {
-            resultCallback({ code: 0 })
-          } else {
-            const error = new Error(`HTTP ${res.statusCode}: ${data}`)
-            resultCallback({ code: 1, error })
-          }
+        req.on('error', (error) => {
+          log.error('Error sending OTLP %s:', this.signalType, error)
+          complete({ code: 1, error })
         })
-      })
 
-      req.on('error', (error) => {
-        log.error('Error sending OTLP %s:', this.signalType, error)
-        resultCallback({ code: 1, error })
-      })
+        req.once('timeout', () => {
+          req.destroy()
+          const error = new Error('Request timeout')
+          complete({ code: 1, error })
+        })
 
-      req.once('timeout', () => {
-        req.destroy()
-        const error = new Error('Request timeout')
-        resultCallback({ code: 1, error })
+        req.write(payload)
+        req.end()
       })
+    } catch (error) {
+      log.error('Error sending OTLP %s:', this.signalType, error)
+      complete({ code: 1, error })
+    }
+  }
 
-      req.write(payload)
-      req.end()
-    })
+  /**
+   * Calls back once Vercel-tracked requests active at the flush boundary complete.
+   * @param {Function} [done]
+   */
+  flush (done) {
+    if (this.#serverlessDeliveryTracker) return this.#serverlessDeliveryTracker.waitForIdle(done)
+    done?.()
   }
 
   /**