dd-trace 5.123.0 → 5.124.0

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -4,6 +4,7 @@ const { URL, format } = require('node:url')
 const path = require('node:path')
 const request = require('../../exporters/common/request')
 const { getEnvironmentVariable } = require('../../config/helper')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 
 const logger = require('../../log')
 
@@ -33,6 +34,9 @@ class LLMObsBuffer {
 
 class BaseLLMObsWriter {
   #destroyer
+  /** @type {Function[]} */
+  #pendingFlushes = []
+  #serverlessDeliveryTracker = createServerlessDeliveryTracker()
   /** @type {Map<string, LLMObsBuffer>} */
   #multiTenantBuffers = new Map()
 
@@ -111,35 +115,52 @@ class BaseLLMObsWriter {
     return true
   }
 
-  flush () {
+  /**
+   * Drains buffered events and joins requests active at this flush boundary.
+   * @param {Function} [done]
+   */
+  flush (done) {
     if (this._agentless == null) {
+      if (done) this.#pendingFlushes.push(done)
       return
     }
 
-    // Flush default buffer
-    if (this._buffer.events.length > 0) {
-      const events = this._buffer.events
-      this._buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
+    const requests = this.#drainBuffers()
+    if (!this.#serverlessDeliveryTracker) {
+      // Only invocation-retaining platforms need completion-aware delivery.
+      for (const request of requests) this.#sendSafely(request)
+      done?.()
+      return
+    }
 
-      log.debug('Encoded LLMObs payload: %s', payload)
+    for (const request of requests) this.#sendSafely(request)
+    this.#serverlessDeliveryTracker.waitForIdle(done)
+  }
 
-      const options = this._getOptions()
+  #sendSafely (requestToSend) {
+    try {
+      if (this.#serverlessDeliveryTracker) {
+        this.#serverlessDeliveryTracker.track(done => this.#send(requestToSend, done))
+      } else {
+        this.#send(requestToSend)
+      }
+    } catch (error) {
+      logger.error('Failed to send LLMObs %s events: %s', this._eventType, error.message)
+    }
+  }
 
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, this.url, this._eventType)
-      })
+  #drainBuffers () {
+    const requests = []
+    if (this._buffer.events.length > 0) {
+      const events = this._buffer.events
+      this._buffer.clear()
+      requests.push({ events, options: this._getOptions(), url: this.url })
     }
 
-    // Flush multi-tenant buffers
     for (const [apiKey, buffer] of this.#multiTenantBuffers) {
       if (buffer.events.length === 0) continue
-
       const events = buffer.events
       buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
       const site = buffer.routing.site || this._config.site
       const options = {
         headers: {
@@ -156,16 +177,21 @@ class BaseLLMObsWriter {
       }
       const url = this.#buildUrl(options.url.href, options.path)
       const maskedApiKey = apiKey ? `****${apiKey.slice(-4)}` : ''
-
       log.debug('Encoding and flushing multi-tenant buffer for %s', maskedApiKey)
-      log.debug('Encoded LLMObs payload: %s', payload)
-
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, url, this._eventType)
-      })
+      requests.push({ events, options, url })
     }
 
     this.#cleanupEmptyBuffers()
+    return requests
+  }
+
+  #send ({ events, options, url }, done) {
+    const payload = this._encode(this.makePayload(events))
+    log.debug('Encoded LLMObs payload: %s', payload)
+    request(payload, options, (err, resp, code) => {
+      parseResponseAndLog(err, code, events.length, url, this._eventType)
+      done?.()
+    })
   }
 
   #cleanupEmptyBuffers () {
@@ -178,13 +204,19 @@ class BaseLLMObsWriter {
 
   makePayload (events) {}
 
-  destroy () {
+  /**
+   * Stops periodic flushing and drains buffered events.
+   * @param {Function} [done] Called after queued and active deliveries complete.
+   */
+  destroy (done) {
     if (this.#destroyer) {
       logger.debug(`Stopping ${this.constructor.name}`)
       clearInterval(this._periodic)
       globalThis[Symbol.for('dd-trace')].beforeExitHandlers.delete(this.#destroyer)
-      this.flush()
+      this.flush(done)
       this.#destroyer = undefined
+    } else {
+      done?.()
     }
   }
 
@@ -196,6 +228,10 @@ class BaseLLMObsWriter {
     this._endpoint = endpoint
 
     logger.debug(`Configuring ${this.constructor.name} to ${this.url}`)
+
+    const pendingFlushes = this.#pendingFlushes
+    this.#pendingFlushes = []
+    for (const done of pendingFlushes) this.flush(done)
   }
 
   _getUrlAndPath () {

package/packages/dd-trace/src/noop/span.js

@@ -26,6 +26,11 @@ class NoopSpan {
   addLink (link) { return this }
   addLinks (links) { return this }
   addSpanPointer (ptrKind, ptrDir, ptrHash) { return this }
+  /**
+   * @param {import('../../../../index').Exception} exception
+   * @param {import('../../../../index').SpanEventAttributes} [attributes]
+   */
+  recordException (exception, attributes) {}
   log () { return this }
   logEvent () {}
   finish (finishTime) {}

package/packages/dd-trace/src/openfeature/writers/exposures.js

@@ -37,6 +37,7 @@ const PENDING_MAX_EVENTS = 1000
  * @property {string} flag.key - Flag key
  * @property {object} variant - Variant information
  * @property {string} variant.key - Variant key
+ * @property {number} [serial_id] - Serial id of the split the subject landed in
  * @property {object} subject - Subject (user/entity) information
  * @property {string} subject.id - Subject identifier
  * @property {string} [subject.type] - Subject type
@@ -204,7 +205,7 @@ class ExposuresWriter extends BaseFFEWriter {
   makePayload (events) {
     const formattedEvents = events.map(event => {
       /** @type {ExposureEvent} */
-      return {
+      const formatted = {
         timestamp: event.timestamp || Date.now(),
         allocation: {
           key: event.allocation?.key || event['allocation.key'],
@@ -221,6 +222,12 @@ class ExposuresWriter extends BaseFFEWriter {
           attributes: event.subject?.attributes,
         },
       }
+
+      if (typeof event.serial_id === 'number') {
+        formatted.serial_id = event.serial_id
+      }
+
+      return formatted
     })
 
     return {

package/packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js

@@ -1,5 +1,8 @@
 'use strict'
 
+const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
+
 /**
  * @typedef {import('@opentelemetry/api-logs').LogRecord} LogRecord
  * @typedef {import('@opentelemetry/core').InstrumentationScope} InstrumentationScope
@@ -54,10 +57,48 @@ class BatchLogRecordProcessor {
 
   /**
    * Forces an immediate flush of all pending log records.
-   * @returns {undefined} Promise that resolves when flush is complete
+   * @param {Function} [done] Called after all pending log exports complete
    */
-  forceFlush () {
-    this.#export()
+  forceFlush (done) {
+    this.#clearTimer()
+
+    const deliveryTracker = createServerlessDeliveryTracker()
+    if (!deliveryTracker) {
+      // Normal processes preserve the existing fire-and-forget batch flush.
+      this.#export()
+      done?.()
+      return
+    }
+
+    // Flush only records present at this boundary. New records belong to the
+    // later request that produced them and must not extend this lifecycle flush.
+    const logRecords = this.#logRecords
+    this.#logRecords = []
+
+    // Join exports already active at this boundary before draining this snapshot.
+    if (typeof this.exporter.flush === 'function') {
+      deliveryTracker.track(complete => this.exporter.flush(complete))
+    }
+
+    deliveryTracker.track(complete => {
+      const flushNext = () => {
+        if (logRecords.length === 0) {
+          complete()
+          return
+        }
+
+        // Drain the boundary snapshot one batch at a time.
+        const batch = logRecords.splice(0, this.#maxExportBatchSize)
+        try {
+          this.exporter.export(batch, flushNext)
+        } catch (error) {
+          log.error('Error exporting OTLP logs:', error)
+          complete()
+        }
+      }
+      flushNext()
+    })
+    deliveryTracker.waitForIdle(done)
   }
 
   /**
@@ -79,6 +120,7 @@ class BatchLogRecordProcessor {
    * @private
    */
   #export () {
+    if (this.#logRecords.length === 0) return
     const logRecords = this.#logRecords.slice(0, this.#maxExportBatchSize)
     this.#logRecords = this.#logRecords.slice(this.#maxExportBatchSize)
 

package/packages/dd-trace/src/opentelemetry/logs/index.js

@@ -27,6 +27,7 @@ const os = require('os')
  * @package
  */
 
+const { registerTelemetryFlusher } = require('../../flush')
 const LoggerProvider = require('./logger_provider')
 const BatchLogRecordProcessor = require('./batch_log_processor')
 const OtlpHttpLogExporter = require('./otlp_http_log_exporter')
@@ -77,8 +78,10 @@ function initializeOpenTelemetryLogs (config) {
   // Create logger provider with processor for Datadog Agent export
   const loggerProvider = new LoggerProvider({ processor })
 
-  // Register the logger provider globally with OpenTelemetry API
+  // Expose this provider to application calls through the OpenTelemetry Logs API.
   loggerProvider.register()
+  // Include final log batches in lifecycle retention with trace delivery.
+  registerTelemetryFlusher(done => loggerProvider.forceFlush(done))
 }
 
 module.exports = {

package/packages/dd-trace/src/opentelemetry/logs/logger_provider.js

@@ -84,12 +84,15 @@ class LoggerProvider {
 
   /**
    * Forces a flush of all pending log records.
-   * @returns {undefined} Promise that resolves when flush is n ssue cncomplete
+   * @param {Function} [done] Called after all pending log exports complete
    */
-  forceFlush () {
-    if (!this.isShutdown) {
-      return this.processor?.forceFlush()
+  forceFlush (done) {
+    if (this.isShutdown || !this.processor) {
+      done?.()
+      return
     }
+
+    this.processor.forceFlush(done)
   }
 
   /**

package/packages/dd-trace/src/opentelemetry/metrics/index.js

@@ -6,12 +6,12 @@ const { metrics } = require('@opentelemetry/api')
 
 const { VERSION } = require('../../../../../version')
 const processTags = require('../../process-tags')
+const { registerTelemetryFlusher } = require('../../flush')
 const MeterProvider = require('./meter_provider')
 const PeriodicMetricReader = require('./periodic_metric_reader')
 const OtlpHttpMetricExporter = require('./otlp_http_metric_exporter')
 
 const RESERVED_TRACER_TAGS = new Set(['service', 'env', 'version', 'runtime_id', 'runtime-id'])
-
 /**
  * @typedef {import('../../config')} Config
  */
@@ -78,6 +78,8 @@ function initializeOpenTelemetryMetrics (config) {
 
   const meterProvider = new MeterProvider({ reader })
   metrics.setGlobalMeterProvider(meterProvider)
+  // Include the final metric collection and export in lifecycle retention.
+  registerTelemetryFlusher(done => meterProvider.forceFlush(done))
 }
 
 /**

package/packages/dd-trace/src/opentelemetry/metrics/meter_provider.js

@@ -49,6 +49,14 @@ class MeterProvider {
     }
     return meter
   }
+
+  /**
+   * @param {Function} [done] Called after the metric export completes
+   */
+  forceFlush (done) {
+    if (this.reader) this.reader.forceFlush(done)
+    else done?.()
+  }
 }
 
 module.exports = MeterProvider

package/packages/dd-trace/src/opentelemetry/metrics/otlp_http_metric_exporter.js

@@ -34,10 +34,11 @@ class OtlpHttpMetricExporter extends OtlpHttpExporterBase {
    *
    * @param {Map<string, AggregatedMetric>} metrics - Map of metric data to export
    *
-   * @returns {void}
+   * @param {Function} [done] Called after the HTTP export completes
    */
-  export (metrics) {
+  export (metrics, done) {
     if (metrics.size === 0) {
+      done?.({ code: 0 })
       return
     }
 
@@ -56,6 +57,7 @@ class OtlpHttpMetricExporter extends OtlpHttpExporterBase {
       if (result.code === 0) {
         this.recordTelemetry('otel.metrics_export_successes', 1, additionalTags)
       }
+      done?.(result)
     })
   }
 }

package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_exporter.js

@@ -22,14 +22,16 @@ class OtlpStatsExporter extends OtlpHttpExporterBase {
   /**
    * @param {Array<{timeNs: number, bucket: import('../../span_stats').SpanBuckets}>} drained
    * @param {number} bucketSizeNs
+   * @param {Function} [done] Called after the HTTP export completes
    */
-  export (drained, bucketSizeNs) {
-    if (drained.length === 0) return
+  export (drained, bucketSizeNs, done) {
+    if (drained.length === 0) return done?.()
     const payload = this.#transformer.transform(drained, bucketSizeNs)
     this.sendPayload(payload, (result) => {
       if (result.code !== 0) {
         log.error('Failed to export span stats: %s', result.error?.message)
       }
+      done?.()
     })
   }
 }

package/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js

@@ -197,14 +197,33 @@ class PeriodicMetricReader {
 
   /**
    * Forces an immediate collection and export of all metrics.
-   * @returns {void}
+   * @param {Function} [done] Called after the metric export completes
    */
-  forceFlush () {
+  forceFlush (done) {
     if (this.#isShutdown) {
       log.warn('PeriodicMetricReader is shutdown. %d measurement(s) were dropped', this.#droppedCount)
+      done?.()
       return
     }
-    this.#collectAndExport()
+    let pending = 2
+    const complete = () => {
+      if (--pending === 0) done?.()
+    }
+
+    // Snapshot requests already active before starting this flush's export.
+    try {
+      if (typeof this.exporter.flush === 'function') this.exporter.flush(complete)
+      else complete()
+    } catch (error) {
+      log.error('Error flushing OTLP metrics:', error)
+      complete()
+    }
+    try {
+      this.#collectAndExport(complete)
+    } catch (error) {
+      log.error('Error exporting OTLP metrics:', error)
+      complete()
+    }
   }
 
   /**
@@ -250,7 +269,8 @@ class PeriodicMetricReader {
    *
    * @param {Function} [callback] - Called after export completes
    */
-  #collectAndExport (callback = () => {}) {
+  #collectAndExport (callback) {
+    // Observable instruments must be collected even without synchronous measurements.
     // Atomically drain measurements for export. New measurements can be recorded
     // during export without interfering with this batch.
     const allMeasurements = this.#measurements
@@ -292,7 +312,7 @@ class PeriodicMetricReader {
     }
 
     if (allMeasurements.length === 0) {
-      callback()
+      callback?.()
       return
     }
 

package/packages/dd-trace/src/opentelemetry/otlp/otlp_http_exporter_base.js

@@ -5,6 +5,7 @@ const https = require('node:https')
 const { URL } = require('node:url')
 const { storage } = require('../../../../datadog-core')
 const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 const telemetryMetrics = require('../../telemetry/metrics')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
@@ -20,6 +21,7 @@ const legacyStorage = storage('legacy')
  */
 class OtlpHttpExporterBase {
   #transport = https
+  #serverlessDeliveryTracker
 
   /**
    * Creates a new OtlpHttpExporterBase instance.
@@ -32,6 +34,7 @@ class OtlpHttpExporterBase {
    * @param {string} signalType - Signal type for error messages (e.g., 'logs', 'metrics')
    */
   constructor (url, headers, timeout, protocol, signalType) {
+    this.#serverlessDeliveryTracker = createServerlessDeliveryTracker()
     this.protocol = protocol
     this.signalType = signalType
 
@@ -80,6 +83,13 @@ class OtlpHttpExporterBase {
    * @protected
    */
   sendPayload (payload, resultCallback) {
+    if (this.#serverlessDeliveryTracker) {
+      return this.#serverlessDeliveryTracker.track(done => this.#sendPayload(payload, resultCallback, done))
+    }
+    this.#sendPayload(payload, resultCallback)
+  }
+
+  #sendPayload (payload, resultCallback, done) {
     const options = {
       ...this.options,
       headers: {
@@ -88,39 +98,65 @@ class OtlpHttpExporterBase {
       },
     }
 
-    legacyStorage.run({ noop: true }, () => {
-      const req = this.#transport.request(options, (res) => {
-        let data = ''
+    let completed = false
+    const complete = result => {
+      if (completed) return
+      completed = true
+      resultCallback(result)
+      done?.()
+    }
 
-        res.on('data', (chunk) => {
-          data += chunk
+    try {
+      legacyStorage.run({ noop: true }, () => {
+        const req = this.#transport.request(options, (res) => {
+          let data = ''
+
+          res.on('data', (chunk) => {
+            data += chunk
+          })
+
+          res.once('error', (error) => {
+            complete({ code: 1, error })
+          })
+
+          res.once('end', () => {
+            // @ts-expect-error - res.statusCode can be undefined
+            if (res.statusCode >= 200 && res.statusCode < 300) {
+              complete({ code: 0 })
+            } else {
+              const error = new Error(`HTTP ${res.statusCode}: ${data}`)
+              complete({ code: 1, error })
+            }
+          })
         })
 
-        res.once('end', () => {
-          // @ts-expect-error - res.statusCode can be undefined
-          if (res.statusCode >= 200 && res.statusCode < 300) {
-            resultCallback({ code: 0 })
-          } else {
-            const error = new Error(`HTTP ${res.statusCode}: ${data}`)
-            resultCallback({ code: 1, error })
-          }
+        req.on('error', (error) => {
+          log.error('Error sending OTLP %s:', this.signalType, error)
+          complete({ code: 1, error })
         })
-      })
 
-      req.on('error', (error) => {
-        log.error('Error sending OTLP %s:', this.signalType, error)
-        resultCallback({ code: 1, error })
-      })
+        req.once('timeout', () => {
+          req.destroy()
+          const error = new Error('Request timeout')
+          complete({ code: 1, error })
+        })
 
-      req.once('timeout', () => {
-        req.destroy()
-        const error = new Error('Request timeout')
-        resultCallback({ code: 1, error })
+        req.write(payload)
+        req.end()
       })
+    } catch (error) {
+      log.error('Error sending OTLP %s:', this.signalType, error)
+      complete({ code: 1, error })
+    }
+  }
 
-      req.write(payload)
-      req.end()
-    })
+  /**
+   * Calls back once Vercel-tracked requests active at the flush boundary complete.
+   * @param {Function} [done]
+   */
+  flush (done) {
+    if (this.#serverlessDeliveryTracker) return this.#serverlessDeliveryTracker.waitForIdle(done)
+    done?.()
   }
 
   /**

package/packages/dd-trace/src/opentelemetry/tracer.js

@@ -3,17 +3,41 @@
 const api = require('@opentelemetry/api')
 const { sanitizeAttributes } = require('../../../../vendor/dist/@opentelemetry/core')
 
+const { AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority')
 const tracer = require('../../')
 
 const id = require('../id')
 const log = require('../log')
-const TextMapPropagator = require('../opentracing/propagation/text_map')
 const TraceState = require('../opentracing/propagation/tracestate')
 const SpanContext = require('./span_context')
 const Span = require('./span')
 const Sampler = require('./sampler')
 const { normalizeLinkContext } = require('./span-helpers')
 
+/**
+ * @param {number} traceparentSampled
+ * @param {number | undefined} tracestateSamplingPriority
+ * @param {string | null} origin
+ * @returns {import('../priority_sampler').SamplingPriority}
+ */
+function getSamplingPriority (traceparentSampled, tracestateSamplingPriority, origin) {
+  const fromRumWithoutPriority = tracestateSamplingPriority === undefined && origin === 'rum'
+
+  let samplingPriority =
+    /** @type {import('../priority_sampler').SamplingPriority} */ (tracestateSamplingPriority ?? AUTO_KEEP)
+  if (!fromRumWithoutPriority) {
+    if (traceparentSampled === 0 &&
+        (!tracestateSamplingPriority || tracestateSamplingPriority >= 0)) {
+      samplingPriority = AUTO_REJECT
+    } else if (traceparentSampled === 1 &&
+              (!tracestateSamplingPriority || tracestateSamplingPriority < 0)) {
+      samplingPriority = AUTO_KEEP
+    }
+  }
+
+  return samplingPriority
+}
+
 class Tracer {
   constructor (library, config, tracerProvider) {
     this._sampler = new Sampler()
@@ -76,7 +100,7 @@ class Tracer {
         Object.assign(meta, otherPropagatedTags)
         // Guard against an undefined/empty `s:` field that would result in NaN.
         const tracestateSamplingPriority = samplingPriorityTs ? Math.trunc(samplingPriorityTs) : undefined
-        samplingPriority = TextMapPropagator._getSamplingPriority(traceFlag, tracestateSamplingPriority, origin)
+        samplingPriority = getSamplingPriority(traceFlag, tracestateSamplingPriority, origin)
       } else {
         log.debug('No dd list member in tracestate from incoming request:', ts)
       }