dd-trace 6.12.0 → 6.13.0

package/packages/dd-trace/src/proxy.js

@@ -7,12 +7,12 @@ const { getEnvironmentVariable } = require('./config/helper')
 const runtimeMetrics = require('./runtime_metrics')
 const log = require('./log')
 const { setStartupLogPluginManager, startupLog } = require('./startup-log')
-const DynamicInstrumentation = require('./debugger')
 const telemetry = require('./telemetry')
 const nomenclature = require('./service-naming')
 const PluginManager = require('./plugin_manager')
 const NoopDogStatsDClient = require('./noop/dogstatsd')
-const { IS_SERVERLESS } = require('./serverless')
+const { IS_SERVERLESS, initializeServerlessTelemetry, supportsServerlessTelemetryRetention } = require('./serverless')
+const { flushServerlessTelemetry, registerTelemetryFlusher } = require('./flush')
 const processTags = require('./process-tags')
 const { isTrue } = require('./util')
 const {
@@ -41,6 +41,13 @@ const OPENFEATURE_STATE_NOOP = 0
 const OPENFEATURE_STATE_LAZY = 1
 const OPENFEATURE_STATE_ACTIVE = 2
 
+let dynamicInstrumentation
+
+function getDynamicInstrumentation () {
+  dynamicInstrumentation ??= require('./debugger')
+  return dynamicInstrumentation
+}
+
 class LazyModule {
   constructor (provider) {
     this.provider = provider
@@ -100,6 +107,15 @@ class Tracer extends NoopProxy {
     this._pluginManager = new PluginManager(this)
     this.dogstatsd = new NoopDogStatsDClient()
     this._tracingInitialized = false
+    // Logs and metrics can need retention even when tracing is disabled.
+    if (supportsServerlessTelemetryRetention()) {
+      this._serverlessTelemetry = {
+        flushAll: (done, options) => {
+          if (typeof this._tracer?.flushAll === 'function') this._tracer.flushAll(done, options)
+          else flushServerlessTelemetry(done, options)
+        },
+      }
+    }
     this._flare = new LazyModule(() => require('./flare'))
     this.setBaggageItem = setBaggageItem
     this.getBaggageItem = getBaggageItem
@@ -205,7 +221,7 @@ class Tracer extends NoopProxy {
         }
 
         if (config.dynamicInstrumentation.enabled) {
-          DynamicInstrumentation.start(config, rc)
+          getDynamicInstrumentation().start(config, rc)
         }
 
         const openfeatureRemoteConfig = require('./openfeature/remote_config')
@@ -255,13 +271,20 @@ class Tracer extends NoopProxy {
 
       if (config.runtimeMetrics.enabled) {
         runtimeMetrics.start(config)
+        // Agent trace response metrics are recorded asynchronously, so drain
+        // runtime metrics after the trace export has completed.
+        registerTelemetryFlusher(done => runtimeMetrics.flush(done), { afterTrace: true })
       }
 
       this.#updateTracing(config)
 
-      this._modules.rewriter.enable(config)
+      if (config.iast.enabled) {
+        this._modules.rewriter.enable(config)
+      }
 
-      if (config.DD_TRACE_ENABLED && config.testOptimization.DD_CIVISIBILITY_MANUAL_API_ENABLED) {
+      if (config.isCiVisibility &&
+        config.DD_TRACE_ENABLED &&
+        config.testOptimization.DD_CIVISIBILITY_MANUAL_API_ENABLED) {
         const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin')
         this._testApiManualPlugin = new TestApiManualPlugin(this)
         // `shouldGetEnvironmentData` is passed as false so that we only lazily calculate it
@@ -269,7 +292,7 @@ class Tracer extends NoopProxy {
         // are lazily configured when the library is imported.
         this._testApiManualPlugin.configure({ ...config, enabled: true }, false)
       }
-      if (config.DD_AGENTLESS_LOG_SUBMISSION_ENABLED) {
+      if (config.isCiVisibility && config.DD_AGENTLESS_LOG_SUBMISSION_ENABLED) {
         if (config.DD_API_KEY) {
           const LogSubmissionPlugin = require('./ci-visibility/log-submission/log-submission-plugin')
           const automaticLogPlugin = new LogSubmissionPlugin(this)
@@ -282,7 +305,7 @@ class Tracer extends NoopProxy {
         }
       }
 
-      if (config.testOptimization.DD_TEST_FAILED_TEST_REPLAY_ENABLED) {
+      if (config.isCiVisibility && config.testOptimization.DD_TEST_FAILED_TEST_REPLAY_ENABLED) {
         const getDynamicInstrumentationClient = require('./ci-visibility/dynamic-instrumentation')
         // We instantiate the client but do not start the Worker here. The worker is started lazily
         getDynamicInstrumentationClient(config)
@@ -389,10 +412,12 @@ class Tracer extends NoopProxy {
     if (this._tracingInitialized) {
       this._tracer.configure(config)
       this._pluginManager.configure(config)
-      DynamicInstrumentation.configure(config)
+      dynamicInstrumentation?.configure(config)
       setStartupLogPluginManager(this._pluginManager)
       startupLog()
     }
+
+    initializeServerlessTelemetry(this._serverlessTelemetry)
   }
 
   /**
@@ -404,6 +429,9 @@ class Tracer extends NoopProxy {
    */
   #updateDebugger (config, rc) {
     const shouldBeEnabled = config.dynamicInstrumentation.enabled
+    if (!shouldBeEnabled && dynamicInstrumentation === undefined) return
+
+    const DynamicInstrumentation = getDynamicInstrumentation()
     const isCurrentlyStarted = DynamicInstrumentation.isStarted()
 
     if (shouldBeEnabled) {

package/packages/dd-trace/src/runtime_metrics/index.js

@@ -13,6 +13,7 @@ const noop = runtimeMetrics = {
   gauge () {},
   increment () {},
   decrement () {},
+  flush (done) { done?.() },
 }
 
 module.exports = {
@@ -42,6 +43,10 @@ module.exports = {
     runtimeMetrics = noop
     Object.setPrototypeOf(module.exports, noop)
   },
+
+  flush (done) {
+    runtimeMetrics.flush(done)
+  },
 }
 
 Object.setPrototypeOf(module.exports, noop)

package/packages/dd-trace/src/runtime_metrics/otlp_runtime_metrics.js

@@ -245,6 +245,11 @@ module.exports = {
   decrement (name, tag) {
     this.count(name, -1, tag)
   },
+
+  flush (done) {
+    if (client) return client.flush(done)
+    done?.()
+  },
 }
 
 /**

package/packages/dd-trace/src/runtime_metrics/runtime_metrics.js

@@ -27,6 +27,7 @@ let client = null
 let lastTime = 0
 let lastCpuUsage = null
 let eventLoopDelayObserver = null
+let capture = null
 
 // !!!!!!!!!!!
 //  IMPORTANT
@@ -76,11 +77,10 @@ module.exports = {
     lastTime = performance.now()
 
     if (nativeMetrics) {
-      interval = setInterval(() => {
+      capture = () => {
         captureNativeMetrics(trackEventLoop, trackGc)
         captureCommonMetrics(trackEventLoop)
-        client.flush()
-      }, flushIntervalMs)
+      }
     } else {
       lastCpuUsage = process.cpuUsage()
 
@@ -92,17 +92,21 @@ module.exports = {
         eventLoopDelayObserver.enable()
       }
 
-      interval = setInterval(() => {
+      capture = () => {
         captureCpuUsage()
         captureCommonMetrics(trackEventLoop)
         captureHeapSpace()
         if (trackEventLoop) {
           captureEventLoopDelay()
         }
-        client.flush()
-      }, flushIntervalMs)
+      }
     }
 
+    interval = setInterval(() => {
+      capture()
+      client.flush()
+    }, flushIntervalMs)
+
     interval.unref?.()
   },
 
@@ -114,6 +118,7 @@ module.exports = {
     interval = null
 
     client = null
+    capture = null
     lastCpuUsage = null
 
     gcObserver?.disconnect()
@@ -158,6 +163,12 @@ module.exports = {
   decrement (name, tag) {
     this.count(name, -1, tag)
   },
+
+  flush (done) {
+    if (!client) return done?.()
+    capture?.()
+    client.flush(done)
+  },
 }
 
 function captureCpuUsage () {

package/packages/dd-trace/src/serverless/telemetry-delivery-tracker.js

@@ -0,0 +1,55 @@
+'use strict'
+
+/**
+ * Tracks transport deliveries that must outlive a serverless request.
+ *
+ * The tracker is created only for platforms with an invocation-retention
+ * boundary. Exporters keep their normal callback path when it is absent.
+ */
+class TelemetryDeliveryTracker {
+  #deliveries = new Set()
+
+  /**
+   * Tracks one asynchronous transport delivery until its callback runs.
+   * @param {(done: () => void) => void} deliver
+   * @param {(() => void)|undefined} done
+   */
+  track (deliver, done) {
+    const delivery = { callbacks: done ? [done] : [] }
+    this.#deliveries.add(delivery)
+
+    let completed = false
+    const complete = () => {
+      if (completed) return
+      completed = true
+      this.#deliveries.delete(delivery)
+      for (const callback of delivery.callbacks) callback()
+    }
+
+    try {
+      deliver(complete)
+    } catch (error) {
+      complete()
+      throw error
+    }
+  }
+
+  /**
+   * Calls back after every delivery active at this boundary has completed.
+   * @param {(() => void)|undefined} done
+   */
+  waitForIdle (done) {
+    if (!done) return
+
+    const deliveries = [...this.#deliveries]
+    if (deliveries.length === 0) return done()
+
+    let pending = deliveries.length
+    const complete = () => {
+      if (--pending === 0) done()
+    }
+    for (const delivery of deliveries) delivery.callbacks.push(complete)
+  }
+}
+
+module.exports = TelemetryDeliveryTracker

package/packages/dd-trace/src/serverless/vercel.js

@@ -0,0 +1,122 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+
+const { getEnvironmentVariable } = require('../config/helper')
+const log = require('../log')
+
+const httpRequestFinishChannel = channel('apm:http:server:request:finish')
+const http2RequestStartChannel = channel('apm:http2:server:request:start')
+const http2ResponseEmitChannel = channel('apm:http2:server:response:emit')
+const VERCEL_REQUEST_CONTEXT = Symbol.for('@vercel/request-context')
+const VERCEL_FLUSH_TIMEOUT = 2000
+const vercelRetentionHandlers = new WeakMap()
+
+/**
+ * @typedef {{ flushAll?: (done: () => void, options?: { timeout?: number }) => void }} TelemetryFlusher
+ */
+
+/**
+ * @param {TelemetryFlusher} tracer
+ * @param {() => void} done
+ * @returns {void}
+ */
+function flushVercelTelemetry (tracer, done) {
+  setImmediate(() => {
+    try {
+      tracer.flushAll(done, { timeout: VERCEL_FLUSH_TIMEOUT })
+    } catch (error) {
+      log.warn('Unable to flush Vercel telemetry:', error)
+      done()
+    }
+  })
+}
+
+function registerVercelRequestFlush (tracer) {
+  const requestContext = getVercelRequestContext()
+  if (!requestContext) return
+
+  const { waitUntil } = requestContext
+  if (typeof waitUntil !== 'function') return
+
+  // Retain the invocation synchronously, then flush after the response completes.
+  let done
+  const pending = new Promise(resolve => { done = resolve })
+  try {
+    waitUntil(pending)
+    flushVercelTelemetry(tracer, done)
+  } catch (error) {
+    log.warn('Unable to retain Vercel telemetry:', error)
+    done()
+  }
+}
+
+function getVercelRequestContext () {
+  return globalThis[VERCEL_REQUEST_CONTEXT]?.get?.()
+}
+
+// HTTP/2 binds response lifecycle events while handling its request-start channel.
+function activateHttp2Lifecycle () {}
+
+/**
+ * Retains a Vercel Node Function until configured telemetry exporters complete.
+ *
+ * @param {TelemetryFlusher} tracer
+ * @returns {(() => void)|undefined}
+ */
+function registerVercelTelemetryRetention (tracer) {
+  const existing = vercelRetentionHandlers.get(tracer)
+  if (existing) return existing
+
+  if (typeof tracer?.flushAll !== 'function') return
+  // The HTTP finish channel activates its response wrapper directly. HTTP/2 needs
+  // a passive request-start subscriber before it can bind response emit events.
+  const flushRequest = () => registerVercelRequestFlush(tracer)
+  const flushHttp2Response = ({ eventName }) => {
+    if (eventName === 'close') flushRequest()
+  }
+  httpRequestFinishChannel.subscribe(flushRequest)
+  http2RequestStartChannel.subscribe(activateHttp2Lifecycle)
+  http2ResponseEmitChannel.subscribe(flushHttp2Response)
+
+  const unregister = () => {
+    httpRequestFinishChannel.unsubscribe(flushRequest)
+    http2RequestStartChannel.unsubscribe(activateHttp2Lifecycle)
+    http2ResponseEmitChannel.unsubscribe(flushHttp2Response)
+    vercelRetentionHandlers.delete(tracer)
+  }
+  vercelRetentionHandlers.set(tracer, unregister)
+  return unregister
+}
+
+/**
+ * Gets Vercel deployment tags to attach to spans.
+ *
+ * @returns {string[]|undefined}
+ */
+function getVercelPlatformTags () {
+  let tags
+  const projectId = getEnvironmentVariable('VERCEL_PROJECT_ID')
+  if (projectId) {
+    tags = ['vercel.project_id', projectId]
+  }
+
+  const environment = getEnvironmentVariable('VERCEL_ENV')
+  if (environment) {
+    tags ??= []
+    tags.push('vercel.environment', environment)
+  }
+
+  const region = getEnvironmentVariable('VERCEL_REGION')
+  if (region) {
+    tags ??= []
+    tags.push('vercel.region', region)
+  }
+
+  return tags
+}
+
+module.exports = {
+  getVercelPlatformTags,
+  registerVercelTelemetryRetention,
+}

package/packages/dd-trace/src/serverless.js

@@ -45,44 +45,61 @@ function isInServerlessEnvironment () {
 /**
  * Gets tags describing the serverless platform where the tracer is running.
  *
+ * @param {{ isVercel: boolean }} [platform] Detected serverless platform.
  * @returns {string[]|undefined}
  */
-function getServerlessPlatformTags () {
-  if (getEnvironmentVariable('VERCEL') === '1') {
-    return getVercelPlatformTags()
+function getServerlessPlatformTags (platform = getServerlessPlatform()) {
+  if (platform.isVercel) {
+    return require('./serverless/vercel').getVercelPlatformTags()
   }
 }
 
 /**
- * @returns {string[]|undefined}
+ * Detects the serverless platform once while configuration is built.
+ * @returns {{ isVercel: boolean }}
  */
-function getVercelPlatformTags () {
-  let tags
-  const projectId = getEnvironmentVariable('VERCEL_PROJECT_ID')
-  if (projectId) {
-    tags = ['vercel.project_id', projectId]
-  }
+function getServerlessPlatform () {
+  return { isVercel: getEnvironmentVariable('VERCEL') === '1' }
+}
 
-  const environment = getEnvironmentVariable('VERCEL_ENV')
-  if (environment) {
-    tags ??= []
-    tags.push('vercel.environment', environment)
-  }
+/**
+ * Whether the current platform can retain an invocation for telemetry delivery.
+ *
+ * Add future serverless platforms here as they gain an equivalent retention hook.
+ * @returns {boolean}
+ */
+function supportsServerlessTelemetryRetention () {
+  return getServerlessPlatform().isVercel
+}
 
-  const region = getEnvironmentVariable('VERCEL_REGION')
-  if (region) {
-    tags ??= []
-    tags.push('vercel.region', region)
+/**
+ * Creates delivery tracking for platforms with an invocation retention hook.
+ */
+function createServerlessDeliveryTracker () {
+  if (supportsServerlessTelemetryRetention()) {
+    return new (require('./serverless/telemetry-delivery-tracker'))()
   }
+}
 
-  return tags
+/**
+ * Registers the lifecycle adapter selected by the detected serverless platform.
+ * @param {{ flushAll?: (done: () => void) => void }} tracer
+ */
+function initializeServerlessTelemetry (tracer) {
+  if (supportsServerlessTelemetryRetention()) {
+    return require('./serverless/vercel').registerVercelTelemetryRetention(tracer)
+  }
 }
 
 module.exports = {
   getServerlessPlatformTags,
+  getServerlessPlatform,
+  supportsServerlessTelemetryRetention,
+  createServerlessDeliveryTracker,
   getIsGCPFunction,
   getIsAzureFunction,
   enableGCPPubSubPushSubscription,
   getIsFlexConsumptionAzureFunction,
+  initializeServerlessTelemetry,
   IS_SERVERLESS: isInServerlessEnvironment(),
 }

package/packages/dd-trace/src/span_stats.js

@@ -15,6 +15,7 @@ const {
 } = require('../../../ext/tags')
 const { ORIGIN_KEY, TOP_LEVEL_KEY, SVC_SRC_KEY, GRPC_STATUS_NAMES } = require('./constants')
 const id = require('./id')
+const log = require('./log')
 
 const GRPC_STATUS_CODE_MAP = Object.fromEntries(GRPC_STATUS_NAMES.map((name, i) => [name, String(i)]))
 const ZERO_ID = id('0')
@@ -235,6 +236,18 @@ class SpanStatsProcessor {
   }
 
   onInterval () {
+    this.#flush()
+  }
+
+  /**
+   * Drains pending span statistics and waits for their export.
+   * @param {Function} [done]
+   */
+  forceFlush (done) {
+    this.#flush(done)
+  }
+
+  #flush (done) {
     const drained = this.#drainBuckets()
 
     if (this.enabled && !this.otlpExporter) {
@@ -248,10 +261,34 @@ class SpanStatsProcessor {
         RuntimeID: this.tags['runtime-id'],
         Sequence: ++this.sequence,
         ProcessTags: processTags.serialized,
-      })
+      }, done)
     } else if (this.otlpExporter && drained.length > 0) {
-      this.otlpExporter.export(drained, this.bucketSizeNs)
-    }
+      if (typeof this.otlpExporter.flush === 'function' && done) {
+        // Snapshot requests already in flight before starting this boundary
+        // export, so a later invocation cannot extend this lifecycle barrier.
+        let pending = 2
+        const complete = () => {
+          if (--pending === 0) done()
+        }
+        try {
+          this.otlpExporter.flush(complete)
+        } catch (error) {
+          log.error('Failed to flush OTLP span stats:', error)
+          complete()
+        }
+        try {
+          this.otlpExporter.export(drained, this.bucketSizeNs, complete)
+        } catch (error) {
+          log.error('Failed to export OTLP span stats:', error)
+          complete()
+        }
+      } else {
+        this.otlpExporter.export(drained, this.bucketSizeNs, done)
+      }
+    } else if (this.otlpExporter) {
+      if (typeof this.otlpExporter.flush === 'function') this.otlpExporter.flush(done)
+      else done?.()
+    } else done?.()
   }
 
   onSpanFinished (span) {

package/packages/dd-trace/src/tracer.js

@@ -13,6 +13,7 @@ const { isError } = require('./util')
 const { setStartupLogConfig } = require('./startup-log')
 const { DataStreamsCheckpointer, DataStreamsManager, DataStreamsProcessor } = require('./datastreams')
 const { IS_SERVERLESS } = require('./serverless')
+const { flushServerlessTelemetry } = require('./flush')
 const log = require('./log')
 // Always-on writer (console.warn), not the channel-gated `log`: these surface regardless of
 // DD_TRACE_DEBUG.
@@ -150,6 +151,27 @@ class DatadogTracer extends Tracer {
     this._dataStreamsProcessor.setUrl(url)
   }
 
+  /**
+   * Flushes every configured telemetry pipeline for a serverless lifecycle.
+   * @param {Function} [done] Called after every configured export completes
+   * @param {{ timeout?: number }} [options] Bounds this flush operation.
+   */
+  flushAll (done, options) {
+    const traceExporter = this._exporter
+    const spanStats = this._processor?._stats
+    const traceFlusher = typeof traceExporter?.flush === 'function'
+      ? callback => traceExporter.flush(callback)
+      : undefined
+    const spanStatsFlusher = typeof spanStats?.forceFlush === 'function'
+      ? callback => spanStats.forceFlush(callback)
+      : undefined
+
+    flushServerlessTelemetry(done, options, {
+      trace: traceFlusher,
+      spanStats: spanStatsFlusher,
+    })
+  }
+
   scope () {
     return this._scope
   }