dd-trace 6.11.0 → 6.13.0

package/packages/dd-trace/src/evp_proxy/discovery.js

@@ -0,0 +1,133 @@
+'use strict'
+
+const { fetchAgentInfo } = require('../agent/info')
+const log = require('../log')
+
+const TRAILING_SLASHES = /\/+$/
+
+/**
+ * Receiver discovery contract
+ *
+ * The tracer sends `GET /info` to its configured local Agent URL. An
+ * Agent-compatible trace receiver produces the response. The tracer does not.
+ *
+ * The full Agent and serverless-init embed the same trace receiver. A future
+ * in-process receiver can expose the same contract. Callers can therefore
+ * select capabilities without detecting the receiver implementation.
+ *
+ * For EVP discovery, `endpoints` advertises registered proxy paths.
+ * `evp_proxy_allowed_headers` advertises headers that the proxy forwards to
+ * intake. It does not include routing headers that the proxy consumes, such as
+ * `X-Datadog-EVP-Subdomain`.
+ *
+ * An advertised route is not a health check. The receiver can register an EVP
+ * route while configuration disables its handler. The request then returns
+ * `405`. The `/info` version also identifies the embedded Agent code, not a
+ * serverless-init image or deployment type.
+ *
+ * This module only discovers a candidate route. A missing or unresponsive
+ * `/info` endpoint returns an error through the shared request timeout and
+ * retry policy. A valid response without a compatible path returns no route.
+ * Discovery sends no events, so the caller can safely select direct intake
+ * after either result. The caller also owns later delivery failures. Exposure
+ * delivery uses retries and can therefore produce duplicates. After local
+ * retries fail, the caller can retry through direct intake and use that route
+ * for future batches.
+ *
+ * Reference implementations:
+ *
+ * Agent `/info` and EVP proxy:
+ * https://github.com/DataDog/datadog-agent/tree/main/pkg/trace/api
+ *
+ * serverless-init entry point and embedded trace receiver:
+ * https://github.com/DataDog/datadog-agent/blob/main/cmd/serverless-init/main.go
+ * https://github.com/DataDog/datadog-agent/blob/main/pkg/serverless/trace/trace.go
+ */
+
+/**
+ * Selects the first advertised EVP proxy path that the caller supports.
+ *
+ * @param {object} agentInfo - Agent `/info` response
+ * @param {object} options - Selection options
+ * @param {string[]} options.supportedPaths - Supported paths in preference order
+ * @param {string[]} [options.requiredHeaders] - Headers that the proxy must forward unchanged to intake. Each
+ * header must appear in `evp_proxy_allowed_headers`. Do not include routing headers that the Agent consumes.
+ * @returns {string|undefined} Selected normalized path
+ */
+function selectEVPProxyPath (agentInfo, { supportedPaths, requiredHeaders = [] } = {}) {
+  if (!Array.isArray(agentInfo?.endpoints) ||
+      !Array.isArray(supportedPaths) ||
+      !Array.isArray(requiredHeaders) ||
+      requiredHeaders.some(header => typeof header !== 'string')) {
+    return
+  }
+
+  const allowedHeaders = agentInfo.evp_proxy_allowed_headers
+  if (allowedHeaders !== undefined) {
+    if (!Array.isArray(allowedHeaders)) return
+
+    const normalizedHeaders = new Set()
+    for (const header of allowedHeaders) {
+      if (typeof header === 'string') {
+        normalizedHeaders.add(header.toLowerCase())
+      }
+    }
+
+    if (requiredHeaders.some(header => !normalizedHeaders.has(header.toLowerCase()))) {
+      return
+    }
+  }
+
+  const advertisedPaths = new Set()
+  for (const endpoint of agentInfo.endpoints) {
+    if (typeof endpoint === 'string') {
+      advertisedPaths.add(endpoint.replace(TRAILING_SLASHES, ''))
+    }
+  }
+
+  for (const supportedPath of supportedPaths) {
+    if (typeof supportedPath !== 'string') continue
+
+    const normalizedPath = supportedPath.replace(TRAILING_SLASHES, '')
+    if (advertisedPaths.has(normalizedPath)) {
+      return normalizedPath
+    }
+  }
+}
+
+/**
+ * Discovers an EVP proxy route through the configured Agent URL.
+ *
+ * This function performs discovery only when the caller invokes it. It stores
+ * no state. The Agent information client owns its existing response cache.
+ *
+ * @param {URL} url - Configured Agent URL
+ * @param {object} options - Selection options
+ * @param {string[]} options.supportedPaths - Supported paths in preference order
+ * @param {string[]} [options.requiredHeaders] - Headers that the proxy must forward unchanged to intake. Each
+ * header must appear in `evp_proxy_allowed_headers`. Do not include routing headers that the Agent consumes.
+ * @param {(error: Error|null, route?: {url: URL, basePath: string}) => void} callback - Result callback
+ * @returns {void}
+ */
+function discoverEVPProxy (url, options, callback) {
+  fetchAgentInfo(url, (error, agentInfo) => {
+    if (error) {
+      callback(error)
+      return
+    }
+
+    const basePath = selectEVPProxyPath(agentInfo, options)
+    if (basePath === undefined) {
+      callback(null)
+      return
+    }
+
+    log.debug('EVP proxy route %s discovered through the configured local receiver', basePath)
+    callback(null, { url, basePath })
+  })
+}
+
+module.exports = {
+  discoverEVPProxy,
+  selectEVPProxyPath,
+}

package/packages/dd-trace/src/evp_proxy/path.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const LEADING_SLASHES = /^\/+/
+const TRAILING_SLASHES = /\/+$/
+
+/**
+ * Joins a caller-supplied EVP proxy path and product endpoint.
+ *
+ * This utility does not perform EVP proxy discovery.
+ *
+ * @param {string} basePath - EVP proxy base path
+ * @param {string} endpoint - Product intake endpoint
+ * @returns {string} Joined request path
+ */
+function joinEVPProxyPath (basePath, endpoint) {
+  const normalizedBasePath = basePath.replace(TRAILING_SLASHES, '')
+  const normalizedEndpoint = endpoint.replace(LEADING_SLASHES, '')
+
+  return `${normalizedBasePath}/${normalizedEndpoint}`
+}
+
+module.exports = { joinEVPProxyPath }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -2,12 +2,15 @@
 
 const { URL } = require('url')
 const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 const Writer = require('./writer')
 
 class AgentExporter {
   #timer
+  #serverlessDeliveryTracker
 
   constructor (config, prioritySampler) {
+    this.#serverlessDeliveryTracker = createServerlessDeliveryTracker()
     this._config = config
     const { lookup, protocolVersion, stats = {}, apmTracingEnabled } = config
     this._url = config.url
@@ -23,6 +26,7 @@ class AgentExporter {
       lookup,
       protocolVersion,
       headers,
+      deliveryTracker: this.#serverlessDeliveryTracker,
     })
 
     globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(this.flush.bind(this))
@@ -54,10 +58,26 @@ class AgentExporter {
     }
   }
 
-  flush (done = () => {}) {
+  flush (done) {
     clearTimeout(this.#timer)
     this.#timer = undefined
-    this._writer.flush(done)
+
+    if (!this.#serverlessDeliveryTracker) {
+      try {
+        return this._writer.flush(done)
+      } catch (error) {
+        log.error('Failed to flush traces: %s', error.message)
+        done?.()
+        return
+      }
+    }
+
+    try {
+      this._writer.flush()
+    } catch (error) {
+      log.error('Failed to flush traces: %s', error.message)
+    }
+    this.#serverlessDeliveryTracker.waitForIdle(done)
   }
 }
 

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -39,18 +39,18 @@ class AgentWriter extends BaseWriter {
   }
 
   /**
-   * Flushes payloads, including requests already in flight during Test Optimization finalization.
-   *
+   * Performs the writer flush without registering a serverless delivery.
+   * Test Optimization owns its own request lifecycle tracking.
    * @param {(error?: Error) => void} [done]
    * @param {{ deadline?: number }} [options]
    * @returns {void}
    */
-  flush (done, options) {
+  flushDirect (done, options) {
     if (this.#requestTracker) {
       this.#requestTracker.flush(done, options)
       return
     }
-    super.flush(done, options)
+    super.flushDirect(done, options)
   }
 
   _sendPayload (data, count, done, flushOptions) {

package/packages/dd-trace/src/exporters/common/request.js

@@ -93,7 +93,10 @@ function request (data, options, callback) {
 
   docker.inject(options.headers)
 
-  options.agent = isSecure ? httpsAgent : httpAgent
+  const connectionOptions = {
+    ...options,
+    agent: options.agent ?? (isSecure ? httpsAgent : httpAgent),
+  }
 
   /**
    * @param {import('node:http').IncomingMessage} res
@@ -218,7 +221,7 @@ function request (data, options, callback) {
         }
       }
 
-      const req = client.request(options, (res) => onResponse(res, complete, handleError))
+      const req = client.request(connectionOptions, (res) => onResponse(res, complete, handleError))
 
       req.once('close', finalize)
       req.once('timeout', finalize)

package/packages/dd-trace/src/exporters/common/writer.js

@@ -10,14 +10,36 @@ const { safeJSONStringify } = require('./util')
 const firstFlushChannel = channel('dd-trace:exporter:first-flush')
 
 class Writer {
-  constructor ({ url, beforeFirstFlush }) {
+  #deliveryTracker
+
+  constructor ({ url, beforeFirstFlush, deliveryTracker }) {
     this._url = url
     this._beforeFirstFlush = beforeFirstFlush
+    this.#deliveryTracker = deliveryTracker
   }
 
   #isFirstFlush = true
 
-  flush (done = () => {}, options) {
+  /**
+   * Flushes queued telemetry, retaining delivery on supported serverless platforms.
+   * @param {(error?: Error) => void} [done]
+   * @param {{ deadline?: number }} [options]
+   * @returns {void}
+   */
+  flush (done, options) {
+    if (this.#deliveryTracker) {
+      return this.#deliveryTracker.track(callback => this.flushDirect(callback, options), done)
+    }
+    this.flushDirect(done, options)
+  }
+
+  /**
+   * Flushes queued telemetry without registering serverless delivery retention.
+   * @param {(error?: Error) => void} [done]
+   * @param {{ deadline?: number }} [options]
+   * @returns {void}
+   */
+  flushDirect (done = () => {}, options) {
     const count = this._encoder.count()
 
     if (!request.writable && options?.deadline === undefined) {

package/packages/dd-trace/src/exporters/span-stats/index.js

@@ -1,16 +1,35 @@
 'use strict'
 
+const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 const { Writer } = require('./writer')
 
 class SpanStatsExporter {
+  #serverlessDeliveryTracker
+
   constructor (config) {
+    this.#serverlessDeliveryTracker = createServerlessDeliveryTracker()
     this._url = config.url
-    this._writer = new Writer({ url: this._url })
+    this._writer = new Writer({
+      url: this._url,
+      deliveryTracker: this.#serverlessDeliveryTracker,
+    })
+  }
+
+  export (payload, done) {
+    try {
+      this._writer.append(payload)
+      this._writer.flush(this.#serverlessDeliveryTracker ? undefined : done)
+    } catch (error) {
+      if (!done) throw error
+      log.error('Failed to flush span stats: %s', error.message)
+    }
+    this.#serverlessDeliveryTracker?.waitForIdle(done)
   }
 
-  export (payload) {
-    this._writer.append(payload)
-    this._writer.flush()
+  flush (done) {
+    this._writer.flush(this.#serverlessDeliveryTracker ? undefined : done)
+    this.#serverlessDeliveryTracker?.waitForIdle(done)
   }
 }
 

package/packages/dd-trace/src/external-logger/src/index.js

@@ -33,11 +33,12 @@ class ExternalLogger {
   }
 
   static tagString (tags) {
-    const tagArray = []
+    let tagString = ''
     for (const key in tags) {
-      tagArray.push(key + ':' + tags[key])
+      if (tagString) tagString += ','
+      tagString += key + ':' + tags[key]
     }
-    return tagArray.join(',')
+    return tagString
   }
 
   // Parses and enqueues a log

package/packages/dd-trace/src/flush.js

@@ -0,0 +1,102 @@
+'use strict'
+
+const log = require('./log')
+const { supportsServerlessTelemetryRetention } = require('./serverless')
+
+/**
+ * @typedef {(done: () => void) => void} TelemetryFlusher
+ */
+
+/** @type {Set<TelemetryFlusher>} */
+const telemetryFlushers = new Set()
+const postTraceTelemetryFlushers = new Set()
+
+/**
+ * @typedef {{
+ *   trace?: TelemetryFlusher,
+ *   spanStats?: TelemetryFlusher
+ * }} TraceFlushers
+ */
+
+/**
+ * Registers a configured telemetry pipeline so serverless lifecycle retention
+ * waits for its final export alongside trace delivery.
+ * @param {TelemetryFlusher} flusher
+ * @param {{ afterTrace?: boolean }} [options]
+ * @returns {() => void} Removes this pipeline when its provider is replaced.
+ */
+function registerTelemetryFlusher (flusher, options) {
+  if (!supportsServerlessTelemetryRetention()) return () => {}
+
+  const flushers = options?.afterTrace ? postTraceTelemetryFlushers : telemetryFlushers
+  flushers.add(flusher)
+  // Avoid retaining a replaced provider or flushing it alongside the new one.
+  return () => flushers.delete(flusher)
+}
+
+/**
+ * Coordinates the configured telemetry flushers for a serverless lifecycle.
+ *
+ * Trace-owned flushers are supplied by DatadogTracer so this module does not
+ * depend on its private implementation details.
+ * @param {() => void} [done]
+ * @param {{ timeout?: number }} [options]
+ * @param {TraceFlushers} [traceFlushers]
+ */
+function flushServerlessTelemetry (done, options, traceFlushers = {}) {
+  const { trace: traceFlusher, spanStats: spanStatsFlusher } = traceFlushers
+  // TODO: Include DSM after DataStreamsProcessor exposes a completion-aware flush API.
+  let pending = telemetryFlushers.size + postTraceTelemetryFlushers.size +
+    (typeof traceFlusher === 'function' ? 1 : 0) +
+    (typeof spanStatsFlusher === 'function' ? 1 : 0)
+  let completed = false
+  let timeout
+
+  const finish = () => {
+    if (completed) return
+    completed = true
+    clearTimeout(timeout)
+    done?.()
+  }
+  const complete = () => {
+    if (--pending === 0) finish()
+  }
+
+  if (pending === 0) return finish()
+  if (options?.timeout) {
+    timeout = setTimeout(() => {
+      log.warn('Timed out waiting for telemetry flush after %dms', options.timeout)
+      finish()
+    }, options.timeout)
+  }
+
+  const flush = (flusher, afterFlushed) => {
+    let flushed = false
+    const onFlushed = error => {
+      if (flushed) return
+      flushed = true
+      if (error) log.error('Error flushing telemetry pipeline:', error)
+      afterFlushed?.()
+      complete()
+    }
+    try {
+      flusher(onFlushed)
+    } catch (error) {
+      onFlushed(error)
+    }
+  }
+
+  if (typeof traceFlusher === 'function') {
+    flush(traceFlusher, () => {
+      for (const flusher of postTraceTelemetryFlushers) flush(flusher)
+    })
+  } else {
+    for (const flusher of postTraceTelemetryFlushers) flush(flusher)
+  }
+  if (typeof spanStatsFlusher === 'function') {
+    flush(spanStatsFlusher)
+  }
+  for (const flusher of telemetryFlushers) flush(flusher)
+}
+
+module.exports = { flushServerlessTelemetry, registerTelemetryFlusher }

package/packages/dd-trace/src/llmobs/experiments/client.js

@@ -33,7 +33,8 @@ function datasetRecordFromResource (resource) {
     attrs.input ?? null,
     attrs.expected_output ?? null,
     attrs.metadata ?? {},
-    id
+    id,
+    attrs.tags ?? []
   )
 }
 
@@ -107,6 +108,10 @@ class ExperimentsClient {
     return this.#site
   }
 
+  get projectName () {
+    return this.#projectName
+  }
+
   // Dashboard URL base for the configured site, e.g. https://app.datadoghq.com
   get appBase () {
     return `https://${appHost(this.#site)}`
@@ -195,7 +200,7 @@ class ExperimentsClient {
     const resources = Array.isArray(response?.records)
       ? response.records
       : (Array.isArray(response?.data) ? response.data : [])
-    return resources.map(datasetRecordFromResource)
+    return datasetMutationResultFromResources(resources)
   }
 
   async batchUpdateDatasetRecords (projectId, datasetId, attributes) {
@@ -226,6 +231,9 @@ class ExperimentsClient {
     const query = new URLSearchParams()
     if (options.cursor) query.set('page[cursor]', options.cursor)
     if (options.version !== undefined && options.version !== null) query.set('filter[version]', String(options.version))
+    if (Array.isArray(options.tags)) {
+      for (const tag of options.tags) query.append('filter[tags]', tag)
+    }
     const response = await this.request(
       'GET',
       `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records?${query.toString()}`