dd-trace 6.12.0 → 6.14.0

package/packages/dd-trace/src/encode/0.4-cross-payload.js

@@ -0,0 +1,170 @@
+'use strict'
+
+const { AgentEncoder } = require('./0.4')
+
+const CROSS_PAYLOAD_CACHE_LIMIT = 256
+const CROSS_PAYLOAD_CANDIDATE_LIMIT = 512
+const CROSS_PAYLOAD_LEARNING_PAYLOAD_LIMIT = 8
+const CROSS_PAYLOAD_MISS_LIMIT = 32
+const CROSS_PAYLOAD_STRING_LIMIT = 256
+
+class CrossPayloadAgentEncoder extends AgentEncoder {
+  #disableCrossPayloadCache
+  #crossPayloadState = createCrossPayloadState()
+
+  /**
+   * @param {{ flush: () => void }} writer
+   * @param {() => void} disableCrossPayloadCache
+   */
+  constructor (writer, disableCrossPayloadCache) {
+    super(writer)
+    this.#disableCrossPayloadCache = disableCrossPayloadCache
+  }
+
+  /**
+   * @param {string} value
+   * @param {boolean} [stable]
+   * @returns {Buffer}
+   */
+  _cacheString (value, stable = false) {
+    if (!stable) return super._cacheString(value)
+
+    const crossPayloadState = this.#crossPayloadState
+    if (crossPayloadState === undefined || value.length === 0 || value.length > CROSS_PAYLOAD_STRING_LIMIT) {
+      return super._cacheString(value)
+    }
+
+    const learning = crossPayloadState.learningPayloadCount < CROSS_PAYLOAD_LEARNING_PAYLOAD_LIMIT
+    if (learning) markPayloadString(crossPayloadState, value)
+    const retainedEntry = crossPayloadState.stringMap?.[value]
+    if (retainedEntry !== undefined) {
+      if (!learning && crossPayloadState.hitCount < CROSS_PAYLOAD_MISS_LIMIT) {
+        crossPayloadState.hitCount++
+      }
+      const start = this._stringBytes.length
+      this._stringBytes.set(retainedEntry)
+      const entry = this._stringBytes.buffer.subarray(start, start + retainedEntry.length)
+      this._stringMap[value] = entry
+      return entry
+    }
+    if (!learning && crossPayloadState.missCount < CROSS_PAYLOAD_MISS_LIMIT) {
+      crossPayloadState.missCount++
+    }
+    if (!learning || crossPayloadState.payloadStableStrings?.[value] !== true) {
+      return super._cacheString(value)
+    }
+
+    let entry = super._cacheString(value)
+    if (entry.length > CROSS_PAYLOAD_STRING_LIMIT) return entry
+
+    const previousPayloadStrings = crossPayloadState.previousPayloadStrings
+    if (previousPayloadStrings?.[value] !== undefined &&
+      crossPayloadState.stringCount < CROSS_PAYLOAD_CACHE_LIMIT) {
+      entry = Buffer.from(entry)
+      crossPayloadState.stringMap ??= Object.create(null)
+      crossPayloadState.stringMap[value] = entry
+      crossPayloadState.stringCount++
+    }
+    return entry
+  }
+
+  /** @returns {Buffer} */
+  makePayload () {
+    this.#prepareReset()
+    return super.makePayload()
+  }
+
+  /** @returns {void} */
+  reset () {
+    this.#prepareReset()
+    super.reset()
+  }
+
+  /** @returns {void} */
+  #prepareReset () {
+    const crossPayloadState = this.#crossPayloadState
+    if (crossPayloadState !== undefined && !prepareCrossPayloadReset(crossPayloadState)) {
+      this.#crossPayloadState = undefined
+      this.#disableCrossPayloadCache()
+    }
+  }
+}
+
+/**
+ * @param {{ flush: () => void }} writer
+ * @param {() => void} disableCrossPayloadCache
+ * @returns {AgentEncoder}
+ */
+function createAgentEncoder (writer, disableCrossPayloadCache) {
+  return new CrossPayloadAgentEncoder(writer, disableCrossPayloadCache)
+}
+
+function createCrossPayloadState () {
+  return {
+    hitCount: 0,
+    learningPayloadCount: 0,
+    lowUseCount: 0,
+    missCount: 0,
+    payloadStableStringCount: 0,
+    payloadStableStrings: undefined,
+    previousPayloadStrings: undefined,
+    stringCount: 0,
+    stringMap: undefined,
+  }
+}
+
+/**
+ * @param {ReturnType<typeof createCrossPayloadState>} crossPayloadState
+ * @param {string} value
+ */
+function markPayloadString (crossPayloadState, value) {
+  if (crossPayloadState.payloadStableStringCount >= CROSS_PAYLOAD_CANDIDATE_LIMIT ||
+    crossPayloadState.payloadStableStrings?.[value] !== undefined) {
+    return
+  }
+  crossPayloadState.payloadStableStrings ??= Object.create(null)
+  crossPayloadState.payloadStableStrings[value] = true
+  crossPayloadState.payloadStableStringCount++
+}
+
+/**
+ * @param {ReturnType<typeof createCrossPayloadState>} crossPayloadState
+ * @returns {boolean}
+ */
+function prepareCrossPayloadReset (crossPayloadState) {
+  const learning = crossPayloadState.learningPayloadCount < CROSS_PAYLOAD_LEARNING_PAYLOAD_LIMIT
+  const lowUse = !learning && crossPayloadState.hitCount < crossPayloadState.missCount
+  if (lowUse) {
+    crossPayloadState.lowUseCount++
+    if (crossPayloadState.lowUseCount >= 2) return false
+    crossPayloadState.learningPayloadCount = 0
+    crossPayloadState.stringCount = 0
+    crossPayloadState.stringMap = undefined
+    crossPayloadState.payloadStableStrings = undefined
+    crossPayloadState.payloadStableStringCount = 0
+    crossPayloadState.previousPayloadStrings = undefined
+    crossPayloadState.hitCount = 0
+    crossPayloadState.missCount = 0
+    return true
+  }
+  if (!learning && crossPayloadState.hitCount > 0) {
+    crossPayloadState.lowUseCount = 0
+  }
+
+  if (learning) {
+    crossPayloadState.learningPayloadCount++
+    if (crossPayloadState.learningPayloadCount < CROSS_PAYLOAD_LEARNING_PAYLOAD_LIMIT) {
+      crossPayloadState.previousPayloadStrings = crossPayloadState.payloadStableStrings
+    } else {
+      crossPayloadState.previousPayloadStrings = undefined
+      if (crossPayloadState.stringMap === undefined) return false
+    }
+  }
+  crossPayloadState.payloadStableStrings = undefined
+  crossPayloadState.payloadStableStringCount = 0
+  crossPayloadState.hitCount = 0
+  crossPayloadState.missCount = 0
+  return true
+}
+
+module.exports = { createAgentEncoder }

package/packages/dd-trace/src/encode/0.4.js

@@ -339,11 +339,11 @@ class AgentEncoder {
       // Replaces up to ten separate `bytes.reserve` calls per span with one.
       let typeEntry
       if (span.type) {
-        typeEntry = stringMap[span.type] ?? this._cacheString(span.type)
+        typeEntry = stringMap[span.type] ?? this._cacheString(span.type, true)
       }
-      const nameEntry = stringMap[span.name] ?? this._cacheString(span.name)
-      const resourceEntry = stringMap[span.resource] ?? this._cacheString(span.resource)
-      const serviceEntry = stringMap[span.service] ?? this._cacheString(span.service)
+      const nameEntry = stringMap[span.name] ?? this._cacheString(span.name, true)
+      const resourceEntry = stringMap[span.resource] ?? this._cacheString(span.resource, true)
+      const serviceEntry = stringMap[span.service] ?? this._cacheString(span.service, true)
       const nameLen = nameEntry.length
       const resourceLen = resourceEntry.length
       const serviceLen = serviceEntry.length
@@ -589,7 +589,7 @@ class AgentEncoder {
       const entryValue = value[key]
       if (typeof entryValue !== 'string' && typeof entryValue !== 'number') continue
 
-      const keyEntry = stringMap[key] ?? this._cacheString(key)
+      const keyEntry = stringMap[key] ?? this._cacheString(key, true)
       const keyEntryLen = keyEntry.length
       const writeOffset = bytes.length
 

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -69,11 +69,11 @@ function truncateTestLevelMetadataTags (tags) {
 }
 
 class AgentlessCiVisibilityEncoder extends AgentEncoder {
-  constructor (writer, { runtimeId, service, env }) {
+  constructor (writer, { tags }) {
     super(writer, INTAKE_SOFT_LIMIT)
-    this.runtimeId = runtimeId
-    this.service = service
-    this.env = env
+    // Holds a reference to the live `tags` object (instead of copying `env`/`runtime-id` out of it)
+    // so a later change (e.g. a MicroVM clone resume) is picked up at flush time.
+    this.tags = tags
 
     // Used to keep track of the number of encoded events to update the
     // length of `payload.events` when calling `makePayload`
@@ -406,11 +406,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       events: [],
     }
 
-    if (this.env) {
-      payload.metadata['*'].env = this.env
+    if (this.tags.env) {
+      payload.metadata['*'].env = this.tags.env
     }
-    if (this.runtimeId) {
-      payload.metadata['*']['runtime-id'] = this.runtimeId
+    const runtimeId = this.tags['runtime-id']
+    if (runtimeId) {
+      payload.metadata['*']['runtime-id'] = runtimeId
     }
 
     bytes.writeMapPrefix(Object.keys(payload).length)

package/packages/dd-trace/src/evp_proxy/direct.js

@@ -1,11 +1,13 @@
 'use strict'
 
-const { format } = require('node:url')
-
 const { HttpsProxyAgent } = require('../../../../vendor/dist/https-proxy-agent')
 const { getProxyForUrl } = require('../../../../vendor/dist/proxy-from-env')
+const { createSiteUrl } = require('../exporters/common/url')
 const log = require('../log')
 
+/** @type {Map<string, import('node:https').Agent>} */
+const proxyAgents = new Map()
+
 /**
  * @typedef {object} DirectEVPRoute
  * @property {URL} url - Direct intake URL
@@ -28,37 +30,39 @@ function createDirectEVPRoute (config, intake) {
   if (!apiKey || !config.site) return
 
   try {
-    const hostname = `${intake}.${config.site}`.toLowerCase()
-    const url = new URL(format({
-      protocol: 'https:',
-      hostname,
-    }))
-    if (
-      url.hostname !== hostname ||
-      url.username ||
-      url.password ||
-      url.port ||
-      url.pathname !== '/' ||
-      url.search ||
-      url.hash
-    ) {
-      throw new Error('Invalid direct EVP intake URL')
-    }
+    const url = createSiteUrl(config.site, intake)
+    if (url === undefined) throw new Error('Invalid direct EVP intake URL')
 
-    const proxyUrl = getProxyForUrl(url.href)
-    const agent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined
+    const agent = getHttpsProxyAgent(url.href)
 
-    return {
+    const route = {
       url,
       basePath: '',
       headers: {
         'DD-API-KEY': apiKey,
       },
-      ...(agent && { agent }),
     }
+    if (agent) route.agent = agent
+    return route
   } catch (error) {
     log.debug('Unable to configure direct EVP intake: %s', error.message)
   }
 }
 
-module.exports = { createDirectEVPRoute }
+/**
+ * @param {string} url
+ * @returns {import('node:https').Agent|undefined}
+ */
+function getHttpsProxyAgent (url) {
+  const proxyUrl = getProxyForUrl(url)
+  if (!proxyUrl) return
+
+  let agent = proxyAgents.get(proxyUrl)
+  if (agent === undefined) {
+    agent = new HttpsProxyAgent(proxyUrl)
+    proxyAgents.set(proxyUrl, agent)
+  }
+  return agent
+}
+
+module.exports = { createDirectEVPRoute, getHttpsProxyAgent }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -2,14 +2,17 @@
 
 const { URL } = require('url')
 const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 const Writer = require('./writer')
 
 class AgentExporter {
   #timer
+  #serverlessDeliveryTracker
 
   constructor (config, prioritySampler) {
+    this.#serverlessDeliveryTracker = createServerlessDeliveryTracker()
     this._config = config
-    const { lookup, protocolVersion, stats = {}, apmTracingEnabled } = config
+    const { lookup, protocolVersion, stats = {}, apmTracingEnabled, flushInterval } = config
     this._url = config.url
 
     const headers = {}
@@ -22,7 +25,9 @@ class AgentExporter {
       prioritySampler,
       lookup,
       protocolVersion,
+      flushInterval,
       headers,
+      deliveryTracker: this.#serverlessDeliveryTracker,
     })
 
     globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(this.flush.bind(this))
@@ -54,10 +59,26 @@ class AgentExporter {
     }
   }
 
-  flush (done = () => {}) {
+  flush (done) {
     clearTimeout(this.#timer)
     this.#timer = undefined
-    this._writer.flush(done)
+
+    if (!this.#serverlessDeliveryTracker) {
+      try {
+        return this._writer.flush(done)
+      } catch (error) {
+        log.error('Failed to flush traces: %s', error.message)
+        done?.()
+        return
+      }
+    }
+
+    try {
+      this._writer.flush()
+    } catch (error) {
+      log.error('Failed to flush traces: %s', error.message)
+    }
+    this.#serverlessDeliveryTracker.waitForIdle(done)
   }
 }
 

package/packages/dd-trace/src/exporters/agent/writer.js

@@ -19,18 +19,19 @@ class AgentWriter extends BaseWriter {
   #requestTracker
 
   constructor (...args) {
+    const { isTestOptimization } = args[0]
     super({
       ...args[0],
       beforeFirstFlush: () => firstFlushChannel.publish(),
+      retainOnBackpressure: isTestOptimization,
     })
-    const { prioritySampler, lookup, protocolVersion, headers, isTestOptimization } = args[0]
-    const AgentEncoder = getEncoder(protocolVersion)
+    const { prioritySampler, lookup, protocolVersion, flushInterval, headers } = args[0]
 
     this._prioritySampler = prioritySampler
     this._lookup = lookup
     this._protocolVersion = protocolVersion
     this._headers = headers
-    this._encoder = new AgentEncoder(this)
+    this._encoder = createEncoder(protocolVersion, flushInterval, this)
     if (isTestOptimization) {
       this.#request = require('../../ci-visibility/exporters/request')
       const TestOptimizationRequestTracker = require('../../ci-visibility/exporters/agentless/request-tracker')
@@ -39,18 +40,18 @@ class AgentWriter extends BaseWriter {
   }
 
   /**
-   * Flushes payloads, including requests already in flight during Test Optimization finalization.
-   *
+   * Performs the writer flush without registering a serverless delivery.
+   * Test Optimization owns its own request lifecycle tracking.
    * @param {(error?: Error) => void} [done]
    * @param {{ deadline?: number }} [options]
    * @returns {void}
    */
-  flush (done, options) {
+  flushDirect (done, options) {
     if (this.#requestTracker) {
       this.#requestTracker.flush(done, options)
       return
     }
-    super.flush(done, options)
+    super.flushDirect(done, options)
   }
 
   _sendPayload (data, count, done, flushOptions) {
@@ -113,10 +114,26 @@ class AgentWriter extends BaseWriter {
   }
 }
 
-function getEncoder (protocolVersion) {
-  return protocolVersion === '0.5'
-    ? require('../../encode/0.5').AgentEncoder
-    : require('../../encode/0.4').AgentEncoder
+/**
+ * @param {string} protocolVersion
+ * @param {number | undefined} flushInterval
+ * @param {AgentWriter} writer
+ * @returns {import('../../encode/0.4').AgentEncoder}
+ */
+function createEncoder (protocolVersion, flushInterval, writer) {
+  if (protocolVersion === '0.5') {
+    const { AgentEncoder } = require('../../encode/0.5')
+    return new AgentEncoder(writer)
+  }
+  if (flushInterval === 0) {
+    const { createAgentEncoder } = require('../../encode/0.4-cross-payload')
+    const disableCrossPayloadCache = () => {
+      writer._encoder = createEncoder('0.4', undefined, writer)
+    }
+    return createAgentEncoder(writer, disableCrossPayloadCache)
+  }
+  const { AgentEncoder } = require('../../encode/0.4')
+  return new AgentEncoder(writer)
 }
 
 function makeRequest (version, data, count, url, headers, lookup, flushOptions, request, requestTracker, cb) {

package/packages/dd-trace/src/exporters/agentless/index.js

@@ -4,8 +4,7 @@ const { URL } = require('node:url')
 const os = require('node:os')
 
 const log = require('../../log')
-const { entityId } = require('../common/docker')
-const tracerVersion = require('../../../../../package.json').version
+const { containerId } = require('../common/docker')
 const Writer = require('./writer')
 const { computeIntakeUrl } = require('./intake')
 
@@ -23,7 +22,7 @@ class AgentlessExporter {
    * @param {string} [config.site] - The Datadog site. Defaults to 'datadoghq.com'.
    * @param {number} [config.flushInterval] - Batch flush interval in ms
    * @param {string} [config.env] - Environment name
-   * @param {object} [config.tags] - Tags including runtime-id
+   * @param {object} config.tags - Tags including runtime-id
    */
   constructor (config) {
     this.#config = config
@@ -40,13 +39,12 @@ class AgentlessExporter {
 
     const metadata = {
       hostname: os.hostname(),
-      env: config.env,
-      languageName: 'nodejs',
-      languageVersion: process.version,
-      tracerVersion,
-      runtimeID: config.tags?.['runtime-id'],
-      ...(entityId ? { containerID: entityId } : {}),
+      // Read live off `config` (instead of copying the value) so a later change
+      // (e.g. a MicroVM clone resume) is picked up by the next data-pipeline export.
+      get env () { return config.env },
+      get runtimeID () { return config.tags['runtime-id'] },
     }
+    if (containerId) metadata.containerId = containerId
 
     this._writer = new Writer({
       url: this._url,