dd-trace 6.12.0 → 6.14.0

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -4,6 +4,7 @@ const { URL, format } = require('node:url')
 const path = require('node:path')
 const request = require('../../exporters/common/request')
 const { getEnvironmentVariable } = require('../../config/helper')
+const { createServerlessDeliveryTracker } = require('../../serverless')
 
 const logger = require('../../log')
 
@@ -16,13 +17,20 @@ const {
 } = require('../constants/writers')
 const { parseResponseAndLog } = require('./util')
 
+const MAX_BUFFER_EVENTS = 1000
+
 class LLMObsBuffer {
-  constructor ({ events, size, routing = {}, isDefault = false, limit = 1000 }) {
+  /**
+   * @param {{
+   *   events: Record<string, unknown>[],
+   *   size: number,
+   *   routing?: { apiKey?: string, site?: string }
+   * }} options
+   */
+  constructor ({ events, size, routing = {} }) {
     this.events = events
     this.size = size
     this.routing = routing
-    this.isDefault = isDefault
-    this.limit = limit
   }
 
   clear () {
@@ -33,6 +41,9 @@ class LLMObsBuffer {
 
 class BaseLLMObsWriter {
   #destroyer
+  /** @type {Function[]} */
+  #pendingFlushes = []
+  #serverlessDeliveryTracker = createServerlessDeliveryTracker()
   /** @type {Map<string, LLMObsBuffer>} */
   #multiTenantBuffers = new Map()
 
@@ -42,7 +53,7 @@ class BaseLLMObsWriter {
     this._eventType = eventType
 
     /** @type {LLMObsBuffer} */
-    this._buffer = new LLMObsBuffer({ events: [], size: 0, isDefault: true })
+    this._buffer = new LLMObsBuffer({ events: [], size: 0 })
 
     /** @type {import('../../config/config-base')} */
     this._config = config
@@ -98,8 +109,8 @@ class BaseLLMObsWriter {
   append (event, routing, byteLength) {
     const buffer = this._getBuffer(routing)
 
-    if (buffer.events.length >= buffer.limit) {
-      logger.warn(`${this.constructor.name} event buffer full (limit is ${buffer.limit}), dropping event`)
+    if (buffer.events.length >= MAX_BUFFER_EVENTS) {
+      logger.warn(`${this.constructor.name} event buffer full (limit is ${MAX_BUFFER_EVENTS}), dropping event`)
       telemetry.recordDroppedPayload(1, this._eventType, 'buffer_full')
       return false
     }
@@ -111,80 +122,118 @@ class BaseLLMObsWriter {
     return true
   }
 
-  flush () {
+  /**
+   * Drains buffered events and joins requests active at this flush boundary.
+   * @param {Function} [done]
+   */
+  flush (done) {
     if (this._agentless == null) {
+      if (done) this.#pendingFlushes.push(done)
       return
     }
 
-    // Flush default buffer
-    if (this._buffer.events.length > 0) {
-      const events = this._buffer.events
-      this._buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
+    const requests = this.#drainBuffers()
+    if (!this.#serverlessDeliveryTracker) {
+      // Only invocation-retaining platforms need completion-aware delivery.
+      for (const request of requests) this.#sendSafely(request)
+      done?.()
+      return
+    }
 
-      log.debug('Encoded LLMObs payload: %s', payload)
+    for (const request of requests) this.#sendSafely(request)
+    this.#serverlessDeliveryTracker.waitForIdle(done)
+  }
 
-      const options = this._getOptions()
+  #sendSafely (requestToSend) {
+    try {
+      if (this.#serverlessDeliveryTracker) {
+        this.#serverlessDeliveryTracker.track(done => this.#send(requestToSend, done))
+      } else {
+        this.#send(requestToSend)
+      }
+    } catch (error) {
+      logger.error('Failed to send LLMObs %s events: %s', this._eventType, error.message)
+    }
+  }
 
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, this.url, this._eventType)
-      })
+  #drainBuffers () {
+    const requests = []
+    if (this._buffer.events.length > 0) {
+      const events = this._buffer.events
+      this._buffer.clear()
+      requests.push({ events, options: this._getOptions(), url: this.url })
     }
 
-    // Flush multi-tenant buffers
     for (const [apiKey, buffer] of this.#multiTenantBuffers) {
-      if (buffer.events.length === 0) continue
-
-      const events = buffer.events
-      buffer.clear()
-
-      const payload = this._encode(this.makePayload(events))
+      if (buffer.events.length === 0) {
+        this.#multiTenantBuffers.delete(apiKey)
+        continue
+      }
       const site = buffer.routing.site || this._config.site
-      const options = {
-        headers: {
-          'Content-Type': 'application/json',
-          'DD-API-KEY': apiKey,
-        },
-        method: 'POST',
-        timeout: this._timeout,
-        url: new URL(format({
-          protocol: 'https:',
-          hostname: `${this._intake}.${site}`,
-        })),
-        path: this._baseEndpoint,
+      const maskedApiKey = `****${apiKey.slice(-4)}`
+      let options
+      let url
+      try {
+        options = {
+          headers: {
+            'Content-Type': 'application/json',
+            'DD-API-KEY': apiKey,
+          },
+          method: 'POST',
+          timeout: this._timeout,
+          url: new URL(format({
+            protocol: 'https:',
+            hostname: `${this._intake}.${site}`,
+          })),
+          path: this._baseEndpoint,
+        }
+        url = this.#buildUrl(options.url.href, options.path)
+      } catch (error) {
+        buffer.clear()
+        this.#multiTenantBuffers.delete(apiKey)
+        logger.error(
+          'Failed to route LLMObs %s events for API key %s: %s',
+          this._eventType,
+          maskedApiKey,
+          error.message
+        )
+        continue
       }
-      const url = this.#buildUrl(options.url.href, options.path)
-      const maskedApiKey = apiKey ? `****${apiKey.slice(-4)}` : ''
 
+      const events = buffer.events
+      buffer.clear()
+      this.#multiTenantBuffers.delete(apiKey)
       log.debug('Encoding and flushing multi-tenant buffer for %s', maskedApiKey)
-      log.debug('Encoded LLMObs payload: %s', payload)
-
-      request(payload, options, (err, resp, code) => {
-        parseResponseAndLog(err, code, events.length, url, this._eventType)
-      })
+      requests.push({ events, options, url })
     }
 
-    this.#cleanupEmptyBuffers()
+    return requests
   }
 
-  #cleanupEmptyBuffers () {
-    for (const [key, buffer] of this.#multiTenantBuffers) {
-      if (buffer.events.length === 0) {
-        this.#multiTenantBuffers.delete(key)
-      }
-    }
+  #send ({ events, options, url }, done) {
+    const payload = this._encode(this.makePayload(events))
+    log.debug('Encoded LLMObs payload: %s', payload)
+    request(payload, options, (err, resp, code) => {
+      parseResponseAndLog(err, code, events.length, url, this._eventType)
+      done?.()
+    })
   }
 
   makePayload (events) {}
 
-  destroy () {
+  /**
+   * Stops periodic flushing and drains buffered events.
+   * @param {Function} [done] Called after queued and active deliveries complete.
+   */
+  destroy (done) {
     if (this.#destroyer) {
       logger.debug(`Stopping ${this.constructor.name}`)
       clearInterval(this._periodic)
       globalThis[Symbol.for('dd-trace')].beforeExitHandlers.delete(this.#destroyer)
-      this.flush()
+      this.flush(done)
       this.#destroyer = undefined
+    } else {
+      done?.()
     }
   }
 
@@ -196,6 +245,10 @@ class BaseLLMObsWriter {
     this._endpoint = endpoint
 
     logger.debug(`Configuring ${this.constructor.name} to ${this.url}`)
+
+    const pendingFlushes = this.#pendingFlushes
+    this.#pendingFlushes = []
+    for (const done of pendingFlushes) this.flush(done)
   }
 
   _getUrlAndPath () {

package/packages/dd-trace/src/log/index.js

@@ -29,7 +29,13 @@ const log = {
       const stack = logRecord.stack.split('\n')
       const fn = stack[1].replace(/^\s+at ([^\s]+) .+/, '$1')
       const options = { depth: 2, breakLength: Infinity, compact: true, maxArrayLength: Infinity }
-      const params = args.map(a => inspect(a, options)).join(', ')
+      let params = ''
+      let isFirstParameter = true
+      for (const argument of args) {
+        if (!isFirstParameter) params += ', '
+        params += inspect(argument, options)
+        isFirstParameter = false
+      }
 
       stack[0] = `Trace: ${fn}(${params})`
 

package/packages/dd-trace/src/log-submission/log-submission-plugin.js

@@ -0,0 +1,249 @@
+'use strict'
+
+const { Writable } = require('node:stream')
+
+const FinalFlushRequestTracker = require('../exporters/common/final-flush-request-tracker')
+const request = require('../exporters/common/request')
+const log = require('../log')
+const Plugin = require('../plugins/plugin')
+
+const MAX_BATCH_BYTES = 5 * 1024 * 1024
+const MAX_BATCH_LOGS = 1000
+const BATCH_FLUSH_INTERVAL = 1000
+const FINAL_FLUSH_TIMEOUT = 60_000
+
+/**
+ * @returns {Error & { code: string }}
+ */
+function createFinalFlushTimeoutError () {
+  return Object.assign(
+    new Error('Timed out waiting for automatic log submission to flush'),
+    { code: 'ERR_DD_LOG_SUBMISSION_FLUSH_TIMEOUT' }
+  )
+}
+
+/**
+ * @param {import('../config/config-base')} config
+ * @returns {URL | void}
+ */
+function getLogSubmissionUrl (config) {
+  if (config.DD_AGENTLESS_LOG_SUBMISSION_URL) {
+    try {
+      const url = new URL(config.DD_AGENTLESS_LOG_SUBMISSION_URL)
+      if (url.protocol === 'http:' || url.protocol === 'https:') return url
+
+      log.error('Unsupported automatic log submission URL protocol: %s', url.protocol)
+    } catch {
+      log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
+    }
+    return
+  }
+
+  const hostname = `http-intake.logs.${config.site}`.toLowerCase()
+  try {
+    const url = new URL(`https://${hostname}`)
+    if (url.hostname === hostname) return url
+  } catch {}
+
+  log.error('Could not parse automatic log submission site: %s', config.site)
+}
+
+/**
+ * @param {unknown} config
+ * @returns {import('../config/config-base')}
+ */
+function asTracerConfig (config) {
+  return /** @type {import('../config/config-base')} */ (config)
+}
+
+/**
+ * @param {import('../config/config-base')} config
+ * @param {string} source
+ * @returns {string}
+ */
+function getLogSubmissionPath (config, source) {
+  return `/api/v2/logs?${new URLSearchParams({ ddsource: source, service: config.service })}`
+}
+
+class LogSubmissionPlugin extends Plugin {
+  static id = 'log-submission'
+
+  /** @type {string[]} */
+  #batch = []
+  #batchBytes = 2
+  #batchSource
+  /** @type {import('../config/config-base') | undefined} */
+  #config
+  #logSubmissionUrl
+  #requestTracker
+  #timer
+  #beforeExitHandler = () => this.#flushLogs()
+  #createWinstonJsonFormat
+  #winstonStreamClass
+  // Winston formats records inside its transports, not at logger.write time, so (unlike Bunyan/Pino)
+  // the instrumentation can't publish a post-format line. A Stream transport pipes format.json()
+  // output through this Writable into the shared sender, reusing Winston's own cycle-safe serializer.
+  #winstonOutput = new Writable({
+    decodeStrings: false,
+    write: (message, encoding, callback) => {
+      this.#enqueueLog({ source: 'winston', message })
+      callback()
+    },
+  })
+
+  /**
+   * @param {object} tracer
+   * @param {unknown} tracerConfig
+   */
+  constructor (tracer, tracerConfig) {
+    super(tracer, asTracerConfig(tracerConfig))
+    this.#requestTracker = new FinalFlushRequestTracker(
+      (done) => {
+        this.#flushLogs()
+        done?.()
+      },
+      createFinalFlushTimeoutError
+    )
+
+    // The main-module hook (configure) and the logger.js hook (add-transport) can fire in either
+    // order depending on how Winston is required, so buffer loggers that arrive before configure.
+    const pendingWinstonLoggers = new Set()
+    const addWinstonTransport = (logger) => {
+      if (!this.#winstonStreamClass || !this.#createWinstonJsonFormat) {
+        pendingWinstonLoggers.add(logger)
+        return
+      }
+
+      logger.add(new this.#winstonStreamClass({
+        format: this.#createWinstonJsonFormat(),
+        stream: this.#winstonOutput,
+      }))
+    }
+
+    this.addSub('ci:log-submission:winston:configure', ({ StreamTransport, createJsonFormat }) => {
+      this.#winstonStreamClass = StreamTransport
+      this.#createWinstonJsonFormat = createJsonFormat
+
+      for (const logger of pendingWinstonLoggers) {
+        addWinstonTransport(logger)
+      }
+      pendingWinstonLoggers.clear()
+    })
+
+    this.addSub('ci:log-submission:winston:add-transport', addWinstonTransport)
+
+    this.addSub('ci:log-submission:log', (payload) => {
+      this.#enqueueLog(payload)
+    })
+    this.addSub('ci:log-submission:flush', ({ onDone } = {}) => {
+      if (!onDone) {
+        this.#flushLogs()
+        return
+      }
+
+      this.#requestTracker.flush(onDone, {
+        deadline: Date.now() + FINAL_FLUSH_TIMEOUT,
+      })
+    })
+  }
+
+  /**
+   * @param {boolean | (Record<string, unknown> & { enabled: boolean })} config
+   * @returns {void}
+   */
+  configure (config) {
+    if (this._enabled) this.#flushLogs()
+
+    const isEnabled = typeof config === 'boolean' ? config : config.enabled
+    this.#config = isEnabled && typeof config !== 'boolean' ? asTracerConfig(config) : undefined
+    this.#logSubmissionUrl = this.#config
+      ? getLogSubmissionUrl(this.#config)
+      : undefined
+    super.configure(config)
+
+    const beforeExitHandlers = globalThis[Symbol.for('dd-trace')].beforeExitHandlers
+    if (this._enabled) {
+      beforeExitHandlers.add(this.#beforeExitHandler)
+    } else {
+      beforeExitHandlers.delete(this.#beforeExitHandler)
+    }
+  }
+
+  /**
+   * @param {{ source: string, message: string | Record<string, unknown> }} payload
+   * @returns {void}
+   */
+  #enqueueLog ({ source, message }) {
+    if (!this.#logSubmissionUrl) return
+
+    let serializedMessage
+    try {
+      serializedMessage = typeof message === 'string' ? message : JSON.stringify(message)
+    } catch (error) {
+      log.error('Could not serialize %s log for automatic submission', source, error)
+      return
+    }
+    if (serializedMessage === undefined) return
+
+    const messageBytes = Buffer.byteLength(serializedMessage)
+    if (messageBytes + 2 > MAX_BATCH_BYTES) {
+      log.error('Could not submit %s log because it exceeds the %d byte payload limit', source, MAX_BATCH_BYTES)
+      return
+    }
+
+    if (this.#batch.length > 0 &&
+        (this.#batchSource !== source || this.#batchBytes + messageBytes + 1 > MAX_BATCH_BYTES)) {
+      this.#flushLogs()
+      if (!this.#logSubmissionUrl) return
+    }
+
+    this.#batchSource = source
+    if (this.#batch.length > 0) this.#batchBytes++
+    this.#batch.push(serializedMessage)
+    this.#batchBytes += messageBytes
+
+    if (this.#batch.length === MAX_BATCH_LOGS || this.#batchBytes === MAX_BATCH_BYTES) {
+      this.#flushLogs()
+    } else if (this.#timer === undefined) {
+      this.#timer = setTimeout(() => this.#flushLogs(), BATCH_FLUSH_INTERVAL)
+      this.#timer.unref?.()
+    }
+  }
+
+  /**
+   * @returns {void}
+   */
+  #flushLogs () {
+    clearTimeout(this.#timer)
+    this.#timer = undefined
+
+    const config = this.#config
+    if (this.#batch.length === 0 || !this.#logSubmissionUrl || !config) return
+
+    const source = this.#batchSource
+    const data = `[${this.#batch.join(',')}]`
+    this.#batch = []
+    this.#batchBytes = 2
+    this.#batchSource = undefined
+    const options = {
+      path: getLogSubmissionPath(config, source),
+      method: 'POST',
+      headers: {
+        'DD-API-KEY': config.DD_API_KEY,
+        'Content-Type': 'application/json',
+      },
+      url: this.#logSubmissionUrl,
+    }
+
+    try {
+      this.#requestTracker.send(request, data, options, error => {
+        if (error) log.error('Error submitting %s logs', source, error)
+      })
+    } catch (error) {
+      this.#logSubmissionUrl = undefined
+      log.error('Error submitting %s logs', source, error)
+    }
+  }
+}
+
+module.exports = LogSubmissionPlugin

package/packages/dd-trace/src/noop/span.js

@@ -26,6 +26,11 @@ class NoopSpan {
   addLink (link) { return this }
   addLinks (links) { return this }
   addSpanPointer (ptrKind, ptrDir, ptrHash) { return this }
+  /**
+   * @param {import('../../../../index').Exception} exception
+   * @param {import('../../../../index').SpanEventAttributes} [attributes]
+   */
+  recordException (exception, attributes) {}
   log () { return this }
   logEvent () {}
   finish (finishTime) {}

package/packages/dd-trace/src/openfeature/writers/base.js

@@ -214,21 +214,23 @@ class BaseFFEWriter {
    * @returns {ActiveWriterRoute} Active route state
    */
   #createRoute (route) {
+    const requestOptions = {
+      headers: {
+        ...route.headers,
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+      retry: true,
+      timeout: this._timeout,
+      url: route.url,
+      path: route.endpoint,
+    }
+    if (route.agent) requestOptions.agent = route.agent
+
     return {
       url: route.url,
       endpoint: route.endpoint,
-      requestOptions: {
-        ...(route.agent && { agent: route.agent }),
-        headers: {
-          ...route.headers,
-          'Content-Type': 'application/json',
-        },
-        method: 'POST',
-        retry: true,
-        timeout: this._timeout,
-        url: route.url,
-        path: route.endpoint,
-      },
+      requestOptions,
     }
   }
 

package/packages/dd-trace/src/openfeature/writers/exposures.js

@@ -37,6 +37,7 @@ const PENDING_MAX_EVENTS = 1000
  * @property {string} flag.key - Flag key
  * @property {object} variant - Variant information
  * @property {string} variant.key - Variant key
+ * @property {number} [serial_id] - Serial id of the split the subject landed in
  * @property {object} subject - Subject (user/entity) information
  * @property {string} subject.id - Subject identifier
  * @property {string} [subject.type] - Subject type
@@ -204,7 +205,7 @@ class ExposuresWriter extends BaseFFEWriter {
   makePayload (events) {
     const formattedEvents = events.map(event => {
       /** @type {ExposureEvent} */
-      return {
+      const formatted = {
         timestamp: event.timestamp || Date.now(),
         allocation: {
           key: event.allocation?.key || event['allocation.key'],
@@ -221,6 +222,12 @@ class ExposuresWriter extends BaseFFEWriter {
           attributes: event.subject?.attributes,
         },
       }
+
+      if (typeof event.serial_id === 'number') {
+        formatted.serial_id = event.serial_id
+      }
+
+      return formatted
     })
 
     return {

package/packages/dd-trace/src/openfeature/writers/util.js

@@ -75,8 +75,8 @@ function setAgentlessStrategy (config, setWriterEnabledValue) {
         headers: {
           [EVP_SUBDOMAIN_HEADER_NAME]: EVP_EVENT_PLATFORM_SUBDOMAIN,
         },
-        ...(directRoute && { fallback: directRoute }),
       }
+      if (directRoute) route.fallback = directRoute
       logger.debug('FFE Writer enabled with local EVP route %s', route.basePath)
       setWriterEnabledValue(true, route)
       return

package/packages/dd-trace/src/opentelemetry/logs/batch_log_processor.js

@@ -1,5 +1,8 @@
 'use strict'
 
+const log = require('../../log')
+const { createServerlessDeliveryTracker } = require('../../serverless')
+
 /**
  * @typedef {import('@opentelemetry/api-logs').LogRecord} LogRecord
  * @typedef {import('@opentelemetry/core').InstrumentationScope} InstrumentationScope
@@ -54,10 +57,48 @@ class BatchLogRecordProcessor {
 
   /**
    * Forces an immediate flush of all pending log records.
-   * @returns {undefined} Promise that resolves when flush is complete
+   * @param {Function} [done] Called after all pending log exports complete
    */
-  forceFlush () {
-    this.#export()
+  forceFlush (done) {
+    this.#clearTimer()
+
+    const deliveryTracker = createServerlessDeliveryTracker()
+    if (!deliveryTracker) {
+      // Normal processes preserve the existing fire-and-forget batch flush.
+      this.#export()
+      done?.()
+      return
+    }
+
+    // Flush only records present at this boundary. New records belong to the
+    // later request that produced them and must not extend this lifecycle flush.
+    const logRecords = this.#logRecords
+    this.#logRecords = []
+
+    // Join exports already active at this boundary before draining this snapshot.
+    if (typeof this.exporter.flush === 'function') {
+      deliveryTracker.track(complete => this.exporter.flush(complete))
+    }
+
+    deliveryTracker.track(complete => {
+      const flushNext = () => {
+        if (logRecords.length === 0) {
+          complete()
+          return
+        }
+
+        // Drain the boundary snapshot one batch at a time.
+        const batch = logRecords.splice(0, this.#maxExportBatchSize)
+        try {
+          this.exporter.export(batch, flushNext)
+        } catch (error) {
+          log.error('Error exporting OTLP logs:', error)
+          complete()
+        }
+      }
+      flushNext()
+    })
+    deliveryTracker.waitForIdle(done)
   }
 
   /**
@@ -79,6 +120,7 @@ class BatchLogRecordProcessor {
    * @private
    */
   #export () {
+    if (this.#logRecords.length === 0) return
     const logRecords = this.#logRecords.slice(0, this.#maxExportBatchSize)
     this.#logRecords = this.#logRecords.slice(this.#maxExportBatchSize)