dd-trace 6.12.0 → 6.13.0

package/packages/dd-trace/src/ci-visibility/exporters/agentless/request-tracker.js

@@ -1,22 +1,21 @@
 'use strict'
 
 const BaseWriter = require('../../../exporters/common/writer')
+const { createFinalFlushTimeoutError } = require('../../final-flush')
 
-const FINAL_FLUSH_TIMEOUT_CODE = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
-
-class TestOptimizationRequestTracker {
-  #writer
+class FinalFlushRequestTracker {
+  #flush
   #pendingRequests = new Set()
   #finalFlushes = new Set()
   #activeFinalFlush
 
   /**
-   * Creates request tracking for a Test Optimization writer.
+   * Creates request tracking for a Test Optimization exporter.
    *
-   * @param {BaseWriter} writer
+   * @param {(done?: (error?: Error) => void, options?: { deadline?: number }) => void} flush
    */
-  constructor (writer) {
-    this.#writer = writer
+  constructor (flush) {
+    this.#flush = flush
   }
 
   /**
@@ -30,7 +29,7 @@ class TestOptimizationRequestTracker {
    */
   flush (done, options) {
     if (options?.deadline === undefined) {
-      BaseWriter.prototype.flush.call(this.#writer, done, options)
+      this.#flush(done, options)
       return
     }
 
@@ -50,8 +49,7 @@ class TestOptimizationRequestTracker {
 
     const remaining = Math.max(0, options.deadline - Date.now())
     finalFlush.timeoutId = setTimeout(() => {
-      const error = new Error('Timed out flushing Test Optimization data')
-      error.code = FINAL_FLUSH_TIMEOUT_CODE
+      const error = createFinalFlushTimeoutError()
 
       finalFlush.error ||= error
       finalFlush.writerDone = true
@@ -72,7 +70,7 @@ class TestOptimizationRequestTracker {
     const previousFinalFlush = this.#activeFinalFlush
     this.#activeFinalFlush = finalFlush
     try {
-      BaseWriter.prototype.flush.call(this.#writer, (error) => {
+      this.#flush((error) => {
         finalFlush.error ||= error
         finalFlush.writerDone = true
         this.#finishFinalFlush(finalFlush)
@@ -100,22 +98,37 @@ class TestOptimizationRequestTracker {
     this.#pendingRequests.add(pendingRequest)
     if (this.#activeFinalFlush) this.#attachRequest(this.#activeFinalFlush, pendingRequest)
 
-    request(data, requestOptions, (error, result, statusCode, headers) => {
-      if (error) {
-        for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
-      }
+    try {
+      request(data, requestOptions, (error, result, statusCode, headers) => {
+        if (error) {
+          for (const finalFlush of pendingRequest.finalFlushes) finalFlush.error ||= error
+        }
 
-      try {
-        callback(error, result, statusCode, headers)
-      } finally {
-        this.#pendingRequests.delete(pendingRequest)
-        for (const finalFlush of pendingRequest.finalFlushes) {
-          finalFlush.requests.delete(pendingRequest)
-          this.#finishFinalFlush(finalFlush)
+        try {
+          callback(error, result, statusCode, headers)
+        } finally {
+          this.#dropRequest(pendingRequest)
         }
-        pendingRequest.finalFlushes.clear()
-      }
-    })
+      })
+    } catch (error) {
+      this.#dropRequest(pendingRequest)
+      throw error
+    }
+  }
+
+  /**
+   * Stops tracking a settled request and releases final flushes waiting for it.
+   *
+   * @param {object} pendingRequest
+   * @returns {void}
+   */
+  #dropRequest (pendingRequest) {
+    this.#pendingRequests.delete(pendingRequest)
+    for (const finalFlush of pendingRequest.finalFlushes) {
+      finalFlush.requests.delete(pendingRequest)
+      this.#finishFinalFlush(finalFlush)
+    }
+    pendingRequest.finalFlushes.clear()
   }
 
   /**
@@ -161,4 +174,16 @@ class TestOptimizationRequestTracker {
   }
 }
 
+class TestOptimizationRequestTracker extends FinalFlushRequestTracker {
+  /**
+   * Creates request tracking for a Test Optimization writer.
+   *
+   * @param {BaseWriter} writer
+   */
+  constructor (writer) {
+    super((done, options) => BaseWriter.prototype.flushDirect.call(writer, done, options))
+  }
+}
+
 module.exports = TestOptimizationRequestTracker
+module.exports.FinalFlushRequestTracker = FinalFlushRequestTracker

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -22,6 +22,11 @@ const { getSegment } = require('../../util')
 const BufferingExporter = require('../../exporters/common/buffering-exporter')
 const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
 const { TEST_STATUS } = require('../../plugins/util/test')
+const {
+  createFinalFlushTimeoutError,
+  FINAL_FLUSH_FALLBACK_DELAY,
+  FINAL_FLUSH_TIMEOUT,
+} = require('../final-flush')
 const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
 
 const hostname = getHostname()
@@ -60,17 +65,6 @@ function getIsTestSessionTrace (trace) {
 const GIT_UPLOAD_TIMEOUT = 60_000 // 60 seconds
 const CAN_USE_CI_VIS_PROTOCOL_TIMEOUT = GIT_UPLOAD_TIMEOUT
 const MAX_COVERAGE_REPORT_FLAGS = 32
-const FINAL_FLUSH_TIMEOUT = 10_000
-const FINAL_FLUSH_FALLBACK_DELAY = 100
-
-/**
- * @returns {Error}
- */
-function createFinalFlushTimeoutError () {
-  const error = new Error('Timed out waiting for Test Optimization to flush')
-  error.code = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
-  return error
-}
 
 function appendLogTag (tags, key, value) {
   if (value !== undefined) {

package/packages/dd-trace/src/ci-visibility/final-flush.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const FINAL_FLUSH_TIMEOUT = 10_000
+const FINAL_FLUSH_FALLBACK_DELAY = 100
+const FINAL_FLUSH_TIMEOUT_CODE = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
+
+/**
+ * @returns {Error & { code: string }}
+ */
+function createFinalFlushTimeoutError () {
+  const error = new Error('Timed out waiting for Test Optimization to flush')
+  error.code = FINAL_FLUSH_TIMEOUT_CODE
+  return error
+}
+
+module.exports = {
+  createFinalFlushTimeoutError,
+  FINAL_FLUSH_FALLBACK_DELAY,
+  FINAL_FLUSH_TIMEOUT,
+}

package/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js

@@ -1,53 +1,220 @@
 'use strict'
 
-const Plugin = require('../../plugins/plugin')
+const { Writable } = require('node:stream')
+
+const request = require('../../exporters/common/request')
 const log = require('../../log')
+const Plugin = require('../../plugins/plugin')
+const { FinalFlushRequestTracker } = require('../exporters/agentless/request-tracker')
+const { FINAL_FLUSH_TIMEOUT } = require('../final-flush')
 
-function getWinstonLogSubmissionParameters (config) {
-  const { site, service, DD_API_KEY, DD_AGENTLESS_LOG_SUBMISSION_URL } = config
+const MAX_BATCH_BYTES = 5 * 1024 * 1024
+const MAX_BATCH_LOGS = 1000
+const BATCH_FLUSH_INTERVAL = 1000
 
-  const defaultParameters = {
-    host: `http-intake.logs.${site}`,
-    path: `/api/v2/logs?ddsource=winston&service=${service}`,
-    ssl: true,
-    headers: {
-      'DD-API-KEY': DD_API_KEY,
-    },
-  }
+/**
+ * @param {import('../../config/config-base')} config
+ * @returns {URL | undefined}
+ */
+function getLogSubmissionUrl (config) {
+  if (config.DD_AGENTLESS_LOG_SUBMISSION_URL) {
+    try {
+      const url = new URL(config.DD_AGENTLESS_LOG_SUBMISSION_URL)
+      if (url.protocol === 'http:' || url.protocol === 'https:') return url
 
-  if (!DD_AGENTLESS_LOG_SUBMISSION_URL) {
-    return defaultParameters
+      log.error('Unsupported automatic log submission URL protocol: %s', url.protocol)
+    } catch {
+      log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
+    }
+    return
   }
 
+  const hostname = `http-intake.logs.${config.site}`.toLowerCase()
   try {
-    const url = new URL(DD_AGENTLESS_LOG_SUBMISSION_URL)
-    return {
-      host: url.hostname,
-      port: url.port,
-      ssl: url.protocol === 'https:',
-      path: defaultParameters.path,
-      headers: defaultParameters.headers,
-    }
-  } catch {
-    log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
-    return defaultParameters
-  }
+    const url = new URL(`https://${hostname}`)
+    if (url.hostname === hostname) return url
+  } catch {}
+
+  log.error('Could not parse automatic log submission site: %s', config.site)
+}
+
+/**
+ * @param {import('../../config/config-base')} config
+ * @param {string} source
+ * @returns {string}
+ */
+function getLogSubmissionPath (config, source) {
+  return `/api/v2/logs?${new URLSearchParams({ ddsource: source, service: config.service })}`
 }
 
 class LogSubmissionPlugin extends Plugin {
   static id = 'log-submission'
 
+  /** @type {string[]} */
+  #batch = []
+  #batchBytes = 2
+  #batchSource
+  #logSubmissionUrl
+  #requestTracker
+  #timer
+  #beforeExitHandler = () => this.#flushLogs()
+  #createWinstonJsonFormat
+  #winstonStreamClass
+  // Winston formats records inside its transports, not at logger.write time, so (unlike Bunyan/Pino)
+  // the instrumentation can't publish a post-format line. A Stream transport pipes format.json()
+  // output through this Writable into the shared sender, reusing Winston's own cycle-safe serializer.
+  #winstonOutput = new Writable({
+    decodeStrings: false,
+    write: (message, encoding, callback) => {
+      this.#enqueueLog({ source: 'winston', message })
+      callback()
+    },
+  })
+
   constructor (...args) {
     super(...args)
+    this.#requestTracker = new FinalFlushRequestTracker((done) => {
+      this.#flushLogs()
+      done?.()
+    })
+
+    // The main-module hook (configure) and the logger.js hook (add-transport) can fire in either
+    // order depending on how Winston is required, so buffer loggers that arrive before configure.
+    const pendingWinstonLoggers = new Set()
+    const addWinstonTransport = (logger) => {
+      if (!this.#winstonStreamClass || !this.#createWinstonJsonFormat) {
+        pendingWinstonLoggers.add(logger)
+        return
+      }
+
+      logger.add(new this.#winstonStreamClass({
+        format: this.#createWinstonJsonFormat(),
+        stream: this.#winstonOutput,
+      }))
+    }
+
+    this.addSub('ci:log-submission:winston:configure', ({ StreamTransport, createJsonFormat }) => {
+      this.#winstonStreamClass = StreamTransport
+      this.#createWinstonJsonFormat = createJsonFormat
 
-    this.addSub('ci:log-submission:winston:configure', (httpClass) => {
-      this.HttpClass = httpClass
+      for (const logger of pendingWinstonLoggers) {
+        addWinstonTransport(logger)
+      }
+      pendingWinstonLoggers.clear()
     })
 
-    this.addSub('ci:log-submission:winston:add-transport', (logger) => {
-      logger.add(new this.HttpClass(getWinstonLogSubmissionParameters(this.config)))
+    this.addSub('ci:log-submission:winston:add-transport', addWinstonTransport)
+
+    this.addSub('ci:log-submission:log', (payload) => {
+      this.#enqueueLog(payload)
+    })
+    this.addSub('ci:log-submission:flush', ({ onDone } = {}) => {
+      if (!onDone) {
+        this.#flushLogs()
+        return
+      }
+
+      this.#requestTracker.flush(onDone, {
+        deadline: Date.now() + FINAL_FLUSH_TIMEOUT,
+      })
     })
   }
+
+  /**
+   * @param {boolean | (Record<string, unknown> & { enabled: boolean })} config
+   * @returns {void}
+   */
+  configure (config) {
+    if (this._enabled) this.#flushLogs()
+
+    const isEnabled = typeof config === 'boolean' ? config : config.enabled
+    this.#logSubmissionUrl = isEnabled && typeof config !== 'boolean'
+      ? getLogSubmissionUrl(config)
+      : undefined
+    super.configure(config)
+
+    const beforeExitHandlers = globalThis[Symbol.for('dd-trace')].beforeExitHandlers
+    if (this._enabled) {
+      beforeExitHandlers.add(this.#beforeExitHandler)
+    } else {
+      beforeExitHandlers.delete(this.#beforeExitHandler)
+    }
+  }
+
+  /**
+   * @param {{ source: string, message: string | Record<string, unknown> }} payload
+   * @returns {void}
+   */
+  #enqueueLog ({ source, message }) {
+    if (!this.#logSubmissionUrl) return
+
+    let serializedMessage
+    try {
+      serializedMessage = typeof message === 'string' ? message : JSON.stringify(message)
+    } catch (error) {
+      log.error('Could not serialize %s log for automatic submission', source, error)
+      return
+    }
+    if (serializedMessage === undefined) return
+
+    const messageBytes = Buffer.byteLength(serializedMessage)
+    if (messageBytes + 2 > MAX_BATCH_BYTES) {
+      log.error('Could not submit %s log because it exceeds the %d byte payload limit', source, MAX_BATCH_BYTES)
+      return
+    }
+
+    if (this.#batch.length > 0 &&
+        (this.#batchSource !== source || this.#batchBytes + messageBytes + 1 > MAX_BATCH_BYTES)) {
+      this.#flushLogs()
+      if (!this.#logSubmissionUrl) return
+    }
+
+    this.#batchSource = source
+    if (this.#batch.length > 0) this.#batchBytes++
+    this.#batch.push(serializedMessage)
+    this.#batchBytes += messageBytes
+
+    if (this.#batch.length === MAX_BATCH_LOGS || this.#batchBytes === MAX_BATCH_BYTES) {
+      this.#flushLogs()
+    } else if (this.#timer === undefined) {
+      this.#timer = setTimeout(() => this.#flushLogs(), BATCH_FLUSH_INTERVAL)
+      this.#timer.unref?.()
+    }
+  }
+
+  /**
+   * @returns {void}
+   */
+  #flushLogs () {
+    clearTimeout(this.#timer)
+    this.#timer = undefined
+
+    if (this.#batch.length === 0 || !this.#logSubmissionUrl) return
+
+    const source = this.#batchSource
+    const data = `[${this.#batch.join(',')}]`
+    this.#batch = []
+    this.#batchBytes = 2
+    this.#batchSource = undefined
+    const options = {
+      path: getLogSubmissionPath(this.config, source),
+      method: 'POST',
+      headers: {
+        'DD-API-KEY': this.config.DD_API_KEY,
+        'Content-Type': 'application/json',
+      },
+      url: this.#logSubmissionUrl,
+    }
+
+    try {
+      this.#requestTracker.send(request, data, options, error => {
+        if (error) log.error('Error submitting %s logs', source, error)
+      })
+    } catch (error) {
+      this.#logSubmissionUrl = undefined
+      log.error('Error submitting %s logs', source, error)
+    }
+  }
 }
 
 module.exports = LogSubmissionPlugin

package/packages/dd-trace/src/config/generated-config-types.d.ts

@@ -272,6 +272,7 @@ export interface GeneratedConfig {
   DD_TRACE_HAPI_ENABLED: boolean;
   DD_TRACE_HAPI_HAPI_ENABLED: boolean;
   DD_TRACE_HONO_ENABLED: boolean;
+  DD_TRACE_HTTP_CLIENT_ERROR_STATUSES: string;
   DD_TRACE_HTTP_ENABLED: boolean;
   DD_TRACE_HTTP_SERVER_ERROR_STATUSES: string;
   DD_TRACE_HTTP2_ENABLED: boolean;
@@ -418,6 +419,7 @@ export interface GeneratedConfig {
       endpoint: string | undefined;
       maxContentSize: number;
       maxMessagesLength: number;
+      redactionEnabled: boolean;
       timeout: number;
     };
     appsec: {
@@ -473,6 +475,7 @@ export interface GeneratedConfig {
     agentlessEnabled: boolean | undefined;
     DD_LLMOBS_ENABLED: boolean;
     mlApp: string | undefined;
+    projectName: string | undefined;
     sampleRate: number;
   };
   logInjection: boolean;
@@ -614,6 +617,7 @@ export interface GeneratedEnvVarConfig {
   DD_AI_GUARD_ENDPOINT: string | undefined;
   DD_AI_GUARD_MAX_CONTENT_SIZE: number;
   DD_AI_GUARD_MAX_MESSAGES_LENGTH: number;
+  DD_AI_GUARD_REDACTION_ENABLED: boolean;
   DD_AI_GUARD_TIMEOUT: number;
   DD_API_KEY: string | undefined;
   DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE: number;
@@ -758,6 +762,7 @@ export interface GeneratedEnvVarConfig {
   DD_LLMOBS_AGENTLESS_ENABLED: boolean | undefined;
   DD_LLMOBS_ENABLED: boolean;
   DD_LLMOBS_ML_APP: string | undefined;
+  DD_LLMOBS_PROJECT_NAME: string | undefined;
   DD_LLMOBS_SAMPLE_RATE: number;
   DD_LOG_LEVEL: "debug" | "info" | "warn" | "error";
   DD_LOGS_INJECTION: boolean;
@@ -970,6 +975,7 @@ export interface GeneratedEnvVarConfig {
   DD_TRACE_HAPI_HAPI_ENABLED: boolean;
   DD_TRACE_HEADER_TAGS: string[];
   DD_TRACE_HONO_ENABLED: boolean;
+  DD_TRACE_HTTP_CLIENT_ERROR_STATUSES: string;
   DD_TRACE_HTTP_ENABLED: boolean;
   DD_TRACE_HTTP_SERVER_ERROR_STATUSES: string;
   DD_TRACE_HTTP2_ENABLED: boolean;

package/packages/dd-trace/src/config/index.js

@@ -510,7 +510,8 @@ class Config extends ConfigBase {
     if (!this.llmobs.DD_LLMOBS_ENABLED &&
         !trackedConfigOrigins.has('llmobs.DD_LLMOBS_ENABLED') &&
         (trackedConfigOrigins.has('llmobs.agentlessEnabled') ||
-        trackedConfigOrigins.has('llmobs.mlApp'))) {
+        trackedConfigOrigins.has('llmobs.mlApp') ||
+        trackedConfigOrigins.has('llmobs.projectName'))) {
       setAndTrack(this, 'llmobs.DD_LLMOBS_ENABLED', true)
     }
 

package/packages/dd-trace/src/config/supported-configurations.json

@@ -93,6 +93,16 @@
         "default": "16"
       }
     ],
+    "DD_AI_GUARD_REDACTION_ENABLED": [
+      {
+        "implementation": "A",
+        "type": "boolean",
+        "configurationNames": [
+          "experimental.aiguard.redactionEnabled"
+        ],
+        "default": "true"
+      }
+    ],
     "DD_AI_GUARD_TIMEOUT": [
       {
         "implementation": "A",
@@ -1341,6 +1351,16 @@
         "default": null
       }
     ],
+    "DD_LLMOBS_PROJECT_NAME": [
+      {
+        "implementation": "A",
+        "type": "string",
+        "configurationNames": [
+          "llmobs.projectName"
+        ],
+        "default": null
+      }
+    ],
     "DD_LLMOBS_SAMPLE_RATE": [
       {
         "implementation": "A",
@@ -2999,6 +3019,13 @@
         "default": "true"
       }
     ],
+    "DD_TRACE_HTTP_CLIENT_ERROR_STATUSES": [
+      {
+        "implementation": "A",
+        "type": "string",
+        "default": "400-499"
+      }
+    ],
     "DD_TRACE_HTTP_ENABLED": [
       {
         "implementation": "A",

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -141,7 +141,7 @@ function setInputPath (newPath) {
  * @returns {string} The serialized tags.
  */
 function buildTags (tags) {
-  const serializedTags = []
+  let serializedTags = ''
 
   for (const [key, rawValue] of tags) {
     if (rawValue === undefined) continue
@@ -151,8 +151,9 @@ function buildTags (tags) {
       continue
     }
 
-    serializedTags.push(`${key}:${rawValue}`)
+    if (serializedTags) serializedTags += ','
+    serializedTags += `${key}:${rawValue}`
   }
 
-  return serializedTags.join(',')
+  return serializedTags
 }