@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/exporters/agentless/writer.js

@@ -0,0 +1,201 @@
+'use strict'
+
+const { getValueFromEnvSources } = require('../../config/helper')
+const log = require('../../log')
+const request = require('../common/request')
+const tracerVersion = require('../../../../../package.json').version
+
+const BaseWriter = require('../common/writer')
+const { AgentlessJSONEncoder } = require('../../encode/agentless-json')
+
+/**
+ * Writer for agentless APM trace intake.
+ * Sends traces directly to the Datadog intake endpoint without an agent.
+ */
+class AgentlessWriter extends BaseWriter {
+  #apiKeyMissing = false
+  #urlMissing = false
+
+  /**
+   * @param {object} options - Writer options
+   * @param {URL} [options.url] - The intake URL. If not provided, constructed from site.
+   * @param {string} [options.site='datadoghq.com'] - The Datadog site
+   * @param {object} [options.metadata={}] - Metadata to pass to the encoder (hostname, env, etc.)
+   */
+  constructor ({ url, site = 'datadoghq.com', metadata = {} }) {
+    super({ url })
+    this._encoder = new AgentlessJSONEncoder(this, metadata)
+
+    if (!url) {
+      try {
+        this._url = new URL(`https://public-trace-http-intake.logs.${site}`)
+      } catch (err) {
+        log.error(
+          'Invalid site value for agentless intake: %s. Cannot construct URL. Error: %s',
+          site,
+          err.message
+        )
+        this._url = null
+      }
+    }
+
+    if (!getValueFromEnvSources('DD_API_KEY')) {
+      this.#apiKeyMissing = true
+      log.error('DD_API_KEY is required for agentless trace intake. Set DD_API_KEY. Traces will not be sent.')
+    }
+  }
+
+  setUrl (url) {
+    super.setUrl(url)
+    if (url) {
+      this.#urlMissing = false
+    }
+  }
+
+  /**
+   * Flushes accumulated traces to the intake as a single request.
+   * @param {Function} [done] - Callback when send completes
+   */
+  flush (done = () => {}) {
+    if (!request.writable) {
+      const count = this._encoder.count()
+      if (count > 0) {
+        log.error('Maximum number of active requests reached. Dropping %d trace(s).', count)
+      }
+      this._encoder.reset()
+      done()
+      return
+    }
+
+    const count = this._encoder.count()
+
+    if (count === 0) {
+      done()
+      return
+    }
+
+    const payload = this._encoder.makePayload()
+
+    if (payload.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    this._sendPayload(payload, count, done)
+  }
+
+  /**
+   * Sends the encoded payload to the intake endpoint.
+   * @param {Buffer} data - The encoded JSON payload
+   * @param {number} count - Number of traces in the payload
+   * @param {Function} done - Callback when complete
+   */
+  _sendPayload (data, count, done) {
+    if (!data || data.length === 0) {
+      log.debug('Skipping send of empty payload')
+      done()
+      return
+    }
+
+    if (!this._url) {
+      if (!this.#urlMissing) {
+        this.#urlMissing = true
+        log.error('No valid URL configured for agentless trace intake. Traces will not be sent.')
+      }
+      log.debug('Dropping %d trace(s) due to missing URL', count)
+      done()
+      return
+    }
+
+    const apiKey = getValueFromEnvSources('DD_API_KEY')
+    if (!apiKey) {
+      if (!this.#apiKeyMissing) {
+        this.#apiKeyMissing = true
+        log.error('DD_API_KEY is required for agentless trace intake. Set DD_API_KEY. Traces will not be sent.')
+      }
+      log.debug('Dropping %d trace(s) due to missing DD_API_KEY', count)
+      done()
+      return
+    }
+    this.#apiKeyMissing = false
+
+    const options = {
+      path: '/v1/input',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'dd-api-key': apiKey,
+        'X-Datadog-Trace-Count': String(count),
+        'Datadog-Meta-Lang': 'nodejs',
+        'Datadog-Meta-Lang-Version': process.version,
+        'Datadog-Meta-Lang-Interpreter': process.versions.bun ? 'JavaScriptCore' : 'v8',
+        'Datadog-Meta-Tracer-Version': tracerVersion,
+      },
+      timeout: 15_000,
+      url: this._url,
+    }
+
+    log.debug('Request to the agentless intake: %j', options)
+
+    request(data, options, (err, res, statusCode) => {
+      if (err) {
+        this._logRequestError(err, statusCode, count)
+        done()
+        return
+      }
+
+      log.debug('Response from the agentless intake: %s', res)
+      done()
+    })
+  }
+
+  /**
+   * Logs request errors with status-specific guidance.
+   * @param {Error} err - The error object
+   * @param {number} statusCode - HTTP status code (if available)
+   * @param {number} count - Number of traces that were being sent
+   */
+  _logRequestError (err, statusCode, count) {
+    if (statusCode === 401 || statusCode === 403) {
+      log.error(
+        'Authentication failed sending %d trace(s) (status %s). Verify DD_API_KEY is valid.',
+        count,
+        statusCode
+      )
+    } else if (statusCode === 404) {
+      log.error(
+        'Trace intake endpoint not found (status %s). Verify DD_SITE is correctly configured. %d trace(s) dropped.',
+        statusCode,
+        count
+      )
+    } else if (statusCode === 429) {
+      log.error(
+        'Rate limited by trace intake (status 429). %d trace(s) dropped.',
+        count
+      )
+    } else if (statusCode >= 500) {
+      log.error(
+        'Trace intake server error (status %s). %d trace(s) dropped. This may be transient.',
+        statusCode,
+        count
+      )
+    } else if (statusCode) {
+      log.error(
+        'Error sending agentless payload (status %s): %s. %d trace(s) dropped.',
+        statusCode,
+        err.message,
+        count
+      )
+    } else {
+      log.error(
+        'Network error sending %d trace(s) to %s: %s',
+        count,
+        this._url?.hostname || 'unknown',
+        err.message
+      )
+    }
+  }
+}
+
+module.exports = AgentlessWriter

package/packages/dd-trace/src/exporters/common/agents.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const { storage } = require('../../../../datadog-core')
+
+const keepAlive = true
+const maxSockets = 1
+
+function createAgentClass (BaseAgent) {
+  class CustomAgent extends BaseAgent {
+    constructor () {
+      super({ keepAlive, maxSockets })
+    }
+
+    createConnection (...args) {
+      return this._noop(() => super.createConnection(...args))
+    }
+
+    keepSocketAlive (...args) {
+      return this._noop(() => super.keepSocketAlive(...args))
+    }
+
+    reuseSocket (...args) {
+      return this._noop(() => super.reuseSocket(...args))
+    }
+
+    _noop (callback) {
+      return storage('legacy').run({ noop: true }, callback)
+    }
+  }
+
+  return CustomAgent
+}
+
+const HttpAgent = createAgentClass(http.Agent)
+const HttpsAgent = createAgentClass(https.Agent)
+
+module.exports = {
+  httpAgent: new HttpAgent(),
+  httpsAgent: new HttpsAgent(),
+}

package/packages/dd-trace/src/exporters/common/buffering-exporter.js

@@ -0,0 +1,62 @@
+'use strict'
+
+const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../../ci-visibility/telemetry')
+const { getAgentUrl } = require('../../agent/url')
+
+/**
+ * Base exporter that buffers traces until a writer is initialized.
+ * Provides common export logic with flush intervals.
+ */
+class BufferingExporter {
+  _traceBuffer = []
+  _isInitialized = false
+  _writer
+
+  constructor (tracerConfig) {
+    this._config = tracerConfig
+    this._url = getAgentUrl(tracerConfig)
+  }
+
+  export (trace) {
+    if (!this._isInitialized) {
+      this._traceBuffer.push(trace)
+      return
+    }
+    this._export(trace)
+  }
+
+  _export (payload, writer = this._writer, timerKey = '_timer') {
+    if (this._config.isCiVisibility) {
+      incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, {}, payload.length)
+    }
+    writer.append(payload)
+
+    const { flushInterval } = this._config
+
+    if (flushInterval === 0) {
+      writer.flush()
+    } else if (this[timerKey] === undefined) {
+      this[timerKey] = setTimeout(() => {
+        writer.flush()
+        this[timerKey] = undefined
+      }, flushInterval).unref()
+    }
+  }
+
+  getUncodedTraces () {
+    return this._traceBuffer
+  }
+
+  exportUncodedTraces () {
+    for (const uncodedTrace of this.getUncodedTraces()) {
+      this.export(uncodedTrace)
+    }
+    this.resetUncodedTraces()
+  }
+
+  resetUncodedTraces () {
+    this._traceBuffer = []
+  }
+}
+
+module.exports = BufferingExporter

package/packages/dd-trace/src/exporters/common/docker.js

@@ -0,0 +1,52 @@
+'use strict'
+
+const fs = require('fs')
+const { getValueFromEnvSources } = require('../../config/helper')
+
+const DD_EXTERNAL_ENV = getValueFromEnvSources('DD_EXTERNAL_ENV')
+
+// The second part is the PCF / Garden regexp. We currently assume no suffix($) to avoid matching pod UIDs
+// See https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/util/cgroups/reader.go#L50
+const uuidSource =
+'[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}|[0-9a-f]{8}(?:-[0-9a-f]{4}){4}$'
+const containerSource = '[0-9a-f]{64}'
+const taskSource = String.raw`[0-9a-f]{32}-\d+`
+const lineReg = /^(\d+):([^:]*):(.+)$/m
+const entityReg = new RegExp(String.raw`.*(${uuidSource}|${containerSource}|${taskSource})(?:\.scope)?$`, 'm')
+
+let inode = 0
+let cgroup = ''
+let containerId
+
+try {
+  cgroup = fs.readFileSync('/proc/self/cgroup', 'utf8').trim()
+  containerId = cgroup.match(entityReg)?.[1]
+} catch { /* Ignore error */ }
+
+const inodePath = cgroup.match(lineReg)?.[3]
+if (inodePath) {
+  const strippedPath = inodePath.replaceAll(/^\/|\/$/g, '')
+
+  try {
+    inode = fs.statSync(`/sys/fs/cgroup/${strippedPath}`).ino
+  } catch { /* Ignore error */ }
+}
+
+const entityId = containerId ? `ci-${containerId}` : inode && `in-${inode}`
+
+module.exports = {
+  entityId,
+
+  inject (carrier) {
+    if (containerId) {
+      carrier['Datadog-Container-Id'] = containerId
+      carrier['Datadog-Entity-ID'] = entityId
+    } else if (inode) {
+      carrier['Datadog-Entity-ID'] = entityId
+    }
+
+    if (DD_EXTERNAL_ENV) {
+      carrier['Datadog-External-Env'] = DD_EXTERNAL_ENV
+    }
+  },
+}

package/packages/dd-trace/src/exporters/common/form-data.js

@@ -0,0 +1,59 @@
+'use strict'
+
+const { Readable } = require('stream')
+const id = require('../../id')
+
+class FormData extends Readable {
+  constructor () {
+    super()
+
+    this._boundary = id().toString()
+    this._data = []
+  }
+
+  append (key, value, options = {}) {
+    this._appendBoundary()
+
+    if (options.filename) {
+      this._appendFile(key, value, options)
+    } else {
+      this._appendMetadata(key, value, options)
+    }
+  }
+
+  size () {
+    return this._data.reduce((size, chunk) => size + chunk.length, 0)
+  }
+
+  getHeaders () {
+    return { 'Content-Type': 'multipart/form-data; boundary=' + this._boundary }
+  }
+
+  _appendBoundary () {
+    this._data.push(`--${this._boundary}\r\n`)
+  }
+
+  _appendMetadata (key, value) {
+    this._data.push(`Content-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`)
+  }
+
+  _appendFile (key, value, { filename, contentType = 'application/octet-stream' }) {
+    this._data.push(
+      `Content-Disposition: form-data; name="${key}"; filename="${filename}"\r\n`,
+      `Content-Type: ${contentType}\r\n\r\n`,
+      value,
+      '\r\n'
+    )
+  }
+
+  _read () {
+    this.push(this._data.shift())
+
+    if (this._data.length === 0) {
+      this.push(`--${this._boundary}--\r\n`)
+      this.push(null)
+    }
+  }
+}
+
+module.exports = FormData

package/packages/dd-trace/src/exporters/common/request.js

@@ -0,0 +1,190 @@
+'use strict'
+
+// TODO: Add test with slow or unresponsive agent.
+// TODO: Add telemetry for things like dropped requests, errors, etc.
+
+const { Readable } = require('stream')
+const http = require('http')
+const https = require('https')
+const zlib = require('zlib')
+
+const { storage } = require('../../../../datadog-core')
+const log = require('../../log')
+const { urlToHttpOptions } = require('./url-to-http-options-polyfill')
+const docker = require('./docker')
+const { httpAgent, httpsAgent } = require('./agents')
+
+const maxActiveRequests = 8
+
+let activeRequests = 0
+
+function parseUrl (urlObjOrString) {
+  if (urlObjOrString !== null && typeof urlObjOrString === 'object') return urlToHttpOptions(urlObjOrString)
+
+  const url = urlToHttpOptions(new URL(urlObjOrString))
+
+  // Special handling if we're using named pipes on Windows
+  if (url.protocol === 'unix:' && url.hostname === '.') {
+    const udsPath = urlObjOrString.slice(5)
+    url.path = udsPath
+    url.pathname = udsPath
+  }
+
+  return url
+}
+
+function request (data, options, callback) {
+  if (!options.headers) {
+    options.headers = {}
+  }
+
+  if (options.url) {
+    const url = parseUrl(options.url)
+    if (url.protocol === 'unix:') {
+      options.socketPath = url.pathname
+    } else {
+      if (!options.path) options.path = url.path
+      options.protocol = url.protocol
+      options.hostname = url.hostname // for IPv6 this should be '::1' and not '[::1]'
+      options.port = url.port
+    }
+  }
+
+  const isReadable = data instanceof Readable
+
+  // The timeout should be kept low to avoid excessive queueing.
+  const timeout = options.timeout || 2000
+  const isSecure = options.protocol === 'https:'
+  const client = isSecure ? https : http
+  let dataArray = data
+
+  if (!isReadable) {
+    if (!Array.isArray(data)) {
+      dataArray = [data]
+    }
+    options.headers['Content-Length'] = byteLength(dataArray)
+  }
+
+  docker.inject(options.headers)
+
+  options.agent = isSecure ? httpsAgent : httpAgent
+
+  const onResponse = (res, finalize) => {
+    const chunks = []
+
+    res.setTimeout(timeout)
+
+    res.on('data', chunk => {
+      chunks.push(chunk)
+    })
+
+    res.once('end', () => {
+      finalize()
+      const buffer = Buffer.concat(chunks)
+
+      if (res.statusCode >= 200 && res.statusCode <= 299) {
+        const isGzip = res.headers['content-encoding'] === 'gzip'
+        if (isGzip) {
+          zlib.gunzip(buffer, (err, result) => {
+            if (err) {
+              log.error('Could not gunzip response: %s', err.message)
+              callback(null, '', res.statusCode, res.headers)
+            } else {
+              callback(null, result.toString(), res.statusCode, res.headers)
+            }
+          })
+        } else {
+          callback(null, buffer.toString(), res.statusCode, res.headers)
+        }
+      } else {
+        let errorMessage = ''
+        try {
+          const fullUrl = new URL(
+            options.path,
+            options.url || options.hostname || `http://localhost:${options.port}`
+          ).href
+          errorMessage = `Error from ${fullUrl}: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}.`
+        } catch {
+          // ignore error
+        }
+
+        const responseData = buffer.toString()
+        if (responseData) {
+          errorMessage += ` Response from the endpoint: "${responseData}"`
+        }
+        const error = new log.NoTransmitError(errorMessage)
+        error.status = res.statusCode
+
+        callback(error, null, res.statusCode, res.headers)
+      }
+    })
+  }
+
+  const makeRequest = onError => {
+    if (!request.writable) {
+      log.debug('Maximum number of active requests reached: payload is discarded.')
+      return callback(null)
+    }
+
+    activeRequests++
+
+    storage('legacy').run({ noop: true }, () => {
+      let finished = false
+      const finalize = () => {
+        if (finished) return
+        finished = true
+        activeRequests--
+      }
+
+      const req = client.request(options, (res) => onResponse(res, finalize))
+
+      req.once('close', finalize)
+      req.once('timeout', finalize)
+
+      req.once('error', err => {
+        finalize()
+        onError(err)
+      })
+
+      req.setTimeout(timeout, () => {
+        try {
+          if (typeof req.abort === 'function') {
+            req.abort()
+          } else {
+            req.destroy()
+          }
+        } catch {
+          // ignore
+        }
+      })
+
+      if (isReadable) {
+        data.pipe(req) // TODO: Validate whether this is actually retriable.
+      } else {
+        for (const buffer of dataArray) req.write(buffer)
+        req.end()
+      }
+    })
+  }
+
+  // The setTimeout is needed to avoid losing the async context in the retry
+  // request before socket.connect() is called. This is a workaround for the
+  // issue that the AsyncLocalStorage.run() method does not call the
+  // AsyncLocalStorage.enterWith() method when not using AsyncContextFrame.
+  //
+  // TODO: Test that this doesn't trace itself on retry when the diagnostics
+  // channel events are available in the agent exporter.
+  makeRequest(() => setTimeout(() => makeRequest(callback)))
+}
+
+function byteLength (data) {
+  return data.length > 0 ? data.reduce((prev, next) => prev + Buffer.byteLength(next, 'utf8'), 0) : 0
+}
+
+Object.defineProperty(request, 'writable', {
+  get () {
+    return activeRequests < maxActiveRequests
+  },
+})
+
+module.exports = request

package/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const { urlToHttpOptions } = require('url')
+
+// TODO: Remove `urlToHttpOptions` polyfill once we drop support for the older Cypress versions that uses a built-in
+// version of Node.js doesn't include that function.
+module.exports = {
+  urlToHttpOptions: urlToHttpOptions ?? function (url) {
+    const { hostname, pathname, port, username, password, search } = url
+    const options = {
+      __proto__: null,
+      ...url, // In case the url object was extended by the user.
+      protocol: url.protocol,
+      hostname: typeof hostname === 'string' && hostname.startsWith('[')
+        ? hostname.slice(1, -1)
+        : hostname,
+      hash: url.hash,
+      search,
+      pathname,
+      path: `${pathname || ''}${search || ''}`,
+      href: url.href,
+    }
+    if (port !== '') {
+      options.port = Number(port)
+    }
+    if (username || password) {
+      options.auth = `${decodeURIComponent(username)}:${decodeURIComponent(password)}`
+    }
+    return options
+  },
+}

package/packages/dd-trace/src/exporters/common/util.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const { getValueFromEnvSources } = require('../../config/helper')
+
+function safeJSONStringify (value) {
+  return JSON.stringify(
+    value,
+    (key, value) => key === 'dd-api-key' ? undefined : value,
+    getValueFromEnvSources('DD_TRACE_BEAUTIFUL_LOGS') ? 2 : undefined
+  )
+}
+
+module.exports = { safeJSONStringify }

package/packages/dd-trace/src/exporters/common/writer.js

@@ -0,0 +1,49 @@
+'use strict'
+
+const log = require('../../log')
+const request = require('./request')
+const { safeJSONStringify } = require('./util')
+
+class Writer {
+  constructor ({ url }) {
+    this._url = url
+  }
+
+  flush (done = () => {}) {
+    const count = this._encoder.count()
+
+    if (!request.writable) {
+      this._encoder.reset()
+      done()
+    } else if (count > 0) {
+      const payload = this._encoder.makePayload()
+
+      this._sendPayload(payload, count, done)
+    } else {
+      done()
+    }
+  }
+
+  append (payload) {
+    if (!request.writable) {
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.debug(() => `Maximum number of active requests reached. Payload discarded: ${safeJSONStringify(payload)}`)
+      return
+    }
+
+    // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+    log.debug(() => `Encoding payload: ${safeJSONStringify(payload)}`)
+
+    this._encode(payload)
+  }
+
+  _encode (payload) {
+    this._encoder.encode(payload)
+  }
+
+  setUrl (url) {
+    this._url = url
+  }
+}
+
+module.exports = Writer