@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -0,0 +1,361 @@
+'use strict'
+const { version: ddTraceVersion } = require('../../../../package.json')
+const { ITR_CORRELATION_ID } = require('../../src/plugins/util/test')
+const id = require('../../src/id')
+const {
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT,
+} = require('../ci-visibility/telemetry')
+const { AgentEncoder } = require('./0.4')
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+
+const ENCODING_VERSION = 1
+const ALLOWED_CONTENT_TYPES = new Set(['test_session_end', 'test_module_end', 'test_suite_end', 'test'])
+
+const TEST_SUITE_KEYS_LENGTH = 12
+const TEST_MODULE_KEYS_LENGTH = 11
+const TEST_SESSION_KEYS_LENGTH = 10
+const TEST_AND_SPAN_KEYS_LENGTH = 11
+
+const INTAKE_SOFT_LIMIT = 2 * 1024 * 1024 // 2MB
+
+function formatSpan (span) {
+  let encodingVersion = ENCODING_VERSION
+  if (span.type === 'test' && span.meta && span.meta.test_session_id) {
+    encodingVersion = 2
+  }
+  return {
+    type: ALLOWED_CONTENT_TYPES.has(span.type) ? span.type : 'span',
+    version: encodingVersion,
+    content: normalizeSpan(truncateSpan(span)),
+  }
+}
+
+class AgentlessCiVisibilityEncoder extends AgentEncoder {
+  constructor (writer, { runtimeId, service, env }) {
+    super(writer, INTAKE_SOFT_LIMIT)
+    this.runtimeId = runtimeId
+    this.service = service
+    this.env = env
+
+    // Used to keep track of the number of encoded events to update the
+    // length of `payload.events` when calling `makePayload`
+    this._eventCount = 0
+
+    this.metadataTags = {}
+
+    this.reset()
+  }
+
+  addMetadataTags (tags) {
+    for (const type of ALLOWED_CONTENT_TYPES) {
+      if (tags[type]) {
+        this.metadataTags[type] = {
+          ...this.metadataTags[type],
+          ...tags[type],
+        }
+      }
+    }
+  }
+
+  _encodeTestSuite (bytes, content) {
+    let keysLength = TEST_SUITE_KEYS_LENGTH
+    const itrCorrelationId = content.meta[ITR_CORRELATION_ID]
+    if (itrCorrelationId) {
+      keysLength++
+    }
+
+    this._encodeMapPrefix(bytes, keysLength)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_module_id')
+    this._encodeId(bytes, content.parent_id)
+
+    this._encodeString(bytes, 'test_suite_id')
+    this._encodeId(bytes, content.span_id)
+
+    if (itrCorrelationId) {
+      this._encodeString(bytes, ITR_CORRELATION_ID)
+      this._encodeString(bytes, itrCorrelationId)
+      delete content.meta[ITR_CORRELATION_ID]
+    }
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeTestModule (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_MODULE_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'test_module_id')
+    this._encodeId(bytes, content.span_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeTestSession (bytes, content) {
+    this._encodeMapPrefix(bytes, TEST_SESSION_KEYS_LENGTH)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, content.type)
+
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, content.trace_id)
+
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeEventContent (bytes, content) {
+    let totalKeysLength = TEST_AND_SPAN_KEYS_LENGTH
+    if (content.meta.test_session_id) {
+      totalKeysLength += 1
+    }
+    if (content.meta.test_module_id) {
+      totalKeysLength += 1
+    }
+    if (content.meta.test_suite_id) {
+      totalKeysLength += 1
+    }
+    const itrCorrelationId = content.meta[ITR_CORRELATION_ID]
+    if (itrCorrelationId) {
+      totalKeysLength += 1
+    }
+    if (content.type) {
+      totalKeysLength += 1
+    }
+    this._encodeMapPrefix(bytes, totalKeysLength)
+    if (content.type) {
+      this._encodeString(bytes, 'type')
+      this._encodeString(bytes, content.type)
+    }
+    this._encodeString(bytes, 'trace_id')
+    this._encodeId(bytes, content.trace_id)
+    this._encodeString(bytes, 'span_id')
+    this._encodeId(bytes, content.span_id)
+    this._encodeString(bytes, 'parent_id')
+    this._encodeId(bytes, content.parent_id)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    /**
+     * We include `test_session_id` and `test_suite_id`
+     * in the root of the event by passing them via the `meta` dict.
+     * This is to avoid changing the span format in packages/dd-trace/src/format.js,
+     * which can have undesired side effects in other products.
+     * But `test_session_id` and `test_suite_id` are *not* supposed to be in `meta`,
+     * so we delete them before enconding the dictionary.
+     * TODO: find a better way to do this.
+     */
+    if (content.meta.test_session_id) {
+      this._encodeString(bytes, 'test_session_id')
+      this._encodeId(bytes, id(content.meta.test_session_id, 10))
+      delete content.meta.test_session_id
+    }
+
+    if (content.meta.test_module_id) {
+      this._encodeString(bytes, 'test_module_id')
+      this._encodeId(bytes, id(content.meta.test_module_id, 10))
+      delete content.meta.test_module_id
+    }
+
+    if (content.meta.test_suite_id) {
+      this._encodeString(bytes, 'test_suite_id')
+      this._encodeId(bytes, id(content.meta.test_suite_id, 10))
+      delete content.meta.test_suite_id
+    }
+
+    if (itrCorrelationId) {
+      this._encodeString(bytes, ITR_CORRELATION_ID)
+      this._encodeString(bytes, itrCorrelationId)
+      delete content.meta[ITR_CORRELATION_ID]
+    }
+
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeEvent (bytes, event) {
+    this._encodeMapPrefix(bytes, Object.keys(event).length)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, event.type)
+
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, event.version)
+
+    this._encodeString(bytes, 'content')
+    if (event.type === 'span' || event.type === 'test') {
+      this._encodeEventContent(bytes, event.content)
+    } else if (event.type === 'test_suite_end') {
+      this._encodeTestSuite(bytes, event.content)
+    } else if (event.type === 'test_module_end') {
+      this._encodeTestModule(bytes, event.content)
+    } else if (event.type === 'test_session_end') {
+      this._encodeTestSession(bytes, event.content)
+    }
+  }
+
+  _encode (bytes, trace) {
+    if (this._isReset) {
+      this._encodePayloadStart(bytes)
+      this._isReset = false
+    }
+    const startTime = Date.now()
+
+    const events = trace.map(formatSpan)
+
+    this._eventCount += events.length
+
+    for (const event of events) {
+      this._encodeEvent(bytes, event)
+    }
+    distributionMetric(
+      TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+      { endpoint: 'test_cycle' },
+      Date.now() - startTime
+    )
+  }
+
+  makePayload () {
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'test_cycle' }, this._eventCount)
+    const bytes = this._traceBytes
+    const eventsOffset = this._eventsOffset
+    const eventsCount = this._eventCount
+
+    bytes.buffer[eventsOffset] = 0xDD
+    bytes.buffer[eventsOffset + 1] = eventsCount >> 24
+    bytes.buffer[eventsOffset + 2] = eventsCount >> 16
+    bytes.buffer[eventsOffset + 3] = eventsCount >> 8
+    bytes.buffer[eventsOffset + 4] = eventsCount
+
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, traceSize)
+
+    this.reset()
+
+    return buffer
+  }
+
+  _encodePayloadStart (bytes) {
+    // encodes the payload up to `events`. `events` will be encoded via _encode
+    const payload = {
+      version: ENCODING_VERSION,
+      metadata: {
+        '*': {
+          language: 'javascript',
+          library_version: ddTraceVersion,
+        },
+        ...this.metadataTags,
+      },
+      events: [],
+    }
+
+    if (this.env) {
+      payload.metadata['*'].env = this.env
+    }
+    if (this.runtimeId) {
+      payload.metadata['*']['runtime-id'] = this.runtimeId
+    }
+
+    this._encodeMapPrefix(bytes, Object.keys(payload).length)
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, payload.version)
+    this._encodeString(bytes, 'metadata')
+    this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length)
+    this._encodeString(bytes, '*')
+    this._encodeMap(bytes, payload.metadata['*'])
+    if (payload.metadata.test) {
+      this._encodeString(bytes, 'test')
+      this._encodeMap(bytes, payload.metadata.test)
+    }
+    if (payload.metadata.test_suite_end) {
+      this._encodeString(bytes, 'test_suite_end')
+      this._encodeMap(bytes, payload.metadata.test_suite_end)
+    }
+    if (payload.metadata.test_module_end) {
+      this._encodeString(bytes, 'test_module_end')
+      this._encodeMap(bytes, payload.metadata.test_module_end)
+    }
+    if (payload.metadata.test_session_end) {
+      this._encodeString(bytes, 'test_session_end')
+      this._encodeMap(bytes, payload.metadata.test_session_end)
+    }
+    this._encodeString(bytes, 'events')
+    // Get offset of the events list to update the length of the array when calling `makePayload`
+    this._eventsOffset = bytes.length
+    bytes.reserve(5)
+  }
+
+  reset () {
+    this._reset()
+    this._eventCount = 0
+    this._isReset = true
+  }
+}
+
+module.exports = { AgentlessCiVisibilityEncoder }

package/packages/dd-trace/src/encode/agentless-json.js

@@ -0,0 +1,200 @@
+'use strict'
+
+const log = require('../log')
+const { TOP_LEVEL_KEY } = require('../constants')
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+
+// Soft limit for estimated payload size. Triggers an early flush to stay under intake request size limits.
+const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB
+
+/**
+ * Formats a span for JSON encoding.
+ * @param {object} span - The span to format
+ * @param {boolean} isFirstSpan - Whether this is the first span in the trace
+ * @returns {object} The formatted span
+ */
+function formatSpan (span, isFirstSpan) {
+  span = normalizeSpan(truncateSpan(span, false))
+
+  if (span.span_events) {
+    span.meta.events = JSON.stringify(span.span_events)
+    delete span.span_events
+  }
+
+  if (isFirstSpan) {
+    span.meta['_dd.compute_stats'] = '1'
+  }
+
+  if (span.parent_id?.toString(10) === '0') {
+    span.metrics._trace_root = 1
+  }
+
+  if (span.metrics[TOP_LEVEL_KEY]) {
+    span.metrics._top_level = 1
+  }
+
+  return span
+}
+
+/**
+ * Converts a span to JSON-serializable format.
+ * IDs are converted to lowercase hex strings. Start time is converted from
+ * nanoseconds to seconds for the intake format.
+ * @param {object} span - The formatted span
+ * @returns {object} JSON-serializable span object
+ */
+function spanToJSON (span) {
+  const result = {
+    trace_id: span.trace_id.toString(16).toLowerCase(),
+    span_id: span.span_id.toString(16).toLowerCase(),
+    parent_id: span.parent_id.toString(16).toLowerCase(),
+    name: span.name,
+    resource: span.resource,
+    service: span.service,
+    error: span.error,
+    start: Math.floor(span.start / 1e9),
+    duration: span.duration,
+    meta: span.meta,
+    metrics: span.metrics,
+  }
+
+  if (span.type) {
+    result.type = span.type
+  }
+
+  if (span.meta_struct) {
+    result.meta_struct = span.meta_struct
+  }
+
+  if (span.links && span.links.length > 0) {
+    result.links = span.links
+  }
+
+  return result
+}
+
+/**
+ * JSON encoder for agentless trace intake.
+ * Encodes multiple traces as JSON with the payload format: {"traces": [{spans: [...], ...metadata}, ...]}
+ *
+ * Traces are accumulated until flushed (timer-based, size-based, or explicit).
+ */
+class AgentlessJSONEncoder {
+  /**
+   * @param {object} writer - Writer instance with a flush() method, called when the buffer exceeds the soft limit
+   * @param {object} [metadata={}] - Shared metadata spread into each trace object (hostname, env, tracerVersion, etc.)
+   */
+  constructor (writer, metadata = {}) {
+    this._writer = writer
+    this._metadata = metadata
+    this._reset()
+  }
+
+  /**
+   * Returns the number of traces encoded.
+   * @returns {number}
+   */
+  count () {
+    return this._traceCount
+  }
+
+  /**
+   * Encodes a trace (array of spans) and adds it to the pending batch.
+   * @param {object[]} trace - Array of spans to encode
+   */
+  encode (trace) {
+    const spanStrings = []
+    let traceSize = 0
+
+    for (const span of trace) {
+      try {
+        const formattedSpan = formatSpan(span, spanStrings.length === 0)
+        const serialized = JSON.stringify(spanToJSON(formattedSpan))
+        spanStrings.push(serialized)
+        traceSize += serialized.length
+      } catch (err) {
+        log.error(
+          'Failed to encode span (name: %s, service: %s). Span will be dropped. Error: %s\n%s',
+          span?.name || 'unknown',
+          span?.service || 'unknown',
+          err.message,
+          err.stack
+        )
+      }
+    }
+
+    if (spanStrings.length > 0) {
+      this._traces.push(spanStrings)
+      this._traceCount++
+      this._estimatedSize += traceSize
+    } else if (trace.length > 0) {
+      log.error('All %d span(s) in trace failed to encode. Entire trace dropped.', trace.length)
+    }
+
+    if (this._estimatedSize > SOFT_LIMIT) {
+      log.debug('Buffer went over soft limit, flushing')
+      try {
+        this._writer.flush()
+      } catch (err) {
+        log.error('Failed to flush on soft limit: %s\n%s', err.message, err.stack)
+      }
+    }
+  }
+
+  /**
+   * Creates the JSON payload for the encoded traces.
+   * Builds the payload via string concatenation from pre-serialized spans to avoid double-stringify.
+   * @returns {Buffer} JSON payload as a buffer, or empty buffer if no traces
+   */
+  makePayload () {
+    if (this._traces.length === 0) {
+      this._reset()
+      return Buffer.alloc(0)
+    }
+
+    try {
+      const metadataJson = JSON.stringify(this._metadata)
+      // Strip trailing '}' so we can append ',"spans":[...]}'
+      const metadataPrefix = metadataJson.slice(0, -1)
+      const hasMetadata = metadataPrefix.length > 1 // more than just '{'
+
+      const traceParts = []
+      for (const spanStrings of this._traces) {
+        const spansJson = '[' + spanStrings.join(',') + ']'
+        if (hasMetadata) {
+          traceParts.push(metadataPrefix + ',"spans":' + spansJson + '}')
+        } else {
+          traceParts.push('{"spans":' + spansJson + '}')
+        }
+      }
+
+      const payload = '{"traces":[' + traceParts.join(',') + ']}'
+      this._reset()
+      return Buffer.from(payload, 'utf8')
+    } catch (err) {
+      log.error(
+        'Failed to encode traces as JSON (%d traces). Traces will be dropped. Error: %s\n%s',
+        this._traces.length,
+        err.message,
+        err.stack
+      )
+      this._reset()
+      return Buffer.alloc(0)
+    }
+  }
+
+  /**
+   * Resets the encoder state.
+   */
+  reset () {
+    this._reset()
+  }
+
+  _reset () {
+    this._traces = []
+    this._traceCount = 0
+    this._estimatedSize = 0
+  }
+}
+
+module.exports = { AgentlessJSONEncoder }

package/packages/dd-trace/src/encode/coverage-ci-visibility.js

@@ -0,0 +1,131 @@
+'use strict'
+const { MsgpackChunk } = require('../msgpack')
+
+const {
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT,
+} = require('../ci-visibility/telemetry')
+const FormData = require('../exporters/common/form-data')
+const { AgentEncoder } = require('./0.4')
+
+const COVERAGE_PAYLOAD_VERSION = 2
+const COVERAGE_KEYS_LENGTH = 2
+
+class CoverageCIVisibilityEncoder extends AgentEncoder {
+  constructor () {
+    super(...arguments)
+    this._coverageBytes = new MsgpackChunk()
+    this.form = new FormData()
+    this._coveragesCount = 0
+    this.reset()
+  }
+
+  count () {
+    return this._coveragesCount
+  }
+
+  encode (coverage) {
+    const startTime = Date.now()
+
+    this._coveragesCount++
+    this.encodeCodeCoverage(this._coverageBytes, coverage)
+
+    distributionMetric(
+      TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+      { endpoint: 'code_coverage' },
+      Date.now() - startTime
+    )
+  }
+
+  encodeCodeCoverage (bytes, coverage) {
+    if (coverage.testId) {
+      this._encodeMapPrefix(bytes, 4)
+    } else {
+      this._encodeMapPrefix(bytes, 3)
+    }
+    this._encodeString(bytes, 'test_session_id')
+    this._encodeId(bytes, coverage.sessionId)
+    this._encodeString(bytes, 'test_suite_id')
+    this._encodeId(bytes, coverage.suiteId)
+    if (coverage.testId) {
+      this._encodeString(bytes, 'span_id')
+      this._encodeId(bytes, coverage.testId)
+    }
+    this._encodeString(bytes, 'files')
+    this._encodeArrayPrefix(bytes, coverage.files)
+    for (const filename of coverage.files) {
+      this._encodeMapPrefix(bytes, 1)
+      this._encodeString(bytes, 'filename')
+      this._encodeString(bytes, filename)
+    }
+  }
+
+  reset () {
+    this._reset()
+    if (this._coverageBytes) {
+      this._coverageBytes.length = 0
+    }
+    this._coveragesCount = 0
+    this._encodePayloadStart(this._coverageBytes)
+  }
+
+  _encodePayloadStart (bytes) {
+    const payload = {
+      version: COVERAGE_PAYLOAD_VERSION,
+      coverages: [],
+    }
+    this._encodeMapPrefix(bytes, COVERAGE_KEYS_LENGTH)
+    this._encodeString(bytes, 'version')
+    this._encodeInteger(bytes, payload.version)
+    this._encodeString(bytes, 'coverages')
+    // Get offset of the coverages list to update the length of the array when calling `makePayload`
+    this._coveragesOffset = bytes.length
+    bytes.reserve(5)
+  }
+
+  makePayload () {
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'code_coverage' }, this._coveragesCount)
+    const bytes = this._coverageBytes
+
+    const coveragesOffset = this._coveragesOffset
+    const coveragesCount = this._coveragesCount
+
+    // update with number of coverages
+    bytes.buffer[coveragesOffset] = 0xDD
+    bytes.buffer[coveragesOffset + 1] = coveragesCount >> 24
+    bytes.buffer[coveragesOffset + 2] = coveragesCount >> 16
+    bytes.buffer[coveragesOffset + 3] = coveragesCount >> 8
+    bytes.buffer[coveragesOffset + 4] = coveragesCount
+
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+
+    this.form.append(
+      'coverage1',
+      buffer,
+      {
+        filename: 'coverage1.msgpack',
+        contentType: 'application/msgpack',
+      }
+    )
+    this.form.append(
+      'event',
+      // The intake requires a populated dictionary here. Simply having {} is not valid.
+      // We use dummy: true but any other key/value pair would be valid.
+      JSON.stringify({ dummy: true }),
+      { filename: 'event.json', contentType: 'application/json' }
+    )
+
+    const form = this.form
+
+    this.form = new FormData()
+    this.reset()
+
+    return form
+  }
+}
+
+module.exports = { CoverageCIVisibilityEncoder }