dd-trace 4.18.0 → 5.6.0

package/packages/dd-trace/src/data_streams_context.js

@@ -6,7 +6,7 @@ function getDataStreamsContext () {
 }
 
 function setDataStreamsContext (dataStreamsContext) {
-  storage.enterWith({ ...(storage.getStore()), dataStreamsContext })
+  if (dataStreamsContext) storage.enterWith({ ...(storage.getStore()), dataStreamsContext })
 }
 
 module.exports = {

package/packages/dd-trace/src/datastreams/pathway.js

@@ -8,6 +8,9 @@ const LRUCache = require('lru-cache')
 const options = { max: 500 }
 const cache = new LRUCache(options)
 
+const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx'
+const CONTEXT_PROPAGATION_KEY_BASE64 = 'dd-pathway-ctx-base64'
+
 function shaHash (checkpointString) {
   const hash = crypto.createHash('md5').update(checkpointString).digest('hex').slice(0, 16)
   return Buffer.from(hash, 'hex')
@@ -33,6 +36,11 @@ function encodePathwayContext (dataStreamsContext) {
   ], 20)
 }
 
+function encodePathwayContextBase64 (dataStreamsContext) {
+  const encodedPathway = encodePathwayContext(dataStreamsContext)
+  return encodedPathway.toString('base64')
+}
+
 function decodePathwayContext (pathwayContext) {
   if (pathwayContext == null || pathwayContext.length < 8) {
     return null
@@ -51,8 +59,57 @@ function decodePathwayContext (pathwayContext) {
   return { hash: pathwayHash, pathwayStartNs: pathwayStartMs * 1e6, edgeStartNs: edgeStartMs * 1e6 }
 }
 
+function decodePathwayContextBase64 (pathwayContext) {
+  if (pathwayContext == null || pathwayContext.length < 8) {
+    return
+  }
+  if (Buffer.isBuffer(pathwayContext)) {
+    pathwayContext = pathwayContext.toString()
+  }
+  const encodedPathway = Buffer.from(pathwayContext, 'base64')
+  return decodePathwayContext(encodedPathway)
+}
+
+class DsmPathwayCodec {
+  // we use a class for encoding / decoding in case we update our encoding/decoding. A class will make updates easier
+  // instead of using individual functions.
+  static encode (dataStreamsContext, carrier) {
+    if (!dataStreamsContext || !dataStreamsContext.hash) {
+      return
+    }
+    carrier[CONTEXT_PROPAGATION_KEY_BASE64] = encodePathwayContextBase64(dataStreamsContext)
+  }
+
+  static decode (carrier) {
+    if (carrier == null) return
+
+    let ctx
+    if (CONTEXT_PROPAGATION_KEY_BASE64 in carrier) {
+      // decode v2 encoding of base64
+      ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY_BASE64])
+    } else if (CONTEXT_PROPAGATION_KEY in carrier) {
+      try {
+        // decode v1 encoding
+        ctx = decodePathwayContext(carrier[CONTEXT_PROPAGATION_KEY])
+      } catch {
+        // pass
+      }
+      // cover case where base64 context was received under wrong key
+      if (!ctx) ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY])
+    }
+    return ctx
+  }
+
+  static contextExists (carrier) {
+    return CONTEXT_PROPAGATION_KEY_BASE64 in carrier || CONTEXT_PROPAGATION_KEY in carrier
+  }
+}
+
 module.exports = {
   computePathwayHash: computeHash,
   encodePathwayContext,
-  decodePathwayContext
+  decodePathwayContext,
+  encodePathwayContextBase64,
+  decodePathwayContextBase64,
+  DsmPathwayCodec
 }

package/packages/dd-trace/src/datastreams/processor.js

@@ -4,9 +4,12 @@ const pkg = require('../../../../package.json')
 const Uint64 = require('int64-buffer').Uint64BE
 
 const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
-
+const { DsmPathwayCodec } = require('./pathway')
 const { DataStreamsWriter } = require('./writer')
 const { computePathwayHash } = require('./pathway')
+const { types } = require('util')
+const { PATHWAY_HASH } = require('../../../../ext/tags')
+
 const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex')
 
 const HIGH_ACCURACY_DISTRIBUTION = 0.0075
@@ -18,6 +21,7 @@ class StatsPoint {
     this.edgeTags = edgeTags
     this.edgeLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
     this.pathwayLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
+    this.payloadSize = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
   }
 
   addLatencies (checkpoint) {
@@ -25,6 +29,7 @@ class StatsPoint {
     const pathwayLatencySec = checkpoint.pathwayLatencyNs / 1e9
     this.edgeLatency.accept(edgeLatencySec)
     this.pathwayLatency.accept(pathwayLatencySec)
+    this.payloadSize.accept(checkpoint.payloadSize)
   }
 
   encode () {
@@ -33,20 +38,123 @@ class StatsPoint {
       ParentHash: this.parentHash,
       EdgeTags: this.edgeTags,
       EdgeLatency: this.edgeLatency.toProto(),
-      PathwayLatency: this.pathwayLatency.toProto()
+      PathwayLatency: this.pathwayLatency.toProto(),
+      PayloadSize: this.payloadSize.toProto()
     }
   }
 }
 
-class StatsBucket extends Map {
+class Backlog {
+  constructor ({ offset, ...tags }) {
+    this._tags = Object.keys(tags).sort().map(key => `${key}:${tags[key]}`)
+    this._hash = this._tags.join(',')
+    this._offset = offset
+  }
+
+  get hash () { return this._hash }
+
+  get offset () { return this._offset }
+
+  get tags () { return this._tags }
+
+  encode () {
+    return {
+      Tags: this.tags,
+      Value: this.offset
+    }
+  }
+}
+
+class StatsBucket {
+  constructor () {
+    this._checkpoints = new Map()
+    this._backlogs = new Map()
+  }
+
+  get checkpoints () {
+    return this._checkpoints
+  }
+
+  get backlogs () {
+    return this._backlogs
+  }
+
   forCheckpoint (checkpoint) {
     const key = checkpoint.hash
-    if (!this.has(key)) {
-      this.set(key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)) // StatsPoint
+    if (!this._checkpoints.has(key)) {
+      this._checkpoints.set(
+        key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)
+      )
     }
 
-    return this.get(key)
+    return this._checkpoints.get(key)
+  }
+
+  /**
+   * Conditionally add a backlog to the bucket. If there is currently an offset
+   * matching the backlog's tags, overwrite the offset IFF the backlog's offset
+   * is greater than the recorded offset.
+   *
+   * @typedef {{[key: string]: string}} BacklogData
+   * @property {number} offset
+   *
+   * @param {BacklogData} backlogData
+   * @returns {Backlog}
+   */
+  forBacklog (backlogData) {
+    const backlog = new Backlog(backlogData)
+    const existingBacklog = this._backlogs.get(backlog.hash)
+    if (existingBacklog !== undefined) {
+      if (existingBacklog.offset > backlog.offset) {
+        return existingBacklog
+      }
+    }
+    this._backlogs.set(backlog.hash, backlog)
+    return backlog
+  }
+}
+
+function getSizeOrZero (obj) {
+  if (typeof obj === 'string') {
+    return Buffer.from(obj, 'utf-8').length
   }
+  if (types.isArrayBuffer(obj)) {
+    return obj.byteLength
+  }
+  if (Buffer.isBuffer(obj)) {
+    return obj.length
+  }
+  if (Array.isArray(obj) && obj.length > 0) {
+    if (typeof obj[0] === 'number') return Buffer.from(obj).length
+    let payloadSize = 0
+    obj.forEach(item => {
+      payloadSize += getSizeOrZero(item)
+    })
+    return payloadSize
+  }
+  if (typeof obj === 'object') {
+    try {
+      return getHeadersSize(obj)
+    } catch {
+      // pass
+    }
+  }
+  return 0
+}
+
+function getHeadersSize (headers) {
+  if (headers === undefined) return 0
+  return Object.entries(headers).reduce((prev, [key, val]) => getSizeOrZero(key) + getSizeOrZero(val) + prev, 0)
+}
+
+function getMessageSize (message) {
+  const { key, value, headers } = message
+  return getSizeOrZero(key) + getSizeOrZero(value) + getHeadersSize(headers)
+}
+
+function getAmqpMessageSize (message) {
+  const { headers, content } = message
+  return getSizeOrZero(content) + getHeadersSize(headers)
 }
 
 class TimeBuckets extends Map {
@@ -68,7 +176,8 @@ class DataStreamsProcessor {
     env,
     tags,
     version,
-    service
+    service,
+    flushInterval
   } = {}) {
     this.writer = new DataStreamsWriter({
       hostname,
@@ -84,20 +193,22 @@ class DataStreamsProcessor {
     this.service = service || 'unnamed-nodejs-service'
     this.version = version || ''
     this.sequence = 0
+    this.flushInterval = flushInterval
 
     if (this.enabled) {
-      this.timer = setInterval(this.onInterval.bind(this), 10000)
+      this.timer = setInterval(this.onInterval.bind(this), flushInterval)
       this.timer.unref()
     }
+    process.once('beforeExit', () => this.onInterval())
   }
 
   onInterval () {
-    const serialized = this._serializeBuckets()
-    if (!serialized) return
+    const { Stats } = this._serializeBuckets()
+    if (Stats.length === 0) return
     const payload = {
       Env: this.env,
       Service: this.service,
-      Stats: serialized,
+      Stats,
       TracerVersion: pkg.version,
       Version: this.version,
       Lang: 'javascript'
@@ -105,15 +216,29 @@ class DataStreamsProcessor {
     this.writer.flush(payload)
   }
 
-  recordCheckpoint (checkpoint) {
+  /**
+   * Given a timestamp in nanoseconds, compute and return the closest TimeBucket
+   * @param {number} timestamp
+   * @returns {StatsBucket}
+   */
+  bucketFromTimestamp (timestamp) {
+    const bucketTime = Math.round(timestamp - (timestamp % this.bucketSizeNs))
+    const bucket = this.buckets.forTime(bucketTime)
+    return bucket
+  }
+
+  recordCheckpoint (checkpoint, span = null) {
     if (!this.enabled) return
-    const bucketTime = Math.round(checkpoint.currentTimestamp - (checkpoint.currentTimestamp % this.bucketSizeNs))
-    this.buckets.forTime(bucketTime)
+    this.bucketFromTimestamp(checkpoint.currentTimestamp)
       .forCheckpoint(checkpoint)
       .addLatencies(checkpoint)
+    // set DSM pathway hash on span to enable related traces feature on DSM tab, convert from buffer to uint64
+    if (span) {
+      span.setTag(PATHWAY_HASH, checkpoint.hash.readBigUInt64BE(0).toString())
+    }
   }
 
-  setCheckpoint (edgeTags, ctx = null) {
+  setCheckpoint (edgeTags, span, ctx = null, payloadSize = 0) {
     if (!this.enabled) return null
     const nowNs = Date.now() * 1e6
     const direction = edgeTags.find(t => t.startsWith('direction:'))
@@ -147,45 +272,84 @@ class DataStreamsProcessor {
     const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash)
     const edgeLatencyNs = nowNs - edgeStartNs
     const pathwayLatencyNs = nowNs - pathwayStartNs
+    const dataStreamsContext = {
+      hash: hash,
+      edgeStartNs: edgeStartNs,
+      pathwayStartNs: pathwayStartNs,
+      previousDirection: direction,
+      closestOppositeDirectionHash: closestOppositeDirectionHash,
+      closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart
+    }
+    if (direction === 'direction:out') {
+      // Add the header for this now, as the callee doesn't have access to context when producing
+      // - 1 to account for extra byte for {
+      const ddInfoContinued = {}
+      DsmPathwayCodec.encode(dataStreamsContext, ddInfoContinued)
+      payloadSize += getSizeOrZero(JSON.stringify(ddInfoContinued)) - 1
+    }
     const checkpoint = {
       currentTimestamp: nowNs,
       parentHash: parentHash,
       hash: hash,
       edgeTags: edgeTags,
       edgeLatencyNs: edgeLatencyNs,
-      pathwayLatencyNs: pathwayLatencyNs
+      pathwayLatencyNs: pathwayLatencyNs,
+      payloadSize: payloadSize
     }
-    this.recordCheckpoint(checkpoint)
-    return {
-      hash: hash,
-      edgeStartNs: edgeStartNs,
-      pathwayStartNs: pathwayStartNs,
-      previousDirection: direction,
-      closestOppositeDirectionHash: closestOppositeDirectionHash,
-      closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart
+    this.recordCheckpoint(checkpoint, span)
+    return dataStreamsContext
+  }
+
+  recordOffset ({ timestamp, ...backlogData }) {
+    if (!this.enabled) return
+    return this.bucketFromTimestamp(timestamp)
+      .forBacklog(backlogData)
+  }
+
+  setOffset (offsetObj) {
+    if (!this.enabled) return
+    const nowNs = Date.now() * 1e6
+    const backlogData = {
+      ...offsetObj,
+      timestamp: nowNs
     }
+    this.recordOffset(backlogData)
   }
 
   _serializeBuckets () {
+    // TimeBuckets
     const serializedBuckets = []
 
     for (const [ timeNs, bucket ] of this.buckets.entries()) {
       const points = []
 
-      for (const stats of bucket.values()) {
+      // bucket: StatsBucket
+      // stats: StatsPoint
+      for (const stats of bucket._checkpoints.values()) {
         points.push(stats.encode())
       }
 
+      const backlogs = []
+      for (const backlog of bucket._backlogs.values()) {
+        backlogs.push(backlog.encode())
+      }
       serializedBuckets.push({
         Start: new Uint64(timeNs),
         Duration: new Uint64(this.bucketSizeNs),
-        Stats: points
+        Stats: points,
+        Backlogs: backlogs
       })
     }
 
     this.buckets.clear()
 
-    return serializedBuckets
+    return {
+      Stats: serializedBuckets
+    }
+  }
+
+  setUrl (url) {
+    this.writer.setUrl(url)
   }
 }
 
@@ -193,6 +357,11 @@ module.exports = {
   DataStreamsProcessor: DataStreamsProcessor,
   StatsPoint: StatsPoint,
   StatsBucket: StatsBucket,
+  Backlog,
   TimeBuckets,
+  getMessageSize,
+  getHeadersSize,
+  getSizeOrZero,
+  getAmqpMessageSize,
   ENTRY_PARENT_HASH
 }

package/packages/dd-trace/src/datastreams/writer.js

@@ -15,13 +15,10 @@ function makeRequest (data, url, cb) {
       'Datadog-Meta-Tracer-Version': pkg.version,
       'Content-Type': 'application/msgpack',
       'Content-Encoding': 'gzip'
-    }
+    },
+    url
   }
 
-  options.protocol = url.protocol
-  options.hostname = url.hostname
-  options.port = url.port
-
   log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
 
   request(data, options, (err, res) => {
@@ -59,6 +56,15 @@ class DataStreamsWriter {
       })
     })
   }
+
+  setUrl (url) {
+    try {
+      url = new URL(url)
+      this._url = url
+    } catch (e) {
+      log.warn(e.stack)
+    }
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/dogstatsd.js

@@ -67,16 +67,14 @@ class DogStatsDClient {
     request(buffer, this._httpOptions, (err) => {
       if (err) {
         log.error('HTTP error from agent: ' + err.stack)
-        if (err.status) {
+        if (err.status === 404) {
           // Inside this if-block, we have connectivity to the agent, but
           // we're not getting a 200 from the proxy endpoint. If it's a 404,
           // then we know we'll never have the endpoint, so just clear out the
           // options. Either way, we can give UDP a try.
-          if (err.status === 404) {
-            this._httpOptions = null
-          }
-          this._sendUdp(queue)
+          this._httpOptions = null
         }
+        this._sendUdp(queue)
       }
     })
   }

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -2,14 +2,21 @@
 const { truncateSpan, normalizeSpan } = require('./tags-processors')
 const { AgentEncoder } = require('./0.4')
 const { version: ddTraceVersion } = require('../../../../package.json')
-const id = require('../../../dd-trace/src/id')
-const ENCODING_VERSION = 1
+const { ITR_CORRELATION_ID } = require('../../src/plugins/util/test')
+const id = require('../../src/id')
+const {
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT
+} = require('../ci-visibility/telemetry')
 
+const ENCODING_VERSION = 1
 const ALLOWED_CONTENT_TYPES = ['test_session_end', 'test_module_end', 'test_suite_end', 'test']
 
 const TEST_SUITE_KEYS_LENGTH = 12
 const TEST_MODULE_KEYS_LENGTH = 11
 const TEST_SESSION_KEYS_LENGTH = 10
+const TEST_AND_SPAN_KEYS_LENGTH = 11
 
 const INTAKE_SOFT_LIMIT = 2 * 1024 * 1024 // 2MB
 
@@ -40,7 +47,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encodeTestSuite (bytes, content) {
-    this._encodeMapPrefix(bytes, TEST_SUITE_KEYS_LENGTH)
+    let keysLength = TEST_SUITE_KEYS_LENGTH
+    const itrCorrelationId = content.meta[ITR_CORRELATION_ID]
+    if (itrCorrelationId) {
+      keysLength++
+    }
+
+    this._encodeMapPrefix(bytes, keysLength)
     this._encodeString(bytes, 'type')
     this._encodeString(bytes, content.type)
 
@@ -53,6 +66,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeString(bytes, 'test_suite_id')
     this._encodeId(bytes, content.span_id)
 
+    if (itrCorrelationId) {
+      this._encodeString(bytes, ITR_CORRELATION_ID)
+      this._encodeString(bytes, itrCorrelationId)
+      delete content.meta[ITR_CORRELATION_ID]
+    }
+
     this._encodeString(bytes, 'error')
     this._encodeNumber(bytes, content.error)
     this._encodeString(bytes, 'name')
@@ -127,9 +146,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encodeEventContent (bytes, content) {
-    const keysLength = Object.keys(content).length
-
-    let totalKeysLength = keysLength
+    let totalKeysLength = TEST_AND_SPAN_KEYS_LENGTH
     if (content.meta.test_session_id) {
       totalKeysLength = totalKeysLength + 1
     }
@@ -139,6 +156,13 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     if (content.meta.test_suite_id) {
       totalKeysLength = totalKeysLength + 1
     }
+    const itrCorrelationId = content.meta[ITR_CORRELATION_ID]
+    if (itrCorrelationId) {
+      totalKeysLength = totalKeysLength + 1
+    }
+    if (content.type) {
+      totalKeysLength = totalKeysLength + 1
+    }
     this._encodeMapPrefix(bytes, totalKeysLength)
     if (content.type) {
       this._encodeString(bytes, 'type')
@@ -189,6 +213,12 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
       delete content.meta.test_suite_id
     }
 
+    if (itrCorrelationId) {
+      this._encodeString(bytes, ITR_CORRELATION_ID)
+      this._encodeString(bytes, itrCorrelationId)
+      delete content.meta[ITR_CORRELATION_ID]
+    }
+
     this._encodeString(bytes, 'meta')
     this._encodeMap(bytes, content.meta)
     this._encodeString(bytes, 'metrics')
@@ -247,6 +277,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encode (bytes, trace) {
+    const startTime = Date.now()
+
     const rawEvents = trace.map(formatSpan)
 
     const testSessionEvents = rawEvents.filter(
@@ -261,9 +293,15 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     for (const event of events) {
       this._encodeEvent(bytes, event)
     }
+    distributionMetric(
+      TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+      { endpoint: 'test_cycle' },
+      Date.now() - startTime
+    )
   }
 
   makePayload () {
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'test_cycle' }, this._eventCount)
     const bytes = this._traceBytes
     const eventsOffset = this._eventsOffset
     const eventsCount = this._eventCount

package/packages/dd-trace/src/encode/coverage-ci-visibility.js

@@ -2,6 +2,11 @@
 const { AgentEncoder } = require('./0.4')
 const Chunk = require('./chunk')
 
+const {
+  distributionMetric,
+  TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+  TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT
+} = require('../ci-visibility/telemetry')
 const FormData = require('../exporters/common/form-data')
 
 const COVERAGE_PAYLOAD_VERSION = 2
@@ -21,8 +26,16 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
   }
 
   encode (coverage) {
+    const startTime = Date.now()
+
     this._coveragesCount++
     this.encodeCodeCoverage(this._coverageBytes, coverage)
+
+    distributionMetric(
+      TELEMETRY_ENDPOINT_PAYLOAD_SERIALIZATION_MS,
+      { endpoint: 'code_coverage' },
+      Date.now() - startTime
+    )
   }
 
   encodeCodeCoverage (bytes, coverage) {
@@ -73,6 +86,7 @@ class CoverageCIVisibilityEncoder extends AgentEncoder {
   }
 
   makePayload () {
+    distributionMetric(TELEMETRY_ENDPOINT_PAYLOAD_EVENTS_COUNT, { endpoint: 'code_coverage' }, this._coveragesCount)
     const bytes = this._coverageBytes
 
     const coveragesOffset = this._coveragesOffset

package/packages/dd-trace/src/exporters/common/agent-info-exporter.js

@@ -1,6 +1,7 @@
 const { URL, format } = require('url')
 
 const request = require('./request')
+const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../../ci-visibility/telemetry')
 
 function fetchAgentInfo (url, callback) {
   request('', {
@@ -49,6 +50,9 @@ class AgentInfoExporter {
   }
 
   _export (payload, writer = this._writer, timerKey = '_timer') {
+    if (this._config.isCiVisibility) {
+      incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION, {}, payload.length)
+    }
     writer.append(payload)
 
     const { flushInterval } = this._config

package/packages/dd-trace/src/exporters/common/form-data.js

@@ -21,6 +21,10 @@ class FormData extends Readable {
     }
   }
 
+  size () {
+    return this._data.reduce((size, chunk) => size + chunk.length, 0)
+  }
+
   getHeaders () {
     return { 'Content-Type': 'multipart/form-data; boundary=' + this._boundary }
   }