dd-trace 4.18.0 → 4.23.0

package/packages/dd-trace/src/config.js

@@ -109,13 +109,6 @@ class Config {
     log.use(this.logger)
     log.toggle(this.debug, this.logLevel, this)
 
-    this.tags = {}
-
-    tagger.add(this.tags, process.env.DD_TAGS)
-    tagger.add(this.tags, process.env.DD_TRACE_TAGS)
-    tagger.add(this.tags, process.env.DD_TRACE_GLOBAL_TAGS)
-    tagger.add(this.tags, options.tags)
-
     const DD_TRACING_ENABLED = coalesce(
       process.env.DD_TRACING_ENABLED,
       true
@@ -184,33 +177,12 @@ class Config {
       false
     )
 
-    const DD_SERVICE = options.service ||
-      process.env.DD_SERVICE ||
-      process.env.DD_SERVICE_NAME ||
-      this.tags.service ||
-      process.env.AWS_LAMBDA_FUNCTION_NAME ||
-      process.env.FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes
-      process.env.K_SERVICE || // Google Cloud Function Name set by newer runtimes
-      process.env.WEBSITE_SITE_NAME || // set by Azure Functions
-      pkg.name ||
-      'node'
     const DD_SERVICE_MAPPING = coalesce(
       options.serviceMapping,
       process.env.DD_SERVICE_MAPPING ? fromEntries(
         process.env.DD_SERVICE_MAPPING.split(',').map(x => x.trim().split(':'))
       ) : {}
     )
-    const DD_ENV = coalesce(
-      options.env,
-      process.env.DD_ENV,
-      this.tags.env
-    )
-    const DD_VERSION = coalesce(
-      options.version,
-      process.env.DD_VERSION,
-      this.tags.version,
-      pkg.version
-    )
     const DD_TRACE_STARTUP_LOGS = coalesce(
       options.startupLogs,
       process.env.DD_TRACE_STARTUP_LOGS,
@@ -235,8 +207,9 @@ class Config {
 
     const inServerlessEnvironment = inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan
 
-    const DD_TRACE_TELEMETRY_ENABLED = coalesce(
-      process.env.DD_TRACE_TELEMETRY_ENABLED,
+    const DD_INSTRUMENTATION_TELEMETRY_ENABLED = coalesce(
+      process.env.DD_TRACE_TELEMETRY_ENABLED, // for backward compatibility
+      process.env.DD_INSTRUMENTATION_TELEMETRY_ENABLED, // to comply with instrumentation telemetry specs
       !inServerlessEnvironment
     )
     const DD_TELEMETRY_HEARTBEAT_INTERVAL = process.env.DD_TELEMETRY_HEARTBEAT_INTERVAL
@@ -308,6 +281,10 @@ class Config {
       options.tracePropagationStyle,
       defaultPropagationStyle
     )
+    const DD_TRACE_PROPAGATION_EXTRACT_FIRST = coalesce(
+      process.env.DD_TRACE_PROPAGATION_EXTRACT_FIRST,
+      false
+    )
     const DD_TRACE_RUNTIME_ID_ENABLED = coalesce(
       options.experimental && options.experimental.runtimeId,
       process.env.DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED,
@@ -367,10 +344,11 @@ class Config {
       isGCPFunction || isAzureFunctionConsumptionPlan
     )
 
+    // the tracer generates 128 bit IDs by default as of v5
     const DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = coalesce(
       options.traceId128BitGenerationEnabled,
       process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED,
-      false
+      true
     )
 
     const DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = coalesce(
@@ -393,7 +371,6 @@ class Config {
       appsec.enabled,
       process.env.DD_APPSEC_ENABLED && isTrue(process.env.DD_APPSEC_ENABLED)
     )
-
     const DD_APPSEC_RULES = coalesce(
       appsec.rules,
       process.env.DD_APPSEC_RULES
@@ -430,11 +407,25 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       maybeFile(appsec.blockedTemplateJson),
       maybeFile(process.env.DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON)
     )
+    const DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON = coalesce(
+      maybeFile(appsec.blockedTemplateGraphql),
+      maybeFile(process.env.DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON)
+    )
     const DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING = coalesce(
       appsec.eventTracking && appsec.eventTracking.mode,
       process.env.DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING,
       'safe'
     ).toLowerCase()
+    const DD_EXPERIMENTAL_API_SECURITY_ENABLED = coalesce(
+      appsec?.apiSecurity?.enabled,
+      isTrue(process.env.DD_EXPERIMENTAL_API_SECURITY_ENABLED),
+      false
+    )
+    const DD_API_SECURITY_REQUEST_SAMPLE_RATE = coalesce(
+      appsec?.apiSecurity?.requestSampling,
+      parseFloat(process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE),
+      0.1
+    )
 
     const remoteConfigOptions = options.remoteConfig || {}
     const DD_REMOTE_CONFIGURATION_ENABLED = coalesce(
@@ -459,6 +450,11 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       DD_IAST_ENABLED
     )
 
+    const DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED = coalesce(
+      process.env.DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED,
+      true
+    )
+
     const defaultIastRequestSampling = 30
     const iastRequestSampling = coalesce(
       parseInt(iastOptions?.requestSampling),
@@ -520,6 +516,25 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       true
     )
 
+    // 0: disabled, 1: logging, 2: garbage collection + logging
+    const DD_TRACE_SPAN_LEAK_DEBUG = coalesce(
+      process.env.DD_TRACE_SPAN_LEAK_DEBUG,
+      0
+    )
+
+    const DD_INSTRUMENTATION_INSTALL_ID = coalesce(
+      process.env.DD_INSTRUMENTATION_INSTALL_ID,
+      null
+    )
+    const DD_INSTRUMENTATION_INSTALL_TIME = coalesce(
+      process.env.DD_INSTRUMENTATION_INSTALL_TIME,
+      null
+    )
+    const DD_INSTRUMENTATION_INSTALL_TYPE = coalesce(
+      process.env.DD_INSTRUMENTATION_INSTALL_TYPE,
+      null
+    )
+
     const ingestion = options.ingestion || {}
     const dogstatsd = coalesce(options.dogstatsd, {})
     const sampler = {
@@ -553,7 +568,6 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.dsmEnabled = isTrue(DD_DATA_STREAMS_ENABLED)
     this.openAiLogsEnabled = DD_OPENAI_LOGS_ENABLED
     this.apiKey = DD_API_KEY
-    this.env = DD_ENV
     this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL)
       : getAgentUrl(DD_TRACE_AGENT_URL, options)
     this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com')
@@ -565,9 +579,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.clientIpEnabled = DD_TRACE_CLIENT_IP_ENABLED
     this.clientIpHeader = DD_TRACE_CLIENT_IP_HEADER
     this.plugins = !!coalesce(options.plugins, true)
-    this.service = DD_SERVICE
     this.serviceMapping = DD_SERVICE_MAPPING
-    this.version = DD_VERSION
     this.dogstatsd = {
       hostname: coalesce(dogstatsd.hostname, process.env.DD_DOGSTATSD_HOSTNAME, this.hostname),
       port: String(coalesce(dogstatsd.port, process.env.DD_DOGSTATSD_PORT, 8125))
@@ -577,6 +589,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       inject: DD_TRACE_PROPAGATION_STYLE_INJECT,
       extract: DD_TRACE_PROPAGATION_STYLE_EXTRACT
     }
+    this.tracePropagationExtractFirst = isTrue(DD_TRACE_PROPAGATION_EXTRACT_FIRST)
     this.experimental = {
       runtimeId: isTrue(DD_TRACE_RUNTIME_ID_ENABLED),
       exporter: DD_TRACE_EXPORTER,
@@ -598,17 +611,18 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
     // Disabled for CI Visibility's agentless
     this.telemetry = {
-      enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_TRACE_TELEMETRY_ENABLED),
+      enabled: DD_TRACE_EXPORTER !== 'datadog' && isTrue(DD_INSTRUMENTATION_TELEMETRY_ENABLED),
       heartbeatInterval: DD_TELEMETRY_HEARTBEAT_INTERVAL,
       debug: isTrue(DD_TELEMETRY_DEBUG),
       logCollection: isTrue(DD_TELEMETRY_LOG_COLLECTION_ENABLED),
-      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED)
+      metrics: isTrue(DD_TELEMETRY_METRICS_ENABLED),
+      dependencyCollection: DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED
     }
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.tagsHeaderMaxLength = parseInt(DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH)
     this.appsec = {
       enabled: DD_APPSEC_ENABLED,
-      rules: DD_APPSEC_RULES ? safeJsonParse(maybeFile(DD_APPSEC_RULES)) : require('./appsec/recommended.json'),
+      rules: DD_APPSEC_RULES,
       customRulesProvided: !!DD_APPSEC_RULES,
       rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
       wafTimeout: DD_APPSEC_WAF_TIMEOUT,
@@ -616,11 +630,18 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
       blockedTemplateHtml: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML,
       blockedTemplateJson: DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON,
+      blockedTemplateGraphql: DD_APPSEC_GRAPHQL_BLOCKED_TEMPLATE_JSON,
       eventTracking: {
         enabled: ['extended', 'safe'].includes(DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING),
         mode: DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING
+      },
+      apiSecurity: {
+        enabled: DD_EXPERIMENTAL_API_SECURITY_ENABLED,
+        // Coerce value between 0 and 1
+        requestSampling: Math.min(1, Math.max(0, DD_API_SECURITY_REQUEST_SAMPLE_RATE))
       }
     }
+
     this.remoteConfig = {
       enabled: DD_REMOTE_CONFIGURATION_ENABLED,
       pollInterval: DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS
@@ -651,6 +672,37 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent
     this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED)
 
+    this.stats = {
+      enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
+    }
+
+    this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
+    this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED)
+
+    this.isGCPFunction = isGCPFunction
+    this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan
+
+    this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG)
+
+    this.installSignature = {
+      id: DD_INSTRUMENTATION_INSTALL_ID,
+      time: DD_INSTRUMENTATION_INSTALL_TIME,
+      type: DD_INSTRUMENTATION_INSTALL_TYPE
+    }
+
+    this._applyDefaults()
+    this._applyEnvironment()
+    this._applyOptions(options)
+    this._applyRemote({})
+    this._merge()
+
+    tagger.add(this.tags, {
+      service: this.service,
+      env: this.env,
+      version: this.version,
+      'runtime-id': uuid()
+    })
+
     if (this.gitMetadataEnabled) {
       this.repositoryUrl = removeUserSensitiveInfo(
         coalesce(
@@ -683,29 +735,6 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
         }
       }
     }
-
-    this.stats = {
-      enabled: isTrue(DD_TRACE_STATS_COMPUTATION_ENABLED)
-    }
-
-    this.traceId128BitGenerationEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
-    this.traceId128BitLoggingEnabled = isTrue(DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED)
-
-    this.isGCPFunction = isGCPFunction
-    this.isAzureFunctionConsumptionPlan = isAzureFunctionConsumptionPlan
-
-    tagger.add(this.tags, {
-      service: this.service,
-      env: this.env,
-      version: this.version,
-      'runtime-id': uuid()
-    })
-
-    this._applyDefaults()
-    this._applyEnvironment()
-    this._applyOptions(options)
-    this._applyRemote({})
-    this._merge()
   }
 
   // Supports only a subset of options for now.
@@ -720,48 +749,93 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
   }
 
   _applyDefaults () {
+    const {
+      AWS_LAMBDA_FUNCTION_NAME,
+      FUNCTION_NAME,
+      K_SERVICE,
+      WEBSITE_SITE_NAME
+    } = process.env
+
+    const service = AWS_LAMBDA_FUNCTION_NAME ||
+      FUNCTION_NAME || // Google Cloud Function Name set by deprecated runtimes
+      K_SERVICE || // Google Cloud Function Name set by newer runtimes
+      WEBSITE_SITE_NAME || // set by Azure Functions
+      pkg.name ||
+      'node'
+
     const defaults = this._defaults = {}
 
+    this._setValue(defaults, 'service', service)
+    this._setValue(defaults, 'env', undefined)
+    this._setValue(defaults, 'version', pkg.version)
     this._setUnit(defaults, 'sampleRate', undefined)
     this._setBoolean(defaults, 'logInjection', false)
     this._setArray(defaults, 'headerTags', [])
+    this._setValue(defaults, 'tags', {})
   }
 
   _applyEnvironment () {
     const {
-      DD_TRACE_SAMPLE_RATE,
+      DD_ENV,
       DD_LOGS_INJECTION,
-      DD_TRACE_HEADER_TAGS
+      DD_SERVICE,
+      DD_SERVICE_NAME,
+      DD_TAGS,
+      DD_TRACE_GLOBAL_TAGS,
+      DD_TRACE_HEADER_TAGS,
+      DD_TRACE_SAMPLE_RATE,
+      DD_TRACE_TAGS,
+      DD_VERSION
     } = process.env
 
+    const tags = {}
     const env = this._env = {}
 
+    tagger.add(tags, DD_TAGS)
+    tagger.add(tags, DD_TRACE_TAGS)
+    tagger.add(tags, DD_TRACE_GLOBAL_TAGS)
+
+    this._setValue(env, 'service', DD_SERVICE || DD_SERVICE_NAME || tags.service)
+    this._setValue(env, 'env', DD_ENV || tags.env)
+    this._setValue(env, 'version', DD_VERSION || tags.version)
     this._setUnit(env, 'sampleRate', DD_TRACE_SAMPLE_RATE)
     this._setBoolean(env, 'logInjection', DD_LOGS_INJECTION)
     this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS)
+    this._setTags(env, 'tags', tags)
   }
 
   _applyOptions (options) {
     const opts = this._options = this._options || {}
+    const tags = {}
 
     options = Object.assign({ ingestion: {} }, options, opts)
 
+    tagger.add(tags, options.tags)
+
+    this._setValue(opts, 'service', options.service || tags.service)
+    this._setValue(opts, 'env', options.env || tags.env)
+    this._setValue(opts, 'version', options.version || tags.version)
     this._setUnit(opts, 'sampleRate', coalesce(options.sampleRate, options.ingestion.sampleRate))
     this._setBoolean(opts, 'logInjection', options.logInjection)
     this._setArray(opts, 'headerTags', options.headerTags)
+    this._setTags(opts, 'tags', tags)
   }
 
   _applyRemote (options) {
     const opts = this._remote = this._remote || {}
+    const tags = {}
     const headerTags = options.tracing_header_tags
       ? options.tracing_header_tags.map(tag => {
         return tag.tag_name ? `${tag.header}:${tag.tag_name}` : tag.header
       })
       : undefined
 
+    tagger.add(tags, options.tracing_tags)
+
     this._setUnit(opts, 'sampleRate', options.tracing_sampling_rate)
     this._setBoolean(opts, 'logInjection', options.log_injection_enabled)
     this._setArray(opts, 'headerTags', headerTags)
+    this._setTags(opts, 'tags', tags)
   }
 
   _setBoolean (obj, name, value) {
@@ -801,6 +875,14 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
     }
   }
 
+  _setTags (obj, name, value) {
+    if (!value || Object.keys(value).length === 0) {
+      return this._setValue(obj, name, null)
+    }
+
+    this._setValue(obj, name, value)
+  }
+
   _setValue (obj, name, value) {
     obj[name] = value
   }

package/packages/dd-trace/src/datastreams/processor.js

@@ -4,12 +4,16 @@ const pkg = require('../../../../package.json')
 const Uint64 = require('int64-buffer').Uint64BE
 
 const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
-
+const { encodePathwayContext } = require('./pathway')
 const { DataStreamsWriter } = require('./writer')
 const { computePathwayHash } = require('./pathway')
+const { types } = require('util')
+const { PATHWAY_HASH } = require('../../../../ext/tags')
+
 const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex')
 
 const HIGH_ACCURACY_DISTRIBUTION = 0.0075
+const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx'
 
 class StatsPoint {
   constructor (hash, parentHash, edgeTags) {
@@ -18,6 +22,7 @@ class StatsPoint {
     this.edgeTags = edgeTags
     this.edgeLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
     this.pathwayLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
+    this.payloadSize = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
   }
 
   addLatencies (checkpoint) {
@@ -25,6 +30,7 @@ class StatsPoint {
     const pathwayLatencySec = checkpoint.pathwayLatencyNs / 1e9
     this.edgeLatency.accept(edgeLatencySec)
     this.pathwayLatency.accept(pathwayLatencySec)
+    this.payloadSize.accept(checkpoint.payloadSize)
   }
 
   encode () {
@@ -33,22 +39,105 @@ class StatsPoint {
       ParentHash: this.parentHash,
       EdgeTags: this.edgeTags,
       EdgeLatency: this.edgeLatency.toProto(),
-      PathwayLatency: this.pathwayLatency.toProto()
+      PathwayLatency: this.pathwayLatency.toProto(),
+      PayloadSize: this.payloadSize.toProto()
     }
   }
 }
 
-class StatsBucket extends Map {
+class Backlog {
+  constructor ({ offset, ...tags }) {
+    this._tags = Object.keys(tags).sort().map(key => `${key}:${tags[key]}`)
+    this._hash = this._tags.join(',')
+    this._offset = offset
+  }
+
+  get hash () { return this._hash }
+
+  get offset () { return this._offset }
+
+  get tags () { return this._tags }
+
+  encode () {
+    return {
+      Tags: this.tags,
+      Value: this.offset
+    }
+  }
+}
+
+class StatsBucket {
+  constructor () {
+    this._checkpoints = new Map()
+    this._backlogs = new Map()
+  }
+
+  get checkpoints () {
+    return this._checkpoints
+  }
+
+  get backlogs () {
+    return this._backlogs
+  }
+
   forCheckpoint (checkpoint) {
     const key = checkpoint.hash
-    if (!this.has(key)) {
-      this.set(key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)) // StatsPoint
+    if (!this._checkpoints.has(key)) {
+      this._checkpoints.set(
+        key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)
+      )
     }
 
-    return this.get(key)
+    return this._checkpoints.get(key)
+  }
+
+  /**
+   * Conditionally add a backlog to the bucket. If there is currently an offset
+   * matching the backlog's tags, overwrite the offset IFF the backlog's offset
+   * is greater than the recorded offset.
+   *
+   * @typedef {{[key: string]: string}} BacklogData
+   * @property {number} offset
+   *
+   * @param {BacklogData} backlogData
+   * @returns {Backlog}
+   */
+  forBacklog (backlogData) {
+    const backlog = new Backlog(backlogData)
+    const existingBacklog = this._backlogs.get(backlog.hash)
+    if (existingBacklog !== undefined) {
+      if (existingBacklog.offset > backlog.offset) {
+        return existingBacklog
+      }
+    }
+    this._backlogs.set(backlog.hash, backlog)
+    return backlog
   }
 }
 
+function getSizeOrZero (obj) {
+  if (typeof obj === 'string') {
+    return Buffer.from(obj, 'utf-8').length
+  }
+  if (types.isArrayBuffer(obj)) {
+    return obj.byteLength
+  }
+  if (Buffer.isBuffer(obj)) {
+    return obj.length
+  }
+  return 0
+}
+
+function getHeadersSize (headers) {
+  if (headers === undefined) return 0
+  return Object.entries(headers).reduce((prev, [key, val]) => getSizeOrZero(key) + getSizeOrZero(val) + prev, 0)
+}
+
+function getMessageSize (message) {
+  const { key, value, headers } = message
+  return getSizeOrZero(key) + getSizeOrZero(value) + getHeadersSize(headers)
+}
+
 class TimeBuckets extends Map {
   forTime (time) {
     if (!this.has(time)) {
@@ -92,12 +181,12 @@ class DataStreamsProcessor {
   }
 
   onInterval () {
-    const serialized = this._serializeBuckets()
-    if (!serialized) return
+    const { Stats } = this._serializeBuckets()
+    if (Stats.length === 0) return
     const payload = {
       Env: this.env,
       Service: this.service,
-      Stats: serialized,
+      Stats,
       TracerVersion: pkg.version,
       Version: this.version,
       Lang: 'javascript'
@@ -105,15 +194,28 @@ class DataStreamsProcessor {
     this.writer.flush(payload)
   }
 
-  recordCheckpoint (checkpoint) {
+  /**
+   * Given a timestamp in nanoseconds, compute and return the closest TimeBucket
+   * @param {number} timestamp
+   * @returns {StatsBucket}
+   */
+  bucketFromTimestamp (timestamp) {
+    const bucketTime = Math.round(timestamp - (timestamp % this.bucketSizeNs))
+    return this.buckets.forTime(bucketTime)
+  }
+
+  recordCheckpoint (checkpoint, span = null) {
     if (!this.enabled) return
-    const bucketTime = Math.round(checkpoint.currentTimestamp - (checkpoint.currentTimestamp % this.bucketSizeNs))
-    this.buckets.forTime(bucketTime)
+    this.bucketFromTimestamp(checkpoint.currentTimestamp)
       .forCheckpoint(checkpoint)
       .addLatencies(checkpoint)
+    // set DSM pathway hash on span to enable related traces feature on DSM tab, convert from buffer to uint64
+    if (span) {
+      span.setTag(PATHWAY_HASH, checkpoint.hash.readBigUInt64BE(0).toString())
+    }
   }
 
-  setCheckpoint (edgeTags, ctx = null) {
+  setCheckpoint (edgeTags, span, ctx = null, payloadSize = 0) {
     if (!this.enabled) return null
     const nowNs = Date.now() * 1e6
     const direction = edgeTags.find(t => t.startsWith('direction:'))
@@ -147,45 +249,78 @@ class DataStreamsProcessor {
     const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash)
     const edgeLatencyNs = nowNs - edgeStartNs
     const pathwayLatencyNs = nowNs - pathwayStartNs
+    const dataStreamsContext = {
+      hash: hash,
+      edgeStartNs: edgeStartNs,
+      pathwayStartNs: pathwayStartNs,
+      previousDirection: direction,
+      closestOppositeDirectionHash: closestOppositeDirectionHash,
+      closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart
+    }
+    if (direction === 'direction:out') {
+      // Add the header for this now, as the callee doesn't have access to context when producing
+      payloadSize += getSizeOrZero(encodePathwayContext(dataStreamsContext))
+      payloadSize += CONTEXT_PROPAGATION_KEY.length
+    }
     const checkpoint = {
       currentTimestamp: nowNs,
       parentHash: parentHash,
       hash: hash,
       edgeTags: edgeTags,
       edgeLatencyNs: edgeLatencyNs,
-      pathwayLatencyNs: pathwayLatencyNs
+      pathwayLatencyNs: pathwayLatencyNs,
+      payloadSize: payloadSize
     }
-    this.recordCheckpoint(checkpoint)
-    return {
-      hash: hash,
-      edgeStartNs: edgeStartNs,
-      pathwayStartNs: pathwayStartNs,
-      previousDirection: direction,
-      closestOppositeDirectionHash: closestOppositeDirectionHash,
-      closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart
+    this.recordCheckpoint(checkpoint, span)
+    return dataStreamsContext
+  }
+
+  recordOffset ({ timestamp, ...backlogData }) {
+    if (!this.enabled) return
+    return this.bucketFromTimestamp(timestamp)
+      .forBacklog(backlogData)
+  }
+
+  setOffset (offsetObj) {
+    if (!this.enabled) return
+    const nowNs = Date.now() * 1e6
+    const backlogData = {
+      ...offsetObj,
+      timestamp: nowNs
     }
+    this.recordOffset(backlogData)
   }
 
   _serializeBuckets () {
+    // TimeBuckets
     const serializedBuckets = []
 
     for (const [ timeNs, bucket ] of this.buckets.entries()) {
       const points = []
 
-      for (const stats of bucket.values()) {
+      // bucket: StatsBucket
+      // stats: StatsPoint
+      for (const stats of bucket._checkpoints.values()) {
         points.push(stats.encode())
       }
 
+      const backlogs = []
+      for (const backlog of bucket._backlogs.values()) {
+        backlogs.push(backlog.encode())
+      }
       serializedBuckets.push({
         Start: new Uint64(timeNs),
         Duration: new Uint64(this.bucketSizeNs),
-        Stats: points
+        Stats: points,
+        Backlogs: backlogs
       })
     }
 
     this.buckets.clear()
 
-    return serializedBuckets
+    return {
+      Stats: serializedBuckets
+    }
   }
 }
 
@@ -193,6 +328,11 @@ module.exports = {
   DataStreamsProcessor: DataStreamsProcessor,
   StatsPoint: StatsPoint,
   StatsBucket: StatsBucket,
+  Backlog,
   TimeBuckets,
-  ENTRY_PARENT_HASH
+  getMessageSize,
+  getHeadersSize,
+  getSizeOrZero,
+  ENTRY_PARENT_HASH,
+  CONTEXT_PROPAGATION_KEY
 }