datadog 2.12.1 → 2.13.0

data/lib/datadog/tracing/distributed/baggage.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require_relative '../metadata/ext'
+require_relative '../trace_digest'
+require_relative 'datadog_tags_codec'
+require_relative '../utils'
+require_relative 'helpers'
+require 'uri'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # W3C Baggage propagator implementation.
+      # The baggage header is propagated through `baggage`.
+      # @see https://www.w3.org/TR/baggage/
+      class Baggage
+        BAGGAGE_KEY = 'baggage'
+        DD_TRACE_BAGGAGE_MAX_ITEMS = 64
+        DD_TRACE_BAGGAGE_MAX_BYTES = 8192
+        SAFE_CHARACTERS_KEY = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$!#&'*+-.^_`|~"
+        SAFE_CHARACTERS_VALUE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$!#&'()*+-./:<>?@[]^_`{|}~"
+
+        def initialize(
+          fetcher:,
+          baggage_key: BAGGAGE_KEY
+        )
+          @baggage_key = baggage_key
+          @fetcher = fetcher
+        end
+
+        def inject!(digest, data)
+          return if digest.nil? || digest.baggage.nil?
+
+          baggage_items = digest.baggage.reject { |k, v| k.nil? || v.nil? }
+          return if baggage_items.empty?
+
+          begin
+            if baggage_items.size > DD_TRACE_BAGGAGE_MAX_ITEMS
+              ::Datadog.logger.warn('Baggage item limit exceeded, dropping excess items')
+              baggage_items = baggage_items.first(DD_TRACE_BAGGAGE_MAX_ITEMS)
+            end
+
+            encoded_items = []
+            total_size = 0
+
+            baggage_items.each do |key, value|
+              item = "#{encode_item(key, SAFE_CHARACTERS_KEY)}=#{encode_item(value, SAFE_CHARACTERS_VALUE)}"
+              item_size = item.bytesize + (encoded_items.empty? ? 0 : 1) # +1 for comma if not first item
+              if total_size + item_size > DD_TRACE_BAGGAGE_MAX_BYTES
+                ::Datadog.logger.warn('Baggage header size exceeded, dropping excess items')
+                break # stop adding items when size limit is reached
+              end
+              encoded_items << item
+              total_size += item_size
+            end
+
+            # edge case where a single item is too large
+            return if encoded_items.empty?
+
+            header_value = encoded_items.join(',')
+            data[@baggage_key] = header_value
+          rescue => e
+            ::Datadog.logger.warn("Failed to encode and inject baggage header: #{e.message}")
+          end
+        end
+
+        def extract(data)
+          fetcher = @fetcher.new(data)
+          data = fetcher[@baggage_key]
+          return unless data
+
+          baggage = parse_baggage_header(fetcher[@baggage_key])
+          return unless baggage
+
+          TraceDigest.new(
+            baggage: baggage,
+          )
+        end
+
+        private
+
+        def encode_item(item, safe_characters)
+          # Strip whitespace and URL-encode the item
+          result = URI.encode_www_form_component(item.strip)
+          # Replace '+' with '%20' for space encoding consistency with W3C spec
+          result = result.gsub('+', '%20')
+          # Selectively decode percent-encoded characters that are considered "safe" in W3C Baggage spec
+          result.gsub(/%[0-9A-F]{2}/) do |encoded|
+            if encoded.size >= 3 && encoded[1..2] =~ /\A[0-9A-F]{2}\z/
+              hex_str = encoded[1..2]
+              next encoded unless hex_str && !hex_str.empty?
+
+              # Convert hex representation back to character
+              char = [hex_str.hex].pack('C')
+              # Keep the character as-is if it's in the safe character set, otherwise keep it encoded
+              safe_characters.include?(char) ? char : encoded
+            else
+              encoded
+            end
+          end
+        end
+
+        # Parses a W3C Baggage header string into a hash of key-value pairs
+        # The header format follows the W3C Baggage specification:
+        # - Multiple baggage items are separated by commas
+        # - Each baggage item is a key-value pair separated by '='
+        # - Keys and values are URL-encoded
+        # - Returns an empty hash if the baggage header is malformed
+        #
+        # @param baggage_header [String] The W3C Baggage header string to parse
+        # @return [Hash<String, String>] A hash of decoded baggage items
+        def parse_baggage_header(baggage_header)
+          baggage = {}
+          baggages = baggage_header.split(',')
+          baggages.each do |key_value|
+            key, value = key_value.split('=', 2)
+            # If baggage is malformed, return an empty hash
+            return {} unless key && value
+
+            key = URI.decode_www_form_component(key.strip)
+            value = URI.decode_www_form_component(value.strip)
+            return {} if key.empty? || value.empty?
+
+            baggage[key] = value
+          end
+          baggage
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/datadog.rb

@@ -116,6 +116,8 @@ module Datadog
         def extract_trace_id!(trace_id, tags)
           return trace_id unless tags
           return trace_id unless (high_order = tags.delete(Tracing::Metadata::Ext::Distributed::TAG_TID))
+          return trace_id unless high_order.size == 16
+          return trace_id unless /\A[0-9a-f]+\z/i.match?(high_order)
 
           Tracing::Utils::TraceId.concatenate(high_order.to_i(16), trace_id)
         end

data/lib/datadog/tracing/distributed/propagation.rb

@@ -4,6 +4,7 @@ require_relative '../configuration/ext'
 require_relative '../trace_digest'
 require_relative '../trace_operation'
 require_relative '../../core/telemetry/logger'
+require_relative 'baggage'
 
 module Datadog
   module Tracing
@@ -26,9 +27,13 @@ module Datadog
         )
           @propagation_styles = propagation_styles
           @propagation_extract_first = propagation_extract_first
-
           @propagation_style_inject = propagation_style_inject.map { |style| propagation_styles[style] }
           @propagation_style_extract = propagation_style_extract.map { |style| propagation_styles[style] }
+
+          # The baggage propagator is unique in that baggage should always be extracted, if present.
+          # Therefore we remove it from the `propagation_style_extract` list.
+          @baggage_propagator = @propagation_style_extract.find { |propagator| propagator.is_a?(Baggage) }
+          @propagation_style_extract.delete(@baggage_propagator) if @baggage_propagator
         end
 
         # inject! populates the env with span ID, trace ID and sampling priority
@@ -57,9 +62,8 @@ module Datadog
           end
 
           digest = digest.to_digest if digest.respond_to?(:to_digest)
-
-          if digest.trace_id.nil?
-            ::Datadog.logger.debug('Cannot inject distributed trace data: digest.trace_id is nil.')
+          if digest.trace_id.nil? && digest.baggage.nil?
+            ::Datadog.logger.debug('Cannot inject distributed trace data: digest.trace_id and digest.baggage are both nil.')
             return nil
           end
 
@@ -138,12 +142,29 @@ module Datadog
               "Error extracting distributed trace data. Cause: #{e} Location: #{Array(e.backtrace).first}"
             )
           end
+          # Handle baggage after all other styles if present
+          extracted_trace_digest = propagate_baggage(data, extracted_trace_digest) if @baggage_propagator
 
           extracted_trace_digest
         end
 
         private
 
+        def propagate_baggage(data, extracted_trace_digest)
+          if extracted_trace_digest
+            # Merge with baggage if present
+            digest = @baggage_propagator.extract(data)
+            if digest
+              extracted_trace_digest.merge(baggage: digest.baggage)
+            else
+              extracted_trace_digest
+            end
+          else
+            # Baggage is the only style
+            @baggage_propagator.extract(data)
+          end
+        end
+
         def last_datadog_parent_id(headers, tracecontext_tags)
           dd_propagator = @propagation_style_extract.find { |propagator| propagator.is_a?(Datadog) }
           if tracecontext_tags&.fetch(

data/lib/datadog/tracing/distributed/propagation_policy.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Helper method to decide when to skip distributed tracing
+      module PropagationPolicy
+        module_function
+
+        # Skips distributed tracing if disabled for this instrumentation
+        # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+        #
+        # Both pin_config and global_config are configuration for integrations.
+        # pin_config is a Datadog::Core::Pin object, which gives the configuration of a single instance of an integration.
+        # global_config is the config for all instances of an integration.
+        def enabled?(pin_config: nil, global_config: nil, trace: nil)
+          return false unless Tracing.enabled?
+
+          unless ::Datadog.configuration.apm.tracing.enabled
+            return false if trace.nil?
+
+            trace_source = trace.get_tag(::Datadog::Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE)&.to_i(16)
+            return false if trace_source.nil?
+
+            # If AppSec is enabled and AppSec bit is set in the trace, we should not skip distributed tracing
+            # Other products that will use dd.p.ts should implement similar behavior here
+            if ::Datadog.configuration.appsec.enabled && (trace_source & ::Datadog::AppSec::Ext::PRODUCT_BIT) != 0
+              return true
+            end
+
+            return false
+          end
+
+          return pin_config[:distributed_tracing] if pin_config && pin_config.key?(:distributed_tracing)
+          return global_config[:distributed_tracing] if global_config
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/ext.rb

@@ -31,6 +31,8 @@ module Datadog
         # See Datadog-internal "RFC: Identifying which spans have profiling enabled " for details
         TAG_PROFILING_ENABLED = '_dd.profiling.enabled'
 
+        TAG_APM_ENABLED = '_dd.apm.enabled'
+
         # Defines constants for trace analytics
         # @public_api
         module Analytics
@@ -55,6 +57,9 @@ module Datadog
           # @see Datadog::Tracing::Sampling::Ext::Mechanism
           TAG_DECISION_MAKER = '_dd.p.dm'
 
+          # Bitmask for which product generated an event. E.g.: 2 for an AppSec event.
+          TAG_TRACE_SOURCE = '_dd.p.ts'
+
           TAG_ORIGIN = '_dd.origin'
           TAG_SAMPLING_PRIORITY = '_sampling_priority_v1'
 

data/lib/datadog/tracing/metadata/metastruct.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # This class is a data structure that is used to store
+      # complex metadata, such as an array of objects.
+      #
+      # It is serialized to MessagePack format when sent to the agent.
+      class Metastruct
+        extend Forwardable
+
+        def_delegators :@metastruct, :[], :[]=, :to_h
+
+        def initialize
+          @metastruct = {}
+        end
+
+        def to_msgpack(packer = nil)
+          # JRuby doesn't pass the packer
+          packer ||= MessagePack::Packer.new
+
+          packer.write(@metastruct.transform_values(&:to_msgpack))
+        end
+
+        def pretty_print(q)
+          q.seplist @metastruct.each do |key, value|
+            q.text "#{key} => #{value}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/metastruct_tagging.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'metastruct'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # Adds data storage for the `meta_struct` field.
+      #
+      # This field is used to send more complex data like an array of objects
+      # in MessagePack format to the agent, and has no size limitations.
+      #
+      # The agent fully supports meta_struct from version v7.35.0 (April 2022).
+      #
+      # On versions older than v7.35.0, sending traces containing meta_struct
+      # has no unexpected side-effects; traces are sent to the backend as expected,
+      # while the meta_struct field is stripped.
+      module MetastructTagging
+        # Set the given key / value tag pair on the metastruct.
+        #
+        # A valid example is:
+        #
+        #   span.set_metastruct_tag('_dd.stack', [])
+        def set_metastruct_tag(key, value)
+          metastruct[key] = value
+        end
+
+        # Return the metastruct tag value for the given key,
+        # returns nil if the key doesn't exist.
+        def get_metastruct_tag(key)
+          metastruct[key]
+        end
+
+        private
+
+        def metastruct
+          @metastruct ||= Metastruct.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata.rb

@@ -2,6 +2,7 @@
 
 require_relative 'metadata/analytics'
 require_relative 'metadata/tagging'
+require_relative 'metadata/metastruct_tagging'
 require_relative 'metadata/errors'
 
 module Datadog
@@ -10,6 +11,7 @@ module Datadog
     module Metadata
       def self.included(base)
         base.include(Metadata::Tagging)
+        base.include(Metadata::MetastructTagging)
         base.include(Metadata::Errors)
 
         # Additional extensions

data/lib/datadog/tracing/sampling/span/rule.rb

@@ -24,7 +24,6 @@ module Datadog
             sample_rate: Span::Ext::DEFAULT_SAMPLE_RATE,
             rate_limit: Span::Ext::DEFAULT_MAX_PER_SECOND
           )
-
             @matcher = matcher
             @sample_rate = sample_rate
             @rate_limit = rate_limit

data/lib/datadog/tracing/span.rb

@@ -33,6 +33,9 @@ module Datadog
         :status,
         :trace_id
 
+      attr_reader \
+        :metastruct
+
       attr_writer \
         :duration
 
@@ -54,6 +57,7 @@ module Datadog
         id: nil,
         meta: nil,
         metrics: nil,
+        metastruct: nil,
         parent_id: 0,
         resource: name,
         service: nil,
@@ -76,6 +80,7 @@ module Datadog
 
         @meta = meta || {}
         @metrics = metrics || {}
+        @metastruct = metastruct || {}
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock
@@ -144,6 +149,7 @@ module Datadog
           error: @status,
           meta: @meta,
           metrics: @metrics,
+          meta_struct: @metastruct.to_h,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -185,12 +191,15 @@ module Datadog
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist @metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 

data/lib/datadog/tracing/span_operation.rb

@@ -269,7 +269,8 @@ module Datadog
 
       def duration
         return @duration_end - @duration_start if @duration_start && @duration_end
-        return @end_time - @start_time if @start_time && @end_time
+
+        @end_time - @start_time if @start_time && @end_time
       end
 
       def set_error(e)
@@ -289,6 +290,7 @@ module Datadog
           id: @id,
           meta: meta,
           metrics: metrics,
+          metastruct: metastruct,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -328,12 +330,15 @@ module Datadog
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 
@@ -456,6 +461,7 @@ module Datadog
           id: @id,
           meta: Core::Utils::SafeDup.frozen_or_dup(meta),
           metrics: Core::Utils::SafeDup.frozen_or_dup(metrics),
+          metastruct: Core::Utils::SafeDup.frozen_or_dup(metastruct),
           parent_id: @parent_id,
           resource: @resource,
           service: @service,

data/lib/datadog/tracing/sync_writer.rb

@@ -32,8 +32,7 @@ module Datadog
         @agent_settings = agent_settings
 
         @transport = transport || begin
-          transport_options = transport_options.merge(agent_settings: agent_settings) if agent_settings
-          Transport::HTTP.default(**transport_options)
+          Transport::HTTP.default(agent_settings: agent_settings, logger: logger, **transport_options)
         end
 
         @events = Writer::Events.new

data/lib/datadog/tracing/trace_digest.rb

@@ -80,6 +80,9 @@ module Datadog
       #   This allows later propagation to include those unknown fields, as they can represent future versions of the spec
       #   sending data through this service. This value ends in a trailing `;` to facilitate serialization.
       #   @return [String]
+      # @!attribute [r] baggage
+      #   The W3C "baggage" extracted from a distributed context. This field is a hash of key/value pairs.
+      #   @return [Hash<String,String>]
       # TODO: The documentation for the last attribute above won't be rendered.
       # TODO: This might be a YARD bug as adding an attribute, making it now second-last attribute, renders correctly.
       attr_reader \
@@ -102,7 +105,8 @@ module Datadog
         :trace_flags,
         :trace_state,
         :trace_state_unknown_fields,
-        :span_remote
+        :span_remote,
+        :baggage
 
       def initialize(
         span_id: nil,
@@ -124,7 +128,8 @@ module Datadog
         trace_flags: nil,
         trace_state: nil,
         trace_state_unknown_fields: nil,
-        span_remote: true
+        span_remote: true,
+        baggage: nil
       )
         @span_id = span_id
         @span_name = span_name && span_name.dup.freeze
@@ -146,6 +151,7 @@ module Datadog
         @trace_state = trace_state && trace_state.dup.freeze
         @trace_state_unknown_fields = trace_state_unknown_fields && trace_state_unknown_fields.dup.freeze
         @span_remote = span_remote
+        @baggage = baggage && baggage.dup.freeze
         freeze
       end
 
@@ -177,6 +183,7 @@ module Datadog
             trace_state: trace_state,
             trace_state_unknown_fields: trace_state_unknown_fields,
             span_remote: span_remote,
+            baggage: baggage
           }.merge!(field_value_pairs)
         )
       end

data/lib/datadog/tracing/trace_operation.rb

@@ -2,7 +2,6 @@
 
 require_relative '../core/environment/identity'
 require_relative '../core/utils'
-require_relative 'tracer'
 require_relative 'event'
 require_relative 'metadata/tagging'
 require_relative 'sampling/ext'
@@ -37,7 +36,8 @@ module Datadog
         :rule_sample_rate,
         :sample_rate,
         :sampling_priority,
-        :remote_parent
+        :remote_parent,
+        :baggage
 
       attr_reader \
         :active_span_count,
@@ -71,19 +71,21 @@ module Datadog
         sampling_priority: nil,
         service: nil,
         profiling_enabled: nil,
+        apm_tracing_enabled: nil,
         tags: nil,
         metrics: nil,
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
-        tracer: nil
+        tracer: nil,
+        baggage: nil
 
       )
         # Attributes
         @id = id || Tracing::Utils::TraceId.next_id
         @max_length = max_length || DEFAULT_MAX_LENGTH
         @parent_span_id = parent_span_id
-        @sampled = sampled.nil? ? true : sampled
+        @sampled = sampled.nil? || sampled
         @remote_parent = remote_parent
 
         # Tags
@@ -98,9 +100,11 @@ module Datadog
         @sampling_priority = sampling_priority
         @service = service
         @profiling_enabled = profiling_enabled
+        @apm_tracing_enabled = apm_tracing_enabled
         @trace_state = trace_state
         @trace_state_unknown_fields = trace_state_unknown_fields
         @tracer = tracer
+        @baggage = baggage
 
         # Generic tags
         set_tags(tags) if tags
@@ -173,6 +177,12 @@ module Datadog
         super || (root_span && root_span.get_metric(key))
       end
 
+      def set_distributed_source(product_bit)
+        source = get_tag(Metadata::Ext::Distributed::TAG_TRACE_SOURCE)&.to_i(16) || 0
+        source |= product_bit
+        set_tag(Metadata::Ext::Distributed::TAG_TRACE_SOURCE, format('%02X', source))
+      end
+
       def tags
         all_tags = {}
         all_tags.merge!(root_span&.tags || {}) if root_span
@@ -315,10 +325,10 @@ module Datadog
 
         TraceDigest.new(
           span_id: span_id,
-          span_name: (@active_span && @active_span.name),
-          span_resource: (@active_span && @active_span.resource),
-          span_service: (@active_span && @active_span.service),
-          span_type: (@active_span && @active_span.type),
+          span_name: @active_span && @active_span.name,
+          span_resource: @active_span && @active_span.resource,
+          span_service: @active_span && @active_span.service,
+          span_type: @active_span && @active_span.type,
           trace_distributed_tags: distributed_tags,
           trace_hostname: @hostname,
           trace_id: @id,
@@ -331,7 +341,8 @@ module Datadog
           trace_service: service,
           trace_state: @trace_state,
           trace_state_unknown_fields: @trace_state_unknown_fields,
-          span_remote: (@remote_parent && @active_span.nil?),
+          span_remote: @remote_parent && @active_span.nil?,
+          baggage: @baggage.nil? || @baggage.empty? ? nil : @baggage
         ).freeze
       end
 
@@ -351,22 +362,22 @@ module Datadog
       def fork_clone
         self.class.new(
           agent_sample_rate: @agent_sample_rate,
-          events: (@events && @events.dup),
-          hostname: (@hostname && @hostname.dup),
+          events: @events && @events.dup,
+          hostname: @hostname && @hostname.dup,
           id: @id,
           max_length: @max_length,
-          name: (name && name.dup),
-          origin: (@origin && @origin.dup),
+          name: name && name.dup,
+          origin: @origin && @origin.dup,
           parent_span_id: (@active_span && @active_span.id) || @parent_span_id,
           rate_limiter_rate: @rate_limiter_rate,
-          resource: (resource && resource.dup),
+          resource: resource && resource.dup,
           rule_sample_rate: @rule_sample_rate,
           sample_rate: @sample_rate,
           sampled: @sampled,
           sampling_priority: @sampling_priority,
-          service: (service && service.dup),
-          trace_state: (@trace_state && @trace_state.dup),
-          trace_state_unknown_fields: (@trace_state_unknown_fields && @trace_state_unknown_fields.dup),
+          service: service && service.dup,
+          trace_state: @trace_state && @trace_state.dup,
+          trace_state_unknown_fields: @trace_state_unknown_fields && @trace_state_unknown_fields.dup,
           tags: meta.dup,
           metrics: metrics.dup,
           remote_parent: @remote_parent
@@ -510,6 +521,7 @@ module Datadog
           metrics: metrics,
           root_span_id: !partial ? root_span && root_span.id : nil,
           profiling_enabled: @profiling_enabled,
+          apm_tracing_enabled: @apm_tracing_enabled
         )
       end