datadog 2.7.1 → 2.18.0

data/lib/datadog/tracing/distributed/baggage.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require_relative '../metadata/ext'
+require_relative '../trace_digest'
+require_relative 'datadog_tags_codec'
+require_relative '../utils'
+require_relative 'helpers'
+require 'uri'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # W3C Baggage propagator implementation.
+      # The baggage header is propagated through `baggage`.
+      # @see https://www.w3.org/TR/baggage/
+      class Baggage
+        BAGGAGE_KEY = 'baggage'
+        DD_TRACE_BAGGAGE_MAX_ITEMS = 64
+        DD_TRACE_BAGGAGE_MAX_BYTES = 8192
+        SAFE_CHARACTERS_KEY = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$!#&'*+-.^_`|~"
+        SAFE_CHARACTERS_VALUE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789$!#&'()*+-./:<>?@[]^_`{|}~"
+
+        def initialize(
+          fetcher:,
+          baggage_key: BAGGAGE_KEY
+        )
+          @baggage_key = baggage_key
+          @fetcher = fetcher
+        end
+
+        def inject!(digest, data)
+          return if digest.nil? || digest.baggage.nil?
+
+          baggage_items = digest.baggage.reject { |k, v| k.nil? || v.nil? }
+          return if baggage_items.empty?
+
+          begin
+            if baggage_items.size > DD_TRACE_BAGGAGE_MAX_ITEMS
+              ::Datadog.logger.warn('Baggage item limit exceeded, dropping excess items')
+              baggage_items = baggage_items.first(DD_TRACE_BAGGAGE_MAX_ITEMS)
+            end
+
+            encoded_items = []
+            total_size = 0
+
+            baggage_items.each do |key, value|
+              item = "#{encode_item(key, SAFE_CHARACTERS_KEY)}=#{encode_item(value, SAFE_CHARACTERS_VALUE)}"
+              item_size = item.bytesize + (encoded_items.empty? ? 0 : 1) # +1 for comma if not first item
+              if total_size + item_size > DD_TRACE_BAGGAGE_MAX_BYTES
+                ::Datadog.logger.warn('Baggage header size exceeded, dropping excess items')
+                break # stop adding items when size limit is reached
+              end
+              encoded_items << item
+              total_size += item_size
+            end
+
+            # edge case where a single item is too large
+            return if encoded_items.empty?
+
+            header_value = encoded_items.join(',')
+            data[@baggage_key] = header_value
+          rescue => e
+            ::Datadog.logger.warn("Failed to encode and inject baggage header: #{e.message}")
+          end
+        end
+
+        def extract(data)
+          fetcher = @fetcher.new(data)
+          data = fetcher[@baggage_key]
+          return unless data
+
+          baggage = parse_baggage_header(fetcher[@baggage_key])
+          return unless baggage
+
+          TraceDigest.new(
+            baggage: baggage,
+          )
+        end
+
+        private
+
+        def encode_item(item, safe_characters)
+          # Strip whitespace and URL-encode the item
+          result = URI.encode_www_form_component(item.strip)
+          # Replace '+' with '%20' for space encoding consistency with W3C spec
+          result = result.gsub('+', '%20')
+          # Selectively decode percent-encoded characters that are considered "safe" in W3C Baggage spec
+          result.gsub(/%[0-9A-F]{2}/) do |encoded|
+            if encoded.size >= 3 && encoded[1..2] =~ /\A[0-9A-F]{2}\z/
+              hex_str = encoded[1..2]
+              next encoded unless hex_str && !hex_str.empty?
+
+              # Convert hex representation back to character
+              char = [hex_str.hex].pack('C')
+              # Keep the character as-is if it's in the safe character set, otherwise keep it encoded
+              safe_characters.include?(char) ? char : encoded
+            else
+              encoded
+            end
+          end
+        end
+
+        # Parses a W3C Baggage header string into a hash of key-value pairs
+        # The header format follows the W3C Baggage specification:
+        # - Multiple baggage items are separated by commas
+        # - Each baggage item is a key-value pair separated by '='
+        # - Keys and values are URL-encoded
+        # - Returns an empty hash if the baggage header is malformed
+        #
+        # @param baggage_header [String] The W3C Baggage header string to parse
+        # @return [Hash<String, String>] A hash of decoded baggage items
+        def parse_baggage_header(baggage_header)
+          baggage = {}
+          baggages = baggage_header.split(',')
+          baggages.each do |key_value|
+            key, value = key_value.split('=', 2)
+            # If baggage is malformed, return an empty hash
+            return {} unless key && value
+
+            key = URI.decode_www_form_component(key.strip)
+            value = URI.decode_www_form_component(value.strip)
+            return {} if key.empty? || value.empty?
+
+            baggage[key] = value
+          end
+          baggage
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/datadog.rb

@@ -89,7 +89,7 @@ module Datadog
           trace_id = Helpers.parse_decimal_id(fetcher_object[@trace_id_key])
 
           return unless trace_id
-          return if trace_id <= 0 || trace_id >= Tracing::Utils::EXTERNAL_MAX_ID
+          return if trace_id <= 0 || trace_id > Tracing::Utils::EXTERNAL_MAX_ID
 
           trace_id
         end
@@ -98,7 +98,7 @@ module Datadog
           parent_id = Helpers.parse_decimal_id(fetcher_object[@parent_id_key])
 
           return unless parent_id
-          return if parent_id <= 0 || parent_id >= Tracing::Utils::EXTERNAL_MAX_ID
+          return if parent_id <= 0 || parent_id > Tracing::Utils::EXTERNAL_MAX_ID
 
           parent_id
         end
@@ -116,6 +116,8 @@ module Datadog
         def extract_trace_id!(trace_id, tags)
           return trace_id unless tags
           return trace_id unless (high_order = tags.delete(Tracing::Metadata::Ext::Distributed::TAG_TID))
+          return trace_id unless high_order.size == 16
+          return trace_id unless /\A[0-9a-f]+\z/i.match?(high_order)
 
           Tracing::Utils::TraceId.concatenate(high_order.to_i(16), trace_id)
         end

data/lib/datadog/tracing/distributed/propagation.rb

@@ -4,6 +4,7 @@ require_relative '../configuration/ext'
 require_relative '../trace_digest'
 require_relative '../trace_operation'
 require_relative '../../core/telemetry/logger'
+require_relative 'baggage'
 
 module Datadog
   module Tracing
@@ -26,9 +27,13 @@ module Datadog
         )
           @propagation_styles = propagation_styles
           @propagation_extract_first = propagation_extract_first
-
           @propagation_style_inject = propagation_style_inject.map { |style| propagation_styles[style] }
           @propagation_style_extract = propagation_style_extract.map { |style| propagation_styles[style] }
+
+          # The baggage propagator is unique in that baggage should always be extracted, if present.
+          # Therefore we remove it from the `propagation_style_extract` list.
+          @baggage_propagator = @propagation_style_extract.find { |propagator| propagator.is_a?(Baggage) }
+          @propagation_style_extract.delete(@baggage_propagator) if @baggage_propagator
         end
 
         # inject! populates the env with span ID, trace ID and sampling priority
@@ -57,9 +62,8 @@ module Datadog
           end
 
           digest = digest.to_digest if digest.respond_to?(:to_digest)
-
-          if digest.trace_id.nil?
-            ::Datadog.logger.debug('Cannot inject distributed trace data: digest.trace_id is nil.')
+          if digest.trace_id.nil? && digest.baggage.nil?
+            ::Datadog.logger.debug('Cannot inject distributed trace data: digest.trace_id and digest.baggage are both nil.')
             return nil
           end
 
@@ -138,12 +142,29 @@ module Datadog
               "Error extracting distributed trace data. Cause: #{e} Location: #{Array(e.backtrace).first}"
             )
           end
+          # Handle baggage after all other styles if present
+          extracted_trace_digest = propagate_baggage(data, extracted_trace_digest) if @baggage_propagator
 
           extracted_trace_digest
         end
 
         private
 
+        def propagate_baggage(data, extracted_trace_digest)
+          if extracted_trace_digest
+            # Merge with baggage if present
+            digest = @baggage_propagator.extract(data)
+            if digest
+              extracted_trace_digest.merge(baggage: digest.baggage)
+            else
+              extracted_trace_digest
+            end
+          else
+            # Baggage is the only style
+            @baggage_propagator.extract(data)
+          end
+        end
+
         def last_datadog_parent_id(headers, tracecontext_tags)
           dd_propagator = @propagation_style_extract.find { |propagator| propagator.is_a?(Datadog) }
           if tracecontext_tags&.fetch(

data/lib/datadog/tracing/distributed/propagation_policy.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Helper method to decide when to skip distributed tracing
+      module PropagationPolicy
+        module_function
+
+        # Skips distributed tracing if disabled for this instrumentation
+        # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+        #
+        # Both pin_config and global_config are configuration for integrations.
+        # pin_config is a Datadog::Core::Pin object, which gives the configuration of a single instance of an integration.
+        # global_config is the config for all instances of an integration.
+        def enabled?(pin_config: nil, global_config: nil, trace: nil)
+          return false unless Tracing.enabled?
+
+          unless ::Datadog.configuration.apm.tracing.enabled
+            return false if trace.nil?
+
+            trace_source = trace.get_tag(::Datadog::Tracing::Metadata::Ext::Distributed::TAG_TRACE_SOURCE)&.to_i(16)
+            return false if trace_source.nil?
+
+            # If AppSec is enabled and AppSec bit is set in the trace, we should not skip distributed tracing
+            # Other products that will use dd.p.ts should implement similar behavior here
+            if ::Datadog.configuration.appsec.enabled && (trace_source & ::Datadog::AppSec::Ext::PRODUCT_BIT) != 0
+              return true
+            end
+
+            return false
+          end
+
+          return pin_config[:distributed_tracing] if pin_config && pin_config.key?(:distributed_tracing)
+          return global_config[:distributed_tracing] if global_config
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/errors.rb

@@ -10,17 +10,17 @@ module Datadog
       # Adds error tagging behavior
       # @public_api
       module Errors
-        def set_error(e)
+        def set_error(error)
           Datadog::Core.log_deprecation do
             'Errors.set_error(..) is deprecated. ' \
             'Use Errors.set_error_tags(..) instead.'
           end
-          set_error_tags(e)
+          set_error_tags(error)
         end
 
         # Mark the span with the given error.
-        def set_error_tags(e)
-          e = Core::Error.build_from(e)
+        def set_error_tags(error)
+          e = Core::Error.build_from(error)
 
           set_tag(Ext::Errors::TAG_TYPE, e.type) unless e.type.empty?
           set_tag(Ext::Errors::TAG_MSG, e.message) unless e.message.empty?

data/lib/datadog/tracing/metadata/ext.rb

@@ -31,6 +31,8 @@ module Datadog
         # See Datadog-internal "RFC: Identifying which spans have profiling enabled " for details
         TAG_PROFILING_ENABLED = '_dd.profiling.enabled'
 
+        TAG_APM_ENABLED = '_dd.apm.enabled'
+
         # Defines constants for trace analytics
         # @public_api
         module Analytics
@@ -55,6 +57,9 @@ module Datadog
           # @see Datadog::Tracing::Sampling::Ext::Mechanism
           TAG_DECISION_MAKER = '_dd.p.dm'
 
+          # Bitmask for which product generated an event. E.g.: 2 for an AppSec event.
+          TAG_TRACE_SOURCE = '_dd.p.ts'
+
           TAG_ORIGIN = '_dd.origin'
           TAG_SAMPLING_PRIORITY = '_sampling_priority_v1'
 

data/lib/datadog/tracing/metadata/metastruct.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # This class is a data structure that is used to store
+      # complex metadata, such as an array of objects.
+      #
+      # It is serialized to MessagePack format when sent to the agent.
+      class Metastruct
+        extend Forwardable
+
+        def_delegators :@metastruct, :[], :[]=, :to_h
+
+        def initialize
+          @metastruct = {}
+        end
+
+        def to_msgpack(packer = nil)
+          # JRuby doesn't pass the packer
+          packer ||= MessagePack::Packer.new
+
+          packer.write(@metastruct.transform_values(&:to_msgpack))
+        end
+
+        def pretty_print(q)
+          q.seplist @metastruct.each do |key, value|
+            q.text "#{key} => #{value}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/metastruct_tagging.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'metastruct'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # Adds data storage for the `meta_struct` field.
+      #
+      # This field is used to send more complex data like an array of objects
+      # in MessagePack format to the agent, and has no size limitations.
+      #
+      # The agent fully supports meta_struct from version v7.35.0 (April 2022).
+      #
+      # On versions older than v7.35.0, sending traces containing meta_struct
+      # has no unexpected side-effects; traces are sent to the backend as expected,
+      # while the meta_struct field is stripped.
+      module MetastructTagging
+        # Set the given key / value tag pair on the metastruct.
+        #
+        # A valid example is:
+        #
+        #   span.set_metastruct_tag('_dd.stack', [])
+        def set_metastruct_tag(key, value)
+          metastruct[key] = value
+        end
+
+        # Return the metastruct tag value for the given key,
+        # returns nil if the key doesn't exist.
+        def get_metastruct_tag(key)
+          metastruct[key]
+        end
+
+        private
+
+        def metastruct
+          @metastruct ||= Metastruct.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata.rb

@@ -2,6 +2,7 @@
 
 require_relative 'metadata/analytics'
 require_relative 'metadata/tagging'
+require_relative 'metadata/metastruct_tagging'
 require_relative 'metadata/errors'
 
 module Datadog
@@ -10,6 +11,7 @@ module Datadog
     module Metadata
       def self.included(base)
         base.include(Metadata::Tagging)
+        base.include(Metadata::MetastructTagging)
         base.include(Metadata::Errors)
 
         # Additional extensions

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -9,6 +9,7 @@ module Datadog
       # {Datadog::Tracing::Sampling::RateSampler} is based on a sample rate.
       class RateSampler < Sampler
         KNUTH_FACTOR = 1111111111111111111
+        UINT64_MODULO = (1 << 64)
 
         # Initialize a {Datadog::Tracing::Sampling::RateSampler}.
         # This sampler keeps a random subset of the traces. Its main purpose is to
@@ -39,7 +40,7 @@ module Datadog
         end
 
         def sample?(trace)
-          ((trace.id * KNUTH_FACTOR) % Tracing::Utils::EXTERNAL_MAX_ID) <= @sampling_id_threshold
+          ((trace.id * KNUTH_FACTOR) % UINT64_MODULO) <= @sampling_id_threshold
         end
 
         def sample!(trace)

data/lib/datadog/tracing/sampling/span/rule.rb

@@ -24,7 +24,6 @@ module Datadog
             sample_rate: Span::Ext::DEFAULT_SAMPLE_RATE,
             rate_limit: Span::Ext::DEFAULT_MAX_PER_SECOND
           )
-
             @matcher = matcher
             @sample_rate = sample_rate
             @rate_limit = rate_limit

data/lib/datadog/tracing/span.rb

@@ -33,6 +33,9 @@ module Datadog
         :status,
         :trace_id
 
+      attr_reader \
+        :metastruct
+
       attr_writer \
         :duration
 
@@ -54,6 +57,7 @@ module Datadog
         id: nil,
         meta: nil,
         metrics: nil,
+        metastruct: nil,
         parent_id: 0,
         resource: name,
         service: nil,
@@ -76,6 +80,7 @@ module Datadog
 
         @meta = meta || {}
         @metrics = metrics || {}
+        @metastruct = metastruct || {}
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock
@@ -112,7 +117,10 @@ module Datadog
 
       def duration
         return @duration if @duration
-        return @end_time - @start_time if @start_time && @end_time
+
+        start_time = @start_time
+        end_time = @end_time
+        end_time - start_time if start_time && end_time
       end
 
       def set_error(e)
@@ -135,10 +143,13 @@ module Datadog
       # TODO: Change this to reflect attributes when serialization
       # isn't handled by this method.
       def to_hash
+        @meta['events'] = @events.map(&:to_hash).to_json unless @events.empty?
+
         h = {
           error: @status,
           meta: @meta,
           metrics: @metrics,
+          meta_struct: @metastruct.to_h,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -154,8 +165,6 @@ module Datadog
           h[:duration] = duration_nano
         end
 
-        h[:meta]['events'] = @events.map(&:to_hash).to_json unless @events.empty?
-
         h
       end
 
@@ -182,12 +191,15 @@ module Datadog
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist @metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 
@@ -196,12 +208,17 @@ module Datadog
       # Used for serialization
       # @return [Integer] in nanoseconds since Epoch
       def start_time_nano
-        @start_time.to_i * 1000000000 + @start_time.nsec
+        return unless (start_time = @start_time)
+
+        start_time.to_i * 1000000000 + start_time.nsec
       end
 
       # Used for serialization
       # @return [Integer] in nanoseconds since Epoch
       def duration_nano
+        duration = self.duration
+        return unless duration
+
         (duration * 1e9).to_i
       end
 

data/lib/datadog/tracing/span_event.rb

@@ -19,23 +19,143 @@ module Datadog
       #   @return [Integer]
       attr_reader :time_unix_nano
 
+      # TODO: Accept {Time} as the time_unix_nano parameter, possibly in addition to the current nano integer.
       def initialize(
         name,
         attributes: nil,
         time_unix_nano: nil
       )
         @name = name
-        @attributes = attributes || {}
+
+        @attributes = attributes.dup || {}
+        validate_attributes!(@attributes)
+        @attributes.transform_keys!(&:to_s)
+
         # OpenTelemetry SDK stores span event timestamps in nanoseconds (not seconds).
         # We will do the same here to avoid unnecessary conversions and inconsistencies.
-        @time_unix_nano = time_unix_nano || (Time.now.to_r * 1_000_000_000).to_i
+        @time_unix_nano = time_unix_nano || (Core::Utils::Time.now.to_r * 1_000_000_000).to_i
       end
 
+      # Converts the span event into a hash to be used by with the span tag serialization
+      # (`span.set_tag('events) = [event.to_hash]`). This serialization format has the drawback
+      # of being limiting span events to the size limit of a span tag.
+      # All Datadog agents support this format.
       def to_hash
-        h = { name: @name, time_unix_nano: @time_unix_nano }
-        h[:attributes] = attributes unless @attributes.empty?
+        h = { 'name' => @name, 'time_unix_nano' => @time_unix_nano }
+        h['attributes'] = @attributes unless @attributes.empty?
+        h
+      end
+
+      # Converts the span event into a hash to be used by the MessagePack serialization as a
+      # top-level span field (span.span_events = [event.to_native_format]).
+      # This serialization format removes the serialization limitations of the `span.set_tag('events)` approach,
+      # but is only supported by newer version of the Datadog agent.
+      def to_native_format
+        h = { 'name' => @name, 'time_unix_nano' => @time_unix_nano }
+
+        attr = {}
+        @attributes.each do |key, value|
+          attr[key] = if value.is_a?(Array)
+                        { type: ARRAY_TYPE, array_value: value.map { |v| serialize_native_attribute(v) } }
+                      else
+                        serialize_native_attribute(value)
+                      end
+        end
+
+        h['attributes'] = attr unless @attributes.empty?
+
         h
       end
+
+      private
+
+      MIN_INT64_SIGNED = -2**63
+      MAX_INT64_SIGNED = 2 << 63 - 1
+
+      # Checks the attributes hash to ensure it only contains serializable values.
+      # Invalid values are removed from the hash.
+      def validate_attributes!(attributes)
+        attributes.select! do |key, value|
+          case value
+          when Array
+            next true if value.empty?
+
+            first = value.first
+            case first
+            when String, Integer, Float
+              first_type = first.class
+              if value.all? { |v| v.is_a?(first_type) }
+                value.all? { |v| validate_scalar_attribute!(key, v) }
+              else
+                Datadog.logger.warn("Attribute #{key} array must be homogenous: #{value}.")
+                false
+              end
+            when TrueClass, FalseClass
+              if value.all? { |v| v.is_a?(TrueClass) || v.is_a?(FalseClass) }
+                value.all? { |v| validate_scalar_attribute!(key, v) }
+              else
+                Datadog.logger.warn("Attribute #{key} array must be homogenous: #{value}.")
+                false
+              end
+            else
+              Datadog.logger.warn("Attribute #{key} must be a string, number, or boolean: #{value}.")
+              false
+            end
+          when String, Numeric, TrueClass, FalseClass
+            validate_scalar_attribute!(key, value)
+          else
+            Datadog.logger.warn("Attribute #{key} must be a string, number, boolean, or array: #{value}.")
+            false
+          end
+        end
+      end
+
+      def validate_scalar_attribute!(key, value)
+        case value
+        when String, TrueClass, FalseClass
+          true
+        when Integer
+          # Cannot be larger than signed 64-bit integer
+          if value < MIN_INT64_SIGNED || value > MAX_INT64_SIGNED
+            Datadog.logger.warn("Attribute #{key} must be within the range of a signed 64-bit integer: #{value}.")
+            false
+          else
+            true
+          end
+        when Float
+          # Has to be finite
+          return true if value.finite?
+
+          Datadog.logger.warn("Attribute #{key} must be a finite number: #{value}.")
+          false
+        else
+          Datadog.logger.warn("Attribute #{key} must be a string, number, or boolean: #{value}.")
+          false
+        end
+      end
+
+      STRING_TYPE = 0
+      BOOLEAN_TYPE = 1
+      INTEGER_TYPE = 2
+      DOUBLE_TYPE = 3
+      ARRAY_TYPE = 4
+
+      # Serializes individual scalar attributes into the native format.
+      def serialize_native_attribute(value)
+        case value
+        when String
+          { type: STRING_TYPE, string_value: value }
+        when TrueClass, FalseClass
+          { type: BOOLEAN_TYPE, bool_value: value }
+        when Integer
+          { type: INTEGER_TYPE, int_value: value }
+        when Float
+          { type: DOUBLE_TYPE, double_value: value }
+        else
+          # This is technically unreachable due to the validation in #initialize.
+          raise ArgumentError, "Attribute must be a string, number, or boolean: #{value}."
+        end
+      end
     end
   end
 end