datadog 2.0.0.beta1

data/lib/datadog/tracing/distributed/datadog.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require_relative '../metadata/ext'
+require_relative '../trace_digest'
+require_relative 'datadog_tags_codec'
+require_relative '../utils'
+require_relative 'helpers'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Datadog-style trace propagation.
+      class Datadog
+        TRACE_ID_KEY = 'x-datadog-trace-id'
+        PARENT_ID_KEY = 'x-datadog-parent-id'
+        SAMPLING_PRIORITY_KEY = 'x-datadog-sampling-priority'
+        ORIGIN_KEY = 'x-datadog-origin'
+        # Distributed trace-level tags
+        TAGS_KEY = 'x-datadog-tags'
+
+        # Prefix used by all Datadog-specific distributed tags
+        TAGS_PREFIX = 'x-datadog-'
+
+        def initialize(
+          fetcher:,
+          trace_id_key: TRACE_ID_KEY,
+          parent_id_key: PARENT_ID_KEY,
+          sampling_priority_key: SAMPLING_PRIORITY_KEY,
+          origin_key: ORIGIN_KEY,
+          tags_key: TAGS_KEY
+        )
+          @trace_id_key = trace_id_key
+          @parent_id_key = parent_id_key
+          @sampling_priority_key = sampling_priority_key
+          @origin_key = origin_key
+          @tags_key = tags_key
+          @fetcher = fetcher
+        end
+
+        def inject!(digest, data)
+          return if digest.nil?
+
+          data[@trace_id_key] = Tracing::Utils::TraceId.to_low_order(digest.trace_id).to_s
+
+          data[@parent_id_key] = digest.span_id.to_s
+          data[@sampling_priority_key] = digest.trace_sampling_priority.to_s if digest.trace_sampling_priority
+          data[@origin_key] = digest.trace_origin.to_s if digest.trace_origin
+
+          build_tags(digest).tap do |tags|
+            inject_tags!(tags, data) unless tags.empty?
+          end
+          data
+        end
+
+        def extract(data)
+          fetcher = @fetcher.new(data)
+
+          trace_id  = parse_trace_id(fetcher)
+          parent_id = parse_parent_id(fetcher)
+
+          sampling_priority = Helpers.parse_decimal_id(fetcher[@sampling_priority_key])
+          origin = fetcher[@origin_key]
+
+          # Return early if this propagation is not valid
+          # DEV: To be valid we need to have a trace id and a parent id
+          #      or when it is a synthetics trace, just the trace id.
+          # DEV: `Fetcher#id` will not return 0
+          return unless (trace_id && parent_id) || (origin && trace_id)
+
+          trace_distributed_tags = extract_tags(fetcher)
+
+          # If trace id is 128 bits long,
+          # Concatentated high order 64 bit hex-encoded `tid` tag
+          trace_id = extract_trace_id!(trace_id, trace_distributed_tags)
+
+          TraceDigest.new(
+            span_id: parent_id,
+            trace_id: trace_id,
+            trace_origin: origin,
+            trace_sampling_priority: sampling_priority,
+            trace_distributed_tags: trace_distributed_tags,
+            span_remote: true,
+          )
+        end
+
+        private
+
+        def parse_trace_id(fetcher_object)
+          trace_id = Helpers.parse_decimal_id(fetcher_object[@trace_id_key])
+
+          return unless trace_id
+          return if trace_id <= 0 || trace_id >= Tracing::Utils::EXTERNAL_MAX_ID
+
+          trace_id
+        end
+
+        def parse_parent_id(fetcher_object)
+          parent_id = Helpers.parse_decimal_id(fetcher_object[@parent_id_key])
+
+          return unless parent_id
+          return if parent_id <= 0 || parent_id >= Tracing::Utils::EXTERNAL_MAX_ID
+
+          parent_id
+        end
+
+        def build_tags(digest)
+          high_order = Tracing::Utils::TraceId.to_high_order(digest.trace_id)
+          tags = digest.trace_distributed_tags || {}
+
+          return tags if high_order == 0
+
+          tags.merge(Tracing::Metadata::Ext::Distributed::TAG_TID => high_order.to_s(16))
+        end
+
+        # Side effect: Remove high order 64 bit hex-encoded `tid` tag from distributed tags
+        def extract_trace_id!(trace_id, tags)
+          return trace_id unless tags
+          return trace_id unless (high_order = tags.delete(Tracing::Metadata::Ext::Distributed::TAG_TID))
+
+          Tracing::Utils::TraceId.concatenate(high_order.to_i(16), trace_id)
+        end
+
+        # Export trace distributed tags through the `x-datadog-tags` key.
+        #
+        # DEV: This method accesses global state (the active trace) to record its error state as a trace tag.
+        # DEV: This means errors cannot be reported if there's not active span.
+        # DEV: Ideally, we'd have a dedicated error reporting stream for all of datadog.
+        def inject_tags!(tags, data)
+          return set_tags_propagation_error(reason: 'disabled') if tags_disabled?
+
+          encoded_tags = DatadogTagsCodec.encode(tags)
+
+          return set_tags_propagation_error(reason: 'inject_max_size') if tags_too_large?(
+            encoded_tags.size,
+            scenario: 'inject'
+          )
+
+          data[@tags_key] = encoded_tags
+        rescue => e
+          set_tags_propagation_error(reason: 'encoding_error')
+          ::Datadog.logger.warn(
+            "Failed to inject x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+          )
+        end
+
+        # Import `x-datadog-tags` tags as trace distributed tags.
+        # Only tags that have the `_dd.p.` prefix are processed.
+        #
+        # DEV: This method accesses global state (the active trace) to record its error state as a trace tag.
+        # DEV: This means errors cannot be reported if there's not active span.
+        # DEV: Ideally, we'd have a dedicated error reporting stream for all of datadog.
+        def extract_tags(fetcher)
+          tags = fetcher[@tags_key]
+
+          return if !tags || tags.empty?
+          return set_tags_propagation_error(reason: 'disabled') if tags_disabled?
+          return set_tags_propagation_error(reason: 'extract_max_size') if tags_too_large?(tags.size, scenario: 'extract')
+
+          tags_hash = DatadogTagsCodec.decode(tags)
+          # Only extract keys with the expected Datadog prefix
+          tags_hash.select! do |key, _|
+            key.start_with?(Tracing::Metadata::Ext::Distributed::TAGS_PREFIX) && key != EXCLUDED_TAG
+          end
+          tags_hash
+        rescue => e
+          set_tags_propagation_error(reason: 'decoding_error')
+          ::Datadog.logger.warn(
+            "Failed to extract x-datadog-tags: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+          )
+        end
+
+        def set_tags_propagation_error(reason:)
+          active_trace = Tracing.active_trace
+          active_trace.set_tag('_dd.propagation_error', reason) if active_trace
+          nil
+        end
+
+        def tags_disabled?
+          ::Datadog.configuration.tracing.x_datadog_tags_max_length <= 0
+        end
+
+        def tags_too_large?(size, scenario:)
+          return false if size <= ::Datadog.configuration.tracing.x_datadog_tags_max_length
+
+          ::Datadog.logger.warn(
+            "Failed to #{scenario} x-datadog-tags: tags are too large for configured limit (size:#{size} >= " \
+              "limit:#{::Datadog.configuration.tracing.x_datadog_tags_max_length}). This limit can be configured " \
+              'through the DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH environment variable.'
+          )
+
+          true
+        end
+
+        # We want to exclude tags that we don't want to propagate downstream.
+        EXCLUDED_TAG = '_dd.p.upstream_services'
+        private_constant :EXCLUDED_TAG
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/datadog_tags_codec.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Encodes and decodes distributed 'x-datadog-tags' tags for transport
+      # to and from external processes.
+      module DatadogTagsCodec
+        # Backport `Regexp::match?` because it is measurably the most performant
+        # way to check if a string matches a regular expression.
+        module RefineRegexp
+          unless Regexp.method_defined?(:match?)
+            refine ::Regexp do
+              def match?(*args)
+                !match(*args).nil?
+              end
+            end
+          end
+        end
+        using RefineRegexp
+
+        # ASCII characters 32-126, except `,`, `=`, and ` `. At least one character.
+        VALID_KEY_CHARS = /\A(?:(?![,= ])[\u0020-\u007E])+\Z/.freeze
+        # ASCII characters 32-126, except `,`. At least one character.
+        VALID_VALUE_CHARS = /\A(?:(?!,)[\u0020-\u007E])+\Z/.freeze
+
+        # Serializes a {Hash<String,String>} into a `x-datadog-tags`-compatible
+        # String.
+        #
+        # @param tags [Hash<String,String>] trace tag hash
+        # @return [String] serialized tags hash
+        # @raise [EncodingError] if tags cannot be serialized to the `x-datadog-tags` format
+        def self.encode(tags)
+          begin
+            tags.map do |raw_key, raw_value|
+              key = raw_key.to_s
+              value = raw_value.to_s
+
+              raise EncodingError, "Invalid key `#{key}` for value `#{value}`" unless VALID_KEY_CHARS.match?(key)
+              raise EncodingError, "Invalid value `#{value}` for key `#{key}`" unless VALID_VALUE_CHARS.match?(value)
+
+              "#{key}=#{value.strip}"
+            end.join(',')
+          rescue => e
+            raise EncodingError, "Error encoding tags `#{tags}`: `#{e}`"
+          end
+        end
+
+        # Deserializes a `x-datadog-tags`-formatted String into a {Hash<String,String>}.
+        #
+        # @param string [String] tags as serialized by {#encode}
+        # @return [Hash<String,String>] decoded input as a hash of strings
+        # @raise [DecodingError] if string does not conform to the `x-datadog-tags` format
+        def self.decode(string)
+          result = Hash[string.split(',').map do |raw_tag|
+            raw_tag.split('=', 2).tap do |raw_key, raw_value|
+              key = raw_key.to_s
+              value = raw_value.to_s
+
+              raise DecodingError, "Invalid key: #{key}" unless VALID_KEY_CHARS.match?(key)
+              raise DecodingError, "Invalid value: #{value}" unless VALID_VALUE_CHARS.match?(value)
+
+              value.strip!
+            end
+          end]
+
+          raise DecodingError, "Invalid empty tags: #{string}" if result.empty? && !string.empty?
+
+          result
+        end
+
+        # An error occurred during distributed tags encoding.
+        # See {#message} for more information.
+        class EncodingError < StandardError
+        end
+
+        # An error occurred during distributed tags decoding.
+        # See {#message} for more information.
+        class DecodingError < StandardError
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/fetcher.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative 'helpers'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Common fetcher that retrieves fields from a Hash data input
+      class Fetcher
+        # @param data [Hash]
+        def initialize(data)
+          @data = data
+        end
+
+        def [](key)
+          @data[key]
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/helpers.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../sampling/ext'
+require_relative '../utils'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Helpers module provides common helper functions for distributed tracing data
+      module Helpers
+        # Base provides common methods for distributed helper classes
+        def self.clamp_sampling_priority(sampling_priority)
+          # B3 doesn't have our -1 (USER_REJECT) and 2 (USER_KEEP) priorities so convert to acceptable 0/1
+          if sampling_priority < 0
+            sampling_priority = Sampling::Ext::Priority::AUTO_REJECT
+          elsif sampling_priority > 1
+            sampling_priority = Sampling::Ext::Priority::AUTO_KEEP
+          end
+
+          sampling_priority
+        end
+
+        def self.parse_decimal_id(value)
+          return unless value
+
+          value = value.to_s
+
+          num   = value.to_i
+
+          # Ensure the parsed number is the same as the original string value
+          # e.g. We want to make sure to throw away `'nan'.to_i == 0`
+          return unless num.to_s(10) == value
+
+          num
+        end
+
+        def self.parse_hex_id(value)
+          return unless value
+
+          # Lowercase if we want to parse base16 e.g. 3E8 => 3e8
+          # DEV: Ruby will parse `3E8` just fine, but to test
+          #      `num.to_s(base) == value` we need to lowercase
+          value = value.to_s.downcase
+
+          # Remove any leading zeros
+          # DEV: When we call `num.to_s(16)` later Ruby will not add leading zeros
+          #      for us so we want to make sure the comparision will work as expected
+          # DEV: regex, remove all leading zeros up until we find the last 0 in the string
+          #      or we find the first non-zero, this allows `'0000' -> '0'` and `'00001' -> '1'`
+          value = value.sub(/^0*(?=(0$)|[^0])/, '')
+
+          # Convert value to an integer
+          # DEV: Ruby `.to_i` will return `0` if a number could not be parsed
+          num = value.to_i(16)
+
+          # Ensure the parsed number is the same as the original string value
+          # e.g. We want to make sure to throw away `'nan'.to_i == 0`
+          return unless num.to_s(16) == value
+
+          num
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/none.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Propagator that does not inject nor extract data. It performs no operation.
+      # Supported for feature parity with OpenTelemetry.
+      # @see https://github.com/open-telemetry/opentelemetry-specification/blob/255a6c52b8914a2ed7e26bb5585abecab276aafc/specification/sdk-environment-variables.md?plain=1#L88
+      class None
+        # No-op
+        def inject!(_digest, _data); end
+
+        # No-op
+        def extract(_data); end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/propagation.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative '../configuration/ext'
+require_relative '../trace_digest'
+require_relative '../trace_operation'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Provides extraction and injection of distributed trace data.
+      class Propagation
+        # @param propagation_styles [Hash<String,Object>]
+        #  a map of propagation styles to their corresponding implementations
+        # @param propagation_style_inject [Array<String>]
+        #   a list of styles to use when injecting distributed trace data
+        # @param propagation_style_extract [Array<String>]
+        #   a list of styles to use when extracting distributed trace data
+        # @param propagation_extract_first [Boolean]
+        #   if true, only the first successfully extracted trace will be used
+        def initialize(
+          propagation_styles:,
+          propagation_style_inject:,
+          propagation_style_extract:,
+          propagation_extract_first:
+        )
+          @propagation_styles = propagation_styles
+          @propagation_extract_first = propagation_extract_first
+
+          @propagation_style_inject = propagation_style_inject.map { |style| propagation_styles[style] }
+          @propagation_style_extract = propagation_style_extract.map { |style| propagation_styles[style] }
+        end
+
+        # inject! populates the env with span ID, trace ID and sampling priority
+        #
+        # This method will never raise errors, but instead log them to `Datadog.logger`.
+        #
+        # DEV-2.0: inject! should work without arguments, injecting the active_trace's digest
+        # DEV-2.0: and returning a new Hash with the injected data.
+        # DEV-2.0: inject! should also accept either a `trace` or a `digest`, as a `trace`
+        # DEV-2.0: argument is the common use case, but also allows us to set error tags in the `trace`
+        # DEV-2.0: if needed.
+        # DEV-2.0: Ideally, we'd have a separate stream to report tracer errors and never
+        # DEV-2.0: touch the active span.
+        #
+        # @param digest [TraceDigest]
+        # @param data [Hash]
+        # @return [Boolean] `true` if injected successfully, `false` if no propagation style is configured
+        # @return [nil] in case of error, see `Datadog.logger` output for details.
+        def inject!(digest, data)
+          if digest.nil?
+            ::Datadog.logger.debug('Cannot inject distributed trace data: digest is nil.')
+            return nil
+          end
+
+          digest = digest.to_digest if digest.respond_to?(:to_digest)
+
+          result = false
+
+          # Inject all configured propagation styles
+          @propagation_style_inject.each do |propagator|
+            propagator.inject!(digest, data)
+            result = true
+          rescue => e
+            result = nil
+            ::Datadog.logger.error(
+              "Error injecting distributed trace data. Cause: #{e} Location: #{Array(e.backtrace).first}"
+            )
+          end
+
+          result
+        end
+
+        # extract returns {TraceDigest} containing the distributed trace information.
+        # sampling priority defined in data.
+        #
+        # This method will never raise errors, but instead log them to `Datadog.logger`.
+        #
+        # @param data [Hash]
+        def extract(data)
+          return unless data
+          return if data.empty?
+
+          extracted_trace_digest = nil
+
+          @propagation_style_extract.each do |propagator|
+            # First extraction?
+            unless extracted_trace_digest
+              extracted_trace_digest = propagator.extract(data)
+              next
+            end
+
+            # Return if we are only inspecting the first valid style.
+            next if @propagation_extract_first
+
+            # Continue parsing styles to find the W3C `tracestate` header, if present.
+            # `tracestate` must always be propagated, as it might contain pass-through data that we don't control.
+            # @see https://www.w3.org/TR/2021/REC-trace-context-1-20211123/#mutating-the-tracestate-field
+            next unless propagator.is_a?(TraceContext)
+
+            if (tracecontext_digest = propagator.extract(data))
+              # Only parse if it represent the same trace as the successfully extracted one
+              next unless tracecontext_digest.trace_id == extracted_trace_digest.trace_id
+
+              # Preserve the `tracestate`
+              extracted_trace_digest = extracted_trace_digest.merge(
+                trace_state: tracecontext_digest.trace_state,
+                trace_state_unknown_fields: tracecontext_digest.trace_state_unknown_fields
+              )
+            end
+          rescue => e
+            ::Datadog.logger.error(
+              "Error extracting distributed trace data. Cause: #{e} Location: #{Array(e.backtrace).first}"
+            )
+          end
+
+          extracted_trace_digest
+        end
+      end
+    end
+  end
+end