ddtrace 1.6.1 → 1.8.0

data/lib/datadog/tracing/distributed/trace_context.rb

@@ -0,0 +1,369 @@
+# frozen_string_literal: true
+# typed: false
+
+module Datadog
+  module Tracing
+    module Distributed
+      # W3C Trace Context propagator implementation, version 00.
+      # The trace is propagated through two fields: `traceparent` and `tracestate`.
+      # @see https://www.w3.org/TR/trace-context/
+      class TraceContext
+        TRACEPARENT_KEY = 'traceparent'
+        TRACESTATE_KEY = 'tracestate'
+        SPEC_VERSION = '00'
+
+        def initialize(
+          fetcher:,
+          traceparent_key: TRACEPARENT_KEY,
+          tracestate_key: TRACESTATE_KEY
+        )
+          @fetcher = fetcher
+          @traceparent_key = traceparent_key
+          @tracestate_key = tracestate_key
+        end
+
+        def inject!(digest, data)
+          return if digest.nil?
+
+          if (traceparent = build_traceparent(digest))
+            data[@traceparent_key] = traceparent
+
+            if (tracestate = build_tracestate(digest))
+              data[@tracestate_key] = tracestate
+            end
+          end
+
+          data
+        end
+
+        def extract(data)
+          fetcher = @fetcher.new(data)
+
+          trace_id, dd_trace_id, parent_id, sampled, trace_flags = extract_traceparent(fetcher[@traceparent_key])
+
+          return unless trace_id # Could not parse traceparent
+
+          tracestate, sampling_priority, origin, tags = extract_tracestate(fetcher[@tracestate_key])
+
+          sampling_priority = parse_priority_sampling(sampled, sampling_priority)
+
+          TraceDigest.new(
+            span_id: parent_id,
+            trace_id: dd_trace_id,
+            trace_origin: origin,
+            trace_sampling_priority: sampling_priority,
+            trace_distributed_tags: tags,
+            trace_distributed_id: trace_id,
+            trace_state: tracestate,
+            trace_flags: trace_flags
+          )
+        end
+
+        private
+
+        # Refinements to ensure newer rubies do not suffer performance impact
+        # by needing to use older APIs.
+        module Refine
+          # Backport `Regexp::match?` because it is measurably the most performant
+          # way to check if a string matches a regular expression.
+          unless Regexp.method_defined?(:match?)
+            refine ::Regexp do
+              def match?(*args)
+                !match(*args).nil?
+              end
+            end
+          end
+
+          unless String.method_defined?(:delete_prefix)
+            refine ::String do
+              def delete_prefix(prefix)
+                prefix = prefix.to_s
+                if rindex(prefix, 0)
+                  self[prefix.length..-1]
+                else
+                  dup
+                end
+              end
+            end
+          end
+        end
+        using Refine
+
+        # @see https://www.w3.org/TR/trace-context/#traceparent-header
+        def build_traceparent(digest)
+          build_traceparent_string(
+            digest.trace_distributed_id || digest.trace_id,
+            digest.span_id,
+            build_trace_flags(digest)
+          )
+        end
+
+        # For the current version (00), the traceparent has the following format:
+        #
+        # `"#{version}-#{trace_id}-#{parent_id}-#{trace_flags}"`
+        #
+        # Where:
+        #   * `version`: 2 hex-encoded digits, zero padded.
+        #   * `trace_id`: 32 hex-encoded digits, zero padded.
+        #   * `parent_id`: 16 hex-encoded digits, zero padded.
+        #   * `trace_flags`: 2 hex-encoded digits, zero padded.
+        #
+        # All hex values should be lowercase.
+        #
+        # @param trace_id [Integer] 128-bit
+        # @param parent_id [Integer] 64-bit
+        # @param trace_flags [Integer] 8-bit
+        def build_traceparent_string(trace_id, parent_id, trace_flags)
+          "00-#{format('%032x', trace_id)}-#{format('%016x', parent_id)}-#{format('%02x', trace_flags)}"
+        end
+
+        # Sets the trace flag to an existing `trace_flags`.
+        def build_trace_flags(digest)
+          trace_flags = digest.trace_flags || DEFAULT_TRACE_FLAGS
+
+          if digest.trace_sampling_priority
+            if Tracing::Sampling::PrioritySampler.sampled?(digest.trace_sampling_priority)
+              trace_flags |= TRACE_FLAGS_SAMPLED
+            else
+              trace_flags &= ~TRACE_FLAGS_SAMPLED
+            end
+          end
+
+          trace_flags
+        end
+
+        # @see https://www.w3.org/TR/trace-context/#tracestate-header
+        def build_tracestate(digest)
+          tracestate = String.new('dd=')
+          tracestate << "s:#{digest.trace_sampling_priority};" if digest.trace_sampling_priority
+          tracestate << "o:#{serialize_origin(digest.trace_origin)};" if digest.trace_origin
+
+          if digest.trace_distributed_tags
+            digest.trace_distributed_tags.each do |name, value|
+              tag = "t.#{serialize_tag_key(name)}:#{serialize_tag_value(value)};"
+
+              # If tracestate size limit is exceed, drop the remaining data.
+              # String#bytesize is used because only ASCII characters are allowed.
+              #
+              # We add 1 to the limit because of the trailing comma, which will be removed before returning.
+              break if tracestate.bytesize + tag.bytesize > (TRACESTATE_VALUE_SIZE_LIMIT + 1)
+
+              tracestate << tag
+            end
+          end
+
+          # Is there any Datadog-specific information to propagate.
+          # Check for > 3 size because the empty prefix `dd=` has 3 characters.
+          if tracestate.size > 3
+            # Propagate upstream tracestate with `dd=...` appended to the list
+            tracestate.chop! # Removes trailing `;` from Datadog trace state string.
+
+            if digest.trace_state
+              # Delete existing `dd=` tracestate fields, if present.
+              vendors = split_tracestate(digest.trace_state)
+              vendors.reject! { |v| v.start_with?('dd=') }
+            end
+
+            if vendors && !vendors.empty?
+              # Ensure the list has at most 31 elements, as we need to prepend Datadog's
+              # entry and the limit is 32 elements total.
+              vendors = vendors[0..30]
+              "#{tracestate},#{vendors.join(',')}"
+            else
+              tracestate.to_s
+            end
+          else
+            digest.trace_state # Propagate upstream tracestate with no Datadog changes
+          end
+        end
+
+        # If any characters in <origin_value> are invalid, replace each invalid character with 0x5F (underscore).
+        # Invalid characters are: characters outside the ASCII range 0x20 to 0x7E, 0x2C (comma), and 0x3D (equals).
+        def serialize_origin(value)
+          # DEV: It's unlikely that characters will be out of range, as they mostly
+          # DEV: come from Datadog-controlled sources.
+          # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
+          if INVALID_ORIGIN_CHARS.match?(value)
+            value.gsub(INVALID_ORIGIN_CHARS, '_')
+          else
+            value
+          end
+        end
+
+        # Serialize `_dd.p.{key}` by first removing the `_dd.p.` prefix.
+        # Then replacing invalid characters with `_`.
+        def serialize_tag_key(name)
+          key = name.delete_prefix(Tracing::Metadata::Ext::Distributed::TAGS_PREFIX)
+
+          # DEV: It's unlikely that characters will be out of range, as they mostly
+          # DEV: come from Datadog-controlled sources.
+          # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
+          if INVALID_TAG_KEY_CHARS.match?(key)
+            key.gsub(INVALID_TAG_KEY_CHARS, '_')
+          else
+            key
+          end
+        end
+
+        # Replaces invalid characters with `_`, then replaces `=` with `:`.
+        def serialize_tag_value(value)
+          # DEV: It's unlikely that characters will be out of range, as they mostly
+          # DEV: come from Datadog-controlled sources.
+          # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
+          ret = if INVALID_TAG_VALUE_CHARS.match?(value)
+                  value.gsub(INVALID_TAG_VALUE_CHARS, '_')
+                else
+                  value
+                end
+
+          # DEV: Checking for an unlikely '=' is faster than a no-op `tr!`.
+          ret.tr!('=', ':') if ret.include?('=')
+          ret
+        end
+
+        def extract_traceparent(traceparent)
+          trace_id, parent_id, trace_flags = parse_traceparent_string(traceparent)
+
+          # Return unless all traceparent fields are valid.
+          return unless trace_id && !trace_id.zero? && parent_id && !parent_id.zero? && trace_flags
+
+          dd_trace_id = parse_datadog_trace_id(trace_id)
+          sampled = parse_sampled_flag(trace_flags)
+
+          [trace_id, dd_trace_id, parent_id, sampled, trace_flags]
+        end
+
+        def parse_traceparent_string(traceparent)
+          return unless traceparent
+
+          version, trace_id, parent_id, trace_flags, extra = traceparent.strip.split('-')
+
+          return if version == INVALID_VERSION
+
+          # Extra fields are not allowed in version 00, but we have to be lenient for future versions.
+          return if version == SPEC_VERSION && extra
+
+          # Invalid field sizes
+          return if version.size != 2 || trace_id.size != 32 || parent_id.size != 16 || trace_flags.size != 2
+
+          [Integer(trace_id, 16), Integer(parent_id, 16), Integer(trace_flags, 16)]
+        rescue ArgumentError # Conversion to integer failed
+          nil
+        end
+
+        # Datadog only allows 64 bits for the trace id.
+        # We extract the lower 64 bits from the original 128-bit id.
+        def parse_datadog_trace_id(trace_id)
+          trace_id & 0xFFFFFFFFFFFFFFFF
+        end
+
+        def parse_sampled_flag(trace_flags)
+          trace_flags & TRACE_FLAGS_SAMPLED
+        end
+
+        # @return [Array<String,Integer,String,Hash>] returns 4 values: tracestate, sampling_priority, origin, tags.
+        def extract_tracestate(tracestate)
+          return unless tracestate
+
+          vendors = split_tracestate(tracestate)
+
+          # Find Datadog's `dd=` tracestate field.
+          dd_tracestate = vendors.find { |v| v.start_with?('dd=') }
+          return tracestate unless dd_tracestate
+
+          # Delete `dd=` prefix
+          dd_tracestate.slice!(0..2)
+
+          origin, sampling_priority, tags = extract_datadog_fields(dd_tracestate)
+
+          [tracestate, sampling_priority, origin, tags]
+        end
+
+        def extract_datadog_fields(dd_tracestate)
+          sampling_priority = nil
+          origin = nil
+          tags = nil
+
+          # DEV: Since Ruby 2.6 `split` can receive a block, so `each` can be removed then.
+          dd_tracestate.split(';').each do |pair|
+            key, value = pair.split(':', 2)
+            case key
+            when 's'
+              sampling_priority = Integer(value) rescue nil
+            when 'o'
+              origin = value
+            when /^t\./
+              key.slice!(0..1) # Delete `t.` prefix
+
+              value = deserialize_tag_value(value)
+
+              tags ||= {}
+              tags["#{Tracing::Metadata::Ext::Distributed::TAGS_PREFIX}#{key}"] = value
+            end
+          end
+
+          [origin, sampling_priority, tags]
+        end
+
+        # Restore `:` back to `=`.
+        def deserialize_tag_value(value)
+          value.tr!(':', '=')
+          value
+        end
+
+        # If `sampled` and `sampling_priority` disagree, `sampled` overrides the decision.
+        # @return [Integer] one of the {Datadog::Tracing::Sampling::Ext::Priority} values
+        def parse_priority_sampling(sampled, sampling_priority)
+          # If both fields agree
+          if sampling_priority &&
+              (!Tracing::Sampling::PrioritySampler.sampled?(sampling_priority) && sampled == 0 || # Both drop
+                Tracing::Sampling::PrioritySampler.sampled?(sampling_priority) && sampled == 1) # Both keep
+
+            return sampling_priority # Return the richer `sampling_priority`
+          end
+
+          sampled # Sampled flag trumps `sampling_priority` on conflict
+        end
+
+        def split_tracestate(tracestate)
+          tracestate.split(/[ \t]*,[ \t]*/)[0..31]
+        end
+
+        # Version 0xFF is invalid as per spec
+        # @see https://www.w3.org/TR/trace-context/#version
+        INVALID_VERSION = 'ff'
+        private_constant :INVALID_VERSION
+
+        # Empty 8-bit `trace-flags`.
+        # @see https://www.w3.org/TR/trace-context/#trace-flags
+        DEFAULT_TRACE_FLAGS = 0b00000000
+        private_constant :DEFAULT_TRACE_FLAGS
+
+        # Bit-mask for `trace-flags` that represents a sampled span (sampled==true).
+        # @see https://www.w3.org/TR/trace-context/#trace-flags
+        TRACE_FLAGS_SAMPLED = 0b00000001
+        private_constant :TRACE_FLAGS_SAMPLED
+
+        # The limit is inclusive: sizes *greater* than 256 are disallowed.
+        # @see https://www.w3.org/TR/trace-context/#value
+        TRACESTATE_VALUE_SIZE_LIMIT = 256
+        private_constant :TRACESTATE_VALUE_SIZE_LIMIT
+
+        # Replace all characters with `_`, except ASCII characters 0x20-0x7E.
+        # Additionally, `,`, ';', and `=` must also be replaced by `_`.
+        INVALID_ORIGIN_CHARS = /[\u0000-\u0019,;=\u007F-\u{10FFFF}]/.freeze
+        private_constant :INVALID_ORIGIN_CHARS
+
+        # Replace all characters with `_`, except ASCII characters 0x21-0x7E.
+        # Additionally, `,` and `=` must also be replaced by `_`.
+        INVALID_TAG_KEY_CHARS = /[\u0000-\u0020,=\u007F-\u{10FFFF}]/.freeze
+        private_constant :INVALID_TAG_KEY_CHARS
+
+        # Replace all characters with `_`, except ASCII characters 0x20-0x7E.
+        # Additionally, `,`, ':' and `;` must also be replaced by `_`.
+        INVALID_TAG_VALUE_CHARS = /[\u0000-\u001F,:;\u007F-\u{10FFFF}]/.freeze
+        private_constant :INVALID_TAG_VALUE_CHARS
+      end
+    end
+  end
+end

data/lib/datadog/tracing/metadata/ext.rb

@@ -60,7 +60,7 @@ module Datadog
         # @public_api
         module Errors
           STATUS = 1
-          TAG_MSG = 'error.msg'
+          TAG_MSG = 'error.message'
           TAG_STACK = 'error.stack'
           TAG_TYPE = 'error.type'
         end

data/lib/datadog/tracing/propagation/http.rb

@@ -1,117 +1,14 @@
 # typed: false
 
-require_relative '../../core'
-
-require_relative '../configuration/ext'
-require_relative '../sampling/ext'
-require_relative '../distributed/headers/b3'
-require_relative '../distributed/headers/b3_single'
-require_relative '../distributed/headers/datadog'
-
-require_relative '../trace_digest'
-require_relative '../trace_operation'
+require_relative '../contrib/http/distributed/propagation'
 
 module Datadog
   module Tracing
     module Propagation
       # Propagation::HTTP helps extracting and injecting HTTP headers.
+      # DEV-2.0: This file has been moved to Contrib. Should be deleted in the next release.
       # @public_api
-      module HTTP
-        PROPAGATION_STYLES = {
-          Configuration::Ext::Distributed::PROPAGATION_STYLE_B3 => Distributed::Headers::B3,
-          Configuration::Ext::Distributed::PROPAGATION_STYLE_B3_SINGLE_HEADER => Distributed::Headers::B3Single,
-          Configuration::Ext::Distributed::PROPAGATION_STYLE_DATADOG => Distributed::Headers::Datadog
-        }.freeze
-
-        # inject! populates the env with span ID, trace ID and sampling priority
-        #
-        # DEV-2.0: inject! should work without arguments, injecting the active_trace's digest
-        # DEV-2.0: and returning a new Hash with the injected headers.
-        # DEV-2.0: inject! should also accept either a `trace` or a `digest`, as a `trace`
-        # DEV-2.0: argument is the common use case, but also allows us to set error tags in the `trace`
-        # DEV-2.0: if needed.
-        # DEV-2.0: Ideally, we'd have a separate stream to report tracer errors and never
-        # DEV-2.0: touch the active span.
-        def self.inject!(digest, env)
-          # Prevent propagation from being attempted if trace headers provided are nil.
-          if digest.nil?
-            Datadog.logger.debug(
-              'Cannot inject trace headers into env to propagate over HTTP: trace headers are nil.'.freeze
-            )
-            return
-          end
-
-          digest = digest.to_digest if digest.is_a?(TraceOperation)
-
-          # Inject all configured propagation styles
-          Datadog.configuration.tracing.distributed_tracing.propagation_inject_style.each do |style|
-            propagator = PROPAGATION_STYLES[style]
-            begin
-              propagator.inject!(digest, env) unless propagator.nil?
-            rescue => e
-              Datadog.logger.error(
-                'Error injecting propagated trace headers into the environment. ' \
-                "Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-          end
-        end
-
-        # extract returns trace headers containing the span ID, trace ID and
-        # sampling priority defined in env.
-        def self.extract(env)
-          trace_digest = nil
-          dd_trace_digest = nil
-
-          Datadog.configuration.tracing.distributed_tracing.propagation_extract_style.each do |style|
-            propagator = PROPAGATION_STYLES[style]
-            next if propagator.nil?
-
-            # Extract trace headers
-            # DEV: `propagator.extract` will return `nil`, where `Propagation::HTTP#extract` will not
-            begin
-              extracted_trace_digest = propagator.extract(env)
-            rescue => e
-              Datadog.logger.error(
-                'Error extracting propagated trace headers from the environment. ' \
-                "Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
-
-            # Skip this style if no valid headers were found
-            next if extracted_trace_digest.nil?
-
-            # Keep track of the Datadog extract trace headers, we want to return
-            #   this one if we have one
-            if extracted_trace_digest && style == Configuration::Ext::Distributed::PROPAGATION_STYLE_DATADOG
-              dd_trace_digest = extracted_trace_digest
-            end
-
-            # No previously extracted trace headers, use the one we just extracted
-            if trace_digest.nil?
-              trace_digest = extracted_trace_digest
-            else
-              unless trace_digest.trace_id == extracted_trace_digest.trace_id \
-                      && trace_digest.span_id == extracted_trace_digest.span_id
-                # Return an empty/new trace headers if we have a mismatch in values extracted
-                msg = "#{trace_digest.trace_id} != #{extracted_trace_digest.trace_id} && " \
-                      "#{trace_digest.span_id} != #{extracted_trace_digest.span_id}"
-                Datadog.logger.debug(
-                  "Cannot extract trace headers from HTTP: extracted trace headers differ, #{msg}"
-                )
-                # DEV: This will return from `self.extract` not this `each` block
-                return TraceDigest.new
-              end
-            end
-          end
-
-          # Return the extracted trace headers if we found one or else a new empty trace headers
-          # Always return the Datadog trace headers if one exists since it has more
-          #   information than the B3 headers e.g. origin, expanded priority
-          #   sampling values, etc
-          dd_trace_digest || trace_digest || nil
-        end
-      end
+      HTTP = Tracing::Contrib::HTTP::Distributed::Propagation.new
     end
   end
 end

data/lib/datadog/tracing/sampling/priority_sampler.rb

@@ -85,6 +85,17 @@ module Datadog
           @priority_sampler.update(rate_by_service, decision: decision)
         end
 
+        # Check if the Priority Sampling decision is to keep or drop the trace.
+        # Other factors can influence the sampling decision; this method is only
+        # responsible for interpreting the Sampling Priority decision.
+        #
+        # @param priority_sampling [Integer] priority sampling number
+        # @return [Boolean] true if trace is "kept" by priority sampling
+        # @return [Boolean] false if trace is "dropped" by priority sampling
+        def self.sampled?(priority_sampling)
+          priority_sampling >= Ext::Priority::AUTO_KEEP
+        end
+
         private
 
         def pre_sample?(trace)

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -3,7 +3,7 @@
 require_relative '../../core'
 
 require_relative 'sampler'
-require_relative '../span'
+require_relative '../utils'
 
 module Datadog
   module Tracing
@@ -49,11 +49,11 @@ module Datadog
 
         def sample_rate=(sample_rate)
           @sample_rate = sample_rate
-          @sampling_id_threshold = sample_rate * Tracing::Span::EXTERNAL_MAX_ID
+          @sampling_id_threshold = sample_rate * Tracing::Utils::EXTERNAL_MAX_ID
         end
 
         def sample?(trace)
-          ((trace.id * KNUTH_FACTOR) % Tracing::Span::EXTERNAL_MAX_ID) <= @sampling_id_threshold
+          ((trace.id * KNUTH_FACTOR) % Tracing::Utils::EXTERNAL_MAX_ID) <= @sampling_id_threshold
         end
 
         def sample!(trace)

data/lib/datadog/tracing/span.rb

@@ -2,8 +2,8 @@
 
 # typed: true
 
-require_relative '../core/utils'
 require_relative '../core/utils/safe_dup'
+require_relative 'utils'
 
 require_relative 'metadata/ext'
 require_relative 'metadata'
@@ -19,22 +19,6 @@ module Datadog
     class Span
       include Metadata
 
-      # The max value for a {Datadog::Tracing::Span} identifier.
-      # Span and trace identifiers should be strictly positive and strictly inferior to this limit.
-      #
-      # Limited to +2<<62-1+ positive integers, as Ruby is able to represent such numbers "inline",
-      # inside a +VALUE+ scalar, thus not requiring memory allocation.
-      #
-      # The range of IDs also has to consider portability across different languages and platforms.
-      RUBY_MAX_ID = (1 << 62) - 1
-
-      # Excludes zero from possible values
-      RUBY_ID_RANGE = (1..RUBY_MAX_ID).freeze
-
-      # While we only generate 63-bit integers due to limitations in other languages, we support
-      # parsing 64-bit integers for distributed tracing since an upstream system may generate one
-      EXTERNAL_MAX_ID = 1 << 64
-
       attr_accessor \
         :end_time,
         :id,
@@ -90,9 +74,9 @@ module Datadog
         @resource = Core::Utils::SafeDup.frozen_or_dup(resource)
         @type = Core::Utils::SafeDup.frozen_or_dup(type)
 
-        @id = id || Core::Utils.next_id
+        @id = id || Tracing::Utils.next_id
         @parent_id = parent_id || 0
-        @trace_id = trace_id || Core::Utils.next_id
+        @trace_id = trace_id || Tracing::Utils.next_id
 
         @meta = meta || {}
         @metrics = metrics || {}

data/lib/datadog/tracing/span_operation.rb

@@ -7,10 +7,11 @@ require_relative '../core/environment/identity'
 require_relative '../core/utils'
 require_relative '../core/utils/time'
 
-require_relative 'metadata/ext'
 require_relative 'event'
-require_relative 'span'
 require_relative 'metadata'
+require_relative 'metadata/ext'
+require_relative 'span'
+require_relative 'utils'
 
 module Datadog
   module Tracing
@@ -61,9 +62,9 @@ module Datadog
         self.type = type
         self.resource = resource
 
-        @id = Core::Utils.next_id
+        @id = Tracing::Utils.next_id
         @parent_id = parent_id || 0
-        @trace_id = trace_id || Core::Utils.next_id
+        @trace_id = trace_id || Tracing::Utils.next_id
 
         @status = 0