ddtrace 1.12.1 → 1.18.0

data/lib/datadog/tracing/distributed/propagation.rb

@@ -18,6 +18,12 @@ module Datadog
         # @param propagation_styles [Hash<String,Object>]
         def initialize(propagation_styles:)
           @propagation_styles = propagation_styles
+          # We need to make sure propagation_style option is evaluated.
+          # Our options are lazy evaluated and it happens that propagation_style has the after_set callback
+          # that affect Datadog.configuration.tracing.distributed_tracing.propagation_inject_style and
+          # Datadog.configuration.tracing.distributed_tracing.propagation_extract_style
+          # By calling it here, we make sure if the customers has set any value either via code or ENV variable is applied.
+          ::Datadog.configuration.tracing.distributed_tracing.propagation_style
         end
 
         # inject! populates the env with span ID, trace ID and sampling priority
@@ -72,53 +78,48 @@ module Datadog
         #
         # @param data [Hash]
         def extract(data)
-          trace_digest = nil
-          dd_trace_digest = nil
+          return unless data
+          return if data.empty?
 
-          ::Datadog.configuration.tracing.distributed_tracing.propagation_extract_style.each do |style|
+          extracted_trace_digest = nil
+
+          config = ::Datadog.configuration.tracing.distributed_tracing
+
+          config.propagation_extract_style.each do |style|
             propagator = @propagation_styles[style]
             next if propagator.nil?
 
             begin
-              extracted_trace_digest = propagator.extract(data)
-            rescue => e
-              ::Datadog.logger.error(
-                'Error extracting distributed trace data. ' \
-              "Cause: #{e} Location: #{Array(e.backtrace).first}"
-              )
-            end
+              if extracted_trace_digest
+                # Return if we are only inspecting the first valid style.
+                next if config.propagation_extract_first
 
-            # Skip if no valid data was found
-            next if extracted_trace_digest.nil?
+                # Continue parsing styles to find the W3C `tracestate` header, if present.
+                # `tracestate` must always be propagated, as it might contain pass-through data that we don't control.
+                # @see https://www.w3.org/TR/2021/REC-trace-context-1-20211123/#mutating-the-tracestate-field
+                next if style != Configuration::Ext::Distributed::PROPAGATION_STYLE_TRACE_CONTEXT
 
-            # Keep track of the Datadog extracted digest, we want to return it if we have it.
-            if extracted_trace_digest && style == Tracing::Configuration::Ext::Distributed::PROPAGATION_STYLE_DATADOG
-              dd_trace_digest = extracted_trace_digest
-            end
+                if (tracecontext_digest = propagator.extract(data))
+                  # Only parse if it represent the same trace as the successfully extracted one
+                  next unless tracecontext_digest.trace_id == extracted_trace_digest.trace_id
 
-            # No previously extracted trace data, use the one we just extracted
-            if trace_digest.nil?
-              trace_digest = extracted_trace_digest
-            else
-              unless trace_digest.trace_id == extracted_trace_digest.trace_id \
-                    && trace_digest.span_id == extracted_trace_digest.span_id
-                # We have two mismatched propagation contexts
-                ::Datadog.logger.debug do
-                  'Cannot extract distributed trace data: extracted styles differ, ' \
-                  "#{trace_digest.trace_id} != #{extracted_trace_digest.trace_id} && " \
-                  "#{trace_digest.span_id} != #{extracted_trace_digest.span_id}"
+                  # Preserve the `tracestate`
+                  extracted_trace_digest = extracted_trace_digest.merge(
+                    trace_state: tracecontext_digest.trace_state,
+                    trace_state_unknown_fields: tracecontext_digest.trace_state_unknown_fields
+                  )
                 end
-
-                # It's safer to create a new context than to attach ourselves to the wrong context
-                return TraceDigest.new # Early return from the whole method
+              else
+                extracted_trace_digest = propagator.extract(data)
               end
+            rescue => e
+              ::Datadog.logger.error(
+                "Error extracting distributed trace data. Cause: #{e} Location: #{Array(e.backtrace).first}"
+              )
             end
           end
 
-          # Return the extracted trace digest if we found one or else a new empty digest.
-          # Always return the Datadog trace digest if one exists since it has more
-          # information than the B3 digest, e.g. origin, priority sampling values.
-          dd_trace_digest || trace_digest || nil
+          extracted_trace_digest
         end
       end
     end

data/lib/datadog/tracing/distributed/trace_context.rb

@@ -44,7 +44,15 @@ module Datadog
 
           tracestate, sampling_priority, origin, tags, unknown_fields = extract_tracestate(fetcher[@tracestate_key])
 
-          sampling_priority = parse_priority_sampling(sampled, sampling_priority)
+          sampling_priority = parse_priority_sampling(sampled, sampling_priority) do |decision|
+            case decision
+            when String
+              tags ||= {}
+              tags[Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER] = decision
+            when :drop
+              tags.delete(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER) if tags
+            end
+          end
 
           TraceDigest.new(
             span_id: parent_id,
@@ -160,8 +168,10 @@ module Datadog
             tracestate.chop! # Removes trailing `;` from Datadog trace state string.
 
             if digest.trace_state
+              trace_state = digest.trace_state.strip
+
               # Delete existing `dd=` tracestate fields, if present.
-              vendors = split_tracestate(digest.trace_state)
+              vendors = split_tracestate(trace_state)
               vendors.reject! { |v| v.start_with?('dd=') }
             end
 
@@ -179,13 +189,21 @@ module Datadog
         end
 
         # If any characters in <origin_value> are invalid, replace each invalid character with 0x5F (underscore).
-        # Invalid characters are: characters outside the ASCII range 0x20 to 0x7E, 0x2C (comma), and 0x3D (equals).
+        # Invalid characters are: characters outside the ASCII range 0x20 to 0x7E,
+        # 0x2C (comma), 0x3B (semi-colon), and 0x7E (tilde).
+        # Then, remap 0x3D (equals) to 0x7E (tilde)
         def serialize_origin(value)
           # DEV: It's unlikely that characters will be out of range, as they mostly
           # DEV: come from Datadog-controlled sources.
           # DEV: Trying to `match?` is measurably faster than a `gsub` that does not match.
-          if INVALID_ORIGIN_CHARS.match?(value)
-            value.gsub(INVALID_ORIGIN_CHARS, '_')
+          value = if INVALID_ORIGIN_CHARS.match?(value)
+                    value.gsub(INVALID_ORIGIN_CHARS, '_')
+                  else
+                    value
+                  end
+
+          if REMAP_ORIGIN_CHARS.match?(value)
+            value.gsub(REMAP_ORIGIN_CHARS, '~')
           else
             value
           end
@@ -245,13 +263,15 @@ module Datadog
 
           version, trace_id, parent_id, trace_flags, extra = traceparent.strip.split('-')
 
+          return if version.size != 2 || version[0] < '0' || version[0] > 'f' || version[1] < '0' || version[1] > 'f'
+
           return if version == INVALID_VERSION
 
           # Extra fields are not allowed in version 00, but we have to be lenient for future versions.
           return if version == SPEC_VERSION && extra
 
           # Invalid field sizes
-          return if version.size != 2 || trace_id.size != 32 || parent_id.size != 16 || trace_flags.size != 2
+          return if trace_id.size != 32 || parent_id.size != 16 || trace_flags.size != 2
 
           [Integer(trace_id, 16), Integer(parent_id, 16), Integer(trace_flags, 16)]
         rescue ArgumentError # Conversion to integer failed
@@ -324,20 +344,31 @@ module Datadog
 
         # If `sampled` and `sampling_priority` disagree, `sampled` overrides the decision.
         # @return [Integer] one of the {Datadog::Tracing::Sampling::Ext::Priority} values
+        # @yieldparam the new decision maker (either :drop or a new decision maker String value).
         def parse_priority_sampling(sampled, sampling_priority)
-          # If both fields agree
-          if sampling_priority &&
-              (!Tracing::Sampling::PrioritySampler.sampled?(sampling_priority) && sampled == 0 || # Both drop
-                Tracing::Sampling::PrioritySampler.sampled?(sampling_priority) && sampled == 1) # Both keep
-
-            return sampling_priority # Return the richer `sampling_priority`
+          if sampled == 1
+            if sampling_priority && Tracing::Sampling::PrioritySampler.sampled?(sampling_priority)
+              # Both sampling fields agree.
+              sampling_priority
+            else
+              # Sampling fields disagree.
+              # Let's force the trace to be kept, while also updating the decision maker to ourselves.
+              yield Tracing::Sampling::Ext::Decision::DEFAULT
+              sampled
+            end
+          elsif sampling_priority && !Tracing::Sampling::PrioritySampler.sampled?(sampling_priority)
+            sampling_priority
+          # Both sampling fields agree.
+          else
+            # Sampling fields disagree.
+            # Let's drop the trace and remove the sampling decision tag, as dropped spans don't carry sampling decision.
+            yield :drop
+            sampled
           end
-
-          sampled # Sampled flag trumps `sampling_priority` on conflict
         end
 
         def split_tracestate(tracestate)
-          tracestate.split(/[ \t]*,[ \t]*/)[0..31]
+          tracestate.split(/[ \t]*+,[ \t]*+/)[0..31]
         end
 
         # Version 0xFF is invalid as per spec
@@ -361,10 +392,14 @@ module Datadog
         private_constant :TRACESTATE_VALUE_SIZE_LIMIT
 
         # Replace all characters with `_`, except ASCII characters 0x20-0x7E.
-        # Additionally, `,`, ';', and `=` must also be replaced by `_`.
-        INVALID_ORIGIN_CHARS = /[\u0000-\u0019,;=\u007F-\u{10FFFF}]/.freeze
+        # Additionally, `,`, ';', and `~` must also be replaced by `_`.
+        INVALID_ORIGIN_CHARS = /[\u0000-\u0019,;~\u007F-\u{10FFFF}]/.freeze
         private_constant :INVALID_ORIGIN_CHARS
 
+        # Additionally, remap `=` to `~`
+        REMAP_ORIGIN_CHARS = /=/.freeze
+        private_constant :REMAP_ORIGIN_CHARS
+
         # Replace all characters with `_`, except ASCII characters 0x21-0x7E.
         # Additionally, `,` and `=` must also be replaced by `_`.
         INVALID_TAG_KEY_CHARS = /[\u0000-\u0020,=\u007F-\u{10FFFF}]/.freeze

data/lib/datadog/tracing/metadata/ext.rb

@@ -23,6 +23,10 @@ module Datadog
         # Set this tag to `1.0` if the span is a Service Entry span.
         TAG_TOP_LEVEL = '_dd.top_level'
 
+        # Set to `1.0` if profiling is enabled together with tracing, and `0.0` otherwise
+        # See Datadog-internal "RFC: Identifying which spans have profiling enabled " for details
+        TAG_PROFILING_ENABLED = '_dd.profiling.enabled'
+
         # Defines constants for trace analytics
         # @public_api
         module Analytics
@@ -90,7 +94,10 @@ module Datadog
             INVALID_TAG_CHARACTERS = %r{[^a-z0-9_\-:./]}.freeze
 
             # Normalizes an HTTP header string into a valid tag string.
-            def to_tag(name)
+            #
+            # By default, tags cannot create nested span tag levels:
+            # `allow_nested` allows you to override this behavior.
+            def to_tag(name, allow_nested: false)
               # Tag normalization based on: https://docs.datadoghq.com/getting_started/tagging/#defining-tags.
               #
               # Only the following characters are accepted.
@@ -109,7 +116,7 @@ module Datadog
               # Additional HTTP header normalization.
               #
               # Periods are replaced with an underscore.
-              tag.tr!('.', '_')
+              tag.tr!('.', '_') unless allow_nested
               tag
             end
           end
@@ -178,10 +185,6 @@ module Datadog
         end
 
         # @public_api
-        module SpanAttributeSchema
-          # current span attribute schema version
-          TAG_SCHEMA_VERSION = '_dd.trace_span_attribute_schema'
-        end
       end
     end
   end

data/lib/datadog/tracing/metadata/tagging.rb

@@ -59,8 +59,8 @@ module Datadog
         # A valid example is:
         #
         #   span.set_tags({ "http.method" => "GET", "user.id" => "234" })
-        def set_tags(tags)
-          tags.each { |k, v| set_tag(k, v) }
+        def set_tags(hash)
+          hash.each { |k, v| set_tag(k, v) }
         end
 
         # Returns true if the provided `tag` was set to a non-nil value.
@@ -111,7 +111,7 @@ module Datadog
         # Returns a copy of all metadata.
         # Keys for `@meta` and `@metrics` don't collide, by construction.
         def tags
-          @meta.merge(@metrics)
+          meta.merge(metrics)
         end
 
         protected

data/lib/datadog/tracing/remote.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative '../core/remote/dispatcher'
+require_relative 'configuration/dynamic'
+
+module Datadog
+  module Tracing
+    # Remote configuration declaration
+    module Remote
+      class ReadError < StandardError; end
+
+      class << self
+        PRODUCT = 'APM_TRACING'
+
+        def products
+          [PRODUCT]
+        end
+
+        def capabilities
+          [] # No capabilities advertised
+        end
+
+        def process_config(config, content)
+          lib_config = config['lib_config']
+
+          env_vars = Datadog::Tracing::Configuration::Dynamic::OPTIONS.map do |name, env_var, option|
+            value = lib_config[name]
+
+            # Guard for RBS/Steep
+            raise "option is a #{option.class}, expected Option" unless option.is_a?(Configuration::Dynamic::Option)
+
+            option.call(value)
+
+            [env_var, value]
+          end
+
+          content.applied
+
+          Datadog.send(:components).telemetry.client_configuration_change!(env_vars)
+        rescue => e
+          content.errored("#{e.class.name} #{e.message}: #{Array(e.backtrace).join("\n")}")
+        end
+
+        def receivers
+          receiver do |repository, _changes|
+            # DEV: Filter our by product. Given it will be very common
+            # DEV: we can filter this out before we receive the data in this method.
+            # DEV: Apply this refactor to AppSec as well if implemented.
+            repository.contents.map do |content|
+              case content.path.product
+              when PRODUCT
+                config = parse_content(content)
+                process_config(config, content)
+              end
+            end
+          end
+        end
+
+        def receiver(products = [PRODUCT], &block)
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
+          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+        end
+
+        private
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/rule_sampler.rb

@@ -48,6 +48,35 @@ module Datadog
                              end
         end
 
+        def self.parse(rules, rate_limit, default_sample_rate)
+          parsed_rules = JSON.parse(rules).map do |rule|
+            sample_rate = rule['sample_rate']
+
+            begin
+              sample_rate = Float(sample_rate)
+            rescue
+              raise "Rule '#{rule.inspect}' does not contain a float property `sample_rate`"
+            end
+
+            kwargs = {
+              name: rule['name'],
+              service: rule['service'],
+              sample_rate: sample_rate,
+            }
+
+            Core::BackportFrom24.hash_compact!(kwargs)
+
+            SimpleRule.new(**kwargs)
+          end
+
+          new(parsed_rules, rate_limit: rate_limit, default_sample_rate: default_sample_rate)
+        rescue => e
+          Datadog.logger.error do
+            "Could not parse trace sampling rules '#{rules}': #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+          end
+          nil
+        end
+
         # /RuleSampler's components (it's rate limiter, for example) are
         # not be guaranteed to be size-effect free.
         # It is not possible to guarantee that a call to {#sample?} will

data/lib/datadog/tracing/span_operation.rb

@@ -3,6 +3,7 @@ require 'time'
 require_relative '../core/environment/identity'
 require_relative '../core/utils'
 require_relative '../core/utils/time'
+require_relative '../core/utils/safe_dup'
 
 require_relative 'event'
 require_relative 'metadata'
@@ -434,19 +435,6 @@ module Datadog
         :parent,
         :span
 
-      if RUBY_VERSION < '2.2' # nil.dup only fails in Ruby 2.1
-        # Ensures #initialize can call nil.dup safely
-        module RefineNil
-          refine NilClass do
-            def dup
-              self
-            end
-          end
-        end
-
-        using RefineNil
-      end
-
       # Create a Span from the operation which represents
       # the finalized measurement. We #dup here to prevent
       # mutation by reference; when this span is returned,
@@ -457,8 +445,8 @@ module Datadog
           duration: duration,
           end_time: @end_time,
           id: @id,
-          meta: meta.dup,
-          metrics: metrics.dup,
+          meta: Core::Utils::SafeDup.frozen_or_dup(meta),
+          metrics: Core::Utils::SafeDup.frozen_or_dup(metrics),
           parent_id: @parent_id,
           resource: @resource,
           service: @service,

data/lib/datadog/tracing/sync_writer.rb

@@ -4,7 +4,7 @@ require_relative 'pipeline'
 require_relative 'runtime/metrics'
 require_relative 'writer'
 
-require_relative '../../ddtrace/transport/http'
+require_relative 'transport/http'
 
 module Datadog
   module Tracing
@@ -20,10 +20,10 @@ module Datadog
         :events,
         :transport
 
-      # @param [Datadog::Transport::Traces::Transport] transport a custom transport instance.
+      # @param [Datadog::Tracing::Transport::Traces::Transport] transport a custom transport instance.
       #   If provided, overrides `transport_options` and `agent_settings`.
       # @param [Hash<Symbol,Object>] transport_options options for the default transport instance.
-      # @param [Datadog::Core::Configuration::AgentSettingsResolver::AgentSettings] agent_settings agent options for
+      # @param [Datadog::Tracing::Configuration::AgentSettingsResolver::AgentSettings] agent_settings agent options for
       #   the default transport instance.
       def initialize(transport: nil, transport_options: {}, agent_settings: nil)
         @transport = transport || begin

data/lib/datadog/tracing/trace_digest.rb

@@ -141,6 +141,37 @@ module Datadog
 
         freeze
       end
+
+      # Creates a copy of this object, modifying the provided fields.
+      # @param field_value_pairs [Hash<String>] the fields to be overwritten
+      # @return [TraceDigest] returns a copy of this object with the `field_value_pairs` modified
+      def merge(field_value_pairs)
+        # DEV: Because we want to sometimes freeze the values provided to `TraceDigest`, it's best
+        # DEV: to let `#initialize` decide how to handle each field, instead of duplicating that logic here.
+        TraceDigest.new(
+          **{
+            span_id: span_id,
+            span_name: span_name,
+            span_resource: span_resource,
+            span_service: span_service,
+            span_type: span_type,
+            trace_distributed_tags: trace_distributed_tags,
+            trace_hostname: trace_hostname,
+            trace_id: trace_id,
+            trace_name: trace_name,
+            trace_origin: trace_origin,
+            trace_process_id: trace_process_id,
+            trace_resource: trace_resource,
+            trace_runtime_id: trace_runtime_id,
+            trace_sampling_priority: trace_sampling_priority,
+            trace_service: trace_service,
+            trace_distributed_id: trace_distributed_id,
+            trace_flags: trace_flags,
+            trace_state: trace_state,
+            trace_state_unknown_fields: trace_state_unknown_fields,
+          }.merge!(field_value_pairs)
+        )
+      end
     end
   end
 end

data/lib/datadog/tracing/trace_operation.rb

@@ -40,7 +40,9 @@ module Datadog
         :active_span,
         :id,
         :max_length,
-        :parent_span_id
+        :parent_span_id,
+        :trace_state,
+        :trace_state_unknown_fields
 
       attr_writer \
         :name,
@@ -64,8 +66,11 @@ module Datadog
         sampled: nil,
         sampling_priority: nil,
         service: nil,
+        profiling_enabled: nil,
         tags: nil,
-        metrics: nil
+        metrics: nil,
+        trace_state: nil,
+        trace_state_unknown_fields: nil
       )
         # Attributes
         @id = id || Tracing::Utils::TraceId.next_id
@@ -84,6 +89,9 @@ module Datadog
         @sample_rate = sample_rate
         @sampling_priority = sampling_priority
         @service = service
+        @profiling_enabled = profiling_enabled
+        @trace_state = trace_state
+        @trace_state_unknown_fields = trace_state_unknown_fields
 
         # Generic tags
         set_tags(tags) if tags
@@ -288,6 +296,8 @@ module Datadog
           trace_runtime_id: Core::Environment::Identity.id,
           trace_sampling_priority: @sampling_priority,
           trace_service: service,
+          trace_state: @trace_state,
+          trace_state_unknown_fields: @trace_state_unknown_fields,
         ).freeze
       end
 
@@ -310,6 +320,8 @@ module Datadog
           sampled: @sampled,
           sampling_priority: @sampling_priority,
           service: (service && service.dup),
+          trace_state: (@trace_state && @trace_state.dup),
+          trace_state_unknown_fields: (@trace_state_unknown_fields && @trace_state_unknown_fields.dup),
           tags: meta.dup,
           metrics: metrics.dup
         )
@@ -450,7 +462,8 @@ module Datadog
           service: service,
           tags: meta,
           metrics: metrics,
-          root_span_id: !partial ? root_span && root_span.id : nil
+          root_span_id: !partial ? root_span && root_span.id : nil,
+          profiling_enabled: @profiling_enabled,
         )
       end
 

data/lib/datadog/tracing/trace_segment.rb

@@ -31,7 +31,8 @@ module Datadog
         :sample_rate,
         :sampling_decision_maker,
         :sampling_priority,
-        :service
+        :service,
+        :profiling_enabled
 
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
@@ -54,7 +55,8 @@ module Datadog
         sampling_priority: nil,
         service: nil,
         tags: nil,
-        metrics: nil
+        metrics: nil,
+        profiling_enabled: nil
       )
         @id = id
         @root_span_id = root_span_id
@@ -80,6 +82,7 @@ module Datadog
         @sampling_decision_maker = sampling_decision_maker_tag
         @sampling_priority = sampling_priority || sampling_priority_tag
         @service = Core::Utils::SafeDup.frozen_or_dup(service || service_tag)
+        @profiling_enabled = profiling_enabled
       end
       # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/CyclomaticComplexity

data/lib/datadog/tracing/tracer.rb

@@ -320,16 +320,20 @@ module Datadog
         if digest
           TraceOperation.new(
             hostname: hostname,
+            profiling_enabled: profiling_enabled,
             id: digest.trace_id,
             origin: digest.trace_origin,
             parent_span_id: digest.span_id,
             sampling_priority: digest.trace_sampling_priority,
             # Distributed tags are just regular trace tags with special meaning to Datadog
             tags: digest.trace_distributed_tags,
+            trace_state: digest.trace_state,
+            trace_state_unknown_fields: digest.trace_state_unknown_fields,
           )
         else
           TraceOperation.new(
-            hostname: hostname
+            hostname: hostname,
+            profiling_enabled: profiling_enabled,
           )
         end
       end
@@ -349,6 +353,8 @@ module Datadog
         end
       end
 
+      # Creates a new TraceOperation, with events bounds to this Tracer instance.
+      # @return [TraceOperation]
       def start_trace(continue_from: nil)
         # Build a new trace using digest if provided.
         trace = build_trace(continue_from)
@@ -525,6 +531,11 @@ module Datadog
           span
         end
       end
+
+      def profiling_enabled
+        @profiling_enabled ||=
+          !!(defined?(Datadog::Profiling) && Datadog::Profiling.respond_to?(:enabled?) && Datadog::Profiling.enabled?)
+      end
     end
   end
 end

data/lib/datadog/{core → tracing}/transport/http/api/instance.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Datadog
-  module Core
+  module Tracing
     module Transport
       module HTTP
         module API

data/lib/datadog/{core → tracing}/transport/http/api/spec.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Datadog
-  module Core
+  module Tracing
     module Transport
       module HTTP
         module API