ddtrace 1.4.1 → 1.6.1

data/lib/datadog/tracing/flush.rb

@@ -3,71 +3,93 @@
 module Datadog
   module Tracing
     module Flush
-      # Consumes only completed traces (where all spans have finished)
-      class Finished
-        # Consumes and returns completed traces (where all spans have finished)
-        # from the provided \trace_op, if any.
+      # Consumes and returns a {TraceSegment} to be flushed, from
+      # the provided {TraceSegment}.
+      #
+      # Only finished spans are consumed. Any spans consumed are
+      # removed from +trace_op+ as a side effect. Unfinished spans are
+      # unaffected.
+      #
+      # @abstract
+      class Base
+        # Consumes and returns a {TraceSegment} to be flushed, from
+        # the provided {TraceSegment}.
         #
-        # Any traces consumed are removed from +trace_op+ as a side effect.
+        # Only finished spans are consumed. Any spans consumed are
+        # removed from +trace_op+ as a side effect. Unfinished spans are
+        # unaffected.
         #
+        # @param [TraceOperation] trace_op
         # @return [TraceSegment] trace to be flushed, or +nil+ if the trace is not finished
         def consume!(trace_op)
-          return unless full_flush?(trace_op)
+          return unless flush?(trace_op)
 
           get_trace(trace_op)
         end
 
-        def full_flush?(trace_op)
-          trace_op && trace_op.sampled? && trace_op.finished?
+        # Should we consume spans from the +trace_op+?
+        # @abstract
+        def flush?(trace_op)
+          raise NotImplementedError
         end
 
         protected
 
+        # Consumes all finished spans from trace.
+        # @return [TraceSegment]
         def get_trace(trace_op)
-          trace_op.flush!
+          trace_op.flush! do |spans|
+            spans.select! { |span| single_sampled?(span) } unless trace_op.sampled?
+
+            spans
+          end
+        end
+
+        # Single Span Sampling has chosen to keep this span
+        # regardless of the trace-level sampling decision
+        def single_sampled?(span)
+          span.get_metric(Sampling::Span::Ext::TAG_MECHANISM) == Sampling::Ext::Mechanism::SPAN_SAMPLING_RATE
+        end
+      end
+
+      # Consumes and returns completed traces (where all spans have finished),
+      # if any, from the provided +trace_op+.
+      #
+      # Spans consumed are removed from +trace_op+ as a side effect.
+      class Finished < Base
+        # Are all spans finished?
+        def flush?(trace_op)
+          trace_op && trace_op.finished?
         end
       end
 
-      # Performs partial trace flushing to avoid large traces residing in memory for too long
-      class Partial
+      # Consumes and returns completed or partially completed
+      # traces from the provided +trace_op+, if any.
+      #
+      # Partial trace flushing avoids large traces residing in memory for too long.
+      #
+      # Partially completed traces, where not all spans have finished,
+      # will only be returned if there are at least
+      # +@min_spans_for_partial+ finished spans.
+      #
+      # Spans consumed are removed from +trace_op+ as a side effect.
+      class Partial < Base
         # Start flushing partial trace after this many active spans in one trace
         DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH = 500
 
         attr_reader :min_spans_for_partial
 
         def initialize(options = {})
+          super()
           @min_spans_for_partial = options.fetch(:min_spans_before_partial_flush, DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH)
         end
 
-        # Consumes and returns completed or partially completed
-        # traces from the provided +trace_op+, if any.
-        #
-        # Partially completed traces, where not all spans have finished,
-        # will only be returned if there are at least
-        # +@min_spans_for_partial+ finished spans.
-        #
-        # Any spans consumed are removed from +trace_op+ as a side effect.
-        #
-        # @return [TraceSegment] partial or complete trace to be flushed, or +nil+ if no spans are finished
-        def consume!(trace_op)
-          return unless partial_flush?(trace_op)
-
-          get_trace(trace_op)
-        end
-
-        def partial_flush?(trace_op)
-          return false unless trace_op.sampled?
+        def flush?(trace_op)
           return true if trace_op.finished?
           return false if trace_op.finished_span_count < @min_spans_for_partial
 
           true
         end
-
-        protected
-
-        def get_trace(trace_op)
-          trace_op.flush!
-        end
       end
     end
   end

data/lib/datadog/tracing/metadata/ext.rb

@@ -45,8 +45,16 @@ module Datadog
         # @public_api
         # Tags related to distributed tracing
         module Distributed
+          # What mechanism was used to make this trace's sampling decision.
+          # @see Datadog::Tracing::Sampling::Ext::Mechanism
+          TAG_DECISION_MAKER = '_dd.p.dm'
+
           TAG_ORIGIN = '_dd.origin'
           TAG_SAMPLING_PRIORITY = '_sampling_priority_v1'
+
+          # Trace tags with this prefix will propagate from a trace through distributed tracing.
+          # Distributed headers tags with this prefix will be injected into the active trace.
+          TAGS_PREFIX = '_dd.p.'
         end
 
         # @public_api
@@ -63,11 +71,14 @@ module Datadog
           TAG_BASE_URL = 'http.base_url'
           TAG_METHOD = 'http.method'
           TAG_STATUS_CODE = 'http.status_code'
+          TAG_USER_AGENT = 'http.useragent'
           TAG_URL = 'http.url'
           TYPE_INBOUND = AppTypes::TYPE_WEB.freeze
           TYPE_OUTBOUND = 'http'
           TYPE_PROXY = 'proxy'
           TYPE_TEMPLATE = 'template'
+          TAG_CLIENT_IP = 'http.client_ip'
+          HEADER_USER_AGENT = 'User-Agent'
 
           # General header functionality
           module Headers
@@ -153,15 +164,6 @@ module Datadog
           TAG_QUERY = 'sql.query'
         end
 
-        # @public_api
-        module DB
-          TAG_INSTANCE = 'db.instance'
-          TAG_USER = 'db.user'
-          TAG_SYSTEM = 'db.system'
-          TAG_STATEMENT = 'db.statement'
-          TAG_ROW_COUNT = 'db.row_count'
-        end
-
         # @public_api
         module SpanKind
           TAG_SERVER = 'server'

data/lib/datadog/tracing/metadata/tagging.rb

@@ -65,6 +65,15 @@ module Datadog
           tags.each { |k, v| set_tag(k, v) }
         end
 
+        # Returns true if the provided `tag` was set to a non-nil value.
+        # False otherwise.
+        #
+        # @param [String] tag the tag or metric to check for presence
+        # @return [Boolean] if the tag is present and not nil
+        def has_tag?(tag) # rubocop:disable Naming/PredicateName
+          !get_tag(tag).nil? # nil is considered not present, thus we can't use `Hash#has_key?`
+        end
+
         # This method removes a tag for the given key.
         def clear_tag(key)
           meta.delete(key)

data/lib/datadog/tracing/propagation/http.rb

@@ -23,7 +23,15 @@ module Datadog
           Configuration::Ext::Distributed::PROPAGATION_STYLE_DATADOG => Distributed::Headers::Datadog
         }.freeze
 
-        # inject! popolates the env with span ID, trace ID and sampling priority
+        # inject! populates the env with span ID, trace ID and sampling priority
+        #
+        # DEV-2.0: inject! should work without arguments, injecting the active_trace's digest
+        # DEV-2.0: and returning a new Hash with the injected headers.
+        # DEV-2.0: inject! should also accept either a `trace` or a `digest`, as a `trace`
+        # DEV-2.0: argument is the common use case, but also allows us to set error tags in the `trace`
+        # DEV-2.0: if needed.
+        # DEV-2.0: Ideally, we'd have a separate stream to report tracer errors and never
+        # DEV-2.0: touch the active span.
         def self.inject!(digest, env)
           # Prevent propagation from being attempted if trace headers provided are nil.
           if digest.nil?

data/lib/datadog/tracing/sampling/ext.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # typed: strict
 
 module Datadog
@@ -21,6 +23,35 @@ module Datadog
           # through the {Datadog::Tracing::Sampling::RuleSampler}.
           USER_KEEP = 2
         end
+
+        # List of what mechanism was used to make the trace-level sampling decision.
+        module Mechanism
+          # Single Span Sampled.
+          SPAN_SAMPLING_RATE = 8
+        end
+
+        # List of how the decision was made for the trace-level sampling.
+        #
+        # These values used to populate the {Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER} tag.
+        #
+        # The decision has two parts, separated by a `-`:
+        # `part1-sampling_mechanism`. `part1` is currently not populated, thus
+        # this tag is currently formatted as `"-sampling_mechanism"`.
+        module Decision
+          # Used before the tracer receives any rates from agent and there are no rules configured.
+          DEFAULT = '-0'
+          # The sampling rate received in the agent's http response.
+          AGENT_RATE = '-1'
+          # Sampling rule or sampling rate based on tracer config.
+          TRACE_SAMPLING_RULE = '-3'
+          # User directly sets sampling priority via {Tracing.reject!} or {Tracing.keep!},
+          # or by a custom sampler implementation.
+          MANUAL = '-4'
+          # Formerly AppSec.
+          ASM = '-5'
+          # Single Span Sampled.
+          SPAN_SAMPLING_RATE = '-8'
+        end
       end
     end
   end

data/lib/datadog/tracing/sampling/priority_sampler.rb

@@ -21,17 +21,26 @@ module Datadog
 
         def initialize(opts = {})
           @pre_sampler = opts[:base_sampler] || AllSampler.new
-          @priority_sampler = opts[:post_sampler] || RateByServiceSampler.new
+          @priority_sampler = opts[:post_sampler] || RateByServiceSampler.new(decision: Sampling::Ext::Decision::AGENT_RATE)
         end
 
         def sample?(trace)
           @pre_sampler.sample?(trace)
         end
 
+        # DEV-2.0:We should get rid of this complicated interaction between @pre_sampler and @priority_sampler.
+        # DEV-2.0:If the user wants to configure a custom sampler, we should only allow them to provide a complete
+        # DEV-2.0:sampling suite, not having this convoluted support for mixing arbitrary provided samplers in
+        # DEV-2.0:the PrioritySampler. Ideally, the PrioritySampler is only used by Datadog.
+        # DEV-2.0:There are too many edge cases and combinations to work around currently in this class.
         def sample!(trace)
+          # The priority that was set before the sampler ran.
+          # This comes from distributed tracing priority propagation.
+          distributed_sampling_priority = priority_assigned?(trace)
+
           # If pre-sampling is configured, do it first. (By default, this will sample at 100%.)
           # NOTE: Pre-sampling at rates < 100% may result in partial traces; not recommended.
-          trace.sampled = pre_sample?(trace) ? @pre_sampler.sample!(trace) : true
+          trace.sampled = pre_sample?(trace) ? preserving_priority_sampling(trace) { @pre_sampler.sample!(trace) } : true
 
           if trace.sampled?
             # If priority sampling has already been applied upstream, use that value.
@@ -53,11 +62,27 @@ module Datadog
           end
 
           trace.sampled?
+        ensure
+          if trace.sampling_priority && trace.sampling_priority > 0
+            # Don't modify decision if priority was set upstream.
+            if !distributed_sampling_priority && !trace.has_tag?(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+              # If no sampling priority being assigned at this point, a custom
+              # sampler implementation is configured: this means the user has
+              # full control over the sampling decision.
+              trace.set_tag(
+                Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+                Sampling::Ext::Decision::MANUAL
+              )
+            end
+          else
+            # The sampler decided to not keep this span, removing sampling decision.
+            trace.clear_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+          end
         end
 
         # (see Datadog::Tracing::Sampling::RateByServiceSampler#update)
-        def update(rate_by_service)
-          @priority_sampler.update(rate_by_service)
+        def update(rate_by_service, decision: nil)
+          @priority_sampler.update(rate_by_service, decision: decision)
         end
 
         private
@@ -83,6 +108,23 @@ module Datadog
           end
         end
 
+        # Ensures the trace's priority sampling decision is not changed by the @pre_sampler.
+        # The @pre_sampler should only change `trace.sampled`.
+        def preserving_priority_sampling(trace)
+          sampling_priority = trace.sampling_priority
+          sampling_decision = trace.get_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+
+          yield.tap do
+            trace.sampling_priority = sampling_priority
+
+            if sampling_decision
+              trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, sampling_decision)
+            else
+              trace.clear_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+            end
+          end
+        end
+
         # Ensures the trace is always propagated to the writer and that
         # the sample rate metric represents the true client-side sampling.
         def preserving_sampling(trace)

data/lib/datadog/tracing/sampling/rate_by_key_sampler.rb

@@ -12,7 +12,7 @@ module Datadog
         attr_reader \
           :default_key
 
-        def initialize(default_key, default_rate = 1.0, &block)
+        def initialize(default_key, default_rate = 1.0, decision: nil, &block)
           super()
 
           raise ArgumentError, 'No resolver given!' unless block
@@ -22,7 +22,7 @@ module Datadog
           @mutex = Mutex.new
           @samplers = {}
 
-          set_rate(default_key, default_rate)
+          set_rate(default_key, default_rate, decision)
         end
 
         def resolve(trace)
@@ -57,15 +57,15 @@ module Datadog
           end
         end
 
-        def update(key, rate)
+        def update(key, rate, decision: nil)
           @mutex.synchronize do
-            set_rate(key, rate)
+            set_rate(key, rate, decision)
           end
         end
 
-        def update_all(rate_by_key)
+        def update_all(rate_by_key, decision: nil)
           @mutex.synchronize do
-            rate_by_key.each { |key, rate| set_rate(key, rate) }
+            rate_by_key.each { |key, rate| set_rate(key, rate, decision) }
           end
         end
 
@@ -87,9 +87,8 @@ module Datadog
 
         private
 
-        def set_rate(key, rate)
-          @samplers[key] ||= RateSampler.new(rate)
-          @samplers[key].sample_rate = rate
+        def set_rate(key, rate, decision)
+          @samplers[key] = RateSampler.new(rate, decision: decision)
         end
       end
     end

data/lib/datadog/tracing/sampling/rate_by_service_sampler.rb

@@ -11,17 +11,23 @@ module Datadog
       class RateByServiceSampler < RateByKeySampler
         DEFAULT_KEY = 'service:,env:'.freeze
 
-        def initialize(default_rate = 1.0, options = {})
-          super(DEFAULT_KEY, default_rate, &method(:key_for))
-          @env = options[:env]
+        def initialize(default_rate = 1.0, env: nil, decision: Datadog::Tracing::Sampling::Ext::Decision::DEFAULT)
+          super(
+            DEFAULT_KEY,
+            default_rate,
+            decision: decision,
+            &method(:key_for)
+          )
+
+          @env = env
         end
 
-        def update(rate_by_service)
+        def update(rate_by_service, decision: nil)
           # Remove any old services
           delete_if { |key, _| key != DEFAULT_KEY && !rate_by_service.key?(key) }
 
           # Update each service rate
-          update_all(rate_by_service)
+          update_all(rate_by_service, decision: decision)
 
           # Emit metric for service cache size
           Datadog.health_metrics.sampling_service_cache_length(length)
@@ -29,6 +35,24 @@ module Datadog
 
         private
 
+        # DEV: Creating a string on every trace to perform a single Hash lookup is expensive.
+        #
+        # Using 2 nested hashes: 1 for env and 1 for service is the fastest option.
+        # This approach requires large API changes to `RateByKeySampler`.
+        #
+        # Reducing the interpolated string size, by using a 1 character separator,
+        # is also measurably faster than the current method. This approach does not
+        # require changes to `RateByKeySampler`.
+        #
+        # Keep in mind that these changes also require changes to `#update`.
+        #
+        # Comparison:
+        #  2 nested hashes: `service_hash.fetch(service, {}).fetch(env, default_rate)`
+        #                   7730045 i/s
+        # 1 char separator: `hash.fetch("#{service}\0#{env}", default_rate)`
+        #                   4302801 i/s - 1.80x slower
+        #          current: `hash.fetch("service:#{service},env:#{env}", default_rate)`
+        #                   2720459 i/s - 2.84x slower
         def key_for(trace)
           # Resolve env dynamically, if Proc is given.
           env = @env.is_a?(Proc) ? @env.call : @env

data/lib/datadog/tracing/sampling/rate_limiter.rb

@@ -39,6 +39,9 @@ module Datadog
         def initialize(rate, max_tokens = rate)
           super()
 
+          raise ArgumentError, "rate must be a number: #{rate}" unless rate.is_a?(Numeric)
+          raise ArgumentError, "max_tokens must be a number: #{max_tokens}" unless max_tokens.is_a?(Numeric)
+
           @rate = rate
           @max_tokens = max_tokens
 

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -20,7 +20,17 @@ module Datadog
         # * +sample_rate+: the sample rate as a {Float} between 0.0 and 1.0. 0.0
         #   means that no trace will be sampled; 1.0 means that all traces will be
         #   sampled.
-        def initialize(sample_rate = 1.0)
+        #
+        # DEV-2.0: Allow for `sample_rate` zero (drop all) to be allowed. This eases
+        # DEV-2.0: usage for all internal users of the {RateSampler} class: both
+        # DEV-2.0: RuleSampler and Single Span Sampling leverage the RateSampler, but want
+        # DEV-2.0: `sample_rate` zero to mean "drop all". They work around this by hard-
+        # DEV-2.0: setting the `sample_rate` to zero like so:
+        # DEV-2.0: ```
+        # DEV-2.0: sampler = RateSampler.new
+        # DEV-2.0: sampler.sample_rate = sample_rate
+        # DEV-2.0: ```
+        def initialize(sample_rate = 1.0, decision: nil)
           super()
 
           unless sample_rate > 0.0 && sample_rate <= 1.0
@@ -29,6 +39,8 @@ module Datadog
           end
 
           self.sample_rate = sample_rate
+
+          @decision = decision
         end
 
         def sample_rate(*_)
@@ -46,8 +58,13 @@ module Datadog
 
         def sample!(trace)
           sampled = trace.sampled = sample?(trace)
-          trace.sample_rate = @sample_rate if sampled
-          sampled
+
+          return false unless sampled
+
+          trace.sample_rate = @sample_rate
+          trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, @decision) if @decision
+
+          true
         end
       end
     end

data/lib/datadog/tracing/sampling/rule_sampler.rb

@@ -48,7 +48,7 @@ module Datadog
                                nil
                              else
                                # TODO: Simplify .tags access, as `Tracer#tags` can't be arbitrarily changed anymore
-                               RateByServiceSampler.new(1.0, env: -> { Tracing.send(:tracer).tags[:env] })
+                               RateByServiceSampler.new(1.0, env: -> { Tracing.send(:tracer).tags['env'] })
                              end
         end
 
@@ -76,10 +76,10 @@ module Datadog
         end
 
         # @!visibility private
-        def update(*args)
+        def update(*args, **kwargs)
           return false unless @default_sampler.respond_to?(:update)
 
-          @default_sampler.update(*args)
+          @default_sampler.update(*args, **kwargs)
         end
 
         private
@@ -100,6 +100,7 @@ module Datadog
           rate_limiter.allow?(1).tap do |allowed|
             set_priority(trace, allowed)
             set_limiter_metrics(trace, rate_limiter.effective_rate)
+            trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, Ext::Decision::TRACE_SAMPLING_RULE)
           end
         rescue StandardError => e
           Datadog.logger.error(

data/lib/datadog/tracing/sampling/span/ext.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Sampling
+      module Span
+        # Single Span Sampling constants.
+        module Ext
+          # Accept all spans (100% retention).
+          DEFAULT_SAMPLE_RATE = 1.0
+          # Unlimited.
+          # @see Datadog::Tracing::Sampling::TokenBucket
+          DEFAULT_MAX_PER_SECOND = -1
+
+          # Sampling decision method used to come to the sampling decision for this span
+          TAG_MECHANISM = '_dd.span_sampling.mechanism'
+          # Sampling rate applied to this span, if a rule applies
+          TAG_RULE_RATE = '_dd.span_sampling.rule_rate'
+          # Rate limit configured for this span, if a rule applies
+          TAG_MAX_PER_SECOND = '_dd.span_sampling.max_per_second'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/span/matcher.rb

@@ -6,6 +6,8 @@ module Datadog
       module Span
         # Checks if a span conforms to a matching criteria.
         class Matcher
+          attr_reader :name, :service
+
           # Pattern that matches any string
           MATCH_ALL_PATTERN = '*'
 
@@ -54,6 +56,13 @@ module Datadog
             end
           end
 
+          def ==(other)
+            return super unless other.is_a?(Matcher)
+
+            name == other.name &&
+              service == other.service
+          end
+
           private
 
           # @param pattern [String]

data/lib/datadog/tracing/sampling/span/rule.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module Tracing
+    module Sampling
+      module Span
+        # Span sampling rule that applies a sampling rate if the span
+        # matches the provided {Matcher}.
+        # Additionally, a rate limiter is also applied.
+        #
+        # If a span does not conform to the matcher, no changes are made.
+        class Rule
+          attr_reader :matcher, :sample_rate, :rate_limit
+
+          # Creates a new span sampling rule.
+          #
+          # @param [Sampling::Span::Matcher] matcher whether this rule applies to a specific span
+          # @param [Float] sample_rate span sampling ratio, between 0.0 (0%) and 1.0 (100%).
+          # @param [Numeric] rate_limit maximum number of spans sampled per second. Negative numbers mean unlimited spans.
+          def initialize(
+            matcher,
+            sample_rate: Span::Ext::DEFAULT_SAMPLE_RATE,
+            rate_limit: Span::Ext::DEFAULT_MAX_PER_SECOND
+          )
+
+            @matcher = matcher
+            @sample_rate = sample_rate
+            @rate_limit = rate_limit
+
+            @sampler = Sampling::RateSampler.new
+            # Set the sample_rate outside of the initializer to allow for
+            # zero to be a "drop all".
+            # The RateSampler initializer enforces non-zero, falling back to 100% sampling
+            # if zero is provided.
+            @sampler.sample_rate = sample_rate
+            @rate_limiter = Sampling::TokenBucket.new(rate_limit)
+          end
+
+          # This method should only be invoked for spans that are part
+          # of a trace that has been dropped by trace-level sampling.
+          # Invoking it for other spans will cause incorrect sampling
+          # metrics to be reported by the Datadog App.
+          #
+          # Returns `true` if the provided span is sampled.
+          # If the span is dropped due to sampling rate or rate limiting,
+          # it returns `false`.
+          #
+          # Returns `nil` if the span did not meet the matching criteria by the
+          # provided matcher.
+          #
+          # This method modifies the `span` if it matches the provided matcher.
+          #
+          # @param [Datadog::Tracing::SpanOperation] span_op span to be sampled
+          # @return [:kept,:rejected] should this span be sampled?
+          # @return [:not_matched] span did not satisfy the matcher, no changes are made to the span
+          def sample!(span_op)
+            return :not_matched unless @matcher.match?(span_op)
+
+            if @sampler.sample?(span_op) && @rate_limiter.allow?(1)
+              span_op.set_metric(Span::Ext::TAG_MECHANISM, Sampling::Ext::Mechanism::SPAN_SAMPLING_RATE)
+              span_op.set_metric(Span::Ext::TAG_RULE_RATE, @sample_rate)
+              span_op.set_metric(Span::Ext::TAG_MAX_PER_SECOND, @rate_limit)
+              :kept
+            else
+              :rejected
+            end
+          end
+
+          def ==(other)
+            return super unless other.is_a?(Rule)
+
+            matcher == other.matcher &&
+              sample_rate == other.sample_rate &&
+              rate_limit == other.rate_limit
+          end
+        end
+      end
+    end
+  end
+end