ddtrace 0.13.0 → 0.47.0

data/lib/ddtrace/runtime/object_space.rb

@@ -0,0 +1,19 @@
+require 'objspace'
+
+module Datadog
+  module Runtime
+    # Retrieves garbage collection statistics
+    module ObjectSpace
+      module_function
+
+      def estimate_bytesize(object)
+        return 0 unless ::ObjectSpace.respond_to?(:memsize_of)
+
+        # Rough calculation of bytesize; not very accurate.
+        object.instance_variables.inject(::ObjectSpace.memsize_of(object)) do |sum, var|
+          sum + ::ObjectSpace.memsize_of(object.instance_variable_get(var))
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/runtime/socket.rb

@@ -0,0 +1,14 @@
+require 'socket'
+
+module Datadog
+  module Runtime
+    # For runtime identity
+    module Socket
+      module_function
+
+      def hostname
+        ::Socket.gethostname
+      end
+    end
+  end
+end

data/lib/ddtrace/runtime/thread_count.rb

@@ -0,0 +1,16 @@
+module Datadog
+  module Runtime
+    # Retrieves number of threads from runtime
+    module ThreadCount
+      module_function
+
+      def value
+        Thread.list.count
+      end
+
+      def available?
+        Thread.respond_to?(:list)
+      end
+    end
+  end
+end

data/lib/ddtrace/sampler.rb

@@ -1,28 +1,43 @@
 require 'forwardable'
 
 require 'ddtrace/ext/priority'
+require 'ddtrace/diagnostics/health'
 
 module Datadog
   # \Sampler performs client-side trace sampling.
   class Sampler
-    def sample(_span)
-      raise NotImplementedError, 'samplers have to implement the sample() method'
+    def sample?(_span)
+      raise NotImplementedError, 'Samplers must implement the #sample? method'
+    end
+
+    def sample!(_span)
+      raise NotImplementedError, 'Samplers must implement the #sample! method'
+    end
+
+    def sample_rate(span)
+      raise NotImplementedError, 'Samplers must implement the #sample_rate method'
     end
   end
 
   # \AllSampler samples all the traces.
   class AllSampler < Sampler
-    def sample(span)
+    def sample?(span)
+      true
+    end
+
+    def sample!(span)
       span.sampled = true
     end
+
+    def sample_rate(*_)
+      1.0
+    end
   end
 
   # \RateSampler is based on a sample rate.
   class RateSampler < Sampler
     KNUTH_FACTOR = 1111111111111111111
-    SAMPLE_RATE_METRIC_KEY = '_sample_rate'.freeze()
-
-    attr_reader :sample_rate
+    SAMPLE_RATE_METRIC_KEY = '_sample_rate'.freeze
 
     # Initialize a \RateSampler.
     # This sampler keeps a random subset of the traces. Its main purpose is to
@@ -33,58 +48,144 @@ module Datadog
     #   sampled.
     def initialize(sample_rate = 1.0)
       unless sample_rate > 0.0 && sample_rate <= 1.0
-        Datadog::Tracer.log.error('sample rate is not between 0 and 1, disabling the sampler')
+        Datadog.logger.error('sample rate is not between 0 and 1, disabling the sampler')
         sample_rate = 1.0
       end
 
       self.sample_rate = sample_rate
     end
 
+    def sample_rate(*_)
+      @sample_rate
+    end
+
     def sample_rate=(sample_rate)
       @sample_rate = sample_rate
-      @sampling_id_threshold = sample_rate * Span::MAX_ID
+      @sampling_id_threshold = sample_rate * Span::EXTERNAL_MAX_ID
     end
 
-    def sample(span)
-      span.set_metric(SAMPLE_RATE_METRIC_KEY, @sample_rate)
-      span.sampled = ((span.trace_id * KNUTH_FACTOR) % Datadog::Span::MAX_ID) <= @sampling_id_threshold
+    def sample?(span)
+      ((span.trace_id * KNUTH_FACTOR) % Datadog::Span::EXTERNAL_MAX_ID) <= @sampling_id_threshold
+    end
+
+    def sample!(span)
+      (span.sampled = sample?(span)).tap do |sampled|
+        span.set_metric(SAMPLE_RATE_METRIC_KEY, @sample_rate) if sampled
+      end
     end
   end
 
-  # \RateByServiceSampler samples different services at different rates
-  class RateByServiceSampler < Sampler
-    DEFAULT_KEY = 'service:,env:'.freeze
+  # Samples at different rates by key.
+  class RateByKeySampler < Sampler
+    attr_reader \
+      :default_key
+
+    def initialize(default_key, default_rate = 1.0, &block)
+      raise ArgumentError, 'No resolver given!' unless block
 
-    def initialize(rate = 1.0, opts = {})
-      @env = opts.fetch(:env, Datadog.tracer.tags[:env])
+      @default_key = default_key
+      @resolver = block
       @mutex = Mutex.new
-      @fallback = RateSampler.new(rate)
-      @sampler = { DEFAULT_KEY => @fallback }
+      @samplers = {}
+
+      set_rate(default_key, default_rate)
+    end
+
+    def resolve(span)
+      @resolver.call(span)
+    end
+
+    def default_sampler
+      @samplers[default_key]
     end
 
-    def sample(span)
-      key = key_for(span)
+    def sample?(span)
+      key = resolve(span)
 
       @mutex.synchronize do
-        @sampler.fetch(key, @fallback).sample(span)
+        @samplers.fetch(key, default_sampler).sample?(span)
       end
     end
 
-    def update(rate_by_service)
+    def sample!(span)
+      key = resolve(span)
+
       @mutex.synchronize do
-        @sampler.delete_if { |key, _| key != DEFAULT_KEY && !rate_by_service.key?(key) }
+        @samplers.fetch(key, default_sampler).sample!(span)
+      end
+    end
 
-        rate_by_service.each do |key, rate|
-          @sampler[key] ||= RateSampler.new(rate)
-          @sampler[key].sample_rate = rate
-        end
+    def sample_rate(span)
+      key = resolve(span)
+
+      @mutex.synchronize do
+        @samplers.fetch(key, default_sampler).sample_rate
+      end
+    end
+
+    def update(key, rate)
+      @mutex.synchronize do
+        set_rate(key, rate)
+      end
+    end
+
+    def update_all(rate_by_key)
+      @mutex.synchronize do
+        rate_by_key.each { |key, rate| set_rate(key, rate) }
+      end
+    end
+
+    def delete(key)
+      @mutex.synchronize do
+        @samplers.delete(key)
+      end
+    end
+
+    def delete_if(&block)
+      @mutex.synchronize do
+        @samplers.delete_if(&block)
       end
     end
 
+    def length
+      @samplers.length
+    end
+
+    private
+
+    def set_rate(key, rate)
+      @samplers[key] ||= RateSampler.new(rate)
+      @samplers[key].sample_rate = rate
+    end
+  end
+
+  # \RateByServiceSampler samples different services at different rates
+  class RateByServiceSampler < RateByKeySampler
+    DEFAULT_KEY = 'service:,env:'.freeze
+
+    def initialize(default_rate = 1.0, options = {})
+      super(DEFAULT_KEY, default_rate, &method(:key_for))
+      @env = options[:env]
+    end
+
+    def update(rate_by_service)
+      # Remove any old services
+      delete_if { |key, _| key != DEFAULT_KEY && !rate_by_service.key?(key) }
+
+      # Update each service rate
+      update_all(rate_by_service)
+
+      # Emit metric for service cache size
+      Datadog.health_metrics.sampling_service_cache_length(length)
+    end
+
     private
 
     def key_for(span)
-      "service:#{span.service},env:#{@env}"
+      # Resolve env dynamically, if Proc is given.
+      env = @env.is_a?(Proc) ? @env.call : @env
+
+      "service:#{span.service},env:#{env}"
     end
   end
 
@@ -92,20 +193,100 @@ module Datadog
   class PrioritySampler
     extend Forwardable
 
+    attr_reader :pre_sampler, :priority_sampler
+
+    SAMPLE_RATE_METRIC_KEY = '_sample_rate'.freeze
+
     def initialize(opts = {})
-      @base_sampler = opts[:base_sampler] || RateSampler.new
-      @post_sampler = opts[:post_sampler] || RateByServiceSampler.new
+      @pre_sampler = opts[:base_sampler] || AllSampler.new
+      @priority_sampler = opts[:post_sampler] || RateByServiceSampler.new
     end
 
-    def sample(span)
-      span.context.sampling_priority = Datadog::Ext::Priority::AUTO_REJECT if span.context
-      return unless @base_sampler.sample(span)
-      return unless @post_sampler.sample(span)
-      span.context.sampling_priority = Datadog::Ext::Priority::AUTO_KEEP if span.context
+    def sample?(span)
+      @pre_sampler.sample?(span)
+    end
 
-      true
+    def sample!(span)
+      # If pre-sampling is configured, do it first. (By default, this will sample at 100%.)
+      # NOTE: Pre-sampling at rates < 100% may result in partial traces; not recommended.
+      span.sampled = pre_sample?(span) ? @pre_sampler.sample!(span) : true
+
+      if span.sampled
+        # If priority sampling has already been applied upstream, use that, otherwise...
+        unless priority_assigned_upstream?(span)
+          # Roll the dice and determine whether how we set the priority.
+          priority = priority_sample!(span) ? Datadog::Ext::Priority::AUTO_KEEP : Datadog::Ext::Priority::AUTO_REJECT
+
+          assign_priority!(span, priority)
+        end
+      else
+        # If discarded by pre-sampling, set "reject" priority, so other
+        # services for the same trace don't sample needlessly.
+        assign_priority!(span, Datadog::Ext::Priority::AUTO_REJECT)
+      end
+
+      span.sampled
+    end
+
+    def_delegators :@priority_sampler, :update
+
+    private
+
+    def pre_sample?(span)
+      case @pre_sampler
+      when RateSampler
+        @pre_sampler.sample_rate < 1.0
+      when RateByServiceSampler
+        @pre_sampler.sample_rate(span) < 1.0
+      else
+        true
+      end
+    end
+
+    def priority_assigned_upstream?(span)
+      span.context && !span.context.sampling_priority.nil?
+    end
+
+    def priority_sample!(span)
+      preserving_sampling(span) do
+        @priority_sampler.sample!(span)
+      end
     end
 
-    def_delegators :@post_sampler, :update
+    # Ensures the span is always propagated to the writer and that
+    # the sample rate metric represents the true client-side sampling.
+    def preserving_sampling(span)
+      pre_sample_rate_metric = span.get_metric(SAMPLE_RATE_METRIC_KEY)
+
+      yield.tap do
+        # NOTE: We'll want to leave `span.sampled = true` here; all spans for priority sampling must
+        #       be sent to the agent. Otherwise metrics for traces will not be accurate, since the
+        #       agent will have an incomplete dataset.
+        #
+        #       We also ensure that the agent knows we that our `post_sampler` is not performing true sampling,
+        #       to avoid erroneous metric upscaling.
+        span.sampled = true
+        if pre_sample_rate_metric
+          # Restore true sampling metric, as only the @pre_sampler can reject traces
+          span.set_metric(SAMPLE_RATE_METRIC_KEY, pre_sample_rate_metric)
+        else
+          # If @pre_sampler is not enable, sending this metric would be misleading
+          span.clear_metric(SAMPLE_RATE_METRIC_KEY)
+        end
+      end
+    end
+
+    def assign_priority!(span, priority)
+      if span.context
+        span.context.sampling_priority = priority
+      else
+        # Set the priority directly on the span instead, since otherwise
+        # it won't receive the appropriate tag.
+        span.set_metric(
+          Ext::DistributedTracing::SAMPLING_PRIORITY_KEY,
+          priority
+        )
+      end
+    end
   end
 end

data/lib/ddtrace/sampling.rb

@@ -0,0 +1,2 @@
+require 'ddtrace/sampling/rule'
+require 'ddtrace/sampling/rule_sampler'

data/lib/ddtrace/sampling/matcher.rb

@@ -0,0 +1,57 @@
+module Datadog
+  module Sampling
+    # Checks if a span conforms to a matching criteria.
+    class Matcher
+      # Returns `true` if the span should conforms to this rule, `false` otherwise
+      #
+      # @abstract
+      # @param [Span] span
+      # @return [Boolean]
+      def match?(span)
+        raise NotImplementedError
+      end
+    end
+
+    # A \Matcher that supports matching a span by
+    # operation name and/or service name.
+    class SimpleMatcher < Matcher
+      # Returns `true` for case equality (===) with any object
+      MATCH_ALL = Class.new do
+        # DEV: A class that implements `#===` is ~20% faster than
+        # DEV: a `Proc` that always returns `true`.
+        def ===(other)
+          true
+        end
+      end.new
+
+      attr_reader :name, :service
+
+      # @param name [String,Regexp,Proc] Matcher for case equality (===) with the span name, defaults to always match
+      # @param service [String,Regexp,Proc] Matcher for case equality (===) with the service name, defaults to always match
+      def initialize(name: MATCH_ALL, service: MATCH_ALL)
+        @name = name
+        @service = service
+      end
+
+      def match?(span)
+        name === span.name && service === span.service
+      end
+    end
+
+    # A \Matcher that allows for arbitrary span matching
+    # based on the return value of a provided block.
+    class ProcMatcher < Matcher
+      attr_reader :block
+
+      # @yield [name, service] Provides span name and service to the block
+      # @yieldreturn [Boolean] Whether the span conforms to this matcher
+      def initialize(&block)
+        @block = block
+      end
+
+      def match?(span)
+        block.call(span.name, span.service)
+      end
+    end
+  end
+end

data/lib/ddtrace/sampling/rate_limiter.rb

@@ -0,0 +1,176 @@
+require 'ddtrace/utils/time'
+
+module Datadog
+  module Sampling
+    # Checks for rate limiting on a resource.
+    class RateLimiter
+      # Checks if resource of specified size can be
+      # conforms with the current limit.
+      #
+      # Implementations of this method are not guaranteed
+      # to be side-effect free.
+      #
+      # @return [Boolean] whether a resource conforms with the current limit
+      def allow?(size); end
+
+      # The effective rate limiting ratio based on
+      # recent calls to `allow?`.
+      #
+      # @return [Float] recent allowance ratio
+      def effective_rate; end
+    end
+
+    # Implementation of the Token Bucket metering algorithm
+    # for rate limiting.
+    #
+    # @see https://en.wikipedia.org/wiki/Token_bucket Token bucket
+    class TokenBucket < RateLimiter
+      attr_reader :rate, :max_tokens
+
+      # @param rate [Numeric] Allowance rate, in units per second
+      #  if rate is negative, always allow
+      #  if rate is zero, never allow
+      # @param max_tokens [Numeric] Limit of available tokens
+      def initialize(rate, max_tokens = rate)
+        @rate = rate
+        @max_tokens = max_tokens
+
+        @tokens = max_tokens
+        @total_messages = 0
+        @conforming_messages = 0
+        @prev_conforming_messages = nil
+        @prev_total_messages = nil
+        @current_window = nil
+
+        @last_refill = Utils::Time.get_time
+      end
+
+      # Checks if a message of provided +size+
+      # conforms with the current bucket limit.
+      #
+      # If it does, return +true+ and remove +size+
+      # tokens from the bucket.
+      # If it does not, return +false+ without affecting
+      # the tokens from the bucket.
+      #
+      # @return [Boolean] +true+ if message conforms with current bucket limit
+      def allow?(size)
+        allowed = should_allow?(size)
+        update_rate_counts(allowed)
+        allowed
+      end
+
+      # Ratio of 'conformance' per 'total messages' checked
+      # averaged for the past 2 buckets
+      #
+      # Returns +1.0+ when no messages have been checked yet.
+      #
+      # @return [Float] Conformance ratio, between +[0,1]+
+      def effective_rate
+        return 0.0 if @rate.zero?
+        return 1.0 if @rate < 0 || @total_messages.zero?
+
+        return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
+
+        (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
+      end
+
+      # Ratio of 'conformance' per 'total messages' checked
+      # on this bucket
+      #
+      # Returns +1.0+ when no messages have been checked yet.
+      #
+      # @return [Float] Conformance ratio, between +[0,1]+
+      def current_window_rate
+        return 1.0 if @total_messages.zero?
+
+        @conforming_messages.to_f / @total_messages
+      end
+
+      # @return [Numeric] number of tokens currently available
+      def available_tokens
+        @tokens
+      end
+
+      private
+
+      def refill_since_last_message
+        now = Utils::Time.get_time
+        elapsed = now - @last_refill
+
+        # Update the number of available tokens, but ensure we do not exceed the max
+        # we return the min of tokens + rate*elapsed, or max tokens
+        refill_tokens(@rate * elapsed)
+
+        @last_refill = now
+      end
+
+      def refill_tokens(size)
+        @tokens += size
+        @tokens = @max_tokens if @tokens > @max_tokens
+      end
+
+      def increment_total_count
+        @total_messages += 1
+      end
+
+      def increment_conforming_count
+        @conforming_messages += 1
+      end
+
+      def should_allow?(size)
+        # rate limit of 0 blocks everything
+        return false if @rate.zero?
+
+        # negative rate limit disables rate limiting
+        return true if @rate < 0
+
+        refill_since_last_message
+
+        # if tokens < 1 we don't allow?
+        return false if @tokens < size
+
+        @tokens -= size
+
+        true
+      end
+
+      # Sets and Updates the past two 1 second windows for which
+      # the rate limiter must compute it's rate over and updates
+      # the total count, and conforming message count if +allowed+
+      def update_rate_counts(allowed)
+        now = Utils::Time.get_time
+
+        # No tokens have been seen yet, start a new window
+        if @current_window.nil?
+          @current_window = now
+        # If more than 1 second has past since last window, reset
+        elsif now - @current_window >= 1
+          @prev_conforming_messages = @conforming_messages
+          @prev_total_messages = @total_messages
+          @conforming_messages = 0
+          @total_messages = 0
+          @current_window = now
+        end
+
+        increment_conforming_count if allowed
+
+        increment_total_count
+      end
+    end
+
+    # \RateLimiter that accepts all resources,
+    # with no limits.
+    class UnlimitedLimiter < RateLimiter
+      # @return [Boolean] always +true+
+      def allow?(_)
+        true
+      end
+
+      # @return [Float] always 100%
+      def effective_rate
+        1.0
+      end
+    end
+  end
+end