ddtrace 0.35.2 → 1.1.0

data/lib/datadog/tracing/sampling/rate_limiter.rb

@@ -0,0 +1,185 @@
+# typed: true
+
+require 'datadog/core/utils/time'
+
+module Datadog
+  module Tracing
+    module Sampling
+      # Checks for rate limiting on a resource.
+      # @public_api
+      class RateLimiter
+        # Checks if resource of specified size can be
+        # conforms with the current limit.
+        #
+        # Implementations of this method are not guaranteed
+        # to be side-effect free.
+        #
+        # @return [Boolean] whether a resource conforms with the current limit
+        def allow?(size); end
+
+        # The effective rate limiting ratio based on
+        # recent calls to `allow?`.
+        #
+        # @return [Float] recent allowance ratio
+        def effective_rate; end
+      end
+
+      # Implementation of the Token Bucket metering algorithm
+      # for rate limiting.
+      #
+      # @see https://en.wikipedia.org/wiki/Token_bucket Token bucket
+      # @public_api
+      class TokenBucket < RateLimiter
+        attr_reader :rate, :max_tokens
+
+        # @param rate [Numeric] Allowance rate, in units per second
+        #  if rate is negative, always allow
+        #  if rate is zero, never allow
+        # @param max_tokens [Numeric] Limit of available tokens
+        def initialize(rate, max_tokens = rate)
+          super()
+
+          @rate = rate
+          @max_tokens = max_tokens
+
+          @tokens = max_tokens
+          @total_messages = 0
+          @conforming_messages = 0
+          @prev_conforming_messages = nil
+          @prev_total_messages = nil
+          @current_window = nil
+
+          @last_refill = Core::Utils::Time.get_time
+        end
+
+        # Checks if a message of provided +size+
+        # conforms with the current bucket limit.
+        #
+        # If it does, return +true+ and remove +size+
+        # tokens from the bucket.
+        # If it does not, return +false+ without affecting
+        # the tokens from the bucket.
+        #
+        # @return [Boolean] +true+ if message conforms with current bucket limit
+        def allow?(size)
+          allowed = should_allow?(size)
+          update_rate_counts(allowed)
+          allowed
+        end
+
+        # Ratio of 'conformance' per 'total messages' checked
+        # averaged for the past 2 buckets
+        #
+        # Returns +1.0+ when no messages have been checked yet.
+        #
+        # @return [Float] Conformance ratio, between +[0,1]+
+        def effective_rate
+          return 0.0 if @rate.zero?
+          return 1.0 if @rate < 0 || @total_messages.zero?
+
+          return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
+
+          (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
+        end
+
+        # Ratio of 'conformance' per 'total messages' checked
+        # on this bucket
+        #
+        # Returns +1.0+ when no messages have been checked yet.
+        #
+        # @return [Float] Conformance ratio, between +[0,1]+
+        def current_window_rate
+          return 1.0 if @total_messages.zero?
+
+          @conforming_messages.to_f / @total_messages
+        end
+
+        # @return [Numeric] number of tokens currently available
+        def available_tokens
+          @tokens
+        end
+
+        private
+
+        def refill_since_last_message
+          now = Core::Utils::Time.get_time
+          elapsed = now - @last_refill
+
+          # Update the number of available tokens, but ensure we do not exceed the max
+          # we return the min of tokens + rate*elapsed, or max tokens
+          refill_tokens(@rate * elapsed)
+
+          @last_refill = now
+        end
+
+        def refill_tokens(size)
+          @tokens += size
+          @tokens = @max_tokens if @tokens > @max_tokens
+        end
+
+        def increment_total_count
+          @total_messages += 1
+        end
+
+        def increment_conforming_count
+          @conforming_messages += 1
+        end
+
+        def should_allow?(size)
+          # rate limit of 0 blocks everything
+          return false if @rate.zero?
+
+          # negative rate limit disables rate limiting
+          return true if @rate < 0
+
+          refill_since_last_message
+
+          # if tokens < 1 we don't allow?
+          return false if @tokens < size
+
+          @tokens -= size
+
+          true
+        end
+
+        # Sets and Updates the past two 1 second windows for which
+        # the rate limiter must compute it's rate over and updates
+        # the total count, and conforming message count if +allowed+
+        def update_rate_counts(allowed)
+          now = Core::Utils::Time.get_time
+
+          # No tokens have been seen yet, start a new window
+          if @current_window.nil?
+            @current_window = now
+          # If more than 1 second has past since last window, reset
+          elsif now - @current_window >= 1
+            @prev_conforming_messages = @conforming_messages
+            @prev_total_messages = @total_messages
+            @conforming_messages = 0
+            @total_messages = 0
+            @current_window = now
+          end
+
+          increment_conforming_count if allowed
+
+          increment_total_count
+        end
+      end
+
+      # {Datadog::Tracing::Sampling::RateLimiter} that accepts all resources,
+      # with no limits.
+      # @public_api
+      class UnlimitedLimiter < RateLimiter
+        # @return [Boolean] always +true+
+        def allow?(_)
+          true
+        end
+
+        # @return [Float] always 100%
+        def effective_rate
+          1.0
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -0,0 +1,55 @@
+# typed: true
+
+require 'datadog/core'
+
+require 'datadog/tracing/sampling/sampler'
+require 'datadog/tracing/span'
+
+module Datadog
+  module Tracing
+    module Sampling
+      # {Datadog:::Tracing::Sampling::RateSampler} is based on a sample rate.
+      # @public_api
+      class RateSampler < Sampler
+        KNUTH_FACTOR = 1111111111111111111
+
+        # Initialize a {Datadog:::Tracing::Sampling::RateSampler}.
+        # This sampler keeps a random subset of the traces. Its main purpose is to
+        # reduce the instrumentation footprint.
+        #
+        # * +sample_rate+: the sample rate as a {Float} between 0.0 and 1.0. 0.0
+        #   means that no trace will be sampled; 1.0 means that all traces will be
+        #   sampled.
+        def initialize(sample_rate = 1.0)
+          super()
+
+          unless sample_rate > 0.0 && sample_rate <= 1.0
+            Datadog.logger.error('sample rate is not between 0 and 1, disabling the sampler')
+            sample_rate = 1.0
+          end
+
+          self.sample_rate = sample_rate
+        end
+
+        def sample_rate(*_)
+          @sample_rate
+        end
+
+        def sample_rate=(sample_rate)
+          @sample_rate = sample_rate
+          @sampling_id_threshold = sample_rate * Span::EXTERNAL_MAX_ID
+        end
+
+        def sample?(trace)
+          ((trace.id * KNUTH_FACTOR) % Span::EXTERNAL_MAX_ID) <= @sampling_id_threshold
+        end
+
+        def sample!(trace)
+          sampled = trace.sampled = sample?(trace)
+          trace.sample_rate = @sample_rate if sampled
+          sampled
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/rule.rb

@@ -0,0 +1,76 @@
+# typed: true
+
+require 'datadog/core'
+
+require 'datadog/tracing/sampling/matcher'
+require 'datadog/tracing/sampling/rate_sampler'
+
+module Datadog
+  module Tracing
+    module Sampling
+      # Sampling rule that dictates if a trace matches
+      # a specific criteria and what sampling strategy to
+      # apply in case of a positive match.
+      # @public_api
+      class Rule
+        attr_reader :matcher, :sampler
+
+        # @param [Matcher] matcher A matcher to verify trace conformity against
+        # @param [Sampler] sampler A sampler to be consulted on a positive match
+        def initialize(matcher, sampler)
+          @matcher = matcher
+          @sampler = sampler
+        end
+
+        # Evaluates if the provided `trace` conforms to the `matcher`.
+        #
+        # @param [TraceOperation] trace
+        # @return [Boolean] whether this rules applies to the trace
+        # @return [NilClass] if the matcher fails errs during evaluation
+        def match?(trace)
+          @matcher.match?(trace)
+        rescue => e
+          Datadog.logger.error(
+            "Matcher failed. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+          )
+          nil
+        end
+
+        # (see Datadog::Tracing::Sampling::Sampler#sample?)
+        def sample?(trace)
+          @sampler.sample?(trace)
+        end
+
+        # (see Datadog::Tracing::Sampling::Sampler#sample_rate)
+        def sample_rate(trace)
+          @sampler.sample_rate(trace)
+        end
+      end
+
+      # A {Datadog::Tracing::Sampling::Rule} that matches a trace based on
+      # trace name and/or service name and
+      # applies a fixed sampling to matching spans.
+      # @public_api
+      class SimpleRule < Rule
+        # @param name [String,Regexp,Proc] Matcher for case equality (===) with the trace name, defaults to always match
+        # @param service [String,Regexp,Proc] Matcher for case equality (===) with the service name,
+        #                defaults to always match
+        # @param sample_rate [Float] Sampling rate between +[0,1]+
+        def initialize(name: SimpleMatcher::MATCH_ALL, service: SimpleMatcher::MATCH_ALL, sample_rate: 1.0)
+          # We want to allow 0.0 to drop all traces, but {Datadog::Tracing::Sampling::RateSampler}
+          # considers 0.0 an invalid rate and falls back to 100% sampling.
+          #
+          # We address that here by not setting the rate in the constructor,
+          # but using the setter method.
+          #
+          # We don't want to make this change directly to {Datadog::Tracing::Sampling::RateSampler}
+          # because it breaks its current contract to existing users.
+          sampler = RateSampler.new
+          sampler.sample_rate = sample_rate
+
+          super(SimpleMatcher.new(name: name, service: service), sampler)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/rule_sampler.rb

@@ -0,0 +1,131 @@
+# typed: true
+
+require 'datadog/core'
+
+require 'datadog/tracing/sampling/ext'
+require 'datadog/tracing/sampling/rate_limiter'
+require 'datadog/tracing/sampling/rule'
+
+module Datadog
+  module Tracing
+    module Sampling
+      # Span {Sampler} that applies a set of {Rule}s to decide
+      # on sampling outcome. Then, a rate limiter is applied.
+      #
+      # If a trace does not conform to any rules, a default
+      # sampling strategy is applied.
+      # @public_api
+      class RuleSampler
+        attr_reader :rules, :rate_limiter, :default_sampler
+
+        # @param rules [Array<Rule>] ordered list of rules to be applied to a trace
+        # @param rate_limit [Float] number of traces per second, defaults to +100+
+        # @param rate_limiter [RateLimiter] limiter applied after rule matching
+        # @param default_sample_rate [Float] fallback sample rate when no rules apply to a trace,
+        #   between +[0,1]+, defaults to +1+
+        # @param default_sampler [Sample] fallback strategy when no rules apply to a trace
+        def initialize(
+          rules = [],
+          rate_limit: Datadog.configuration.tracing.sampling.rate_limit,
+          rate_limiter: nil,
+          default_sample_rate: Datadog.configuration.tracing.sampling.default_rate,
+          default_sampler: nil
+        )
+          @rules = rules
+          @rate_limiter = if rate_limiter
+                            rate_limiter
+                          elsif rate_limit
+                            TokenBucket.new(rate_limit)
+                          else
+                            UnlimitedLimiter.new
+                          end
+
+          @default_sampler = if default_sampler
+                               default_sampler
+                             elsif default_sample_rate
+                               # Add to the end of the rule list a rule always matches any trace
+                               @rules << SimpleRule.new(sample_rate: default_sample_rate)
+                               nil
+                             else
+                               # TODO: Simplify .tags access, as `Tracer#tags` can't be arbitrarily changed anymore
+                               RateByServiceSampler.new(1.0, env: -> { Tracing.send(:tracer).tags[:env] })
+                             end
+        end
+
+        # /RuleSampler's components (it's rate limiter, for example) are
+        # not be guaranteed to be size-effect free.
+        # It is not possible to guarantee that a call to {#sample?} will
+        # return the same result as a successive call to {#sample!} with the same trace.
+        #
+        # Use {#sample!} instead
+        def sample?(_trace)
+          raise 'RuleSampler cannot be evaluated without side-effects'
+        end
+
+        def sample!(trace)
+          sampled = sample_trace(trace) do |t|
+            @default_sampler.sample!(t).tap do
+              # We want to make sure the trace is tagged with the agent-derived
+              # service rate. Retrieve this from the rate by service sampler.
+              # Only do this if it was set by a RateByServiceSampler.
+              trace.agent_sample_rate = @default_sampler.sample_rate(trace) if @default_sampler.is_a?(RateByServiceSampler)
+            end
+          end
+
+          trace.sampled = sampled
+        end
+
+        # @!visibility private
+        def update(*args)
+          return false unless @default_sampler.respond_to?(:update)
+
+          @default_sampler.update(*args)
+        end
+
+        private
+
+        def sample_trace(trace)
+          rule = @rules.find { |r| r.match?(trace) }
+
+          return yield(trace) if rule.nil?
+
+          sampled = rule.sample?(trace)
+          sample_rate = rule.sample_rate(trace)
+
+          set_priority(trace, sampled)
+          set_rule_metrics(trace, sample_rate)
+
+          return false unless sampled
+
+          rate_limiter.allow?(1).tap do |allowed|
+            set_priority(trace, allowed)
+            set_limiter_metrics(trace, rate_limiter.effective_rate)
+          end
+        rescue StandardError => e
+          Datadog.logger.error(
+            "Rule sampling failed. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+          )
+          yield(trace)
+        end
+
+        # Span priority should only be set when the {RuleSampler}
+        # was responsible for the sampling decision.
+        def set_priority(trace, sampled)
+          trace.sampling_priority = if sampled
+                                      Sampling::Ext::Priority::USER_KEEP
+                                    else
+                                      Sampling::Ext::Priority::USER_REJECT
+                                    end
+        end
+
+        def set_rule_metrics(trace, sample_rate)
+          trace.rule_sample_rate = sample_rate
+        end
+
+        def set_limiter_metrics(trace, limiter_rate)
+          trace.rate_limiter_rate = limiter_rate
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/sampling/sampler.rb

@@ -0,0 +1,44 @@
+# typed: true
+
+module Datadog
+  module Tracing
+    module Sampling
+      # Interface for client-side trace sampling.
+      # @abstract
+      # @public_api
+      class Sampler
+        # Returns `true` if the provided trace should be kept.
+        # Otherwise, `false`.
+        #
+        # This method *must not* modify the `trace`.
+        #
+        # @param [Datadog::Tracing::TraceOperation] trace
+        # @return [Boolean] should this trace be kept?
+        def sample?(trace)
+          raise NotImplementedError, 'Samplers must implement the #sample? method'
+        end
+
+        # Returns `true` if the provided trace should be kept.
+        # Otherwise, `false`.
+        #
+        # This method *may* modify the `trace`, in case changes are necessary based on the
+        # sampling decision.
+        #
+        # @param [Datadog::Tracing::TraceOperation] trace
+        # @return [Boolean] should this trace be kept?
+        def sample!(trace)
+          raise NotImplementedError, 'Samplers must implement the #sample! method'
+        end
+
+        # The sampling rate, if this sampler has such concept.
+        # Otherwise, `nil`.
+        #
+        # @param [Datadog::Tracing::TraceOperation] trace
+        # @return [Float,nil] sampling ratio between 0.0 and 1.0 (inclusive), or `nil` if not applicable
+        def sample_rate(trace)
+          raise NotImplementedError, 'Samplers must implement the #sample_rate method'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/span.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require 'datadog/core/utils'
+require 'datadog/core/utils/safe_dup'
+
+require 'datadog/tracing/metadata/ext'
+require 'datadog/tracing/metadata'
+
+module Datadog
+  module Tracing
+    # Represents a logical unit of work in the system. Each trace consists of one or more spans.
+    # Each span consists of a start time and a duration. For example, a span can describe the time
+    # spent on a distributed call on a separate machine, or the time spent in a small component
+    # within a larger operation. Spans can be nested within each other, and in those instances
+    # will have a parent-child relationship.
+    # @public_api
+    class Span
+      include Metadata
+
+      # The max value for a {Datadog::Tracing::Span} identifier.
+      # Span and trace identifiers should be strictly positive and strictly inferior to this limit.
+      #
+      # Limited to +2<<62-1+ positive integers, as Ruby is able to represent such numbers "inline",
+      # inside a +VALUE+ scalar, thus not requiring memory allocation.
+      #
+      # The range of IDs also has to consider portability across different languages and platforms.
+      RUBY_MAX_ID = (1 << 62) - 1
+
+      # While we only generate 63-bit integers due to limitations in other languages, we support
+      # parsing 64-bit integers for distributed tracing since an upstream system may generate one
+      EXTERNAL_MAX_ID = 1 << 64
+
+      attr_accessor \
+        :end_time,
+        :id,
+        :meta,
+        :metrics,
+        :name,
+        :parent_id,
+        :resource,
+        :service,
+        :type,
+        :start_time,
+        :status,
+        :trace_id
+
+      attr_writer \
+        :duration
+
+      # For backwards compatiblity
+      # TODO: Deprecate and remove these.
+      alias :span_id :id
+      alias :span_type :type
+
+      # Create a new span manually. Call the <tt>start()</tt> method to start the time
+      # measurement and then <tt>stop()</tt> once the timing operation is over.
+      #
+      # * +service+: the service name for this span
+      # * +resource+: the resource this span refers, or +name+ if it's missing.
+      #     +nil+ can be used as a placeholder, when the resource value is not yet known at +#initialize+ time.
+      # * +type+: the type of the span (such as +http+, +db+ and so on)
+      # * +parent_id+: the identifier of the parent span
+      # * +trace_id+: the identifier of the root span for this trace
+      # TODO: Remove span_type
+      def initialize(
+        name,
+        duration: nil,
+        end_time: nil,
+        id: nil,
+        meta: nil,
+        metrics: nil,
+        parent_id: 0,
+        resource: name,
+        service: nil,
+        span_type: nil,
+        start_time: nil,
+        status: 0,
+        type: span_type,
+        trace_id: nil
+      )
+        @name = Core::Utils::SafeDup.frozen_or_dup(name)
+        @service = Core::Utils::SafeDup.frozen_or_dup(service)
+        @resource = Core::Utils::SafeDup.frozen_or_dup(resource)
+        @type = Core::Utils::SafeDup.frozen_or_dup(type)
+
+        @id = id || Core::Utils.next_id
+        @parent_id = parent_id || 0
+        @trace_id = trace_id || Core::Utils.next_id
+
+        @meta = meta || {}
+        @metrics = metrics || {}
+        @status = status || 0
+
+        # start_time and end_time track wall clock. In Ruby, wall clock
+        # has less accuracy than monotonic clock, so if possible we look to only use wall clock
+        # to measure duration when a time is supplied by the user, or if monotonic clock
+        # is unsupported.
+        @start_time = start_time
+        @end_time = end_time
+
+        # duration_start and duration_end track monotonic clock, and may remain nil in cases where it
+        # is known that we have to use wall clock to measure duration.
+        @duration = duration
+      end
+
+      # Return whether the duration is started or not
+      def started?
+        !@start_time.nil?
+      end
+
+      # Return whether the duration is stopped or not.
+      def stopped?
+        !@end_time.nil?
+      end
+      alias :finished? :stopped?
+
+      def duration
+        return @duration if @duration
+        return @end_time - @start_time if @start_time && @end_time
+      end
+
+      def set_error(e)
+        @status = Metadata::Ext::Errors::STATUS
+        super
+      end
+
+      # Spans with the same ID are considered the same span
+      def ==(other)
+        other.instance_of?(Span) &&
+          @id == other.id
+      end
+
+      # Return a string representation of the span.
+      def to_s
+        "Span(name:#{@name},sid:#{@id},tid:#{@trace_id},pid:#{@parent_id})"
+      end
+
+      # Return the hash representation of the current span.
+      # TODO: Change this to reflect attributes when serialization
+      # isn't handled by this method.
+      def to_hash
+        h = {
+          error: @status,
+          meta: @meta,
+          metrics: @metrics,
+          name: @name,
+          parent_id: @parent_id,
+          resource: @resource,
+          service: @service,
+          span_id: @id,
+          trace_id: @trace_id,
+          type: @type
+        }
+
+        if stopped?
+          h[:start] = start_time_nano
+          h[:duration] = duration_nano
+        end
+
+        h
+      end
+
+      # Return a human readable version of the span
+      def pretty_print(q)
+        start_time = (self.start_time.to_f * 1e9).to_i
+        end_time = (self.end_time.to_f * 1e9).to_i
+        q.group 0 do
+          q.breakable
+          q.text "Name: #{@name}\n"
+          q.text "Span ID: #{@id}\n"
+          q.text "Parent ID: #{@parent_id}\n"
+          q.text "Trace ID: #{@trace_id}\n"
+          q.text "Type: #{@type}\n"
+          q.text "Service: #{@service}\n"
+          q.text "Resource: #{@resource}\n"
+          q.text "Error: #{@status}\n"
+          q.text "Start: #{start_time}\n"
+          q.text "End: #{end_time}\n"
+          q.text "Duration: #{duration.to_f}\n"
+          q.group(2, 'Tags: [', "]\n") do
+            q.breakable
+            q.seplist @meta.each do |key, value|
+              q.text "#{key} => #{value}"
+            end
+          end
+          q.group(2, 'Metrics: [', ']') do
+            q.breakable
+            q.seplist @metrics.each do |key, value|
+              q.text "#{key} => #{value}"
+            end
+          end
+        end
+      end
+
+      private
+
+      # Used for serialization
+      # @return [Integer] in nanoseconds since Epoch
+      def start_time_nano
+        @start_time.to_i * 1000000000 + @start_time.nsec
+      end
+
+      # Used for serialization
+      # @return [Integer] in nanoseconds since Epoch
+      def duration_nano
+        (duration * 1e9).to_i
+      end
+    end
+  end
+end