ddtrace 0.36.0 → 0.41.0

data/lib/ddtrace.rb

@@ -55,8 +55,11 @@ require 'ddtrace/contrib/grape/integration'
 require 'ddtrace/contrib/graphql/integration'
 require 'ddtrace/contrib/grpc/integration'
 require 'ddtrace/contrib/http/integration'
+require 'ddtrace/contrib/httprb/integration'
 require 'ddtrace/contrib/integration'
+require 'ddtrace/contrib/kafka/integration'
 require 'ddtrace/contrib/presto/integration'
+require 'ddtrace/contrib/que/integration'
 require 'ddtrace/contrib/mysql2/integration'
 require 'ddtrace/contrib/mongodb/integration'
 require 'ddtrace/contrib/racecar/integration'
@@ -70,5 +73,6 @@ require 'ddtrace/contrib/sequel/integration'
 require 'ddtrace/contrib/shoryuken/integration'
 require 'ddtrace/contrib/sidekiq/integration'
 require 'ddtrace/contrib/sinatra/integration'
+require 'ddtrace/contrib/sneakers/integration'
 require 'ddtrace/contrib/sucker_punch/integration'
 require 'ddtrace/monkey'

data/lib/ddtrace/buffer.rb

@@ -2,23 +2,79 @@ require 'thread'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/runtime/object_space'
 
+# Trace buffer that accumulates traces for a consumer.
+# Consumption can happen from a different thread.
 module Datadog
-  # Trace buffer that stores application traces. The buffer has a maximum size and when
-  # the buffer is full, a random trace is discarded. This class is thread-safe and is used
-  # automatically by the ``Tracer`` instance when a ``Span`` is finished.
-  class TraceBuffer
-    def initialize(max_size)
-      @max_size = max_size
+  # Aggregate metrics:
+  # They reflect buffer activity since last #pop.
+  # These may not be as accurate or as granular, but they
+  # don't use as much network traffic as live stats.
+  class MeasuredBuffer
+    def initialize
+      @buffer_accepted = 0
+      @buffer_accepted_lengths = 0
+      @buffer_dropped = 0
+      @buffer_spans = 0
+    end
 
-      @mutex = Mutex.new()
-      @traces = []
-      @closed = false
+    def measure_accept(trace)
+      @buffer_accepted += 1
+      @buffer_accepted_lengths += trace.length
+
+      @buffer_spans += trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
 
-      # Initialize metric values
+    def measure_drop(trace)
+      @buffer_dropped += 1
+
+      @buffer_spans -= trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+
+    def measure_pop(traces)
+      # Accepted, cumulative totals
+      Datadog.health_metrics.queue_accepted(@buffer_accepted)
+      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+
+      # Dropped, cumulative totals
+      Datadog.health_metrics.queue_dropped(@buffer_dropped)
+      # TODO: are we missing a +queue_dropped_lengths+ metric?
+
+      # Queue gauges, current values
+      Datadog.health_metrics.queue_max_length(@max_size)
+      Datadog.health_metrics.queue_spans(@buffer_spans)
+      Datadog.health_metrics.queue_length(traces.length)
+
+      # Reset aggregated metrics
       @buffer_accepted = 0
       @buffer_accepted_lengths = 0
       @buffer_dropped = 0
       @buffer_spans = 0
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+  end
+
+  # Trace buffer that stores application traces and
+  # can be safely used concurrently on any environment.
+  #
+  # This implementation uses a {Mutex} around public methods, incurring
+  # overhead in order to ensure full thread-safety.
+  #
+  # This is implementation is recommended for non-CRuby environments.
+  # If using CRuby, {Datadog::CRubyTraceBuffer} is a faster implementation with minimal compromise.
+  class ThreadSafeBuffer < MeasuredBuffer
+    def initialize(max_size)
+      super()
+
+      @max_size = max_size
+
+      @mutex = Mutex.new()
+      @traces = []
+      @closed = false
     end
 
     # Add a new ``trace`` in the local queue. This method doesn't block the execution
@@ -72,48 +128,103 @@ module Datadog
         @closed = true
       end
     end
+  end
+
+  # Trace buffer that stores application traces and
+  # can be safely used concurrently with CRuby.
+  #
+  # Under extreme concurrency scenarios, this class can exceed
+  # its +max_size+ by up to 4%.
+  #
+  # Because singular +Array+ operations are thread-safe in CRuby,
+  # we can implement the trace buffer without an explicit lock,
+  # while making the compromise of allowing the buffer to go
+  # over its maximum limit under extreme circumstances.
+  #
+  # On the following scenario:
+  # * 4.5 million spans/second.
+  # * Pushed into a single CRubyTraceBuffer from 1000 threads.
+  # The buffer can exceed its maximum size by no more than 4%.
+  #
+  # This implementation allocates less memory and is faster
+  # than {Datadog::ThreadSafeBuffer}.
+  #
+  # @see spec/ddtrace/benchmark/buffer_benchmark_spec.rb Buffer benchmarks
+  # @see https://github.com/ruby-concurrency/concurrent-ruby/blob/c1114a0c6891d9634f019f1f9fe58dcae8658964/lib/concurrent-ruby/concurrent/array.rb#L23-L27
+  class CRubyTraceBuffer < MeasuredBuffer
+    def initialize(max_size)
+      super()
+
+      @max_size = max_size
+
+      @traces = []
+      @closed = false
+    end
 
-    # Aggregate metrics:
-    # They reflect buffer activity since last #pop.
-    # These may not be as accurate or as granular, but they
-    # don't use as much network traffic as live stats.
+    # Add a new ``trace`` in the local queue. This method doesn't block the execution
+    # even if the buffer is full. In that case, a random trace is discarded.
+    def push(trace)
+      return if @closed
+      len = @traces.length
+      if len < @max_size || @max_size <= 0
+        @traces << trace
+      else
+        # we should replace a random trace with the new one
+        replace_index = rand(len)
+        replaced_trace = @traces.delete_at(replace_index)
+        @traces << trace
+
+        # Check if we deleted the element right when the buffer
+        # was popped. In that case we didn't actually delete anything,
+        # we just inserted into a newly cleared buffer instead.
+        measure_drop(replaced_trace) if replaced_trace
+      end
 
-    def measure_accept(trace)
-      @buffer_spans += trace.length
-      @buffer_accepted += 1
-      @buffer_accepted_lengths += trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
+      measure_accept(trace)
     end
 
-    def measure_drop(trace)
-      @buffer_dropped += 1
-      @buffer_spans -= trace.length
-      @buffer_accepted_lengths -= trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    # Return the current number of stored traces.
+    def length
+      @traces.length
     end
 
-    def measure_pop(traces)
-      # Accepted
-      Datadog.health_metrics.queue_accepted(@buffer_accepted)
-      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+    # Return if the buffer is empty.
+    def empty?
+      @traces.empty?
+    end
 
-      # Dropped
-      Datadog.health_metrics.queue_dropped(@buffer_dropped)
+    # Return all traces stored and reset buffer.
+    def pop
+      traces = @traces.pop(VERY_LARGE_INTEGER)
 
-      # Queue gauges
-      Datadog.health_metrics.queue_max_length(@max_size)
-      Datadog.health_metrics.queue_spans(@buffer_spans)
-      Datadog.health_metrics.queue_length(traces.length)
+      measure_pop(traces)
 
-      # Reset aggregated metrics
-      @buffer_accepted = 0
-      @buffer_accepted_lengths = 0
-      @buffer_dropped = 0
-      @buffer_spans = 0
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+      traces
+    end
+
+    # Very large value, to ensure that we drain the whole buffer.
+    # 1<<62-1 happens to be the largest integer that can be stored inline in CRuby.
+    VERY_LARGE_INTEGER = 1 << 62 - 1
+
+    def close
+      @closed = true
     end
   end
+
+  # Choose default TraceBuffer implementation for current platform.
+  BUFFER_IMPLEMENTATION = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby'
+                            CRubyTraceBuffer
+                          else
+                            ThreadSafeBuffer
+                          end
+  private_constant :BUFFER_IMPLEMENTATION
+
+  # Trace buffer that stores application traces. The buffer has a maximum size and when
+  # the buffer is full, a random trace is discarded. This class is thread-safe and is used
+  # automatically by the ``Tracer`` instance when a ``Span`` is finished.
+  #
+  # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
+  # TODO Making such a change is potentially breaking for users manually configuring the tracer.
+  class TraceBuffer < BUFFER_IMPLEMENTATION
+  end
 end

data/lib/ddtrace/configuration.rb

@@ -22,9 +22,9 @@ module Datadog
         # Build immutable components from settings
         @components ||= nil
         @components = if @components
-                        Components.replace!(@components, target)
+                        replace_components!(target, @components)
                       else
-                        Components.new(target)
+                        build_components(target)
                       end
 
         target
@@ -36,18 +36,52 @@ module Datadog
     def_delegators \
       :components,
       :health_metrics,
-      :logger,
       :runtime_metrics,
       :tracer
 
+    def logger
+      if instance_variable_defined?(:@components) && @components
+        @temp_logger = nil
+        components.logger
+      else
+        # Use default logger without initializing components.
+        # This prevents recursive loops while initializing.
+        # e.g. Get logger --> Build components --> Log message --> Repeat...
+        @temp_logger ||= begin
+          logger = configuration.logger.instance || Datadog::Logger.new(STDOUT)
+          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+          logger
+        end
+      end
+    end
+
     def shutdown!
-      components.teardown! if @components
+      if instance_variable_defined?(:@components) && @components
+        components.shutdown!
+        @components = nil
+      end
     end
 
     protected
 
     def components
-      @components ||= Components.new(configuration)
+      @components ||= build_components(configuration)
+    end
+
+    private
+
+    def build_components(settings)
+      components = Components.new(settings)
+      components.startup!(settings)
+      components
+    end
+
+    def replace_components!(settings, old)
+      components = Components.new(settings)
+
+      old.shutdown!(components)
+      components.startup!(settings)
+      components
     end
   end
 end

data/lib/ddtrace/configuration/components.rb

@@ -10,12 +10,6 @@ module Datadog
     # rubocop:disable Metrics/LineLength
     class Components
       class << self
-        def replace!(old, settings)
-          replacement = new(settings)
-          old.teardown!(replacement)
-          replacement
-        end
-
         def build_health_metrics(settings)
           settings = settings.diagnostics.health_metrics
           options = { enabled: settings.enabled }
@@ -118,10 +112,13 @@ module Datadog
         @health_metrics = self.class.build_health_metrics(settings)
       end
 
+      # Starts up components
+      def startup!(settings); end
+
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
       # and avoid tearing down parts still in use.
-      def teardown!(replacement = nil)
+      def shutdown!(replacement = nil)
         # Shutdown the old tracer, unless it's still being used.
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer

data/lib/ddtrace/configuration/options.rb

@@ -26,7 +26,9 @@ module Datadog
           builder = OptionDefinition::Builder.new(name, meta, &block)
           options[name] = builder.to_definition.tap do
             # Resolve and define helper functions
-            helpers = default_helpers(name).merge(builder.helpers)
+            helpers = default_helpers(name)
+            # Prevent unnecessary creation of an identical copy of helpers if there's nothing to merge
+            helpers = helpers.merge(builder.helpers) unless builder.helpers.empty?
             define_helpers(helpers)
           end
         end

data/lib/ddtrace/configuration/pin_setup.rb

@@ -13,7 +13,7 @@ module Datadog
 
         ATTRS.each { |key| pin.public_send("#{key}=", opts[key]) if opts[key] }
 
-        pin.config = opts.reject { |key, _| ATTRS.include?(key) }
+        pin.config = opts.reject { |key, _| ATTRS.include?(key) || DEPRECATED_ATTRS.include?(key) }
 
         true
       end
@@ -22,7 +22,8 @@ module Datadog
 
       attr_reader :pin, :opts
 
-      ATTRS = [:app, :tags, :app_type, :name, :tracer, :service_name].freeze
+      ATTRS = [:app, :tags, :app_type, :name, :service_name].freeze
+      DEPRECATED_ATTRS = [:tracer].freeze
 
       private_constant :ATTRS
     end

data/lib/ddtrace/configuration/settings.rb

@@ -31,8 +31,21 @@ module Datadog
         end
       end
 
+      option :api_key do |o|
+        o.default { ENV.fetch(Ext::Environment::ENV_API_KEY, nil) }
+        o.lazy
+      end
+
       settings :diagnostics do
-        option :debug, default: false
+        option :debug do |o|
+          o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_DEBUG, false) }
+          o.lazy
+          o.on_set do |enabled|
+            # Enable rich debug print statements.
+            # We do not need to unnecessarily load 'pp' unless in debugging mode.
+            require 'pp' if enabled
+          end
+        end
 
         settings :health_metrics do
           option :enabled do |o|
@@ -42,6 +55,14 @@ module Datadog
 
           option :statsd
         end
+
+        settings :startup_logs do
+          option :enabled do |o|
+            # Defaults to nil as we want to know when the default value is being used
+            o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_STARTUP_LOGS, nil) }
+            o.lazy
+          end
+        end
       end
 
       settings :distributed_tracing do
@@ -75,11 +96,10 @@ module Datadog
 
       settings :logger do
         option :instance do |o|
-          o.setter { |value, old_value| value.is_a?(::Logger) ? value : old_value }
           o.on_set { |value| set_option(:level, value.level) unless value.nil? }
         end
 
-        option :level, default: ::Logger::WARN
+        option :level, default: ::Logger::INFO
       end
 
       def logger=(logger)
@@ -138,6 +158,11 @@ module Datadog
         o.lazy
       end
 
+      option :site do |o|
+        o.default { ENV.fetch(Ext::Environment::ENV_SITE, nil) }
+        o.lazy
+      end
+
       option :tags do |o|
         o.default do
           tags = {}
@@ -180,7 +205,10 @@ module Datadog
       end
 
       settings :tracer do
-        option :enabled, default: true
+        option :enabled do |o|
+          o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_ENABLED, true) }
+          o.lazy
+        end
         option :hostname # TODO: Deprecate
         option :instance
 

data/lib/ddtrace/contrib/action_cable/configuration/settings.rb

@@ -7,13 +7,18 @@ module Datadog
       module Configuration
         # Custom settings for the ActionCable integration
         class Settings < Contrib::Configuration::Settings
+          option :enabled do |o|
+            o.default { env_to_bool(Ext::ENV_ENABLED, true) }
+            o.lazy
+          end
+
           option :analytics_enabled do |o|
-            o.default { env_to_bool(Ext::ENV_ANALYTICS_ENABLED, false) }
+            o.default { env_to_bool([Ext::ENV_ANALYTICS_ENABLED, Ext::ENV_ANALYTICS_ENABLED_OLD], false) }
             o.lazy
           end
 
           option :analytics_sample_rate do |o|
-            o.default { env_to_float(Ext::ENV_ANALYTICS_SAMPLE_RATE, 1.0) }
+            o.default { env_to_float([Ext::ENV_ANALYTICS_SAMPLE_RATE, Ext::ENV_ANALYTICS_SAMPLE_RATE_OLD], 1.0) }
             o.lazy
           end