ddtrace 0.43.0 → 0.48.0

data/lib/ddtrace.rb

@@ -1,5 +1,3 @@
-require 'thread'
-
 # During development, we load `ddtrace` by through `ddtrace.gemspec`,
 # which in turn eager loads 'ddtrace/version'.
 #
@@ -15,13 +13,14 @@ require 'ddtrace/quantization/http'
 require 'ddtrace/pipeline'
 require 'ddtrace/configuration'
 require 'ddtrace/patcher'
-require 'ddtrace/augmentation'
 require 'ddtrace/metrics'
+require 'ddtrace/auto_instrument_base'
+require 'ddtrace/profiling'
 
 # \Datadog global namespace that includes all tracing functionality for Tracer and Span classes.
 module Datadog
-  extend Augmentation
   extend Configuration
+  extend AutoInstrumentBase
 
   # Load and extend Contrib by default
   require 'ddtrace/contrib/extensions'
@@ -31,6 +30,10 @@ module Datadog
   require 'ddtrace/opentelemetry/extensions'
   extend OpenTelemetry::Extensions
 
+  # Load and extend AutoInstrument
+  require 'ddtrace/contrib/auto_instrument'
+  extend Contrib::AutoInstrument
+
   # Add shutdown hook:
   # Ensures the tracer has an opportunity to flush traces
   # and cleanup before terminating the process.
@@ -56,6 +59,7 @@ require 'ddtrace/contrib/grape/integration'
 require 'ddtrace/contrib/graphql/integration'
 require 'ddtrace/contrib/grpc/integration'
 require 'ddtrace/contrib/http/integration'
+require 'ddtrace/contrib/httpclient/integration'
 require 'ddtrace/contrib/httprb/integration'
 require 'ddtrace/contrib/integration'
 require 'ddtrace/contrib/kafka/integration'
@@ -63,6 +67,7 @@ require 'ddtrace/contrib/presto/integration'
 require 'ddtrace/contrib/que/integration'
 require 'ddtrace/contrib/mysql2/integration'
 require 'ddtrace/contrib/mongodb/integration'
+require 'ddtrace/contrib/qless/integration'
 require 'ddtrace/contrib/racecar/integration'
 require 'ddtrace/contrib/rack/integration'
 require 'ddtrace/contrib/rails/integration'
@@ -77,4 +82,3 @@ require 'ddtrace/contrib/sidekiq/integration'
 require 'ddtrace/contrib/sinatra/integration'
 require 'ddtrace/contrib/sneakers/integration'
 require 'ddtrace/contrib/sucker_punch/integration'
-require 'ddtrace/monkey'

data/lib/ddtrace/analytics.rb

@@ -6,11 +6,13 @@ module Datadog
     class << self
       def set_sample_rate(span, sample_rate)
         return if span.nil? || !sample_rate.is_a?(Numeric)
+
         span.set_metric(Datadog::Ext::Analytics::TAG_SAMPLE_RATE, sample_rate)
       end
 
       def set_measured(span, value = true)
         return if span.nil?
+
         # rubocop:disable Style/MultipleComparison
         value = value == true || value == 1 ? 1 : 0
         span.set_metric(Datadog::Ext::Analytics::TAG_MEASURED, value)

data/lib/ddtrace/auto_instrument.rb

@@ -0,0 +1,3 @@
+require 'ddtrace'
+
+Datadog.add_auto_instrument

data/lib/ddtrace/auto_instrument_base.rb

@@ -0,0 +1,6 @@
+module Datadog
+  # base methods stubbed for adding auto instrument extensions
+  module AutoInstrumentBase
+    def add_auto_instrument; end
+  end
+end

data/lib/ddtrace/buffer.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/runtime/object_space'
 
@@ -18,6 +17,7 @@ module Datadog
     # even if the buffer is full. In that case, a random item is discarded.
     def push(item)
       return if closed?
+
       full? ? replace!(item) : add!(item)
       item
     end
@@ -168,8 +168,8 @@ module Datadog
       synchronize { super }
     end
 
-    def synchronize
-      @mutex.synchronize { yield }
+    def synchronize(&block)
+      @mutex.synchronize(&block)
     end
   end
 
@@ -318,7 +318,7 @@ module Datadog
   #
   # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
   # TODO Making such a change is potentially breaking for users manually configuring the tracer.
-  TraceBuffer = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby' # rubocop:disable Style/ConstantName
+  TraceBuffer = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby'
                   CRubyTraceBuffer
                 else
                   ThreadSafeTraceBuffer

data/lib/ddtrace/configuration.rb

@@ -1,14 +1,41 @@
 require 'forwardable'
-
 require 'ddtrace/configuration/pin_setup'
 require 'ddtrace/configuration/settings'
 require 'ddtrace/configuration/components'
+require 'ddtrace/utils/only_once'
 
 module Datadog
   # Configuration provides a unique access point for configurations
-  module Configuration
+  module Configuration # rubocop:disable Metrics/ModuleLength
     extend Forwardable
 
+    # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+    #
+    # This is important because components can end up being accessed from multiple application threads (for instance on
+    # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+    #
+    # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+    # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+    #
+    # Every method that directly or indirectly mutates @components should be holding the lock (through
+    # #safely_synchronize) while doing so.
+    COMPONENTS_WRITE_LOCK = Mutex.new
+    private_constant :COMPONENTS_WRITE_LOCK
+
+    # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+    # This was needed because we ran into several issues where we still needed to read the old
+    # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+    # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+    #
+    # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+    # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+    # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+    # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+    # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+    # Concurrency is hard.
+    COMPONENTS_READ_LOCK = Mutex.new
+    private_constant :COMPONENTS_READ_LOCK
+
     attr_writer :configuration
 
     def configuration
@@ -16,16 +43,20 @@ module Datadog
     end
 
     def configure(target = configuration, opts = {})
+      ruby_version_deprecation_warning
+
       if target.is_a?(Settings)
         yield(target) if block_given?
 
-        # Build immutable components from settings
-        @components ||= nil
-        @components = if @components
-                        replace_components!(target, @components)
-                      else
-                        build_components(target)
-                      end
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(target, @components)
+            else
+              build_components(target)
+            end
+          )
+        end
 
         target
       else
@@ -36,22 +67,19 @@ module Datadog
     def_delegators \
       :components,
       :health_metrics,
+      :profiler,
       :runtime_metrics,
       :tracer
 
     def logger
-      if instance_variable_defined?(:@components) && @components
+      # avoid initializing components if they didn't already exist
+      current_components = components(allow_initialization: false)
+
+      if current_components
         @temp_logger = nil
-        components.logger
+        current_components.logger
       else
-        # Use default logger without initializing components.
-        # This prevents recursive loops while initializing.
-        # e.g. Get logger --> Build components --> Log message --> Repeat...
-        @temp_logger ||= begin
-          logger = configuration.logger.instance || Datadog::Logger.new(STDOUT)
-          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
-          logger
-        end
+        logger_without_components
       end
     end
 
@@ -65,26 +93,64 @@ module Datadog
     #
     # Components won't be automatically reinitialized after a shutdown.
     def shutdown!
-      components.shutdown! if instance_variable_defined?(:@components) && @components
+      safely_synchronize do
+        @components.shutdown! if components?
+      end
     end
 
+    protected
+
+    def components(allow_initialization: true)
+      current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+      return current_components if current_components || !allow_initialization
+
+      safely_synchronize do |write_components|
+        (defined?(@components) && @components) || write_components.call(build_components(configuration))
+      end
+    end
+
+    private
+
     # Gracefully shuts down the tracer and disposes of component references,
     # allowing execution to start anew.
     #
     # In contrast with +#shutdown!+, components will be automatically
     # reinitialized after a reset.
+    #
+    # Used internally to ensure a clean environment between test runs.
     def reset!
-      shutdown!
-      @components = nil
+      safely_synchronize do |write_components|
+        @components.shutdown! if components?
+        write_components.call(nil)
+        configuration.reset!
+      end
     end
 
-    protected
+    def safely_synchronize
+      # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+      # safely_synchronize, this forces all writers to go through this method.
+      write_components = proc do |new_value|
+        COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+      end
 
-    def components
-      @components ||= build_components(configuration)
+      COMPONENTS_WRITE_LOCK.synchronize do
+        begin
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during ddtrace initialization. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{e.backtrace.join("\n\t")}"
+          )
+          nil
+        end
+      end
     end
 
-    private
+    def components?
+      # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+      (defined?(@components) && @components) != nil
+    end
 
     def build_components(settings)
       components = Components.new(settings)
@@ -99,5 +165,34 @@ module Datadog
       components.startup!(settings)
       components
     end
+
+    def logger_without_components
+      # Use default logger without initializing components.
+      # This enables logging during initialization, otherwise we'd run into deadlocks.
+      @temp_logger ||= begin
+        logger = configuration.logger.instance || Datadog::Logger.new($stdout)
+        logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+        logger
+      end
+    end
+
+    # Perform version check only once
+    DEPRECATED_RUBY_VERSION = Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.1')
+    private_constant :DEPRECATED_RUBY_VERSION
+
+    RUBY_VERSION_DEPRECATION_ONLY_ONCE = Datadog::Utils::OnlyOnce.new
+    private_constant :RUBY_VERSION_DEPRECATION_ONLY_ONCE
+
+    def ruby_version_deprecation_warning
+      return unless DEPRECATED_RUBY_VERSION
+
+      RUBY_VERSION_DEPRECATION_ONLY_ONCE.run do
+        Datadog.logger.warn(
+          "Support for Ruby versions < 2.1 in dd-trace-rb is DEPRECATED.\n" \
+          "Last version to support Ruby < 2.1 will be 0.49.x, which will only receive critical bugfixes.\n" \
+          'Support for Ruby versions < 2.1 will be REMOVED in version 0.50.0.'
+        )
+      end
+    end
   end
 end

data/lib/ddtrace/configuration/base.rb

@@ -37,7 +37,7 @@ module Datadog
 
         def new_settings_class(&block)
           Class.new { include Datadog::Configuration::Base }.tap do |klass|
-            klass.instance_eval(&block) if block_given?
+            klass.instance_eval(&block) if block
           end
         end
       end

data/lib/ddtrace/configuration/components.rb

@@ -1,5 +1,6 @@
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/logger'
+require 'ddtrace/profiling'
 require 'ddtrace/runtime/metrics'
 require 'ddtrace/tracer'
 require 'ddtrace/workers/runtime_metrics'
@@ -7,7 +8,8 @@ require 'ddtrace/workers/runtime_metrics'
 module Datadog
   module Configuration
     # Global components for the trace library.
-    # rubocop:disable Metrics/LineLength
+    # rubocop:disable Layout/LineLength
+    # rubocop:disable Metrics/ClassLength
     class Components
       class << self
         def build_health_metrics(settings)
@@ -19,7 +21,7 @@ module Datadog
         end
 
         def build_logger(settings)
-          logger = settings.logger.instance || Datadog::Logger.new(STDOUT)
+          logger = settings.logger.instance || Datadog::Logger.new($stdout)
           logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
           logger
@@ -66,6 +68,22 @@ module Datadog
           tracer
         end
 
+        def build_profiler(settings)
+          return unless Datadog::Profiling.supported? && settings.profiling.enabled
+
+          # Load extensions needed to support some of the Profiling features
+          Datadog::Profiling::Tasks::Setup.new.run
+
+          # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+          recorder = build_profiler_recorder(settings)
+          collectors = build_profiler_collectors(settings, recorder)
+          exporters = build_profiler_exporters(settings)
+          scheduler = build_profiler_scheduler(settings, recorder, exporters)
+
+          Datadog::Profiler.new(collectors, scheduler)
+        end
+
         private
 
         def build_tracer_tags(settings)
@@ -90,11 +108,51 @@ module Datadog
             opts[:writer_options] = settings.writer_options if settings.writer.nil?
           end
         end
+
+        def build_profiler_recorder(settings)
+          event_classes = [Datadog::Profiling::Events::StackSample]
+
+          Datadog::Profiling::Recorder.new(event_classes, settings.profiling.max_events)
+        end
+
+        def build_profiler_collectors(settings, recorder)
+          [
+            Datadog::Profiling::Collectors::Stack.new(
+              recorder,
+              max_frames: settings.profiling.max_frames
+              # TODO: Provide proc that identifies Datadog worker threads?
+              # ignore_thread: settings.profiling.ignore_profiler
+            )
+          ]
+        end
+
+        def build_profiler_exporters(settings)
+          if settings.profiling.exporter.instances.is_a?(Array)
+            settings.profiling.exporter.instances
+          else
+            transport = if settings.profiling.exporter.transport
+                          settings.profiling.exporter.transport
+                        else
+                          transport_options = settings.profiling.exporter.transport_options.dup
+                          transport_options[:site] ||= settings.site if settings.site
+                          transport_options[:api_key] ||= settings.api_key if settings.api_key
+                          transport_options[:timeout] ||= settings.profiling.upload.timeout
+                          Datadog::Profiling::Transport::HTTP.default(transport_options)
+                        end
+
+            [Datadog::Profiling::Exporter.new(transport)]
+          end
+        end
+
+        def build_profiler_scheduler(settings, recorder, exporters)
+          Datadog::Profiling::Scheduler.new(recorder, exporters)
+        end
       end
 
       attr_reader \
         :health_metrics,
         :logger,
+        :profiler,
         :runtime_metrics,
         :tracer
 
@@ -105,6 +163,9 @@ module Datadog
         # Tracer
         @tracer = self.class.build_tracer(settings)
 
+        # Profiler
+        @profiler = self.class.build_profiler(settings)
+
         # Runtime metrics
         @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
 
@@ -113,7 +174,20 @@ module Datadog
       end
 
       # Starts up components
-      def startup!(settings); end
+      def startup!(settings)
+        if settings.profiling.enabled
+          if profiler
+            @logger.debug('Profiling started')
+            profiler.start
+          else
+            # Display a warning for users who expected profiling to autostart
+            protobuf = Datadog::Profiling.google_protobuf_supported?
+            logger.warn("Profiling was enabled but is not supported; profiling disabled. (google-protobuf?: #{protobuf})")
+          end
+        else
+          @logger.debug('Profiling is disabled')
+        end
+      end
 
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
@@ -123,12 +197,20 @@ module Datadog
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer
 
+        # Shutdown old profiler
+        profiler.shutdown! unless profiler.nil?
+
         # Shutdown workers
-        runtime_metrics.enabled = false
-        runtime_metrics.stop(true)
+        runtime_metrics.stop(true, close_metrics: false)
 
         # Shutdown the old metrics, unless they are still being used.
         # (e.g. custom Statsd instances.)
+        #
+        # TODO: This violates the encapsulation created by Runtime::Metrics and
+        # Health::Metrics, by directly manipulating `statsd` and changing
+        # it's lifecycle management.
+        # If we need to directly have ownership of `statsd` lifecycle, we should
+        # have direct ownership of it.
         old_statsd = [
           runtime_metrics.metrics.statsd,
           health_metrics.statsd