ddtrace 0.40.0 → 0.54.1

data/lib/ddtrace/workers/async.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'ddtrace/logger'
 
 module Datadog
@@ -12,13 +13,13 @@ module Datadog
         SHUTDOWN_TIMEOUT = 1
 
         def self.included(base)
-          base.send(:prepend, PrependedMethods)
+          base.prepend(PrependedMethods)
         end
 
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            start { self.result = super(*args) } unless started?
+            start_async { self.result = super(*args) } unless started?
           end
         end
 
@@ -31,18 +32,22 @@ module Datadog
 
         def join(timeout = nil)
           return true unless running?
+
           !worker.join(timeout).nil?
         end
 
         def terminate
           return false unless running?
+
           @run_async = false
+          Datadog.logger.debug { "Forcibly terminating worker thread for: #{self}" }
           worker.terminate
           true
         end
 
         def run_async?
           return false unless instance_variable_defined?(:@run_async)
+
           @run_async == true
         end
 
@@ -56,6 +61,7 @@ module Datadog
 
         def error?
           return false unless instance_variable_defined?(:@error)
+
           !@error.nil?
         end
 
@@ -101,9 +107,10 @@ module Datadog
           @worker ||= nil
         end
 
-        def start(&block)
+        def start_async(&block)
           mutex.synchronize do
             return if running?
+
             if forked?
               case fork_policy
               when FORK_POLICY_STOP
@@ -121,7 +128,7 @@ module Datadog
           @run_async = true
           @pid = Process.pid
           @error = nil
-          Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
+          Datadog.logger.debug { "Starting thread for: #{self}" }
 
           @worker = ::Thread.new do
             begin
@@ -129,10 +136,13 @@ module Datadog
             # rubocop:disable Lint/RescueException
             rescue Exception => e
               @error = e
-              Datadog.logger.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
+              Datadog.logger.debug("Worker thread error. Cause #{e.message} Location: #{Array(e.backtrace).first}")
               raise
             end
           end
+          @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+          nil
         end
 
         def stop_fork

data/lib/ddtrace/workers/loop.rb

@@ -1,3 +1,4 @@
+# typed: false
 module Datadog
   module Workers
     # Adds looping behavior to workers, with a sleep
@@ -8,7 +9,7 @@ module Datadog
       BASE_INTERVAL = 1
 
       def self.included(base)
-        base.send(:prepend, PrependedMethods)
+        base.prepend(PrependedMethods)
       end
 
       # Methods that must be prepended
@@ -21,6 +22,7 @@ module Datadog
       def stop_loop
         mutex.synchronize do
           return false unless run_loop?
+
           @run_loop = false
           shutdown.signal
         end
@@ -34,6 +36,7 @@ module Datadog
 
       def run_loop?
         return false unless instance_variable_defined?(:@run_loop)
+
         @run_loop == true
       end
 
@@ -53,12 +56,29 @@ module Datadog
         @loop_wait_time ||= loop_base_interval
       end
 
+      def loop_wait_time=(value)
+        @loop_wait_time = value
+      end
+
+      def reset_loop_wait_time
+        self.loop_wait_time = loop_base_interval
+      end
+
+      # Should the loop "back off" when there's no work?
       def loop_back_off?
         false
       end
 
-      def loop_back_off!(amount = nil)
-        @loop_wait_time = amount || [loop_wait_time * BACK_OFF_RATIO, BACK_OFF_MAX].min
+      def loop_back_off!
+        self.loop_wait_time = [loop_wait_time * BACK_OFF_RATIO, BACK_OFF_MAX].min
+      end
+
+      # Should perform_loop just straight into work, or start by waiting?
+      #
+      # The use case is if we want to report some information (like profiles) from time to time, we may not want to
+      # report empty/zero/some residual value immediately when the worker starts.
+      def loop_wait_before_first_iteration?
+        false
       end
 
       protected
@@ -75,16 +95,22 @@ module Datadog
       private
 
       def perform_loop
-        @run_loop = true
+        mutex.synchronize do
+          @run_loop = true
+
+          shutdown.wait(mutex, loop_wait_time) if loop_wait_before_first_iteration?
+        end
 
         loop do
           if work_pending?
+            # There's work to do...
             # Run the task
             yield
 
             # Reset the wait interval
-            loop_back_off!(loop_base_interval)
+            reset_loop_wait_time if loop_back_off?
           elsif loop_back_off?
+            # There's no work to do...
             # Back off the wait interval a bit
             loop_back_off!
           end
@@ -92,6 +118,7 @@ module Datadog
           # Wait for an interval, unless shutdown has been signaled.
           mutex.synchronize do
             return unless run_loop? || work_pending?
+
             shutdown.wait(mutex, loop_wait_time) if run_loop?
           end
         end

data/lib/ddtrace/workers/polling.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'ddtrace/workers/async'
 require 'ddtrace/workers/loop'
 
@@ -8,9 +9,9 @@ module Datadog
       SHUTDOWN_TIMEOUT = 1
 
       def self.included(base)
-        base.send(:include, Workers::IntervalLoop)
-        base.send(:include, Workers::Async::Thread)
-        base.send(:prepend, PrependedMethods)
+        base.include(Workers::IntervalLoop)
+        base.include(Workers::Async::Thread)
+        base.prepend(PrependedMethods)
       end
 
       # Methods that must be prepended
@@ -26,8 +27,14 @@ module Datadog
           stop_loop
           graceful = join(timeout)
 
-          # If timeout and force stop...
-          !graceful && force_stop ? terminate : graceful
+          if !graceful && force_stop
+            Datadog.logger.debug do
+              "Timeout while waiting for worker to finish gracefully, forcing termination for: #{self}"
+            end
+            terminate
+          else
+            graceful
+          end
         else
           false
         end
@@ -35,6 +42,7 @@ module Datadog
 
       def enabled?
         return true unless instance_variable_defined?(:@enabled)
+
         @enabled
       end
 

data/lib/ddtrace/workers/queue.rb

@@ -1,10 +1,11 @@
+# typed: false
 module Datadog
   module Workers
     # Adds queue behavior to workers, with a buffer
     # to which items can be queued then dequeued.
     module Queue
       def self.included(base)
-        base.send(:prepend, PrependedMethods)
+        base.prepend(PrependedMethods)
       end
 
       # Methods that must be prepended
@@ -26,6 +27,7 @@ module Datadog
         buffer.shift
       end
 
+      # Are there more items to be processed next?
       def work_pending?
         !buffer.empty?
       end

data/lib/ddtrace/workers/runtime_metrics.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'forwardable'
 
 require 'ddtrace/runtime/metrics'
@@ -12,19 +13,23 @@ module Datadog
       extend Forwardable
       include Workers::Polling
 
+      # In seconds
+      DEFAULT_FLUSH_INTERVAL = 10
+      DEFAULT_BACK_OFF_MAX = 30
+
       attr_reader \
         :metrics
 
       def initialize(options = {})
-        @metrics = options.fetch(:metrics, Runtime::Metrics.new)
+        @metrics = options.fetch(:metrics) { Runtime::Metrics.new }
 
         # Workers::Async::Thread settings
         self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)
 
         # Workers::IntervalLoop settings
-        self.interval = options[:interval] if options.key?(:interval)
-        self.back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
-        self.back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+        self.loop_base_interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options.fetch(:back_off_max, DEFAULT_BACK_OFF_MAX)
 
         self.enabled = options.fetch(:enabled, false)
       end
@@ -39,6 +44,19 @@ module Datadog
         metrics.associate_with_span(*args).tap { perform }
       end
 
+      # TODO: `close_metrics` is only needed because
+      # Datadog::Components directly manipulates the lifecycle of
+      # Runtime::Metrics.statsd instances.
+      # This should be avoided, as it prevents this class from
+      # ensuring correct resource decommission of its internal
+      # dependencies.
+      def stop(*args, close_metrics: true)
+        self.enabled = false
+        result = super(*args)
+        @metrics.close if close_metrics
+        result
+      end
+
       def_delegators \
         :metrics,
         :register_service

data/lib/ddtrace/workers/trace_writer.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'ddtrace/logger'
 require 'ddtrace/transport/http'
 
@@ -16,15 +17,10 @@ module Datadog
       def initialize(options = {})
         transport_options = options.fetch(:transport_options, {})
 
-        if transport_options.is_a?(Proc)
-          transport_options = { on_build: transport_options }
-        end
-
-        transport_options[:hostname] = options[:hostname] if options.key?(:hostname)
-        transport_options[:port] = options[:port] if options.key?(:port)
+        transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
 
         @transport = options.fetch(:transport) do
-          Transport::HTTP.default(transport_options)
+          Transport::HTTP.default(**transport_options)
         end
       end
 
@@ -41,7 +37,7 @@ module Datadog
         flush_traces(traces)
       rescue StandardError => e
         Datadog.logger.error(
-          "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
+          "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
         )
       end
 
@@ -65,10 +61,8 @@ module Datadog
         traces.each do |trace|
           next if trace.first.nil?
 
-          hostname = Datadog::Runtime::Socket.hostname
-          unless hostname.nil? || hostname.empty?
-            trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname)
-          end
+          hostname = Datadog::Core::Environment::Socket.hostname
+          trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname) unless hostname.nil? || hostname.empty?
         end
       end
 
@@ -84,10 +78,6 @@ module Datadog
         def initialize
           super(:flush_completed)
         end
-
-        def publish(response)
-          super(response)
-        end
       end
     end
 
@@ -127,10 +117,15 @@ module Datadog
 
       # NOTE: #perform is wrapped by other modules:
       #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
+      #
+      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#perform is spec'd to return the
+      # result from the writer, whereas this method always returns nil.
       def perform(traces)
         super(traces).tap do |responses|
           loop_back_off! if responses.find(&:server_error?)
         end
+
+        nil
       end
 
       def stop(*args)
@@ -148,6 +143,7 @@ module Datadog
         [buffer.pop]
       end
 
+      # Are there more traces to be processed next?
       def work_pending?
         !buffer.empty?
       end
@@ -187,6 +183,8 @@ module Datadog
         @async = false if @writer_fork_policy == FORK_POLICY_SYNC
       end
 
+      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#write is spec'd to return the
+      # result from the writer, whereas this method returns something else when running in async mode.
       def write(trace)
         # Start worker thread. If the process has forked, it will trigger #after_fork to
         # reconfigure the worker accordingly.

data/lib/ddtrace/workers.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'time'
 
 require 'ddtrace/workers/trace_writer'
@@ -58,7 +59,7 @@ module Datadog
           # TODO[manu]: findout the reason and reschedule the send if it's not
           # a fatal exception
           Datadog.logger.error(
-            "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
+            "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
           )
         end
       end
@@ -67,9 +68,13 @@ module Datadog
       def start
         @mutex.synchronize do
           return if @run
+
           @run = true
-          Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
+          Datadog.logger.debug { "Starting thread for: #{self}" }
           @worker = Thread.new { perform }
+          @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+          nil
         end
       end
 
@@ -108,6 +113,7 @@ module Datadog
 
           @mutex.synchronize do
             return if !@run && @trace_buffer.empty?
+
             @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down
           end
         end

data/lib/ddtrace/writer.rb

@@ -1,17 +1,22 @@
+# typed: true
 require 'json'
 
 require 'ddtrace/ext/net'
-require 'ddtrace/runtime/socket'
+require 'datadog/core/environment/socket'
 
+require 'ddtrace/configuration/agent_settings_resolver'
 require 'ddtrace/transport/http'
 require 'ddtrace/transport/io'
 require 'ddtrace/encoding'
 require 'ddtrace/workers'
 require 'ddtrace/diagnostics/environment_logger'
+require 'ddtrace/utils/only_once'
 
 module Datadog
   # Processor that sends traces and metadata to the agent
   class Writer
+    DEPRECATION_WARN_ONLY_ONCE = Datadog::Utils::OnlyOnce.new
+
     attr_reader \
       :priority_sampler,
       :transport,
@@ -23,6 +28,8 @@ module Datadog
       @flush_interval = options.fetch(:flush_interval, Workers::AsyncTransport::DEFAULT_FLUSH_INTERVAL)
       transport_options = options.fetch(:transport_options, {})
 
+      transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+
       # priority sampling
       if options[:priority_sampler]
         @priority_sampler = options[:priority_sampler]
@@ -31,7 +38,7 @@ module Datadog
 
       # transport and buffers
       @transport = options.fetch(:transport) do
-        Transport::HTTP.default(transport_options)
+        Transport::HTTP.default(**transport_options)
       end
 
       # handles the thread creation after an eventual fork
@@ -42,13 +49,23 @@ module Datadog
 
       # one worker for traces
       @worker = nil
+
+      # Once stopped, this writer instance cannot be restarted.
+      # This allow for graceful shutdown, while preventing
+      # the host application from inadvertently start new
+      # threads during shutdown.
+      @stopped = false
     end
 
     def start
       @mutex_after_fork.synchronize do
+        return false if @stopped
+
         pid = Process.pid
         return if @worker && pid == @pid
+
         @pid = pid
+
         start_worker
         true
       end
@@ -67,14 +84,24 @@ module Datadog
       @worker.start
     end
 
+    # Gracefully shuts down this writer.
+    #
+    # Once stopped methods calls won't fail, but
+    # no internal work will be performed.
+    #
+    # It is not possible to restart a stopped writer instance.
     def stop
       @mutex_after_fork.synchronize { stop_worker }
     end
 
     def stop_worker
+      @stopped = true
+
       return if @worker.nil?
+
       @worker.stop
       @worker = nil
+
       true
     end
 
@@ -107,7 +134,7 @@ module Datadog
     # enqueue the trace for submission to the API
     def write(trace, services = nil)
       unless services.nil?
-        Datadog::Patcher.do_once('Writer#write') do
+        DEPRECATION_WARN_ONLY_ONCE.run do
           Datadog.logger.warn(%(
             write: Writing services has been deprecated and no longer need to be provided.
             write(traces, services) can be updated to write(traces)
@@ -136,7 +163,7 @@ module Datadog
 
       if worker_local
         worker_local.enqueue_trace(trace)
-      else
+      elsif !@stopped
         Datadog.logger.debug('Writer either failed to start or was stopped before #write could complete')
       end
     end
@@ -155,10 +182,8 @@ module Datadog
       traces.each do |trace|
         next if trace.first.nil?
 
-        hostname = Datadog::Runtime::Socket.hostname
-        unless hostname.nil? || hostname.empty?
-          trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname)
-        end
+        hostname = Datadog::Core::Environment::Socket.hostname
+        trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname) unless hostname.nil? || hostname.empty?
       end
     end
 

data/lib/ddtrace.rb

@@ -1,5 +1,4 @@
-require 'thread'
-
+# typed: strict
 # During development, we load `ddtrace` by through `ddtrace.gemspec`,
 # which in turn eager loads 'ddtrace/version'.
 #
@@ -15,64 +14,39 @@ require 'ddtrace/quantization/http'
 require 'ddtrace/pipeline'
 require 'ddtrace/configuration'
 require 'ddtrace/patcher'
-require 'ddtrace/augmentation'
 require 'ddtrace/metrics'
+require 'ddtrace/auto_instrument_base'
+require 'ddtrace/profiling'
+
+require 'datadog/contrib'
+require 'ddtrace/contrib/auto_instrument'
+require 'ddtrace/contrib/extensions'
+
+require 'ddtrace/opentelemetry/extensions'
 
 # \Datadog global namespace that includes all tracing functionality for Tracer and Span classes.
 module Datadog
-  extend Augmentation
   extend Configuration
+  extend AutoInstrumentBase
 
-  # Load and extend Contrib by default
-  require 'ddtrace/contrib/extensions'
+  # Load built-in Datadog integrations
   extend Contrib::Extensions
+  # Load Contrib auto instrumentation
+  extend Contrib::AutoInstrument
+  # Load Contrib extension to global Datadog objects
+  Configuration::Settings.include Contrib::Extensions::Configuration::Settings
 
   # Load and extend OpenTelemetry compatibility by default
-  require 'ddtrace/opentelemetry/extensions'
   extend OpenTelemetry::Extensions
 
   # Add shutdown hook:
   # Ensures the tracer has an opportunity to flush traces
   # and cleanup before terminating the process.
-  at_exit { Datadog.shutdown! }
+  at_exit do
+    if Interrupt === $! # rubocop:disable Style/SpecialGlobalVars is process terminating due to a ctrl+c or similar?
+      Datadog.send(:handle_interrupt_shutdown!)
+    else
+      Datadog.shutdown!
+    end
+  end
 end
-
-require 'ddtrace/contrib/action_cable/integration'
-require 'ddtrace/contrib/action_pack/integration'
-require 'ddtrace/contrib/action_view/integration'
-require 'ddtrace/contrib/active_model_serializers/integration'
-require 'ddtrace/contrib/active_record/integration'
-require 'ddtrace/contrib/active_support/integration'
-require 'ddtrace/contrib/aws/integration'
-require 'ddtrace/contrib/concurrent_ruby/integration'
-require 'ddtrace/contrib/dalli/integration'
-require 'ddtrace/contrib/delayed_job/integration'
-require 'ddtrace/contrib/elasticsearch/integration'
-require 'ddtrace/contrib/ethon/integration'
-require 'ddtrace/contrib/excon/integration'
-require 'ddtrace/contrib/faraday/integration'
-require 'ddtrace/contrib/grape/integration'
-require 'ddtrace/contrib/graphql/integration'
-require 'ddtrace/contrib/grpc/integration'
-require 'ddtrace/contrib/http/integration'
-require 'ddtrace/contrib/httprb/integration'
-require 'ddtrace/contrib/integration'
-require 'ddtrace/contrib/kafka/integration'
-require 'ddtrace/contrib/presto/integration'
-require 'ddtrace/contrib/que/integration'
-require 'ddtrace/contrib/mysql2/integration'
-require 'ddtrace/contrib/mongodb/integration'
-require 'ddtrace/contrib/racecar/integration'
-require 'ddtrace/contrib/rack/integration'
-require 'ddtrace/contrib/rails/integration'
-require 'ddtrace/contrib/rake/integration'
-require 'ddtrace/contrib/redis/integration'
-require 'ddtrace/contrib/resque/integration'
-require 'ddtrace/contrib/rest_client/integration'
-require 'ddtrace/contrib/sequel/integration'
-require 'ddtrace/contrib/shoryuken/integration'
-require 'ddtrace/contrib/sidekiq/integration'
-require 'ddtrace/contrib/sinatra/integration'
-require 'ddtrace/contrib/sneakers/integration'
-require 'ddtrace/contrib/sucker_punch/integration'
-require 'ddtrace/monkey'