ddtrace 0.13.0 → 0.47.0

data/lib/ddtrace/chunker.rb

@@ -0,0 +1,34 @@
+require 'json'
+require 'msgpack'
+
+module Datadog
+  # Chunks list of elements into batches
+  module Chunker
+    module_function
+
+    # Chunks a list into batches of at most +max_chunk_size+ elements each.
+    #
+    # An exception can occur if a single element is too large. That single
+    # element will be returned in its own chunk. You have to verify by yourself
+    # when such elements are returned.
+    #
+    # @param list [Enumerable] list of elements
+    # @param max_chunk_size [Numeric] maximum acceptable chunk size
+    # @return [Enumerable] lazy list of chunks
+    def chunk_by_size(list, max_chunk_size)
+      chunk_agg = 0
+      list.slice_before do |elem|
+        size = elem.size
+        chunk_agg += size
+        if chunk_agg > max_chunk_size
+          # Can't fit element in current chunk, start a new one.
+          chunk_agg = size
+          true
+        else
+          # Add to current chunk
+          false
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/configuration.rb

@@ -1,47 +1,172 @@
-require_relative 'configuration/proxy'
-require_relative 'configuration/resolver'
-require_relative 'configuration/pin_setup'
+require 'forwardable'
+require 'ddtrace/configuration/pin_setup'
+require 'ddtrace/configuration/settings'
+require 'ddtrace/configuration/components'
 
 module Datadog
   # Configuration provides a unique access point for configurations
-  class Configuration
-    InvalidIntegrationError = Class.new(StandardError)
+  module Configuration
+    extend Forwardable
 
-    def initialize(options = {})
-      @registry = options.fetch(:registry, Datadog.registry)
-      @wrapped_registry = {}
+    # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+    #
+    # This is important because components can end up being accessed from multiple application threads (for instance on
+    # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+    #
+    # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+    # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+    #
+    # Every method that directly or indirectly mutates @components should be holding the lock (through
+    # #safely_synchronize) while doing so.
+    COMPONENTS_WRITE_LOCK = Mutex.new
+    private_constant :COMPONENTS_WRITE_LOCK
+
+    # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+    # This was needed because we ran into several issues where we still needed to read the old
+    # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+    # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+    #
+    # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+    # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+    # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+    # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+    # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+    # Concurrency is hard.
+    COMPONENTS_READ_LOCK = Mutex.new
+    private_constant :COMPONENTS_READ_LOCK
+
+    attr_writer :configuration
+
+    def configuration
+      @configuration ||= Settings.new
     end
 
-    def [](integration_name)
-      @wrapped_registry[integration_name] ||= Proxy.new(fetch_integration(integration_name))
+    def configure(target = configuration, opts = {})
+      if target.is_a?(Settings)
+        yield(target) if block_given?
+
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(target, @components)
+            else
+              build_components(target)
+            end
+          )
+        end
+
+        target
+      else
+        PinSetup.new(target, opts).call
+      end
     end
 
-    def use(integration_name, options = {})
-      integration = fetch_integration(integration_name)
-      settings = Proxy.new(integration)
+    def_delegators \
+      :components,
+      :health_metrics,
+      :runtime_metrics,
+      :tracer
+
+    def logger
+      # avoid initializing components if they didn't already exist
+      current_components = components(allow_initialization: false)
 
-      integration.sorted_options.each do |name|
-        settings[name] = options.fetch(name, settings[name])
+      if current_components
+        @temp_logger = nil
+        current_components.logger
+      else
+        logger_without_components
       end
+    end
 
-      integration.patch if integration.respond_to?(:patch)
+    # Gracefully shuts down all components.
+    #
+    # Components will still respond to method calls as usual,
+    # but might not internally perform their work after shutdown.
+    #
+    # This avoids errors being raised across the host application
+    # during shutdown, while allowing for graceful decommission of resources.
+    #
+    # Components won't be automatically reinitialized after a shutdown.
+    def shutdown!
+      safely_synchronize do
+        @components.shutdown! if components?
+      end
     end
 
-    def tracer(options = {})
-      instance = options.fetch(:instance, Datadog.tracer)
+    # Gracefully shuts down the tracer and disposes of component references,
+    # allowing execution to start anew.
+    #
+    # In contrast with +#shutdown!+, components will be automatically
+    # reinitialized after a reset.
+    def reset!
+      safely_synchronize do |write_components|
+        @components.shutdown! if components?
+        write_components.call(nil)
+      end
+    end
 
-      instance.configure(options)
-      instance.class.log = options[:log] if options[:log]
-      instance.set_tags(options[:tags]) if options[:tags]
-      instance.set_tags(env: options[:env]) if options[:env]
-      instance.class.debug_logging = options.fetch(:debug, false)
+    protected
+
+    def components(allow_initialization: true)
+      current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+      return current_components if current_components || !allow_initialization
+
+      safely_synchronize do |write_components|
+        (defined?(@components) && @components) || write_components.call(build_components(configuration))
+      end
     end
 
     private
 
-    def fetch_integration(name)
-      @registry[name] ||
-        raise(InvalidIntegrationError, "'#{name}' is not a valid integration.")
+    def safely_synchronize
+      # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+      # safely_synchronize, this forces all writers to go through this method.
+      write_components = proc do |new_value|
+        COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+      end
+
+      COMPONENTS_WRITE_LOCK.synchronize do
+        begin
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during ddtrace initialization. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{e.backtrace.join("\n\t")}"
+          )
+          nil
+        end
+      end
+    end
+
+    def components?
+      # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+      (defined?(@components) && @components) != nil
+    end
+
+    def build_components(settings)
+      components = Components.new(settings)
+      components.startup!(settings)
+      components
+    end
+
+    def replace_components!(settings, old)
+      components = Components.new(settings)
+
+      old.shutdown!(components)
+      components.startup!(settings)
+      components
+    end
+
+    def logger_without_components
+      # Use default logger without initializing components.
+      # This enables logging during initialization, otherwise we'd run into deadlocks.
+      @temp_logger ||= begin
+        logger = configuration.logger.instance || Datadog::Logger.new($stdout)
+        logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+        logger
+      end
     end
   end
 end

data/lib/ddtrace/configuration/base.rb

@@ -0,0 +1,84 @@
+require 'ddtrace/environment'
+require 'ddtrace/configuration/options'
+
+module Datadog
+  module Configuration
+    # Basic configuration behavior
+    module Base
+      def self.included(base)
+        base.send(:extend, Datadog::Environment::Helpers)
+        base.send(:include, Datadog::Environment::Helpers)
+        base.send(:include, Options)
+
+        base.send(:extend, ClassMethods)
+        base.send(:include, InstanceMethods)
+      end
+
+      # Class methods for configuration
+      module ClassMethods
+        protected
+
+        # Allows subgroupings of settings to be defined.
+        # e.g. `settings :foo { option :bar }` --> `config.foo.bar`
+        def settings(name, &block)
+          settings_class = new_settings_class(&block)
+
+          option(name) do |o|
+            o.default { settings_class.new }
+            o.lazy
+            o.resetter do |value|
+              value.reset! if value.respond_to?(:reset!)
+              value
+            end
+          end
+        end
+
+        private
+
+        def new_settings_class(&block)
+          Class.new { include Datadog::Configuration::Base }.tap do |klass|
+            klass.instance_eval(&block) if block
+          end
+        end
+      end
+
+      # Instance methods for configuration
+      module InstanceMethods
+        def initialize(options = {})
+          configure(options) unless options.empty?
+        end
+
+        def configure(opts = {})
+          # Sort the options in preference of dependency order first
+          ordering = self.class.options.dependency_order
+          sorted_opts = opts.sort_by do |name, _value|
+            ordering.index(name) || (ordering.length + 1)
+          end
+
+          # Ruby 2.0 doesn't support Array#to_h
+          sorted_opts = Hash[*sorted_opts.flatten]
+
+          # Apply options in sort order
+          sorted_opts.each do |name, value|
+            if respond_to?("#{name}=")
+              send("#{name}=", value)
+            elsif option_defined?(name)
+              set_option(name, value)
+            end
+          end
+
+          # Apply any additional settings from block
+          yield(self) if block_given?
+        end
+
+        def to_h
+          options_hash
+        end
+
+        def reset!
+          reset_options!
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/configuration/components.rb

@@ -0,0 +1,151 @@
+require 'ddtrace/diagnostics/health'
+require 'ddtrace/logger'
+require 'ddtrace/runtime/metrics'
+require 'ddtrace/tracer'
+require 'ddtrace/workers/runtime_metrics'
+
+module Datadog
+  module Configuration
+    # Global components for the trace library.
+    # rubocop:disable Layout/LineLength
+    class Components
+      class << self
+        def build_health_metrics(settings)
+          settings = settings.diagnostics.health_metrics
+          options = { enabled: settings.enabled }
+          options[:statsd] = settings.statsd unless settings.statsd.nil?
+
+          Datadog::Diagnostics::Health::Metrics.new(options)
+        end
+
+        def build_logger(settings)
+          logger = settings.logger.instance || Datadog::Logger.new($stdout)
+          logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
+
+          logger
+        end
+
+        def build_runtime_metrics(settings)
+          options = { enabled: settings.runtime_metrics.enabled }
+          options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
+          options[:services] = [settings.service] unless settings.service.nil?
+
+          Datadog::Runtime::Metrics.new(options)
+        end
+
+        def build_runtime_metrics_worker(settings)
+          # NOTE: Should we just ignore building the worker if its not enabled?
+          options = settings.runtime_metrics.opts.merge(
+            enabled: settings.runtime_metrics.enabled,
+            metrics: build_runtime_metrics(settings)
+          )
+
+          Datadog::Workers::RuntimeMetrics.new(options)
+        end
+
+        def build_tracer(settings)
+          # If a custom tracer has been provided, use it instead.
+          # Ignore all other options (they should already be configured.)
+          tracer = settings.tracer.instance
+          return tracer unless tracer.nil?
+
+          tracer = Tracer.new(
+            default_service: settings.service,
+            enabled: settings.tracer.enabled,
+            partial_flush: settings.tracer.partial_flush.enabled,
+            tags: build_tracer_tags(settings)
+          )
+
+          # TODO: We reconfigure the tracer here because it has way too many
+          #       options it allows to mutate, and it's overwhelming to rewrite
+          #       tracer initialization for now. Just reconfigure using the
+          #       existing mutable #configure function. Remove when these components
+          #       are extracted.
+          tracer.configure(build_tracer_options(settings))
+
+          tracer
+        end
+
+        private
+
+        def build_tracer_tags(settings)
+          settings.tags.dup.tap do |tags|
+            tags['env'] = settings.env unless settings.env.nil?
+            tags['version'] = settings.version unless settings.version.nil?
+          end
+        end
+
+        def build_tracer_options(settings)
+          settings = settings.tracer
+
+          {}.tap do |opts|
+            opts[:hostname] = settings.hostname unless settings.hostname.nil?
+            opts[:min_spans_before_partial_flush] = settings.partial_flush.min_spans_threshold unless settings.partial_flush.min_spans_threshold.nil?
+            opts[:partial_flush] = settings.partial_flush.enabled unless settings.partial_flush.enabled.nil?
+            opts[:port] = settings.port unless settings.port.nil?
+            opts[:priority_sampling] = settings.priority_sampling unless settings.priority_sampling.nil?
+            opts[:sampler] = settings.sampler unless settings.sampler.nil?
+            opts[:transport_options] = settings.transport_options
+            opts[:writer] = settings.writer unless settings.writer.nil?
+            opts[:writer_options] = settings.writer_options if settings.writer.nil?
+          end
+        end
+      end
+
+      attr_reader \
+        :health_metrics,
+        :logger,
+        :runtime_metrics,
+        :tracer
+
+      def initialize(settings)
+        # Logger
+        @logger = self.class.build_logger(settings)
+
+        # Tracer
+        @tracer = self.class.build_tracer(settings)
+
+        # Runtime metrics
+        @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
+
+        # Health metrics
+        @health_metrics = self.class.build_health_metrics(settings)
+      end
+
+      # Starts up components
+      def startup!(settings); end
+
+      # Shuts down all the components in use.
+      # If it has another instance to compare to, it will compare
+      # and avoid tearing down parts still in use.
+      def shutdown!(replacement = nil)
+        # Shutdown the old tracer, unless it's still being used.
+        # (e.g. a custom tracer instance passed in.)
+        tracer.shutdown! unless replacement && tracer == replacement.tracer
+
+        # Shutdown workers
+        runtime_metrics.enabled = false
+        runtime_metrics.stop(true)
+
+        # Shutdown the old metrics, unless they are still being used.
+        # (e.g. custom Statsd instances.)
+        old_statsd = [
+          runtime_metrics.metrics.statsd,
+          health_metrics.statsd
+        ].compact.uniq
+
+        new_statsd =  if replacement
+                        [
+                          replacement.runtime_metrics.metrics.statsd,
+                          replacement.health_metrics.statsd
+                        ].compact.uniq
+                      else
+                        []
+                      end
+
+        unused_statsd = (old_statsd - (old_statsd & new_statsd))
+        unused_statsd.each(&:close)
+      end
+    end
+  end
+end