ddtrace 0.44.0 → 0.49.0

data/lib/ddtrace/configuration/agent_settings_resolver.rb

@@ -0,0 +1,241 @@
+require 'uri'
+
+require 'ddtrace/ext/transport'
+require 'ddtrace/configuration/settings'
+
+module Datadog
+  module Configuration
+    # This class unifies all the different ways that users can configure how we talk to the agent.
+    #
+    # It has quite a lot of complexity, but this complexity just reflects the actual complexity we have around our
+    # configuration today. E.g., this is just all of the complexity regarding agent settings gathered together in a
+    # single place. As we deprecate more and more of the different ways that these things can be configured,
+    # this class will reflect that simplification as well.
+    #
+    # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
+    # about it and pick a value based on the following priority: code > environment variable > defaults.
+    #
+    # rubocop:disable Metrics/ClassLength
+    class AgentSettingsResolver
+      AgentSettings = \
+        Struct.new(
+          :ssl,
+          :hostname,
+          :port,
+          :timeout_seconds,
+          :deprecated_for_removal_transport_configuration_proc,
+          :deprecated_for_removal_transport_configuration_options
+        ) do
+          def initialize(
+            # Hacky required kw args, we can get rid of this when we drop Ruby 2.0
+            ssl: raise(ArgumentError, 'missing keyword :ssl'),
+            hostname: raise(ArgumentError, 'missing keyword :hostname'),
+            port: raise(ArgumentError, 'missing keyword :port'),
+            timeout_seconds: raise(ArgumentError, 'missing keyword :timeout_seconds'),
+            deprecated_for_removal_transport_configuration_proc: \
+              raise(ArgumentError, 'missing keyword :deprecated_for_removal_transport_configuration_proc'),
+            deprecated_for_removal_transport_configuration_options: \
+              raise(ArgumentError, 'missing keyword :deprecated_for_removal_transport_configuration_options')
+          )
+            super(ssl, hostname, port, timeout_seconds, deprecated_for_removal_transport_configuration_proc, \
+              deprecated_for_removal_transport_configuration_options)
+            freeze
+          end
+        end
+
+      def self.call(settings, logger: Datadog.logger)
+        new(settings, logger: logger).send(:call)
+      end
+
+      private
+
+      attr_reader \
+        :logger,
+        :settings
+
+      def initialize(settings, logger: Datadog.logger)
+        @settings = settings
+        @logger = logger
+      end
+
+      def call
+        AgentSettings.new(
+          ssl: ssl?,
+          hostname: hostname,
+          port: port,
+          timeout_seconds: timeout_seconds,
+          # NOTE: When provided, the deprecated_for_removal_transport_configuration_proc can override all
+          # values above (ssl, hostname, port, timeout), or even make them irrelevant (by using an unix socket or
+          # enabling test mode instead).
+          # That is the main reason why it is deprecated -- it's an opaque function that may set a bunch of settings
+          # that we know nothing of until we actually call it.
+          deprecated_for_removal_transport_configuration_proc: deprecated_for_removal_transport_configuration_proc,
+          deprecated_for_removal_transport_configuration_options: deprecated_for_removal_transport_configuration_options
+        )
+      end
+
+      def hostname
+        pick_from(
+          configurations_in_priority_order: [
+            DetectedConfiguration.new(
+              friendly_name: "'c.tracer.hostname'",
+              value: settings.tracer.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url && parsed_url.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST} environment variable",
+              value: ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST]
+            )
+          ],
+          or_use_default: Datadog::Ext::Transport::HTTP::DEFAULT_HOST
+        )
+      end
+
+      def port
+        port_from_env = ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT]
+        parsed_port_from_env =
+          if port_from_env
+            begin
+              Integer(port_from_env)
+            rescue ArgumentError
+              log_warning(
+                "Invalid value for #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT} environment variable " \
+                "('#{port_from_env}'). Ignoring this configuration."
+              )
+            end
+          end
+
+        pick_from(
+          configurations_in_priority_order: [
+            DetectedConfiguration.new(
+              friendly_name: '"c.tracer.port"',
+              value: settings.tracer.port
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url && parsed_url.port
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT} environment variable",
+              value: parsed_port_from_env
+            )
+          ],
+          or_use_default: Datadog::Ext::Transport::HTTP::DEFAULT_PORT
+        )
+      end
+
+      def ssl?
+        !parsed_url.nil? && parsed_url.scheme == 'https'
+      end
+
+      def timeout_seconds
+        Datadog::Ext::Transport::HTTP::DEFAULT_TIMEOUT_SECONDS
+      end
+
+      def deprecated_for_removal_transport_configuration_proc
+        settings.tracer.transport_options if settings.tracer.transport_options.is_a?(Proc)
+      end
+
+      def deprecated_for_removal_transport_configuration_options
+        options = settings.tracer.transport_options
+
+        if options.is_a?(Hash) && !options.empty?
+          log_warning(
+            'Configuring the tracer via a c.tracer.transport_options hash is deprecated for removal in a future ' \
+            "ddtrace version (c.tracer.transport_options contained '#{options.inspect}')."
+          )
+
+          options
+        end
+      end
+
+      def parsed_url
+        return @parsed_url if defined?(@parsed_url)
+
+        @parsed_url =
+          if unparsed_url_from_env
+            parsed = URI.parse(unparsed_url_from_env)
+
+            if %w[http https].include?(parsed.scheme)
+              parsed
+            else
+              log_warning(
+                "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} " \
+                "environment variable ('#{unparsed_url_from_env}'). " \
+                "Ignoring the contents of #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL}."
+              )
+
+              nil
+            end
+          end
+      end
+
+      # NOTE: This should only be used AFTER parsing, via `#parsed_url`. The only other use-case where this can be used
+      # directly without parsing, is when displaying in warning messages, to show users what it actually contains.
+      def unparsed_url_from_env
+        @unparsed_url_from_env ||= ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL]
+      end
+
+      def pick_from(
+        # Hacky required kw args, we can get rid of this when we drop Ruby 2.0
+        configurations_in_priority_order: raise(ArgumentError, 'missing keyword :configurations'),
+        or_use_default: raise(ArgumentError, 'missing keyword :or_use_default')
+      )
+        detected_configurations_in_priority_order = configurations_in_priority_order.select(&:value?)
+
+        if detected_configurations_in_priority_order.any?
+          warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+
+          # The configurations are listed in priority, so we only need to look at the first; if there's more than
+          # one, we emit a warning above
+          detected_configurations_in_priority_order.first.value
+        else
+          or_use_default
+        end
+      end
+
+      def warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+        return unless detected_configurations_in_priority_order.map(&:value).uniq.size > 1
+
+        log_warning(
+          'Configuration mismatch: values differ between ' \
+          "#{detected_configurations_in_priority_order
+            .map { |config| "#{config.friendly_name} ('#{config.value}')" }.join(' and ')}" \
+          ". Using '#{detected_configurations_in_priority_order.first.value}'."
+        )
+      end
+
+      def log_warning(message)
+        logger.warn(message) if logger
+      end
+
+      DetectedConfiguration = Struct.new(:friendly_name, :value) do
+        def initialize(
+          # Hacky required kw args, we can get rid of this when we drop Ruby 2.0
+          friendly_name: raise(ArgumentError, 'missing keyword :friendly_name'),
+          value: raise(ArgumentError, 'missing keyword :value')
+        )
+          super(friendly_name, value)
+          freeze
+        end
+
+        def value?
+          !value.nil?
+        end
+      end
+      private_constant :DetectedConfiguration
+
+      # NOTE: Due to... legacy reasons... Some classes like having an `AgentSettings` instance to fall back to.
+      # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
+      # represents only settings specified via environment variables + the usual defaults.
+      #
+      # YOU DO NOT WANT TO USE THE BELOW INSTANCE ON ANY NEWLY WRITTEN CODE, as it ignores any settings specified
+      # by users via `Datadog.configure`.
+      ENVIRONMENT_AGENT_SETTINGS = call(Settings.new, logger: nil)
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/ddtrace/configuration/base.rb

@@ -37,7 +37,7 @@ module Datadog
 
         def new_settings_class(&block)
           Class.new { include Datadog::Configuration::Base }.tap do |klass|
-            klass.instance_eval(&block) if block_given?
+            klass.instance_eval(&block) if block
           end
         end
       end

data/lib/ddtrace/configuration/components.rb

@@ -1,5 +1,7 @@
+require 'ddtrace/configuration/agent_settings_resolver'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/logger'
+require 'ddtrace/profiling'
 require 'ddtrace/runtime/metrics'
 require 'ddtrace/tracer'
 require 'ddtrace/workers/runtime_metrics'
@@ -7,7 +9,7 @@ require 'ddtrace/workers/runtime_metrics'
 module Datadog
   module Configuration
     # Global components for the trace library.
-    # rubocop:disable Metrics/LineLength
+    # rubocop:disable Layout/LineLength
     class Components
       class << self
         def build_health_metrics(settings)
@@ -19,7 +21,7 @@ module Datadog
         end
 
         def build_logger(settings)
-          logger = settings.logger.instance || Datadog::Logger.new(STDOUT)
+          logger = settings.logger.instance || Datadog::Logger.new($stdout)
           logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
           logger
@@ -43,7 +45,7 @@ module Datadog
           Datadog::Workers::RuntimeMetrics.new(options)
         end
 
-        def build_tracer(settings)
+        def build_tracer(settings, agent_settings)
           # If a custom tracer has been provided, use it instead.
           # Ignore all other options (they should already be configured.)
           tracer = settings.tracer.instance
@@ -61,11 +63,27 @@ module Datadog
           #       tracer initialization for now. Just reconfigure using the
           #       existing mutable #configure function. Remove when these components
           #       are extracted.
-          tracer.configure(build_tracer_options(settings))
+          tracer.configure(agent_settings: agent_settings, **build_tracer_options(settings))
 
           tracer
         end
 
+        def build_profiler(settings, agent_settings)
+          return unless Datadog::Profiling.supported? && settings.profiling.enabled
+
+          # Load extensions needed to support some of the Profiling features
+          Datadog::Profiling::Tasks::Setup.new.run
+
+          # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+          recorder = build_profiler_recorder(settings)
+          collectors = build_profiler_collectors(settings, recorder)
+          exporters = build_profiler_exporters(settings, agent_settings)
+          scheduler = build_profiler_scheduler(settings, recorder, exporters)
+
+          Datadog::Profiler.new(collectors, scheduler)
+        end
+
         private
 
         def build_tracer_tags(settings)
@@ -79,22 +97,53 @@ module Datadog
           settings = settings.tracer
 
           {}.tap do |opts|
-            opts[:hostname] = settings.hostname unless settings.hostname.nil?
             opts[:min_spans_before_partial_flush] = settings.partial_flush.min_spans_threshold unless settings.partial_flush.min_spans_threshold.nil?
             opts[:partial_flush] = settings.partial_flush.enabled unless settings.partial_flush.enabled.nil?
-            opts[:port] = settings.port unless settings.port.nil?
             opts[:priority_sampling] = settings.priority_sampling unless settings.priority_sampling.nil?
             opts[:sampler] = settings.sampler unless settings.sampler.nil?
-            opts[:transport_options] = settings.transport_options
             opts[:writer] = settings.writer unless settings.writer.nil?
             opts[:writer_options] = settings.writer_options if settings.writer.nil?
           end
         end
+
+        def build_profiler_recorder(settings)
+          event_classes = [Datadog::Profiling::Events::StackSample]
+
+          Datadog::Profiling::Recorder.new(event_classes, settings.profiling.max_events)
+        end
+
+        def build_profiler_collectors(settings, recorder)
+          [
+            Datadog::Profiling::Collectors::Stack.new(
+              recorder,
+              max_frames: settings.profiling.max_frames
+              # TODO: Provide proc that identifies Datadog worker threads?
+              # ignore_thread: settings.profiling.ignore_profiler
+            )
+          ]
+        end
+
+        def build_profiler_exporters(settings, agent_settings)
+          transport =
+            settings.profiling.exporter.transport || Datadog::Profiling::Transport::HTTP.default(
+              agent_settings: agent_settings,
+              site: settings.site,
+              api_key: settings.api_key,
+              profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
+            )
+
+          [Datadog::Profiling::Exporter.new(transport)]
+        end
+
+        def build_profiler_scheduler(settings, recorder, exporters)
+          Datadog::Profiling::Scheduler.new(recorder, exporters)
+        end
       end
 
       attr_reader \
         :health_metrics,
         :logger,
+        :profiler,
         :runtime_metrics,
         :tracer
 
@@ -102,8 +151,13 @@ module Datadog
         # Logger
         @logger = self.class.build_logger(settings)
 
+        agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
         # Tracer
-        @tracer = self.class.build_tracer(settings)
+        @tracer = self.class.build_tracer(settings, agent_settings)
+
+        # Profiler
+        @profiler = self.class.build_profiler(settings, agent_settings)
 
         # Runtime metrics
         @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
@@ -113,7 +167,20 @@ module Datadog
       end
 
       # Starts up components
-      def startup!(settings); end
+      def startup!(settings)
+        if settings.profiling.enabled
+          if profiler
+            @logger.debug('Profiling started')
+            profiler.start
+          else
+            # Display a warning for users who expected profiling to be enabled
+            unsupported_reason = Datadog::Profiling.unsupported_reason
+            logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+          end
+        else
+          @logger.debug('Profiling is disabled')
+        end
+      end
 
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
@@ -123,12 +190,20 @@ module Datadog
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer
 
+        # Shutdown old profiler
+        profiler.shutdown! unless profiler.nil?
+
         # Shutdown workers
-        runtime_metrics.enabled = false
-        runtime_metrics.stop(true)
+        runtime_metrics.stop(true, close_metrics: false)
 
         # Shutdown the old metrics, unless they are still being used.
         # (e.g. custom Statsd instances.)
+        #
+        # TODO: This violates the encapsulation created by Runtime::Metrics and
+        # Health::Metrics, by directly manipulating `statsd` and changing
+        # it's lifecycle management.
+        # If we need to directly have ownership of `statsd` lifecycle, we should
+        # have direct ownership of it.
         old_statsd = [
           runtime_metrics.metrics.statsd,
           health_metrics.statsd

data/lib/ddtrace/configuration/option_definition.rb

@@ -60,7 +60,7 @@ module Datadog
         end
 
         def default(value = nil, &block)
-          @default = block_given? ? block : value
+          @default = block || value
         end
 
         def delegate_to(&block)
@@ -71,8 +71,6 @@ module Datadog
           @helpers[name] = block
         end
 
-        # rubocop:disable Style/TrivialAccessors
-        # (Rubocop erroneously thinks #lazy == #lazy= )
         def lazy(value = true)
           @lazy = value
         end

data/lib/ddtrace/configuration/options.rb

@@ -14,10 +14,8 @@ module Datadog
       # Class behavior for a configuration object with options
       module ClassMethods
         def options
-          @options ||= begin
-            # Allows for class inheritance of option definitions
-            superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
-          end
+          # Allows for class inheritance of option definitions
+          @options ||= superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
         end
 
         protected
@@ -51,6 +49,7 @@ module Datadog
         def define_helpers(helpers)
           helpers.each do |name, block|
             next unless block.is_a?(Proc)
+
             define_method(name, &block)
           end
         end
@@ -102,9 +101,7 @@ module Datadog
         end
 
         def assert_valid_option!(name)
-          unless option_defined?(name)
-            raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}")
-          end
+          raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}") unless option_defined?(name)
         end
       end