ddtrace 0.49.0 → 1.12.1

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -0,0 +1,396 @@
+require 'uri'
+
+require_relative 'settings'
+require_relative '../../tracing/configuration/ext'
+require_relative '../../../ddtrace/transport/ext'
+
+module Datadog
+  module Core
+    module Configuration
+      # This class unifies all the different ways that users can configure how we talk to the agent.
+      #
+      # It has quite a lot of complexity, but this complexity just reflects the actual complexity we have around our
+      # configuration today. E.g., this is just all of the complexity regarding agent settings gathered together in a
+      # single place. As we deprecate more and more of the different ways that these things can be configured,
+      # this class will reflect that simplification as well.
+      #
+      # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
+      # about it and pick a value based on the following priority: code > environment variable > defaults.
+      class AgentSettingsResolver
+        AgentSettings = \
+          Struct.new(
+            :adapter,
+            :ssl,
+            :hostname,
+            :port,
+            :uds_path,
+            :timeout_seconds,
+            :deprecated_for_removal_transport_configuration_proc,
+          ) do
+            def initialize(
+              adapter:,
+              ssl:,
+              hostname:,
+              port:,
+              uds_path:,
+              timeout_seconds:,
+              deprecated_for_removal_transport_configuration_proc:
+            )
+              super(
+                adapter,
+                ssl,
+                hostname,
+                port,
+                uds_path,
+                timeout_seconds,
+                deprecated_for_removal_transport_configuration_proc
+              )
+              freeze
+            end
+          end
+
+        def self.call(settings, logger: Datadog.logger)
+          new(settings, logger: logger).send(:call)
+        end
+
+        private
+
+        attr_reader \
+          :logger,
+          :settings
+
+        def initialize(settings, logger: Datadog.logger)
+          @settings = settings
+          @logger = logger
+        end
+
+        def call
+          # A transport_options proc configured for unix domain socket overrides most of the logic on this file
+          if transport_options.adapter == Datadog::Transport::Ext::UnixSocket::ADAPTER
+            return AgentSettings.new(
+              adapter: Datadog::Transport::Ext::UnixSocket::ADAPTER,
+              ssl: false,
+              hostname: nil,
+              port: nil,
+              uds_path: transport_options.uds_path,
+              timeout_seconds: timeout_seconds,
+              deprecated_for_removal_transport_configuration_proc: nil,
+            )
+          end
+
+          AgentSettings.new(
+            adapter: adapter,
+            ssl: ssl?,
+            hostname: hostname,
+            port: port,
+            uds_path: uds_path,
+            timeout_seconds: timeout_seconds,
+            # NOTE: When provided, the deprecated_for_removal_transport_configuration_proc can override all
+            # values above (ssl, hostname, port, timeout), or even make them irrelevant (by using an unix socket or
+            # enabling test mode instead).
+            # That is the main reason why it is deprecated -- it's an opaque function that may set a bunch of settings
+            # that we know nothing of until we actually call it.
+            deprecated_for_removal_transport_configuration_proc: deprecated_for_removal_transport_configuration_proc,
+          )
+        end
+
+        def adapter
+          if should_use_uds? && !mixed_http_and_uds?
+            Datadog::Transport::Ext::UnixSocket::ADAPTER
+          else
+            Datadog::Transport::Ext::HTTP::ADAPTER
+          end
+        end
+
+        def configured_hostname
+          return @configured_hostname if defined?(@configured_hostname)
+
+          @configured_hostname = pick_from(
+            DetectedConfiguration.new(
+              friendly_name: "'c.tracing.transport_options'",
+              value: transport_options.hostname,
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "'c.agent.host'",
+              value: settings.agent.host
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
+              value: parsed_http_url && parsed_http_url.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Transport::ENV_DEFAULT_HOST} environment variable",
+              value: ENV[Datadog::Core::Configuration::Ext::Transport::ENV_DEFAULT_HOST]
+            )
+          )
+        end
+
+        def configured_port
+          return @configured_port if defined?(@configured_port)
+
+          @configured_port = pick_from(
+            try_parsing_as_integer(
+              friendly_name: "'c.tracing.transport_options'",
+              value: transport_options.port,
+            ),
+            try_parsing_as_integer(
+              friendly_name: '"c.agent.port"',
+              value: settings.agent.port,
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
+              value: parsed_http_url && parsed_http_url.port,
+            ),
+            try_parsing_as_integer(
+              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_PORT} environment variable",
+              value: ENV[Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_PORT],
+            )
+          )
+        end
+
+        def try_parsing_as_integer(value:, friendly_name:)
+          value =
+            begin
+              Integer(value) if value
+            rescue ArgumentError, TypeError
+              log_warning("Invalid value for #{friendly_name} (#{value.inspect}). Ignoring this configuration.")
+
+              nil
+            end
+
+          DetectedConfiguration.new(friendly_name: friendly_name, value: value)
+        end
+
+        def ssl?
+          transport_options.ssl ||
+            (!parsed_url.nil? && parsed_url.scheme == 'https')
+        end
+
+        def hostname
+          configured_hostname || (should_use_uds? ? nil : Datadog::Transport::Ext::HTTP::DEFAULT_HOST)
+        end
+
+        def port
+          configured_port || (should_use_uds? ? nil : Datadog::Transport::Ext::HTTP::DEFAULT_PORT)
+        end
+
+        # Unix socket path in the file system
+        def uds_path
+          if mixed_http_and_uds?
+            nil
+          elsif parsed_url && unix_scheme?(parsed_url)
+            path = parsed_url.to_s
+            # Some versions of the built-in uri gem leave the original url untouched, and others remove the //, so this
+            # supports both
+            if path.start_with?('unix://')
+              path.sub('unix://', '')
+            else
+              path.sub('unix:', '')
+            end
+          else
+            uds_fallback
+          end
+        end
+
+        # Defaults to +nil+, letting the adapter choose what default
+        # works best in their case.
+        def timeout_seconds
+          transport_options.timeout_seconds
+        end
+
+        # In transport_options, we try to invoke the transport_options proc and get its configuration. In case that
+        # doesn't work, we include the proc directly in the agent settings result.
+        def deprecated_for_removal_transport_configuration_proc
+          if settings.tracing.transport_options.is_a?(Proc) && transport_options.adapter.nil?
+            settings.tracing.transport_options
+          end
+        end
+
+        # We only use the default unix socket if it is already present.
+        # This is by design, as we still want to use the default host:port if no unix socket is present.
+        def uds_fallback
+          return @uds_fallback if defined?(@uds_fallback)
+
+          @uds_fallback =
+            if configured_hostname.nil? &&
+                configured_port.nil? &&
+                deprecated_for_removal_transport_configuration_proc.nil? &&
+                File.exist?(Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH)
+
+              Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH
+            end
+        end
+
+        def should_use_uds?
+          parsed_url && unix_scheme?(parsed_url) ||
+            # If no agent settings have been provided, we try to connect using a local unix socket.
+            # We only do so if the socket is present when `ddtrace` runs.
+            !uds_fallback.nil?
+        end
+
+        def parsed_url
+          return @parsed_url if defined?(@parsed_url)
+
+          unparsed_url_from_env = ENV[Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL]
+
+          @parsed_url =
+            if unparsed_url_from_env
+              parsed = URI.parse(unparsed_url_from_env)
+
+              if http_scheme?(parsed) || unix_scheme?(parsed)
+                parsed
+              else
+                # rubocop:disable Layout/LineLength
+                log_warning(
+                  "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} " \
+                  "environment variable ('#{unparsed_url_from_env}'). " \
+                  "Ignoring the contents of #{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL}."
+                )
+                # rubocop:enable Layout/LineLength
+
+                nil
+              end
+            end
+        end
+
+        def pick_from(*configurations_in_priority_order)
+          detected_configurations_in_priority_order = configurations_in_priority_order.select(&:value?)
+
+          if detected_configurations_in_priority_order.any?
+            warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+
+            # The configurations are listed in priority, so we only need to look at the first; if there's more than
+            # one, we emit a warning above
+            detected_configurations_in_priority_order.first.value
+          end
+        end
+
+        def warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+          return unless detected_configurations_in_priority_order.map(&:value).uniq.size > 1
+
+          log_warning(
+            'Configuration mismatch: values differ between ' \
+            "#{detected_configurations_in_priority_order
+              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(' and ')}" \
+            ". Using #{detected_configurations_in_priority_order.first.value.inspect}."
+          )
+        end
+
+        def log_warning(message)
+          logger.warn(message) if logger
+        end
+
+        def http_scheme?(uri)
+          ['http', 'https'].include?(uri.scheme)
+        end
+
+        # Expected to return nil (not false!) when it's not http
+        def parsed_http_url
+          parsed_url if parsed_url && http_scheme?(parsed_url)
+        end
+
+        def unix_scheme?(uri)
+          uri.scheme == 'unix'
+        end
+
+        # When we have mixed settings for http/https and uds, we print a warning and ignore the uds settings
+        def mixed_http_and_uds?
+          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
+
+          @mixed_http_and_uds = (configured_hostname || configured_port) && should_use_uds?
+
+          if @mixed_http_and_uds
+            warn_if_configuration_mismatch(
+              [
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration of hostname/port for http/https use',
+                  value: "hostname: '#{configured_hostname}', port: #{configured_port.inspect}",
+                ),
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration for unix domain socket',
+                  value: "unix://#{uds_path}",
+                ),
+              ]
+            )
+          end
+
+          @mixed_http_and_uds
+        end
+
+        # The settings.tracing.transport_options allows users to have full control over the settings used to
+        # communicate with the agent. In the general case, we can't extract the configuration from this proc, but
+        # in the specific case of the http and unix socket adapters we can, and we use this method together with the
+        # `TransportOptionsResolver` to call the proc and extract its information.
+        def transport_options
+          return @transport_options if defined?(@transport_options)
+
+          transport_options_proc = settings.tracing.transport_options
+
+          @transport_options = TransportOptions.new
+
+          if transport_options_proc.is_a?(Proc)
+            begin
+              transport_options_proc.call(TransportOptionsResolver.new(@transport_options))
+            rescue NoMethodError => e
+              if logger
+                logger.debug do
+                  'Could not extract configuration from transport_options proc. ' \
+                  "Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+                end
+              end
+
+              # Reset the object; we shouldn't return the same one we passed into the proc as it may have
+              # some partial configuration and we want all-or-nothing.
+              @transport_options = TransportOptions.new
+            end
+          end
+
+          @transport_options.freeze
+        end
+
+        # Represents a given configuration value and where we got it from
+        class DetectedConfiguration
+          attr_reader :friendly_name, :value
+
+          def initialize(friendly_name:, value:)
+            @friendly_name = friendly_name
+            @value = value
+            freeze
+          end
+
+          def value?
+            !value.nil?
+          end
+        end
+        private_constant :DetectedConfiguration
+
+        # Used to contain information extracted from the transport_options proc (see #transport_options above)
+        TransportOptions = Struct.new(:adapter, :hostname, :port, :timeout_seconds, :ssl, :uds_path)
+        private_constant :TransportOptions
+
+        # Used to extract information from the transport_options proc (see #transport_options above)
+        class TransportOptionsResolver
+          def initialize(transport_options)
+            @transport_options = transport_options
+          end
+
+          def adapter(kind_or_custom_adapter, *args, **kwargs)
+            case kind_or_custom_adapter
+            when Datadog::Transport::Ext::HTTP::ADAPTER
+              @transport_options.adapter = Datadog::Transport::Ext::HTTP::ADAPTER
+              @transport_options.hostname = args[0] || kwargs[:hostname]
+              @transport_options.port = args[1] || kwargs[:port]
+              @transport_options.timeout_seconds = kwargs[:timeout]
+              @transport_options.ssl = kwargs[:ssl]
+            when Datadog::Transport::Ext::UnixSocket::ADAPTER
+              @transport_options.adapter = Datadog::Transport::Ext::UnixSocket::ADAPTER
+              @transport_options.uds_path = args[0] || kwargs[:uds_path]
+            end
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/base.rb

@@ -0,0 +1,99 @@
+require_relative '../environment/variable_helpers'
+require_relative 'options'
+
+module Datadog
+  module Core
+    module Configuration
+      # Basic configuration behavior
+      # @public_api
+      module Base
+        def self.included(base)
+          base.extend(Core::Environment::VariableHelpers)
+          base.include(Core::Environment::VariableHelpers)
+          base.include(Options)
+
+          base.extend(ClassMethods)
+          base.include(InstanceMethods)
+        end
+
+        # Class methods for configuration
+        # @public_api
+        module ClassMethods
+          protected
+
+          # Allows subgroupings of settings to be defined.
+          # e.g. `settings :foo { option :bar }` --> `config.foo.bar`
+          # @param [Symbol] name option name. Methods will be created based on this name.
+          def settings(name, &block)
+            settings_class = new_settings_class(&block)
+
+            option(name) do |o|
+              o.default { settings_class.new }
+              o.lazy
+
+              o.resetter do |value|
+                value.reset! if value.respond_to?(:reset!)
+                value
+              end
+
+              o.type settings_class
+            end
+          end
+
+          private
+
+          def new_settings_class(&block)
+            Class.new { include Configuration::Base }.tap do |klass|
+              klass.instance_eval(&block) if block
+            end
+          end
+        end
+
+        # Instance methods for configuration
+        # @public_api
+        module InstanceMethods
+          def initialize(options = {})
+            configure(options) unless options.empty?
+          end
+
+          def configure(opts = {})
+            # Sort the options in preference of dependency order first
+            ordering = self.class.options.dependency_order
+            sorted_opts = opts.sort_by do |name, _value|
+              ordering.index(name) || (ordering.length + 1)
+            end.to_h
+
+            # Apply options in sort order
+            sorted_opts.each do |name, value|
+              if respond_to?("#{name}=")
+                send("#{name}=", value)
+              elsif option_defined?(name)
+                set_option(name, value)
+              end
+            end
+
+            # Apply any additional settings from block
+            yield(self) if block_given?
+          end
+
+          def to_h
+            options_hash
+          end
+
+          # Retrieves a nested option from a list of symbols
+          def dig(*options)
+            raise ArgumentError, 'expected at least one option' if options.empty?
+
+            options.inject(self) do |receiver, option|
+              receiver.send(option)
+            end
+          end
+
+          def reset!
+            reset_options!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/components.rb

@@ -0,0 +1,161 @@
+require_relative 'agent_settings_resolver'
+require_relative '../diagnostics/environment_logger'
+require_relative '../diagnostics/health'
+require_relative '../logger'
+require_relative '../runtime/metrics'
+require_relative '../telemetry/client'
+require_relative '../workers/runtime_metrics'
+
+require_relative '../remote/component'
+require_relative '../../tracing/component'
+require_relative '../../profiling/component'
+require_relative '../../appsec/component'
+
+module Datadog
+  module Core
+    module Configuration
+      # Global components for the trace library.
+      class Components
+        class << self
+          include Datadog::Tracing::Component
+
+          def build_health_metrics(settings)
+            settings = settings.diagnostics.health_metrics
+            options = { enabled: settings.enabled }
+            options[:statsd] = settings.statsd unless settings.statsd.nil?
+
+            Core::Diagnostics::Health::Metrics.new(**options)
+          end
+
+          def build_logger(settings)
+            logger = settings.logger.instance || Core::Logger.new($stdout)
+            logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
+
+            logger
+          end
+
+          def build_runtime_metrics(settings)
+            options = { enabled: settings.runtime_metrics.enabled }
+            options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
+            options[:services] = [settings.service] unless settings.service.nil?
+
+            Core::Runtime::Metrics.new(**options)
+          end
+
+          def build_runtime_metrics_worker(settings)
+            # NOTE: Should we just ignore building the worker if its not enabled?
+            options = settings.runtime_metrics.opts.merge(
+              enabled: settings.runtime_metrics.enabled,
+              metrics: build_runtime_metrics(settings)
+            )
+
+            Core::Workers::RuntimeMetrics.new(options)
+          end
+
+          def build_telemetry(settings, agent_settings, logger)
+            enabled = settings.telemetry.enabled
+            if agent_settings.adapter != Datadog::Transport::Ext::HTTP::ADAPTER
+              enabled = false
+              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+            end
+
+            Telemetry::Client.new(enabled: enabled)
+          end
+        end
+
+        attr_reader \
+          :health_metrics,
+          :logger,
+          :remote,
+          :profiler,
+          :runtime_metrics,
+          :telemetry,
+          :tracer,
+          :appsec
+
+        def initialize(settings)
+          @logger = self.class.build_logger(settings)
+
+          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
+          @remote = Remote::Component.build(settings, agent_settings)
+          @tracer = self.class.build_tracer(settings, agent_settings)
+          @profiler = Datadog::Profiling::Component.build_profiler_component(
+            settings: settings,
+            agent_settings: agent_settings,
+            optional_tracer: @tracer,
+          )
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
+          @health_metrics = self.class.build_health_metrics(settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
+        end
+
+        # Starts up components
+        def startup!(settings)
+          if settings.profiling.enabled
+            if profiler
+              @logger.debug('Profiling started')
+              profiler.start
+            else
+              # Display a warning for users who expected profiling to be enabled
+              unsupported_reason = Profiling.unsupported_reason
+              logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+            end
+          else
+            @logger.debug('Profiling is disabled')
+          end
+        end
+
+        # Shuts down all the components in use.
+        # If it has another instance to compare to, it will compare
+        # and avoid tearing down parts still in use.
+        def shutdown!(replacement = nil)
+          # Shutdown remote configuration
+          remote.shutdown! if remote
+
+          # Decommission AppSec
+          appsec.shutdown! if appsec
+
+          # Shutdown the old tracer, unless it's still being used.
+          # (e.g. a custom tracer instance passed in.)
+          tracer.shutdown! unless replacement && tracer == replacement.tracer
+
+          # Shutdown old profiler
+          profiler.shutdown! unless profiler.nil?
+
+          # Shutdown workers
+          runtime_metrics.stop(true, close_metrics: false)
+
+          # Shutdown the old metrics, unless they are still being used.
+          # (e.g. custom Statsd instances.)
+          #
+          # TODO: This violates the encapsulation created by Runtime::Metrics and
+          # Health::Metrics, by directly manipulating `statsd` and changing
+          # it's lifecycle management.
+          # If we need to directly have ownership of `statsd` lifecycle, we should
+          # have direct ownership of it.
+          old_statsd = [
+            runtime_metrics.metrics.statsd,
+            health_metrics.statsd
+          ].compact.uniq
+
+          new_statsd =  if replacement
+                          [
+                            replacement.runtime_metrics.metrics.statsd,
+                            replacement.health_metrics.statsd
+                          ].compact.uniq
+                        else
+                          []
+                        end
+
+          unused_statsd = (old_statsd - (old_statsd & new_statsd))
+          unused_statsd.each(&:close)
+
+          telemetry.stop!
+          telemetry.emit_closing! unless replacement
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/dependency_resolver.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'tsort'
+
+module Datadog
+  module Core
+    module Configuration
+      # Resolver performs a topological sort over the dependency graph
+      class DependencyResolver
+        include TSort
+
+        def initialize(dependency_graph = {})
+          @dependency_graph = dependency_graph
+        end
+
+        def tsort_each_node(&blk)
+          @dependency_graph.each_key(&blk)
+        end
+
+        def tsort_each_child(node, &blk)
+          @dependency_graph.fetch(node).each(&blk)
+        end
+
+        alias call tsort
+      end
+    end
+  end
+end