ddtrace 0.45.0 → 0.50.0

data/lib/datadog/ci/ext/settings.rb

@@ -0,0 +1,10 @@
+module Datadog
+  module CI
+    module Ext
+      # Defines constants for test tags
+      module Settings
+        ENV_MODE_ENABLED = 'DD_TRACE_CI_MODE_ENABLED'.freeze
+      end
+    end
+  end
+end

data/lib/datadog/ci/ext/test.rb

@@ -0,0 +1,28 @@
+module Datadog
+  module CI
+    module Ext
+      # Defines constants for test tags
+      module Test
+        CONTEXT_ORIGIN = 'ciapp-test'.freeze
+
+        TAG_ARGUMENTS = 'test.arguments'.freeze
+        TAG_FRAMEWORK = 'test.framework'.freeze
+        TAG_NAME = 'test.name'.freeze
+        TAG_SKIP_REASON = 'test.skip_reason'.freeze
+        TAG_STATUS = 'test.status'.freeze
+        TAG_SUITE = 'test.suite'.freeze
+        TAG_TRAITS = 'test.traits'.freeze
+        TAG_TYPE = 'test.type'.freeze
+
+        # TODO: is there a better place for SPAN_KIND?
+        TAG_SPAN_KIND = 'span.kind'.freeze
+
+        module Status
+          PASS = 'pass'.freeze
+          FAIL = 'fail'.freeze
+          SKIP = 'skip'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/extensions.rb

@@ -0,0 +1,17 @@
+require 'ddtrace/configuration/settings'
+require 'ddtrace/configuration/components'
+
+require 'datadog/ci/configuration/settings'
+require 'datadog/ci/configuration/components'
+
+module Datadog
+  module CI
+    # Extends Datadog tracing with CI features
+    module Extensions
+      def self.activate!
+        Datadog::Configuration::Settings.extend(CI::Configuration::Settings)
+        Datadog::Configuration::Components.prepend(CI::Configuration::Components)
+      end
+    end
+  end
+end

data/lib/datadog/ci/test.rb

@@ -0,0 +1,63 @@
+require 'ddtrace/contrib/analytics'
+
+require 'datadog/ci/ext/app_types'
+require 'datadog/ci/ext/test'
+
+module Datadog
+  module CI
+    # Common behavior for CI tests
+    module Test
+      # Creates a new span for a CI test
+      def self.trace(tracer, span_name, options = {})
+        span_options = {
+          span_type: Ext::AppTypes::TEST
+        }.merge(options[:span_options] || {})
+
+        if block_given?
+          tracer.trace(span_name, span_options) do |span|
+            set_tags!(span, options)
+            yield(span)
+          end
+        else
+          span = tracer.trace(span_name, span_options)
+          set_tags!(span, options)
+          span
+        end
+      end
+
+      # Adds tags to a CI test span.
+      def self.set_tags!(span, tags = {})
+        tags ||= {}
+
+        # Set default tags
+        span.context.origin = Ext::Test::CONTEXT_ORIGIN if span.context
+        Datadog::Contrib::Analytics.set_measured(span)
+        span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TEST)
+        Ext::Environment.tags(ENV).each { |k, v| span.set_tag(k, v) }
+
+        # Set contextual tags
+        span.set_tag(Ext::Test::TAG_FRAMEWORK, tags[:framework]) if tags[:framework]
+        span.set_tag(Ext::Test::TAG_NAME, tags[:test_name]) if tags[:test_name]
+        span.set_tag(Ext::Test::TAG_SUITE, tags[:test_suite]) if tags[:test_suite]
+        span.set_tag(Ext::Test::TAG_TYPE, tags[:test_type]) if tags[:test_type]
+
+        span
+      end
+
+      def self.passed!(span)
+        span.set_tag(Ext::Test::TAG_STATUS, Ext::Test::Status::PASS)
+      end
+
+      def self.failed!(span, exception = nil)
+        span.status = 1
+        span.set_tag(Ext::Test::TAG_STATUS, Ext::Test::Status::FAIL)
+        span.set_error(exception) unless exception.nil?
+      end
+
+      def self.skipped!(span, exception = nil)
+        span.set_tag(Ext::Test::TAG_STATUS, Ext::Test::Status::SKIP)
+        span.set_error(exception) unless exception.nil?
+      end
+    end
+  end
+end

data/lib/ddtrace.rb

@@ -1,5 +1,3 @@
-require 'thread'
-
 # During development, we load `ddtrace` by through `ddtrace.gemspec`,
 # which in turn eager loads 'ddtrace/version'.
 #
@@ -15,13 +13,12 @@ require 'ddtrace/quantization/http'
 require 'ddtrace/pipeline'
 require 'ddtrace/configuration'
 require 'ddtrace/patcher'
-require 'ddtrace/augmentation'
 require 'ddtrace/metrics'
 require 'ddtrace/auto_instrument_base'
+require 'ddtrace/profiling'
 
 # \Datadog global namespace that includes all tracing functionality for Tracer and Span classes.
 module Datadog
-  extend Augmentation
   extend Configuration
   extend AutoInstrumentBase
 
@@ -51,7 +48,6 @@ require 'ddtrace/contrib/active_record/integration'
 require 'ddtrace/contrib/active_support/integration'
 require 'ddtrace/contrib/aws/integration'
 require 'ddtrace/contrib/concurrent_ruby/integration'
-require 'ddtrace/contrib/cucumber/integration'
 require 'ddtrace/contrib/dalli/integration'
 require 'ddtrace/contrib/delayed_job/integration'
 require 'ddtrace/contrib/elasticsearch/integration'
@@ -78,11 +74,9 @@ require 'ddtrace/contrib/rake/integration'
 require 'ddtrace/contrib/redis/integration'
 require 'ddtrace/contrib/resque/integration'
 require 'ddtrace/contrib/rest_client/integration'
-require 'ddtrace/contrib/rspec/integration'
 require 'ddtrace/contrib/sequel/integration'
 require 'ddtrace/contrib/shoryuken/integration'
 require 'ddtrace/contrib/sidekiq/integration'
 require 'ddtrace/contrib/sinatra/integration'
 require 'ddtrace/contrib/sneakers/integration'
 require 'ddtrace/contrib/sucker_punch/integration'
-require 'ddtrace/monkey'

data/lib/ddtrace/analytics.rb

@@ -6,11 +6,13 @@ module Datadog
     class << self
       def set_sample_rate(span, sample_rate)
         return if span.nil? || !sample_rate.is_a?(Numeric)
+
         span.set_metric(Datadog::Ext::Analytics::TAG_SAMPLE_RATE, sample_rate)
       end
 
       def set_measured(span, value = true)
         return if span.nil?
+
         # rubocop:disable Style/MultipleComparison
         value = value == true || value == 1 ? 1 : 0
         span.set_metric(Datadog::Ext::Analytics::TAG_MEASURED, value)

data/lib/ddtrace/buffer.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/runtime/object_space'
 
@@ -18,6 +17,7 @@ module Datadog
     # even if the buffer is full. In that case, a random item is discarded.
     def push(item)
       return if closed?
+
       full? ? replace!(item) : add!(item)
       item
     end
@@ -168,8 +168,8 @@ module Datadog
       synchronize { super }
     end
 
-    def synchronize
-      @mutex.synchronize { yield }
+    def synchronize(&block)
+      @mutex.synchronize(&block)
     end
   end
 
@@ -318,7 +318,7 @@ module Datadog
   #
   # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
   # TODO Making such a change is potentially breaking for users manually configuring the tracer.
-  TraceBuffer = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby' # rubocop:disable Style/ConstantName
+  TraceBuffer = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby'
                   CRubyTraceBuffer
                 else
                   ThreadSafeTraceBuffer

data/lib/ddtrace/configuration.rb

@@ -1,5 +1,4 @@
 require 'forwardable'
-
 require 'ddtrace/configuration/pin_setup'
 require 'ddtrace/configuration/settings'
 require 'ddtrace/configuration/components'
@@ -9,6 +8,33 @@ module Datadog
   module Configuration
     extend Forwardable
 
+    # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+    #
+    # This is important because components can end up being accessed from multiple application threads (for instance on
+    # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+    #
+    # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+    # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+    #
+    # Every method that directly or indirectly mutates @components should be holding the lock (through
+    # #safely_synchronize) while doing so.
+    COMPONENTS_WRITE_LOCK = Mutex.new
+    private_constant :COMPONENTS_WRITE_LOCK
+
+    # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+    # This was needed because we ran into several issues where we still needed to read the old
+    # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+    # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+    #
+    # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+    # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+    # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+    # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+    # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+    # Concurrency is hard.
+    COMPONENTS_READ_LOCK = Mutex.new
+    private_constant :COMPONENTS_READ_LOCK
+
     attr_writer :configuration
 
     def configuration
@@ -19,13 +45,15 @@ module Datadog
       if target.is_a?(Settings)
         yield(target) if block_given?
 
-        # Build immutable components from settings
-        @components ||= nil
-        @components = if @components
-                        replace_components!(target, @components)
-                      else
-                        build_components(target)
-                      end
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(target, @components)
+            else
+              build_components(target)
+            end
+          )
+        end
 
         target
       else
@@ -36,22 +64,19 @@ module Datadog
     def_delegators \
       :components,
       :health_metrics,
+      :profiler,
       :runtime_metrics,
       :tracer
 
     def logger
-      if instance_variable_defined?(:@components) && @components
+      # avoid initializing components if they didn't already exist
+      current_components = components(allow_initialization: false)
+
+      if current_components
         @temp_logger = nil
-        components.logger
+        current_components.logger
       else
-        # Use default logger without initializing components.
-        # This prevents recursive loops while initializing.
-        # e.g. Get logger --> Build components --> Log message --> Repeat...
-        @temp_logger ||= begin
-          logger = configuration.logger.instance || Datadog::Logger.new(STDOUT)
-          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
-          logger
-        end
+        logger_without_components
       end
     end
 
@@ -65,26 +90,64 @@ module Datadog
     #
     # Components won't be automatically reinitialized after a shutdown.
     def shutdown!
-      components.shutdown! if instance_variable_defined?(:@components) && @components
+      safely_synchronize do
+        @components.shutdown! if components?
+      end
     end
 
+    protected
+
+    def components(allow_initialization: true)
+      current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+      return current_components if current_components || !allow_initialization
+
+      safely_synchronize do |write_components|
+        (defined?(@components) && @components) || write_components.call(build_components(configuration))
+      end
+    end
+
+    private
+
     # Gracefully shuts down the tracer and disposes of component references,
     # allowing execution to start anew.
     #
     # In contrast with +#shutdown!+, components will be automatically
     # reinitialized after a reset.
+    #
+    # Used internally to ensure a clean environment between test runs.
     def reset!
-      shutdown!
-      @components = nil
+      safely_synchronize do |write_components|
+        @components.shutdown! if components?
+        write_components.call(nil)
+        configuration.reset!
+      end
     end
 
-    protected
+    def safely_synchronize
+      # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+      # safely_synchronize, this forces all writers to go through this method.
+      write_components = proc do |new_value|
+        COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+      end
 
-    def components
-      @components ||= build_components(configuration)
+      COMPONENTS_WRITE_LOCK.synchronize do
+        begin
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during ddtrace initialization. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{e.backtrace.join("\n\t")}"
+          )
+          nil
+        end
+      end
     end
 
-    private
+    def components?
+      # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+      (defined?(@components) && @components) != nil
+    end
 
     def build_components(settings)
       components = Components.new(settings)
@@ -99,5 +162,15 @@ module Datadog
       components.startup!(settings)
       components
     end
+
+    def logger_without_components
+      # Use default logger without initializing components.
+      # This enables logging during initialization, otherwise we'd run into deadlocks.
+      @temp_logger ||= begin
+        logger = configuration.logger.instance || Datadog::Logger.new($stdout)
+        logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+        logger
+      end
+    end
   end
 end

data/lib/ddtrace/configuration/agent_settings_resolver.rb

@@ -0,0 +1,230 @@
+require 'uri'
+
+require 'ddtrace/ext/transport'
+require 'ddtrace/configuration/settings'
+
+module Datadog
+  module Configuration
+    # This class unifies all the different ways that users can configure how we talk to the agent.
+    #
+    # It has quite a lot of complexity, but this complexity just reflects the actual complexity we have around our
+    # configuration today. E.g., this is just all of the complexity regarding agent settings gathered together in a
+    # single place. As we deprecate more and more of the different ways that these things can be configured,
+    # this class will reflect that simplification as well.
+    #
+    # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
+    # about it and pick a value based on the following priority: code > environment variable > defaults.
+    #
+    # rubocop:disable Metrics/ClassLength
+    class AgentSettingsResolver
+      AgentSettings = \
+        Struct.new(
+          :ssl,
+          :hostname,
+          :port,
+          :timeout_seconds,
+          :deprecated_for_removal_transport_configuration_proc,
+          :deprecated_for_removal_transport_configuration_options
+        ) do
+          def initialize(
+            ssl:,
+            hostname:,
+            port:,
+            timeout_seconds:,
+            deprecated_for_removal_transport_configuration_proc:,
+            deprecated_for_removal_transport_configuration_options:
+          )
+            super(ssl, hostname, port, timeout_seconds, deprecated_for_removal_transport_configuration_proc, \
+              deprecated_for_removal_transport_configuration_options)
+            freeze
+          end
+        end
+
+      def self.call(settings, logger: Datadog.logger)
+        new(settings, logger: logger).send(:call)
+      end
+
+      private
+
+      attr_reader \
+        :logger,
+        :settings
+
+      def initialize(settings, logger: Datadog.logger)
+        @settings = settings
+        @logger = logger
+      end
+
+      def call
+        AgentSettings.new(
+          ssl: ssl?,
+          hostname: hostname,
+          port: port,
+          timeout_seconds: timeout_seconds,
+          # NOTE: When provided, the deprecated_for_removal_transport_configuration_proc can override all
+          # values above (ssl, hostname, port, timeout), or even make them irrelevant (by using an unix socket or
+          # enabling test mode instead).
+          # That is the main reason why it is deprecated -- it's an opaque function that may set a bunch of settings
+          # that we know nothing of until we actually call it.
+          deprecated_for_removal_transport_configuration_proc: deprecated_for_removal_transport_configuration_proc,
+          deprecated_for_removal_transport_configuration_options: deprecated_for_removal_transport_configuration_options
+        )
+      end
+
+      def hostname
+        pick_from(
+          configurations_in_priority_order: [
+            DetectedConfiguration.new(
+              friendly_name: "'c.tracer.hostname'",
+              value: settings.tracer.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url && parsed_url.hostname
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST} environment variable",
+              value: ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_HOST]
+            )
+          ],
+          or_use_default: Datadog::Ext::Transport::HTTP::DEFAULT_HOST
+        )
+      end
+
+      def port
+        port_from_env = ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT]
+        parsed_port_from_env =
+          if port_from_env
+            begin
+              Integer(port_from_env)
+            rescue ArgumentError
+              log_warning(
+                "Invalid value for #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT} environment variable " \
+                "('#{port_from_env}'). Ignoring this configuration."
+              )
+            end
+          end
+
+        pick_from(
+          configurations_in_priority_order: [
+            DetectedConfiguration.new(
+              friendly_name: '"c.tracer.port"',
+              value: settings.tracer.port
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} environment variable",
+              value: parsed_url && parsed_url.port
+            ),
+            DetectedConfiguration.new(
+              friendly_name: "#{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_PORT} environment variable",
+              value: parsed_port_from_env
+            )
+          ],
+          or_use_default: Datadog::Ext::Transport::HTTP::DEFAULT_PORT
+        )
+      end
+
+      def ssl?
+        !parsed_url.nil? && parsed_url.scheme == 'https'
+      end
+
+      def timeout_seconds
+        Datadog::Ext::Transport::HTTP::DEFAULT_TIMEOUT_SECONDS
+      end
+
+      def deprecated_for_removal_transport_configuration_proc
+        settings.tracer.transport_options if settings.tracer.transport_options.is_a?(Proc)
+      end
+
+      def deprecated_for_removal_transport_configuration_options
+        options = settings.tracer.transport_options
+
+        if options.is_a?(Hash) && !options.empty?
+          log_warning(
+            'Configuring the tracer via a c.tracer.transport_options hash is deprecated for removal in a future ' \
+            "ddtrace version (c.tracer.transport_options contained '#{options.inspect}')."
+          )
+
+          options
+        end
+      end
+
+      def parsed_url
+        return @parsed_url if defined?(@parsed_url)
+
+        @parsed_url =
+          if unparsed_url_from_env
+            parsed = URI.parse(unparsed_url_from_env)
+
+            if %w[http https].include?(parsed.scheme)
+              parsed
+            else
+              log_warning(
+                "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL} " \
+                "environment variable ('#{unparsed_url_from_env}'). " \
+                "Ignoring the contents of #{Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL}."
+              )
+
+              nil
+            end
+          end
+      end
+
+      # NOTE: This should only be used AFTER parsing, via `#parsed_url`. The only other use-case where this can be used
+      # directly without parsing, is when displaying in warning messages, to show users what it actually contains.
+      def unparsed_url_from_env
+        @unparsed_url_from_env ||= ENV[Datadog::Ext::Transport::HTTP::ENV_DEFAULT_URL]
+      end
+
+      def pick_from(configurations_in_priority_order:, or_use_default:)
+        detected_configurations_in_priority_order = configurations_in_priority_order.select(&:value?)
+
+        if detected_configurations_in_priority_order.any?
+          warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+
+          # The configurations are listed in priority, so we only need to look at the first; if there's more than
+          # one, we emit a warning above
+          detected_configurations_in_priority_order.first.value
+        else
+          or_use_default
+        end
+      end
+
+      def warn_if_configuration_mismatch(detected_configurations_in_priority_order)
+        return unless detected_configurations_in_priority_order.map(&:value).uniq.size > 1
+
+        log_warning(
+          'Configuration mismatch: values differ between ' \
+          "#{detected_configurations_in_priority_order
+            .map { |config| "#{config.friendly_name} ('#{config.value}')" }.join(' and ')}" \
+          ". Using '#{detected_configurations_in_priority_order.first.value}'."
+        )
+      end
+
+      def log_warning(message)
+        logger.warn(message) if logger
+      end
+
+      DetectedConfiguration = Struct.new(:friendly_name, :value) do
+        def initialize(friendly_name:, value:)
+          super(friendly_name, value)
+          freeze
+        end
+
+        def value?
+          !value.nil?
+        end
+      end
+      private_constant :DetectedConfiguration
+
+      # NOTE: Due to... legacy reasons... Some classes like having an `AgentSettings` instance to fall back to.
+      # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
+      # represents only settings specified via environment variables + the usual defaults.
+      #
+      # YOU DO NOT WANT TO USE THE BELOW INSTANCE ON ANY NEWLY WRITTEN CODE, as it ignores any settings specified
+      # by users via `Datadog.configure`.
+      ENVIRONMENT_AGENT_SETTINGS = call(Settings.new, logger: nil)
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end