ddtrace 0.54.2 → 1.10.1

data/lib/datadog/core/configuration.rb

@@ -0,0 +1,285 @@
+require_relative 'configuration/components'
+require_relative 'configuration/settings'
+require_relative 'telemetry/emitter'
+require_relative 'logger'
+require_relative 'pin'
+
+module Datadog
+  module Core
+    # Configuration provides a unique access point for configurations
+    module Configuration
+      # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+      #
+      # This is important because components can end up being accessed from multiple application threads (for instance on
+      # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+      #
+      # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+      # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+      #
+      # Every method that directly or indirectly mutates @components should be holding the lock (through
+      # #safely_synchronize) while doing so.
+      COMPONENTS_WRITE_LOCK = Mutex.new
+      private_constant :COMPONENTS_WRITE_LOCK
+
+      # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+      # This was needed because we ran into several issues where we still needed to read the old
+      # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+      # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+      #
+      # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+      # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+      # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+      # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+      # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+      # Concurrency is hard.
+      COMPONENTS_READ_LOCK = Mutex.new
+      private_constant :COMPONENTS_READ_LOCK
+
+      attr_writer :configuration
+
+      # Current Datadog configuration.
+      #
+      # Access to non-global configuration will raise an error.
+      #
+      # To modify the configuration, use {.configure}.
+      #
+      # @return [Datadog::Core::Configuration::Settings]
+      # @!attribute [r] configuration
+      # @public_api
+      def configuration
+        @configuration ||= Settings.new
+      end
+
+      # Apply global configuration changes to `Datadog`. An example of a {.configure} call:
+      #
+      # ```
+      # Datadog.configure do |c|
+      #   c.service = 'my-service'
+      #   c.env = 'staging'
+      #   # c.diagnostics.debug = true # Enables debug output
+      # end
+      # ```
+      #
+      # See {Datadog::Core::Configuration::Settings} for all available options, defaults, and
+      # available environment variables for configuration.
+      #
+      # Only permits access to global configuration settings; others will raise an error.
+      # If you wish to configure a setting for a specific Datadog component (e.g. Tracing),
+      # use the corresponding `Datadog::COMPONENT.configure` method instead.
+      #
+      # Because many configuration changes require restarting internal components,
+      # invoking {.configure} is the only safe way to change `Datadog` configuration.
+      #
+      # Successive calls to {.configure} maintain the previous configuration values:
+      # configuration is additive between {.configure} calls.
+      #
+      # The yielded configuration `c` comes pre-populated from environment variables, if
+      # any are applicable.
+      #
+      # @yieldparam [Datadog::Core::Configuration::Settings] c the mutable configuration object
+      def configure
+        configuration = self.configuration
+        yield(configuration)
+
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(configuration, @components)
+            else
+              components = build_components(configuration)
+              components.telemetry.started!
+              components
+            end
+          )
+        end
+
+        configuration
+      end
+
+      # Apply configuration changes only to a specific Ruby object.
+      #
+      # Certain integrations or Datadog features may use these
+      # settings to customize behavior for this object.
+      #
+      # An example of a {.configure_onto} call:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # ```
+      #
+      # In this example, it will configure the `client` object with custom options
+      # `service_name: 'api-requests', split_by_domain: true`. The `Net::HTTP` integration
+      # will then use these customized options when the `client` is used, whereas other
+      # clients will use the `service_name: 'http-requests'` configuration provided to the
+      # `Datadog.configure` call block.
+      #
+      # {.configure_onto} is used to separate cases where spans generated by certain objects
+      # require exceptional options.
+      #
+      # The configuration keyword arguments provided should match well known options defined
+      # in the integration or feature that would use them.
+      #
+      # For example, for `Datadog.configure_onto(redis_client, **opts)`, `opts` can be
+      # any of the options in the Redis {Datadog::Tracing::Contrib::Redis::Configuration::Settings} class.
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Hash] opts keyword arguments respective to the integration this object belongs to
+      # @public_api
+      def configure_onto(target, **opts)
+        Pin.set_on(target, **opts)
+      end
+
+      # Get configuration changes applied only to a specific Ruby object, via {.configure_onto}.
+      # An example of an object with specific configuration:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # config = Datadog.configuration_for(client)
+      # config[:service_name] # => 'api-requests'
+      # config[:split_by_domain] # => true
+      # ```
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Object] option an option to retrieve from the object configuration
+      # @public_api
+      def configuration_for(target, option = nil)
+        pin = Pin.get_from(target)
+        return pin unless option
+
+        pin[option] if pin
+      end
+
+      # Internal {Datadog::Statsd} metrics collection.
+      #
+      # The list of metrics collected can be found in {Datadog::Core::Diagnostics::Ext::Health::Metrics}.
+      # @public_api
+      def health_metrics
+        components.health_metrics
+      end
+
+      def logger
+        # avoid initializing components if they didn't already exist
+        current_components = components(allow_initialization: false)
+
+        if current_components
+          @temp_logger = nil
+          current_components.logger
+        else
+          logger_without_components
+        end
+      end
+
+      # Gracefully shuts down all components.
+      #
+      # Components will still respond to method calls as usual,
+      # but might not internally perform their work after shutdown.
+      #
+      # This avoids errors being raised across the host application
+      # during shutdown, while allowing for graceful decommission of resources.
+      #
+      # Components won't be automatically reinitialized after a shutdown.
+      def shutdown!
+        safely_synchronize do
+          @components.shutdown! if components?
+        end
+      end
+
+      protected
+
+      def components(allow_initialization: true)
+        current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+        return current_components if current_components || !allow_initialization
+
+        safely_synchronize do |write_components|
+          (defined?(@components) && @components) || write_components.call(build_components(configuration))
+        end
+      end
+
+      private
+
+      # Gracefully shuts down Datadog components and disposes of component references,
+      # allowing execution to start anew.
+      #
+      # In contrast with +#shutdown!+, components will be automatically
+      # reinitialized after a reset.
+      #
+      # Used internally to ensure a clean environment between test runs.
+      def reset!
+        safely_synchronize do |write_components|
+          @components.shutdown! if components?
+          write_components.call(nil)
+          configuration.reset!
+        end
+      end
+
+      def safely_synchronize
+        # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+        # safely_synchronize, this forces all writers to go through this method.
+        write_components = proc do |new_value|
+          COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+        end
+
+        COMPONENTS_WRITE_LOCK.synchronize do
+          begin
+            yield write_components
+          rescue ThreadError => e
+            logger_without_components.error(
+              'Detected deadlock during ddtrace initialization. ' \
+              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
+            )
+            nil
+          end
+        end
+      end
+
+      def components?
+        # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+        (defined?(@components) && @components) != nil
+      end
+
+      def build_components(settings)
+        components = Components.new(settings)
+        components.startup!(settings)
+        components
+      end
+
+      def replace_components!(settings, old)
+        components = Components.new(settings)
+
+        old.shutdown!(components)
+        components.startup!(settings)
+        components
+      end
+
+      def logger_without_components
+        # Use default logger without initializing components.
+        # This enables logging during initialization, otherwise we'd run into deadlocks.
+        @temp_logger ||= begin
+          logger = configuration.logger.instance || Core::Logger.new($stdout)
+          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+          logger
+        end
+      end
+
+      # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
+      # to print a nice message whenever we're taking a bit longer than usual to finish the process.
+      def handle_interrupt_shutdown!
+        logger = Datadog.logger
+        shutdown_thread = Thread.new { shutdown! }
+        print_message_treshold_seconds = 0.2
+
+        slow_shutdown = shutdown_thread.join(print_message_treshold_seconds).nil?
+
+        if slow_shutdown
+          logger.info 'Reporting remaining data... Press ctrl+c to exit immediately.'
+          shutdown_thread.join
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -0,0 +1,286 @@
+require 'date'
+require 'json'
+require 'rbconfig'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # A holistic collection of the environment in which ddtrace is running.
+      # This logger should allow for easy reporting by users to Datadog support.
+      #
+      module EnvironmentLogger
+        class << self
+          # Outputs environment information to {Datadog.logger}.
+          # Executes only once for the lifetime of the program.
+          def log!(transport_responses)
+            return if (defined?(@executed) && @executed) || !log?
+
+            @executed = true
+
+            data = EnvironmentCollector.new.collect!(transport_responses)
+            data.reject! { |_, v| v.nil? } # Remove empty values from hash output
+
+            log_environment!(data.to_json)
+            log_error!('Agent Error'.freeze, data[:agent_error]) if data[:agent_error]
+          rescue => e
+            Datadog.logger.warn("Failed to collect environment information: #{e} Location: #{Array(e.backtrace).first}")
+          end
+
+          private
+
+          def log_environment!(line)
+            Datadog.logger.info("DATADOG CONFIGURATION - #{line}")
+          end
+
+          def log_error!(type, error)
+            Datadog.logger.warn("DATADOG DIAGNOSTIC - #{type}: #{error}")
+          end
+
+          # Are we logging the environment data?
+          def log?
+            startup_logs_enabled = Datadog.configuration.diagnostics.startup_logs.enabled
+            if startup_logs_enabled.nil?
+              !repl? # Suppress logs if we running in a REPL
+            else
+              startup_logs_enabled
+            end
+          end
+
+          REPL_PROGRAM_NAMES = %w[irb pry].freeze
+
+          def repl?
+            REPL_PROGRAM_NAMES.include?($PROGRAM_NAME)
+          end
+        end
+      end
+
+      # Collects environment information for diagnostic logging
+      class EnvironmentCollector
+        # @return [String] current time in ISO8601 format
+        def date
+          DateTime.now.iso8601
+        end
+
+        # Best portable guess of OS information.
+        # @return [String] platform string
+        def os_name
+          RbConfig::CONFIG['host'.freeze]
+        end
+
+        # @return [String] ddtrace version
+        def version
+          DDTrace::VERSION::STRING
+        end
+
+        # @return [String] "ruby"
+        def lang
+          Core::Environment::Ext::LANG
+        end
+
+        # Supported Ruby language version.
+        # Will be distinct from VM version for non-MRI environments.
+        # @return [String]
+        def lang_version
+          Core::Environment::Ext::LANG_VERSION
+        end
+
+        # @return [String] configured application environment
+        def env
+          Datadog.configuration.env
+        end
+
+        # @return [Boolean, nil]
+        def enabled
+          Datadog.configuration.tracing.enabled
+        end
+
+        # @return [String] configured application service name
+        def service
+          Datadog.configuration.service
+        end
+
+        # @return [String] configured application version
+        def dd_version
+          Datadog.configuration.version
+        end
+
+        # @return [String, nil] target agent URL for trace flushing
+        def agent_url
+          # Retrieve the effect agent URL, regardless of how it was configured
+          transport = Tracing.send(:tracer).writer.transport
+
+          # return `nil` with IO transport
+          return unless transport.respond_to?(:client)
+
+          adapter = transport.client.api.adapter
+          adapter.url
+        end
+
+        # Error returned by Datadog agent during a tracer flush attempt
+        # @return [String] concatenated list of transport errors
+        def agent_error(transport_responses)
+          error_responses = transport_responses.reject(&:ok?)
+
+          return nil if error_responses.empty?
+
+          error_responses.map(&:inspect).join(','.freeze)
+        end
+
+        # @return [Boolean, nil] debug mode enabled in configuration
+        def debug
+          !!Datadog.configuration.diagnostics.debug
+        end
+
+        # @return [Boolean, nil] analytics enabled in configuration
+        def analytics_enabled
+          !!Datadog.configuration.tracing.analytics.enabled
+        end
+
+        # @return [Numeric, nil] tracer sample rate configured
+        def sample_rate
+          sampler = Datadog.configuration.tracing.sampler
+          return nil unless sampler
+
+          sampler.sample_rate(nil) rescue nil
+        end
+
+        # DEV: We currently only support SimpleRule instances.
+        # DEV: These are the most commonly used rules.
+        # DEV: We should expand support for other rules in the future,
+        # DEV: although it is tricky to serialize arbitrary rules.
+        #
+        # @return [Hash, nil] sample rules configured
+        def sampling_rules
+          sampler = Datadog.configuration.tracing.sampler
+          return nil unless sampler.is_a?(Tracing::Sampling::PrioritySampler) &&
+            sampler.priority_sampler.is_a?(Tracing::Sampling::RuleSampler)
+
+          sampler.priority_sampler.rules.map do |rule|
+            next unless rule.is_a?(Tracing::Sampling::SimpleRule)
+
+            {
+              name: rule.matcher.name,
+              service: rule.matcher.service,
+              sample_rate: rule.sampler.sample_rate(nil)
+            }
+          end.compact
+        end
+
+        # @return [Hash, nil] concatenated list of global tracer tags configured
+        def tags
+          tags = Datadog.configuration.tags
+          return nil if tags.empty?
+
+          hash_serializer(tags)
+        end
+
+        # @return [Boolean, nil] runtime metrics enabled in configuration
+        def runtime_metrics_enabled
+          Datadog.configuration.runtime_metrics.enabled
+        end
+
+        # Concatenated list of integrations activated, with their gem version.
+        # Example: "rails@6.0.3,rack@2.2.3"
+        #
+        # @return [String, nil]
+        def integrations_loaded
+          integrations = instrumented_integrations
+          return if integrations.empty?
+
+          integrations.map { |name, integration| "#{name}@#{integration.class.version}" }.join(','.freeze)
+        end
+
+        # Ruby VM name and version.
+        # Examples: "ruby-2.7.1", "jruby-9.2.11.1", "truffleruby-20.1.0"
+        # @return [String, nil]
+        def vm
+          # RUBY_ENGINE_VERSION returns the VM version, which
+          # will differ from RUBY_VERSION for non-mri VMs.
+          if defined?(RUBY_ENGINE_VERSION)
+            "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}"
+          else
+            # Ruby < 2.3 doesn't support RUBY_ENGINE_VERSION
+            "#{RUBY_ENGINE}-#{RUBY_VERSION}"
+          end
+        end
+
+        # @return [Boolean, nil] partial flushing enabled in configuration
+        def partial_flushing_enabled
+          !!Datadog.configuration.tracing.partial_flush.enabled
+        end
+
+        # @return [Boolean, nil] priority sampling enabled in configuration
+        def priority_sampling_enabled
+          !!Datadog.configuration.tracing.priority_sampling
+        end
+
+        # @return [Boolean, nil] health metrics enabled in configuration
+        def health_metrics_enabled
+          !!Datadog.configuration.diagnostics.health_metrics.enabled
+        end
+
+        def profiling_enabled
+          !!Datadog.configuration.profiling.enabled
+        end
+
+        # TODO: Populate when automatic log correlation is implemented
+        # def logs_correlation_enabled
+        # end
+
+        # @return [Hash] environment information available at call time
+        def collect!(transport_responses)
+          {
+            date: date,
+            os_name: os_name,
+            version: version,
+            lang: lang,
+            lang_version: lang_version,
+            env: env,
+            enabled: enabled,
+            service: service,
+            dd_version: dd_version,
+            agent_url: agent_url,
+            agent_error: agent_error(transport_responses),
+            debug: debug,
+            analytics_enabled: analytics_enabled,
+            sample_rate: sample_rate,
+            sampling_rules: sampling_rules,
+            tags: tags,
+            runtime_metrics_enabled: runtime_metrics_enabled,
+            integrations_loaded: integrations_loaded,
+            vm: vm,
+            partial_flushing_enabled: partial_flushing_enabled,
+            priority_sampling_enabled: priority_sampling_enabled,
+            health_metrics_enabled: health_metrics_enabled,
+            profiling_enabled: profiling_enabled,
+            **instrumented_integrations_settings
+          }
+        end
+
+        private
+
+        def instrumented_integrations
+          Datadog.configuration.tracing.instrumented_integrations
+        end
+
+        # Capture all active integration settings into "integrationName_settingName: value" entries.
+        def instrumented_integrations_settings
+          instrumented_integrations.flat_map do |name, integration|
+            integration.configuration.to_h.flat_map do |setting, value|
+              next [] if setting == :tracer # Skip internal Ruby objects
+
+              # Convert value to a string to avoid custom #to_json
+              # handlers possibly causing errors.
+              [[:"integration_#{name}_#{setting}", value.to_s]]
+            end
+          end.to_h
+        end
+
+        # Outputs "k1:v1,k2:v2,..."
+        def hash_serializer(h)
+          h.map { |k, v| "#{k}:#{v}" }.join(','.freeze)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/health.rb

@@ -0,0 +1,17 @@
+require_relative '../metrics/client'
+require_relative '../../tracing/diagnostics/health'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # Health-related diagnostics
+      module Health
+        # Health metrics for diagnostics
+        class Metrics < Core::Metrics::Client
+          # TODO: Don't reference this. Have tracing add its metrics behavior.
+          extend Tracing::Diagnostics::Health::Metrics
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/encoding.rb

@@ -0,0 +1,72 @@
+require 'json'
+require 'msgpack'
+
+module Datadog
+  module Core
+    # Encoding module that encodes data for the AgentTransport
+    module Encoding
+      # Encoder interface that provides the logic to encode traces and service
+      # @abstract
+      module Encoder
+        def content_type
+          raise NotImplementedError
+        end
+
+        # Concatenates a list of elements previously encoded by +#encode+.
+        def join(encoded_elements)
+          raise NotImplementedError
+        end
+
+        # Serializes a single trace into a String suitable for network transmission.
+        def encode(_)
+          raise NotImplementedError
+        end
+      end
+
+      # Encoder for the JSON format
+      module JSONEncoder
+        extend Encoder
+
+        CONTENT_TYPE = 'application/json'.freeze
+
+        module_function
+
+        def content_type
+          CONTENT_TYPE
+        end
+
+        def encode(obj)
+          JSON.dump(obj)
+        end
+
+        def join(encoded_data)
+          "[#{encoded_data.join(',')}]"
+        end
+      end
+
+      # Encoder for the Msgpack format
+      module MsgpackEncoder
+        extend Encoder
+
+        module_function
+
+        CONTENT_TYPE = 'application/msgpack'.freeze
+
+        def content_type
+          CONTENT_TYPE
+        end
+
+        def encode(obj)
+          MessagePack.pack(obj)
+        end
+
+        def join(encoded_data)
+          packer = MessagePack::Packer.new
+          packer.write_array_header(encoded_data.size)
+
+          (packer.buffer.to_a + encoded_data).join
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/environment/cgroup.rb

@@ -1,5 +1,4 @@
-# typed: true
-require 'datadog/core/environment/ext'
+require_relative 'ext'
 
 module Datadog
   module Core
@@ -9,8 +8,6 @@ module Datadog
       # about the current Linux container identity.
       # @see https://man7.org/linux/man-pages/man7/cgroups.7.html
       module Cgroup
-        include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
-
         LINE_REGEX = /^(\d+):([^:]*):(.+)$/.freeze
 
         Descriptor = Struct.new(
@@ -34,7 +31,9 @@ module Datadog
                 end
               end
             rescue StandardError => e
-              Datadog.logger.error("Error while parsing cgroup. Cause: #{e.message} Location: #{Array(e.backtrace).first}")
+              Datadog.logger.error(
+                "Error while parsing cgroup. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+              )
             end
           end
         end

data/lib/datadog/core/environment/class_count.rb

@@ -1,4 +1,3 @@
-# typed: true
 module Datadog
   module Core
     module Environment