ddtrace 0.54.2 → 1.4.0

data/lib/datadog/core/configuration.rb

@@ -0,0 +1,289 @@
+# typed: true
+
+require_relative 'configuration/components'
+require_relative 'configuration/settings'
+require_relative 'telemetry/emitter'
+require_relative 'logger'
+require_relative 'pin'
+
+module Datadog
+  module Core
+    # Configuration provides a unique access point for configurations
+    module Configuration # rubocop:disable Metrics/ModuleLength
+      include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
+
+      # Used to ensure that @components initialization/reconfiguration is performed one-at-a-time, by a single thread.
+      #
+      # This is important because components can end up being accessed from multiple application threads (for instance on
+      # a threaded webserver), and we don't want their initialization to clash (for instance, starting two profilers...).
+      #
+      # Note that a Mutex **IS NOT** reentrant: the same thread cannot grab the same Mutex more than once.
+      # This means below we are careful not to nest calls to methods that would trigger initialization and grab the lock.
+      #
+      # Every method that directly or indirectly mutates @components should be holding the lock (through
+      # #safely_synchronize) while doing so.
+      COMPONENTS_WRITE_LOCK = Mutex.new
+      private_constant :COMPONENTS_WRITE_LOCK
+
+      # We use a separate lock when reading the @components, so that they continue to be accessible during reconfiguration.
+      # This was needed because we ran into several issues where we still needed to read the old
+      # components while the COMPONENTS_WRITE_LOCK was being held (see https://github.com/DataDog/dd-trace-rb/pull/1387
+      # and https://github.com/DataDog/dd-trace-rb/pull/1373#issuecomment-799593022 ).
+      #
+      # Technically on MRI we could get away without this lock, but on non-MRI Rubies, we may run into issues because
+      # we fall into the "UnsafeDCLFactory" case of https://shipilev.net/blog/2014/safe-public-construction/ .
+      # Specifically, on JRuby reads from the @components do NOT have volatile semantics, and on TruffleRuby they do
+      # BUT just as an implementation detail, see https://github.com/jruby/jruby/wiki/Concurrency-in-jruby#volatility and
+      # https://github.com/DataDog/dd-trace-rb/pull/1329#issuecomment-776750377 .
+      # Concurrency is hard.
+      COMPONENTS_READ_LOCK = Mutex.new
+      private_constant :COMPONENTS_READ_LOCK
+
+      attr_writer :configuration
+
+      # Current Datadog configuration.
+      #
+      # Access to non-global configuration will raise an error.
+      #
+      # To modify the configuration, use {.configure}.
+      #
+      # @return [Datadog::Core::Configuration::Settings]
+      # @!attribute [r] configuration
+      # @public_api
+      def configuration
+        @configuration ||= Settings.new
+      end
+
+      # Apply global configuration changes to `Datadog`. An example of a {.configure} call:
+      #
+      # ```
+      # Datadog.configure do |c|
+      #   c.service = 'my-service'
+      #   c.env = 'staging'
+      #   # c.diagnostics.debug = true # Enables debug output
+      # end
+      # ```
+      #
+      # See {Datadog::Core::Configuration::Settings} for all available options, defaults, and
+      # available environment variables for configuration.
+      #
+      # Only permits access to global configuration settings; others will raise an error.
+      # If you wish to configure a setting for a specific Datadog component (e.g. Tracing),
+      # use the corresponding `Datadog::COMPONENT.configure` method instead.
+      #
+      # Because many configuration changes require restarting internal components,
+      # invoking {.configure} is the only safe way to change `Datadog` configuration.
+      #
+      # Successive calls to {.configure} maintain the previous configuration values:
+      # configuration is additive between {.configure} calls.
+      #
+      # The yielded configuration `c` comes pre-populated from environment variables, if
+      # any are applicable.
+      #
+      # @yieldparam [Datadog::Core::Configuration::Settings] c the mutable configuration object
+      def configure
+        configuration = self.configuration
+        yield(configuration)
+
+        safely_synchronize do |write_components|
+          write_components.call(
+            if components?
+              replace_components!(configuration, @components)
+            else
+              components = build_components(configuration)
+              components.telemetry.started!
+              components
+            end
+          )
+        end
+
+        configuration
+      end
+
+      # Apply configuration changes only to a specific Ruby object.
+      #
+      # Certain integrations or Datadog features may use these
+      # settings to customize behavior for this object.
+      #
+      # An example of a {.configure_onto} call:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # ```
+      #
+      # In this example, it will configure the `client` object with custom options
+      # `service_name: 'api-requests', split_by_domain: true`. The `Net::HTTP` integration
+      # will then use these customized options when the `client` is used, whereas other
+      # clients will use the `service_name: 'http-requests'` configuration provided to the
+      # `Datadog.configure` call block.
+      #
+      # {.configure_onto} is used to separate cases where spans generated by certain objects
+      # require exceptional options.
+      #
+      # The configuration keyword arguments provided should match well known options defined
+      # in the integration or feature that would use them.
+      #
+      # For example, for `Datadog.configure_onto(redis_client, **opts)`, `opts` can be
+      # any of the options in the Redis {Datadog::Tracing::Contrib::Redis::Configuration::Settings} class.
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Hash] opts keyword arguments respective to the integration this object belongs to
+      # @public_api
+      def configure_onto(target, **opts)
+        Pin.set_on(target, **opts)
+      end
+
+      # Get configuration changes applied only to a specific Ruby object, via {.configure_onto}.
+      # An example of an object with specific configuration:
+      #
+      # ```
+      # client = Net::HTTP.new(host, port)
+      # Datadog.configure_onto(client, service_name: 'api-requests', split_by_domain: true)
+      # config = Datadog.configuration_for(client)
+      # config[:service_name] # => 'api-requests'
+      # config[:split_by_domain] # => true
+      # ```
+      #
+      # @param [Object] target the object to receive configuration options
+      # @param [Object] option an option to retrieve from the object configuration
+      # @public_api
+      def configuration_for(target, option = nil)
+        pin = Pin.get_from(target)
+        return pin unless option
+
+        pin[option] if pin
+      end
+
+      # Internal {Datadog::Statsd} metrics collection.
+      #
+      # The list of metrics collected can be found in {Datadog::Core::Diagnostics::Ext::Health::Metrics}.
+      # @public_api
+      def health_metrics
+        components.health_metrics
+      end
+
+      def logger
+        # avoid initializing components if they didn't already exist
+        current_components = components(allow_initialization: false)
+
+        if current_components
+          @temp_logger = nil
+          current_components.logger
+        else
+          logger_without_components
+        end
+      end
+
+      # Gracefully shuts down all components.
+      #
+      # Components will still respond to method calls as usual,
+      # but might not internally perform their work after shutdown.
+      #
+      # This avoids errors being raised across the host application
+      # during shutdown, while allowing for graceful decommission of resources.
+      #
+      # Components won't be automatically reinitialized after a shutdown.
+      def shutdown!
+        safely_synchronize do
+          @components.shutdown! if components?
+        end
+      end
+
+      protected
+
+      def components(allow_initialization: true)
+        current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
+        return current_components if current_components || !allow_initialization
+
+        safely_synchronize do |write_components|
+          (defined?(@components) && @components) || write_components.call(build_components(configuration))
+        end
+      end
+
+      private
+
+      # Gracefully shuts down Datadog components and disposes of component references,
+      # allowing execution to start anew.
+      #
+      # In contrast with +#shutdown!+, components will be automatically
+      # reinitialized after a reset.
+      #
+      # Used internally to ensure a clean environment between test runs.
+      def reset!
+        safely_synchronize do |write_components|
+          @components.shutdown! if components?
+          write_components.call(nil)
+          configuration.reset!
+        end
+      end
+
+      def safely_synchronize
+        # Writes to @components should only happen through this proc. Because this proc is only accessible to callers of
+        # safely_synchronize, this forces all writers to go through this method.
+        write_components = proc do |new_value|
+          COMPONENTS_READ_LOCK.synchronize { @components = new_value }
+        end
+
+        COMPONENTS_WRITE_LOCK.synchronize do
+          begin
+            yield write_components
+          rescue ThreadError => e
+            logger_without_components.error(
+              'Detected deadlock during ddtrace initialization. ' \
+              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+              "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
+            )
+            nil
+          end
+        end
+      end
+
+      def components?
+        # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
+        (defined?(@components) && @components) != nil
+      end
+
+      def build_components(settings)
+        components = Components.new(settings)
+        components.startup!(settings)
+        components
+      end
+
+      def replace_components!(settings, old)
+        components = Components.new(settings)
+
+        old.shutdown!(components)
+        components.startup!(settings)
+        components
+      end
+
+      def logger_without_components
+        # Use default logger without initializing components.
+        # This enables logging during initialization, otherwise we'd run into deadlocks.
+        @temp_logger ||= begin
+          logger = configuration.logger.instance || Core::Logger.new($stdout)
+          logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
+          logger
+        end
+      end
+
+      # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
+      # to print a nice message whenever we're taking a bit longer than usual to finish the process.
+      def handle_interrupt_shutdown!
+        logger = Datadog.logger
+        shutdown_thread = Thread.new { shutdown! }
+        print_message_treshold_seconds = 0.2
+
+        slow_shutdown = shutdown_thread.join(print_message_treshold_seconds).nil?
+
+        if slow_shutdown
+          logger.info 'Reporting remaining data... Press ctrl+c to exit immediately.'
+          shutdown_thread.join
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -0,0 +1,283 @@
+# typed: true
+
+require 'date'
+require 'json'
+require 'rbconfig'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # A holistic collection of the environment in which ddtrace is running.
+      # This logger should allow for easy reporting by users to Datadog support.
+      #
+      module EnvironmentLogger
+        class << self
+          # Outputs environment information to {Datadog.logger}.
+          # Executes only once for the lifetime of the program.
+          def log!(transport_responses)
+            return if (defined?(@executed) && @executed) || !log?
+
+            @executed = true
+
+            data = EnvironmentCollector.new.collect!(transport_responses)
+            data.reject! { |_, v| v.nil? } # Remove empty values from hash output
+
+            log_environment!(data.to_json)
+            log_error!('Agent Error'.freeze, data[:agent_error]) if data[:agent_error]
+          rescue => e
+            Datadog.logger.warn("Failed to collect environment information: #{e} Location: #{Array(e.backtrace).first}")
+          end
+
+          private
+
+          def log_environment!(line)
+            Datadog.logger.info("DATADOG CONFIGURATION - #{line}")
+          end
+
+          def log_error!(type, error)
+            Datadog.logger.warn("DATADOG DIAGNOSTIC - #{type}: #{error}")
+          end
+
+          # Are we logging the environment data?
+          def log?
+            startup_logs_enabled = Datadog.configuration.diagnostics.startup_logs.enabled
+            if startup_logs_enabled.nil?
+              !repl? # Suppress logs if we running in a REPL
+            else
+              startup_logs_enabled
+            end
+          end
+
+          REPL_PROGRAM_NAMES = %w[irb pry].freeze
+
+          def repl?
+            REPL_PROGRAM_NAMES.include?($PROGRAM_NAME)
+          end
+        end
+      end
+
+      # Collects environment information for diagnostic logging
+      class EnvironmentCollector
+        # @return [String] current time in ISO8601 format
+        def date
+          DateTime.now.iso8601
+        end
+
+        # Best portable guess of OS information.
+        # @return [String] platform string
+        def os_name
+          RbConfig::CONFIG['host'.freeze]
+        end
+
+        # @return [String] ddtrace version
+        def version
+          DDTrace::VERSION::STRING
+        end
+
+        # @return [String] "ruby"
+        def lang
+          Core::Environment::Ext::LANG
+        end
+
+        # Supported Ruby language version.
+        # Will be distinct from VM version for non-MRI environments.
+        # @return [String]
+        def lang_version
+          Core::Environment::Ext::LANG_VERSION
+        end
+
+        # @return [String] configured application environment
+        def env
+          Datadog.configuration.env
+        end
+
+        # @return [Boolean, nil]
+        def enabled
+          Datadog.configuration.tracing.enabled
+        end
+
+        # @return [String] configured application service name
+        def service
+          Datadog.configuration.service
+        end
+
+        # @return [String] configured application version
+        def dd_version
+          Datadog.configuration.version
+        end
+
+        # @return [String] target agent URL for trace flushing
+        def agent_url
+          # Retrieve the effect agent URL, regardless of how it was configured
+          transport = Tracing.send(:tracer).writer.transport
+          adapter = transport.client.api.adapter
+          adapter.url
+        end
+
+        # Error returned by Datadog agent during a tracer flush attempt
+        # @return [String] concatenated list of transport errors
+        def agent_error(transport_responses)
+          error_responses = transport_responses.reject(&:ok?)
+
+          return nil if error_responses.empty?
+
+          error_responses.map(&:inspect).join(','.freeze)
+        end
+
+        # @return [Boolean, nil] debug mode enabled in configuration
+        def debug
+          !!Datadog.configuration.diagnostics.debug
+        end
+
+        # @return [Boolean, nil] analytics enabled in configuration
+        def analytics_enabled
+          !!Datadog.configuration.tracing.analytics.enabled
+        end
+
+        # @return [Numeric, nil] tracer sample rate configured
+        def sample_rate
+          sampler = Datadog.configuration.tracing.sampler
+          return nil unless sampler
+
+          sampler.sample_rate(nil) rescue nil
+        end
+
+        # DEV: We currently only support SimpleRule instances.
+        # DEV: These are the most commonly used rules.
+        # DEV: We should expand support for other rules in the future,
+        # DEV: although it is tricky to serialize arbitrary rules.
+        #
+        # @return [Hash, nil] sample rules configured
+        def sampling_rules
+          sampler = Datadog.configuration.tracing.sampler
+          return nil unless sampler.is_a?(Tracing::Sampling::PrioritySampler) &&
+            sampler.priority_sampler.is_a?(Tracing::Sampling::RuleSampler)
+
+          sampler.priority_sampler.rules.map do |rule|
+            next unless rule.is_a?(Tracing::Sampling::SimpleRule)
+
+            {
+              name: rule.matcher.name,
+              service: rule.matcher.service,
+              sample_rate: rule.sampler.sample_rate(nil)
+            }
+          end.compact
+        end
+
+        # @return [Hash, nil] concatenated list of global tracer tags configured
+        def tags
+          tags = Datadog.configuration.tags
+          return nil if tags.empty?
+
+          hash_serializer(tags)
+        end
+
+        # @return [Boolean, nil] runtime metrics enabled in configuration
+        def runtime_metrics_enabled
+          Datadog.configuration.runtime_metrics.enabled
+        end
+
+        # Concatenated list of integrations activated, with their gem version.
+        # Example: "rails@6.0.3,rack@2.2.3"
+        #
+        # @return [String, nil]
+        def integrations_loaded
+          integrations = instrumented_integrations
+          return if integrations.empty?
+
+          integrations.map { |name, integration| "#{name}@#{integration.class.version}" }.join(','.freeze)
+        end
+
+        # Ruby VM name and version.
+        # Examples: "ruby-2.7.1", "jruby-9.2.11.1", "truffleruby-20.1.0"
+        # @return [String, nil]
+        def vm
+          # RUBY_ENGINE_VERSION returns the VM version, which
+          # will differ from RUBY_VERSION for non-mri VMs.
+          if defined?(RUBY_ENGINE_VERSION)
+            "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}"
+          else
+            # Ruby < 2.3 doesn't support RUBY_ENGINE_VERSION
+            "#{RUBY_ENGINE}-#{RUBY_VERSION}"
+          end
+        end
+
+        # @return [Boolean, nil] partial flushing enabled in configuration
+        def partial_flushing_enabled
+          !!Datadog.configuration.tracing.partial_flush.enabled
+        end
+
+        # @return [Boolean, nil] priority sampling enabled in configuration
+        def priority_sampling_enabled
+          !!Datadog.configuration.tracing.priority_sampling
+        end
+
+        # @return [Boolean, nil] health metrics enabled in configuration
+        def health_metrics_enabled
+          !!Datadog.configuration.diagnostics.health_metrics.enabled
+        end
+
+        # TODO: Populate when profiling is implemented
+        # def profiling_enabled
+        # end
+
+        # TODO: Populate when automatic log correlation is implemented
+        # def logs_correlation_enabled
+        # end
+
+        # @return [Hash] environment information available at call time
+        def collect!(transport_responses)
+          {
+            date: date,
+            os_name: os_name,
+            version: version,
+            lang: lang,
+            lang_version: lang_version,
+            env: env,
+            enabled: enabled,
+            service: service,
+            dd_version: dd_version,
+            agent_url: agent_url,
+            agent_error: agent_error(transport_responses),
+            debug: debug,
+            analytics_enabled: analytics_enabled,
+            sample_rate: sample_rate,
+            sampling_rules: sampling_rules,
+            tags: tags,
+            runtime_metrics_enabled: runtime_metrics_enabled,
+            integrations_loaded: integrations_loaded,
+            vm: vm,
+            partial_flushing_enabled: partial_flushing_enabled,
+            priority_sampling_enabled: priority_sampling_enabled,
+            health_metrics_enabled: health_metrics_enabled,
+            **instrumented_integrations_settings
+          }
+        end
+
+        private
+
+        def instrumented_integrations
+          Datadog.configuration.tracing.instrumented_integrations
+        end
+
+        # Capture all active integration settings into "integrationName_settingName: value" entries.
+        def instrumented_integrations_settings
+          instrumented_integrations.flat_map do |name, integration|
+            integration.configuration.to_h.flat_map do |setting, value|
+              next [] if setting == :tracer # Skip internal Ruby objects
+
+              # Convert value to a string to avoid custom #to_json
+              # handlers possibly causing errors.
+              [[:"integration_#{name}_#{setting}", value.to_s]]
+            end
+          end.to_h
+        end
+
+        # Outputs "k1:v1,k2:v2,..."
+        def hash_serializer(h)
+          h.map { |k, v| "#{k}:#{v}" }.join(','.freeze)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/ext.rb

@@ -0,0 +1,41 @@
+# typed: true
+
+module Datadog
+  module Core
+    module Diagnostics
+      # @public_api
+      module Ext
+        DD_TRACE_STARTUP_LOGS = 'DD_TRACE_STARTUP_LOGS'.freeze
+        DD_TRACE_DEBUG = 'DD_TRACE_DEBUG'.freeze
+        DD_TRACE_ENABLED = 'DD_TRACE_ENABLED'.freeze
+        # Health
+        module Health
+          # Metrics
+          module Metrics
+            ENV_ENABLED = 'DD_HEALTH_METRICS_ENABLED'.freeze
+
+            METRIC_API_ERRORS = 'datadog.tracer.api.errors'.freeze
+            METRIC_API_REQUESTS = 'datadog.tracer.api.requests'.freeze
+            METRIC_API_RESPONSES = 'datadog.tracer.api.responses'.freeze
+            METRIC_ERROR_CONTEXT_OVERFLOW = 'datadog.tracer.error.context_overflow'.freeze
+            METRIC_ERROR_INSTRUMENTATION_PATCH = 'datadog.tracer.error.instrumentation_patch'.freeze
+            METRIC_ERROR_SPAN_FINISH = 'datadog.tracer.error.span_finish'.freeze
+            METRIC_ERROR_UNFINISHED_SPANS = 'datadog.tracer.error.unfinished_spans'.freeze
+            METRIC_INSTRUMENTATION_PATCHED = 'datadog.tracer.instrumentation_patched'.freeze
+            METRIC_QUEUE_ACCEPTED = 'datadog.tracer.queue.accepted'.freeze
+            METRIC_QUEUE_ACCEPTED_LENGTHS = 'datadog.tracer.queue.accepted_lengths'.freeze
+            METRIC_QUEUE_DROPPED = 'datadog.tracer.queue.dropped'.freeze
+            METRIC_QUEUE_LENGTH = 'datadog.tracer.queue.length'.freeze
+            METRIC_QUEUE_MAX_LENGTH = 'datadog.tracer.queue.max_length'.freeze
+            METRIC_QUEUE_SPANS = 'datadog.tracer.queue.spans'.freeze
+            METRIC_SAMPLING_SERVICE_CACHE_LENGTH = 'datadog.tracer.sampling.service_cache_length'.freeze
+            METRIC_TRACES_FILTERED = 'datadog.tracer.traces.filtered'.freeze
+            METRIC_TRANSPORT_CHUNKED = 'datadog.tracer.transport.chunked'.freeze
+            METRIC_TRANSPORT_TRACE_TOO_LARGE = 'datadog.tracer.transport.trace_too_large'.freeze
+            METRIC_WRITER_CPU_TIME = 'datadog.tracer.writer.cpu_time'.freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/diagnostics/health.rb

@@ -0,0 +1,37 @@
+# typed: strict
+
+require_relative 'ext'
+require_relative '../metrics/client'
+
+module Datadog
+  module Core
+    module Diagnostics
+      # Health-related diagnostics
+      module Health
+        # Health metrics for diagnostics
+        class Metrics < Core::Metrics::Client
+          count :api_errors, Ext::Health::Metrics::METRIC_API_ERRORS
+          count :api_requests, Ext::Health::Metrics::METRIC_API_REQUESTS
+          count :api_responses, Ext::Health::Metrics::METRIC_API_RESPONSES
+          count :error_context_overflow, Ext::Health::Metrics::METRIC_ERROR_CONTEXT_OVERFLOW
+          count :error_instrumentation_patch, Ext::Health::Metrics::METRIC_ERROR_INSTRUMENTATION_PATCH
+          count :error_span_finish, Ext::Health::Metrics::METRIC_ERROR_SPAN_FINISH
+          count :error_unfinished_spans, Ext::Health::Metrics::METRIC_ERROR_UNFINISHED_SPANS
+          count :instrumentation_patched, Ext::Health::Metrics::METRIC_INSTRUMENTATION_PATCHED
+          count :queue_accepted, Ext::Health::Metrics::METRIC_QUEUE_ACCEPTED
+          count :queue_accepted_lengths, Ext::Health::Metrics::METRIC_QUEUE_ACCEPTED_LENGTHS
+          count :queue_dropped, Ext::Health::Metrics::METRIC_QUEUE_DROPPED
+          count :traces_filtered, Ext::Health::Metrics::METRIC_TRACES_FILTERED
+          count :transport_trace_too_large, Ext::Health::Metrics::METRIC_TRANSPORT_TRACE_TOO_LARGE
+          count :transport_chunked, Ext::Health::Metrics::METRIC_TRANSPORT_CHUNKED
+          count :writer_cpu_time, Ext::Health::Metrics::METRIC_WRITER_CPU_TIME
+
+          gauge :queue_length, Ext::Health::Metrics::METRIC_QUEUE_LENGTH
+          gauge :queue_max_length, Ext::Health::Metrics::METRIC_QUEUE_MAX_LENGTH
+          gauge :queue_spans, Ext::Health::Metrics::METRIC_QUEUE_SPANS
+          gauge :sampling_service_cache_length, Ext::Health::Metrics::METRIC_SAMPLING_SERVICE_CACHE_LENGTH
+        end
+      end
+    end
+  end
+end