ddtrace 1.0.0 → 1.10.1

data/lib/datadog/profiling/tasks/help.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Profiling
     module Tasks

data/lib/datadog/profiling/tasks/setup.rb

@@ -1,8 +1,5 @@
-# typed: false
-
-require 'datadog/core/utils/only_once'
-require 'datadog/profiling'
-require 'datadog/profiling/ext/forking'
+require_relative '../../core/utils/only_once'
+require_relative '../ext/forking'
 
 module Datadog
   module Profiling
@@ -19,7 +16,8 @@ module Datadog
               setup_at_fork_hooks
             rescue StandardError, ScriptError => e
               Datadog.logger.warn do
-                "Profiler extensions unavailable. Cause: #{e.message} Location: #{Array(e.backtrace).first}"
+                "Profiler extensions unavailable. Cause: #{e.class.name} #{e.message} " \
+                "Location: #{Array(e.backtrace).first}"
               end
             end
           end
@@ -35,7 +33,8 @@ module Datadog
           end
         rescue StandardError, ScriptError => e
           Datadog.logger.warn do
-            "Profiler forking extensions unavailable. Cause: #{e.message} Location: #{Array(e.backtrace).first}"
+            "Profiler forking extensions unavailable. Cause: #{e.class.name} #{e.message} " \
+            "Location: #{Array(e.backtrace).first}"
           end
         end
 
@@ -54,18 +53,12 @@ module Datadog
           if Process.respond_to?(:at_fork)
             Process.at_fork(:child) do
               begin
-                # When Ruby forks, clock IDs for each of the threads
-                # will change. We can only update these IDs from the
-                # execution context of the thread that owns it.
-                # This hook will update the IDs for the main thread
-                # after a fork occurs.
-                Thread.current.send(:update_native_ids) if Thread.current.respond_to?(:update_native_ids, true)
-
                 # Restart profiler, if enabled
                 Profiling.start_if_enabled
               rescue StandardError => e
                 Datadog.logger.warn do
-                  "Error during post-fork hooks. Cause: #{e.message} Location: #{Array(e.backtrace).first}"
+                  "Error during post-fork hooks. Cause: #{e.class.name} #{e.message} " \
+                  "Location: #{Array(e.backtrace).first}"
                 end
               end
             end

data/lib/datadog/profiling/trace_identifiers/ddtrace.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-# typed: true
-
-require 'datadog/tracing'
-require 'datadog/tracing/metadata/ext'
+require_relative '../../tracing'
+require_relative '../../tracing/metadata/ext'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/trace_identifiers/helper.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-# typed: true
-
-require 'datadog/profiling/trace_identifiers/ddtrace'
+require_relative 'ddtrace'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling.rb

@@ -1,18 +1,13 @@
-# typed: true
-
-require 'datadog/core'
-require 'datadog/core/environment/variable_helpers'
-require 'datadog/core/utils/only_once'
+require_relative 'core'
+require_relative 'core/environment/variable_helpers'
+require_relative 'core/utils/only_once'
 
 module Datadog
-  # Contains profiler for generating stack profiles, etc.
-  module Profiling # rubocop:disable Metrics/ModuleLength
+  # Datadog Continuous Profiler implementation: https://docs.datadoghq.com/profiler/
+  module Profiling
     GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
     private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
 
-    SKIPPED_NATIVE_EXTENSION_ONLY_ONCE = Core::Utils::OnlyOnce.new
-    private_constant :SKIPPED_NATIVE_EXTENSION_ONLY_ONCE
-
     def self.supported?
       unsupported_reason.nil?
     end
@@ -21,7 +16,7 @@ module Datadog
       # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons -- e.g. tell users
       # first that they can't use this on JRuby before telling them that they are missing protobuf
 
-      ruby_engine_unsupported? ||
+      native_library_compilation_skipped? ||
         native_library_failed_to_load? ||
         protobuf_gem_unavailable? ||
         protobuf_version_unsupported? ||
@@ -44,15 +39,64 @@ module Datadog
       !!profiler
     end
 
-    private_class_method def self.ruby_engine_unsupported?
-      'JRuby is not supported' if RUBY_ENGINE == 'jruby'
+    # Returns an ever-increasing counter of the number of allocations observed by the profiler in this thread.
+    #
+    # Note 1: This counter may not start from zero on new threads. It should only be used to measure how many
+    # allocations have happened between two calls to this API:
+    # ```
+    # allocations_before = Datadog::Profiling.allocation_count
+    # do_some_work()
+    # allocations_after = Datadog::Profiling.allocation_count
+    # puts "Allocations during do_some_work: #{allocations_after - allocations_before}"
+    # ```
+    # (This is similar to some OS-based time representations.)
+    #
+    # Note 2: All fibers in the same thread will share the same counter values.
+    #
+    # Only available when the profiler is running, the new CPU Profiling 2.0 profiler is in use, and allocation-related
+    # features are not disabled via configuration.
+    # For instructions on enabling CPU Profiling 2.0 see the ddtrace release notes.
+    #
+    # @return [Integer] number of allocations observed in the current thread.
+    # @return [nil] when not available.
+    # @public_api
+    def self.allocation_count
+      # This no-op implementation is used when profiling failed to load.
+      # It gets replaced inside #replace_noop_allocation_count.
+      nil
+    end
+
+    private_class_method def self.replace_noop_allocation_count
+      def self.allocation_count # rubocop:disable Lint/DuplicateMethods, Lint/NestedMethodDefinition (On purpose!)
+        Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
+      end
+    end
+
+    private_class_method def self.native_library_compilation_skipped?
+      skipped_reason = try_reading_skipped_reason_file
+
+      "Your ddtrace installation is missing support for the Continuous Profiler because #{skipped_reason}" if skipped_reason
+    end
+
+    private_class_method def self.try_reading_skipped_reason_file(file_api = File)
+      # This file, if it exists, is recorded by extconf.rb during compilation of the native extension
+      skipped_reason_file = "#{__dir__}/../../ext/ddtrace_profiling_native_extension/skipped_reason.txt"
+
+      begin
+        return unless file_api.exist?(skipped_reason_file)
+
+        contents = file_api.read(skipped_reason_file).strip
+        contents unless contents.empty?
+      rescue StandardError
+        # Do nothing
+      end
     end
 
     private_class_method def self.protobuf_gem_unavailable?
       # NOTE: On environments where protobuf is already loaded, we skip the check. This allows us to support environments
       # where no Gem.loaded_version is NOT available but customers are able to load protobuf; see for instance
       # https://github.com/teamcapybara/capybara/commit/caf3bcd7664f4f2691d0ca9ef3be9a2a954fecfb
-      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].nil?
+      if !protobuf_already_loaded? && Gem.loaded_specs['google-protobuf'].nil?
         "Missing google-protobuf dependency; please add `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
       end
     end
@@ -61,12 +105,16 @@ module Datadog
       # See above for why we skip the check when protobuf is already loaded; note that when protobuf was already loaded
       # we skip the version check to avoid the call to Gem.loaded_specs. Unfortunately, protobuf does not seem to
       # expose the gem version constant elsewhere, so in that setup we are not able to check the version.
-      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
+      if !protobuf_already_loaded? && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
         'Your google-protobuf is too old; ensure that you have google-protobuf >= 3.0 by ' \
         "adding `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
       end
     end
 
+    private_class_method def self.protobuf_already_loaded?
+      defined?(::Google::Protobuf) && !defined?(::Protobuf)
+    end
+
     private_class_method def self.protobuf_failed_to_load?
       unless protobuf_loaded_successfully?
         'There was an error loading the google-protobuf library; see previous warning message for details'
@@ -92,13 +140,12 @@ module Datadog
         # NOTE: We use Kernel#warn here because this code gets run BEFORE Datadog.logger is actually set up.
         # In the future it'd be nice to shuffle the logger startup to happen first to avoid this special case.
         Kernel.warn(
-          '[DDTRACE] Error while loading google-protobuf gem. ' \
-          "Cause: '#{e.message}' Location: '#{Array(e.backtrace).first}'. " \
+          '[ddtrace] Error while loading google-protobuf gem. ' \
+          "Cause: '#{e.class.name} #{e.message}' Location: '#{Array(e.backtrace).first}'. " \
           'This can happen when google-protobuf is missing its native components. ' \
           'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
           '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
-          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
-          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+          'If the error persists, please contact Datadog support at <https://docs.datadoghq.com/help/>.'
         )
         @protobuf_loaded = false
       end
@@ -110,31 +157,18 @@ module Datadog
       unless success
         if exception
           'There was an error loading the profiling native extension due to ' \
-          "'#{exception.message}' at '#{exception.backtrace.first}'"
+          "'#{exception.class.name} #{exception.message}' at '#{exception.backtrace.first}'"
         else
           'The profiling native extension did not load correctly. ' \
-          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
-          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+          'For help solving this issue, please contact Datadog support at <https://docs.datadoghq.com/help/>.' \
         end
       end
     end
 
     private_class_method def self.try_loading_native_library
-      if Core::Environment::VariableHelpers.env_to_bool('DD_PROFILING_NO_EXTENSION', false)
-        SKIPPED_NATIVE_EXTENSION_ONLY_ONCE.run do
-          Kernel.warn(
-            '[DDTRACE] Skipped loading of profiling native extension due to DD_PROFILING_NO_EXTENSION environment ' \
-            'variable being set. ' \
-            'This option is experimental and will lead to the profiler not working in future releases. ' \
-            'If you needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.'
-          )
-        end
-
-        return [true, nil]
-      end
-
       begin
-        require "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+        require_relative 'profiling/load_native_extension'
+
         success =
           defined?(Profiling::NativeExtension) && Profiling::NativeExtension.send(:native_working?)
         [success, nil]
@@ -146,23 +180,31 @@ module Datadog
     private_class_method def self.load_profiling
       return false unless supported?
 
-      require 'datadog/profiling/ext/forking'
-      require 'datadog/profiling/collectors/code_provenance'
-      require 'datadog/profiling/collectors/stack'
-      require 'datadog/profiling/exporter'
-      require 'datadog/profiling/recorder'
-      require 'datadog/profiling/scheduler'
-      require 'datadog/profiling/tasks/setup'
-      require 'datadog/profiling/transport/io'
-      require 'datadog/profiling/transport/http'
-      require 'datadog/profiling/profiler'
-      require 'datadog/profiling/native_extension'
-      require 'datadog/profiling/trace_identifiers/helper'
-      require 'datadog/profiling/pprof/pprof_pb'
+      require_relative 'profiling/ext/forking'
+      require_relative 'profiling/collectors/code_provenance'
+      require_relative 'profiling/collectors/cpu_and_wall_time_worker'
+      require_relative 'profiling/collectors/dynamic_sampling_rate'
+      require_relative 'profiling/collectors/idle_sampling_helper'
+      require_relative 'profiling/collectors/old_stack'
+      require_relative 'profiling/collectors/stack'
+      require_relative 'profiling/collectors/thread_context'
+      require_relative 'profiling/stack_recorder'
+      require_relative 'profiling/old_recorder'
+      require_relative 'profiling/exporter'
+      require_relative 'profiling/scheduler'
+      require_relative 'profiling/tasks/setup'
+      require_relative 'profiling/profiler'
+      require_relative 'profiling/native_extension'
+      require_relative 'profiling/trace_identifiers/helper'
+      require_relative 'profiling/pprof/pprof_pb'
+      require_relative 'profiling/tag_builder'
+      require_relative 'profiling/http_transport'
+
+      replace_noop_allocation_count
 
       true
     end
 
-    load_profiling if supported?
+    load_profiling
   end
 end

data/lib/datadog/tracing/analytics.rb

@@ -1,6 +1,4 @@
-# typed: true
-
-require 'datadog/tracing/metadata/ext'
+require_relative 'metadata/ext'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/buffer.rb

@@ -1,17 +1,13 @@
-# typed: true
-
-require 'datadog/core'
-require 'datadog/core/environment/ext'
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/buffer/cruby'
-require 'datadog/core/diagnostics/health'
+require_relative '../core'
+require_relative '../core/environment/ext'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/buffer/cruby'
+require_relative '../core/diagnostics/health'
 
 module Datadog
   module Tracing
     # Health metrics for trace buffers.
     module MeasuredBuffer
-      include Kernel # Ensure that kernel methods are always available (https://sorbet.org/docs/error-reference#7003)
-
       def initialize(*_)
         super
 
@@ -58,7 +54,9 @@ module Datadog
 
         @buffer_spans += trace.length
       rescue StandardError => e
-        Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+        Datadog.logger.debug(
+          "Failed to measure queue accept. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
       end
 
       def measure_drop(trace)
@@ -66,7 +64,9 @@ module Datadog
 
         @buffer_spans -= trace.length
       rescue StandardError => e
-        Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+        Datadog.logger.debug(
+          "Failed to measure queue drop. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
       end
 
       def measure_pop(traces)
@@ -89,7 +89,9 @@ module Datadog
         @buffer_dropped = 0
         @buffer_spans = 0
       rescue StandardError => e
-        Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{Array(e.backtrace).first}")
+        Datadog.logger.debug(
+          "Failed to measure queue. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
+        )
       end
     end
 

data/lib/datadog/tracing/client_ip.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative '../core/configuration'
+require_relative '../core/utils/network'
+require_relative 'metadata/ext'
+require_relative 'span'
+
+module Datadog
+  module Tracing
+    # Common functions for supporting the `http.client_ip` span attribute.
+    module ClientIp
+      class << self
+        # Sets the `http.client_ip` tag on the given span.
+        #
+        # This function respects the user's settings: if they disable the client IP tagging,
+        #   or provide a different IP header name.
+        #
+        # @param [Span] span The span that's associated with the request.
+        # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+        # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+        def set_client_ip_tag(span, headers: nil, remote_ip: nil)
+          return unless configuration.enabled
+
+          set_client_ip_tag!(span, headers: headers, remote_ip: remote_ip)
+        end
+
+        # Forcefully sets the `http.client_ip` tag on the given span.
+        #
+        # This function ignores the user's `enabled` setting.
+        #
+        # @param [Span] span The span that's associated with the request.
+        # @param [HeaderCollection, #get, nil] headers A collection with the request headers.
+        # @param [String, nil] remote_ip The remote IP the request associated with the span is sent to.
+        def set_client_ip_tag!(span, headers: nil, remote_ip: nil)
+          ip = extract_client_ip(headers, remote_ip)
+
+          span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP, ip) if ip
+        end
+
+        def extract_client_ip(headers, remote_ip)
+          if headers && configuration.header_name
+            return Datadog::Core::Utils::Network.stripped_ip_from_request_headers(
+              headers,
+              ip_headers_to_check: Array(configuration.header_name)
+            )
+          end
+
+          ip_from_headers = Datadog::Core::Utils::Network.stripped_ip_from_request_headers(headers) if headers
+
+          ip_from_headers || Datadog::Core::Utils::Network.stripped_ip(remote_ip)
+        end
+
+        private
+
+        def configuration
+          Datadog.configuration.tracing.client_ip
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/component.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative 'tracer'
+require_relative 'flush'
+require_relative 'sync_writer'
+require_relative 'sampling/span/rule_parser'
+require_relative 'sampling/span/sampler'
+
+module Datadog
+  module Tracing
+    # Tracing component
+    module Component
+      def build_tracer(settings, agent_settings)
+        # If a custom tracer has been provided, use it instead.
+        # Ignore all other options (they should already be configured.)
+        tracer = settings.tracing.instance
+        return tracer unless tracer.nil?
+
+        # Apply test mode settings if test mode is activated
+        if settings.tracing.test_mode.enabled
+          trace_flush = build_test_mode_trace_flush(settings)
+          sampler = build_test_mode_sampler
+          writer = build_test_mode_writer(settings, agent_settings)
+        else
+          trace_flush = build_trace_flush(settings)
+          sampler = build_sampler(settings)
+          writer = build_writer(settings, agent_settings)
+        end
+
+        subscribe_to_writer_events!(writer, sampler, settings.tracing.test_mode.enabled)
+
+        Tracing::Tracer.new(
+          default_service: settings.service,
+          enabled: settings.tracing.enabled,
+          trace_flush: trace_flush,
+          sampler: sampler,
+          span_sampler: build_span_sampler(settings),
+          writer: writer,
+          tags: build_tracer_tags(settings),
+        )
+      end
+
+      def build_trace_flush(settings)
+        if settings.tracing.partial_flush.enabled
+          Tracing::Flush::Partial.new(
+            min_spans_before_partial_flush: settings.tracing.partial_flush.min_spans_threshold
+          )
+        else
+          Tracing::Flush::Finished.new
+        end
+      end
+
+      # TODO: Sampler should be a top-level component.
+      # It is currently part of the Tracer initialization
+      # process, but can take a variety of options (including
+      # a fully custom instance) that makes the Tracer
+      # initialization process complex.
+      def build_sampler(settings)
+        if (sampler = settings.tracing.sampler)
+          if settings.tracing.priority_sampling == false
+            sampler
+          else
+            ensure_priority_sampling(sampler, settings)
+          end
+        elsif settings.tracing.priority_sampling == false
+          Tracing::Sampling::RuleSampler.new(
+            rate_limit: settings.tracing.sampling.rate_limit,
+            default_sample_rate: settings.tracing.sampling.default_rate
+          )
+        else
+          Tracing::Sampling::PrioritySampler.new(
+            base_sampler: Tracing::Sampling::AllSampler.new,
+            post_sampler: Tracing::Sampling::RuleSampler.new(
+              rate_limit: settings.tracing.sampling.rate_limit,
+              default_sample_rate: settings.tracing.sampling.default_rate
+            )
+          )
+        end
+      end
+
+      def ensure_priority_sampling(sampler, settings)
+        if sampler.is_a?(Tracing::Sampling::PrioritySampler)
+          sampler
+        else
+          Tracing::Sampling::PrioritySampler.new(
+            base_sampler: sampler,
+            post_sampler: Tracing::Sampling::RuleSampler.new(
+              rate_limit: settings.tracing.sampling.rate_limit,
+              default_sample_rate: settings.tracing.sampling.default_rate
+            )
+          )
+        end
+      end
+
+      # TODO: Writer should be a top-level component.
+      # It is currently part of the Tracer initialization
+      # process, but can take a variety of options (including
+      # a fully custom instance) that makes the Tracer
+      # initialization process complex.
+      def build_writer(settings, agent_settings)
+        if (writer = settings.tracing.writer)
+          return writer
+        end
+
+        Tracing::Writer.new(agent_settings: agent_settings, **settings.tracing.writer_options)
+      end
+
+      def subscribe_to_writer_events!(writer, sampler, test_mode)
+        return unless writer.respond_to?(:events) # Check if it's a custom, external writer
+
+        writer.events.after_send.subscribe(&WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK)
+
+        return unless sampler.is_a?(Tracing::Sampling::PrioritySampler)
+
+        # DEV: We need to ignore priority sampling updates coming from the agent in test mode
+        # because test mode wants to *unconditionally* sample all traces.
+        #
+        # This can cause trace metrics to be overestimated, but that's a trade-off we take
+        # here to achieve 100% sampling rate.
+        return if test_mode
+
+        writer.events.after_send.subscribe(&writer_update_priority_sampler_rates_callback(sampler))
+      end
+
+      WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK = lambda do |_, responses|
+        Core::Diagnostics::EnvironmentLogger.log!(responses)
+      end
+
+      # Create new lambda for writer callback,
+      # capture the current sampler in the callback closure.
+      def writer_update_priority_sampler_rates_callback(sampler)
+        lambda do |_, responses|
+          response = responses.last
+
+          next unless response && !response.internal_error? && response.service_rates
+
+          sampler.update(response.service_rates, decision: Tracing::Sampling::Ext::Decision::AGENT_RATE)
+        end
+      end
+
+      def build_span_sampler(settings)
+        rules = Tracing::Sampling::Span::RuleParser.parse_json(settings.tracing.sampling.span_rules)
+        Tracing::Sampling::Span::Sampler.new(rules || [])
+      end
+
+      private
+
+      def build_tracer_tags(settings)
+        settings.tags.dup.tap do |tags|
+          tags[Core::Environment::Ext::TAG_ENV] = settings.env unless settings.env.nil?
+          tags[Core::Environment::Ext::TAG_VERSION] = settings.version unless settings.version.nil?
+        end
+      end
+
+      def build_test_mode_trace_flush(settings)
+        # If context flush behavior is provided, use it instead.
+        settings.tracing.test_mode.trace_flush || build_trace_flush(settings)
+      end
+
+      def build_test_mode_sampler
+        # Do not sample any spans for tests; all must be preserved.
+        # Set priority sampler to ensure the agent doesn't drop any traces.
+        Tracing::Sampling::PrioritySampler.new(
+          base_sampler: Tracing::Sampling::AllSampler.new,
+          post_sampler: Tracing::Sampling::AllSampler.new
+        )
+      end
+
+      def build_test_mode_writer(settings, agent_settings)
+        # Flush traces synchronously, to guarantee they are written.
+        writer_options = settings.tracing.test_mode.writer_options || {}
+        Tracing::SyncWriter.new(agent_settings: agent_settings, **writer_options)
+      end
+    end
+  end
+end