cw-datadog 2.23.0.2

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger the periodic execution of Collectors::ThreadState, which implements all of the sampling logic
+      # itself; this class only implements the "when to do it" part.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time_worker.c`
+      class CpuAndWallTimeWorker
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize(
+          gc_profiling_enabled:,
+          no_signals_workaround_enabled:,
+          thread_context_collector:,
+          dynamic_sampling_rate_overhead_target_percentage:,
+          allocation_profiling_enabled:,
+          allocation_counting_enabled:,
+          gvl_profiling_enabled:,
+          sighandler_sampling_enabled:,
+          # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
+          # profiler overhead!
+          dynamic_sampling_rate_enabled: true,
+          skip_idle_samples_for_testing: false,
+          idle_sampling_helper: IdleSamplingHelper.new
+        )
+          unless dynamic_sampling_rate_enabled
+            Datadog.logger.warn(
+              "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
+            )
+            Datadog::Core::Telemetry::Logger.error(
+              "Profiling dynamic sampling rate disabled. This should only be used for testing, and will increase overhead!"
+            )
+          end
+
+          self.class._native_initialize(
+            self_instance: self,
+            thread_context_collector: thread_context_collector,
+            gc_profiling_enabled: gc_profiling_enabled,
+            idle_sampling_helper: idle_sampling_helper,
+            no_signals_workaround_enabled: no_signals_workaround_enabled,
+            dynamic_sampling_rate_enabled: dynamic_sampling_rate_enabled,
+            dynamic_sampling_rate_overhead_target_percentage: dynamic_sampling_rate_overhead_target_percentage,
+            allocation_profiling_enabled: allocation_profiling_enabled,
+            allocation_counting_enabled: allocation_counting_enabled,
+            gvl_profiling_enabled: gvl_profiling_enabled,
+            sighandler_sampling_enabled: sighandler_sampling_enabled,
+            skip_idle_samples_for_testing: skip_idle_samples_for_testing,
+          )
+          @worker_thread = nil
+          @failure_exception = nil
+          @start_stop_mutex = Mutex.new
+          @idle_sampling_helper = idle_sampling_helper
+          @wait_until_running_mutex = Mutex.new
+          @wait_until_running_condition = ConditionVariable.new
+        end
+
+        def start(on_failure_proc: nil)
+          @start_stop_mutex.synchronize do
+            return if @worker_thread&.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            @idle_sampling_helper.start
+
+            @worker_thread = Thread.new do
+              Thread.current.name = self.class.name
+
+              self.class._native_sampling_loop(self)
+
+              Datadog.logger.debug("CpuAndWallTimeWorker thread stopping cleanly")
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              @failure_exception = e
+              Datadog.logger.warn(
+                "CpuAndWallTimeWorker thread error. " \
+                "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+              )
+              on_failure_proc&.call
+              Datadog::Core::Telemetry::Logger.report(e, description: "CpuAndWallTimeWorker thread error")
+            end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
+            @worker_thread.thread_variable_set(:fork_safe, true)
+          end
+
+          true
+        end
+
+        def stop
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug("Requesting CpuAndWallTimeWorker thread shut down")
+
+            @idle_sampling_helper.stop
+
+            return unless @worker_thread
+
+            self.class._native_stop(self, @worker_thread)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        def stats
+          self.class._native_stats(self)
+        end
+
+        def stats_and_reset_not_thread_safe
+          stats = self.stats
+          self.class._native_stats_reset_not_thread_safe(self)
+          stats
+        end
+
+        # Useful for testing, to e.g. make sure the profiler is running before we start running some code we want to observe
+        def wait_until_running(timeout_seconds: 5)
+          @wait_until_running_mutex.synchronize do
+            return true if self.class._native_is_running?(self)
+
+            @wait_until_running_condition.wait(@wait_until_running_mutex, timeout_seconds)
+
+            if self.class._native_is_running?(self)
+              true
+            else
+              raise "Timeout waiting for #{self.class.name} to start (waited for #{timeout_seconds} seconds)"
+            end
+          end
+        end
+
+        private
+
+        def signal_running
+          @wait_until_running_mutex.synchronize { @wait_until_running_condition.broadcast }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/dynamic_sampling_rate.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to pace the rate of profiling samples based on the last observed time for a sample.
+      # All of this module is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_dynamic_sampling_rate.c`
+      module DynamicSamplingRate
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used by the Collectors::CpuAndWallTimeWorker to gather samples when the Ruby process is idle.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_idle_sampling_helper.c`
+      class IdleSamplingHelper
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize
+          @worker_thread = nil
+          @start_stop_mutex = Mutex.new
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread&.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            # The same instance of the IdleSamplingHelper can be reused multiple times, and this resets it back to
+            # a pristine state before recreating the worker thread
+            self.class._native_reset(self)
+
+            @worker_thread = Thread.new do
+              Thread.current.name = self.class.name
+
+              self.class._native_idle_sampling_loop(self)
+
+              Datadog.logger.debug("IdleSamplingHelper thread stopping cleanly")
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              @failure_exception = e
+              Datadog.logger.warn(
+                "IdleSamplingHelper thread error. " \
+                "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+              )
+              Datadog::Core::Telemetry::Logger.report(e, description: "IdleSamplingHelper thread error")
+            end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
+            @worker_thread.thread_variable_set(:fork_safe, true)
+          end
+
+          true
+        end
+
+        def stop
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug("Requesting IdleSamplingHelper thread shut down")
+
+            return unless @worker_thread
+
+            self.class._native_stop(self)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/info.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "set"
+require "time"
+require "libdatadog"
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Collects information of relevance for profiler. This will get sent alongside
+      # the profile and show up in the UI or potentially influence processing in some way.
+      #
+      # Information is currently collected and frozen at construction time. A full collector
+      # could be seen as overkill for this case but it allows us to centralize information
+      # gathering and easily support more flexible/dynamic info collection in the future.
+      class Info
+        def initialize(settings)
+          @profiler_info = nil
+          @info = {
+            platform: collect_platform_info,
+            runtime: collect_runtime_info,
+            application: collect_application_info(settings),
+            profiler: collect_profiler_info(settings),
+          }.freeze
+        end
+
+        attr_reader :info
+
+        private
+
+        # Ruby GC tuning environment variables
+        RUBY_GC_TUNING_ENV_VARS = [
+          "RUBY_GC_HEAP_FREE_SLOTS",
+          "RUBY_GC_HEAP_GROWTH_FACTOR",
+          "RUBY_GC_HEAP_GROWTH_MAX_SLOTS",
+          "RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO",
+          "RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO",
+          "RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO",
+          "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR",
+          "RUBY_GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO",
+          "RUBY_GC_MALLOC_LIMIT",
+          "RUBY_GC_MALLOC_LIMIT_MAX",
+          "RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR",
+          "RUBY_GC_OLDMALLOC_LIMIT",
+          "RUBY_GC_OLDMALLOC_LIMIT_MAX",
+          "RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR",
+          # INIT_SLOTS changed for Ruby 3.3+:
+          # * https://bugs.ruby-lang.org/issues/19785
+          # * https://www.ruby-lang.org/en/news/2023/12/25/ruby-3-3-0-released/#:~:text=Removed%20environment%20variables
+          "RUBY_GC_HEAP_0_INIT_SLOTS",
+          "RUBY_GC_HEAP_1_INIT_SLOTS",
+          "RUBY_GC_HEAP_2_INIT_SLOTS",
+          "RUBY_GC_HEAP_3_INIT_SLOTS",
+          "RUBY_GC_HEAP_4_INIT_SLOTS",
+          # There was only one setting for older Rubies:
+          "RUBY_GC_HEAP_INIT_SLOTS",
+          # Ruby 2.x only, alias for others:
+          "RUBY_FREE_MIN",
+          "RUBY_HEAP_MIN_SLOTS",
+        ].freeze
+
+        # Instead of trying to figure out real process start time by checking
+        # /proc or some other complex/non-portable way, approximate start time
+        # by time of requirement of this file.
+        #
+        # Note: this does not use Core::Utils::Time.now because this constant
+        # gets initialized before a user has a chance to configure the library.
+        START_TIME = Time.now.utc.freeze
+
+        def collect_platform_info
+          @platform_info ||= {
+            container_id: Datadog::Core::Environment::Container.container_id,
+            hostname: Datadog::Core::Environment::Platform.hostname,
+            kernel_name: Datadog::Core::Environment::Platform.kernel_name,
+            kernel_release: Datadog::Core::Environment::Platform.kernel_release,
+            kernel_version: Datadog::Core::Environment::Platform.kernel_version
+          }.freeze
+        end
+
+        def collect_runtime_info
+          @runtime_info ||= {
+            engine: Datadog::Core::Environment::Identity.lang_engine,
+            version: Datadog::Core::Environment::Identity.lang_version,
+            platform: Datadog::Core::Environment::Identity.lang_platform,
+            gc_tuning: collect_gc_tuning_info,
+          }.freeze
+        end
+
+        def collect_application_info(settings)
+          @application_info ||= {
+            start_time: START_TIME.iso8601,
+            env: settings.env,
+            service: settings.service,
+            version: settings.version,
+          }.freeze
+        end
+
+        def collect_profiler_info(settings)
+          unless @profiler_info
+            lib_datadog_gem = ::Gem.loaded_specs["libdatadog"]
+
+            libdatadog_version =
+              if lib_datadog_gem
+                "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}"
+              else
+                # In some cases, Gem.loaded_specs may not be available, as in
+                # https://github.com/DataDog/dd-trace-rb/pull/1506; let's use the version directly
+                "#{Libdatadog::VERSION}-(unknown)"
+              end
+
+            @profiler_info = {
+              version: Datadog::Core::Environment::Identity.gem_datadog_version,
+              libdatadog: libdatadog_version,
+              settings: collect_settings_recursively(settings.profiling),
+            }.freeze
+          end
+          @profiler_info
+        end
+
+        # The settings/option model isn't directly serializable because
+        # of subsettings and options that link to full blown custom object
+        # instances without proper serialization.
+        # This method navigates a settings object recursively, converting
+        # it into more basic types that are trivially convertible to JSON.
+        def collect_settings_recursively(v)
+          v = v.options_hash if v.respond_to?(:options_hash)
+
+          if v.nil? || v.is_a?(Symbol) || v.is_a?(Numeric) || v.is_a?(String) || v.equal?(true) || v.equal?(false)
+            Core::Utils::SafeDup.frozen_or_dup(v)
+          elsif v.is_a?(Hash)
+            collected_hash = v.each_with_object({}) do |(key, value), hash|
+              collected_value = collect_settings_recursively(value)
+              hash[key] = collected_value
+            end
+            collected_hash.freeze
+          elsif v.is_a?(Enumerable)
+            collected_list = v
+              .map { |value| collect_settings_recursively(value) }
+            collected_list.freeze
+          else
+            v.inspect
+          end
+        end
+
+        def collect_gc_tuning_info
+          return @gc_tuning_info if defined?(@gc_tuning_info)
+
+          @gc_tuning_info = RUBY_GC_TUNING_ENV_VARS.each_with_object({}) do |var, hash|
+            current_value = DATADOG_ENV[var]
+            hash[var.to_sym] = current_value if current_value
+          end.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/stack.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
+      #
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger sampling of threads, based on external "events", such as:
+      # * periodic timer for cpu-time and wall-time
+      # * VM garbage collection events
+      # * VM object allocation events
+      # Triggering of this component (e.g. watching for the above "events") is implemented by
+      # Collectors::CpuAndWallTimeWorker.
+      # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
+      class ThreadContext
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          endpoint_collection_enabled:,
+          timeline_enabled:,
+          waiting_for_gvl_threshold_ns:,
+          otel_context_enabled:,
+          native_filenames_enabled:
+        )
+          tracer_context_key = safely_extract_context_key_from(tracer)
+          self.class._native_initialize(
+            self_instance: self,
+            recorder: recorder,
+            max_frames: max_frames,
+            tracer_context_key: tracer_context_key,
+            endpoint_collection_enabled: endpoint_collection_enabled,
+            timeline_enabled: timeline_enabled,
+            waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
+            otel_context_enabled: otel_context_enabled,
+            native_filenames_enabled: validate_native_filenames(native_filenames_enabled),
+          )
+        end
+
+        def self.for_testing(
+          recorder:,
+          max_frames: 400,
+          tracer: nil,
+          endpoint_collection_enabled: false,
+          timeline_enabled: false,
+          waiting_for_gvl_threshold_ns: 10_000_000,
+          otel_context_enabled: false,
+          native_filenames_enabled: true,
+          **options
+        )
+          new(
+            recorder: recorder,
+            max_frames: max_frames,
+            tracer: tracer,
+            endpoint_collection_enabled: endpoint_collection_enabled,
+            timeline_enabled: timeline_enabled,
+            waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
+            otel_context_enabled: otel_context_enabled,
+            native_filenames_enabled: native_filenames_enabled,
+            **options,
+          )
+        end
+
+        def inspect
+          # Compose Ruby's default inspect with our custom inspect for the native parts
+          result = super
+          result[-1] = "#{self.class._native_inspect(self)}>"
+          result
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        private
+
+        def safely_extract_context_key_from(tracer)
+          return unless tracer
+
+          provider = tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context&.instance_variable_get(:@key)
+        end
+
+        def validate_native_filenames(native_filenames_enabled)
+          if native_filenames_enabled && !Datadog::Profiling::Collectors::Stack._native_filenames_available?
+            Datadog.logger.debug(
+              "Native filenames are enabled, but the required dladdr API was not available. Disabling native filenames."
+            )
+            false
+          else
+            native_filenames_enabled
+          end
+        end
+      end
+    end
+  end
+end