datadog 2.39.0 → 2.47.0

data/lib/datadog/profiling/collectors/info.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require "set"
-require "time"
+require "json"
 require "libdatadog"
+require_relative "../../core/utils/time"
 
 module Datadog
   module Profiling
@@ -10,11 +10,11 @@ module Datadog
       # Collects information of relevance for profiler. This will get sent alongside
       # the profile and show up in the UI or potentially influence processing in some way.
       #
-      # Information is currently collected and frozen at construction time. A full collector
-      # could be seen as overkill for this case but it allows us to centralize information
-      # gathering and easily support more flexible/dynamic info collection in the future.
+      # Most information is static: collected and frozen at construction time. The exception is
+      # what `collect_dynamic_info` collects, which can change during the process lifetime.
       class Info
-        # @rbs @info: ::Hash[::Symbol, ::Hash[::Symbol, untyped]]
+        # @rbs @static_info: ::Hash[::Symbol, ::Hash[::Symbol, untyped]]
+        # @rbs @info_json_cache: [::Hash[::Symbol, bool], ::String]?
         # @rbs @platform_info: ::Hash[::Symbol, untyped]
         # @rbs @runtime_info: ::Hash[::Symbol, untyped]
         # @rbs @application_info: ::Hash[::Symbol, untyped]
@@ -25,18 +25,48 @@ module Datadog
         def initialize(settings)
           @profiler_info = nil
 
-          @info = {
+          @static_info = {
             platform: collect_platform_info,
             runtime: collect_runtime_info,
             application: collect_application_info(settings),
             profiler: collect_profiler_info(settings),
           }.freeze
+          @info_json_cache = nil
         end
 
-        attr_reader :info #: ::Hash[::Symbol, ::Hash[::Symbol, untyped]]
+        #: () -> ::Hash[::Symbol, ::Hash[::Symbol, untyped]]
+        def info
+          build_info(collect_dynamic_info)
+        end
+
+        # Serializing the same info on every flush would be wasteful, so the latest result is
+        # cached and reused for as long as the dynamic info is unchanged.
+        #: () -> ::String
+        def info_json
+          dynamic_info = collect_dynamic_info
+          cached_dynamic_info, cached_json = @info_json_cache
+          return cached_json if cached_json && cached_dynamic_info == dynamic_info
+
+          json = JSON.generate(build_info(dynamic_info)).freeze
+          @info_json_cache = [dynamic_info, json].freeze #: [::Hash[::Symbol, bool], ::String]
+          json
+        end
 
         private
 
+        #: (::Hash[::Symbol, bool]) -> ::Hash[::Symbol, ::Hash[::Symbol, untyped]]
+        def build_info(dynamic_info)
+          @static_info.merge(runtime: @static_info.fetch(:runtime).merge(dynamic_info))
+        end
+
+        #: () -> ::Hash[::Symbol, bool]
+        def collect_dynamic_info
+          {
+            yjit_enabled: defined?(::RubyVM::YJIT) ? ::RubyVM::YJIT.enabled? : false,
+            zjit_enabled: defined?(::RubyVM::ZJIT) ? ::RubyVM::ZJIT.enabled? : false,
+          }
+        end
+
         # Ruby GC tuning environment variables
         RUBY_GC_TUNING_ENV_VARS = [
           "RUBY_GC_HEAP_FREE_SLOTS",
@@ -71,10 +101,7 @@ module Datadog
         # Instead of trying to figure out real process start time by checking
         # /proc or some other complex/non-portable way, approximate start time
         # by time of requirement of this file.
-        #
-        # Note: this does not use Core::Utils::Time.now because this constant
-        # gets initialized before a user has a chance to configure the library.
-        START_TIME = Time.now.utc.freeze
+        START_TIME = Datadog::Core::Utils::Time.now.utc.freeze
 
         #: () -> ::Hash[::Symbol, untyped]
         def collect_platform_info
@@ -83,7 +110,7 @@ module Datadog
             hostname: Datadog::Core::Environment::Platform.hostname,
             kernel_name: Datadog::Core::Environment::Platform.kernel_name,
             kernel_release: Datadog::Core::Environment::Platform.kernel_release,
-            kernel_version: Datadog::Core::Environment::Platform.kernel_version
+            kernel_version: Datadog::Core::Environment::Platform.kernel_version,
           }.freeze
         end
 

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -19,9 +19,9 @@ module Datadog
           max_frames:,
           tracer:,
           endpoint_collection_enabled:,
-          waiting_for_gvl_threshold_ns:,
           otel_context_enabled:,
-          native_filenames_enabled:
+          native_filenames_enabled:,
+          show_classes:
         )
           tracer_context_key = safely_extract_context_key_from(tracer)
           self.class._native_initialize(
@@ -30,9 +30,9 @@ module Datadog
             max_frames: max_frames,
             tracer_context_key: tracer_context_key,
             endpoint_collection_enabled: endpoint_collection_enabled,
-            waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
             otel_context_enabled: otel_context_enabled,
-            native_filenames_enabled: validate_native_filenames(native_filenames_enabled),
+            native_filenames_enabled: native_filenames_enabled,
+            show_classes: show_classes,
             overhead_filename: __FILE__,
           )
         end
@@ -42,9 +42,9 @@ module Datadog
           max_frames: 400,
           tracer: nil,
           endpoint_collection_enabled: false,
-          waiting_for_gvl_threshold_ns: 10_000_000,
           otel_context_enabled: false,
           native_filenames_enabled: true,
+          show_classes: false,
           trigger_global_reset: true,
           **options
         )
@@ -53,9 +53,9 @@ module Datadog
             max_frames: max_frames,
             tracer: tracer,
             endpoint_collection_enabled: endpoint_collection_enabled,
-            waiting_for_gvl_threshold_ns: waiting_for_gvl_threshold_ns,
             otel_context_enabled: otel_context_enabled,
             native_filenames_enabled: native_filenames_enabled,
+            show_classes: show_classes,
             **options,
           )
 
@@ -90,17 +90,6 @@ module Datadog
           context = provider.instance_variable_get(:@context)
           context&.instance_variable_get(:@key)
         end
-
-        def validate_native_filenames(native_filenames_enabled)
-          if native_filenames_enabled && !Datadog::Profiling::Collectors::Stack._native_filenames_available?
-            Datadog.logger.debug(
-              "Native filenames are enabled, but the required dladdr API was not available. Disabling native filenames."
-            )
-            false
-          else
-            native_filenames_enabled
-          end
-        end
       end
     end
   end

data/lib/datadog/profiling/component.rb

@@ -65,6 +65,7 @@ module Datadog
           gvl_profiling_enabled: enable_gvl_profiling?(settings, logger),
           sighandler_sampling_enabled: settings.profiling.advanced.sighandler_sampling_enabled,
           cpu_sampling_interval_ms: cpu_sampling_interval_ms,
+          waiting_for_gvl_threshold_ns: settings.profiling.advanced.waiting_for_gvl_threshold_ns,
         )
 
         internal_metadata = {
@@ -72,7 +73,7 @@ module Datadog
           heap_sample_every: heap_sample_every,
         }.freeze
 
-        exporter = build_profiler_exporter(settings, recorder, worker, internal_metadata: internal_metadata)
+        exporter = build_profiler_exporter(settings, worker, internal_metadata: internal_metadata)
         transport = build_profiler_transport(settings, agent_settings)
         scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport, interval: upload_period_seconds)
         profiler = Profiling::Profiler.new(worker: worker, scheduler: scheduler)
@@ -102,19 +103,18 @@ module Datadog
           max_frames: settings.profiling.advanced.max_frames,
           tracer: optional_tracer,
           endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
-          waiting_for_gvl_threshold_ns: settings.profiling.advanced.waiting_for_gvl_threshold_ns,
           otel_context_enabled: settings.profiling.advanced.preview_otel_context_enabled,
           native_filenames_enabled: settings.profiling.advanced.native_filenames_enabled,
+          show_classes: settings.profiling.advanced.experimental_show_classes_enabled,
         )
       end
 
-      private_class_method def self.build_profiler_exporter(settings, recorder, worker, internal_metadata:)
+      private_class_method def self.build_profiler_exporter(settings, worker, internal_metadata:)
         info_collector = Profiling::Collectors::Info.new(settings)
         code_provenance_collector =
           (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
 
         Profiling::Exporter.new(
-          pprof_recorder: recorder,
           worker: worker,
           info_collector: info_collector,
           code_provenance_collector: code_provenance_collector,
@@ -226,11 +226,13 @@ module Datadog
           return false
         end
 
-        # Heap profiling relies on `ObjectSpace._id2ref`, which was removed on Ruby 4.1
-        # (https://bugs.ruby-lang.org/issues/22135).
-        if RubyVersion.is?(">= 4.1")
+        # Heap profiling tracks live objects using an `ObjectSpace::WeakMap`, which could corrupt its internal
+        # state during compaction before these versions (https://bugs.ruby-lang.org/issues/19529).
+        if RubyVersion.is?("< 3.1.4") || RubyVersion.is?(">= 3.2", "< 3.2.3")
           logger.warn(
-            "Heap profiling is currently incompatible with Ruby 4.1+ and has been disabled."
+            "Current Ruby version (#{RUBY_VERSION}) cannot support heap profiling due to a VM bug. " \
+            "Please upgrade to Ruby >= 3.1.4 or >= 3.2.3 in order to use this feature. " \
+            "Heap profiling has been disabled."
           )
           return false
         end
@@ -251,17 +253,7 @@ module Datadog
       private_class_method def self.enable_heap_size_profiling?(settings, heap_profiling_enabled, logger)
         heap_size_profiling_enabled = settings.profiling.advanced.experimental_heap_size_enabled
 
-        return false unless heap_profiling_enabled && heap_size_profiling_enabled
-
-        if RubyVersion.is?(">= 4")
-          logger.info(
-            "Heap live size profiling is currently incompatible with Ruby 4 and has been disabled. " \
-            "Heap live objects is not affected and remains enabled."
-          )
-          return false
-        end
-
-        true
+        heap_profiling_enabled && heap_size_profiling_enabled
       end
 
       private_class_method def self.no_signals_workaround_enabled?(settings, logger) # rubocop:disable Metrics/MethodLength
@@ -435,33 +427,35 @@ module Datadog
         end
       end
 
-      # To add just a bit more complexity to our detection code, in https://github.com/DataDog/dd-trace-rb/issues/3334
-      # a user reported that our code was incorrectly flagging the mariadb variant of libmysqlclient as being
-      # incompatible. In fact we have no reports of the mariadb variant needing the "no signals" workaround,
-      # so we flag it as compatible when it's in use.
+      # MariaDB's client has diverged from Oracle's and has no reported need for the "no signals" workaround, as
+      # reported in https://github.com/DataDog/dd-trace-rb/issues/3334, so we flag it as compatible when it's in use.
       #
-      # A problem is that there doesn't seem to be an obvious way to query the mysql2 gem on which kind of
-      # libmysqlclient it's using, so we detect it by looking at the version.
+      # There's no way to ask the mysql2 gem which client it was built against, so we infer it from the two versions
+      # that `Mysql2::Client.info` exposes:
       #
-      # The info method for mysql2 with mariadb looks something like this:
-      # `{:id=>30308, :version=>"3.3.8", :header_version=>"11.2.2"}`
+      # * `version` is `mysql_get_client_info()`. For MariaDB that's the version of
+      #   https://github.com/mariadb-corporation/mariadb-connector-c, which is still 3.x.
+      # * `header_version` is a constant baked in when mysql2 was compiled. Up to mysql2 0.5.6 it was the MariaDB
+      #   server version, which is what people usually call "the mariadb version"; 0.5.7 switched it to the
+      #   Connector/C version (https://github.com/brianmario/mysql2/pull/1406) and that broke our detection, as
+      #   reported in https://github.com/DataDog/dd-trace-rb/issues/6536.
       #
-      # * The version seems to come from https://github.com/mariadb-corporation/mariadb-connector-c and the latest
-      # one is 3.x.
-      # * The header_version is what people usually see as the "mariadb version"
+      # So MariaDB shows up as `{version: "3.3.8", header_version: "11.2.2"}` with mysql2 < 0.5.7 and as
+      # `{version: "3.4.9", header_version: "3.4.9"}` from 0.5.7 on. We accept both shapes: a header that is a server
+      # version (10 or newer, a sane range for modern releases per https://endoflife.date/mariadb) or one that is a
+      # Connector/C version (3.x, matching the client version).
       #
-      # As a comparison, for libmysql the info looks like:
-      # * `{:id=>80035, :version=>"8.0.35", :header_version=>"8.0.35"}`
-      #
-      # Thus our detection is version 4 or older, because libmysqlclient 4 is almost 20 years old so it's most probably
-      # not that one + header_version being 10 or newer, since according to https://endoflife.date/mariadb that's a
-      # sane range for modern mariadb releases.
+      # Oracle's client repeats one version in both fields (`{version: "8.0.35", header_version: "8.0.35"}`), and our
+      # caller already considers 8.0.0 and above compatible, so only a client older than 5.0.0 gets this far -- and
+      # no Oracle release has been one since libmysqlclient 4.1, which is from 2004.
       private_class_method def self.looks_like_mariadb?(info, libmysqlclient_version)
         header_version = Gem::Version.new(info[:header_version]) if info[:header_version]
 
-        !!(header_version &&
-          libmysqlclient_version < Gem::Version.new("5.0.0") &&
-          header_version >= Gem::Version.new("10.0.0"))
+        return false unless header_version && libmysqlclient_version < Gem::Version.new("5.0.0")
+        return true if header_version >= Gem::Version.new("10.0.0")
+
+        connector_c_versions = Gem::Requirement.new(">= 3.0.0", "< 4.0.0")
+        connector_c_versions.satisfied_by?(libmysqlclient_version) && connector_c_versions.satisfied_by?(header_version)
       end
 
       private_class_method def self.dir_interruption_workaround_enabled?(settings, no_signals_workaround_enabled)

data/lib/datadog/profiling/exporter.rb

@@ -20,7 +20,6 @@ module Datadog
 
       private
 
-      attr_reader :pprof_recorder #: Datadog::Profiling::StackRecorder
       # The code provenance collector acts both as collector and as a recorder
       attr_reader :code_provenance_collector #: Datadog::Profiling::Collectors::CodeProvenance?
       attr_reader :minimum_duration_seconds #: ::Integer
@@ -28,12 +27,11 @@ module Datadog
       attr_reader :last_flush_finish_at #: ::Time?
       attr_reader :created_at #: ::Time
       attr_reader :internal_metadata #: ::Hash[::Symbol, untyped]
-      attr_reader :info_json #: ::String
+      attr_reader :info_collector #: Datadog::Profiling::Collectors::Info
       attr_reader :sequence_tracker #: singleton(Datadog::Profiling::SequenceTracker)
 
       public
 
-      # @rbs pprof_recorder: Datadog::Profiling::StackRecorder
       # @rbs worker: Datadog::Profiling::Collectors::CpuAndWallTimeWorker
       # @rbs info_collector: Datadog::Profiling::Collectors::Info
       # @rbs code_provenance_collector: Datadog::Profiling::Collectors::CodeProvenance?
@@ -43,7 +41,6 @@ module Datadog
       # @rbs sequence_tracker: singleton(Datadog::Profiling::SequenceTracker)
       # @rbs return: void
       def initialize(
-        pprof_recorder:,
         worker:,
         info_collector:,
         code_provenance_collector:,
@@ -52,7 +49,6 @@ module Datadog
         time_provider: Time,
         sequence_tracker: Datadog::Profiling::SequenceTracker
       )
-        @pprof_recorder = pprof_recorder
         @worker = worker
         @code_provenance_collector = code_provenance_collector
         @minimum_duration_seconds = minimum_duration_seconds
@@ -60,14 +56,13 @@ module Datadog
         @last_flush_finish_at = nil
         @created_at = time_provider.now.utc
         @internal_metadata = internal_metadata
-        # NOTE: At the time of this comment collected info does not change over time so we'll hardcode
-        #       it on startup to prevent serializing the same info on every flush.
-        @info_json = JSON.generate(info_collector.info).freeze
+        @info_collector = info_collector
         @sequence_tracker = sequence_tracker
       end
 
       #: () -> Datadog::Profiling::Flush?
       def flush
+        pprof_recorder = @worker.prepare_serialize
         worker_stats = @worker.stats_and_reset_not_thread_safe
         serialization_result = pprof_recorder.serialize
         return if serialization_result.nil?
@@ -111,10 +106,10 @@ module Datadog
               worker_stats: worker_stats,
               profile_stats: profile_stats,
               recorder_stats: pprof_recorder.stats,
-              gc: GC.stat,
+              gc: gc_stats,
             }
           ),
-          info_json: info_json,
+          info_json: info_collector.info_json,
         )
       end
 
@@ -131,6 +126,11 @@ module Datadog
 
       private
 
+      # @rbs return: Hash[Symbol, Integer | bool]
+      def gc_stats
+        GC.stat.merge(auto_compact: GC.respond_to?(:auto_compact) && GC.auto_compact)
+      end
+
       #: (::Time, ::Time) -> bool
       def duration_below_threshold?(start, finish)
         (finish - start) < minimum_duration_seconds

data/lib/datadog/profiling/tasks/exec.rb

@@ -21,7 +21,7 @@ module Datadog
         #: () -> ::Array["-rdatadog/profiling/preload"]
         def rubyopts
           [
-            "-rdatadog/profiling/preload"
+            "-rdatadog/profiling/preload",
           ]
         end
 

data/lib/datadog/profiling.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "rbconfig"
 require_relative "core"
 require_relative "core/environment/variable_helpers"
 require_relative "core/utils/only_once"
@@ -7,6 +8,9 @@ require_relative "core/utils/only_once"
 module Datadog
   # Datadog Continuous Profiler implementation: https://docs.datadoghq.com/profiler/
   module Profiling
+    STATIC_RUBY_PATH = (RbConfig::CONFIG["ENABLE_SHARED"] == "no") ? RbConfig.ruby.freeze : nil
+    private_constant :STATIC_RUBY_PATH
+
     def self.supported?
       unsupported_reason.nil?
     end