ddtrace 1.12.1 → 1.18.0

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -14,9 +14,24 @@ module Datadog
       #
       # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
       class ThreadContext
-        def initialize(recorder:, max_frames:, tracer:, endpoint_collection_enabled:)
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          endpoint_collection_enabled:,
+          timeline_enabled:,
+          allocation_type_enabled: true
+        )
           tracer_context_key = safely_extract_context_key_from(tracer)
-          self.class._native_initialize(self, recorder, max_frames, tracer_context_key, endpoint_collection_enabled)
+          self.class._native_initialize(
+            self,
+            recorder,
+            max_frames,
+            tracer_context_key,
+            endpoint_collection_enabled,
+            timeline_enabled,
+            allocation_type_enabled,
+          )
         end
 
         def inspect

data/lib/datadog/profiling/component.rb

@@ -8,6 +8,10 @@ module Datadog
       # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
       # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
       def self.build_profiler_component(settings:, agent_settings:, optional_tracer:)
+        require_relative '../profiling/diagnostics/environment_logger'
+
+        Profiling::Diagnostics::EnvironmentLogger.collect_and_log!
+
         return unless settings.profiling.enabled
 
         # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
@@ -30,89 +34,53 @@ module Datadog
         require_relative '../profiling'
         return unless Profiling.supported?
 
-        unless defined?(Profiling::Tasks::Setup)
-          # In #1545 a user reported a NameError due to this constant being uninitialized
-          # I've documented my suspicion on why that happened in
-          # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
-          #
-          # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
-          # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
-          # > after Datadog.configure gets run.
-          # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
-          # > either profiler is supported, or profiler is not supported.
-          # > In the problematic case, it looks like in your case check 1 decides that profiler is not
-          # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
-          # > start it.
-          #
-          # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
-          # just in case it happens again, I've left this check which avoids breaking the user's application AND
-          # would instead direct them to report it to us instead, so that we can investigate what's wrong.
-          #
-          # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
-          # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
-          # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
-          Datadog.logger.error(
-            'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
-            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
-          )
-
-          return
-        end
-
-        # Load extensions needed to support some of the Profiling features
+        # Activate forking extensions
         Profiling::Tasks::Setup.new.run
 
         # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
 
-        if enable_new_profiler?(settings)
-          recorder = Datadog::Profiling::StackRecorder.new(
-            cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
-            alloc_samples_enabled: false, # Always disabled for now -- work in progress
-          )
-          collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
-            recorder: recorder,
-            max_frames: settings.profiling.advanced.max_frames,
-            tracer: optional_tracer,
-            endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
-            gc_profiling_enabled: enable_gc_profiling?(settings),
-            allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
-            no_signals_workaround_enabled: no_signals_workaround_enabled?(settings),
-          )
-        else
-          load_pprof_support
+        no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
+        timeline_enabled = settings.profiling.advanced.experimental_timeline_enabled
 
-          recorder = build_profiler_old_recorder(settings)
-          collector = build_profiler_oldstack_collector(settings, recorder, optional_tracer)
-        end
+        recorder = Datadog::Profiling::StackRecorder.new(
+          cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
+          alloc_samples_enabled: false, # Always disabled for now -- work in progress
+        )
+        thread_context_collector = Datadog::Profiling::Collectors::ThreadContext.new(
+          recorder: recorder,
+          max_frames: settings.profiling.advanced.max_frames,
+          tracer: optional_tracer,
+          endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
+          timeline_enabled: timeline_enabled,
+        )
+        worker = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
+          gc_profiling_enabled: enable_gc_profiling?(settings),
+          allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
+          no_signals_workaround_enabled: no_signals_workaround_enabled,
+          thread_context_collector: thread_context_collector,
+          allocation_sample_every: 0,
+        )
+
+        internal_metadata = {
+          no_signals_workaround_enabled: no_signals_workaround_enabled,
+          timeline_enabled: timeline_enabled,
+        }.freeze
 
-        exporter = build_profiler_exporter(settings, recorder)
+        exporter = build_profiler_exporter(settings, recorder, internal_metadata: internal_metadata)
         transport = build_profiler_transport(settings, agent_settings)
         scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
 
-        Profiling::Profiler.new([collector], scheduler)
+        Profiling::Profiler.new(worker: worker, scheduler: scheduler)
       end
 
-      private_class_method def self.build_profiler_old_recorder(settings)
-        Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
-      end
-
-      private_class_method def self.build_profiler_exporter(settings, recorder)
+      private_class_method def self.build_profiler_exporter(settings, recorder, internal_metadata:)
         code_provenance_collector =
           (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
 
-        Profiling::Exporter.new(pprof_recorder: recorder, code_provenance_collector: code_provenance_collector)
-      end
-
-      private_class_method def self.build_profiler_oldstack_collector(settings, old_recorder, tracer)
-        trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
-          tracer: tracer,
-          endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
-        )
-
-        Profiling::Collectors::OldStack.new(
-          old_recorder,
-          trace_identifiers_helper: trace_identifiers_helper,
-          max_frames: settings.profiling.advanced.max_frames
+        Profiling::Exporter.new(
+          pprof_recorder: recorder,
+          code_provenance_collector: code_provenance_collector,
+          internal_metadata: internal_metadata,
         )
       end
 
@@ -142,17 +110,6 @@ module Datadog
         end
       end
 
-      private_class_method def self.enable_new_profiler?(settings)
-        if settings.profiling.advanced.force_enable_legacy_profiler
-          Datadog.logger.warn(
-            'Legacy profiler has been force-enabled via configuration. Do not use unless instructed to by support.'
-          )
-          return false
-        end
-
-        true
-      end
-
       private_class_method def self.no_signals_workaround_enabled?(settings) # rubocop:disable Metrics/MethodLength
         setting_value = settings.profiling.advanced.no_signals_workaround_enabled
         legacy_ruby_that_should_use_workaround = RUBY_VERSION.start_with?('2.3.', '2.4.', '2.5.')
@@ -198,7 +155,7 @@ module Datadog
         if Gem.loaded_specs['mysql2'] && incompatible_libmysqlclient_version?(settings)
           Datadog.logger.warn(
             'Enabling the profiling "no signals" workaround because an incompatible version of the mysql2 gem is ' \
-            'installed. Profiling data will have lower quality.' \
+            'installed. Profiling data will have lower quality. ' \
             'To fix this, upgrade the libmysqlclient in your OS image to version 8.0.0 or above.'
           )
           return true
@@ -214,6 +171,15 @@ module Datadog
           return true
         end
 
+        if (defined?(::PhusionPassenger) || Gem.loaded_specs['passenger']) && incompatible_passenger_version?
+          Datadog.logger.warn(
+            'Enabling the profiling "no signals" workaround because an incompatible version of the passenger gem is ' \
+            'installed. Profiling data will have lower quality.' \
+            'To fix this, upgrade the passenger gem to version 6.0.19 or above.'
+          )
+          return true
+        end
+
         false
       end
 
@@ -236,9 +202,22 @@ module Datadog
         begin
           require 'mysql2'
 
-          return true unless defined?(Mysql2::Client) && Mysql2::Client.respond_to?(:info)
+          # The mysql2-aurora gem likes to monkey patch itself in replacement of Mysql2::Client, and uses
+          # `method_missing` to delegate to the original BUT unfortunately does not implement `respond_to_missing?` and
+          # thus our `respond_to?(:info)` below was failing.
+          #
+          # But on the bright side, the gem does stash a reference to the original Mysql2::Client class in a constant,
+          # so if that constant exists, we use that for our probing.
+          mysql2_client_class =
+            if defined?(Mysql2::Aurora::ORIGINAL_CLIENT_CLASS)
+              Mysql2::Aurora::ORIGINAL_CLIENT_CLASS
+            elsif defined?(Mysql2::Client)
+              Mysql2::Client
+            end
+
+          return true unless mysql2_client_class && mysql2_client_class.respond_to?(:info)
 
-          libmysqlclient_version = Gem::Version.new(Mysql2::Client.info[:version])
+          libmysqlclient_version = Gem::Version.new(mysql2_client_class.info[:version])
 
           compatible = libmysqlclient_version >= Gem::Version.new('8.0.0')
 
@@ -258,17 +237,13 @@ module Datadog
         end
       end
 
-      # The old profiler's pprof support conflicts with the ruby-cloud-profiler gem.
-      #
-      # This is not a problem for almost all customers, since we now default everyone to use the new CPU Profiling 2.0
-      # profiler. But the issue was still triggered, because currently we still _load_ both the old and new profiling
-      # code paths.
-      #
-      # To work around this issue, and because we plan on deleting the old profiler soon, rather than poking at the
-      # pprof support code, we only load the conflicting file when the old profiler is in use. This way customers using
-      # the new profiler will not be affected by the issue any longer.
-      private_class_method def self.load_pprof_support
-        require_relative 'pprof/pprof_pb'
+      # See https://github.com/datadog/dd-trace-rb/issues/2976 for details.
+      private_class_method def self.incompatible_passenger_version?
+        if Gem.loaded_specs['passenger']
+          Gem.loaded_specs['passenger'].version < Gem::Version.new('6.0.19')
+        else
+          true
+        end
       end
     end
   end

data/lib/datadog/profiling/diagnostics/environment_logger.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'rbconfig'
+require_relative '../../core/diagnostics/environment_logger'
+
+module Datadog
+  module Profiling
+    module Diagnostics
+      # Collects and logs Profiling diagnostic information
+      module EnvironmentLogger
+        extend Core::Diagnostics::EnvironmentLogging
+
+        def self.collect_and_log!
+          log_once! do
+            data = EnvironmentCollector.collect_config!
+            log_configuration!('PROFILING', data.to_json)
+          end
+        rescue => e
+          logger.warn("Failed to collect profiling environment information: #{e} Location: #{Array(e.backtrace).first}")
+        end
+      end
+
+      # Collects environment information for Profiling diagnostic logging
+      module EnvironmentCollector
+        def self.collect_config!(*args)
+          {
+            profiling_enabled: profiling_enabled
+          }
+        end
+
+        def self.profiling_enabled
+          !!Datadog.configuration.profiling.enabled
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/exporter.rb

@@ -22,13 +22,15 @@ module Datadog
         :minimum_duration_seconds,
         :time_provider,
         :last_flush_finish_at,
-        :created_at
+        :created_at,
+        :internal_metadata
 
       public
 
       def initialize(
         pprof_recorder:,
         code_provenance_collector:,
+        internal_metadata:,
         minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
         time_provider: Time
       )
@@ -38,6 +40,7 @@ module Datadog
         @time_provider = time_provider
         @last_flush_finish_at = nil
         @created_at = time_provider.now.utc
+        @internal_metadata = internal_metadata
       end
 
       def flush
@@ -61,6 +64,7 @@ module Datadog
           code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
           code_provenance_data: uncompressed_code_provenance,
           tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
+          internal_metadata: internal_metadata,
         )
       end
 

data/lib/datadog/profiling/ext.rb

@@ -9,18 +9,6 @@ module Datadog
       ENV_AGENTLESS = 'DD_PROFILING_AGENTLESS'
       ENV_ENDPOINT_COLLECTION_ENABLED = 'DD_PROFILING_ENDPOINT_COLLECTION_ENABLED'
 
-      # TODO: Consider removing this once the Ruby-based pprof encoding is removed and replaced by libdatadog
-      module Pprof
-        LABEL_KEY_LOCAL_ROOT_SPAN_ID = 'local root span id'
-        LABEL_KEY_SPAN_ID = 'span id'
-        LABEL_KEY_THREAD_ID = 'thread id'
-        LABEL_KEY_TRACE_ENDPOINT = 'trace endpoint'
-        SAMPLE_VALUE_NO_VALUE = 0
-        VALUE_TYPE_CPU = 'cpu-time'
-        VALUE_TYPE_WALL = 'wall-time'
-        VALUE_UNIT_NANOSECONDS = 'nanoseconds'
-      end
-
       module Transport
         module HTTP
           FORM_FIELD_TAG_ENV = 'env'

data/lib/datadog/profiling/flush.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Datadog
   module Profiling
-    # Represents a collection of events of a specific type being flushed.
-    EventGroup = Struct.new(:event_class, :events)
-
     # Entity class used to represent metadata for a given profile
     class Flush
       attr_reader \
@@ -14,7 +13,8 @@ module Datadog
         :pprof_data, # gzipped pprof bytes
         :code_provenance_file_name,
         :code_provenance_data, # gzipped json bytes
-        :tags_as_array
+        :tags_as_array,
+        :internal_metadata_json
 
       def initialize(
         start:,
@@ -23,7 +23,8 @@ module Datadog
         pprof_data:,
         code_provenance_file_name:,
         code_provenance_data:,
-        tags_as_array:
+        tags_as_array:,
+        internal_metadata:
       )
         @start = start
         @finish = finish
@@ -32,6 +33,7 @@ module Datadog
         @code_provenance_file_name = code_provenance_file_name
         @code_provenance_data = code_provenance_data
         @tags_as_array = tags_as_array
+        @internal_metadata_json = JSON.fast_generate(internal_metadata.map { |k, v| [k, v.to_s] }.to_h)
       end
     end
   end

data/lib/datadog/profiling/http_transport.rb

@@ -1,3 +1,5 @@
+require_relative '../core/transport/ext'
+
 module Datadog
   module Profiling
     # Used to report profiling data to Datadog.
@@ -40,6 +42,7 @@ module Datadog
           code_provenance_data: flush.code_provenance_data,
 
           tags_as_array: flush.tags_as_array,
+          internal_metadata_json: flush.internal_metadata_json,
         )
 
         if status == :ok
@@ -47,11 +50,14 @@ module Datadog
             Datadog.logger.debug('Successfully reported profiling data')
             true
           else
-            Datadog.logger.error("Failed to report profiling data: server returned unexpected HTTP #{result} status code")
+            Datadog.logger.error(
+              "Failed to report profiling data (#{config_without_api_key}): " \
+              "server returned unexpected HTTP #{result} status code"
+            )
             false
           end
         else
-          Datadog.logger.error("Failed to report profiling data: #{result}")
+          Datadog.logger.error("Failed to report profiling data (#{config_without_api_key}): #{result}")
           false
         end
       end
@@ -66,9 +72,9 @@ module Datadog
 
       def base_url_from(agent_settings)
         case agent_settings.adapter
-        when Datadog::Transport::Ext::HTTP::ADAPTER
+        when Datadog::Core::Transport::Ext::HTTP::ADAPTER
           "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
-        when Datadog::Transport::Ext::UnixSocket::ADAPTER
+        when Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
           "unix://#{agent_settings.uds_path}"
         else
           raise ArgumentError, "Unexpected adapter: #{agent_settings.adapter}"
@@ -76,7 +82,8 @@ module Datadog
       end
 
       def validate_agent_settings(agent_settings)
-        supported_adapters = [Datadog::Transport::Ext::HTTP::ADAPTER, Datadog::Transport::Ext::UnixSocket::ADAPTER]
+        supported_adapters = [Datadog::Core::Transport::Ext::HTTP::ADAPTER,
+                              Datadog::Core::Transport::Ext::UnixSocket::ADAPTER]
         unless supported_adapters.include?(agent_settings.adapter)
           raise ArgumentError,
             "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
@@ -109,7 +116,8 @@ module Datadog
         pprof_data:,
         code_provenance_file_name:,
         code_provenance_data:,
-        tags_as_array:
+        tags_as_array:,
+        internal_metadata_json:
       )
         self.class._native_do_export(
           exporter_configuration,
@@ -123,8 +131,13 @@ module Datadog
           code_provenance_file_name,
           code_provenance_data,
           tags_as_array,
+          internal_metadata_json,
         )
       end
+
+      def config_without_api_key
+        [@exporter_configuration[0..1]].to_h
+      end
     end
   end
 end

data/lib/datadog/profiling/load_native_extension.rb

@@ -9,7 +9,13 @@
 # All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
 # the first time it gets required, to avoid any issues with the native extension being initialized more than once.
 
-require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+begin
+  require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+rescue LoadError => e
+  raise LoadError,
+    'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
+    "(Details: #{e.message})"
+end
 
 extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
 full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"

data/lib/datadog/profiling/native_extension.rb

@@ -15,27 +15,6 @@ module Datadog
           false
         end
       end
-
-      unless singleton_class.method_defined?(:clock_id_for)
-        def self.clock_id_for(_)
-          nil
-        end
-      end
-
-      def self.cpu_time_ns_for(thread)
-        clock_id =
-          begin
-            clock_id_for(thread)
-          rescue Errno::ESRCH
-            nil
-          end
-
-        begin
-          ::Process.clock_gettime(clock_id, :nanosecond) if clock_id
-        rescue Errno::EINVAL
-          nil
-        end
-      end
     end
   end
 end

data/lib/datadog/profiling/profiler.rb

@@ -1,39 +1,62 @@
 module Datadog
   module Profiling
-    # Profiling entry point, which coordinates collectors and a scheduler
+    # Profiling entry point, which coordinates the worker and scheduler threads
     class Profiler
       include Datadog::Core::Utils::Forking
 
-      attr_reader \
-        :collectors,
-        :scheduler
+      private
 
-      def initialize(collectors, scheduler)
-        @collectors = collectors
+      attr_reader :worker, :scheduler
+
+      public
+
+      def initialize(worker:, scheduler:)
+        @worker = worker
         @scheduler = scheduler
       end
 
       def start
         after_fork! do
-          collectors.each(&:reset_after_fork)
+          worker.reset_after_fork
           scheduler.reset_after_fork
         end
 
-        collectors.each(&:start)
-        scheduler.start
+        worker.start(on_failure_proc: proc { component_failed(:worker) })
+        scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
       end
 
       def shutdown!
         Datadog.logger.debug('Shutting down profiler')
 
-        collectors.each do |collector|
-          collector.enabled = false
-          collector.stop(true)
-        end
+        stop_worker
+        stop_scheduler
+      end
+
+      private
 
+      def stop_worker
+        worker.stop
+      end
+
+      def stop_scheduler
         scheduler.enabled = false
         scheduler.stop(true)
       end
+
+      def component_failed(failed_component)
+        Datadog.logger.warn(
+          "Detected issue with profiler (#{failed_component} component), stopping profiling. " \
+          'See previous log messages for details.'
+        )
+
+        if failed_component == :worker
+          stop_scheduler
+        elsif failed_component == :scheduler
+          stop_worker
+        else
+          raise ArgumentError, "Unexpected failed_component: #{failed_component.inspect}"
+        end
+      end
     end
   end
 end

data/lib/datadog/profiling/scheduler.rb

@@ -45,20 +45,27 @@ module Datadog
         self.enabled = enabled
       end
 
-      def start
-        perform
+      def start(on_failure_proc: nil)
+        perform(on_failure_proc)
       end
 
-      def perform
-        # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
-        # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
-        # successfully finish in the strict timeout.
-        # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
-        interrupted = true
-
+      def perform(on_failure_proc)
         begin
+          # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
+          # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
+          # successfully finish in the strict timeout.
+          # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
+          interrupted = true
+
           flush_and_wait
           interrupted = false
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          Datadog.logger.warn(
+            'Profiling::Scheduler thread error. ' \
+            "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+          on_failure_proc&.call
+          raise
         ensure
           Datadog.logger.debug('#flush was interrupted or failed before it could complete') if interrupted
         end