ddtrace 0.49.0 → 1.12.1

data/lib/datadog/profiling/collectors/stack.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
+      #
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger sampling of threads, based on external "events", such as:
+      # * periodic timer for cpu-time and wall-time
+      # * VM garbage collection events
+      # * VM object allocation events
+      # Triggering of this component (e.g. watching for the above "events") is implemented by
+      # Collectors::CpuAndWallTimeWorker.
+      # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
+      class ThreadContext
+        def initialize(recorder:, max_frames:, tracer:, endpoint_collection_enabled:)
+          tracer_context_key = safely_extract_context_key_from(tracer)
+          self.class._native_initialize(self, recorder, max_frames, tracer_context_key, endpoint_collection_enabled)
+        end
+
+        def inspect
+          # Compose Ruby's default inspect with our custom inspect for the native parts
+          result = super()
+          result[-1] = "#{self.class._native_inspect(self)}>"
+          result
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        private
+
+        def safely_extract_context_key_from(tracer)
+          provider = tracer && tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context && context.instance_variable_get(:@key)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/component.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    # Responsible for wiring up the Profiler for execution
+    module Component
+      # Passing in a `nil` tracer is supported and will disable the following profiling features:
+      # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
+      # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
+      def self.build_profiler_component(settings:, agent_settings:, optional_tracer:)
+        return unless settings.profiling.enabled
+
+        # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
+        # dependency on individual products, in this case the Profiler.
+        # (Note "currently": in the future we want to change this so core classes don't depend on specific products)
+        #
+        # If the current file included a `require 'datadog/profiler'` at its beginning, we would generate circular
+        # requires when used from profiling:
+        #
+        # datadog/profiling
+        #     └─requires─> datadog/core
+        #                      └─requires─> datadog/core/configuration/components
+        #                                       └─requires─> datadog/profiling       # Loop!
+        #
+        # ...thus in #1998 we removed such a require.
+        #
+        # On the other hand, if datadog/core is loaded by a different product and no general `require 'ddtrace'` is
+        # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
+        # no-op if profiling is already loaded).
+        require_relative '../profiling'
+        return unless Profiling.supported?
+
+        unless defined?(Profiling::Tasks::Setup)
+          # In #1545 a user reported a NameError due to this constant being uninitialized
+          # I've documented my suspicion on why that happened in
+          # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
+          #
+          # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
+          # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
+          # > after Datadog.configure gets run.
+          # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
+          # > either profiler is supported, or profiler is not supported.
+          # > In the problematic case, it looks like in your case check 1 decides that profiler is not
+          # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
+          # > start it.
+          #
+          # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
+          # just in case it happens again, I've left this check which avoids breaking the user's application AND
+          # would instead direct them to report it to us instead, so that we can investigate what's wrong.
+          #
+          # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
+          # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
+          # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
+          Datadog.logger.error(
+            'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
+          )
+
+          return
+        end
+
+        # Load extensions needed to support some of the Profiling features
+        Profiling::Tasks::Setup.new.run
+
+        # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+        if enable_new_profiler?(settings)
+          recorder = Datadog::Profiling::StackRecorder.new(
+            cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
+            alloc_samples_enabled: false, # Always disabled for now -- work in progress
+          )
+          collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
+            recorder: recorder,
+            max_frames: settings.profiling.advanced.max_frames,
+            tracer: optional_tracer,
+            endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
+            gc_profiling_enabled: enable_gc_profiling?(settings),
+            allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
+            no_signals_workaround_enabled: no_signals_workaround_enabled?(settings),
+          )
+        else
+          load_pprof_support
+
+          recorder = build_profiler_old_recorder(settings)
+          collector = build_profiler_oldstack_collector(settings, recorder, optional_tracer)
+        end
+
+        exporter = build_profiler_exporter(settings, recorder)
+        transport = build_profiler_transport(settings, agent_settings)
+        scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
+
+        Profiling::Profiler.new([collector], scheduler)
+      end
+
+      private_class_method def self.build_profiler_old_recorder(settings)
+        Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
+      end
+
+      private_class_method def self.build_profiler_exporter(settings, recorder)
+        code_provenance_collector =
+          (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
+
+        Profiling::Exporter.new(pprof_recorder: recorder, code_provenance_collector: code_provenance_collector)
+      end
+
+      private_class_method def self.build_profiler_oldstack_collector(settings, old_recorder, tracer)
+        trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
+          tracer: tracer,
+          endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
+        )
+
+        Profiling::Collectors::OldStack.new(
+          old_recorder,
+          trace_identifiers_helper: trace_identifiers_helper,
+          max_frames: settings.profiling.advanced.max_frames
+        )
+      end
+
+      private_class_method def self.build_profiler_transport(settings, agent_settings)
+        settings.profiling.exporter.transport ||
+          Profiling::HttpTransport.new(
+            agent_settings: agent_settings,
+            site: settings.site,
+            api_key: settings.api_key,
+            upload_timeout_seconds: settings.profiling.upload.timeout_seconds,
+          )
+      end
+
+      private_class_method def self.enable_gc_profiling?(settings)
+        # See comments on the setting definition for more context on why it exists.
+        if settings.profiling.advanced.force_enable_gc_profiling
+          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
+            Datadog.logger.debug(
+              'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
+              'with this option as profiles will be incomplete.'
+            )
+          end
+
+          true
+        else
+          false
+        end
+      end
+
+      private_class_method def self.enable_new_profiler?(settings)
+        if settings.profiling.advanced.force_enable_legacy_profiler
+          Datadog.logger.warn(
+            'Legacy profiler has been force-enabled via configuration. Do not use unless instructed to by support.'
+          )
+          return false
+        end
+
+        true
+      end
+
+      private_class_method def self.no_signals_workaround_enabled?(settings) # rubocop:disable Metrics/MethodLength
+        setting_value = settings.profiling.advanced.no_signals_workaround_enabled
+        legacy_ruby_that_should_use_workaround = RUBY_VERSION.start_with?('2.3.', '2.4.', '2.5.')
+
+        unless [true, false, :auto].include?(setting_value)
+          Datadog.logger.error(
+            "Ignoring invalid value for profiling no_signals_workaround_enabled setting: #{setting_value.inspect}. " \
+            'Valid options are `true`, `false` or (default) `:auto`.'
+          )
+
+          setting_value = :auto
+        end
+
+        if setting_value == false
+          if legacy_ruby_that_should_use_workaround
+            Datadog.logger.warn(
+              'The profiling "no signals" workaround has been disabled via configuration on a legacy Ruby version ' \
+              '(< 2.6). This is not recommended ' \
+              'in production environments, as due to limitations in Ruby APIs, we suspect it may lead to crashes ' \
+              'in very rare situations. Please report any issues you run into to Datadog support or ' \
+              'via <https://github.com/datadog/dd-trace-rb/issues/new>!'
+            )
+          else
+            Datadog.logger.warn('Profiling "no signals" workaround disabled via configuration')
+          end
+
+          return false
+        end
+
+        if setting_value == true
+          Datadog.logger.warn(
+            'Profiling "no signals" workaround enabled via configuration. Profiling data will have lower quality.'
+          )
+
+          return true
+        end
+
+        # Setting is in auto mode. Let's probe to see if we should enable it:
+
+        # We don't warn users in this situation because "upgrade your Ruby" is not a great warning
+        return true if legacy_ruby_that_should_use_workaround
+
+        if Gem.loaded_specs['mysql2'] && incompatible_libmysqlclient_version?(settings)
+          Datadog.logger.warn(
+            'Enabling the profiling "no signals" workaround because an incompatible version of the mysql2 gem is ' \
+            'installed. Profiling data will have lower quality.' \
+            'To fix this, upgrade the libmysqlclient in your OS image to version 8.0.0 or above.'
+          )
+          return true
+        end
+
+        if Gem.loaded_specs['rugged']
+          Datadog.logger.warn(
+            'Enabling the profiling "no signals" workaround because the rugged gem is installed. ' \
+            'This is needed because some operations on this gem are currently incompatible with the normal working mode ' \
+            'of the profiler, as detailed in <https://github.com/datadog/dd-trace-rb/issues/2721>. ' \
+            'Profiling data will have lower quality.'
+          )
+          return true
+        end
+
+        false
+      end
+
+      # Versions of libmysqlclient prior to 8.0.0 are known to have buggy handling of system call interruptions.
+      # The profiler can sometimes cause system call interruptions, and so this combination can cause queries to fail.
+      #
+      # See https://bugs.mysql.com/bug.php?id=83109 and
+      # https://docs.datadoghq.com/profiler/profiler_troubleshooting/ruby/#unexpected-run-time-failures-and-errors-from-ruby-gems-that-use-native-extensions-in-dd-trace-rb-1110
+      # for details.
+      #
+      # The `mysql2` gem's `info` method can be used to determine which `libmysqlclient` version is in use, and thus to
+      # detect if it's safe for the profiler to use signals or if we need to employ a fallback.
+      private_class_method def self.incompatible_libmysqlclient_version?(settings)
+        return true if settings.profiling.advanced.skip_mysql2_check
+
+        Datadog.logger.debug(
+          'Requiring `mysql2` to check if the `libmysqlclient` version it uses is compatible with profiling'
+        )
+
+        begin
+          require 'mysql2'
+
+          return true unless defined?(Mysql2::Client) && Mysql2::Client.respond_to?(:info)
+
+          libmysqlclient_version = Gem::Version.new(Mysql2::Client.info[:version])
+
+          compatible = libmysqlclient_version >= Gem::Version.new('8.0.0')
+
+          Datadog.logger.debug(
+            "The `mysql2` gem is using #{compatible ? 'a compatible' : 'an incompatible'} version of " \
+            "the `libmysqlclient` library (#{libmysqlclient_version})"
+          )
+
+          !compatible
+        rescue StandardError, LoadError => e
+          Datadog.logger.warn(
+            'Failed to probe `mysql2` gem information. ' \
+            "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+
+          true
+        end
+      end
+
+      # The old profiler's pprof support conflicts with the ruby-cloud-profiler gem.
+      #
+      # This is not a problem for almost all customers, since we now default everyone to use the new CPU Profiling 2.0
+      # profiler. But the issue was still triggered, because currently we still _load_ both the old and new profiling
+      # code paths.
+      #
+      # To work around this issue, and because we plan on deleting the old profiler soon, rather than poking at the
+      # pprof support code, we only load the conflicting file when the old profiler is in use. This way customers using
+      # the new profiler will not be affected by the issue any longer.
+      private_class_method def self.load_pprof_support
+        require_relative 'pprof/pprof_pb'
+      end
+    end
+  end
+end

data/lib/datadog/profiling/encoding/profile.rb

@@ -0,0 +1,41 @@
+require 'time'
+
+require_relative '../pprof/template'
+
+module Datadog
+  module Profiling
+    module Encoding
+      module Profile
+        # Encodes gathered data into the pprof format
+        module Protobuf
+          module_function
+
+          def encode(event_count:, event_groups:, start:, finish:)
+            # Create a pprof template from the list of event types
+            event_classes = event_groups.collect(&:event_class).uniq
+            template = Pprof::Template.for_event_classes(event_classes)
+
+            # Add all events to the pprof
+            event_groups.each { |event_group| template.add_events!(event_group.event_class, event_group.events) }
+
+            Datadog.logger.debug do
+              max_events = Datadog.configuration.profiling.advanced.max_events
+              events_sampled =
+                if event_count == max_events
+                  'max events limit hit, events were sampled [profile will be biased], '
+                else
+                  ''
+                end
+
+              "Encoding profile covering #{start.iso8601} to #{finish.iso8601}, " \
+              "events: #{event_count} (#{events_sampled}#{template.debug_statistics})"
+            end
+
+            # Build the profile and encode it
+            template.to_pprof(start: start, finish: finish)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/event.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    # Describes a sample of some data obtained from the runtime.
+    class Event
+      attr_reader \
+        :timestamp
+
+      def initialize(timestamp = nil)
+        @timestamp = timestamp || Time.now.utc.to_f
+      end
+    end
+  end
+end

data/lib/datadog/profiling/events/stack.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative '../event'
+
+module Datadog
+  module Profiling
+    module Events
+      # Describes a stack profiling event
+      class Stack < Event
+        attr_reader \
+          :hash,
+          :frames,
+          :total_frame_count,
+          :thread_id,
+          :root_span_id,
+          :span_id,
+          :trace_resource
+
+        def initialize(
+          timestamp,
+          frames,
+          total_frame_count,
+          thread_id,
+          root_span_id,
+          span_id,
+          trace_resource
+        )
+          super(timestamp)
+
+          @frames = frames
+          @total_frame_count = total_frame_count
+          @thread_id = thread_id
+          @root_span_id = root_span_id
+          @span_id = span_id
+          @trace_resource = trace_resource
+
+          @hash = [
+            thread_id,
+            root_span_id,
+            span_id,
+            # trace_resource is deliberately not included -- events that share the same (root_span_id, span_id) refer
+            # to the same trace
+            frames.collect(&:hash),
+            total_frame_count
+          ].hash
+        end
+      end
+
+      # Describes a stack sample
+      class StackSample < Stack
+        attr_reader \
+          :cpu_time_interval_ns,
+          :wall_time_interval_ns
+
+        def initialize(
+          timestamp,
+          frames,
+          total_frame_count,
+          thread_id,
+          root_span_id,
+          span_id,
+          trace_resource,
+          cpu_time_interval_ns,
+          wall_time_interval_ns
+        )
+          super(
+            timestamp,
+            frames,
+            total_frame_count,
+            thread_id,
+            root_span_id,
+            span_id,
+            trace_resource
+          )
+
+          @cpu_time_interval_ns = cpu_time_interval_ns
+          @wall_time_interval_ns = wall_time_interval_ns
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/exporter.rb

@@ -0,0 +1,83 @@
+require_relative 'ext'
+require_relative 'tag_builder'
+
+module Datadog
+  module Profiling
+    # Exports profiling data gathered by the multiple recorders in a `Flush`.
+    #
+    # @ivoanjo: Note that the recorder that gathers pprof data is special, since we use its start/finish/empty? to
+    # decide if there's data to flush, as well as the timestamp for that data.
+    # I could've made the whole design more generic, but I'm unsure if we'll ever have more than a handful of
+    # recorders, so I've decided to make it specific until we actually need to support more recorders.
+    #
+    class Exporter
+      # Profiles with duration less than this will not be reported
+      PROFILE_DURATION_THRESHOLD_SECONDS = 1
+
+      private
+
+      attr_reader \
+        :pprof_recorder,
+        :code_provenance_collector, # The code provenance collector acts both as collector and as a recorder
+        :minimum_duration_seconds,
+        :time_provider,
+        :last_flush_finish_at,
+        :created_at
+
+      public
+
+      def initialize(
+        pprof_recorder:,
+        code_provenance_collector:,
+        minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
+        time_provider: Time
+      )
+        @pprof_recorder = pprof_recorder
+        @code_provenance_collector = code_provenance_collector
+        @minimum_duration_seconds = minimum_duration_seconds
+        @time_provider = time_provider
+        @last_flush_finish_at = nil
+        @created_at = time_provider.now.utc
+      end
+
+      def flush
+        start, finish, uncompressed_pprof = pprof_recorder.serialize
+        @last_flush_finish_at = finish
+
+        return if uncompressed_pprof.nil? # We don't want to report empty profiles
+
+        if duration_below_threshold?(start, finish)
+          Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
+          return
+        end
+
+        uncompressed_code_provenance = code_provenance_collector.refresh.generate_json if code_provenance_collector
+
+        Flush.new(
+          start: start,
+          finish: finish,
+          pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
+          pprof_data: uncompressed_pprof.to_s,
+          code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
+          code_provenance_data: uncompressed_code_provenance,
+          tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
+        )
+      end
+
+      def can_flush?
+        !duration_below_threshold?(last_flush_finish_at || created_at, time_provider.now.utc)
+      end
+
+      def reset_after_fork
+        @last_flush_finish_at = time_provider.now.utc
+        nil
+      end
+
+      private
+
+      def duration_below_threshold?(start, finish)
+        (finish - start) < minimum_duration_seconds
+      end
+    end
+  end
+end

data/lib/datadog/profiling/ext/forking.rb

@@ -0,0 +1,96 @@
+module Datadog
+  module Profiling
+    module Ext
+      # Monkey patches `Kernel#fork`, adding a `Kernel#at_fork` callback mechanism which is used to restore
+      # profiling abilities after the VM forks.
+      #
+      # Known limitations: Does not handle `BasicObject`s that include `Kernel` directly; e.g.
+      # `Class.new(BasicObject) { include(::Kernel); def call; fork { }; end }.new.call`.
+      #
+      # This will be fixed once we moved to hooking into `Process._fork`
+      module Forking
+        def self.supported?
+          Process.respond_to?(:fork)
+        end
+
+        def self.apply!
+          return false unless supported?
+
+          [
+            ::Process.singleton_class, # Process.fork
+            ::Kernel.singleton_class,  # Kernel.fork
+            ::Object,                  # fork without explicit receiver (it's defined as a method in ::Kernel)
+            # Note: Modifying Object as we do here is irreversible. During tests, this
+            # change will stick around even if we otherwise stub `Process` and `Kernel`
+          ].each { |target| target.prepend(Kernel) }
+
+          ::Process.singleton_class.prepend(ProcessDaemonPatch)
+        end
+
+        # Extensions for kernel
+        #
+        # TODO: Consider hooking into `Process._fork` on Ruby 3.1+ instead, see
+        #       https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
+        module Kernel
+          def fork
+            # If a block is provided, it must be wrapped to trigger callbacks.
+            child_block = if block_given?
+                            proc do
+                              # Trigger :child callback
+                              ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
+
+                              # Invoke original block
+                              yield
+                            end
+                          end
+
+            # Start fork
+            # If a block is provided, use the wrapped version.
+            result = child_block.nil? ? super : super(&child_block)
+
+            # Trigger correct callbacks depending on whether we're in the parent or child.
+            # If we're in the fork, result = nil: trigger child callbacks.
+            # If we're in the parent, result = fork PID: trigger parent callbacks.
+            ddtrace_at_fork_blocks[:child].each(&:call) if result.nil? && ddtrace_at_fork_blocks.key?(:child)
+
+            # Return PID from #fork
+            result
+          end
+
+          def at_fork(stage, &block)
+            raise ArgumentError, 'Bad \'stage\' for ::at_fork' unless stage == :child
+
+            ddtrace_at_fork_blocks[stage] = [] unless ddtrace_at_fork_blocks.key?(stage)
+            ddtrace_at_fork_blocks[stage] << block
+          end
+
+          module_function
+
+          def ddtrace_at_fork_blocks
+            # Blocks should be shared across all users of this module,
+            # e.g. Process#fork, Kernel#fork, etc. should all invoke the same callbacks.
+            # rubocop:disable Style/ClassVars
+            @@ddtrace_at_fork_blocks ||= {}
+            # rubocop:enable Style/ClassVars
+          end
+        end
+
+        # A call to Process.daemon ( https://rubyapi.org/3.1/o/process#method-c-daemon ) forks the current process and
+        # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
+        #
+        # This monkey patch makes the `Kernel#at_fork` mechanism defined above also work in this situation.
+        module ProcessDaemonPatch
+          def daemon(*args)
+            ddtrace_at_fork_blocks = Datadog::Profiling::Ext::Forking::Kernel.ddtrace_at_fork_blocks
+
+            result = super
+
+            ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
+
+            result
+          end
+        end
+      end
+    end
+  end
+end