ddtrace 1.12.1 → 1.23.2

data/lib/datadog/profiling/load_native_extension.rb

@@ -1,18 +1,37 @@
 # This file is used to load the profiling native extension. It works in two steps:
 #
-# 1. Load the ddtrace_profiling_loader extension. This extension will be used to load the actual extension, but in
-#    a special way that avoids exposing native-level code symbols. See `ddtrace_profiling_loader.c` for more details.
+# 1. Load the datadog_profiling_loader extension. This extension will be used to load the actual extension, but in
+#    a special way that avoids exposing native-level code symbols. See `datadog_profiling_loader.c` for more details.
 #
-# 2. Use the Datadog::Profiling::Loader exposed by the ddtrace_profiling_loader extension to load the actual
+# 2. Use the Datadog::Profiling::Loader exposed by the datadog_profiling_loader extension to load the actual
 #    profiling native extension.
 #
 # All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
 # the first time it gets required, to avoid any issues with the native extension being initialized more than once.
 
-require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+begin
+  require "datadog_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+rescue LoadError => e
+  raise LoadError,
+    'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
+    "(Details: #{e.message})"
+end
+
+extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+full_file_path = "#{__dir__}/../../#{file_name}"
+
+unless File.exist?(full_file_path)
+  extension_dir = Gem.loaded_specs['ddtrace'].extension_dir
+  candidate_path = "#{extension_dir}/#{file_name}"
+  if File.exist?(candidate_path)
+    full_file_path = candidate_path
+  else # rubocop:disable Style/EmptyElse
+    # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
+    # down anyway.
+  end
+end
 
-extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
-full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
 init_function_name = "Init_#{extension_name.split('.').first}"
 
 status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)

data/lib/datadog/profiling/native_extension.rb

@@ -3,7 +3,7 @@
 module Datadog
   module Profiling
     # This module contains classes and methods which are implemented using native code in the
-    # ext/ddtrace_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
+    # ext/datadog_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
     # write using C
     module NativeExtension
       private_class_method def self.working?
@@ -15,27 +15,6 @@ module Datadog
           false
         end
       end
-
-      unless singleton_class.method_defined?(:clock_id_for)
-        def self.clock_id_for(_)
-          nil
-        end
-      end
-
-      def self.cpu_time_ns_for(thread)
-        clock_id =
-          begin
-            clock_id_for(thread)
-          rescue Errno::ESRCH
-            nil
-          end
-
-        begin
-          ::Process.clock_gettime(clock_id, :nanosecond) if clock_id
-        rescue Errno::EINVAL
-          nil
-        end
-      end
     end
   end
 end

data/lib/datadog/profiling/profiler.rb

@@ -1,39 +1,62 @@
 module Datadog
   module Profiling
-    # Profiling entry point, which coordinates collectors and a scheduler
+    # Profiling entry point, which coordinates the worker and scheduler threads
     class Profiler
       include Datadog::Core::Utils::Forking
 
-      attr_reader \
-        :collectors,
-        :scheduler
+      private
 
-      def initialize(collectors, scheduler)
-        @collectors = collectors
+      attr_reader :worker, :scheduler
+
+      public
+
+      def initialize(worker:, scheduler:)
+        @worker = worker
         @scheduler = scheduler
       end
 
       def start
         after_fork! do
-          collectors.each(&:reset_after_fork)
+          worker.reset_after_fork
           scheduler.reset_after_fork
         end
 
-        collectors.each(&:start)
-        scheduler.start
+        worker.start(on_failure_proc: proc { component_failed(:worker) })
+        scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
       end
 
       def shutdown!
         Datadog.logger.debug('Shutting down profiler')
 
-        collectors.each do |collector|
-          collector.enabled = false
-          collector.stop(true)
-        end
+        stop_worker
+        stop_scheduler
+      end
+
+      private
 
+      def stop_worker
+        worker.stop
+      end
+
+      def stop_scheduler
         scheduler.enabled = false
         scheduler.stop(true)
       end
+
+      def component_failed(failed_component)
+        Datadog.logger.warn(
+          "Detected issue with profiler (#{failed_component} component), stopping profiling. " \
+          'See previous log messages for details.'
+        )
+
+        if failed_component == :worker
+          stop_scheduler
+        elsif failed_component == :scheduler
+          stop_worker
+        else
+          raise ArgumentError, "Unexpected failed_component: #{failed_component.inspect}"
+        end
+      end
     end
   end
 end

data/lib/datadog/profiling/scheduler.rb

@@ -5,12 +5,11 @@ require_relative '../core/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes a profile from the `Exporter` and reports it using the
+    # Periodically (every interval, 60 seconds by default) takes a profile from the `Exporter` and reports it using the
     # configured transport. Runs on its own background thread.
     class Scheduler < Core::Worker
       include Core::Workers::Polling
 
-      DEFAULT_INTERVAL_SECONDS = 60
       MINIMUM_INTERVAL_SECONDS = 0
 
       # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
@@ -28,8 +27,7 @@ module Datadog
       def initialize(
         exporter:,
         transport:,
-        fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
-        interval: DEFAULT_INTERVAL_SECONDS,
+        interval:, fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default, # seconds
         enabled: true
       )
         @exporter = exporter
@@ -45,20 +43,27 @@ module Datadog
         self.enabled = enabled
       end
 
-      def start
-        perform
+      def start(on_failure_proc: nil)
+        perform(on_failure_proc)
       end
 
-      def perform
-        # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
-        # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
-        # successfully finish in the strict timeout.
-        # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
-        interrupted = true
-
+      def perform(on_failure_proc)
         begin
+          # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
+          # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
+          # successfully finish in the strict timeout.
+          # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
+          interrupted = true
+
           flush_and_wait
           interrupted = false
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          Datadog.logger.warn(
+            'Profiling::Scheduler thread error. ' \
+            "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+          on_failure_proc&.call
+          raise
         ensure
           Datadog.logger.debug('#flush was interrupted or failed before it could complete') if interrupted
         end
@@ -108,8 +113,8 @@ module Datadog
         #
         # During PR review (https://github.com/DataDog/dd-trace-rb/pull/1807) we discussed the possible alternative of
         # just sleeping before starting the scheduler loop. We ended up not going with that option to avoid the first
-        # profile containing up to DEFAULT_INTERVAL_SECONDS + DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS instead of the
-        # usual DEFAULT_INTERVAL_SECONDS size.
+        # profile containing up to interval + DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS instead of the
+        # usual interval seconds.
         if run_loop?
           jitter_seconds = rand * DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS # floating point number between (0.0...maximum)
           sleep(jitter_seconds)

data/lib/datadog/profiling/stack_recorder.rb

@@ -4,7 +4,10 @@ module Datadog
     # Note that `record_sample` is only accessible from native code.
     # Methods prefixed with _native_ are implemented in `stack_recorder.c`
     class StackRecorder
-      def initialize(cpu_time_enabled:, alloc_samples_enabled:)
+      def initialize(
+        cpu_time_enabled:, alloc_samples_enabled:, heap_samples_enabled:, heap_size_enabled:,
+        heap_sample_every:, timeline_enabled:
+      )
         # This mutex works in addition to the fancy C-level mutexes we have in the native side (see the docs there).
         # It prevents multiple Ruby threads calling serialize at the same time -- something like
         # `10.times { Thread.new { stack_recorder.serialize } }`.
@@ -13,18 +16,26 @@ module Datadog
         # accidentally happening.
         @no_concurrent_synchronize_mutex = Mutex.new
 
-        self.class._native_initialize(self, cpu_time_enabled, alloc_samples_enabled)
+        self.class._native_initialize(
+          self,
+          cpu_time_enabled,
+          alloc_samples_enabled,
+          heap_samples_enabled,
+          heap_size_enabled,
+          heap_sample_every,
+          timeline_enabled,
+        )
       end
 
       def serialize
         status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
-          start, finish, encoded_pprof = result
+          start, finish, encoded_pprof, profile_stats = result
 
           Datadog.logger.debug { "Encoded profile covering #{start.iso8601} to #{finish.iso8601}" }
 
-          [start, finish, encoded_pprof]
+          [start, finish, encoded_pprof, profile_stats]
         else
           error_message = result
 
@@ -51,6 +62,10 @@ module Datadog
       def reset_after_fork
         self.class._native_reset_after_fork(self)
       end
+
+      def stats
+        self.class._native_stats(self)
+      end
     end
   end
 end

data/lib/datadog/profiling/tag_builder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../core/utils'
+require_relative '../core/environment/git'
 
 module Datadog
   module Profiling
@@ -23,6 +24,8 @@ module Datadog
         runtime_id: Core::Environment::Identity.id,
         runtime_platform: Core::Environment::Identity.lang_platform,
         runtime_version: Core::Environment::Identity.lang_version,
+        git_repository_url: Core::Environment::Git.git_repository_url,
+        git_commit_sha: Core::Environment::Git.git_commit_sha,
         # User-provided tags
         user_tags: settings.tags
       )
@@ -42,6 +45,8 @@ module Datadog
         tags[FORM_FIELD_TAG_ENV] = env if env
         tags[FORM_FIELD_TAG_SERVICE] = service if service
         tags[FORM_FIELD_TAG_VERSION] = version if version
+        tags[TAG_GIT_REPOSITORY_URL] = git_repository_url if git_repository_url
+        tags[TAG_GIT_COMMIT_SHA] = git_commit_sha if git_commit_sha
 
         # Make sure everything is an utf-8 string, to avoid encoding issues in native code/libddprof/further downstream
         user_tags.merge(tags).map do |key, value|

data/lib/datadog/profiling/tasks/exec.rb

@@ -1,7 +1,7 @@
 module Datadog
   module Profiling
     module Tasks
-      # Wraps command with Datadog tracing
+      # Wraps command with Datadog profiling
       class Exec
         attr_reader :args
 
@@ -36,10 +36,10 @@ module Datadog
         def exec_with_error_handling(args)
           Kernel.exec(*args)
         rescue Errno::ENOENT => e
-          Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
+          Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
           Kernel.exit 127
         rescue Errno::EACCES, Errno::ENOEXEC => e
-          Kernel.warn "ddtracerb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
+          Kernel.warn "ddprofrb exec failed: #{e.class.name} #{e.message} (command was '#{args.join(' ')}')"
           Kernel.exit 126
         end
       end

data/lib/datadog/profiling/tasks/help.rb

@@ -1,12 +1,12 @@
 module Datadog
   module Profiling
     module Tasks
-      # Prints help message for usage of `ddtrace`
+      # Prints help message for usage of `ddprofrb`
       class Help
         def run
           puts %(
-  Usage: ddtracerb [command] [arguments]
-    exec [command]: Executes command with tracing & profiling preloaded.
+  Usage: ddprofrb [command] [arguments]
+    exec [command]: Executes command with profiling preloaded.
     help:           Prints this help message.
           )
         end

data/lib/datadog/profiling.rb

@@ -5,22 +5,15 @@ require_relative 'core/utils/only_once'
 module Datadog
   # Datadog Continuous Profiler implementation: https://docs.datadoghq.com/profiler/
   module Profiling
-    GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
-    private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
-
     def self.supported?
       unsupported_reason.nil?
     end
 
     def self.unsupported_reason
       # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons -- e.g. tell users
-      # first that they can't use this on JRuby before telling them that they are missing protobuf
+      # first that they can't use this on JRuby before telling them that something else failed
 
-      native_library_compilation_skipped? ||
-        native_library_failed_to_load? ||
-        protobuf_gem_unavailable? ||
-        protobuf_version_unsupported? ||
-        protobuf_failed_to_load?
+      native_library_compilation_skipped? || native_library_failed_to_load?
     end
 
     # Starts the profiler, if the profiler is supported by in
@@ -53,9 +46,7 @@ module Datadog
     #
     # Note 2: All fibers in the same thread will share the same counter values.
     #
-    # Only available when the profiler is running, the new CPU Profiling 2.0 profiler is in use, and allocation-related
-    # features are not disabled via configuration.
-    # For instructions on enabling CPU Profiling 2.0 see the ddtrace release notes.
+    # Only available when the profiler is running, and allocation-related features are not disabled via configuration.
     #
     # @return [Integer] number of allocations observed in the current thread.
     # @return [nil] when not available.
@@ -66,8 +57,25 @@ module Datadog
       nil
     end
 
+    def self.enabled?
+      profiler = Datadog.send(:components).profiler
+      # Use .send(...) to avoid exposing the attr_reader as an API to the outside
+      !!(profiler.send(:scheduler).running? if profiler)
+    end
+
+    def self.wait_until_running(timeout_seconds: 5)
+      profiler = Datadog.send(:components).profiler
+      if profiler
+        # Use .send(...) to avoid exposing the attr_reader as an API to the outside
+        worker = profiler.send(:worker)
+        worker.wait_until_running(timeout_seconds: timeout_seconds)
+      else
+        raise 'Profiler not enabled or available'
+      end
+    end
+
     private_class_method def self.replace_noop_allocation_count
-      def self.allocation_count # rubocop:disable Lint/DuplicateMethods, Lint/NestedMethodDefinition (On purpose!)
+      def self.allocation_count # rubocop:disable Lint/NestedMethodDefinition (On purpose!)
         Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
       end
     end
@@ -80,7 +88,7 @@ module Datadog
 
     private_class_method def self.try_reading_skipped_reason_file(file_api = File)
       # This file, if it exists, is recorded by extconf.rb during compilation of the native extension
-      skipped_reason_file = "#{__dir__}/../../ext/ddtrace_profiling_native_extension/skipped_reason.txt"
+      skipped_reason_file = "#{__dir__}/../../ext/datadog_profiling_native_extension/skipped_reason.txt"
 
       begin
         return unless file_api.exist?(skipped_reason_file)
@@ -92,66 +100,6 @@ module Datadog
       end
     end
 
-    private_class_method def self.protobuf_gem_unavailable?
-      # NOTE: On environments where protobuf is already loaded, we skip the check. This allows us to support environments
-      # where no Gem.loaded_version is NOT available but customers are able to load protobuf; see for instance
-      # https://github.com/teamcapybara/capybara/commit/caf3bcd7664f4f2691d0ca9ef3be9a2a954fecfb
-      if !protobuf_already_loaded? && Gem.loaded_specs['google-protobuf'].nil?
-        "Missing google-protobuf dependency; please add `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
-      end
-    end
-
-    private_class_method def self.protobuf_version_unsupported?
-      # See above for why we skip the check when protobuf is already loaded; note that when protobuf was already loaded
-      # we skip the version check to avoid the call to Gem.loaded_specs. Unfortunately, protobuf does not seem to
-      # expose the gem version constant elsewhere, so in that setup we are not able to check the version.
-      if !protobuf_already_loaded? && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
-        'Your google-protobuf is too old; ensure that you have google-protobuf >= 3.0 by ' \
-        "adding `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
-      end
-    end
-
-    private_class_method def self.protobuf_already_loaded?
-      defined?(::Google::Protobuf) && !defined?(::Protobuf)
-      !!(defined?(::Google::Protobuf) && !defined?(::Protobuf))
-    end
-
-    private_class_method def self.protobuf_failed_to_load?
-      unless protobuf_loaded_successfully?
-        'There was an error loading the google-protobuf library; see previous warning message for details'
-      end
-    end
-
-    # The `google-protobuf` gem depends on a native component, and its creators helpfully tried to provide precompiled
-    # versions of this extension on rubygems.org.
-    #
-    # Unfortunately, for a long time, the supported Ruby versions metadata on these precompiled versions of the extension
-    # was not correctly set. (This is fixed in newer versions -- but not all Ruby versions we want to support can use
-    # these.)
-    #
-    # Thus, the gem can still be installed, but can be in a broken state. To avoid breaking customer applications, we
-    # use this helper to load it and gracefully handle failures.
-    private_class_method def self.protobuf_loaded_successfully?
-      return @protobuf_loaded if defined?(@protobuf_loaded)
-
-      begin
-        require 'google/protobuf'
-        @protobuf_loaded = true
-      rescue LoadError => e
-        # NOTE: We use Kernel#warn here because this code gets run BEFORE Datadog.logger is actually set up.
-        # In the future it'd be nice to shuffle the logger startup to happen first to avoid this special case.
-        Kernel.warn(
-          '[ddtrace] Error while loading google-protobuf gem. ' \
-          "Cause: '#{e.class.name} #{e.message}' Location: '#{Array(e.backtrace).first}'. " \
-          'This can happen when google-protobuf is missing its native components. ' \
-          'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
-          '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
-          'If the error persists, please contact Datadog support at <https://docs.datadoghq.com/help/>.'
-        )
-        @protobuf_loaded = false
-      end
-    end
-
     private_class_method def self.native_library_failed_to_load?
       success, exception = try_loading_native_library
 
@@ -178,27 +126,28 @@ module Datadog
       end
     end
 
+    # All requires for the profiler should be directly added here; and everything should be loaded eagerly.
+    # (Currently there's a leftovers that need to be cleaned up, but we should avoid other exceptions.)
+    #
+    # All of the profiler should be loaded and ready to go when this method returns `true`.
     private_class_method def self.load_profiling
       return false unless supported?
 
       require_relative 'profiling/ext/forking'
+      require_relative 'profiling/collectors/info'
       require_relative 'profiling/collectors/code_provenance'
       require_relative 'profiling/collectors/cpu_and_wall_time_worker'
       require_relative 'profiling/collectors/dynamic_sampling_rate'
       require_relative 'profiling/collectors/idle_sampling_helper'
-      require_relative 'profiling/collectors/old_stack'
       require_relative 'profiling/collectors/stack'
       require_relative 'profiling/collectors/thread_context'
       require_relative 'profiling/stack_recorder'
-      require_relative 'profiling/old_recorder'
       require_relative 'profiling/exporter'
+      require_relative 'profiling/flush'
       require_relative 'profiling/scheduler'
       require_relative 'profiling/tasks/setup'
       require_relative 'profiling/profiler'
       require_relative 'profiling/native_extension'
-      require_relative 'profiling/trace_identifiers/helper'
-      # This file is loaded in Profiling::Component#load_pprof_support; see notes there for why
-      # require_relative 'profiling/pprof/pprof_pb'
       require_relative 'profiling/tag_builder'
       require_relative 'profiling/http_transport'