ddtrace 0.34.1 → 1.23.1

data/lib/datadog/profiling/exporter.rb

@@ -0,0 +1,104 @@
+require_relative 'ext'
+require_relative 'tag_builder'
+
+module Datadog
+  module Profiling
+    # Exports profiling data gathered by the multiple recorders in a `Flush`.
+    #
+    # @ivoanjo: Note that the recorder that gathers pprof data is special, since we use its start/finish/empty? to
+    # decide if there's data to flush, as well as the timestamp for that data.
+    # I could've made the whole design more generic, but I'm unsure if we'll ever have more than a handful of
+    # recorders, so I've decided to make it specific until we actually need to support more recorders.
+    #
+    class Exporter
+      # Profiles with duration less than this will not be reported
+      PROFILE_DURATION_THRESHOLD_SECONDS = 1
+
+      private
+
+      attr_reader \
+        :pprof_recorder,
+        :code_provenance_collector, # The code provenance collector acts both as collector and as a recorder
+        :minimum_duration_seconds,
+        :time_provider,
+        :last_flush_finish_at,
+        :created_at,
+        :internal_metadata,
+        :info_json
+
+      public
+
+      def initialize(
+        pprof_recorder:,
+        worker:,
+        info_collector:,
+        code_provenance_collector:,
+        internal_metadata:,
+        minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
+        time_provider: Time
+      )
+        @pprof_recorder = pprof_recorder
+        @worker = worker
+        @code_provenance_collector = code_provenance_collector
+        @minimum_duration_seconds = minimum_duration_seconds
+        @time_provider = time_provider
+        @last_flush_finish_at = nil
+        @created_at = time_provider.now.utc
+        @internal_metadata = internal_metadata
+        # NOTE: At the time of this comment collected info does not change over time so we'll hardcode
+        #       it on startup to prevent serializing the same info on every flush.
+        @info_json = JSON.fast_generate(info_collector.info).freeze
+      end
+
+      def flush
+        worker_stats = @worker.stats_and_reset_not_thread_safe
+        serialization_result = pprof_recorder.serialize
+        return if serialization_result.nil?
+
+        start, finish, compressed_pprof, profile_stats = serialization_result
+        @last_flush_finish_at = finish
+
+        if duration_below_threshold?(start, finish)
+          Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
+          return
+        end
+
+        uncompressed_code_provenance = code_provenance_collector.refresh.generate_json if code_provenance_collector
+
+        Flush.new(
+          start: start,
+          finish: finish,
+          pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
+          pprof_data: compressed_pprof.to_s,
+          code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
+          code_provenance_data: uncompressed_code_provenance,
+          tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
+          internal_metadata: internal_metadata.merge(
+            {
+              worker_stats: worker_stats,
+              profile_stats: profile_stats,
+              recorder_stats: pprof_recorder.stats,
+              gc: GC.stat,
+            }
+          ),
+          info_json: info_json,
+        )
+      end
+
+      def can_flush?
+        !duration_below_threshold?(last_flush_finish_at || created_at, time_provider.now.utc)
+      end
+
+      def reset_after_fork
+        @last_flush_finish_at = time_provider.now.utc
+        nil
+      end
+
+      private
+
+      def duration_below_threshold?(start, finish)
+        (finish - start) < minimum_duration_seconds
+      end
+    end
+  end
+end

data/lib/datadog/profiling/ext/forking.rb

@@ -0,0 +1,96 @@
+module Datadog
+  module Profiling
+    module Ext
+      # Monkey patches `Kernel#fork`, adding a `Kernel#at_fork` callback mechanism which is used to restore
+      # profiling abilities after the VM forks.
+      #
+      # Known limitations: Does not handle `BasicObject`s that include `Kernel` directly; e.g.
+      # `Class.new(BasicObject) { include(::Kernel); def call; fork { }; end }.new.call`.
+      #
+      # This will be fixed once we moved to hooking into `Process._fork`
+      module Forking
+        def self.supported?
+          Process.respond_to?(:fork)
+        end
+
+        def self.apply!
+          return false unless supported?
+
+          [
+            ::Process.singleton_class, # Process.fork
+            ::Kernel.singleton_class,  # Kernel.fork
+            ::Object,                  # fork without explicit receiver (it's defined as a method in ::Kernel)
+            # Note: Modifying Object as we do here is irreversible. During tests, this
+            # change will stick around even if we otherwise stub `Process` and `Kernel`
+          ].each { |target| target.prepend(Kernel) }
+
+          ::Process.singleton_class.prepend(ProcessDaemonPatch)
+        end
+
+        # Extensions for kernel
+        #
+        # TODO: Consider hooking into `Process._fork` on Ruby 3.1+ instead, see
+        #       https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
+        module Kernel
+          def fork
+            # If a block is provided, it must be wrapped to trigger callbacks.
+            child_block = if block_given?
+                            proc do
+                              # Trigger :child callback
+                              ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
+
+                              # Invoke original block
+                              yield
+                            end
+                          end
+
+            # Start fork
+            # If a block is provided, use the wrapped version.
+            result = child_block.nil? ? super : super(&child_block)
+
+            # Trigger correct callbacks depending on whether we're in the parent or child.
+            # If we're in the fork, result = nil: trigger child callbacks.
+            # If we're in the parent, result = fork PID: trigger parent callbacks.
+            ddtrace_at_fork_blocks[:child].each(&:call) if result.nil? && ddtrace_at_fork_blocks.key?(:child)
+
+            # Return PID from #fork
+            result
+          end
+
+          def at_fork(stage, &block)
+            raise ArgumentError, 'Bad \'stage\' for ::at_fork' unless stage == :child
+
+            ddtrace_at_fork_blocks[stage] = [] unless ddtrace_at_fork_blocks.key?(stage)
+            ddtrace_at_fork_blocks[stage] << block
+          end
+
+          module_function
+
+          def ddtrace_at_fork_blocks
+            # Blocks should be shared across all users of this module,
+            # e.g. Process#fork, Kernel#fork, etc. should all invoke the same callbacks.
+            # rubocop:disable Style/ClassVars
+            @@ddtrace_at_fork_blocks ||= {}
+            # rubocop:enable Style/ClassVars
+          end
+        end
+
+        # A call to Process.daemon ( https://rubyapi.org/3.1/o/process#method-c-daemon ) forks the current process and
+        # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
+        #
+        # This monkey patch makes the `Kernel#at_fork` mechanism defined above also work in this situation.
+        module ProcessDaemonPatch
+          def daemon(*args)
+            ddtrace_at_fork_blocks = Datadog::Profiling::Ext::Forking::Kernel.ddtrace_at_fork_blocks
+
+            result = super
+
+            ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
+
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/ext.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Ext
+      ENV_ENABLED = 'DD_PROFILING_ENABLED'
+      ENV_UPLOAD_TIMEOUT = 'DD_PROFILING_UPLOAD_TIMEOUT'
+      ENV_MAX_FRAMES = 'DD_PROFILING_MAX_FRAMES'
+      ENV_AGENTLESS = 'DD_PROFILING_AGENTLESS'
+      ENV_ENDPOINT_COLLECTION_ENABLED = 'DD_PROFILING_ENDPOINT_COLLECTION_ENABLED'
+
+      module Transport
+        module HTTP
+          FORM_FIELD_TAG_ENV = 'env'
+          FORM_FIELD_TAG_HOST = 'host'
+          FORM_FIELD_TAG_LANGUAGE = 'language'
+          FORM_FIELD_TAG_PID = 'process_id'
+          FORM_FIELD_TAG_PROFILER_VERSION = 'profiler_version'
+          FORM_FIELD_TAG_RUNTIME = 'runtime'
+          FORM_FIELD_TAG_RUNTIME_ENGINE = 'runtime_engine'
+          FORM_FIELD_TAG_RUNTIME_ID = 'runtime-id'
+          FORM_FIELD_TAG_RUNTIME_PLATFORM = 'runtime_platform'
+          FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'
+          FORM_FIELD_TAG_SERVICE = 'service'
+          FORM_FIELD_TAG_VERSION = 'version'
+          TAG_GIT_REPOSITORY_URL = 'git.repository_url'
+          TAG_GIT_COMMIT_SHA = 'git.commit.sha'
+
+          PPROF_DEFAULT_FILENAME = 'rubyprofile.pprof'
+          CODE_PROVENANCE_FILENAME = 'code-provenance.json'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/flush.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module Profiling
+    # Entity class used to represent metadata for a given profile
+    class Flush
+      attr_reader \
+        :start,
+        :finish,
+        :pprof_file_name,
+        :pprof_data, # gzipped pprof bytes
+        :code_provenance_file_name,
+        :code_provenance_data, # gzipped json bytes
+        :tags_as_array,
+        :internal_metadata_json,
+        :info_json
+
+      def initialize(
+        start:,
+        finish:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:,
+        internal_metadata:,
+        info_json:
+      )
+        @start = start
+        @finish = finish
+        @pprof_file_name = pprof_file_name
+        @pprof_data = pprof_data
+        @code_provenance_file_name = code_provenance_file_name
+        @code_provenance_data = code_provenance_data
+        @tags_as_array = tags_as_array
+        @internal_metadata_json = JSON.fast_generate(internal_metadata)
+        @info_json = info_json
+      end
+    end
+  end
+end

data/lib/datadog/profiling/http_transport.rb

@@ -0,0 +1,147 @@
+require_relative '../core/transport/ext'
+
+module Datadog
+  module Profiling
+    # Used to report profiling data to Datadog.
+    # Methods prefixed with _native_ are implemented in `http_transport.c`
+    class HttpTransport
+      def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
+        @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
+
+        validate_agent_settings(agent_settings)
+
+        @exporter_configuration =
+          if agentless?(site, api_key)
+            [:agentless, site, api_key]
+          else
+            [:agent, base_url_from(agent_settings)]
+          end
+
+        status, result = validate_exporter(@exporter_configuration)
+
+        raise(ArgumentError, "Failed to initialize transport: #{result}") if status == :error
+      end
+
+      def export(flush)
+        status, result = do_export(
+          exporter_configuration: @exporter_configuration,
+          upload_timeout_milliseconds: @upload_timeout_milliseconds,
+
+          # why "timespec"?
+          # libdatadog represents time using POSIX's struct timespec, see
+          # https://www.gnu.org/software/libc/manual/html_node/Time-Types.html
+          # aka it represents the seconds part separate from the nanoseconds part
+          start_timespec_seconds: flush.start.tv_sec,
+          start_timespec_nanoseconds: flush.start.tv_nsec,
+          finish_timespec_seconds: flush.finish.tv_sec,
+          finish_timespec_nanoseconds: flush.finish.tv_nsec,
+
+          pprof_file_name: flush.pprof_file_name,
+          pprof_data: flush.pprof_data,
+          code_provenance_file_name: flush.code_provenance_file_name,
+          code_provenance_data: flush.code_provenance_data,
+
+          tags_as_array: flush.tags_as_array,
+          internal_metadata_json: flush.internal_metadata_json,
+
+          info_json: flush.info_json
+        )
+
+        if status == :ok
+          if (200..299).cover?(result)
+            Datadog.logger.debug('Successfully reported profiling data')
+            true
+          else
+            Datadog.logger.error(
+              "Failed to report profiling data (#{config_without_api_key}): " \
+              "server returned unexpected HTTP #{result} status code"
+            )
+            false
+          end
+        else
+          Datadog.logger.error("Failed to report profiling data (#{config_without_api_key}): #{result}")
+          false
+        end
+      end
+
+      # Used to log soft failures in `ddog_Vec_tag_push` (e.g. we still report the profile in these cases)
+      # Called from native code
+      def self.log_failure_to_process_tag(failure_details)
+        Datadog.logger.warn("Failed to add tag to profiling request: #{failure_details}")
+      end
+
+      private
+
+      def base_url_from(agent_settings)
+        case agent_settings.adapter
+        when Datadog::Core::Transport::Ext::HTTP::ADAPTER
+          "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+        when Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+          "unix://#{agent_settings.uds_path}"
+        else
+          raise ArgumentError, "Unexpected adapter: #{agent_settings.adapter}"
+        end
+      end
+
+      def validate_agent_settings(agent_settings)
+        supported_adapters = [Datadog::Core::Transport::Ext::HTTP::ADAPTER,
+                              Datadog::Core::Transport::Ext::UnixSocket::ADAPTER]
+        unless supported_adapters.include?(agent_settings.adapter)
+          raise ArgumentError,
+            "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
+                        ' is not supported'
+        end
+
+        if agent_settings.deprecated_for_removal_transport_configuration_proc
+          Datadog.logger.warn(
+            'Ignoring custom c.tracing.transport_options setting as the profiler does not support it.'
+          )
+        end
+      end
+
+      def agentless?(site, api_key)
+        site && api_key && Core::Environment::VariableHelpers.env_to_bool(Profiling::Ext::ENV_AGENTLESS, false)
+      end
+
+      def validate_exporter(exporter_configuration)
+        self.class._native_validate_exporter(exporter_configuration)
+      end
+
+      def do_export(
+        exporter_configuration:,
+        upload_timeout_milliseconds:,
+        start_timespec_seconds:,
+        start_timespec_nanoseconds:,
+        finish_timespec_seconds:,
+        finish_timespec_nanoseconds:,
+        pprof_file_name:,
+        pprof_data:,
+        code_provenance_file_name:,
+        code_provenance_data:,
+        tags_as_array:,
+        internal_metadata_json:,
+        info_json:
+      )
+        self.class._native_do_export(
+          exporter_configuration,
+          upload_timeout_milliseconds,
+          start_timespec_seconds,
+          start_timespec_nanoseconds,
+          finish_timespec_seconds,
+          finish_timespec_nanoseconds,
+          pprof_file_name,
+          pprof_data,
+          code_provenance_file_name,
+          code_provenance_data,
+          tags_as_array,
+          internal_metadata_json,
+          info_json,
+        )
+      end
+
+      def config_without_api_key
+        [@exporter_configuration[0..1]].to_h
+      end
+    end
+  end
+end

data/lib/datadog/profiling/load_native_extension.rb

@@ -0,0 +1,39 @@
+# This file is used to load the profiling native extension. It works in two steps:
+#
+# 1. Load the datadog_profiling_loader extension. This extension will be used to load the actual extension, but in
+#    a special way that avoids exposing native-level code symbols. See `datadog_profiling_loader.c` for more details.
+#
+# 2. Use the Datadog::Profiling::Loader exposed by the datadog_profiling_loader extension to load the actual
+#    profiling native extension.
+#
+# All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
+# the first time it gets required, to avoid any issues with the native extension being initialized more than once.
+
+begin
+  require "datadog_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+rescue LoadError => e
+  raise LoadError,
+    'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
+    "(Details: #{e.message})"
+end
+
+extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+full_file_path = "#{__dir__}/../../#{file_name}"
+
+unless File.exist?(full_file_path)
+  extension_dir = Gem.loaded_specs['ddtrace'].extension_dir
+  candidate_path = "#{extension_dir}/#{file_name}"
+  if File.exist?(candidate_path)
+    full_file_path = candidate_path
+  else # rubocop:disable Style/EmptyElse
+    # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
+    # down anyway.
+  end
+end
+
+init_function_name = "Init_#{extension_name.split('.').first}"
+
+status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)
+
+raise "Failure to load #{extension_name} due to #{result}" if status == :error

data/lib/datadog/profiling/native_extension.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    # This module contains classes and methods which are implemented using native code in the
+    # ext/datadog_profiling_native_extension folder, as well as some Ruby-level utilities that don't make sense to
+    # write using C
+    module NativeExtension
+      private_class_method def self.working?
+        native_working?
+      end
+
+      unless singleton_class.private_method_defined?(:native_working?)
+        private_class_method def self.native_working?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/preload.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative '../../ddtrace'
+
+Datadog::Profiling.start_if_enabled

data/lib/datadog/profiling/profiler.rb

@@ -0,0 +1,62 @@
+module Datadog
+  module Profiling
+    # Profiling entry point, which coordinates the worker and scheduler threads
+    class Profiler
+      include Datadog::Core::Utils::Forking
+
+      private
+
+      attr_reader :worker, :scheduler
+
+      public
+
+      def initialize(worker:, scheduler:)
+        @worker = worker
+        @scheduler = scheduler
+      end
+
+      def start
+        after_fork! do
+          worker.reset_after_fork
+          scheduler.reset_after_fork
+        end
+
+        worker.start(on_failure_proc: proc { component_failed(:worker) })
+        scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
+      end
+
+      def shutdown!
+        Datadog.logger.debug('Shutting down profiler')
+
+        stop_worker
+        stop_scheduler
+      end
+
+      private
+
+      def stop_worker
+        worker.stop
+      end
+
+      def stop_scheduler
+        scheduler.enabled = false
+        scheduler.stop(true)
+      end
+
+      def component_failed(failed_component)
+        Datadog.logger.warn(
+          "Detected issue with profiler (#{failed_component} component), stopping profiling. " \
+          'See previous log messages for details.'
+        )
+
+        if failed_component == :worker
+          stop_scheduler
+        elsif failed_component == :scheduler
+          stop_worker
+        else
+          raise ArgumentError, "Unexpected failed_component: #{failed_component.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/scheduler.rb

@@ -0,0 +1,135 @@
+require_relative '../core/utils/time'
+
+require_relative '../core/worker'
+require_relative '../core/workers/polling'
+
+module Datadog
+  module Profiling
+    # Periodically (every interval, 60 seconds by default) takes a profile from the `Exporter` and reports it using the
+    # configured transport. Runs on its own background thread.
+    class Scheduler < Core::Worker
+      include Core::Workers::Polling
+
+      MINIMUM_INTERVAL_SECONDS = 0
+
+      # We sleep for at most this duration seconds before reporting data to avoid multi-process applications all
+      # reporting profiles at the exact same time
+      DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS = 3
+
+      private
+
+      attr_reader \
+        :exporter,
+        :transport
+
+      public
+
+      def initialize(
+        exporter:,
+        transport:,
+        interval:, fork_policy: Core::Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default, # seconds
+        enabled: true
+      )
+        @exporter = exporter
+        @transport = transport
+
+        # Workers::Async::Thread settings
+        self.fork_policy = fork_policy
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = interval
+
+        # Workers::Polling settings
+        self.enabled = enabled
+      end
+
+      def start(on_failure_proc: nil)
+        perform(on_failure_proc)
+      end
+
+      def perform(on_failure_proc)
+        begin
+          # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
+          # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
+          # successfully finish in the strict timeout.
+          # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
+          interrupted = true
+
+          flush_and_wait
+          interrupted = false
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          Datadog.logger.warn(
+            'Profiling::Scheduler thread error. ' \
+            "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+          on_failure_proc&.call
+          raise
+        ensure
+          Datadog.logger.debug('#flush was interrupted or failed before it could complete') if interrupted
+        end
+      end
+
+      # Configure Workers::IntervalLoop to not report immediately when scheduler starts
+      #
+      # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
+      # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
+      # application just started but this thread took a bit longer so there's already profiling data in the exporter.
+      def loop_wait_before_first_iteration?
+        true
+      end
+
+      def work_pending?
+        exporter.can_flush?
+      end
+
+      def reset_after_fork
+        exporter.reset_after_fork
+      end
+
+      private
+
+      def flush_and_wait
+        run_time = Core::Utils::Time.measure do
+          flush_events
+        end
+
+        # Update wait time to try to wake consistently on time.
+        # Don't drop below the minimum interval.
+        self.loop_wait_time = [loop_base_interval - run_time, MINIMUM_INTERVAL_SECONDS].max
+      end
+
+      def flush_events
+        # Collect data to be exported
+        flush = exporter.flush
+
+        return false unless flush
+
+        # Sleep for a bit to cause misalignment between profilers in multi-process applications
+        #
+        # When not being run in a loop, it means the scheduler has not been started or was stopped, and thus
+        # a) it's being shut down (and is trying to report the last profile)
+        # b) it's being run as a one-shot, usually in a test
+        # ...so in those cases we don't sleep
+        #
+        # During PR review (https://github.com/DataDog/dd-trace-rb/pull/1807) we discussed the possible alternative of
+        # just sleeping before starting the scheduler loop. We ended up not going with that option to avoid the first
+        # profile containing up to interval + DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS instead of the
+        # usual interval seconds.
+        if run_loop?
+          jitter_seconds = rand * DEFAULT_FLUSH_JITTER_MAXIMUM_SECONDS # floating point number between (0.0...maximum)
+          sleep(jitter_seconds)
+        end
+
+        begin
+          transport.export(flush)
+        rescue StandardError => e
+          Datadog.logger.error(
+            "Unable to report profile. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+        end
+
+        true
+      end
+    end
+  end
+end