datadog 2.0.0.beta1 → 2.0.0.rc1

data/lib/datadog/profiling/component.rb

@@ -72,8 +72,10 @@ module Datadog
         exporter = build_profiler_exporter(settings, recorder, worker, internal_metadata: internal_metadata)
         transport = build_profiler_transport(settings, agent_settings)
         scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport, interval: upload_period_seconds)
+        crashtracker = build_crashtracker(settings, transport)
+        profiler = Profiling::Profiler.new(worker: worker, scheduler: scheduler, optional_crashtracker: crashtracker)
 
-        [Profiling::Profiler.new(worker: worker, scheduler: scheduler), { profiling_enabled: true }]
+        [profiler, { profiling_enabled: true }]
       end
 
       private_class_method def self.build_thread_context_collector(settings, recorder, optional_tracer, timeline_enabled)
@@ -110,20 +112,53 @@ module Datadog
           )
       end
 
+      private_class_method def self.build_crashtracker(settings, transport)
+        return unless settings.profiling.advanced.experimental_crash_tracking_enabled
+
+        # By default, the transport is an instance of HttpTransport, which validates the configuration and makes
+        # it available for us to use here.
+        # But we support overriding the transport with a user-specific one, which may e.g. write stuff to a file,
+        # and thus can't really provide a valid configuration to talk to a Datadog agent. Thus, in this situation,
+        # we can't use the crashtracker, even if enabled.
+        unless transport.respond_to?(:exporter_configuration)
+          Datadog.logger.warn(
+            'Cannot enable profiling crash tracking as a custom settings.profiling.exporter.transport is configured'
+          )
+          return
+        end
+
+        Datadog::Profiling::Crashtracker.new(
+          exporter_configuration: transport.exporter_configuration,
+          tags: Datadog::Profiling::TagBuilder.call(settings: settings),
+          upload_timeout_seconds: settings.profiling.upload.timeout_seconds,
+        )
+      end
+
       private_class_method def self.enable_gc_profiling?(settings)
-        # See comments on the setting definition for more context on why it exists.
-        if settings.profiling.advanced.force_enable_gc_profiling
-          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
-            Datadog.logger.debug(
-              'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
-              'with this option as profiles will be incomplete.'
-            )
-          end
+        return false unless settings.profiling.advanced.gc_enabled
 
-          true
-        else
-          false
+        # SEVERE - Only with Ractors
+        # On Ruby versions 3.0 (all), 3.1.0 to 3.1.3, and 3.2.0 to 3.2.2 gc profiling can trigger a VM bug
+        # that causes a segmentation fault during garbage collection of Ractors
+        # (https://bugs.ruby-lang.org/issues/18464). We don't allow enabling gc profiling on such Rubies.
+        # This bug is fixed on Ruby versions 3.1.4, 3.2.3 and 3.3.0.
+        if RUBY_VERSION.start_with?('3.0.') ||
+            (RUBY_VERSION.start_with?('3.1.') && RUBY_VERSION < '3.1.4') ||
+            (RUBY_VERSION.start_with?('3.2.') && RUBY_VERSION < '3.2.3')
+          Datadog.logger.warn(
+            "Current Ruby version (#{RUBY_VERSION}) has a VM bug where enabling GC profiling would cause "\
+            'crashes (https://bugs.ruby-lang.org/issues/18464). GC profiling has been disabled.'
+          )
+          return false
+        elsif RUBY_VERSION.start_with?('3.')
+          Datadog.logger.debug(
+            'In all known versions of Ruby 3.x, using Ractors may result in GC profiling unexpectedly ' \
+            'stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your ' \
+            'application stability or performance. This does not happen if Ractors are not used.'
+          )
         end
+
+        true
       end
 
       private_class_method def self.get_heap_sample_every(settings)
@@ -135,10 +170,7 @@ module Datadog
       end
 
       private_class_method def self.enable_allocation_profiling?(settings)
-        unless settings.profiling.allocation_enabled
-          # Allocation profiling disabled, short-circuit out
-          return false
-        end
+        return false unless settings.profiling.allocation_enabled
 
         # Allocation sampling is safe and supported on Ruby 2.x, but has a few caveats on Ruby 3.x.
 
@@ -235,7 +267,7 @@ module Datadog
 
       private_class_method def self.no_signals_workaround_enabled?(settings) # rubocop:disable Metrics/MethodLength
         setting_value = settings.profiling.advanced.no_signals_workaround_enabled
-        legacy_ruby_that_should_use_workaround = RUBY_VERSION.start_with?('2.3.', '2.4.', '2.5.')
+        legacy_ruby_that_should_use_workaround = RUBY_VERSION.start_with?('2.5.')
 
         unless [true, false, :auto].include?(setting_value)
           Datadog.logger.error(

data/lib/datadog/profiling/crashtracker.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'libdatadog'
+
+module Datadog
+  module Profiling
+    # Used to report Ruby VM crashes.
+    # The interesting bits are implemented as native code and using libdatadog.
+    #
+    # NOTE: The crashtracker native state is a singleton; so even if you create multiple instances of `Crashtracker`
+    # and start them, it only works as "last writer wins". Same for stop -- there's only one state, so calling stop
+    # on it will stop the crash tracker, regardless of which instance started it.
+    #
+    # Methods prefixed with _native_ are implemented in `crashtracker.c`
+    class Crashtracker
+      private
+
+      attr_reader \
+        :exporter_configuration,
+        :tags_as_array,
+        :path_to_crashtracking_receiver_binary,
+        :ld_library_path,
+        :upload_timeout_seconds
+
+      public
+
+      def initialize(
+        exporter_configuration:,
+        tags:,
+        upload_timeout_seconds:,
+        path_to_crashtracking_receiver_binary: Libdatadog.path_to_crashtracking_receiver_binary,
+        ld_library_path: Libdatadog.ld_library_path
+      )
+        @exporter_configuration = exporter_configuration
+        @tags_as_array = tags.to_a
+        @upload_timeout_seconds = upload_timeout_seconds
+        @path_to_crashtracking_receiver_binary = path_to_crashtracking_receiver_binary
+        @ld_library_path = ld_library_path
+      end
+
+      def start
+        start_or_update_on_fork(action: :start)
+      end
+
+      def reset_after_fork
+        start_or_update_on_fork(action: :update_on_fork)
+      end
+
+      def stop
+        begin
+          self.class._native_stop
+          Datadog.logger.debug('Crash tracking stopped successfully')
+        rescue => e
+          Datadog.logger.error("Failed to stop crash tracking: #{e.message}")
+        end
+      end
+
+      private
+
+      def start_or_update_on_fork(action:)
+        unless path_to_crashtracking_receiver_binary
+          Datadog.logger.warn(
+            "Cannot #{action} profiling crash tracking as no path_to_crashtracking_receiver_binary was found"
+          )
+          return
+        end
+
+        unless ld_library_path
+          Datadog.logger.warn(
+            "Cannot #{action} profiling crash tracking as no ld_library_path was found"
+          )
+          return
+        end
+
+        begin
+          self.class._native_start_or_update_on_fork(
+            action: action,
+            exporter_configuration: exporter_configuration,
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            ld_library_path: ld_library_path,
+            tags_as_array: tags_as_array,
+            upload_timeout_seconds: Integer(upload_timeout_seconds),
+          )
+          Datadog.logger.debug("Crash tracking #{action} successful")
+        rescue => e
+          Datadog.logger.error("Failed to #{action} crash tracking: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/exporter.rb

@@ -54,10 +54,11 @@ module Datadog
 
       def flush
         worker_stats = @worker.stats_and_reset_not_thread_safe
-        start, finish, compressed_pprof = pprof_recorder.serialize
-        @last_flush_finish_at = finish
+        serialization_result = pprof_recorder.serialize
+        return if serialization_result.nil?
 
-        return if compressed_pprof.nil? # We don't want to report empty profiles
+        start, finish, compressed_pprof, profile_stats = serialization_result
+        @last_flush_finish_at = finish
 
         if duration_below_threshold?(start, finish)
           Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
@@ -77,6 +78,8 @@ module Datadog
           internal_metadata: internal_metadata.merge(
             {
               worker_stats: worker_stats,
+              profile_stats: profile_stats,
+              recorder_stats: pprof_recorder.stats,
               gc: GC.stat,
             }
           ),

data/lib/datadog/profiling/http_transport.rb

@@ -7,6 +7,8 @@ module Datadog
     # Used to report profiling data to Datadog.
     # Methods prefixed with _native_ are implemented in `http_transport.c`
     class HttpTransport
+      attr_reader :exporter_configuration
+
       def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
         @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
 
@@ -14,19 +16,19 @@ module Datadog
 
         @exporter_configuration =
           if agentless?(site, api_key)
-            [:agentless, site, api_key]
+            [:agentless, site, api_key].freeze
           else
-            [:agent, base_url_from(agent_settings)]
+            [:agent, base_url_from(agent_settings)].freeze
           end
 
-        status, result = validate_exporter(@exporter_configuration)
+        status, result = validate_exporter(exporter_configuration)
 
         raise(ArgumentError, "Failed to initialize transport: #{result}") if status == :error
       end
 
       def export(flush)
         status, result = do_export(
-          exporter_configuration: @exporter_configuration,
+          exporter_configuration: exporter_configuration,
           upload_timeout_milliseconds: @upload_timeout_milliseconds,
 
           # why "timespec"?
@@ -66,12 +68,6 @@ module Datadog
         end
       end
 
-      # Used to log soft failures in `ddog_Vec_tag_push` (e.g. we still report the profile in these cases)
-      # Called from native code
-      def self.log_failure_to_process_tag(failure_details)
-        Datadog.logger.warn("Failed to add tag to profiling request: #{failure_details}")
-      end
-
       private
 
       def base_url_from(agent_settings)
@@ -136,7 +132,7 @@ module Datadog
       end
 
       def config_without_api_key
-        [@exporter_configuration[0..1]].to_h
+        [exporter_configuration[0..1]].to_h
       end
     end
   end

data/lib/datadog/profiling/load_native_extension.rb

@@ -20,7 +20,20 @@ rescue LoadError => e
 end
 
 extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
-full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+full_file_path = "#{__dir__}/../../#{file_name}"
+
+unless File.exist?(full_file_path)
+  extension_dir = Gem.loaded_specs['datadog'].extension_dir
+  candidate_path = "#{extension_dir}/#{file_name}"
+  if File.exist?(candidate_path)
+    full_file_path = candidate_path
+  else # rubocop:disable Style/EmptyElse
+    # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
+    # down anyway.
+  end
+end
+
 init_function_name = "Init_#{extension_name.split('.').first}"
 
 status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)

data/lib/datadog/profiling/profiler.rb

@@ -8,21 +8,24 @@ module Datadog
 
       private
 
-      attr_reader :worker, :scheduler
+      attr_reader :worker, :scheduler, :optional_crashtracker
 
       public
 
-      def initialize(worker:, scheduler:)
+      def initialize(worker:, scheduler:, optional_crashtracker:)
         @worker = worker
         @scheduler = scheduler
+        @optional_crashtracker = optional_crashtracker
       end
 
       def start
         after_fork! do
+          optional_crashtracker.reset_after_fork if optional_crashtracker
           worker.reset_after_fork
           scheduler.reset_after_fork
         end
 
+        optional_crashtracker.start if optional_crashtracker
         worker.start(on_failure_proc: proc { component_failed(:worker) })
         scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
       end
@@ -32,6 +35,7 @@ module Datadog
 
         stop_worker
         stop_scheduler
+        optional_crashtracker.stop if optional_crashtracker
       end
 
       private
@@ -51,6 +55,9 @@ module Datadog
           'See previous log messages for details.'
         )
 
+        # We explicitly not stop the crash tracker in this situation, under the assumption that, if a component failed,
+        # we're operating in a degraded state and crash tracking may still be helpful.
+
         if failed_component == :worker
           stop_scheduler
         elsif failed_component == :scheduler

data/lib/datadog/profiling/stack_recorder.rb

@@ -33,11 +33,11 @@ module Datadog
         status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
-          start, finish, encoded_pprof = result
+          start, finish, encoded_pprof, profile_stats = result
 
           Datadog.logger.debug { "Encoded profile covering #{start.iso8601} to #{finish.iso8601}" }
 
-          [start, finish, encoded_pprof]
+          [start, finish, encoded_pprof, profile_stats]
         else
           error_message = result
 
@@ -64,6 +64,10 @@ module Datadog
       def reset_after_fork
         self.class._native_reset_after_fork(self)
       end
+
+      def stats
+        self.class._native_stats(self)
+      end
     end
   end
 end

data/lib/datadog/profiling.rb

@@ -65,6 +65,17 @@ module Datadog
       !!(profiler.send(:scheduler).running? if profiler)
     end
 
+    def self.wait_until_running(timeout_seconds: 5)
+      profiler = Datadog.send(:components).profiler
+      if profiler
+        # Use .send(...) to avoid exposing the attr_reader as an API to the outside
+        worker = profiler.send(:worker)
+        worker.wait_until_running(timeout_seconds: timeout_seconds)
+      else
+        raise 'Profiler not enabled or available'
+      end
+    end
+
     private_class_method def self.replace_noop_allocation_count
       def self.allocation_count # rubocop:disable Lint/NestedMethodDefinition (On purpose!)
         Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
@@ -132,6 +143,7 @@ module Datadog
       require_relative 'profiling/collectors/idle_sampling_helper'
       require_relative 'profiling/collectors/stack'
       require_relative 'profiling/collectors/thread_context'
+      require_relative 'profiling/crashtracker'
       require_relative 'profiling/stack_recorder'
       require_relative 'profiling/exporter'
       require_relative 'profiling/flush'

data/lib/datadog/tracing/component.rb

@@ -124,9 +124,13 @@ module Datadog
       end
 
       WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK = lambda do |_, responses|
-        Tracing::Diagnostics::EnvironmentLogger.collect_and_log!(responses: responses)
+        WRITER_RECORD_ENVIRONMENT_INFORMATION_ONLY_ONCE.run do
+          Tracing::Diagnostics::EnvironmentLogger.collect_and_log!(responses: responses)
+        end
       end
 
+      WRITER_RECORD_ENVIRONMENT_INFORMATION_ONLY_ONCE = Core::Utils::OnlyOnce.new
+
       # Create new lambda for writer callback,
       # capture the current sampler in the callback closure.
       def writer_update_priority_sampler_rates_callback(sampler)

data/lib/datadog/tracing/configuration/dynamic.rb

@@ -53,8 +53,46 @@ module Datadog
           end
         end
 
+        # Dynamic configuration for `DD_TRACE_SAMPLING_RULES`.
+        class TracingSamplingRules < SimpleOption
+          def initialize
+            super('tracing_sampling_rules', 'DD_TRACE_SAMPLING_RULES', :rules)
+          end
+
+          # Ensures sampler is rebuilt and new configuration is applied
+          def call(tracing_sampling_rules)
+            # Modify the remote configuration value that it matches the
+            # local environment variable it configures.
+            if tracing_sampling_rules
+              tracing_sampling_rules.each do |rule|
+                next unless (tags = rule['tags'])
+
+                # Tag maps come in as arrays of 'key' and `value_glob`.
+                # We need to convert them into a hash for local use.
+                tag_array = tags.map! do |tag|
+                  [tag['key'], tag['value_glob']]
+                end
+
+                rule['tags'] = tag_array.to_h
+              end
+
+              # The configuration is stored as JSON, so we need to convert it back
+              tracing_sampling_rules = tracing_sampling_rules.to_json
+            end
+
+            super(tracing_sampling_rules)
+            Datadog.send(:components).reconfigure_live_sampler
+          end
+
+          protected
+
+          def configuration_object
+            Datadog.configuration.tracing.sampling
+          end
+        end
+
         # List of all tracing dynamic configuration options supported.
-        OPTIONS = [LogInjectionEnabled, TracingHeaderTags, TracingSamplingRate].map do |option_class|
+        OPTIONS = [LogInjectionEnabled, TracingHeaderTags, TracingSamplingRate, TracingSamplingRules].map do |option_class|
           option = option_class.new
           [option.name, option.env_var, option]
         end

data/lib/datadog/tracing/configuration/settings.rb

@@ -274,6 +274,7 @@ module Datadog
                 # @return [String,nil]
                 # @public_api
                 option :rules do |o|
+                  o.type :string, nilable: true
                   o.default { ENV.fetch(Configuration::Ext::Sampling::ENV_RULES, nil) }
                 end
 

data/lib/datadog/tracing/contrib/action_pack/integration.rb

@@ -3,7 +3,7 @@
 require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative '../integration'
-require_relative '../rails/integration'
+require_relative '../rails/ext'
 require_relative '../rails/utils'
 
 module Datadog

data/lib/datadog/tracing/contrib/action_view/integration.rb

@@ -3,7 +3,7 @@
 require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative '../integration'
-require_relative '../rails/integration'
+require_relative '../rails/ext'
 require_relative '../rails/utils'
 
 module Datadog

data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb

@@ -32,6 +32,7 @@ module Datadog
           # based on addition order (`#add`).
           class Resolver < Contrib::Configuration::Resolver
             prepend MakaraResolver
+            prepend Contrib::Configuration::CachingResolver
 
             def initialize(active_record_configuration = nil)
               super()

data/lib/datadog/tracing/contrib/active_record/integration.rb

@@ -4,8 +4,9 @@ require_relative 'configuration/resolver'
 require_relative 'configuration/settings'
 require_relative 'events'
 require_relative 'patcher'
+require_relative '../component'
 require_relative '../integration'
-require_relative '../rails/integration'
+require_relative '../rails/ext'
 require_relative '../rails/utils'
 
 module Datadog
@@ -50,6 +51,15 @@ module Datadog
           def resolver
             @resolver ||= Configuration::Resolver.new
           end
+
+          def reset_resolver_cache
+            @resolver&.reset_cache
+          end
+
+          Contrib::Component.register('activerecord') do |_config|
+            # Ensure resolver cache is reset on configuration change
+            Datadog.configuration.tracing.fetch_integration(:active_record).reset_resolver_cache
+          end
         end
       end
     end

data/lib/datadog/tracing/contrib/active_support/integration.rb

@@ -4,7 +4,7 @@ require_relative '../integration'
 require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative 'cache/redis'
-require_relative '../rails/integration'
+require_relative '../rails/ext'
 require_relative '../rails/utils'
 
 module Datadog

data/lib/datadog/tracing/contrib/configuration/resolver.rb

@@ -79,6 +79,49 @@ module Datadog
             matcher
           end
         end
+
+        # The {CachingResolver} is a mixin that provides caching functionality to the {Resolver} class.
+        # This is useful when {Resolver#resolve} values that are expensive to compute.
+        # This is a size-limited, FIFO cache.
+        #
+        # @example
+        #   class MyResolver < Datadog::Tracing::Contrib::Configuration::Resolver
+        #     prepend Datadog::Tracing::Contrib::Configuration::CachingResolver
+        #     # ...
+        #   end
+        module CachingResolver
+          # @param [Integer] cache_limit maximum number of entries to cache
+          def initialize(*args, cache_limit: 200)
+            super(*args)
+
+            @cache_limit = cache_limit
+            @cache = {}
+          end
+
+          # (see Resolver#resolve)
+          def resolve(value)
+            if @cache.key?(value)
+              @cache[value]
+            else
+              if @cache.size >= @cache_limit
+                @cache.shift # Remove the oldest entry if cache is full
+              end
+
+              @cache[value] = super
+            end
+          end
+
+          # (see Resolver#add)
+          def add(matcher, value)
+            reset_cache # Bust the cache when a new matcher is added
+            super
+          end
+
+          # Clears the internal cache.
+          def reset_cache
+            @cache.clear
+          end
+        end
       end
     end
   end

data/lib/datadog/tracing/contrib/grape/endpoint.rb

@@ -92,8 +92,7 @@ module Datadog
                 # Measure service stats
                 Contrib::Analytics.set_measured(span)
 
-                # catch thrown exceptions
-                handle_error(span, payload[:exception_object]) if payload[:exception_object]
+                handle_error_and_status_code(span, endpoint, payload)
 
                 # override the current span with this notification values
                 span.set_tag(Ext::TAG_ROUTE_ENDPOINT, api_view) unless api_view.nil?
@@ -110,6 +109,30 @@ module Datadog
               Datadog.logger.error(e.message)
             end
 
+            # Status code resolution is tied to the exception handling
+            def handle_error_and_status_code(span, endpoint, payload)
+              status = nil
+
+              # Handle exceptions and status code
+              if (exception_object = payload[:exception_object])
+                # If the exception is not an internal Grape error, we won't have a status code at this point.
+                status = exception_object.status if exception_object.respond_to?(:status)
+
+                handle_error(span, exception_object, status)
+              else
+                # Status code is unreliable in `endpoint_run.grape` if there was an exception.
+                # Only after `Grape::Middleware::Error#run_rescue_handler` that the error status code of a request with a
+                # Ruby exception error is resolved. But that handler is called further down the Grape middleware stack.
+                # Rack span will then be the most reliable source for status codes.
+                # DEV: As a corollary, instrumenting Grape without Rack will provide incomplete
+                # DEV: status quote information.
+                status = endpoint.status
+                span.set_error(endpoint) if error_status_codes.include?(status)
+              end
+
+              span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, status) if status
+            end
+
             def endpoint_start_render(*)
               return if Thread.current[KEY_RENDER]
               return unless enabled?
@@ -193,9 +216,10 @@ module Datadog
 
             private
 
-            def handle_error(span, exception)
-              if exception.respond_to?('status')
-                span.set_error(exception) if error_status_codes.include?(exception.status)
+            def handle_error(span, exception, status = nil)
+              status ||= (exception.status if exception.respond_to?(:status))
+              if status
+                span.set_error(exception) if error_status_codes.include?(status)
               else
                 on_error.call(span, exception)
               end
@@ -240,6 +264,20 @@ module Datadog
               datadog_configuration[:analytics_sample_rate]
             end
 
+            def exception_is_error?(exception)
+              return false unless exception
+              return true unless exception.respond_to?(:status)
+
+              error_status?(status.exception)
+            end
+
+            def error_status?(status)
+              matcher = datadog_configuration[:error_statuses]
+              return true unless matcher
+
+              matcher.include?(status) if matcher
+            end
+
             def enabled?
               Datadog.configuration.tracing.enabled && \
                 datadog_configuration[:enabled] == true

data/lib/datadog/tracing/contrib/trilogy/instrumentation.rb

@@ -24,7 +24,7 @@ module Datadog
 
               Tracing.trace(Ext::SPAN_QUERY, service: service) do |span, trace_op|
                 span.resource = sql
-                span.span_type = Tracing::Metadata::Ext::SQL::TYPE
+                span.type = Tracing::Metadata::Ext::SQL::TYPE
 
                 if datadog_configuration[:peer_service]
                   span.set_tag(