datadog 2.2.0 → 2.4.0

data/lib/datadog/appsec/processor.rb

@@ -73,13 +73,15 @@ module Datadog
 
       attr_reader :diagnostics, :addresses
 
-      def initialize(ruleset:)
+      def initialize(ruleset:, telemetry:)
         @diagnostics = nil
         @addresses = []
         settings = Datadog.configuration.appsec
+        @telemetry = telemetry
 
-        unless load_libddwaf && create_waf_handle(settings, ruleset)
-          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+        # TODO: Refactor to make it easier to test
+        unless require_libddwaf && libddwaf_provides_waf? && create_waf_handle(settings, ruleset)
+          Datadog.logger.warn('AppSec is disabled, see logged errors above')
         end
       end
 
@@ -97,8 +99,27 @@ module Datadog
 
       private
 
-      def load_libddwaf
-        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      # libddwaf raises a LoadError on unsupported platforms; it may at some
+      # point succeed in being required yet not provide a specific needed feature.
+      def require_libddwaf
+        Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+        require 'libddwaf'
+
+        true
+      rescue LoadError => e
+        Datadog.logger.error do
+          'libddwaf failed to load,' \
+            "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+        end
+        @telemetry.report(e, description: 'libddwaf failed to load')
+
+        false
+      end
+
+      # check whether libddwaf is required *and* able to provide the needed feature
+      def libddwaf_provides_waf?
+        defined?(Datadog::AppSec::WAF) ? true : false
       end
 
       def create_waf_handle(settings, ruleset)
@@ -119,6 +140,7 @@ module Datadog
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
+        @telemetry.report(e, description: 'libddwaf failed to initialize')
 
         @diagnostics = e.diagnostics if e.diagnostics
 
@@ -127,44 +149,21 @@ module Datadog
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
+        @telemetry.report(e, description: 'libddwaf failed to initialize')
 
         false
       end
 
-      class << self
-        # check whether libddwaf is required *and* able to provide the needed feature
-        def libddwaf_provides_waf?
-          defined?(Datadog::AppSec::WAF) ? true : false
-        end
-
-        # libddwaf raises a LoadError on unsupported platforms; it may at some
-        # point succeed in being required yet not provide a specific needed feature.
-        def require_libddwaf
-          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
-
-          require 'libddwaf'
-
-          true
-        rescue LoadError => e
-          Datadog.logger.error do
-            'libddwaf failed to load,' \
-              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
-          end
-
-          false
-        end
-
-        def libddwaf_spec
-          Gem.loaded_specs['libddwaf']
-        end
-
-        def libddwaf_platform
-          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+      def libddwaf_platform
+        if Gem.loaded_specs['libddwaf']
+          Gem.loaded_specs['libddwaf'].platform.to_s
+        else
+          'unknown'
         end
+      end
 
-        def ruby_platforms
-          Gem.platforms.map(&:to_s)
-        end
+      def ruby_platforms
+        Gem.platforms.map(&:to_s)
       end
     end
   end

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,60 +1,45 @@
 # frozen_string_literal: true
 
+require_relative '../core/rate_limiter'
+
 module Datadog
   module AppSec
-    # Simple per-thread rate limiter
-    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    # Per-thread rate limiter based on token bucket rate limiter.
+    #
+    # Since AppSec marks sampling to keep on a security event, this limits
+    # the flood of egress traces involving AppSec
     class RateLimiter
-      def initialize(rate)
-        @rate = rate
-        @timestamps = []
-      end
-
-      def limit
-        now = Time.now.to_f
-
-        loop do
-          oldest = @timestamps.first
-
-          break if oldest.nil? || now - oldest < 1
-
-          @timestamps.shift
-        end
-
-        @timestamps << now
-
-        if (count = @timestamps.count) <= @rate
-          yield
-        else
-          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
-        end
-      end
+      THREAD_KEY = :datadog_security_appsec_rate_limiter
 
       class << self
-        def limit(name, &block)
-          rate_limiter(name).limit(&block)
+        def thread_local
+          rate_limiter = Thread.current.thread_variable_get(THREAD_KEY)
+          return rate_limiter unless rate_limiter.nil?
+
+          Thread.current.thread_variable_set(THREAD_KEY, new(trace_rate_limit))
         end
 
         # reset a rate limiter: used for testing
-        def reset!(name)
-          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        def reset!
+          Thread.current.thread_variable_set(THREAD_KEY, nil)
         end
 
-        protected
-
-        def rate_limiter(name)
-          case name
-          when :traces
-            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
-          else
-            raise "unsupported rate limiter: #{name.inspect}"
-          end
-        end
+        private
 
         def trace_rate_limit
           Datadog.configuration.appsec.trace_rate_limit
         end
       end
+
+      def initialize(rate)
+        @rate_limiter = Core::TokenBucket.new(rate)
+      end
+
+      def limit
+        return yield if @rate_limiter.allow?
+
+        Datadog.logger.debug { "Rate limit hit: #{@rate_limiter.current_window_rate} AppSec traces/second" }
+      end
     end
   end
 end

data/lib/datadog/appsec/remote.rb

@@ -53,7 +53,7 @@ module Datadog
         end
 
         # rubocop:disable Metrics/MethodLength
-        def receivers
+        def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
@@ -86,7 +86,10 @@ module Datadog
             end
 
             if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(ruleset: Datadog.configuration.appsec.ruleset)
+              settings_rules = AppSec::Processor::RuleLoader.load_rules(
+                telemetry: telemetry,
+                ruleset: Datadog.configuration.appsec.ruleset
+              )
 
               raise NoRulesError, 'no default rules available' unless settings_rules
 
@@ -99,9 +102,10 @@ module Datadog
               overrides: overrides,
               exclusions: exclusions,
               custom_rules: custom_rules,
+              telemetry: telemetry
             )
 
-            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions)
+            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
           end
 
           [receiver]

data/lib/datadog/appsec/response.rb

@@ -36,7 +36,7 @@ module Datadog
             # I rather use break to stop the execution
             next if configured_response
 
-            action_configuration = AppSec::Processor::Actions.fecth_configuration(action)
+            action_configuration = AppSec::Processor::Actions.fetch_configuration(action)
             next unless action_configuration
 
             configured_response = case action_configuration['type']
@@ -50,6 +50,20 @@ module Datadog
           configured_response || default_response(env)
         end
 
+        def graphql_response(gateway_multiplex)
+          multiplex_return = []
+          gateway_multiplex.queries.each do |query|
+            # This method is only called in places where GraphQL-Ruby is already required
+            query_result = ::GraphQL::Query::Result.new(
+              query: query,
+              values: JSON.parse(content('application/json'))
+            )
+            multiplex_return << query_result
+          end
+
+          multiplex_return
+        end
+
         private
 
         def default_response(env)

data/lib/datadog/appsec.rb

@@ -23,12 +23,12 @@ module Datadog
         appsec_component.processor if appsec_component
       end
 
-      def reconfigure(ruleset:, actions:)
+      def reconfigure(ruleset:, actions:, telemetry:)
         appsec_component = components.appsec
 
         return unless appsec_component
 
-        appsec_component.reconfigure(ruleset: ruleset, actions: actions)
+        appsec_component.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
       end
 
       def reconfigure_lock(&block)
@@ -56,5 +56,6 @@ require_relative 'appsec/contrib/rack/integration'
 require_relative 'appsec/contrib/sinatra/integration'
 require_relative 'appsec/contrib/rails/integration'
 require_relative 'appsec/contrib/devise/integration'
+require_relative 'appsec/contrib/graphql/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/core/configuration/components.rb

@@ -13,6 +13,7 @@ require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
+require_relative '../crashtracking/component'
 
 module Datadog
   module Core
@@ -56,19 +57,18 @@ module Datadog
           end
 
           def build_telemetry(settings, agent_settings, logger)
-            enabled = settings.telemetry.enabled
-            if agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-              enabled = false
-              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+            Telemetry::Component.build(settings, agent_settings, logger)
+          end
+
+          def build_crashtracker(settings, agent_settings, logger:)
+            return unless settings.crashtracking.enabled
+
+            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+              logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
+              return
             end
 
-            Telemetry::Component.new(
-              enabled: enabled,
-              metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-              metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-              dependency_collection: settings.telemetry.dependency_collection
-            )
+            Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
         end
 
@@ -82,6 +82,7 @@ module Datadog
           :runtime_metrics,
           :telemetry,
           :tracer,
+          :crashtracker,
           :appsec
 
         def initialize(settings)
@@ -93,20 +94,22 @@ module Datadog
           # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
+
+          @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)
           @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
+          @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer,
+            optional_tracer: @tracer
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
           @health_metrics = self.class.build_health_metrics(settings)
-          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
-          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
 
           self.class.configure_tracing(settings)
         end

data/lib/datadog/core/configuration/settings.rb

@@ -301,6 +301,16 @@ module Datadog
               o.default true
             end
 
+            # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
+            #
+            # Requires allocation profiling to be enabled.
+            #
+            # @default false
+            option :allocation_counting_enabled do |o|
+              o.type :bool
+              o.default false
+            end
+
             # Can be used to enable/disable the collection of heap profiles.
             #
             # This feature is alpha and disabled by default
@@ -400,10 +410,8 @@ module Datadog
             # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
             #
             # We've discovered that this can trigger a bug in a number of Ruby APIs in the `Dir` class, as
-            # described in https://github.com/DataDog/dd-trace-rb/issues/3450 . This workaround prevents the issue
-            # from happening by monkey patching the affected APIs.
-            #
-            # (In the future, once a fix lands upstream, we'll disable this workaround for Rubies that don't need it)
+            # described in https://bugs.ruby-lang.org/issues/20586 .
+            # This was fixed for Ruby 3.4+, and this setting is a no-op for those versions.
             #
             # @default `DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED` environment variable as a boolean,
             # otherwise `true`
@@ -441,14 +449,70 @@ module Datadog
               o.default 60
             end
 
-            # Enables reporting of information when the Ruby VM crashes.
-            #
-            # @default `DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED` environment variable as a boolean,
-            # otherwise `false`
+            # DEV-3.0: Remove `experimental_crash_tracking_enabled` option
             option :experimental_crash_tracking_enabled do |o|
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Core.log_deprecation(key: :experimental_crash_tracking_enabled) do
+                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal '\
+                    'and no longer does anything. Please remove it from your Datadog.configure block.'
+                  end
+                end
+              end
+            end
+
+            # Enables GVL profiling. This will show when threads are waiting for GVL in the timeline view.
+            #
+            # This is a preview feature and disabled by default. It requires Ruby 3.2+.
+            #
+            # @default `DD_PROFILING_PREVIEW_GVL_ENABLED` environment variable as a boolean, otherwise `false`
+            option :preview_gvl_enabled do |o|
               o.type :bool
-              o.env 'DD_PROFILING_EXPERIMENTAL_CRASH_TRACKING_ENABLED'
+              o.env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
+              o.default false
+            end
+
+            # Controls the smallest time period the profiler will report a thread waiting for the GVL.
+            #
+            # The default value was set to minimize overhead. Periods smaller than the set value will not be reported (e.g.
+            # the thread will be reported as whatever it was doing before it waited for the GVL).
+            #
+            # We do not recommend setting this to less than 1ms. Tweaking this value can increase application latency and
+            # memory use.
+            #
+            # @default 10_000_000 (10ms)
+            option :waiting_for_gvl_threshold_ns do |o|
+              o.type :int
+              o.default 10_000_000
+            end
+
+            # Controls if the profiler should attempt to read context from the otel library
+            #
+            # @default false
+            option :preview_otel_context_enabled do |o|
+              o.env 'DD_PROFILING_PREVIEW_OTEL_CONTEXT_ENABLED'
               o.default false
+              o.env_parser do |value|
+                if value
+                  value = value.strip.downcase
+                  if ['only', 'both'].include?(value)
+                    value
+                  elsif ['true', '1'].include?(value)
+                    'both'
+                  else
+                    'false'
+                  end
+                end
+              end
+              o.setter do |value|
+                if value == true
+                  :both
+                elsif ['only', 'both', :only, :both].include?(value)
+                  value.to_sym
+                else
+                  false
+                end
+              end
             end
           end
 
@@ -618,6 +682,33 @@ module Datadog
           end
         end
 
+        # The monotonic clock time provider used by Datadog. This option is internal and is used by `datadog-ci`
+        # gem to avoid traces' durations being skewed by timecop.
+        #
+        # It must respect the interface of [Datadog::Core::Utils::Time#get_time] method.
+        #
+        # For [Timecop](https://rubygems.org/gems/timecop), for example,
+        # `->(unit = :float_second) { ::Process.clock_gettime_without_mock(::Process::CLOCK_MONOTONIC, unit) }`
+        # allows Datadog features to use the real monotonic time when time is frozen with
+        # `Timecop.mock_process_clock = true`.
+        #
+        # @default `->(unit = :float_second) { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit)}`
+        # @return [Proc<Numeric>]
+        option :get_time_provider do |o|
+          o.default_proc { |unit = :float_second| ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit) }
+          o.type :proc
+
+          o.after_set do |get_time_provider|
+            Core::Utils::Time.get_time_provider = get_time_provider
+          end
+
+          o.resetter do |_value|
+            ->(unit = :float_second) { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit) }.tap do |default|
+              Core::Utils::Time.get_time_provider = default
+            end
+          end
+        end
+
         # The `version` tag in Datadog. Use it to enable [Deployment Tracking](https://docs.datadoghq.com/tracing/deployment_tracking/).
         # @see https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging
         # @default `DD_VERSION` environment variable, otherwise `nils`
@@ -663,6 +754,24 @@ module Datadog
             o.type :bool
           end
 
+          # Enable agentless mode for telemetry: submit telemetry events directly to the intake without Datadog Agent.
+          #
+          # @return [Boolean]
+          # @!visibility private
+          option :agentless_enabled do |o|
+            o.type :bool
+            o.default false
+          end
+
+          # Overrides agentless telemetry URL. To be used internally for testing purposes only.
+          #
+          # @return [String]
+          # @!visibility private
+          option :agentless_url_override do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_AGENTLESS_URL_OVERRIDE
+          end
+
           # Enable metrics collection for telemetry. Metrics collection only works when telemetry is enabled and
           # metrics are enabled.
           # @default `DD_TELEMETRY_METRICS_ENABLED` environment variable, otherwise `true`.
@@ -734,6 +843,14 @@ module Datadog
             o.type :string, nilable: true
             o.env Core::Telemetry::Ext::ENV_INSTALL_TIME
           end
+
+          # Telemetry shutdown timeout in seconds
+          #
+          # @!visibility private
+          option :shutdown_timeout_seconds do |o|
+            o.type :float
+            o.default 1.0
+          end
         end
 
         # Remote configuration
@@ -794,6 +911,15 @@ module Datadog
           option :service
         end
 
+        settings :crashtracking do
+          # Enables reporting of information when Ruby VM crashes.
+          option :enabled do |o|
+            o.type :bool
+            o.default false
+            o.env 'DD_CRASHTRACKING_ENABLED'
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings

data/lib/datadog/core/crashtracking/agent_base_url.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../configuration/ext'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # This module provides a method to resolve the base URL of the agent
+      module AgentBaseUrl
+        def self.resolve(agent_settings)
+          case agent_settings.adapter
+          when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+            "#{agent_settings.ssl ? 'https' : 'http'}://#{agent_settings.hostname}:#{agent_settings.port}/"
+          when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+            "unix://#{agent_settings.uds_path}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/crashtracking/component.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'libdatadog'
+
+require_relative 'tag_builder'
+require_relative 'agent_base_url'
+require_relative '../utils/only_once'
+require_relative '../utils/at_fork_monkey_patch'
+
+module Datadog
+  module Core
+    module Crashtracking
+      # Used to report Ruby VM crashes.
+      #
+      # NOTE: The crashtracker native state is a singleton;
+      # so even if you create multiple instances of `Crashtracking::Component` and start them,
+      # it only works as "last writer wins". Same for stop -- there's only one state, so calling stop
+      # on it will stop the crash tracker, regardless of which instance started it.
+      #
+      # Methods prefixed with _native_ are implemented in `crashtracker.c`
+      class Component
+        LIBDATADOG_API_FAILURE =
+          begin
+            require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+            nil
+          rescue LoadError => e
+            e.message
+          end
+
+        ONLY_ONCE = Core::Utils::OnlyOnce.new
+
+        def self.build(settings, agent_settings, logger:)
+          tags = TagBuilder.call(settings)
+          agent_base_url = AgentBaseUrl.resolve(agent_settings)
+          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+
+          ld_library_path = ::Libdatadog.ld_library_path
+          logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path
+
+          path_to_crashtracking_receiver_binary = ::Libdatadog.path_to_crashtracking_receiver_binary
+          unless path_to_crashtracking_receiver_binary
+            logger.warn('Missing path_to_crashtracking_receiver_binary; cannot enable crash tracking')
+          end
+
+          return unless agent_base_url
+          return unless ld_library_path
+          return unless path_to_crashtracking_receiver_binary
+
+          new(
+            tags: tags,
+            agent_base_url: agent_base_url,
+            ld_library_path: ld_library_path,
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            logger: logger
+          ).tap(&:start)
+        end
+
+        def initialize(tags:, agent_base_url:, ld_library_path:, path_to_crashtracking_receiver_binary:, logger:)
+          @tags = tags
+          @agent_base_url = agent_base_url
+          @ld_library_path = ld_library_path
+          @path_to_crashtracking_receiver_binary = path_to_crashtracking_receiver_binary
+          @logger = logger
+        end
+
+        def start
+          Utils::AtForkMonkeyPatch.apply!
+
+          start_or_update_on_fork(action: :start)
+          ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Must NOT reference `self` here, as only the first instance will
+              # be captured by the ONLY_ONCE and we want to pick the latest active one
+              # (which may have different tags or agent config)
+              Datadog.send(:components).crashtracker&.update_on_fork
+            end
+          end
+        end
+
+        def update_on_fork
+          start_or_update_on_fork(action: :update_on_fork)
+        end
+
+        def stop
+          self.class._native_stop
+          logger.debug('Crash tracking stopped successfully')
+        rescue => e
+          logger.error("Failed to stop crash tracking: #{e.message}")
+        end
+
+        private
+
+        attr_reader :tags, :agent_base_url, :ld_library_path, :path_to_crashtracking_receiver_binary, :logger
+
+        def start_or_update_on_fork(action:)
+          self.class._native_start_or_update_on_fork(
+            action: action,
+            agent_base_url: agent_base_url,
+            path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
+            ld_library_path: ld_library_path,
+            tags_as_array: tags.to_a,
+            upload_timeout_seconds: 1
+          )
+          logger.debug("Crash tracking #{action} successfully")
+        rescue => e
+          logger.error("Failed to #{action} crash tracking: #{e.message}")
+        end
+      end
+    end
+  end
+end