datadog 2.3.0 → 2.5.0

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,60 +1,45 @@
 # frozen_string_literal: true
 
+require_relative '../core/rate_limiter'
+
 module Datadog
   module AppSec
-    # Simple per-thread rate limiter
-    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    # Per-thread rate limiter based on token bucket rate limiter.
+    #
+    # Since AppSec marks sampling to keep on a security event, this limits
+    # the flood of egress traces involving AppSec
     class RateLimiter
-      def initialize(rate)
-        @rate = rate
-        @timestamps = []
-      end
-
-      def limit
-        now = Time.now.to_f
-
-        loop do
-          oldest = @timestamps.first
-
-          break if oldest.nil? || now - oldest < 1
-
-          @timestamps.shift
-        end
-
-        @timestamps << now
-
-        if (count = @timestamps.count) <= @rate
-          yield
-        else
-          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
-        end
-      end
+      THREAD_KEY = :datadog_security_appsec_rate_limiter
 
       class << self
-        def limit(name, &block)
-          rate_limiter(name).limit(&block)
+        def thread_local
+          rate_limiter = Thread.current.thread_variable_get(THREAD_KEY)
+          return rate_limiter unless rate_limiter.nil?
+
+          Thread.current.thread_variable_set(THREAD_KEY, new(trace_rate_limit))
         end
 
         # reset a rate limiter: used for testing
-        def reset!(name)
-          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        def reset!
+          Thread.current.thread_variable_set(THREAD_KEY, nil)
         end
 
-        protected
-
-        def rate_limiter(name)
-          case name
-          when :traces
-            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
-          else
-            raise "unsupported rate limiter: #{name.inspect}"
-          end
-        end
+        private
 
         def trace_rate_limit
           Datadog.configuration.appsec.trace_rate_limit
         end
       end
+
+      def initialize(rate)
+        @rate_limiter = Core::TokenBucket.new(rate)
+      end
+
+      def limit
+        return yield if @rate_limiter.allow?
+
+        Datadog.logger.debug { "Rate limit hit: #{@rate_limiter.current_window_rate} AppSec traces/second" }
+      end
     end
   end
 end

data/lib/datadog/appsec/remote.rb

@@ -53,7 +53,7 @@ module Datadog
         end
 
         # rubocop:disable Metrics/MethodLength
-        def receivers
+        def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
@@ -86,7 +86,10 @@ module Datadog
             end
 
             if rules.empty?
-              settings_rules = AppSec::Processor::RuleLoader.load_rules(ruleset: Datadog.configuration.appsec.ruleset)
+              settings_rules = AppSec::Processor::RuleLoader.load_rules(
+                telemetry: telemetry,
+                ruleset: Datadog.configuration.appsec.ruleset
+              )
 
               raise NoRulesError, 'no default rules available' unless settings_rules
 
@@ -99,9 +102,10 @@ module Datadog
               overrides: overrides,
               exclusions: exclusions,
               custom_rules: custom_rules,
+              telemetry: telemetry
             )
 
-            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions)
+            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
           end
 
           [receiver]

data/lib/datadog/appsec/scope.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'processor'
-
 module Datadog
   module AppSec
     # Capture context essential to consistently call processor and report via traces
@@ -22,8 +20,7 @@ module Datadog
         def activate_scope(trace, service_entry_span, processor)
           raise ActiveScopeError, 'another scope is active, nested scopes are not supported' if active_scope
 
-          context = Datadog::AppSec::Processor::Context.new(processor)
-
+          context = processor.new_context
           self.active_scope = new(trace, service_entry_span, context)
         end
 

data/lib/datadog/appsec/utils/trace_operation.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      # Utility class to to AppSec-specific trace operations
+      class TraceOperation
+        def self.appsec_standalone_reject?(trace)
+          Datadog.configuration.appsec.standalone.enabled &&
+            (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'utils/trace_operation'
+
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -4,6 +4,7 @@ require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
 require_relative 'appsec/scope'
 require_relative 'appsec/ext'
+require_relative 'appsec/utils'
 
 module Datadog
   # Namespace for Datadog AppSec instrumentation
@@ -23,12 +24,12 @@ module Datadog
         appsec_component.processor if appsec_component
       end
 
-      def reconfigure(ruleset:, actions:)
+      def reconfigure(ruleset:, actions:, telemetry:)
         appsec_component = components.appsec
 
         return unless appsec_component
 
-        appsec_component.reconfigure(ruleset: ruleset, actions: actions)
+        appsec_component.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
       end
 
       def reconfigure_lock(&block)

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -232,7 +232,32 @@ module Datadog
         end
 
         def should_use_uds?
-          can_use_uds? && !mixed_http_and_uds?
+          # When we have mixed settings for http/https and uds, we print a warning
+          # and use the uds settings.
+          mixed_http_and_uds
+          can_use_uds?
+        end
+
+        def mixed_http_and_uds
+          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
+
+          @mixed_http_and_uds = (configured_hostname || configured_port) && can_use_uds?
+          if @mixed_http_and_uds
+            warn_if_configuration_mismatch(
+              [
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration for unix domain socket',
+                  value: parsed_url.to_s,
+                ),
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration of hostname/port for http/https use',
+                  value: "hostname: '#{hostname}', port: '#{port}'",
+                ),
+              ]
+            )
+          end
+
+          @mixed_http_and_uds
         end
 
         def can_use_uds?
@@ -307,30 +332,6 @@ module Datadog
           uri.scheme == 'unix'
         end
 
-        # When we have mixed settings for http/https and uds, we print a warning and ignore the uds settings
-        def mixed_http_and_uds?
-          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
-
-          @mixed_http_and_uds = (configured_hostname || configured_port) && can_use_uds?
-
-          if @mixed_http_and_uds
-            warn_if_configuration_mismatch(
-              [
-                DetectedConfiguration.new(
-                  friendly_name: 'configuration of hostname/port for http/https use',
-                  value: "hostname: '#{hostname}', port: '#{port}'",
-                ),
-                DetectedConfiguration.new(
-                  friendly_name: 'configuration for unix domain socket',
-                  value: parsed_url.to_s,
-                ),
-              ]
-            )
-          end
-
-          @mixed_http_and_uds
-        end
-
         # Represents a given configuration value and where we got it from
         class DetectedConfiguration
           attr_reader :friendly_name, :value

data/lib/datadog/core/configuration/components.rb

@@ -94,7 +94,9 @@ module Datadog
           # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
+
+          @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)
           @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
@@ -107,8 +109,7 @@ module Datadog
 
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
           @health_metrics = self.class.build_health_metrics(settings)
-          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
-          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
 
           self.class.configure_tracing(settings)
         end

data/lib/datadog/core/configuration/settings.rb

@@ -410,10 +410,8 @@ module Datadog
             # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
             #
             # We've discovered that this can trigger a bug in a number of Ruby APIs in the `Dir` class, as
-            # described in https://github.com/DataDog/dd-trace-rb/issues/3450 . This workaround prevents the issue
-            # from happening by monkey patching the affected APIs.
-            #
-            # (In the future, once a fix lands upstream, we'll disable this workaround for Rubies that don't need it)
+            # described in https://bugs.ruby-lang.org/issues/20586 .
+            # This was fixed for Ruby 3.4+, and this setting is a no-op for those versions.
             #
             # @default `DD_PROFILING_DIR_INTERRUPTION_WORKAROUND_ENABLED` environment variable as a boolean,
             # otherwise `true`
@@ -462,6 +460,72 @@ module Datadog
                 end
               end
             end
+
+            # Enables GVL profiling. This will show when threads are waiting for GVL in the timeline view.
+            #
+            # This is a preview feature and disabled by default. It requires Ruby 3.2+.
+            #
+            # @default `DD_PROFILING_PREVIEW_GVL_ENABLED` environment variable as a boolean, otherwise `false`
+            option :preview_gvl_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
+              o.default false
+            end
+
+            # Controls the smallest time period the profiler will report a thread waiting for the GVL.
+            #
+            # The default value was set to minimize overhead. Periods smaller than the set value will not be reported (e.g.
+            # the thread will be reported as whatever it was doing before it waited for the GVL).
+            #
+            # We do not recommend setting this to less than 1ms. Tweaking this value can increase application latency and
+            # memory use.
+            #
+            # @default 10_000_000 (10ms)
+            option :waiting_for_gvl_threshold_ns do |o|
+              o.type :int
+              o.default 10_000_000
+            end
+
+            # Controls if the profiler should attempt to read context from the otel library
+            #
+            # @default false
+            option :preview_otel_context_enabled do |o|
+              o.env 'DD_PROFILING_PREVIEW_OTEL_CONTEXT_ENABLED'
+              o.default false
+              o.env_parser do |value|
+                if value
+                  value = value.strip.downcase
+                  if ['only', 'both'].include?(value)
+                    value
+                  elsif ['true', '1'].include?(value)
+                    'both'
+                  else
+                    'false'
+                  end
+                end
+              end
+              o.setter do |value|
+                if value == true
+                  :both
+                elsif ['only', 'both', :only, :both].include?(value)
+                  value.to_sym
+                else
+                  false
+                end
+              end
+            end
+
+            # Controls if the heap profiler should attempt to clean young objects after GC, rather than just at
+            # serialization time. This lowers memory usage and high percentile latency.
+            #
+            # Only takes effect when used together with `gc_enabled: true` and `experimental_heap_enabled: true`.
+            #
+            # @default false
+            option :heap_clean_after_gc_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_HEAP_CLEAN_AFTER_GC_ENABLED'
+              o.default false
+            end
           end
 
           # @public_api
@@ -630,6 +694,33 @@ module Datadog
           end
         end
 
+        # The monotonic clock time provider used by Datadog. This option is internal and is used by `datadog-ci`
+        # gem to avoid traces' durations being skewed by timecop.
+        #
+        # It must respect the interface of [Datadog::Core::Utils::Time#get_time] method.
+        #
+        # For [Timecop](https://rubygems.org/gems/timecop), for example,
+        # `->(unit = :float_second) { ::Process.clock_gettime_without_mock(::Process::CLOCK_MONOTONIC, unit) }`
+        # allows Datadog features to use the real monotonic time when time is frozen with
+        # `Timecop.mock_process_clock = true`.
+        #
+        # @default `->(unit = :float_second) { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit)}`
+        # @return [Proc<Numeric>]
+        option :get_time_provider do |o|
+          o.default_proc { |unit = :float_second| ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit) }
+          o.type :proc
+
+          o.after_set do |get_time_provider|
+            Core::Utils::Time.get_time_provider = get_time_provider
+          end
+
+          o.resetter do |_value|
+            ->(unit = :float_second) { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit) }.tap do |default|
+              Core::Utils::Time.get_time_provider = default
+            end
+          end
+        end
+
         # The `version` tag in Datadog. Use it to enable [Deployment Tracking](https://docs.datadoghq.com/tracing/deployment_tracking/).
         # @see https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging
         # @default `DD_VERSION` environment variable, otherwise `nils`
@@ -836,7 +927,7 @@ module Datadog
           # Enables reporting of information when Ruby VM crashes.
           option :enabled do |o|
             o.type :bool
-            o.default true
+            o.default false
             o.env 'DD_CRASHTRACKING_ENABLED'
           end
         end

data/lib/datadog/core/configuration.rb

@@ -278,9 +278,7 @@ module Datadog
       def handle_interrupt_shutdown!
         logger = Datadog.logger
         shutdown_thread = Thread.new { shutdown! }
-        unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
-          shutdown_thread.name = Datadog::Core::Configuration.name
-        end
+        shutdown_thread.name = Datadog::Core::Configuration.name
 
         print_message_treshold_seconds = 0.2
 

data/lib/datadog/core/crashtracking/component.rb

@@ -66,7 +66,8 @@ module Datadog
         def start
           Utils::AtForkMonkeyPatch.apply!
 
-          start_or_update_on_fork(action: :start)
+          start_or_update_on_fork(action: :start, tags: tags)
+
           ONLY_ONCE.run do
             Utils::AtForkMonkeyPatch.at_fork(:child) do
               # Must NOT reference `self` here, as only the first instance will
@@ -77,8 +78,10 @@ module Datadog
           end
         end
 
-        def update_on_fork
-          start_or_update_on_fork(action: :update_on_fork)
+        def update_on_fork(settings: Datadog.configuration)
+          # Here we pick up the latest settings, so that we pick up any tags that change after forking
+          # such as the pid or runtime-id
+          start_or_update_on_fork(action: :update_on_fork, tags: TagBuilder.call(settings))
         end
 
         def stop
@@ -92,16 +95,16 @@ module Datadog
 
         attr_reader :tags, :agent_base_url, :ld_library_path, :path_to_crashtracking_receiver_binary, :logger
 
-        def start_or_update_on_fork(action:)
+        def start_or_update_on_fork(action:, tags:)
           self.class._native_start_or_update_on_fork(
             action: action,
-            exporter_configuration: [:agent, agent_base_url],
+            agent_base_url: agent_base_url,
             path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
             ld_library_path: ld_library_path,
             tags_as_array: tags.to_a,
             upload_timeout_seconds: 1
           )
-          logger.debug("Crash tracking #{action} successfully")
+          logger.debug("Crash tracking action: #{action} successful")
         rescue => e
           logger.error("Failed to #{action} crash tracking: #{e.message}")
         end

data/lib/datadog/core/environment/execution.rb

@@ -25,9 +25,9 @@ module Datadog
           #   2. Checking if `Net::HTTP` is referring to the original one
           #   => ::Net::HTTP.equal?(::WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetHTTP)
           def webmock_enabled?
-            defined?(::WebMock::HttpLibAdapters::NetHttpAdapter) &&
+            !!(defined?(::WebMock::HttpLibAdapters::NetHttpAdapter) &&
               defined?(::Net::HTTP) &&
-              ::Net::HTTP.equal?(::WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP))
+              ::Net::HTTP.equal?(::WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP)))
           end
 
           private
@@ -68,7 +68,7 @@ module Datadog
 
           # Check if we are running from `bin/cucumber` or `cucumber/rake/task`.
           def cucumber?
-            defined?(::Cucumber::Cli)
+            !!defined?(::Cucumber::Cli)
           end
 
           # If this is a Rails application, use different heuristics to detect
@@ -80,7 +80,7 @@ module Datadog
             # detecting its presence is enough to deduct if this is a development environment.
             #
             # @see https://github.com/rails/spring/blob/48b299348ace2188444489a0c216a6f3e9687281/README.md?plain=1#L204-L207
-            defined?(::Spring) || rails_env_development?
+            !!defined?(::Spring) || rails_env_development?
           end
 
           RAILS_ENV_DEVELOPMENT = Set['development', 'test'].freeze
@@ -94,7 +94,7 @@ module Datadog
           # it's common to have a custom "staging" environment, and such environment normally want to run as close
           # to production as possible.
           def rails_env_development?
-            defined?(::Rails.env) && RAILS_ENV_DEVELOPMENT.include?(::Rails.env)
+            !!defined?(::Rails.env) && RAILS_ENV_DEVELOPMENT.include?(::Rails.env)
           end
         end
       end

data/lib/datadog/core/environment/yjit.rb

@@ -52,6 +52,11 @@ module Datadog
           ::RubyVM::YJIT.runtime_stats[:yjit_alloc_size]
         end
 
+        # Ratio of YJIT-executed instructions
+        def ratio_in_yjit
+          ::RubyVM::YJIT.runtime_stats[:ratio_in_yjit]
+        end
+
         def available?
           defined?(::RubyVM::YJIT) \
             && ::RubyVM::YJIT.enabled? \

data/lib/datadog/core/metrics/client.rb

@@ -2,6 +2,7 @@
 
 require_relative '../utils/time'
 require_relative '../utils/only_once'
+require_relative '../telemetry/logger'
 require_relative '../configuration/ext'
 
 require_relative 'ext'
@@ -100,6 +101,7 @@ module Datadog
           Datadog.logger.error(
             "Failed to send count stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
+          Telemetry::Logger.report(e, description: 'Failed to send count stat')
         end
 
         def distribution(stat, value = nil, options = nil, &block)
@@ -113,6 +115,7 @@ module Datadog
           Datadog.logger.error(
             "Failed to send distribution stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
+          Telemetry::Logger.report(e, description: 'Failed to send distribution stat')
         end
 
         def increment(stat, options = nil)
@@ -125,6 +128,7 @@ module Datadog
           Datadog.logger.error(
             "Failed to send increment stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
+          Telemetry::Logger.report(e, description: 'Failed to send increment stat')
         end
 
         def gauge(stat, value = nil, options = nil, &block)
@@ -138,6 +142,7 @@ module Datadog
           Datadog.logger.error(
             "Failed to send gauge stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
+          Telemetry::Logger.report(e, description: 'Failed to send gauge stat')
         end
 
         def time(stat, options = nil)
@@ -153,9 +158,11 @@ module Datadog
               distribution(stat, ((finished - start) * 1000), options)
             end
           rescue StandardError => e
+            # TODO: Likely to be redundant, since `distribution` handles its own errors.
             Datadog.logger.error(
               "Failed to send time stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
             )
+            Telemetry::Logger.report(e, description: 'Failed to send time stat')
           end
         end