datadog 2.28.0 → 2.30.0

data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb

@@ -11,6 +11,8 @@ module Datadog
               context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
               return super unless context
 
+              context.state[:web_framework] = 'rails'
+
               # TODO: handle exceptions, except for super
               gateway_request = Gateway::Request.new(request)
               http_response, _gateway_request = Instrumentation.gateway.push('rails.request.action', gateway_request) do

data/lib/datadog/appsec/contrib/rest_client/patcher.rb

@@ -20,6 +20,8 @@ module Datadog
             require_relative 'request_ssrf_detection_patch'
 
             ::RestClient::Request.prepend(RequestSSRFDetectionPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -3,6 +3,8 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/media_type'
+require_relative '../../utils/http/body'
 
 module Datadog
   module AppSec
@@ -10,35 +12,97 @@ module Datadog
       module RestClient
         # Module that adds SSRF detection to RestClient::Request#execute
         module RequestSSRFDetectionPatch
+          REDIRECT_STATUS_CODES = (300..399).freeze
+
           def execute(&block)
             context = AppSec.active_context
             return super unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            headers = normalize_request_headers
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               'server.io.net.url' => url,
               'server.io.net.request.method' => method.to_s.upcase,
-              'server.io.net.request.headers' => normalize_request_headers
+              'server.io.net.request.headers' => headers
             }
 
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              sample_body = true
+            else
+              sample_body = mark_body_sampling!(context)
+            end
+
+            if !is_redirect && sample_body
+              body = parse_body(payload.to_s, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
             handle(result, context: context) if result.match?
 
-            response = super
+            # NOTE: RestClient raises exceptions for non-2xx responses. For POST/PUT/PATCH
+            #       requests with 3xx redirects, RestClient raises instead of auto-following.
+            #       We rescue to process the response before re-raising.
+            begin
+              response = super
+            rescue ::RestClient::Exception => e
+              response = e.response
+              process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+
+              raise
+            end
 
+            process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+            response
+          end
+
+          def process_response(response, sample_body:)
+            context = AppSec.active_context
+            return unless context
+
+            headers = normalize_response_headers(response)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               'server.io.net.response.status' => response.code.to_s,
-              'server.io.net.response.headers' => normalize_response_headers(response)
+              'server.io.net.response.headers' => headers
             }
 
+            is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?('location')
+            context.state[:downstream_redirect_url] = URI.join(url, headers['location']).to_s if is_redirect && sample_body
+
+            if sample_body && !is_redirect
+              body = parse_body(response.body, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
             handle(result, context: context) if result.match?
-
-            response
           end
 
           private
 
+          def mark_body_sampling!(context)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return false if context.state[:downstream_body_analyzed_count] >= max
+            return false unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            true
+          end
+
+          def parse_body(body, content_type:)
+            return if body.empty?
+
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            Utils::HTTP::Body.parse(body, media_type: media_type)
+          end
+
           # NOTE: Starting version 2.1.0 headers are already normalized via internal
           #       variable `@processed_headers_lowercase`. In case it's available,
           #       we use it to avoid unnecessary transformation.
@@ -55,7 +119,8 @@ module Datadog
           # FIXME: Steep has issues with `transform_values!` modifying the original
           #        type and it failed with "Cannot allow block body" error
           def normalize_response_headers(response) # steep:ignore MethodBodyTypeMismatch
-            response.net_http_res.to_hash.transform_values! { |value| Array(value).join(', ') } # steep:ignore BlockBodyTypeMismatch
+            response.net_http_res.to_hash
+              .transform_values! { |value| Array(value).join(', ') } # steep:ignore BlockBodyTypeMismatch
           end
 
           def handle(result, context:)

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -22,9 +22,11 @@ module Datadog
               end
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
+                gateway.watch('sinatra.request.dispatch') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
+                  context.state[:web_framework] = 'sinatra'
+
                   persistent_data = {
                     'server.request.body' => gateway_request.form_hash
                   }
@@ -49,7 +51,7 @@ module Datadog
               end
 
               def watch_request_routed(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.routed', :appsec) do |stack, args|
+                gateway.watch('sinatra.request.routed') do |stack, args|
                   gateway_request, gateway_route_params = args # : [Gateway::Request, Gateway::RouteParams]
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
@@ -75,7 +77,7 @@ module Datadog
               end
 
               def watch_response_body_json(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.response.body.json', :appsec) do |stack, container|
+                gateway.watch('sinatra.response.body.json') do |stack, container|
                   context = container.context # : Context
 
                   persistent_data = {

data/lib/datadog/appsec/counter_sampler.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative '../core/knuth_sampler'
+
+module Datadog
+  module AppSec
+    # Sampler that uses an internal counter to make deterministic sampling decisions.
+    #
+    # Each call to {#sample?} increments the counter and uses it as input to
+    # the underlying Knuth multiplicative hash algorithm.
+    #
+    # @api private
+    class CounterSampler
+      def initialize(rate = 1.0)
+        @sampler = Core::KnuthSampler.new(rate)
+        @counter = 0
+      end
+
+      def sample?
+        @counter += 1
+        @sampler.sample?(@counter)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/event.rb

@@ -32,13 +32,6 @@ module Datadog
         accept-language
       ].freeze
 
-      ALLOWED_RESPONSE_HEADERS = %w[
-        content-length
-        content-type
-        content-encoding
-        content-language
-      ].freeze
-
       class << self
         def tag(context, waf_result)
           return if context.span.nil?
@@ -50,7 +43,7 @@ module Datadog
           context.span.set_tag('appsec.event', 'true')
         end
 
-        def record(context, request: nil, response: nil)
+        def record(context, request: nil)
           return if context.events.empty? || context.span.nil?
 
           Datadog::AppSec::RateLimiter.thread_local.limit do
@@ -66,7 +59,6 @@ module Datadog
 
                 context.span['_dd.origin'] = 'appsec'
                 context.span.set_tags(request_tags(request)) if request
-                context.span.set_tags(response_tags(response)) if response
               end
 
               context.span.set_tags(waf_tags(event_group))
@@ -90,14 +82,6 @@ module Datadog
           end
         end
 
-        def response_tags(response)
-          response.headers.each_with_object({}) do |(name, value), memo|
-            next unless ALLOWED_RESPONSE_HEADERS.include?(name)
-
-            memo["http.response.headers.#{name}"] = value
-          end
-        end
-
         def waf_tags(security_events)
           triggers = []
 

data/lib/datadog/appsec/instrumentation/gateway/middleware.rb

@@ -7,10 +7,9 @@ module Datadog
         # NOTE: This class extracted as-is and will be deprecated
         # Instrumentation gateway middleware
         class Middleware
-          attr_reader :key, :block
+          attr_reader :block
 
-          def initialize(key, &block)
-            @key = key
+          def initialize(&block)
             @block = block
           end
 

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -41,8 +41,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
+        def watch(name, &block)
+          @middlewares[name] << Middleware.new(&block)
         end
 
         def pushed?(name)

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -23,6 +23,24 @@ module Datadog
             }
           )
         end
+
+        def export_api_security_metrics(context)
+          web_framework = context.state[:web_framework]
+          return unless web_framework
+
+          if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
+            AppSec.telemetry.inc(
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, 'api_security.missing_route', 1,
+              tags: {framework: web_framework}
+            )
+          end
+
+          metric_name = context.state[:schema_extracted] ? 'schema' : 'no_schema'
+          AppSec.telemetry.inc(
+            AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
+            tags: {framework: web_framework}
+          )
+        end
       end
     end
   end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -24,7 +24,7 @@ module Datadog
             end
 
             def watch_user_id(gateway = Instrumentation.gateway)
-              gateway.watch('identity.set_user', :appsec) do |stack, user|
+              gateway.watch('identity.set_user') do |stack, user|
                 context = AppSec.active_context
 
                 if user.id.nil? && user.login.nil? && user.session_id.nil?
@@ -55,7 +55,7 @@ module Datadog
             end
 
             def watch_user_login(gateway = Instrumentation.gateway)
-              gateway.watch('appsec.events.user_lifecycle', :appsec) do |stack, kind|
+              gateway.watch('appsec.events.user_lifecycle') do |stack, kind|
                 context = AppSec.active_context
 
                 next stack.call(kind) unless WATCHED_LOGIN_EVENTS.include?(kind)

data/lib/datadog/appsec/security_engine/engine.rb

@@ -40,12 +40,16 @@ module Datadog
           @ruleset_version = diagnostics['ruleset_version']
 
           @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
+
+          metric('init', success: true, ruleset_version: @ruleset_version, telemetry: telemetry)
         rescue WAF::Error => e
-          error_message = "AppSec security engine failed to initialize"
+          error_message = 'AppSec security engine failed to initialize'
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           telemetry.report(e, description: error_message)
 
+          metric('init', success: false, ruleset_version: @ruleset_version, telemetry: telemetry)
+
           raise e
         end
 
@@ -103,17 +107,34 @@ module Datadog
           @ruleset_version = @reconfigured_ruleset_version
 
           @handle_ref.current = new_waf_handle
+
+          metric('updates', success: true, ruleset_version: @ruleset_version, telemetry: AppSec.telemetry)
         rescue WAF::Error => e
           # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
           # This means that the current WAF handle was not yet substituted.
-          error_message = "AppSec security engine failed to reconfigure, reverting to the previous configuration"
+          error_message = 'AppSec security engine failed to reconfigure, reverting to the previous configuration'
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           AppSec.telemetry.report(e, description: error_message)
+
+          metric('updates', success: false, ruleset_version: @reconfigured_ruleset_version, telemetry: AppSec.telemetry)
         end
 
         private
 
+        def metric(metric_name, success:, ruleset_version:, telemetry:)
+          telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE,
+            "waf.#{metric_name}",
+            1,
+            tags: {
+              waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+              event_rules_version: ruleset_version.to_s,
+              success: success.to_s
+            }
+          )
+        end
+
         def load_default_config(telemetry:)
           config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
 

data/lib/datadog/appsec/utils/http/body.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'cgi'
+
+require_relative 'url_encoded'
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for handling HTTP body parsing
+        module Body
+          def self.parse(body, media_type:)
+            return if body.nil?
+
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+            # @type var content: ::String?
+            content = body.respond_to?(:read) ? body.read : body # steep:ignore NoMethod, IncompatibleAssignment
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+
+            return if content.nil? || content.empty?
+
+            if media_type.subtype == 'json' || media_type.subtype.end_with?('+json')
+              JSON.parse(content)
+            elsif media_type.subtype == 'x-www-form-urlencoded'
+              URLEncoded.parse(content)
+            end
+          rescue => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to parse body')
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_range.rb

@@ -168,7 +168,8 @@ module Datadog
           #
           # returns true if the MediaType is accepted by this MediaRange
           def ===(other)
-            return self === MediaType.new(other) if other.is_a?(::String)
+            return false if other.nil?
+            return self === MediaType.parse(other) if other.is_a?(::String)
 
             type == other.type && subtype == other.subtype && other.parameters.all? { |k, v| parameters[k] == v } ||
               type == other.type && wildcard?(:subtype) ||

data/lib/datadog/appsec/utils/http/media_type.rb

@@ -10,8 +10,6 @@ module Datadog
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.1
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
         class MediaType
-          ParseError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
-
           WILDCARD = '*'
 
           # See: https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6
@@ -45,48 +43,43 @@ module Datadog
 
           attr_reader :type, :subtype, :parameters
 
-          def self.json?(media_type)
-            return false if media_type.nil? || media_type.empty?
-
-            match = MEDIA_TYPE_RE.match(media_type)
-            return false if match.nil?
+          def self.parse(media)
+            match = MEDIA_TYPE_RE.match(media)
+            return if match.nil?
 
-            subtype = match['subtype']
-            return false if subtype.nil? || subtype.empty?
+            type = match['type'] || WILDCARD
+            type.downcase!
 
+            subtype = match['subtype'] || WILDCARD
             subtype.downcase!
-            subtype == 'json' || subtype.end_with?('+json')
-          end
-
-          def initialize(media_type)
-            match = MEDIA_TYPE_RE.match(media_type)
-            raise ParseError, media_type.inspect if match.nil?
 
-            @type = match['type'] || WILDCARD
-            @type.downcase!
+            parameters = {}
+            params = match['parameters']
 
-            @subtype = match['subtype'] || WILDCARD
-            @subtype.downcase!
+            unless params.nil? || params.empty?
+              params.scan(PARAMETER_RE) do |name, unquoted_value, quoted_value|
+                # NOTE: Order of unquoted_value and quoted_value does not matter,
+                #       as they are mutually exclusive by the regex.
+                # @type var value: ::String?
+                value = unquoted_value || quoted_value
+                next if name.nil? || value.nil?
 
-            @parameters = {}
+                # See https://github.com/soutaro/steep/issues/2051
+                name.downcase! # steep:ignore NoMethod
+                value.downcase!
 
-            parameters = match['parameters']
-            return if parameters.nil? || parameters.empty?
-
-            parameters.scan(PARAMETER_RE) do |name, unquoted_value, quoted_value|
-              # NOTE: Order of unquoted_value and quoted_value does not matter,
-              #       as they are mutually exclusive by the regex.
-              # @type var value: ::String?
-              value = unquoted_value || quoted_value
-              next if name.nil? || value.nil?
+                # See https://github.com/soutaro/steep/issues/2051
+                parameters[name] = value # steep:ignore ArgumentTypeMismatch
+              end
+            end
 
-              # See https://github.com/soutaro/steep/issues/2051
-              name.downcase! # steep:ignore NoMethod
-              value.downcase!
+            new(type: type, subtype: subtype, parameters: parameters)
+          end
 
-              # See https://github.com/soutaro/steep/issues/2051
-              @parameters[name] = value # steep:ignore ArgumentTypeMismatch
-            end
+          def initialize(type:, subtype:, parameters: {})
+            @type = type
+            @subtype = subtype
+            @parameters = parameters
           end
 
           def to_s

data/lib/datadog/appsec/utils/http/url_encoded.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for parsing URL encoded payloads
+        module URLEncoded
+          # Parses a URL encoded payload (query string or form data) into a hash
+          # of keys and values, merging duplicate keys.
+          #
+          # Example:
+          #
+          #   URLEncoded.parse("foo=bar&foo=baz&qux=quux") # => {"foo" => ["bar", "baz"], "qux" => "quux"}
+          #
+          # NOTE: Use it in the absence of `Rack::Utils.parse_query`
+          #
+          # WARNING: This method doesn't limit params byte size.
+          #          See: https://github.com/rack/rack/blob/603b799de38b5eb9b2ff1657c8036a20f4c4db7b/lib/rack/query_parser.rb#L231-L233
+          def self.parse(payload)
+            return {} if payload.nil? || payload.empty?
+
+            payload.split('&').each_with_object({}) do |pair, memo|
+              next if pair.empty?
+
+              # NOTE: Steep has issues with mutation methods
+              #       See https://github.com/ruby/rbs/issues/2819
+              #
+              # @type var key: ::String
+              # @type var value: ::String
+              key, value = pair.split('=', 2).map! do |value| #: ::String
+                CGI.unescape(value)
+              end
+
+              if (stored = memo[key])
+                if stored.is_a?(Array)
+                  stored.push(value)
+                else
+                  memo[key] = [stored, value]
+                end
+              else
+                memo[key] = value
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/components.rb

@@ -11,6 +11,8 @@ require_relative '../runtime/metrics'
 require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
 require_relative '../remote/component'
+require_relative '../utils/at_fork_monkey_patch'
+require_relative '../utils/only_once'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
@@ -28,6 +30,9 @@ module Datadog
     module Configuration
       # Global components for the trace library.
       class Components
+        # Class-level constant to ensure fork patch is applied only once
+        AT_FORK_ONLY_ONCE = Utils::OnlyOnce.new
+
         class << self
           def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
@@ -38,7 +43,7 @@ module Datadog
           end
 
           def build_logger(settings)
-            logger = settings.logger.instance || Core::Logger.new($stdout)
+            logger = settings.logger.instance || Core::Logger.new($stderr)
             logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
             logger
@@ -80,14 +85,15 @@ module Datadog
             Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
 
-          def build_data_streams(settings, agent_settings, logger)
+          def build_data_streams(settings, agent_settings, logger, agent_info)
             return unless settings.data_streams.enabled
 
             Datadog::DataStreams::Processor.new(
               interval: settings.data_streams.interval,
               logger: logger,
               settings: settings,
-              agent_settings: agent_settings
+              agent_settings: agent_settings,
+              agent_info: agent_info
             )
           rescue => e
             logger.warn("Failed to initialize Data Streams Monitoring: #{e.class}: #{e}")
@@ -121,6 +127,17 @@ module Datadog
           StableConfig.log_result(@logger)
           Deprecations.log_deprecations_from_all_sources(@logger)
 
+          # Register fork handling once globally
+          self.class::AT_FORK_ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.apply!
+
+            # Register callback that calls Components.after_fork
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Access via global to avoid capturing 'self'
+              Datadog.send(:components, allow_initialization: false)&.after_fork
+            end
+          end
+
           # This agent_settings is intended for use within Core. If you require
           # agent_settings within a product outside of core you should extend
           # the Core resolver from within your product/component's namespace.
@@ -150,13 +167,21 @@ module Datadog
           @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
-          @data_streams = self.class.build_data_streams(settings, agent_settings, @logger)
+          @data_streams = self.class.build_data_streams(settings, agent_settings, @logger, @agent_info)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
 
           # Configure non-privileged components.
           Datadog::Tracing::Contrib::Component.configure(settings)
         end
 
+        # Called when a fork is detected
+        def after_fork
+          telemetry.after_fork
+          remote&.after_fork
+          crashtracker&.update_on_fork
+          ProcessDiscovery.after_fork
+        end
+
         # Hot-swaps with a new sampler.
         # This operation acquires the Components lock to ensure
         # there is no concurrent modification of the sampler.

data/lib/datadog/core/configuration/option.rb

@@ -167,7 +167,8 @@ module Datadog
 
         def default_value
           if definition.default.instance_of?(Proc)
-            context_eval(&definition.default)
+            # Steep: https://github.com/soutaro/steep/issues/335
+            context_eval(&definition.default) # steep:ignore BlockTypeMismatch
           else
             definition.default_proc || Core::Utils::SafeDup.frozen_or_dup(definition.default)
           end

data/lib/datadog/core/configuration/options.rb

@@ -120,7 +120,7 @@ module Datadog
 
             assert_valid_option!(name)
             definition = self.class.options[name]
-            # @type self: Configuration::Options::GenericSettingsClass
+            # @type self: Configuration::Options::_Settings
             options[name] = definition.build(self)
           end