datadog 2.8.0 → 2.39.0

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/utils/only_once'
+require_relative "../../../core/utils/only_once"
 
-require_relative '../patcher'
-require_relative 'framework'
-require_relative '../../response'
-require_relative '../rack/request_middleware'
-require_relative '../rack/request_body_middleware'
-require_relative 'gateway/watcher'
-require_relative 'gateway/request'
+require_relative "framework"
+require_relative "../../response"
+require_relative "../rack/request_middleware"
+require_relative "../rack/request_body_middleware"
+require_relative "gateway/watcher"
+require_relative "gateway/request"
+require_relative "patches/render_to_body_patch"
+require_relative "patches/process_action_patch"
+require_relative "../../api_security/endpoint_collection/rails_collector"
 
-require_relative '../../../tracing/contrib/rack/middlewares'
+require_relative "../../../tracing/contrib/rack/middlewares"
 
 module Datadog
   module AppSec
@@ -18,8 +20,8 @@ module Datadog
       module Rails
         # Patcher for AppSec on Rails
         module Patcher
-          include Datadog::AppSec::Contrib::Patcher
-
+          GUARD_ACTION_CONTROLLER_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
+          GUARD_ROUTES_REPORTING_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           BEFORE_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           AFTER_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
 
@@ -37,6 +39,8 @@ module Datadog
             Gateway::Watcher.watch
             patch_before_initialize
             patch_after_initialize
+            patch_action_controller
+            subscribe_to_routes_loaded
 
             Patcher.instance_variable_set(:@patched, true)
           end
@@ -52,7 +56,8 @@ module Datadog
               # Middleware must be added before the application is initialized.
               # Otherwise the middleware stack will be frozen.
               add_middleware(app) if Datadog.configuration.tracing[:rails][:middleware]
-              patch_process_action
+
+              ::ActionController::Metal.prepend(Patches::ProcessActionPatch)
             end
           end
 
@@ -68,68 +73,33 @@ module Datadog
             end
           end
 
-          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
-          module ProcessActionPatch
-            def process_action(*args)
-              env = request.env
-
-              context = env[Datadog::AppSec::Ext::SCOPE_KEY]
-
-              return super unless context
-
-              # TODO: handle exceptions, except for super
-
-              gateway_request = Gateway::Request.new(request)
-              request_return, request_response = Instrumentation.gateway.push('rails.request.action', gateway_request) do
-                super
-              end
-
-              if request_response
-                blocked_event = request_response.find { |action, _options| action == :block }
-                if blocked_event
-                  @_response = AppSec::Response.negotiate(
-                    env,
-                    blocked_event.last[:actions]
-                  ).to_action_dispatch_response
-                  request_return = @_response.body
-                end
-              end
-
-              request_return
-            end
-          end
-
-          def patch_process_action
-            ::ActionController::Metal.prepend(ProcessActionPatch)
-          end
-
           def include_middleware?(middleware, app)
             found = false
 
             # find tracer middleware reference in Rails::Configuration::MiddlewareStackProxy
             app.middleware.instance_variable_get(:@operations).each do |operation|
               args = case operation
-                     when Array
-                       # rails 5.2
-                       _op, args = operation
-                       args
-                     when Proc
-                       if operation.binding.local_variables.include?(:args)
-                         # rails 6.0, 6.1
-                         operation.binding.local_variable_get(:args)
-                       else
-                         # rails 7.0 uses ... to pass args
-                         args_getter = Class.new do
-                           def method_missing(_op, *args) # rubocop:disable Style/MissingRespondToMissing
-                             args
-                           end
-                         end.new
-                         operation.call(args_getter)
-                       end
-                     else
-                       # unknown, pass through
-                       []
-                     end
+              when Array
+                # rails 5.2
+                _op, args = operation
+                args
+              when Proc
+                if operation.binding.local_variables.include?(:args)
+                  # rails 6.0, 6.1
+                  operation.binding.local_variable_get(:args)
+                else
+                  # rails 7.0 uses ... to pass args
+                  args_getter = Class.new do
+                    def method_missing(_op, *args) # standard:disable Style/MissingRespondToMissing
+                      args
+                    end
+                  end.new
+                  operation.call(args_getter)
+                end
+              else
+                # unknown, pass through
+                []
+              end
 
               found = true if args.include?(middleware)
             end
@@ -138,7 +108,7 @@ module Datadog
           end
 
           def inspect_middlewares(app)
-            Datadog.logger.debug { +'Rails middlewares: ' << app.middleware.map(&:inspect).inspect }
+            Datadog.logger.debug { +"Rails middlewares: " << app.middleware.map(&:inspect).inspect }
           end
 
           def patch_after_initialize
@@ -156,6 +126,49 @@ module Datadog
             end
           end
 
+          def patch_action_controller
+            ::ActiveSupport.on_load(:action_controller) do
+              GUARD_ACTION_CONTROLLER_ONCE_PER_APP[self].run do
+                ::ActionController::Base.prepend(Patches::RenderToBodyPatch)
+              end
+
+              # Rails 7.1 adds `after_routes_loaded` hook
+              if Datadog::AppSec::Contrib::Rails::Patcher.target_version < Gem::Version.new("7.1")
+                Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+              end
+            rescue => e
+              error_message = "Failed to get application routes"
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
+            end
+          end
+
+          def subscribe_to_routes_loaded
+            ::ActiveSupport.on_load(:after_routes_loaded) do
+              Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+            rescue => e
+              error_message = "Failed to get application routes"
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
+            end
+          end
+
+          def report_routes_via_telemetry(routes)
+            # We do not support Rails 4.x for Endpoint Collection,
+            # mainly because the Route#verb was a Regexp before Rails 5.0
+            return if target_version < Gem::Version.new("5.0")
+            return unless Datadog.configuration.appsec.api_security.endpoint_collection.enabled
+            return unless AppSec.telemetry
+
+            GUARD_ROUTES_REPORTING_ONCE_PER_APP[::Rails.application].run do
+              AppSec.telemetry.app_endpoints_loaded(
+                APISecurity::EndpointCollection::RailsCollector.new(routes).to_enum
+              )
+            end
+          rescue => e
+            AppSec.telemetry&.report(e, description: "failed to report application endpoints")
+          end
+
           def setup_security
             Datadog::AppSec::Contrib::Rails::Framework.setup
           end

data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Patches
+          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
+          module ProcessActionPatch
+            def process_action(*args)
+              context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              return super unless context
+
+              context.state[:web_framework] = "rails"
+
+              # TODO: handle exceptions, except for super
+              gateway_request = Gateway::Request.new(request)
+              http_response, _gateway_request = Instrumentation.gateway.push("rails.request.action", gateway_request) do
+                super
+              end
+
+              http_response
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/patches/render_to_body_patch.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "../../../utils/hash_coercion"
+require_relative "../../../instrumentation/gateway/argument"
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Patches
+          # A patch targeting `AbstractController::Rendering#render_to_body`
+          # method to capture JSON response body right before it is serialized.
+          module RenderToBodyPatch
+            def render_to_body(options = {})
+              return super unless options.key?(:json)
+
+              context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              return super unless context
+
+              data = Utils::HashCoercion.coerce(options[:json])
+              return super unless data
+
+              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              Instrumentation.gateway.push("rails.response.body.json", container)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -13,19 +13,19 @@ module Datadog
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON or
             # with Rails advanced param square bracket parsing
-            body = request.env['action_dispatch.request.request_parameters']
+            body = request.env["action_dispatch.request.request_parameters"]
 
             return if body.nil?
 
             body.reject do |k, _v|
-              request.env['action_dispatch.request.path_parameters'].key?(k)
+              request.env["action_dispatch.request.path_parameters"].key?(k)
             end
           end
 
           def self.route_params(request)
             excluded = [:controller, :action]
 
-            request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+            request.env["action_dispatch.request.path_parameters"].reject do |k, _v|
               excluded.include?(k)
             end
           end

data/lib/datadog/appsec/contrib/rest_client/integration.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "patcher"
+
+module Datadog
+  module AppSec
+    module Contrib
+      module RestClient
+        # This class defines properties of rest-client AppSec integration
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new("1.8")
+
+          register_as :rest_client
+
+          def self.gem_name
+            "rest-client"
+          end
+
+          def self.version
+            Gem.loaded_specs["rest-client"]&.version
+          end
+
+          def self.loaded?
+            !defined?(::RestClient::Request).nil?
+          end
+
+          def self.compatible?
+            super && !!(version&.>= MINIMUM_VERSION)
+          end
+
+          def self.auto_instrument?
+            false
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rest_client/patcher.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module RestClient
+        # Patcher for RestClient gem
+        module Patcher
+          module_function
+
+          def patched?
+            !!Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            require_relative "request_ssrf_detection_patch"
+
+            ::RestClient::Request.prepend(RequestSSRFDetectionPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
+
+module Datadog
+  module AppSec
+    module Contrib
+      module RestClient
+        # Module that adds SSRF detection to RestClient::Request#execute
+        module RequestSSRFDetectionPatch
+          REDIRECT_STATUS_CODES = (300..399).freeze
+
+          def execute(&block)
+            context = AppSec.active_context
+            return super unless context && AppSec.rasp_enabled?
+
+            headers = normalize_request_headers
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => method.to_s.upcase,
+              "server.io.net.request.headers" => headers
+            }
+
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              sample_body = true
+            else
+              sample_body = mark_body_sampling!(context)
+            end
+
+            if !is_redirect && sample_body
+              body = parse_request_body(payload, content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
+            handle(result, context: context) if result.match?
+
+            # NOTE: RestClient raises exceptions for non-2xx responses. For POST/PUT/PATCH
+            #       requests with 3xx redirects, RestClient raises instead of auto-following.
+            #       We rescue to process the response before re-raising.
+            begin
+              response = super
+            rescue ::RestClient::Exception => e
+              response = e.response
+              process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+
+              raise
+            end
+
+            process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+            response
+          end
+
+          def process_response(response, sample_body:)
+            context = AppSec.active_context
+            return unless context
+
+            headers = normalize_response_headers(response)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              "server.io.net.response.status" => response.code.to_s,
+              "server.io.net.response.headers" => headers
+            }
+
+            is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?("location")
+            context.state[:downstream_redirect_url] = URI.join(url, headers["location"]).to_s if is_redirect && sample_body
+
+            if sample_body && !is_redirect
+              body = parse_response_body(response.body, headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
+            handle(result, context: context) if result.match?
+          end
+
+          private
+
+          def mark_body_sampling!(context)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return false if context.state[:downstream_body_analyzed_count] >= max
+            return false unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            # NOTE: Request body analysis is best-effort, non-rewindable payloads are skipped
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
+          # NOTE: Starting version 2.1.0 headers are already normalized via internal
+          #       variable `@processed_headers_lowercase`. In case it's available,
+          #       we use it to avoid unnecessary transformation.
+          def normalize_request_headers
+            return @processed_headers_lowercase if defined?(@processed_headers_lowercase)
+
+            processed_headers.transform_keys(&:downcase)
+          end
+
+          # NOTE: Headers values are always an `Array` in `Net::HTTPResponse`,
+          #       but we want to avoid accidents and will wrap them in no-op
+          #       `Array` call just in case of a breaking change in the future
+          #
+          # FIXME: Steep has issues with `transform_values!` modifying the original
+          #        type and it failed with "Cannot allow block body" error
+          def normalize_response_headers(response) # steep:ignore MethodBodyTypeMismatch
+            response.net_http_res.to_hash
+              .transform_values! { |value| Array(value).join(", ") } # steep:ignore BlockBodyTypeMismatch
+          end
+
+          def handle(result, context:)
+            AppSec::Event.tag(context, result)
+            TraceKeeper.keep!(context.trace) if result.keep?
+
+            context.events.push(
+              AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+            )
+
+            AppSec::ActionsHandler.handle(result.actions)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/sinatra/gateway/request.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../rack/gateway/request'
+require_relative "../../rack/gateway/request"
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/gateway/route_params.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec