datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/contrib/rack/input_peeker.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative 'buffered_input'
+require_relative '../../utils/http/body_reader'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Peeks at `rack.input` without changing what downstream Rack code can read
+        #
+        # @api private
+        module InputPeeker
+          module_function
+
+          # Peeks at `env['rack.input']` and returns the body bytesize when it
+          # can be measured within the given limit
+          #
+          # NOTE: Over-limit bodies cannot be fully measured, so returning `nil`
+          #       is the tradeoff we accept
+          #
+          # WARNING: For forward-only input, replaces `env['rack.input']` with a
+          #          replay stream over the peeked bytes, preserving the rest
+          #          of the original stream
+          #
+          # @param env [Hash] Rack environment
+          # @param limit [Integer] Maximum body bytesize to measure
+          # @return [Integer, nil] `Integer` bytesize when measured within
+          #   `limit`, including `0`; `nil` when input is missing, over the
+          #   limit, or cannot be preserved for downstream reads
+          def peek_bytesize(env, limit:)
+            rack_input = env['rack.input']
+            return unless rack_input
+
+            rewindable = rewind? && rack_input.respond_to?(:rewind)
+
+            # NOTE: Rack 2 requires `rack.input` to be rewindable. Rewind before peeking
+            #       in case an upstream framework already consumed part of the stream
+            return if rewindable && !rewind(rack_input)
+
+            # NOTE: Read one byte past the limit to distinguish an exact-limit body
+            #       from an over-limit body without reading the whole stream.
+            buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
+            over_limit = buffer.bytesize > limit
+
+            if rewindable
+              # NOTE: If we cannot rewind after peeking, downstream code would observe
+              #       a partially consumed body. Treat it as not safely collectable
+              return unless rewind(rack_input)
+            else
+              env['rack.input'] = if over_limit
+                BufferedInput.new(rack_input, buffer: StringIO.new(buffer))
+              else
+                StringIO.new(buffer)
+              end
+            end
+
+            over_limit ? nil : buffer.bytesize
+          end
+
+          private_class_method def rewind?
+            return @rewind if defined?(@rewind)
+
+            @rewind = ::Gem::Version.new(::Rack.release) < ::Gem::Version.new('3')
+          end
+
+          private_class_method def rewind(io)
+            io.rewind
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/integration.rb

@@ -27,7 +27,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -2,15 +2,19 @@
 
 require 'json'
 
+require_relative 'response_body'
 require_relative 'gateway/request'
 require_relative 'gateway/response'
 
 require_relative '../../event'
 require_relative '../../response'
-require_relative '../../processor'
+require_relative '../../api_security'
+require_relative '../../default_header_tags'
+require_relative '../../route_normalizer'
 require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
+require_relative '../../../core/header_collection'
 require_relative '../../../tracing/client_ip'
 require_relative '../../../tracing/contrib/rack/header_collection'
 
@@ -18,17 +22,6 @@ module Datadog
   module AppSec
     module Contrib
       module Rack
-        WAF_VENDOR_HEADERS_TAGS = %w[
-          X-Amzn-Trace-Id
-          Cloudfront-Viewer-Ja3-Fingerprint
-          Cf-Ray
-          X-Cloud-Trace-Context
-          X-Appgw-Trace-id
-          X-SigSci-RequestID
-          X-SigSci-Tags
-          Akamai-User-Risk
-        ].map(&:downcase).freeze
-
         # Topmost Rack middleware for AppSec
         # This should be inserted just below Datadog::Tracing::Contrib::Rack::TraceMiddleware
         class RequestMiddleware
@@ -36,7 +29,6 @@ module Datadog
             @app = app
 
             @oneshot_tags_sent = false
-            @rack_headers = {}
           end
 
           # rubocop:disable Metrics/MethodLength
@@ -46,33 +38,22 @@ module Datadog
             boot = Datadog::Core::Remote::Tie.boot
             Datadog::Core::Remote::Tie::Tracing.tag(boot, active_span)
 
-            processor = nil
-            ready = false
-            ctx = nil
-
             # For a given request, keep using the first Rack stack scope for
             # nested apps. Don't set `context` local variable so that on popping
             # out of this nested stack we don't finalize the parent's context
             return @app.call(env) if active_context(env)
 
-            Datadog::AppSec.reconfigure_lock do
-              processor = Datadog::AppSec.processor
-
-              if !processor.nil? && processor.ready?
-                ctx = Datadog::AppSec::Context.activate(
-                  Datadog::AppSec::Context.new(active_trace, active_span, processor)
-                )
-
-                env[Datadog::AppSec::Ext::CONTEXT_KEY] = ctx
-                ready = true
-              end
-            end
+            security_engine = Datadog::AppSec.security_engine
 
             # TODO: handle exceptions, except for @app.call
+            return @app.call(env) unless security_engine
 
-            return @app.call(env) unless ready
+            ctx = Datadog::AppSec::Context.activate(
+              Datadog::AppSec::Context.new(active_trace, active_span, security_engine.new_runner)
+            )
+            env[Datadog::AppSec::Ext::CONTEXT_KEY] = ctx
 
-            add_appsec_tags(processor, ctx)
+            add_appsec_tags(ctx)
             add_request_tags(ctx, env)
 
             http_response = nil
@@ -96,22 +77,42 @@ module Datadog
               nil
             end
 
+            add_normalized_route_tag(ctx, env)
+
             if interrupt_params
+              ctx.mark_as_interrupted!
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
             end
 
-            if AppSec.perform_api_security_check?
-              ctx.events.push(
-                AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
-              )
+            # NOTE: This is not optimal, but in the current implementation
+            #       `gateway_response` is a container to dispatch response event
+            #       and in case of interruption it suppose to be `nil`.
+            #
+            #       `http_response` is a real response object in both cases, but
+            #       to save us some computations, we will use already pre-computed
+            #       `gateway_response` instead of re-creating it.
+            #
+            # WARNING: This part will be refactored.
+            tmp_response = if interrupt_params
+              Gateway::Response.new(http_response[2], http_response[0], http_response[1], context: ctx)
+            else
+              gateway_response
             end
 
-            AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)
+            if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
+                AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
+              ctx.extract_schema!
+            end
+
+            AppSec::Event.record(ctx, request: gateway_request)
 
+            add_response_tags(ctx, tmp_response)
             http_response
           ensure
             if ctx
               ctx.export_metrics
+              ctx.export_request_telemetry
+
               Datadog::AppSec::Context.deactivate
             end
           end
@@ -140,7 +141,7 @@ module Datadog
           end
 
           # standard:disable Metrics/MethodLength
-          def add_appsec_tags(processor, context)
+          def add_appsec_tags(context)
             span = context.span
             trace = context.trace
 
@@ -150,20 +151,15 @@ module Datadog
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            if processor.diagnostics
-              diagnostics = processor.diagnostics
-
-              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
+            if context.waf_runner_ruleset_version
+              span.set_tag('_dd.appsec.event_rules.version', context.waf_runner_ruleset_version)
 
-              unless @oneshot_tags_sent
+              unless oneshot_tags_sent?
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
-                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
-                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
+                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(context.waf_runner_known_addresses))
 
                 # Ensure these tags reach the backend
                 trace.keep!
@@ -176,33 +172,55 @@ module Datadog
           end
           # standard:enable Metrics/MethodLength
 
-          # standard:disable Metrics/MethodLength
           def add_request_tags(context, env)
             span = context.span
-
             return unless span
 
-            # Always add WAF vendors headers
-            WAF_VENDOR_HEADERS_TAGS.each do |lowercase_header|
-              rack_header = to_rack_header(lowercase_header)
-              span.set_tag("http.request.headers.#{lowercase_header}", env[rack_header]) if env[rack_header]
-            end
-
-            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
-              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+            headers = Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+            AppSec::DefaultHeaderTags.tag_request(span, headers)
 
+            if span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
               # always collect client ip, as this is part of AppSec provided functionality
               Datadog::Tracing::ClientIp.set_client_ip_tag!(
-                span,
-                headers: request_header_collection,
-                remote_ip: env['REMOTE_ADDR']
+                span, headers: headers, remote_ip: env['REMOTE_ADDR']
               )
             end
           end
-          # standard:enable Metrics/MethodLength
 
-          def to_rack_header(header)
-            @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)
+          def add_response_tags(context, response)
+            span = context.span
+            return unless span
+
+            AppSec::DefaultHeaderTags.tag_response(
+              span, Datadog::Core::HeaderCollection.from_hash(response.headers)
+            )
+
+            unless response.headers.key?('content-length')
+              length = ResponseBody.content_length(response.body)
+              span.set_tag('http.response.headers.content-length', length.to_s) if length
+            end
+          end
+
+          def add_normalized_route_tag(context, env)
+            return unless AppSec::APISecurity.enabled?
+
+            span = context.span
+            return unless span
+
+            pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
+            return unless pattern
+
+            # NOTE: To build full path that covers mounted engines we need to add
+            #       pre-computed by Tracer route path tag to the normalized route
+            prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env['SCRIPT_NAME']
+            normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
+            return unless normalized_route
+
+            span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
+          end
+
+          def oneshot_tags_sent?
+            @oneshot_tags_sent
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/response_body.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module ResponseBody
+          # NOTE: We compute content length only for fixed-size response bodies,
+          #       ignoring streaming bodies to avoid buffering.
+          #
+          #       On Rack 3.x, `body.to_ary` on a BodyProxy triggers `close` on all
+          #       nested BodyProxy layers. This is safe because web servers, like Puma
+          #       handles already-closed bodies (its own `to_ary` becomes a no-op).
+          #
+          # @see Puma::Response#prepare_response
+          # @see https://github.com/puma/puma/blob/b1271222cbf21868f3fb64154caa4d03936a7b9e/lib/puma/response.rb#L165-L168
+          def self.content_length(body)
+            return unless body.respond_to?(:to_ary)
+
+            # NOTE: When `to_ary` exists but returns `nil`, the body will be
+            #       transferred in chunks and we can't compute content length
+            #       without buffering it.
+            body_ary = body.to_ary
+            return unless body_ary.is_a?(Array)
+
+            body_ary.sum { |part| part.is_a?(String) ? part.bytesize : 0 }
+          rescue => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to compute body content length')
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../../rack/input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 
 module Datadog
@@ -46,6 +47,7 @@ module Datadog
               body = request.env['action_dispatch.request.request_parameters']
 
               return if body.nil?
+              return body unless request.env['action_dispatch.request.path_parameters']
 
               body.reject do |k, _v|
                 request.env['action_dispatch.request.path_parameters'].key?(k)
@@ -55,10 +57,42 @@ module Datadog
             def route_params
               excluded = [:controller, :action]
 
-              request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+              request.env.fetch('action_dispatch.request.path_parameters', {}).reject do |k, _v|
                 excluded.include?(k)
               end
             end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       raw posted data, raw form vars, size if known, raw
+            #       Content-Length, then buffering to the limit if unknown-length
+            def body_bytesize(limit)
+              raw_body = env['RAW_POST_DATA']
+              return raw_body.bytesize if raw_body
+
+              form_vars = env['rack.request.form_vars']
+              return form_vars.bytesize if form_vars
+
+              io = request.body
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: Read raw `CONTENT_LENGTH` as {ActionDispatch::Request#content_length}
+              #       drains `rack.input` into `RAW_POST_DATA` on chunked Transfer-Encoding
+              content_length = env['CONTENT_LENGTH']
+              return content_length.to_i if content_length
+
+              # NOTE: An already-read body (e.g. late-parsed multipart on Rack 3+) peeks
+              #       as 0, so we skip byte_length but still collect the parsed body
+              begin
+                Rack::InputPeeker.peek_bytesize(env, limit: limit)
+              rescue => e
+                Datadog.logger.debug { "AppSec: Failed to measure Rails request body: #{e.class}: #{e.message}" }
+                nil
+              end
+            end
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -16,31 +17,72 @@ module Datadog
                 gateway = Instrumentation.gateway
 
                 watch_request_action(gateway)
+                watch_response_body_json(gateway)
               end
 
               def watch_request_action(gateway = Instrumentation.gateway)
-                gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
+                gateway.watch('rails.request.action') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
 
                   persistent_data = {
-                    'server.request.body' => gateway_request.parsed_body,
                     'server.request.path_params' => gateway_request.route_params
                   }
 
+                  unless limit.zero?
+                    byte_length = gateway_request.body_bytesize(limit)
+
+                    if byte_length
+                      # NOTE: Params may be parsed before this hook, leaving the
+                      #       body stream at EOF. Keep byte_length unset for that
+                      #       case, but still inspect cached params
+                      persistent_data['server.request.body.byte_length'] = byte_length if byte_length.positive?
+
+                      if byte_length <= limit
+                        body = gateway_request.parsed_body
+                        persistent_data['server.request.body'] = body unless body.nil? || body.empty?
+                      end
+                    end
+                  end
+
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match? || !result.derivatives.empty?
+                  if result.match?
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
+
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
+                    AppSec::ActionsHandler.handle(result.actions)
                   end
 
+                  stack.call(gateway_request.request)
+                end
+              end
+
+              def watch_response_body_json(gateway = Instrumentation.gateway)
+                gateway.watch('rails.response.body.json') do |stack, container|
+                  context = container.context
+
+                  persistent_data = {
+                    'server.response.body' => container.data
+                  }
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 
-                  stack.call(gateway_request.request)
+                  stack.call(container)
                 end
               end
             end

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -26,7 +26,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -8,6 +8,9 @@ require_relative '../rack/request_middleware'
 require_relative '../rack/request_body_middleware'
 require_relative 'gateway/watcher'
 require_relative 'gateway/request'
+require_relative 'patches/render_to_body_patch'
+require_relative 'patches/process_action_patch'
+require_relative '../../api_security/endpoint_collection/rails_collector'
 
 require_relative '../../../tracing/contrib/rack/middlewares'
 
@@ -17,6 +20,8 @@ module Datadog
       module Rails
         # Patcher for AppSec on Rails
         module Patcher
+          GUARD_ACTION_CONTROLLER_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
+          GUARD_ROUTES_REPORTING_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           BEFORE_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
           AFTER_INITIALIZE_ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Datadog::Core::Utils::OnlyOnce.new }
 
@@ -34,6 +39,8 @@ module Datadog
             Gateway::Watcher.watch
             patch_before_initialize
             patch_after_initialize
+            patch_action_controller
+            subscribe_to_routes_loaded
 
             Patcher.instance_variable_set(:@patched, true)
           end
@@ -49,7 +56,8 @@ module Datadog
               # Middleware must be added before the application is initialized.
               # Otherwise the middleware stack will be frozen.
               add_middleware(app) if Datadog.configuration.tracing[:rails][:middleware]
-              patch_process_action
+
+              ::ActionController::Metal.prepend(Patches::ProcessActionPatch)
             end
           end
 
@@ -65,31 +73,6 @@ module Datadog
             end
           end
 
-          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
-          module ProcessActionPatch
-            def process_action(*args)
-              env = request.env
-
-              context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
-
-              return super unless context
-
-              # TODO: handle exceptions, except for super
-
-              gateway_request = Gateway::Request.new(request)
-
-              http_response, _gateway_request = Instrumentation.gateway.push('rails.request.action', gateway_request) do
-                super
-              end
-
-              http_response
-            end
-          end
-
-          def patch_process_action
-            ::ActionController::Metal.prepend(ProcessActionPatch)
-          end
-
           def include_middleware?(middleware, app)
             found = false
 
@@ -143,6 +126,49 @@ module Datadog
             end
           end
 
+          def patch_action_controller
+            ::ActiveSupport.on_load(:action_controller) do
+              GUARD_ACTION_CONTROLLER_ONCE_PER_APP[self].run do
+                ::ActionController::Base.prepend(Patches::RenderToBodyPatch)
+              end
+
+              # Rails 7.1 adds `after_routes_loaded` hook
+              if Datadog::AppSec::Contrib::Rails::Patcher.target_version < Gem::Version.new('7.1')
+                Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+              end
+            rescue => e
+              error_message = 'Failed to get application routes'
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
+            end
+          end
+
+          def subscribe_to_routes_loaded
+            ::ActiveSupport.on_load(:after_routes_loaded) do
+              Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+            rescue => e
+              error_message = 'Failed to get application routes'
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
+            end
+          end
+
+          def report_routes_via_telemetry(routes)
+            # We do not support Rails 4.x for Endpoint Collection,
+            # mainly because the Route#verb was a Regexp before Rails 5.0
+            return if target_version < Gem::Version.new('5.0')
+            return unless Datadog.configuration.appsec.api_security.endpoint_collection.enabled
+            return unless AppSec.telemetry
+
+            GUARD_ROUTES_REPORTING_ONCE_PER_APP[::Rails.application].run do
+              AppSec.telemetry.app_endpoints_loaded(
+                APISecurity::EndpointCollection::RailsCollector.new(routes).to_enum
+              )
+            end
+          rescue => e
+            AppSec.telemetry&.report(e, description: 'failed to report application endpoints')
+          end
+
           def setup_security
             Datadog::AppSec::Contrib::Rails::Framework.setup
           end

data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Patches
+          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
+          module ProcessActionPatch
+            def process_action(*args)
+              context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              return super unless context
+
+              context.state[:web_framework] = 'rails'
+
+              # TODO: handle exceptions, except for super
+              gateway_request = Gateway::Request.new(request)
+              http_response, _gateway_request = Instrumentation.gateway.push('rails.request.action', gateway_request) do
+                super
+              end
+
+              http_response
+            end
+          end
+        end
+      end
+    end
+  end
+end