datadog 2.35.0 → 2.37.0

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -7,6 +7,8 @@ require_relative '../../trace_keeper'
 require_relative '../../security_event'
 require_relative '../../utils/http/url_encoded'
 require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -39,7 +41,7 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data[:body], content_type: headers['content-type'])
+              body = parse_request_body(data[:body], content_type: headers['content-type'])
               ephemeral_data['server.io.net.request.body'] = body if body
             end
 
@@ -67,7 +69,7 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
               ephemeral_data['server.io.net.response.body'] = body if body
             end
 
@@ -89,11 +91,64 @@ module Datadog
             data[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def request_url(data)

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -3,8 +3,9 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
-require_relative '../../utils/http/media_type'
 require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -37,7 +38,7 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
+              body = parse_request_body(env.body, content_type: headers['content-type'])
               ephemeral_data['server.io.net.request.body'] = body if body
             end
 
@@ -64,7 +65,7 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
+              body = parse_response_body(env.body, headers: headers, context: context)
               ephemeral_data['server.io.net.response.body'] = body if body
             end
 
@@ -82,11 +83,64 @@ module Datadog
             env[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def normalize_headers(headers)

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -50,38 +50,83 @@ module Datadog
                 selected_operation = query.selected_operation
                 next unless selected_operation
 
-                arguments_from_selections(selected_operation.selections, query.variables, args_hash)
+                arguments_from_selections(selected_operation.selections, query.variables, args_hash, query.fragments)
               end
             end
 
-            def arguments_from_selections(selections, query_variables, args_hash)
+            def arguments_from_selections(selections, query_variables, args_hash, fragments, visited_fragments = {})
               selections.each do |selection|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless selection.class.name == Integration::AST_NODE_CLASS_NAMES[:field]
-                # rubocop:enable Style/ClassEqualityComparison
-
-                selection_name = selection.alias || selection.name
-
-                if !selection.arguments.empty? || !selection.directives.empty?
-                  args_hash[selection_name] ||= []
-                  args_hash[selection_name] <<
-                    arguments_hash(selection.arguments, query_variables).merge!(
-                      arguments_from_directives(selection.directives, query_variables)
-                    )
+                case selection
+                when ::GraphQL::Language::Nodes::FragmentSpread
+                  fragment_name = selection.name
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  next if visited_fragments[fragment_name]
+
+                  fragment = fragments[fragment_name]
+                  next unless fragment
+
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
+                  )
+
+                  # re-used fragments are not expanded
+                  visited_fragments[fragment_name] = true
+                  arguments_from_selections(
+                    fragment.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::Field
+                  selection_name = selection.alias || selection.name
+                  field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
+                  append_arguments(
+                    args_hash,
+                    selection_name,
+                    field_arguments,
+                    arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::InlineFragment
+                  append_arguments(
+                    args_hash, selection.type.name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
                 end
+              end
+            end
 
-                arguments_from_selections(selection.selections, query_variables, args_hash)
+            def append_arguments(args_hash, selection_name, arguments, directive_arguments)
+              combined_arguments = if arguments
+                arguments.merge!(directive_arguments) if directive_arguments
+                arguments
+              else
+                directive_arguments
               end
+              return unless combined_arguments
+
+              args_hash[selection_name] ||= []
+              args_hash[selection_name] << combined_arguments
             end
 
             def arguments_from_directives(directives, query_variables)
-              directives.each_with_object({}) do |directive, args_hash|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless directive.class.name == Integration::AST_NODE_CLASS_NAMES[:directive]
-                # rubocop:enable Style/ClassEqualityComparison
+              return if directives.empty?
+
+              directive_arguments = directives.each_with_object({}) do |directive, args_hash|
+                next unless directive.is_a?(::GraphQL::Language::Nodes::Directive)
 
                 args_hash[directive.name] = arguments_hash(directive.arguments, query_variables)
               end
+
+              return if directive_arguments.empty?
+
+              directive_arguments
             end
 
             def arguments_hash(arguments, query_variables)

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -16,6 +16,7 @@ module Datadog
           AST_NODE_CLASS_NAMES = {
             field: 'GraphQL::Language::Nodes::Field',
             directive: 'GraphQL::Language::Nodes::Directive',
+            fragment_spread: 'GraphQL::Language::Nodes::FragmentSpread',
             variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
             input_object: 'GraphQL::Language::Nodes::InputObject',
           }.freeze

data/lib/datadog/appsec/contrib/rack/buffered_input.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Wraps a `rack.input` stream with a buffer placed in front of it.
+        # Every read drains the buffer first, then continues from the stream
+        #
+        # NOTE: Forward-only: no rewind, no seek
+        #
+        # NOTE: Rack 3 dropped the rewind requirement from the input stream contract
+        # @see https://github.com/rack/rack/blob/v3.2.6/SPEC.rdoc
+        class BufferedInput
+          # NOTE: Rack's multipart parser reads in 1 MiB chunks, used to bound
+          #       {#each} the same way
+          # @see https://github.com/rack/rack/blob/v3.2.6/lib/rack/multipart/parser.rb#L54
+          READ_BUFSIZE_BYTES = 1_048_576
+
+          def initialize(stream, buffer:)
+            @stream = stream
+            @buffer = buffer
+          end
+
+          def read(length = nil, outbuf = nil)
+            if length.nil?
+              data = @buffer.read(nil, outbuf) || +''
+              more = @stream.read
+
+              data << more if more
+
+              return data
+            end
+
+            data = @buffer.read(length, outbuf)
+
+            if data.nil?
+              more = @stream.read(length, outbuf)
+              return more if more && !more.empty?
+
+              # NOTE: Match `IO#read(length, outbuf)` at EOF. Return nil and clear
+              #       the caller's buffer so stale bytes are not mistaken for data
+              outbuf&.clear
+              return
+            end
+
+            remaining = length - data.bytesize
+            return data if remaining <= 0
+
+            more = @stream.read(remaining)
+            data << more if more
+
+            data
+          end
+
+          def gets
+            line = @buffer.gets
+
+            return @stream.gets if line.nil?
+            return line if line.end_with?("\n")
+
+            more = @stream.gets
+            more ? (line << more) : line
+          end
+
+          def each
+            while (chunk = read(READ_BUFSIZE_BYTES))
+              yield chunk
+            end
+
+            self
+          end
+
+          def close
+            @buffer.close
+          ensure
+            @stream.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 require_relative '../../../../core/header_collection'
 require_relative '../../../../tracing/client_ip'
@@ -75,14 +76,51 @@ module Datadog
             end
 
             def form_hash
-              # force form data processing
+              # NOTE: Rack populates `rack.request.form_hash` only as a side effect
+              #       of {::Rack::Request#POST}, which reads and parses the body.
               request.POST if request.form_data?
 
-              # usually Hash<String,String> but can be a more complex
-              # Hash<String,String||Array||Hash> when e.g coming from JSON
+              # usually Hash[String, String] but can be a more complex
+              # Hash[String, (String|Array|Hash)] when e.g coming from JSON
               env['rack.request.form_hash']
             end
 
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       size if it's known, content-length if provided, and buffering
+            #       to the limit if unknown-length
+            #
+            # WARNING: The buffering path adds overhead for streaming web-servers
+            #          (Rack 3+) when the body length is unknown
+            def body_bytesize(limit)
+              io = request.body
+
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: {::Rack::Request#content_length} is a plain `CONTENT_LENGTH`
+              #       getter forces no body read
+              content_length = request.content_length
+              return content_length.to_i if content_length
+
+              # NOTE: A parsed-body cache without a raw byte count means the input
+              #       stream was consumed before measurement, consider it unknown-length
+              return if env.key?('rack.request.form_hash')
+
+              InputPeeker.peek_bytesize(env, limit: limit)
+            end
+
+            # Whether a request body can be collected without forcing a parse:
+            # either form data parseable on demand, or a body already parsed upstream
+            #
+            # NOTE: Rack does not parse JSON itself, a body parser middleware such as
+            #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
+            def collectable_body?
+              request.form_data? || request.parseable_data? || env.key?('rack.request.form_hash')
+            end
+
             def client_ip
               remote_ip = remote_addr
               header_collection = Datadog::Core::HeaderCollection.from_hash(headers)

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -87,9 +87,30 @@ module Datadog
                 gateway.watch('rack.request.body') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?('rack.request.form_hash')
+                    body = gateway_request.env['rack.request.form_hash']
+                    persistent_data['server.request.body'] = body if body
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
@@ -119,10 +140,6 @@ module Datadog
                   next stack.call(gateway_request.request) if context.span.nil?
 
                   gateway_request.headers.each do |name, value|
-                    if Ext::COLLECTABLE_REQUEST_HEADERS.include?(name)
-                      context.span["http.request.headers.#{name}"] ||= value
-                    end
-
                     if context.state[:has_identity_event] && Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
                       context.span["http.request.headers.#{name}"] ||= value
                     end

data/lib/datadog/appsec/contrib/rack/input_peeker.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative 'buffered_input'
+require_relative '../../utils/http/body_reader'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Peeks at `rack.input` without changing what downstream Rack code can read
+        #
+        # @api private
+        module InputPeeker
+          module_function
+
+          # Peeks at `env['rack.input']` and returns the body bytesize when it
+          # can be measured within the given limit
+          #
+          # NOTE: Over-limit bodies cannot be fully measured, so returning `nil`
+          #       is the tradeoff we accept
+          #
+          # WARNING: For forward-only input, replaces `env['rack.input']` with a
+          #          replay stream over the peeked bytes, preserving the rest
+          #          of the original stream
+          #
+          # @param env [Hash] Rack environment
+          # @param limit [Integer] Maximum body bytesize to measure
+          # @return [Integer, nil] `Integer` bytesize when measured within
+          #   `limit`, including `0`; `nil` when input is missing, over the
+          #   limit, or cannot be preserved for downstream reads
+          def peek_bytesize(env, limit:)
+            rack_input = env['rack.input']
+            return unless rack_input
+
+            rewindable = rewind? && rack_input.respond_to?(:rewind)
+
+            # NOTE: Rack 2 requires `rack.input` to be rewindable. Rewind before peeking
+            #       in case an upstream framework already consumed part of the stream
+            return if rewindable && !rewind(rack_input)
+
+            # NOTE: Read one byte past the limit to distinguish an exact-limit body
+            #       from an over-limit body without reading the whole stream.
+            buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
+            over_limit = buffer.bytesize > limit
+
+            if rewindable
+              # NOTE: If we cannot rewind after peeking, downstream code would observe
+              #       a partially consumed body. Treat it as not safely collectable
+              return unless rewind(rack_input)
+            else
+              env['rack.input'] = if over_limit
+                BufferedInput.new(rack_input, buffer: StringIO.new(buffer))
+              else
+                StringIO.new(buffer)
+              end
+            end
+
+            over_limit ? nil : buffer.bytesize
+          end
+
+          private_class_method def rewind?
+            return @rewind if defined?(@rewind)
+
+            @rewind = ::Gem::Version.new(::Rack.release) < ::Gem::Version.new('3')
+          end
+
+          private_class_method def rewind(io)
+            io.rewind
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -10,6 +10,7 @@ require_relative '../../event'
 require_relative '../../response'
 require_relative '../../api_security'
 require_relative '../../default_header_tags'
+require_relative '../../route_normalizer'
 require_relative '../../security_event'
 require_relative '../../instrumentation/gateway'
 
@@ -76,6 +77,8 @@ module Datadog
               nil
             end
 
+            add_normalized_route_tag(ctx, env)
+
             if interrupt_params
               ctx.mark_as_interrupted!
               http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
@@ -198,6 +201,24 @@ module Datadog
             end
           end
 
+          def add_normalized_route_tag(context, env)
+            return unless AppSec::APISecurity.enabled?
+
+            span = context.span
+            return unless span
+
+            pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
+            return unless pattern
+
+            # NOTE: To build full path that covers mounted engines we need to add
+            #       pre-computed by Tracer route path tag to the normalized route
+            prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env['SCRIPT_NAME']
+            normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
+            return unless normalized_route
+
+            span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
+          end
+
           def oneshot_tags_sent?
             @oneshot_tags_sent
           end

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../../rack/input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 
 module Datadog
@@ -46,6 +47,7 @@ module Datadog
               body = request.env['action_dispatch.request.request_parameters']
 
               return if body.nil?
+              return body unless request.env['action_dispatch.request.path_parameters']
 
               body.reject do |k, _v|
                 request.env['action_dispatch.request.path_parameters'].key?(k)
@@ -55,10 +57,42 @@ module Datadog
             def route_params
               excluded = [:controller, :action]
 
-              request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+              request.env.fetch('action_dispatch.request.path_parameters', {}).reject do |k, _v|
                 excluded.include?(k)
               end
             end
+
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       raw posted data, raw form vars, size if known, raw
+            #       Content-Length, then buffering to the limit if unknown-length
+            def body_bytesize(limit)
+              raw_body = env['RAW_POST_DATA']
+              return raw_body.bytesize if raw_body
+
+              form_vars = env['rack.request.form_vars']
+              return form_vars.bytesize if form_vars
+
+              io = request.body
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: Read raw `CONTENT_LENGTH` as {ActionDispatch::Request#content_length}
+              #       drains `rack.input` into `RAW_POST_DATA` on chunked Transfer-Encoding
+              content_length = env['CONTENT_LENGTH']
+              return content_length.to_i if content_length
+
+              # NOTE: An already-read body (e.g. late-parsed multipart on Rack 3+) peeks
+              #       as 0, so we skip byte_length but still collect the parsed body
+              begin
+                Rack::InputPeeker.peek_bytesize(env, limit: limit)
+              rescue => e
+                Datadog.logger.debug { "AppSec: Failed to measure Rails request body: #{e.class}: #{e.message}" }
+                nil
+              end
+            end
           end
         end
       end