datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,8 +1,11 @@
-# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -10,32 +13,154 @@ module Datadog
       module Faraday
         # AppSec SSRF detection Middleware for Faraday
         class SSRFDetectionMiddleware < ::Faraday::Middleware
-          def call(request_env)
+          REDIRECT_STATUS_CODES = (300..399).freeze
+          SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
+
+          def call(env)
             context = AppSec.active_context
+            return @app.call(env) unless context && AppSec.rasp_enabled?
+
+            url = env.url.to_s
+            headers = normalize_headers(env.request_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              'server.io.net.url' => url,
+              'server.io.net.request.method' => env.method.to_s.upcase,
+              'server.io.net.request.headers' => headers
+            }
+
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              env[SAMPLE_BODY_KEY] = true
+            else
+              mark_body_sampling!(env, context: context)
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_request_body(env.body, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
+            handle(result, context: context) if result.match?
+
+            @app.call(env).on_complete { |response_env| on_complete(response_env, context: context) }
+          end
 
-            return @app.call(request_env) unless context && AppSec.rasp_enabled?
+          private
 
+          def on_complete(env, context:)
+            headers = normalize_headers(env.response_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => request_env.url.to_s
+              'server.io.net.response.status' => env.status.to_s,
+              'server.io.net.response.headers' => headers
             }
 
-            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
+            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
+            if is_redirect && env[SAMPLE_BODY_KEY]
+              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_response_body(env.body, headers: headers, context: context)
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
+            handle(result, context: context) if result.match?
+          end
+
+          def mark_body_sampling!(env, context:)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return if context.state[:downstream_body_analyzed_count] >= max
+            return unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            env[SAMPLE_BODY_KEY] = true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
 
-            if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
 
-              context.events.push(
-                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
-              )
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
 
-              AppSec::ActionsHandler.handle(result.actions)
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
             end
 
-            @app.call(request_env)
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
+          def normalize_headers(headers)
+            return {} if headers.nil? || headers.empty?
+
+            headers.transform_keys(&:downcase)
+          end
+
+          def handle(result, context:)
+            AppSec::Event.tag(context, result)
+            TraceKeeper.keep!(context.trace) if result.keep?
+
+            context.events.push(
+              AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+            )
+
+            AppSec::ActionsHandler.handle(result.actions)
           end
         end
       end
     end
   end
 end
-# rubocop:enable Naming/FileName

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -47,40 +47,86 @@ module Datadog
             # Note that the `comments` "include" directive is included in the arguments list
             def build_arguments_hash
               queries.each_with_object({}) do |query, args_hash|
-                next unless query.selected_operation
+                selected_operation = query.selected_operation
+                next unless selected_operation
 
-                arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
+                arguments_from_selections(selected_operation.selections, query.variables, args_hash, query.fragments)
               end
             end
 
-            def arguments_from_selections(selections, query_variables, args_hash)
+            def arguments_from_selections(selections, query_variables, args_hash, fragments, visited_fragments = {})
               selections.each do |selection|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless selection.class.name == Integration::AST_NODE_CLASS_NAMES[:field]
-                # rubocop:enable Style/ClassEqualityComparison
-
-                selection_name = selection.alias || selection.name
-
-                if !selection.arguments.empty? || !selection.directives.empty?
-                  args_hash[selection_name] ||= []
-                  args_hash[selection_name] <<
-                    arguments_hash(selection.arguments, query_variables).merge!(
-                      arguments_from_directives(selection.directives, query_variables)
-                    )
+                case selection
+                when ::GraphQL::Language::Nodes::FragmentSpread
+                  fragment_name = selection.name
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  next if visited_fragments[fragment_name]
+
+                  fragment = fragments[fragment_name]
+                  next unless fragment
+
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
+                  )
+
+                  # re-used fragments are not expanded
+                  visited_fragments[fragment_name] = true
+                  arguments_from_selections(
+                    fragment.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::Field
+                  selection_name = selection.alias || selection.name
+                  field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
+                  append_arguments(
+                    args_hash,
+                    selection_name,
+                    field_arguments,
+                    arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::InlineFragment
+                  append_arguments(
+                    args_hash, selection.type.name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
                 end
+              end
+            end
 
-                arguments_from_selections(selection.selections, query_variables, args_hash)
+            def append_arguments(args_hash, selection_name, arguments, directive_arguments)
+              combined_arguments = if arguments
+                arguments.merge!(directive_arguments) if directive_arguments
+                arguments
+              else
+                directive_arguments
               end
+              return unless combined_arguments
+
+              args_hash[selection_name] ||= []
+              args_hash[selection_name] << combined_arguments
             end
 
             def arguments_from_directives(directives, query_variables)
-              directives.each_with_object({}) do |directive, args_hash|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless directive.class.name == Integration::AST_NODE_CLASS_NAMES[:directive]
-                # rubocop:enable Style/ClassEqualityComparison
+              return if directives.empty?
+
+              directive_arguments = directives.each_with_object({}) do |directive, args_hash|
+                next unless directive.is_a?(::GraphQL::Language::Nodes::Directive)
 
                 args_hash[directive.name] = arguments_hash(directive.arguments, query_variables)
               end
+
+              return if directive_arguments.empty?
+
+              directive_arguments
             end
 
             def arguments_hash(arguments, query_variables)
@@ -90,15 +136,19 @@ module Datadog
             end
 
             def argument_value(argument, query_variables)
-              case argument.value.class.name
+              value = argument.value
+
+              case value.class.name
               when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
+                # @type var value: GraphQL::Language::Nodes::VariableIdentifier
                 # we need to pass query.variables here instead of query.provided_variables,
                 # since #provided_variables don't know anything about variable default value
-                query_variables[argument.value.name]
+                query_variables[value.name]
               when Integration::AST_NODE_CLASS_NAMES[:input_object]
-                arguments_hash(argument.value.arguments, query_variables)
+                # @type var value: GraphQL::Language::Nodes::InputObject
+                arguments_hash(value.arguments, query_variables)
               else
-                argument.value
+                value
               end
             end
           end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -3,6 +3,7 @@
 require 'json'
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -21,7 +22,7 @@ module Datadog
               end
 
               def watch_multiplex(gateway = Instrumentation.gateway)
-                gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
+                gateway.watch('graphql.multiplex') do |stack, gateway_multiplex|
                   context = AppSec::Context.active
 
                   if context
@@ -32,7 +33,8 @@ module Datadog
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                     if result.match?
-                      AppSec::Event.tag_and_keep!(context, result)
+                      AppSec::Event.tag(context, result)
+                      TraceKeeper.keep!(context.trace) if result.keep?
 
                       context.events.push(
                         AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -16,6 +16,7 @@ module Datadog
           AST_NODE_CLASS_NAMES = {
             field: 'GraphQL::Language::Nodes::Field',
             directive: 'GraphQL::Language::Nodes::Directive',
+            fragment_spread: 'GraphQL::Language::Nodes::FragmentSpread',
             variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
             input_object: 'GraphQL::Language::Nodes::InputObject',
           }.freeze
@@ -31,7 +32,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION && ast_node_classes_defined?
+            !!(super && version&.>=(MINIMUM_VERSION) && ast_node_classes_defined?)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rack/buffered_input.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Wraps a `rack.input` stream with a buffer placed in front of it.
+        # Every read drains the buffer first, then continues from the stream
+        #
+        # NOTE: Forward-only: no rewind, no seek
+        #
+        # NOTE: Rack 3 dropped the rewind requirement from the input stream contract
+        # @see https://github.com/rack/rack/blob/v3.2.6/SPEC.rdoc
+        class BufferedInput
+          # NOTE: Rack's multipart parser reads in 1 MiB chunks, used to bound
+          #       {#each} the same way
+          # @see https://github.com/rack/rack/blob/v3.2.6/lib/rack/multipart/parser.rb#L54
+          READ_BUFSIZE_BYTES = 1_048_576
+
+          def initialize(stream, buffer:)
+            @stream = stream
+            @buffer = buffer
+          end
+
+          def read(length = nil, outbuf = nil)
+            if length.nil?
+              data = @buffer.read(nil, outbuf) || +''
+              more = @stream.read
+
+              data << more if more
+
+              return data
+            end
+
+            data = @buffer.read(length, outbuf)
+
+            if data.nil?
+              more = @stream.read(length, outbuf)
+              return more if more && !more.empty?
+
+              # NOTE: Match `IO#read(length, outbuf)` at EOF. Return nil and clear
+              #       the caller's buffer so stale bytes are not mistaken for data
+              outbuf&.clear
+              return
+            end
+
+            remaining = length - data.bytesize
+            return data if remaining <= 0
+
+            more = @stream.read(remaining)
+            data << more if more
+
+            data
+          end
+
+          def gets
+            line = @buffer.gets
+
+            return @stream.gets if line.nil?
+            return line if line.end_with?("\n")
+
+            more = @stream.gets
+            more ? (line << more) : line
+          end
+
+          def each
+            while (chunk = read(READ_BUFSIZE_BYTES))
+              yield chunk
+            end
+
+            self
+          end
+
+          def close
+            @buffer.close
+          ensure
+            @stream.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -13,7 +13,7 @@ module Datadog
             'cloudfront-viewer-ja3-fingerprint',
             'content-type',
             'user-agent',
-            'x-amzn-trace-Id',
+            'x-amzn-trace-id',
             'x-appgw-trace-id',
             'x-cloud-trace-context',
             'x-sigsci-requestid',

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../input_peeker'
 require_relative '../../../instrumentation/gateway/argument'
 require_relative '../../../../core/header_collection'
 require_relative '../../../../tracing/client_ip'
@@ -23,16 +24,12 @@ module Datadog
             end
 
             def query
-              # Downstream libddwaf expects keys and values to be extractable
-              # separately so we can't use [[k, v], ...]. We also want to allow
-              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
-              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
-              request.query_string.split('&').each.with_object({}) do |e, hash|
-                k, v = e.split('=').map { |s| CGI.unescape(s) }
-                hash[k] ||= []
-
-                hash[k] << v
-              end
+              ::Rack::Utils.parse_query(request.query_string)
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request query string: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request query string')
+
+              {}
             end
 
             def method
@@ -79,14 +76,51 @@ module Datadog
             end
 
             def form_hash
-              # force form data processing
+              # NOTE: Rack populates `rack.request.form_hash` only as a side effect
+              #       of {::Rack::Request#POST}, which reads and parses the body.
               request.POST if request.form_data?
 
-              # usually Hash<String,String> but can be a more complex
-              # Hash<String,String||Array||Hash> when e.g coming from JSON
+              # usually Hash[String, String] but can be a more complex
+              # Hash[String, (String|Array|Hash)] when e.g coming from JSON
               env['rack.request.form_hash']
             end
 
+            # Returns the request body size in bytes using all available methods,
+            # or nil when the size cannot be measured within the limit
+            #
+            # NOTE: The priority of the measurement is the following:
+            #       size if it's known, content-length if provided, and buffering
+            #       to the limit if unknown-length
+            #
+            # WARNING: The buffering path adds overhead for streaming web-servers
+            #          (Rack 3+) when the body length is unknown
+            def body_bytesize(limit)
+              io = request.body
+
+              return 0 unless io
+              return io.size if io.respond_to?(:size)
+
+              # NOTE: {::Rack::Request#content_length} is a plain `CONTENT_LENGTH`
+              #       getter forces no body read
+              content_length = request.content_length
+              return content_length.to_i if content_length
+
+              # NOTE: A parsed-body cache without a raw byte count means the input
+              #       stream was consumed before measurement, consider it unknown-length
+              return if env.key?('rack.request.form_hash')
+
+              InputPeeker.peek_bytesize(env, limit: limit)
+            end
+
+            # Whether a request body can be collected without forcing a parse:
+            # either form data parseable on demand, or a body already parsed upstream
+            #
+            # NOTE: Rack does not parse JSON itself, a body parser middleware such as
+            #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
+            def collectable_body?
+              request.form_data? || request.parseable_data? || env.key?('rack.request.form_hash')
+            end
+
             def client_ip
               remote_ip = remote_addr
               header_collection = Datadog::Core::HeaderCollection.from_hash(headers)

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -2,6 +2,7 @@
 
 require_relative '../ext'
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -23,7 +24,7 @@ module Datadog
               end
 
               def watch_request(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
                   persistent_data = {
@@ -38,14 +39,16 @@ module Datadog
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match? || !result.derivatives.empty?
+                  if result.match? || !result.attributes.empty?
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
                   end
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 
@@ -54,7 +57,7 @@ module Datadog
               end
 
               def watch_response(gateway = Instrumentation.gateway)
-                gateway.watch('rack.response', :appsec) do |stack, gateway_response|
+                gateway.watch('rack.response') do |stack, gateway_response|
                   context = gateway_response.context
 
                   persistent_data = {
@@ -66,7 +69,8 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
 
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
@@ -80,17 +84,39 @@ module Datadog
               end
 
               def watch_request_body(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request.body') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?('rack.request.form_hash')
+                    body = gateway_request.env['rack.request.form_hash']
+                    persistent_data['server.request.body'] = body if body
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
 
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
@@ -108,20 +134,15 @@ module Datadog
               #       somewhere closer to identity related monitor.
               # WARNING: The Gateway is a subject of refactoring
               def watch_request_finish(gateway = Instrumentation.gateway)
-                gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
+                gateway.watch('rack.request.finish') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
 
-                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
-                    next stack.call(gateway_request.request)
-                  end
+                  next stack.call(gateway_request.request) if context.span.nil?
 
                   gateway_request.headers.each do |name, value|
-                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
-                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
-                      next
+                    if context.state[:has_identity_event] && Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      context.span["http.request.headers.#{name}"] ||= value
                     end
-
-                    context.span["http.request.headers.#{name}"] ||= value
                   end
 
                   stack.call(gateway_request.request)