datadog 2.28.0 → 2.41.0

data/lib/datadog/appsec/contrib/excon/integration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
-require_relative 'patcher'
+require_relative "../integration"
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -11,12 +11,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('0.50.0')
+          MINIMUM_VERSION = Gem::Version.new("0.50.0")
 
           register_as :excon
 
           def self.version
-            Gem.loaded_specs['excon']&.version
+            Gem.loaded_specs["excon"]&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/excon/patcher.rb

@@ -17,9 +17,11 @@ module Datadog
           end
 
           def patch
-            require_relative 'ssrf_detection_middleware'
+            require_relative "ssrf_detection_middleware"
 
             ::Excon.defaults[:middlewares].insert(0, SSRFDetectionMiddleware)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 
-require 'excon'
+require "excon"
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/url_encoded"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -12,17 +16,36 @@ module Datadog
       module Excon
         # AppSec Middleware for Excon
         class SSRFDetectionMiddleware < ::Excon::Middleware::Base
+          REDIRECT_STATUS_CODES = (300..399).freeze
+          SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
+
           def request_call(data)
             context = AppSec.active_context
             return super unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            url = request_url(data)
+            headers = normalize_headers(data[:headers])
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => request_url(data),
-              'server.io.net.request.method' => data[:method].to_s.upcase,
-              'server.io.net.request.headers' => normalize_headers(data[:headers])
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => data[:method].to_s.upcase,
+              "server.io.net.request.headers" => headers,
             }
 
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              data[SAMPLE_BODY_KEY] = true
+            else
+              mark_body_sampling!(data, context: context)
+            end
+
+            if !is_redirect && data[SAMPLE_BODY_KEY]
+              body = parse_request_body(data[:body], content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
             handle(result, context: context) if result.match?
 
@@ -33,12 +56,24 @@ module Datadog
             context = AppSec.active_context
             return super unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            headers = normalize_headers(data.dig(:response, :headers))
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => data.dig(:response, :status).to_s,
-              'server.io.net.response.headers' => normalize_headers(data.dig(:response, :headers))
+              "server.io.net.response.status" => data.dig(:response, :status).to_s,
+              "server.io.net.response.headers" => headers,
             }
 
+            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?("location")
+            if is_redirect && data[SAMPLE_BODY_KEY]
+              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers["location"]).to_s
+            end
+
+            if !is_redirect && data[SAMPLE_BODY_KEY]
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
             handle(result, context: context) if result.match?
 
@@ -47,16 +82,85 @@ module Datadog
 
           private
 
+          def mark_body_sampling!(data, context:)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return if context.state[:downstream_body_analyzed_count] >= max
+            return unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            data[SAMPLE_BODY_KEY] = true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
           def request_url(data)
-            klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
-            klass.build(host: data[:host], path: data[:path], query: data[:query]).to_s
+            klass = (data[:scheme] == "https") ? URI::HTTPS : URI::HTTP
+            klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
           end
 
           def normalize_headers(headers)
             return {} if headers.nil? || headers.empty?
 
             headers.each_with_object({}) do |(key, value), memo|
-              memo[key.downcase] = value.is_a?(Array) ? value.join(', ') : value
+              memo[key.downcase] = value.is_a?(Array) ? value.join(", ") : value
             end
           end
 

data/lib/datadog/appsec/contrib/faraday/integration.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
+require_relative "../integration"
 
-require_relative 'patcher'
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -12,12 +12,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('0.14.0')
+          MINIMUM_VERSION = Gem::Version.new("0.14.0")
 
           register_as :faraday, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['faraday']&.version
+            Gem.loaded_specs["faraday"]&.version
           end
 
           def self.loaded?
@@ -25,7 +25,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/faraday/patcher.rb

@@ -17,9 +17,9 @@ module Datadog
           end
 
           def patch
-            require_relative 'ssrf_detection_middleware'
-            require_relative 'connection_patch'
-            require_relative 'rack_builder_patch'
+            require_relative "ssrf_detection_middleware"
+            require_relative "connection_patch"
+            require_relative "rack_builder_patch"
 
             ::Faraday::Middleware.register_middleware(datadog_appsec: SSRFDetectionMiddleware)
             configure_default_faraday_connection
@@ -28,7 +28,7 @@ module Datadog
           end
 
           def configure_default_faraday_connection
-            if target_version >= Gem::Version.new('1.0.0')
+            if target_version&.>= Gem::Version.new("1.0.0")
               # Patch the default connection (e.g. +Faraday.get+)
               ::Faraday.default_connection.use(:datadog_appsec)
 

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -10,17 +13,36 @@ module Datadog
       module Faraday
         # AppSec SSRF detection Middleware for Faraday
         class SSRFDetectionMiddleware < ::Faraday::Middleware
+          REDIRECT_STATUS_CODES = (300..399).freeze
+          SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
+
           def call(env)
             context = AppSec.active_context
             return @app.call(env) unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            url = env.url.to_s
+            headers = normalize_headers(env.request_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => env.url.to_s,
-              'server.io.net.request.method' => env.method.to_s.upcase,
-              'server.io.net.request.headers' => env.request_headers.transform_keys(&:downcase)
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => env.method.to_s.upcase,
+              "server.io.net.request.headers" => headers,
             }
 
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              env[SAMPLE_BODY_KEY] = true
+            else
+              mark_body_sampling!(env, context: context)
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_request_body(env.body, content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
             handle(result, context: context) if result.match?
 
@@ -30,18 +52,103 @@ module Datadog
           private
 
           def on_complete(env, context:)
-            timeout = Datadog.configuration.appsec.waf_timeout
-
-            response_headers = env.response_headers || {}
+            headers = normalize_headers(env.response_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => env.status.to_s,
-              'server.io.net.response.headers' => response_headers.transform_keys(&:downcase)
+              "server.io.net.response.status" => env.status.to_s,
+              "server.io.net.response.headers" => headers,
             }
 
+            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?("location")
+            if is_redirect && env[SAMPLE_BODY_KEY]
+              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers["location"]).to_s
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_response_body(env.body, headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
             handle(result, context: context) if result.match?
           end
 
+          def mark_body_sampling!(env, context:)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return if context.state[:downstream_body_analyzed_count] >= max
+            return unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            env[SAMPLE_BODY_KEY] = true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
+          def normalize_headers(headers)
+            return {} if headers.nil? || headers.empty?
+
+            headers.transform_keys(&:downcase)
+          end
+
           def handle(result, context:)
             AppSec::Event.tag(context, result)
             TraceKeeper.keep!(context.trace) if result.keep?

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require_relative 'gateway/multiplex'
-require_relative '../../instrumentation/gateway'
+require "json"
+require_relative "gateway/multiplex"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -16,7 +16,7 @@ module Datadog
 
             gateway_multiplex = Gateway::Multiplex.new(multiplex)
 
-            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push("graphql.multiplex", gateway_multiplex) do
               super
             end
 

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'graphql'
+require "graphql"
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec
@@ -47,40 +47,86 @@ module Datadog
             # Note that the `comments` "include" directive is included in the arguments list
             def build_arguments_hash
               queries.each_with_object({}) do |query, args_hash|
-                next unless query.selected_operation
+                selected_operation = query.selected_operation
+                next unless selected_operation
 
-                arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
+                arguments_from_selections(selected_operation.selections, query.variables, args_hash, query.fragments)
               end
             end
 
-            def arguments_from_selections(selections, query_variables, args_hash)
+            def arguments_from_selections(selections, query_variables, args_hash, fragments, visited_fragments = {})
               selections.each do |selection|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless selection.class.name == Integration::AST_NODE_CLASS_NAMES[:field]
-                # rubocop:enable Style/ClassEqualityComparison
-
-                selection_name = selection.alias || selection.name
-
-                if !selection.arguments.empty? || !selection.directives.empty?
-                  args_hash[selection_name] ||= []
-                  args_hash[selection_name] <<
-                    arguments_hash(selection.arguments, query_variables).merge!(
-                      arguments_from_directives(selection.directives, query_variables)
-                    )
+                case selection
+                when ::GraphQL::Language::Nodes::FragmentSpread
+                  fragment_name = selection.name
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  next if visited_fragments[fragment_name]
+
+                  fragment = fragments[fragment_name]
+                  next unless fragment
+
+                  append_arguments(
+                    args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
+                  )
+
+                  # re-used fragments are not expanded
+                  visited_fragments[fragment_name] = true
+                  arguments_from_selections(
+                    fragment.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::Field
+                  selection_name = selection.alias || selection.name
+                  field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
+                  append_arguments(
+                    args_hash,
+                    selection_name,
+                    field_arguments,
+                    arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
+                when ::GraphQL::Language::Nodes::InlineFragment
+                  append_arguments(
+                    args_hash, selection.type.name, nil, arguments_from_directives(selection.directives, query_variables)
+                  )
+
+                  arguments_from_selections(
+                    selection.selections, query_variables, args_hash, fragments, visited_fragments
+                  )
                 end
+              end
+            end
 
-                arguments_from_selections(selection.selections, query_variables, args_hash)
+            def append_arguments(args_hash, selection_name, arguments, directive_arguments)
+              combined_arguments = if arguments
+                arguments.merge!(directive_arguments) if directive_arguments
+                arguments
+              else
+                directive_arguments
               end
+              return unless combined_arguments
+
+              args_hash[selection_name] ||= []
+              args_hash[selection_name] << combined_arguments
             end
 
             def arguments_from_directives(directives, query_variables)
-              directives.each_with_object({}) do |directive, args_hash|
-                # rubocop:disable Style/ClassEqualityComparison
-                next unless directive.class.name == Integration::AST_NODE_CLASS_NAMES[:directive]
-                # rubocop:enable Style/ClassEqualityComparison
+              return if directives.empty?
+
+              directive_arguments = directives.each_with_object({}) do |directive, args_hash|
+                next unless directive.is_a?(::GraphQL::Language::Nodes::Directive)
 
                 args_hash[directive.name] = arguments_hash(directive.arguments, query_variables)
               end
+
+              return if directive_arguments.empty?
+
+              directive_arguments
             end
 
             def arguments_hash(arguments, query_variables)
@@ -90,15 +136,19 @@ module Datadog
             end
 
             def argument_value(argument, query_variables)
-              case argument.value.class.name
+              value = argument.value
+
+              case value.class.name
               when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
+                # @type var value: GraphQL::Language::Nodes::VariableIdentifier
                 # we need to pass query.variables here instead of query.provided_variables,
                 # since #provided_variables don't know anything about variable default value
-                query_variables[argument.value.name]
+                query_variables[value.name]
               when Integration::AST_NODE_CLASS_NAMES[:input_object]
-                arguments_hash(argument.value.arguments, query_variables)
+                # @type var value: GraphQL::Language::Nodes::InputObject
+                arguments_hash(value.arguments, query_variables)
               else
-                argument.value
+                value
               end
             end
           end

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
-require_relative '../../../event'
-require_relative '../../../trace_keeper'
-require_relative '../../../security_event'
-require_relative '../../../instrumentation/gateway'
+require_relative "../../../event"
+require_relative "../../../trace_keeper"
+require_relative "../../../security_event"
+require_relative "../../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -22,12 +22,12 @@ module Datadog
               end
 
               def watch_multiplex(gateway = Instrumentation.gateway)
-                gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
+                gateway.watch("graphql.multiplex") do |stack, gateway_multiplex|
                   context = AppSec::Context.active
 
                   if context
                     persistent_data = {
-                      'graphql.server.all_resolvers' => gateway_multiplex.arguments
+                      "graphql.server.all_resolvers" => gateway_multiplex.arguments,
                     }
 
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)