datadog 2.36.0 → 2.40.0

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
-require 'excon'
+require "excon"
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
-require_relative '../../utils/http/url_encoded'
-require_relative '../../utils/http/body'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/url_encoded"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -25,9 +27,9 @@ module Datadog
             headers = normalize_headers(data[:headers])
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => url,
-              'server.io.net.request.method' => data[:method].to_s.upcase,
-              'server.io.net.request.headers' => headers
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => data[:method].to_s.upcase,
+              "server.io.net.request.headers" => headers
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -39,8 +41,8 @@ module Datadog
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data[:body], content_type: headers['content-type'])
-              ephemeral_data['server.io.net.request.body'] = body if body
+              body = parse_request_body(data[:body], content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -57,18 +59,18 @@ module Datadog
             headers = normalize_headers(data.dig(:response, :headers))
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => data.dig(:response, :status).to_s,
-              'server.io.net.response.headers' => headers
+              "server.io.net.response.status" => data.dig(:response, :status).to_s,
+              "server.io.net.response.headers" => headers
             }
 
-            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?('location')
+            is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?("location")
             if is_redirect && data[SAMPLE_BODY_KEY]
-              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers['location']).to_s
+              context.state[:downstream_redirect_url] = URI.join(request_url(data), headers["location"]).to_s
             end
 
             if !is_redirect && data[SAMPLE_BODY_KEY]
-              body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
-              ephemeral_data['server.io.net.response.body'] = body if body
+              body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -89,15 +91,68 @@ module Datadog
             data[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def request_url(data)
-            klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
+            klass = (data[:scheme] == "https") ? URI::HTTPS : URI::HTTP
             klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
           end
 
@@ -105,7 +160,7 @@ module Datadog
             return {} if headers.nil? || headers.empty?
 
             headers.each_with_object({}) do |(key, value), memo|
-              memo[key.downcase] = value.is_a?(Array) ? value.join(', ') : value
+              memo[key.downcase] = value.is_a?(Array) ? value.join(", ") : value
             end
           end
 

data/lib/datadog/appsec/contrib/faraday/integration.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
+require_relative "../integration"
 
-require_relative 'patcher'
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -12,12 +12,12 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('0.14.0')
+          MINIMUM_VERSION = Gem::Version.new("0.14.0")
 
           register_as :faraday, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['faraday']&.version
+            Gem.loaded_specs["faraday"]&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/faraday/patcher.rb

@@ -17,9 +17,9 @@ module Datadog
           end
 
           def patch
-            require_relative 'ssrf_detection_middleware'
-            require_relative 'connection_patch'
-            require_relative 'rack_builder_patch'
+            require_relative "ssrf_detection_middleware"
+            require_relative "connection_patch"
+            require_relative "rack_builder_patch"
 
             ::Faraday::Middleware.register_middleware(datadog_appsec: SSRFDetectionMiddleware)
             configure_default_faraday_connection
@@ -28,7 +28,7 @@ module Datadog
           end
 
           def configure_default_faraday_connection
-            if target_version&.>= Gem::Version.new('1.0.0')
+            if target_version&.>= Gem::Version.new("1.0.0")
               # Patch the default connection (e.g. +Faraday.get+)
               ::Faraday.default_connection.use(:datadog_appsec)
 

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../event'
-require_relative '../../trace_keeper'
-require_relative '../../security_event'
-require_relative '../../utils/http/media_type'
-require_relative '../../utils/http/body'
+require_relative "../../event"
+require_relative "../../trace_keeper"
+require_relative "../../security_event"
+require_relative "../../utils/http/body"
+require_relative "../../utils/http/body_reader"
+require_relative "../../utils/http/media_type"
 
 module Datadog
   module AppSec
@@ -23,9 +24,9 @@ module Datadog
             headers = normalize_headers(env.request_headers)
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => url,
-              'server.io.net.request.method' => env.method.to_s.upcase,
-              'server.io.net.request.headers' => headers
+              "server.io.net.url" => url,
+              "server.io.net.request.method" => env.method.to_s.upcase,
+              "server.io.net.request.headers" => headers
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -37,8 +38,8 @@ module Datadog
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
-              ephemeral_data['server.io.net.request.body'] = body if body
+              body = parse_request_body(env.body, content_type: headers["content-type"])
+              ephemeral_data["server.io.net.request.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -54,18 +55,18 @@ module Datadog
             headers = normalize_headers(env.response_headers)
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.response.status' => env.status.to_s,
-              'server.io.net.response.headers' => headers
+              "server.io.net.response.status" => env.status.to_s,
+              "server.io.net.response.headers" => headers
             }
 
-            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
+            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?("location")
             if is_redirect && env[SAMPLE_BODY_KEY]
-              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
+              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers["location"]).to_s
             end
 
             if !is_redirect && env[SAMPLE_BODY_KEY]
-              body = parse_body(env.body, content_type: headers['content-type'])
-              ephemeral_data['server.io.net.response.body'] = body if body
+              body = parse_response_body(env.body, headers: headers, context: context)
+              ephemeral_data["server.io.net.response.body"] = body if body
             end
 
             timeout = Datadog.configuration.appsec.waf_timeout
@@ -82,11 +83,64 @@ module Datadog
             env[SAMPLE_BODY_KEY] = true
           end
 
-          def parse_body(body, content_type:)
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: limit)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers["content-type"])
+            if !media_type || media_type.type != "application"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != "json" && !subtype.end_with?("+json") && subtype != "x-www-form-urlencoded"
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers["content-length"]
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type, limit: content_length)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           def normalize_headers(headers)

data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require_relative 'gateway/multiplex'
-require_relative '../../instrumentation/gateway'
+require "json"
+require_relative "gateway/multiplex"
+require_relative "../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -16,7 +16,7 @@ module Datadog
 
             gateway_multiplex = Gateway::Multiplex.new(multiplex)
 
-            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push('graphql.multiplex', gateway_multiplex) do
+            multiplex_return, _gateway_multiplex = Instrumentation.gateway.push("graphql.multiplex", gateway_multiplex) do
               super
             end
 

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'graphql'
+require "graphql"
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec
@@ -72,11 +72,11 @@ module Datadog
                     args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
                   )
 
+                  # re-used fragments are not expanded
                   visited_fragments[fragment_name] = true
                   arguments_from_selections(
                     fragment.selections, query_variables, args_hash, fragments, visited_fragments
                   )
-                  visited_fragments.delete(fragment_name)
                 when ::GraphQL::Language::Nodes::Field
                   selection_name = selection.alias || selection.name
                   field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
-require_relative '../../../event'
-require_relative '../../../trace_keeper'
-require_relative '../../../security_event'
-require_relative '../../../instrumentation/gateway'
+require_relative "../../../event"
+require_relative "../../../trace_keeper"
+require_relative "../../../security_event"
+require_relative "../../../instrumentation/gateway"
 
 module Datadog
   module AppSec
@@ -22,12 +22,12 @@ module Datadog
               end
 
               def watch_multiplex(gateway = Instrumentation.gateway)
-                gateway.watch('graphql.multiplex') do |stack, gateway_multiplex|
+                gateway.watch("graphql.multiplex") do |stack, gateway_multiplex|
                   context = AppSec::Context.active
 
                   if context
                     persistent_data = {
-                      'graphql.server.all_resolvers' => gateway_multiplex.arguments
+                      "graphql.server.all_resolvers" => gateway_multiplex.arguments
                     }
 
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../integration'
-require_relative 'patcher'
+require_relative "../integration"
+require_relative "patcher"
 
 module Datadog
   module AppSec
@@ -11,20 +11,20 @@ module Datadog
         class Integration
           include Datadog::AppSec::Contrib::Integration
 
-          MINIMUM_VERSION = Gem::Version.new('2.0.19')
+          MINIMUM_VERSION = Gem::Version.new("2.0.19")
 
           AST_NODE_CLASS_NAMES = {
-            field: 'GraphQL::Language::Nodes::Field',
-            directive: 'GraphQL::Language::Nodes::Directive',
-            fragment_spread: 'GraphQL::Language::Nodes::FragmentSpread',
-            variable_identifier: 'GraphQL::Language::Nodes::VariableIdentifier',
-            input_object: 'GraphQL::Language::Nodes::InputObject',
+            field: "GraphQL::Language::Nodes::Field",
+            directive: "GraphQL::Language::Nodes::Directive",
+            fragment_spread: "GraphQL::Language::Nodes::FragmentSpread",
+            variable_identifier: "GraphQL::Language::Nodes::VariableIdentifier",
+            input_object: "GraphQL::Language::Nodes::InputObject",
           }.freeze
 
           register_as :graphql, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['graphql']&.version
+            Gem.loaded_specs["graphql"]&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/graphql/patcher.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'gateway/watcher'
+require_relative "gateway/watcher"
 
-if Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version >= Gem::Version.new('2.0.19')
-  require_relative 'appsec_trace'
+if Gem.loaded_specs["graphql"] && Gem.loaded_specs["graphql"].version >= Gem::Version.new("2.0.19")
+  require_relative "appsec_trace"
 end
 
 module Datadog

data/lib/datadog/appsec/contrib/rack/buffered_input.rb

@@ -24,7 +24,7 @@ module Datadog
 
           def read(length = nil, outbuf = nil)
             if length.nil?
-              data = @buffer.read(nil, outbuf) || +''
+              data = @buffer.read(nil, outbuf) || +""
               more = @stream.read
 
               data << more if more

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -7,38 +7,38 @@ module Datadog
         # Rack integration constants
         module Ext
           COLLECTABLE_REQUEST_HEADERS = [
-            'accept',
-            'akamai-user-risk',
-            'cf-ray',
-            'cloudfront-viewer-ja3-fingerprint',
-            'content-type',
-            'user-agent',
-            'x-amzn-trace-id',
-            'x-appgw-trace-id',
-            'x-cloud-trace-context',
-            'x-sigsci-requestid',
-            'x-sigsci-tags'
+            "accept",
+            "akamai-user-risk",
+            "cf-ray",
+            "cloudfront-viewer-ja3-fingerprint",
+            "content-type",
+            "user-agent",
+            "x-amzn-trace-id",
+            "x-appgw-trace-id",
+            "x-cloud-trace-context",
+            "x-sigsci-requestid",
+            "x-sigsci-tags"
           ].freeze
 
           IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
-            'accept-encoding',
-            'accept-language',
-            'cf-connecting-ip',
-            'cf-connecting-ipv6',
-            'content-encoding',
-            'content-language',
-            'content-length',
-            'fastly-client-ip',
-            'forwarded',
-            'forwarded-for',
-            'host',
-            'true-client-ip',
-            'via',
-            'x-client-ip',
-            'x-cluster-client-ip',
-            'x-forwarded',
-            'x-forwarded-for',
-            'x-real-ip'
+            "accept-encoding",
+            "accept-language",
+            "cf-connecting-ip",
+            "cf-connecting-ipv6",
+            "content-encoding",
+            "content-language",
+            "content-length",
+            "fastly-client-ip",
+            "forwarded",
+            "forwarded-for",
+            "host",
+            "true-client-ip",
+            "via",
+            "x-client-ip",
+            "x-cluster-client-ip",
+            "x-forwarded",
+            "x-forwarded-for",
+            "x-real-ip"
           ].freeze
         end
       end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../input_peeker'
-require_relative '../../../instrumentation/gateway/argument'
-require_relative '../../../../core/header_collection'
-require_relative '../../../../tracing/client_ip'
+require_relative "../input_peeker"
+require_relative "../../../instrumentation/gateway/argument"
+require_relative "../../../utils/http/url_encoded"
+require_relative "../../../../core/header_collection"
+require_relative "../../../../tracing/client_ip"
 
 module Datadog
   module AppSec
@@ -24,12 +25,7 @@ module Datadog
             end
 
             def query
-              ::Rack::Utils.parse_query(request.query_string)
-            rescue => e
-              Datadog.logger.debug { "AppSec: Failed to parse request query string: #{e.class}: #{e.message}" }
-              AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request query string')
-
-              {}
+              Utils::HTTP::URLEncoded.parse(request.query_string)
             end
 
             def method
@@ -38,12 +34,12 @@ module Datadog
 
             def headers
               result = request.env.each_with_object({}) do |(k, v), h|
-                h[k.delete_prefix('HTTP_').tap(&:downcase!).tap { |s| s.tr!('_', '-') }] = v if k.start_with?('HTTP_')
+                h[k.delete_prefix("HTTP_").tap(&:downcase!).tap { |s| s.tr!("_", "-") }] = v if k.start_with?("HTTP_")
               end
 
-              result['content-type'] = request.content_type if request.content_type
+              result["content-type"] = request.content_type if request.content_type
               # Since Rack 3.1, content-length is nil if the body is empty, but we still want to send it to the WAF.
-              result['content-length'] = request.content_length || '0'
+              result["content-length"] = request.content_length || "0"
               result
             end
 
@@ -72,7 +68,7 @@ module Datadog
             end
 
             def remote_addr
-              env['REMOTE_ADDR']
+              env["REMOTE_ADDR"]
             end
 
             def form_hash
@@ -82,7 +78,12 @@ module Datadog
 
               # usually Hash[String, String] but can be a more complex
               # Hash[String, (String|Array|Hash)] when e.g coming from JSON
-              env['rack.request.form_hash']
+              env["rack.request.form_hash"]
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request body: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: "AppSec: Failed to parse request body")
+
+              nil
             end
 
             # Returns the request body size in bytes using all available methods,
@@ -107,7 +108,7 @@ module Datadog
 
               # NOTE: A parsed-body cache without a raw byte count means the input
               #       stream was consumed before measurement, consider it unknown-length
-              return if env.key?('rack.request.form_hash')
+              return if env.key?("rack.request.form_hash")
 
               InputPeeker.peek_bytesize(env, limit: limit)
             end
@@ -118,7 +119,7 @@ module Datadog
             # NOTE: Rack does not parse JSON itself, a body parser middleware such as
             #       {Rack::JSONBodyParser} populates the form hash read by {#form_hash}
             def collectable_body?
-              request.form_data? || request.parseable_data? || env.key?('rack.request.form_hash')
+              request.form_data? || request.parseable_data? || env.key?("rack.request.form_hash")
             end
 
             def client_ip

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../../instrumentation/gateway/argument'
+require_relative "../../../instrumentation/gateway/argument"
 
 module Datadog
   module AppSec