datadog 2.35.0 → 2.37.0

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -23,12 +23,28 @@ module Datadog
               def watch_request_action(gateway = Instrumentation.gateway)
                 gateway.watch('rails.request.action') do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
 
                   persistent_data = {
-                    'server.request.body' => gateway_request.parsed_body,
                     'server.request.path_params' => gateway_request.route_params
                   }
 
+                  unless limit.zero?
+                    byte_length = gateway_request.body_bytesize(limit)
+
+                    if byte_length
+                      # NOTE: Params may be parsed before this hook, leaving the
+                      #       body stream at EOF. Keep byte_length unset for that
+                      #       case, but still inspect cached params
+                      persistent_data['server.request.body.byte_length'] = byte_length if byte_length.positive?
+
+                      if byte_length <= limit
+                        body = gateway_request.parsed_body
+                        persistent_data['server.request.body'] = body unless body.nil? || body.empty?
+                      end
+                    end
+                  end
+
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -3,8 +3,9 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
-require_relative '../../utils/http/media_type'
 require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -35,7 +36,7 @@ module Datadog
             end
 
             if !is_redirect && sample_body
-              body = parse_body(payload.to_s, content_type: headers['content-type'])
+              body = parse_request_body(payload, content_type: headers['content-type'])
               ephemeral_data['server.io.net.request.body'] = body if body
             end
 
@@ -74,7 +75,7 @@ module Datadog
             context.state[:downstream_redirect_url] = URI.join(url, headers['location']).to_s if is_redirect && sample_body
 
             if sample_body && !is_redirect
-              body = parse_body(response.body, content_type: headers['content-type'])
+              body = parse_response_body(response.body, headers: headers, context: context)
               ephemeral_data['server.io.net.response.body'] = body if body
             end
 
@@ -94,13 +95,65 @@ module Datadog
             true
           end
 
-          def parse_body(body, content_type:)
-            return if body.empty?
-
+          def parse_request_body(body, content_type:)
             media_type = Utils::HTTP::MediaType.parse(content_type)
             return unless media_type
 
-            Utils::HTTP::Body.parse(body, media_type: media_type)
+            # NOTE: Request body analysis is best-effort, non-rewindable payloads are skipped
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
+            end
+
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
           end
 
           # NOTE: Starting version 2.1.0 headers are already normalized via internal

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -27,9 +27,30 @@ module Datadog
 
                   context.state[:web_framework] = 'sinatra'
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?('rack.request.form_hash')
+                    body = gateway_request.env['rack.request.form_hash']
+                    persistent_data['server.request.body'] = body if body
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/default_header_tags.rb

@@ -10,20 +10,24 @@ module Datadog
     #
     # @api private
     module DefaultHeaderTags
-      WAF_VENDOR_HEADERS_TAGS = %w[
+      # NOTE: Contains additional WAF vendor headers
+      REQUEST_HEADERS_TAGS = %w[
+        accept
+        content-type
+        user-agent
+        akamai-user-risk
         x-amzn-trace-id
-        cloudfront-viewer-ja3-fingerprint
-        cf-ray
         x-cloud-trace-context
         x-appgw-trace-id
         x-sigsci-requestid
         x-sigsci-tags
-        akamai-user-risk
+        cf-ray
+        cloudfront-viewer-ja3-fingerprint
       ].freeze
 
       RESPONSE_HEADERS_TAGS = %w[
-        content-length
         content-type
+        content-length
         content-encoding
         content-language
       ].freeze
@@ -31,7 +35,7 @@ module Datadog
       module_function
 
       def tag_request(span, headers)
-        WAF_VENDOR_HEADERS_TAGS.each do |name|
+        REQUEST_HEADERS_TAGS.each do |name|
           value = headers.get(name)
           span.set_tag("http.request.headers.#{name}", value) if value
         end

data/lib/datadog/appsec/ext.rb

@@ -17,6 +17,7 @@ module Datadog
       EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit'
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
+      TAG_NORMALIZED_ROUTE = '_dd.appsec.normalized_route'
       TAG_METASTRUCT_STACK_TRACE = '_dd.stack'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'

data/lib/datadog/appsec/metrics/collector.rb

@@ -17,7 +17,15 @@ module Datadog
           keyword_init: true
         )
 
-        attr_reader :waf, :rasp
+        DownstreamResponseStore = Struct.new(
+          :content_type_invalid,
+          :content_length_missing,
+          :content_length_too_big,
+          :content_exceed_content_length,
+          keyword_init: true
+        )
+
+        attr_reader :waf, :rasp, :downstream_responses
 
         def initialize
           @mutex = Mutex.new
@@ -31,6 +39,11 @@ module Datadog
             evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0,
             duration_ext_ns: 0, inputs_truncated: 0, downstream_requests: 0
           )
+
+          @downstream_responses = DownstreamResponseStore.new(
+            content_type_invalid: 0, content_length_missing: 0,
+            content_length_too_big: 0, content_exceed_content_length: 0
+          )
         end
 
         def record_waf(result)
@@ -57,6 +70,10 @@ module Datadog
             @rasp.downstream_requests += 1 if type == Ext::RASP_SSRF && phase == Ext::RASP_REQUEST_PHASE
           end
         end
+
+        def record_ignored_downstream_response_body(reason)
+          @mutex.synchronize { @downstream_responses[reason] += 1 }
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics/exporter.rb

@@ -3,7 +3,7 @@
 module Datadog
   module AppSec
     module Metrics
-      # A class responsible for exporting WAF and RASP call metrics.
+      # A class responsible for exporting WAF and RASP call metrics
       module Exporter
         module_function
 
@@ -31,9 +31,15 @@ module Datadog
           end
         end
 
-        # private
+        def export_downstream_response_metrics(metrics, span)
+          metrics.each_pair do |reason, count|
+            next if count.zero?
 
-        def convert_ns_to_us(value)
+            span.set_tag("_dd.appsec.downstream_request.response_body_ignored.#{reason}", count)
+          end
+        end
+
+        private_class_method def convert_ns_to_us(value)
           value / 1000.0
         end
       end

data/lib/datadog/appsec/remote.rb

@@ -78,17 +78,23 @@ module Datadog
             engine = AppSec.security_engine
             next unless engine
 
+            # We must process delete operations before insert and update operations
+            # to prevent duplicate-rule errors when the config name changes
+            changes.each do |change|
+              next unless change.type == :delete
+
+              engine.remove_config_at_path(change.path.to_s)
+            end
+
             changes.each do |change|
               content = repository[change.path]
-              next unless content || change.type == :delete
+              next unless content
 
               case change.type
               when :insert, :update
                 # @type var content: Core::Remote::Configuration::Content
                 engine.add_or_update_config(parse_content(content), path: change.path.to_s)
                 content.applied
-              when :delete
-                engine.remove_config_at_path(change.path.to_s)
               end
             end
 

data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative 'route_text'
+require_relative 'route_pattern'
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes Rails route patterns into route format, inspired by
+      # OpenAPI v3 path templating
+      #
+      # NOTE: Uses the parsed Journey AST when available
+      #
+      # @api private
+      class RailsRoutePattern
+        DOT_CHAR = '.'
+        GROUP_OPEN_CHAR = '('
+        OPTIONAL_GROUP_PATTERN = /\(([^()]*)\)/
+        NAMED_PARAM_PREFIX_CHAR = ':'
+        GLOB_PARAM_PREFIX_CHAR = '*'
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path_params:, path:)
+          @path_params = path_params
+          @path = path
+
+          if @pattern.is_a?(String)
+            # NOTE: Journey groups without route params are never kept
+            #       Example: `/foo(/bar)` with request `/foo/bar` normalizes to `/foo`
+            pattern = remove_paramless_optional_groups(@pattern)
+            return RoutePattern.new(pattern).normalize(path: path)
+          end
+
+          route_path = @pattern.path
+          route_spec = route_path.spec
+
+          unless route_spec.respond_to?(:type)
+            return RoutePattern.new(route_spec.to_s).normalize(path: path)
+          end
+
+          if route_path.names.empty?
+            route_string = route_spec.to_s
+            return RouteText.escape(route_string) unless route_string.include?(GROUP_OPEN_CHAR)
+          end
+
+          buffer = Buffer.new
+          visit_route_node(route_spec, buffer)
+          buffer.to_path
+        end
+
+        private
+
+        def remove_paramless_optional_groups(pattern)
+          result = pattern
+
+          loop do
+            substituted = result.gsub(OPTIONAL_GROUP_PATTERN) do
+              # NOTE: OPTIONAL_GROUP_PATTERN always captures a string for each gsub match
+              group = ::Regexp.last_match(1) # : String
+              optional_group_has_route_params?(group) ? "(#{group})" : ''
+            end
+
+            return result if substituted == result
+
+            result = substituted
+          end
+        end
+
+        def visit_route_node(node, buffer)
+          case node.type
+          when :CAT
+            visit_route_node(node.left, buffer)
+            visit_route_node(node.right, buffer)
+          when :SLASH
+            buffer.flush
+          when :LITERAL
+            buffer.add_literal(node.left)
+          when :DOT
+            buffer.add_literal(DOT_CHAR)
+          when :SYMBOL, :STAR
+            buffer.add_param(node.name)
+          when :GROUP
+            visit_route_node(node.left, buffer) if group_present?(node.left)
+          end
+        end
+
+        def group_present?(node)
+          param_names = collect_group_param_names(node, [])
+          return false if param_names.empty?
+
+          param_names.all? { |name| param_matched_path?(name.to_sym) }
+        end
+
+        def param_matched_path?(name)
+          return @path_params[name].is_a?(String) unless name == :format
+
+          format = @path_params[:format]
+          return false if !format.is_a?(String) || format.empty?
+
+          dot_index = @path.length - format.length - 1
+          return false if dot_index < 0 || @path[dot_index] != DOT_CHAR
+
+          @path.end_with?(format)
+        end
+
+        def collect_group_param_names(node, memo)
+          case node.type
+          when :SYMBOL, :STAR
+            memo << node.name
+          when :CAT
+            collect_group_param_names(node.left, memo)
+            collect_group_param_names(node.right, memo)
+          end
+
+          memo
+        end
+
+        def optional_group_has_route_params?(group)
+          group.include?(NAMED_PARAM_PREFIX_CHAR) || group.include?(GLOB_PARAM_PREFIX_CHAR)
+        end
+
+        class Buffer
+          def initialize
+            @segments = []
+            @text = +''
+            @params = []
+            @nameless_param_count = 0
+          end
+
+          def add_literal(text)
+            @text << text
+          end
+
+          def add_param(name)
+            @params << name
+          end
+
+          def flush
+            return if @text.empty? && @params.empty?
+
+            @segments << if @params.empty?
+              RouteText.escape(@text)
+            else
+              render_params(@params)
+            end
+
+            @text = +''
+            @params.clear
+          end
+
+          def to_path
+            flush
+
+            "/#{@segments.join('/')}"
+          end
+
+          private
+
+          def render_params(params)
+            names = params.map do |name| # $ String
+              next name unless name.empty?
+
+              @nameless_param_count += 1
+              "param#{@nameless_param_count}"
+            end
+
+            "{#{names.join('+')}}"
+          end
+        end
+      end
+    end
+  end
+end