datadog 2.17.0 → 2.37.0

data/lib/datadog/appsec/contrib/rails/patches/render_to_body_patch.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative '../../../utils/hash_coercion'
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Patches
+          # A patch targeting `AbstractController::Rendering#render_to_body`
+          # method to capture JSON response body right before it is serialized.
+          module RenderToBodyPatch
+            def render_to_body(options = {})
+              return super unless options.key?(:json)
+
+              context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              return super unless context
+
+              data = Utils::HashCoercion.coerce(options[:json])
+              return super unless data
+
+              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              Instrumentation.gateway.push('rails.response.body.json', container)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rest_client/integration.rb

@@ -28,7 +28,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            super && !!(version&.>= MINIMUM_VERSION)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rest_client/patcher.rb

@@ -9,7 +9,7 @@ module Datadog
           module_function
 
           def patched?
-            Patcher.instance_variable_get(:@patched)
+            !!Patcher.instance_variable_get(:@patched)
           end
 
           def target_version
@@ -20,6 +20,8 @@ module Datadog
             require_relative 'request_ssrf_detection_patch'
 
             ::RestClient::Request.prepend(RequestSSRFDetectionPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -1,8 +1,11 @@
-# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/body'
+require_relative '../../utils/http/body_reader'
+require_relative '../../utils/http/media_type'
 
 module Datadog
   module AppSec
@@ -10,29 +13,181 @@ module Datadog
       module RestClient
         # Module that adds SSRF detection to RestClient::Request#execute
         module RequestSSRFDetectionPatch
+          REDIRECT_STATUS_CODES = (300..399).freeze
+
           def execute(&block)
-            return super unless AppSec.rasp_enabled? && AppSec.active_context
+            context = AppSec.active_context
+            return super unless context && AppSec.rasp_enabled?
+
+            headers = normalize_request_headers
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              'server.io.net.url' => url,
+              'server.io.net.request.method' => method.to_s.upcase,
+              'server.io.net.request.headers' => headers
+            }
+
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              sample_body = true
+            else
+              sample_body = mark_body_sampling!(context)
+            end
+
+            if !is_redirect && sample_body
+              body = parse_request_body(payload, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
+            handle(result, context: context) if result.match?
+
+            # NOTE: RestClient raises exceptions for non-2xx responses. For POST/PUT/PATCH
+            #       requests with 3xx redirects, RestClient raises instead of auto-following.
+            #       We rescue to process the response before re-raising.
+            begin
+              response = super
+            rescue ::RestClient::Exception => e
+              response = e.response
+              process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+
+              raise
+            end
+
+            process_response(response, sample_body: sample_body) if response.is_a?(::RestClient::AbstractResponse)
+            response
+          end
 
+          def process_response(response, sample_body:)
             context = AppSec.active_context
+            return unless context
+
+            headers = normalize_response_headers(response)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
+            ephemeral_data = {
+              'server.io.net.response.status' => response.code.to_s,
+              'server.io.net.response.headers' => headers
+            }
+
+            is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?('location')
+            context.state[:downstream_redirect_url] = URI.join(url, headers['location']).to_s if is_redirect && sample_body
+
+            if sample_body && !is_redirect
+              body = parse_response_body(response.body, headers: headers, context: context)
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
+            handle(result, context: context) if result.match?
+          end
+
+          private
+
+          def mark_body_sampling!(context)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return false if context.state[:downstream_body_analyzed_count] >= max
+            return false unless context.downstream_body_sampler.sample?
 
-            ephemeral_data = {'server.io.net.url' => url}
-            result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
+            context.state[:downstream_body_analyzed_count] += 1
+            true
+          end
+
+          def parse_request_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
 
-            if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+            # NOTE: Request body analysis is best-effort, non-rewindable payloads are skipped
+            limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            content = read_body(body, limit: limit)
+            return if content.nil? || content.bytesize > limit
 
-              context.events.push(
-                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
-              )
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
 
-              AppSec::ActionsHandler.handle(result.actions)
+          def parse_response_body(body, headers:, context:)
+            return unless readable_body?(body)
+
+            media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
+            if !media_type || media_type.type != 'application'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            subtype = media_type.subtype
+            if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
+              context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
+              return
+            end
+
+            content_length_value = headers['content-length']
+            if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
+              context.metrics.record_ignored_downstream_response_body(:content_length_missing)
+              return
+            end
+
+            content_length = content_length_value.to_i
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
+            if content_length > max
+              context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
+              return
+            end
+
+            content = read_body(body, limit: content_length)
+            return if content.nil?
+
+            if content.bytesize > content_length
+              context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
+              return
             end
 
-            super
+            Utils::HTTP::Body.parse(content, media_type: media_type)
+          end
+
+          def readable_body?(body)
+            body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
+          end
+
+          def read_body(body, limit:)
+            Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
+          rescue
+            nil
+          end
+
+          # NOTE: Starting version 2.1.0 headers are already normalized via internal
+          #       variable `@processed_headers_lowercase`. In case it's available,
+          #       we use it to avoid unnecessary transformation.
+          def normalize_request_headers
+            return @processed_headers_lowercase if defined?(@processed_headers_lowercase)
+
+            processed_headers.transform_keys(&:downcase)
+          end
+
+          # NOTE: Headers values are always an `Array` in `Net::HTTPResponse`,
+          #       but we want to avoid accidents and will wrap them in no-op
+          #       `Array` call just in case of a breaking change in the future
+          #
+          # FIXME: Steep has issues with `transform_values!` modifying the original
+          #        type and it failed with "Cannot allow block body" error
+          def normalize_response_headers(response) # steep:ignore MethodBodyTypeMismatch
+            response.net_http_res.to_hash
+              .transform_values! { |value| Array(value).join(', ') } # steep:ignore BlockBodyTypeMismatch
+          end
+
+          def handle(result, context:)
+            AppSec::Event.tag(context, result)
+            TraceKeeper.keep!(context.trace) if result.keep?
+
+            context.events.push(
+              AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+            )
+
+            AppSec::ActionsHandler.handle(result.actions)
           end
         end
       end
     end
   end
 end
-# rubocop:enable Naming/FileName

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -17,26 +18,52 @@ module Datadog
 
                 watch_request_dispatch(gateway)
                 watch_request_routed(gateway)
+                watch_response_body_json(gateway)
               end
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                gateway.watch('sinatra.request.dispatch') do |stack, gateway_request|
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
-                  persistent_data = {
-                    'server.request.body' => gateway_request.form_hash
-                  }
+                  context.state[:web_framework] = 'sinatra'
+
+                  request = gateway_request.request
+                  next stack.call(request) unless gateway_request.collectable_body?
+
+                  # NOTE: A limit of 0 disables request body collection entirely.
+                  limit = Datadog.configuration.appsec.body_parsing_size_limit
+                  next stack.call(request) if limit.zero?
+
+                  persistent_data = {}
+                  byte_length = gateway_request.body_bytesize(limit)
+
+                  if byte_length
+                    persistent_data['server.request.body.byte_length'] = byte_length
+
+                    if byte_length <= limit
+                      body = gateway_request.form_hash
+                      persistent_data['server.request.body'] = body if body
+                    end
+                  # NOTE: Body was parsed before measurement, keep byte_length unset
+                  elsif gateway_request.env.key?('rack.request.form_hash')
+                    body = gateway_request.env['rack.request.form_hash']
+                    persistent_data['server.request.body'] = body if body
+                  end
+
+                  next stack.call(request) if persistent_data.empty?
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
-                  if result.match? || !result.derivatives.empty?
+                  if result.match? || !result.attributes.empty?
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
                     )
                   end
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
                     AppSec::ActionsHandler.handle(result.actions)
                   end
 
@@ -45,8 +72,9 @@ module Datadog
               end
 
               def watch_request_routed(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
-                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                gateway.watch('sinatra.request.routed') do |stack, args|
+                  gateway_request, gateway_route_params = args # : [Gateway::Request, Gateway::RouteParams]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
                   persistent_data = {
                     'server.request.path_params' => gateway_route_params.params
@@ -55,7 +83,8 @@ module Datadog
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                   if result.match?
-                    AppSec::Event.tag_and_keep!(context, result)
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
 
                     context.events.push(
                       AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
@@ -67,6 +96,30 @@ module Datadog
                   stack.call(gateway_request.request)
                 end
               end
+
+              def watch_response_body_json(gateway = Instrumentation.gateway)
+                gateway.watch('sinatra.response.body.json') do |stack, container|
+                  context = container.context # : Context
+
+                  persistent_data = {
+                    'server.response.body' => container.data
+                  }
+                  result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                  if result.match?
+                    context.events.push(
+                      AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                    )
+
+                    AppSec::Event.tag(context, result)
+                    TraceKeeper.keep!(context.trace) if result.keep?
+
+                    AppSec::ActionsHandler.handle(result.actions)
+                  end
+
+                  stack.call(container)
+                end
+              end
             end
           end
         end

data/lib/datadog/appsec/contrib/sinatra/integration.rb

@@ -26,7 +26,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -8,6 +8,7 @@ require_relative 'framework'
 require_relative 'gateway/watcher'
 require_relative 'gateway/route_params'
 require_relative 'gateway/request'
+require_relative 'patches/json_patch'
 require_relative '../../../tracing/contrib/sinatra/framework'
 
 module Datadog
@@ -51,7 +52,6 @@ module Datadog
         module DispatchPatch
           def dispatch!
             env = @request.env
-
             context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
 
             return super unless context
@@ -59,7 +59,6 @@ module Datadog
             # TODO: handle exceptions, except for super
 
             gateway_request = Gateway::Request.new(env)
-
             request_return, _gateway_request = Instrumentation.gateway.push('sinatra.request.dispatch', gateway_request) do
               super
             end
@@ -72,7 +71,7 @@ module Datadog
         # path params are returned by pattern.params in process_route, then
         # merged with normal params, so we get both
         module RoutePatch
-          def process_route(*)
+          def process_route(*args)
             env = @request.env
 
             context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
@@ -84,7 +83,7 @@ module Datadog
             # Capture normal params.
             base_params = params
 
-            super do |*args|
+            super do |*super_args|
               # This block is called only once the route is found.
               # At this point params has both route params and normal params.
               route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
@@ -94,7 +93,7 @@ module Datadog
 
               Instrumentation.gateway.push('sinatra.request.routed', [gateway_request, gateway_route_params])
 
-              yield(*args)
+              yield(*super_args)
             end
           end
         end
@@ -113,27 +112,18 @@ module Datadog
 
           def patch
             Gateway::Watcher.watch
-            patch_default_middlewares
-            patch_dispatch
-            patch_route
-            setup_security
-            Patcher.instance_variable_set(:@patched, true)
-          end
-
-          def setup_security
-            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
-          end
 
-          def patch_default_middlewares
             ::Sinatra::Base.singleton_class.prepend(DefaultMiddlewarePatch)
-          end
-
-          def patch_dispatch
             ::Sinatra::Base.prepend(DispatchPatch)
+            ::Sinatra::Base.prepend(RoutePatch)
+            ::Sinatra::Base.prepend(Patches::JsonPatch) if patch_json?
+            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
+
+            Patcher.instance_variable_set(:@patched, true)
           end
 
-          def patch_route
-            ::Sinatra::Base.prepend(RoutePatch)
+          def patch_json?
+            !!(defined?(::Sinatra::JSON) && ::Sinatra::Base < ::Sinatra::JSON)
           end
         end
       end

data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../../../utils/hash_coercion'
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        module Patches
+          # A patch targeting `Sinatra::JSON#json` method to capture JSON response
+          # body right before it is serialized.
+          module JsonPatch
+            def json(object, options = {})
+              context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY] # : Context?
+              return super unless context
+
+              data = Utils::HashCoercion.coerce(object)
+              return super unless data
+
+              container = Instrumentation::Gateway::DataContainer.new(data, context: context)
+              Instrumentation.gateway.push('sinatra.response.body.json', container)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/counter_sampler.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative '../core/knuth_sampler'
+
+module Datadog
+  module AppSec
+    # Sampler that uses an internal counter to make deterministic sampling decisions.
+    #
+    # Each call to {#sample?} increments the counter and uses it as input to
+    # the underlying Knuth multiplicative hash algorithm.
+    #
+    # @api private
+    class CounterSampler
+      def initialize(rate = 1.0)
+        @sampler = Core::KnuthSampler.new(rate)
+        @counter = 0
+      end
+
+      def sample?
+        @counter += 1
+        @sampler.sample?(@counter)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/default_header_tags.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Sets the always-on span tags built from a fixed allowlist of request and
+    # response headers.
+    #
+    # NOTE: Unlike {Event#record}, which reports request headers only when
+    #       a security event fires, these are tagged on every request/response
+    #
+    # @api private
+    module DefaultHeaderTags
+      # NOTE: Contains additional WAF vendor headers
+      REQUEST_HEADERS_TAGS = %w[
+        accept
+        content-type
+        user-agent
+        akamai-user-risk
+        x-amzn-trace-id
+        x-cloud-trace-context
+        x-appgw-trace-id
+        x-sigsci-requestid
+        x-sigsci-tags
+        cf-ray
+        cloudfront-viewer-ja3-fingerprint
+      ].freeze
+
+      RESPONSE_HEADERS_TAGS = %w[
+        content-type
+        content-length
+        content-encoding
+        content-language
+      ].freeze
+
+      module_function
+
+      def tag_request(span, headers)
+        REQUEST_HEADERS_TAGS.each do |name|
+          value = headers.get(name)
+          span.set_tag("http.request.headers.#{name}", value) if value
+        end
+      end
+
+      def tag_response(span, headers)
+        RESPONSE_HEADERS_TAGS.each do |name|
+          value = headers.get(name)
+          span.set_tag("http.response.headers.#{name}", value.to_s) if value
+        end
+      end
+    end
+  end
+end