datadog 2.27.0 → 2.29.0

data/lib/datadog/appsec/contrib/faraday/patcher.rb

@@ -28,7 +28,7 @@ module Datadog
           end
 
           def configure_default_faraday_connection
-            if target_version >= Gem::Version.new('1.0.0')
+            if target_version&.>= Gem::Version.new('1.0.0')
               # Patch the default connection (e.g. +Faraday.get+)
               ::Faraday.default_connection.use(:datadog_appsec)
 

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -3,6 +3,8 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/media_type'
+require_relative '../../utils/http/body'
 
 module Datadog
   module AppSec
@@ -10,17 +12,36 @@ module Datadog
       module Faraday
         # AppSec SSRF detection Middleware for Faraday
         class SSRFDetectionMiddleware < ::Faraday::Middleware
+          REDIRECT_STATUS_CODES = (300..399).freeze
+          SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
+
           def call(env)
             context = AppSec.active_context
             return @app.call(env) unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            url = env.url.to_s
+            headers = normalize_headers(env.request_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
-              'server.io.net.url' => env.url.to_s,
+              'server.io.net.url' => url,
               'server.io.net.request.method' => env.method.to_s.upcase,
-              'server.io.net.request.headers' => env.request_headers.transform_keys(&:downcase)
+              'server.io.net.request.headers' => headers
             }
 
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              env[SAMPLE_BODY_KEY] = true
+            else
+              mark_body_sampling!(env, context: context)
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_body(env.body, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
             handle(result, context: context) if result.match?
 
@@ -30,18 +51,50 @@ module Datadog
           private
 
           def on_complete(env, context:)
-            timeout = Datadog.configuration.appsec.waf_timeout
-
-            response_headers = env.response_headers || {}
+            headers = normalize_headers(env.response_headers)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               'server.io.net.response.status' => env.status.to_s,
-              'server.io.net.response.headers' => response_headers.transform_keys(&:downcase)
+              'server.io.net.response.headers' => headers
             }
 
+            is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
+            if is_redirect && env[SAMPLE_BODY_KEY]
+              context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
+            end
+
+            if !is_redirect && env[SAMPLE_BODY_KEY]
+              body = parse_body(env.body, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
             handle(result, context: context) if result.match?
           end
 
+          def mark_body_sampling!(env, context:)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return if context.state[:downstream_body_analyzed_count] >= max
+            return unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            env[SAMPLE_BODY_KEY] = true
+          end
+
+          def parse_body(body, content_type:)
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            Utils::HTTP::Body.parse(body, media_type: media_type)
+          end
+
+          def normalize_headers(headers)
+            return {} if headers.nil? || headers.empty?
+
+            headers.transform_keys(&:downcase)
+          end
+
           def handle(result, context:)
             AppSec::Event.tag(context, result)
             TraceKeeper.keep!(context.trace) if result.keep?

data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb

@@ -47,9 +47,10 @@ module Datadog
             # Note that the `comments` "include" directive is included in the arguments list
             def build_arguments_hash
               queries.each_with_object({}) do |query, args_hash|
-                next unless query.selected_operation
+                selected_operation = query.selected_operation
+                next unless selected_operation
 
-                arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
+                arguments_from_selections(selected_operation.selections, query.variables, args_hash)
               end
             end
 
@@ -90,15 +91,19 @@ module Datadog
             end
 
             def argument_value(argument, query_variables)
-              case argument.value.class.name
+              value = argument.value
+
+              case value.class.name
               when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
+                # @type var value: GraphQL::Language::Nodes::VariableIdentifier
                 # we need to pass query.variables here instead of query.provided_variables,
                 # since #provided_variables don't know anything about variable default value
-                query_variables[argument.value.name]
+                query_variables[value.name]
               when Integration::AST_NODE_CLASS_NAMES[:input_object]
-                arguments_hash(argument.value.arguments, query_variables)
+                # @type var value: GraphQL::Language::Nodes::InputObject
+                arguments_hash(value.arguments, query_variables)
               else
-                argument.value
+                value
               end
             end
           end

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -31,7 +31,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION && ast_node_classes_defined?
+            !!(super && version&.>=(MINIMUM_VERSION) && ast_node_classes_defined?)
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -23,16 +23,12 @@ module Datadog
             end
 
             def query
-              # Downstream libddwaf expects keys and values to be extractable
-              # separately so we can't use [[k, v], ...]. We also want to allow
-              # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
-              # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
-              request.query_string.split('&').each.with_object({}) do |e, hash|
-                k, v = e.split('=').map { |s| CGI.unescape(s) }
-                hash[k] ||= []
-
-                hash[k] << v
-              end
+              ::Rack::Utils.parse_query(request.query_string)
+            rescue => e
+              Datadog.logger.debug { "AppSec: Failed to parse request query string: #{e.class}: #{e.message}" }
+              AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request query string')
+
+              {}
             end
 
             def method

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -107,9 +107,7 @@ module Datadog
 
             if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
                 AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
-              ctx.events.push(
-                AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
-              )
+              ctx.extract_schema!
             end
 
             AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -136,12 +136,20 @@ module Datadog
               if Datadog::AppSec::Contrib::Rails::Patcher.target_version < Gem::Version.new('7.1')
                 Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
               end
+            rescue => e
+              error_message = 'Failed to get application routes'
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
             end
           end
 
           def subscribe_to_routes_loaded
-            ::ActiveSupport.on_load(:after_routes_loaded) do |app|
-              Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(app.routes.routes)
+            ::ActiveSupport.on_load(:after_routes_loaded) do
+              Datadog::AppSec::Contrib::Rails::Patcher.report_routes_via_telemetry(::Rails.application.routes.routes)
+            rescue => e
+              error_message = 'Failed to get application routes'
+              Datadog.logger.error("#{error_message}, #{e.class}: #{e.message}")
+              AppSec.telemetry.report(e, description: error_message)
             end
           end
 

data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb

@@ -11,6 +11,8 @@ module Datadog
               context = request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
               return super unless context
 
+              context.state[:web_framework] = 'rails'
+
               # TODO: handle exceptions, except for super
               gateway_request = Gateway::Request.new(request)
               http_response, _gateway_request = Instrumentation.gateway.push('rails.request.action', gateway_request) do

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -3,6 +3,8 @@
 require_relative '../../event'
 require_relative '../../trace_keeper'
 require_relative '../../security_event'
+require_relative '../../utils/http/media_type'
+require_relative '../../utils/http/body'
 
 module Datadog
   module AppSec
@@ -10,35 +12,97 @@ module Datadog
       module RestClient
         # Module that adds SSRF detection to RestClient::Request#execute
         module RequestSSRFDetectionPatch
+          REDIRECT_STATUS_CODES = (300..399).freeze
+
           def execute(&block)
             context = AppSec.active_context
             return super unless context && AppSec.rasp_enabled?
 
-            timeout = Datadog.configuration.appsec.waf_timeout
+            headers = normalize_request_headers
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               'server.io.net.url' => url,
               'server.io.net.request.method' => method.to_s.upcase,
-              'server.io.net.request.headers' => normalize_request_headers
+              'server.io.net.request.headers' => headers
             }
 
+            is_redirect = context.state[:downstream_redirect_url] == url
+            if is_redirect
+              context.state.delete(:downstream_redirect_url)
+              sample_body = true
+            else
+              sample_body = mark_body_sampling!(context)
+            end
+
+            if !is_redirect && sample_body
+              body = parse_body(payload.to_s, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.request.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
             handle(result, context: context) if result.match?
 
-            response = super
+            # NOTE: RestClient raises exceptions for non-2xx responses. For POST/PUT/PATCH
+            #       requests with 3xx redirects, RestClient raises instead of auto-following.
+            #       We rescue to process the response before re-raising.
+            begin
+              response = super
+            rescue ::RestClient::Exception => e
+              response = e.response
+              process_response(response, sample_body: sample_body) if response
+
+              raise
+            end
 
+            process_response(response, sample_body: sample_body)
+            response
+          end
+
+          def process_response(response, sample_body:)
+            context = AppSec.active_context
+            return unless context
+
+            headers = normalize_response_headers(response)
+            # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               'server.io.net.response.status' => response.code.to_s,
-              'server.io.net.response.headers' => normalize_response_headers(response)
+              'server.io.net.response.headers' => headers
             }
 
+            is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?('location')
+            context.state[:downstream_redirect_url] = URI.join(url, headers['location']).to_s if is_redirect && sample_body
+
+            if sample_body && !is_redirect
+              body = parse_body(response.body, content_type: headers['content-type'])
+              ephemeral_data['server.io.net.response.body'] = body if body
+            end
+
+            timeout = Datadog.configuration.appsec.waf_timeout
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
             handle(result, context: context) if result.match?
-
-            response
           end
 
           private
 
+          def mark_body_sampling!(context)
+            max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
+            return false if context.state[:downstream_body_analyzed_count] >= max
+            return false unless context.downstream_body_sampler.sample?
+
+            context.state[:downstream_body_analyzed_count] += 1
+            true
+          end
+
+          def parse_body(body, content_type:)
+            return if body.empty?
+
+            media_type = Utils::HTTP::MediaType.parse(content_type)
+            return unless media_type
+
+            Utils::HTTP::Body.parse(body, media_type: media_type)
+          end
+
           # NOTE: Starting version 2.1.0 headers are already normalized via internal
           #       variable `@processed_headers_lowercase`. In case it's available,
           #       we use it to avoid unnecessary transformation.
@@ -55,7 +119,8 @@ module Datadog
           # FIXME: Steep has issues with `transform_values!` modifying the original
           #        type and it failed with "Cannot allow block body" error
           def normalize_response_headers(response) # steep:ignore MethodBodyTypeMismatch
-            response.net_http_res.to_hash.transform_values! { |value| Array(value).join(', ') } # steep:ignore BlockBodyTypeMismatch
+            response.net_http_res.to_hash
+              .transform_values! { |value| Array(value).join(', ') } # steep:ignore BlockBodyTypeMismatch
           end
 
           def handle(result, context:)

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -23,7 +23,9 @@ module Datadog
 
               def watch_request_dispatch(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
-                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
+
+                  context.state[:web_framework] = 'sinatra'
 
                   persistent_data = {
                     'server.request.body' => gateway_request.form_hash
@@ -49,8 +51,9 @@ module Datadog
               end
 
               def watch_request_routed(gateway = Instrumentation.gateway)
-                gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
-                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
+                gateway.watch('sinatra.request.routed', :appsec) do |stack, args|
+                  gateway_request, gateway_route_params = args # : [Gateway::Request, Gateway::RouteParams]
+                  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
                   persistent_data = {
                     'server.request.path_params' => gateway_route_params.params
@@ -75,7 +78,7 @@ module Datadog
 
               def watch_response_body_json(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.response.body.json', :appsec) do |stack, container|
-                  context = container.context
+                  context = container.context # : Context
 
                   persistent_data = {
                     'server.response.body' => container.data

data/lib/datadog/appsec/contrib/sinatra/integration.rb

@@ -26,7 +26,7 @@ module Datadog
           end
 
           def self.compatible?
-            super && version >= MINIMUM_VERSION
+            !!(super && (version&.>= MINIMUM_VERSION))
           end
 
           def self.auto_instrument?

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -71,7 +71,7 @@ module Datadog
         # path params are returned by pattern.params in process_route, then
         # merged with normal params, so we get both
         module RoutePatch
-          def process_route(*)
+          def process_route(*args)
             env = @request.env
 
             context = env[Datadog::AppSec::Ext::CONTEXT_KEY]
@@ -83,7 +83,7 @@ module Datadog
             # Capture normal params.
             base_params = params
 
-            super do |*args|
+            super do |*super_args|
               # This block is called only once the route is found.
               # At this point params has both route params and normal params.
               route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
@@ -93,7 +93,7 @@ module Datadog
 
               Instrumentation.gateway.push('sinatra.request.routed', [gateway_request, gateway_route_params])
 
-              yield(*args)
+              yield(*super_args)
             end
           end
         end
@@ -123,7 +123,7 @@ module Datadog
           end
 
           def patch_json?
-            defined?(::Sinatra::JSON) && ::Sinatra::Base < ::Sinatra::JSON
+            !!(defined?(::Sinatra::JSON) && ::Sinatra::Base < ::Sinatra::JSON)
           end
         end
       end

data/lib/datadog/appsec/contrib/sinatra/patches/json_patch.rb

@@ -12,7 +12,7 @@ module Datadog
           # body right before it is serialized.
           module JsonPatch
             def json(object, options = {})
-              context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY]
+              context = @request.env[Datadog::AppSec::Ext::CONTEXT_KEY] # : Context?
               return super unless context
 
               data = Utils::HashCoercion.coerce(object)

data/lib/datadog/appsec/counter_sampler.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative '../core/knuth_sampler'
+
+module Datadog
+  module AppSec
+    # Sampler that uses an internal counter to make deterministic sampling decisions.
+    #
+    # Each call to {#sample?} increments the counter and uses it as input to
+    # the underlying Knuth multiplicative hash algorithm.
+    #
+    # @api private
+    class CounterSampler
+      def initialize(rate = 1.0)
+        @sampler = Core::KnuthSampler.new(rate)
+        @counter = 0
+      end
+
+      def sample?
+        @counter += 1
+        @sampler.sample?(@counter)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -23,6 +23,24 @@ module Datadog
             }
           )
         end
+
+        def export_api_security_metrics(context)
+          web_framework = context.state[:web_framework]
+          return unless web_framework
+
+          if context.span&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE).nil?
+            AppSec.telemetry.inc(
+              AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, 'api_security.missing_route', 1,
+              tags: {framework: web_framework}
+            )
+          end
+
+          metric_name = context.state[:schema_extracted] ? 'schema' : 'no_schema'
+          AppSec.telemetry.inc(
+            AppSec::Ext::TELEMETRY_METRICS_NAMESPACE, "api_security.request.#{metric_name}", 1,
+            tags: {framework: web_framework}
+          )
+        end
       end
     end
   end

data/lib/datadog/appsec/security_engine/engine.rb

@@ -40,12 +40,16 @@ module Datadog
           @ruleset_version = diagnostics['ruleset_version']
 
           @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
+
+          metric('init', success: true, ruleset_version: @ruleset_version, telemetry: telemetry)
         rescue WAF::Error => e
-          error_message = "AppSec security engine failed to initialize"
+          error_message = 'AppSec security engine failed to initialize'
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           telemetry.report(e, description: error_message)
 
+          metric('init', success: false, ruleset_version: @ruleset_version, telemetry: telemetry)
+
           raise e
         end
 
@@ -103,17 +107,34 @@ module Datadog
           @ruleset_version = @reconfigured_ruleset_version
 
           @handle_ref.current = new_waf_handle
+
+          metric('updates', success: true, ruleset_version: @ruleset_version, telemetry: AppSec.telemetry)
         rescue WAF::Error => e
           # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
           # This means that the current WAF handle was not yet substituted.
-          error_message = "AppSec security engine failed to reconfigure, reverting to the previous configuration"
+          error_message = 'AppSec security engine failed to reconfigure, reverting to the previous configuration'
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           AppSec.telemetry.report(e, description: error_message)
+
+          metric('updates', success: false, ruleset_version: @reconfigured_ruleset_version, telemetry: AppSec.telemetry)
         end
 
         private
 
+        def metric(metric_name, success:, ruleset_version:, telemetry:)
+          telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE,
+            "waf.#{metric_name}",
+            1,
+            tags: {
+              waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+              event_rules_version: ruleset_version.to_s,
+              success: success.to_s
+            }
+          )
+        end
+
         def load_default_config(telemetry:)
           config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
 

data/lib/datadog/appsec/utils/http/body.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'cgi'
+
+require_relative 'url_encoded'
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for handling HTTP body parsing
+        module Body
+          def self.parse(body, media_type:)
+            return if body.nil?
+
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+            # @type var content: ::String?
+            content = body.respond_to?(:read) ? body.read : body # steep:ignore NoMethod, IncompatibleAssignment
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+
+            return if content.nil? || content.empty?
+
+            if media_type.subtype == 'json' || media_type.subtype.end_with?('+json')
+              JSON.parse(content)
+            elsif media_type.subtype == 'x-www-form-urlencoded'
+              URLEncoded.parse(content)
+            end
+          rescue => e
+            AppSec.telemetry.report(e, description: 'AppSec: Failed to parse body')
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_range.rb

@@ -168,7 +168,8 @@ module Datadog
           #
           # returns true if the MediaType is accepted by this MediaRange
           def ===(other)
-            return self === MediaType.new(other) if other.is_a?(::String)
+            return false if other.nil?
+            return self === MediaType.parse(other) if other.is_a?(::String)
 
             type == other.type && subtype == other.subtype && other.parameters.all? { |k, v| parameters[k] == v } ||
               type == other.type && wildcard?(:subtype) ||