contrast-agent 4.11.0 → 4.14.0

data/lib/contrast/agent/reporting/reporting_utilities/reporting_storage.rb

@@ -0,0 +1,66 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/logger'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This module will be used to store everything that would be sent
+      # and depends on the response we receive
+      module ReportingStorage
+        class << self
+          include Contrast::Components::Logger::InstanceMethods
+
+          # So the collection will be used to store those events
+          def collection
+            @_collection ||= {}
+          end
+
+          # @param key[String] the key for the pair
+          # @param value[Object] the value we need to store
+          # @return [Object] the value we saved
+          def []= key, value
+            return unless key
+            return unless value
+
+            logger.debug('Saving new value', key: key)
+
+            key = key.to_s.downcase.strip
+            collection[key] = value
+
+            value # rubocop:disable Lint/Void
+          end
+
+          # @param key[String] the key for the pair
+          def [] key
+            return unless key
+
+            collection[key]
+          end
+          alias_method :get, :[]
+          alias_method :set, :[]=
+
+          def delete key
+            return unless key
+
+            logger.debug('Starting deleting value for', key: key)
+
+            deleted_value = collection.delete(key)
+            logger.debug('Key wasn\'t found') unless deleted_value
+
+            deleted_value
+          end
+
+          # @param rule_id [String] the rule_id
+          # @return [Hash, nil] return array with key and value of the pair
+          def find_by_rule_id rule_id
+            return unless rule_id
+
+            collection.find { |_, v| v.rule_id == rule_id }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/request.rb

@@ -9,6 +9,7 @@ require 'contrast/utils/string_utils'
 require 'contrast/utils/hash_digest'
 require 'contrast/components/logger'
 require 'contrast/components/scope'
+require 'contrast/utils/request_utils'
 
 module Contrast
   module Agent
@@ -17,6 +18,7 @@ module Contrast
     # data in a format that the Agent expects, caching those transformations in
     # order to avoid repeatedly creating Strings & thrashing GC.
     class Request
+      include Contrast::Utils::RequestUtils
       include Contrast::Components::Logger::InstanceMethods
       include Contrast::Components::Scope::InstanceMethods
 
@@ -152,87 +154,6 @@ module Contrast
 
         accepts.start_with?(*MEDIA_TYPE_MARKERS)
       end
-
-      private
-
-      # Return a flattened hash of params with realized paths for keys, in
-      # addition to a separate, valueless, entry for each nest key.
-      # See RUBY-621 for more details.
-      # { key : { nested_key : ['x','y','z' ] } }
-      # becomes
-      # {
-      #   key[nested_key][0] : 'x'
-      #   key[nested_key][1] : 'y'
-      #   key[nested_key][2] : 'z'
-      #   key :                ''
-      #   nested_key :         ''
-      # }
-      def normalize_params val, prefix: nil
-        # In non-recursive invocations, val should always be a Hash
-        # (rather than breaking this out into two methods)
-        case val
-        when Tempfile
-          # Skip if it's the auto-generated value from rails when it handles
-          # file uploads. The file name will still be sent to SR for analysis.
-          {}
-        when Hash
-          res = val.each_with_object({}) do |(k, v), hash|
-            k = Contrast::Utils::StringUtils.force_utf8(k)
-            nested_prefix = prefix.nil? ? k : "#{ prefix }[#{ k }]"
-            hash[k] = Contrast::Utils::ObjectShare::EMPTY_STRING
-            hash.merge! normalize_params(v, prefix: nested_prefix)
-          end
-          res[prefix] = Contrast::Utils::ObjectShare::EMPTY_STRING if prefix
-          res
-        when Enumerable
-          idx = 0
-          res = {}
-          while idx < val.length
-            res.merge! normalize_params(val[idx], prefix: "#{ prefix }[#{ idx }]")
-            idx += 1
-          end
-          res[prefix] = Contrast::Utils::ObjectShare::EMPTY_STRING if prefix
-          res
-        else
-          { prefix => Contrast::Utils::StringUtils.force_utf8(val) }
-        end
-      end
-
-      def read_body body
-        return body if body.is_a?(String)
-
-        begin
-          can_rewind = Contrast::Utils::DuckUtils.quacks_to?(body, :rewind)
-          # if we are after a middleware that failed to rewind
-          body.rewind if can_rewind
-          body.read
-        rescue StandardError => e
-          logger.error('Error in attempt to read body', message: e.message)
-          logger.trace('With Stack', e)
-          body.to_s
-        ensure
-          # be a good citizen and rewind
-          body.rewind if can_rewind
-        end
-      end
-
-      def traverse_parsed_multipart multipart_data, current_names
-        return current_names unless multipart_data
-
-        multipart_data.each_value do |data_value|
-          next unless data_value.is_a?(Hash)
-
-          tempfile = data_value[:tempfile]
-          if tempfile.nil?
-            traverse_parsed_multipart(data_value, current_names)
-          else
-            name = data_value[:name].to_s
-            file_name = data_value[:filename].to_s
-            current_names[name] = file_name
-          end
-        end
-        current_names
-      end
     end
   end
 end

data/lib/contrast/agent/request_context.rb

@@ -7,6 +7,8 @@ require 'contrast/agent/response'
 require 'contrast/agent/inventory/database_config'
 require 'contrast/components/logger'
 require 'contrast/components/scope'
+require 'contrast/utils/request_utils'
+require 'contrast/agent/request_context_extend'
 
 module Contrast
   module Agent
@@ -27,6 +29,8 @@ module Contrast
     class RequestContext
       include Contrast::Components::Logger::InstanceMethods
       include Contrast::Components::Scope::InstanceMethods
+      include Contrast::Utils::RequestUtils
+      include Contrast::Agent::RequestContextExtend
 
       EMPTY_INPUT_ANALYSIS_PB = Contrast::Api::Settings::InputAnalysis.new
 
@@ -60,14 +64,10 @@ module Contrast
           # generic holder for properties that can be set throughout this request
           @_properties = {}
 
-          @sample = true
-
           if ::Contrast::ASSESS.enabled?
-            @sample_request, @sample_response = Contrast::Utils::Assess::SamplingUtil.instance.sample?(@request)
+            @sample_req, @sample_res = Contrast::Utils::Assess::SamplingUtil.instance.sample?(@request)
           end
 
-          @sample_response &&= ::Contrast::ASSESS.scan_response?
-
           append_route_coverage(Contrast::Agent.framework_manager.get_route_dtm(@request))
         end
       end
@@ -77,103 +77,31 @@ module Contrast
       end
 
       def analyze_request?
-        @sample_request
+        analyze_request_assess? || analyze_req_res_protect?
       end
 
       def analyze_response?
-        @sample_response
+        analyze_response_assess? || analyze_req_res_protect?
       end
 
-      # Convert the discovered route for this request to appropriate forms and disseminate it to those locations
-      # where it is necessary for our route coverage and finding vulnerability discovery features to function.
-      #
-      # @param route [Contrast::Api::Dtm::RouteCoverage, nil] the route of the current request, as determined from the
-      #   framework
-      def append_route_coverage route
-        return unless route
-
-        # For our findings
-        @route = route
-
-        # For SR findings
-        @activity.routes << route
-
-        # For TS routes
-        @observed_route.signature  = route.route
-        @observed_route.verb       = route.verb
-        @observed_route.url        = route.url if route.url
-        @request.route = route
-        @request.observed_route = @observed_route
-      end
-
-      # Collect the results for the given rule with the given action
-      #
-      # @param rule [String] the id of the rule to which the results apply
-      # @param response_type [Symbol] the result of the response, matching a value of
-      #   Contrast::Api::Dtm::AttackResult::ResponseType
-      # @return [Array<Contrast::Api::Dtm::AttackResult>]
-      def results_for rule, response_type = nil
-        if response_type.nil?
-          activity.results.select { |r| r.rule_id == rule }
-        else
-          activity.results.select { |r| r.rule_id == rule && r.response == response_type }
-        end
+      def analyze_req_res_protect?
+        ::Contrast::PROTECT.enabled?
       end
 
-      def service_extract_request
-        return false unless ::Contrast::AGENT.enabled?
-        return false unless ::Contrast::PROTECT.enabled?
-        return false if @do_not_track
+      def analyze_request_assess?
+        return false unless analyze_req_res_assess?
 
-        service_response = Contrast::Agent.messaging_queue.send_event_immediately(@activity.http_request)
-        return false unless service_response
-
-        handle_protect_state(service_response)
-        ia = service_response.input_analysis
-        if ia
-          if logger.trace?
-            logger.trace('Analysis from Contrast Service', evaluations: ia.results.length)
-            logger.trace('Results', input_analysis: ia.inspect)
-          end
-          @speedracer_input_analysis = ia
-          speedracer_input_analysis.request = request
-        else
-          logger.trace('Analysis from Contrast Service was empty.')
-          false
-        end
-      rescue Contrast::SecurityException => e
-        raise e
-      rescue StandardError => e
-        logger.warn('Unable to extract Contrast Service information from request', e)
-        false
+        @sample_req
       end
 
-      # NOTE: this method is only used as a backstop if Speedracer sends Input Evaluations when the protect state
-      # indicates a security exception should be thrown. This method ensures that the attack reports are generated.
-      # Normally these should be generated on Speedracer for any attacks detected during prefilter.
-      #
-      # @param agent_settings [Contrast::Api::Settings::AgentSettings]
-      def handle_protect_state agent_settings
-        return unless agent_settings&.protect_state
-
-        state = agent_settings.protect_state
-        @uuid = state.uuid
-        @do_not_track = true unless state.track_request
-        return unless state.security_exception
-
-        # If Contrast Service has NOT handled the input analysis, handle them here
-        build_attack_results(agent_settings)
-        logger.debug('Contrast Service said to block this request')
-        raise Contrast::SecurityException.new(nil, (state.security_message || 'Blocking suspicious behavior'))
+      def analyze_response_assess?
+        return false unless analyze_req_res_assess?
+
+        @sample_res &&= ::Contrast::ASSESS.scan_response?
       end
 
-      # append anything we've learned to the request seen message this is the sum-total of all inventory information
-      # that has been accumulated since the last request
-      def extract_after rack_response
-        @response = Contrast::Agent::Response.new(rack_response)
-        activity.http_response = @response.dtm if @sample_response
-      rescue StandardError => e
-        logger.error('Unable to extract information after request', e)
+      def analyze_req_res_assess?
+        ::Contrast::ASSESS.enabled?
       end
 
       def add_property key, value
@@ -189,42 +117,6 @@ module Contrast
         @server_activity = Contrast::Api::Dtm::ServerActivity.new
         @observed_route = Contrast::Api::Dtm::ObservedRoute.new
       end
-
-      private
-
-      # Generate attack results directly from any evaluations on the agent settings object.
-      #
-      # @param agent_settings [Contrast::Api::Settings::AgentSettings]
-      def build_attack_results agent_settings
-        return unless agent_settings&.input_analysis&.results&.any?
-
-        attack_results_by_rule = {}
-        agent_settings.input_analysis.results.each do |ia_result|
-          rule_id = ia_result.rule_id
-          rule = ::Contrast::PROTECT.rule(rule_id)
-          next unless rule
-
-          if logger.debug?
-            logger.debug('Building attack result from Contrast Service input analysis result',
-                         result: ia_result.inspect)
-          end
-
-          attack_result = if rule.mode == :BLOCK
-                            # special case for rules (like reflected xss) that used to have an infilter / block mode
-                            # but now are just block at perimeter
-                            rule.build_attack_with_match(self, ia_result, attack_results_by_rule[rule_id],
-                                                         ia_result.value)
-                          else
-                            rule.build_attack_without_match(self, ia_result, attack_results_by_rule[rule_id])
-                          end
-          attack_results_by_rule[rule_id] = attack_result
-        end
-
-        attack_results_by_rule.each_pair do |_, attack_result|
-          logger.info('Blocking attack result', rule: attack_result.rule_id)
-          activity.results << attack_result
-        end
-      end
     end
   end
 end

data/lib/contrast/agent/request_context_extend.rb

@@ -0,0 +1,138 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Agent
+    # This class extends RequestContexts: this class acts to encapsulate information about the currently
+    # executed request, making it available to the Agent for the duration of the request in a standardized
+    # and normalized format which the Agent understands.
+    module RequestContextExtend
+      BUILD_ATTACK_LOGGER_MESSAGE = 'Building attack result from Contrast Service input analysis result'
+      # Convert the discovered route for this request to appropriate forms and disseminate it to those locations
+      # where it is necessary for our route coverage and finding vulnerability discovery features to function.
+      #
+      # @param route [Contrast::Api::Dtm::RouteCoverage, nil] the route of the current request, as determined from the
+      #   framework
+      def append_route_coverage route
+        return unless route
+
+        # For our findings
+        @route = route
+
+        # For SR findings
+        @activity.routes << route
+
+        # For TS routes
+        @observed_route.signature  = route.route
+        @observed_route.verb       = route.verb
+        @observed_route.url        = route.url if route.url
+        @request.route = route
+        @request.observed_route = @observed_route
+      end
+
+      # Collect the results for the given rule with the given action
+      #
+      # @param rule [String] the id of the rule to which the results apply
+      # @param response_type [Symbol] the result of the response, matching a value of
+      #   Contrast::Api::Dtm::AttackResult::ResponseType
+      # @return [Array<Contrast::Api::Dtm::AttackResult>]
+      def results_for rule, response_type = nil
+        if response_type.nil?
+          activity.results.select { |r| r.rule_id == rule }
+        else
+          activity.results.select { |r| r.rule_id == rule && r.response == response_type }
+        end
+      end
+
+      def service_extract_request
+        return false unless ::Contrast::AGENT.enabled?
+        return false unless ::Contrast::PROTECT.enabled?
+        return false if @do_not_track
+
+        service_response = Contrast::Agent.messaging_queue.send_event_immediately(@activity.http_request)
+        return false unless service_response
+
+        handle_protect_state(service_response)
+        ia = service_response.input_analysis
+        if ia
+          if logger.trace?
+            logger.trace('Analysis from Contrast Service', evaluations: ia.results.length)
+            logger.trace('Results', input_analysis: ia.inspect)
+          end
+          @speedracer_input_analysis = ia
+          speedracer_input_analysis.request = request
+        else
+          logger.trace('Analysis from Contrast Service was empty.')
+          false
+        end
+      rescue Contrast::SecurityException => e
+        raise e
+      rescue StandardError => e
+        logger.warn('Unable to extract Contrast Service information from request', e)
+        false
+      end
+
+      # NOTE: this method is only used as a backstop if Speedracer sends Input Evaluations when the protect state
+      # indicates a security exception should be thrown. This method ensures that the attack reports are generated.
+      # Normally these should be generated on Speedracer for any attacks detected during prefilter.
+      #
+      # @param agent_settings [Contrast::Api::Settings::AgentSettings]
+      def handle_protect_state agent_settings
+        return unless agent_settings&.protect_state
+
+        state = agent_settings.protect_state
+        @uuid = state.uuid
+        @do_not_track = true unless state.track_request
+        return unless state.security_exception
+
+        # If Contrast Service has NOT handled the input analysis, handle them here
+        build_attack_results(agent_settings)
+        logger.debug('Contrast Service said to block this request')
+        raise Contrast::SecurityException.new(nil, (state.security_message || 'Blocking suspicious behavior'))
+      end
+
+      # append anything we've learned to the request seen message this is the sum-total of all inventory information
+      # that has been accumulated since the last request
+      def extract_after rack_response
+        @response = Contrast::Agent::Response.new(rack_response)
+        activity.http_response = @response.dtm if @sample_res
+      rescue StandardError => e
+        logger.error('Unable to extract information after request', e)
+      end
+
+      private
+
+      # Generate attack results directly from any evaluations on the agent settings object.
+      #
+      # @param agent_settings [Contrast::Api::Settings::AgentSettings]
+      def build_attack_results agent_settings
+        return unless agent_settings&.input_analysis&.results&.any?
+
+        results_by_rule = {}
+        agent_settings.input_analysis.results.each do |ia_result|
+          rule_id = ia_result.rule_id
+          rule = ::Contrast::PROTECT.rule(rule_id)
+          next unless rule
+
+          logger.debug(BUILD_ATTACK_LOGGER_MESSAGE, result: ia_result.inspect) if logger.debug?
+          results_by_rule[rule_id] = attack_result rule, rule_id, ia_result, results_by_rule
+        end
+
+        results_by_rule.each_pair do |_, attack_result|
+          logger.info('Blocking attack result', rule: attack_result.rule_id)
+          activity.results << attack_result
+        end
+      end
+
+      def attack_result rule, rule_id, ia_result, results_by_rule
+        @_attack_result = if rule.mode == :BLOCK
+                            # special case for rules (like reflected xss) that used to have an infilter / block mode
+                            # but now are just block at perimeter
+                            rule.build_attack_with_match(self, ia_result, results_by_rule[rule_id], ia_result.value)
+                          else
+                            rule.build_attack_without_match(self, ia_result, results_by_rule[rule_id])
+                          end
+      end
+    end
+  end
+end

data/lib/contrast/agent/request_handler.rb

@@ -6,19 +6,23 @@ require 'contrast/components/scope'
 
 module Contrast
   module Agent
-    # This class is instantiated when we receive a request and the agent is enabled to process
-    # that request. It holds the ruleset that we perform filtering operations on (currently
-    # prefilter and postfilter).
+    # This class is instantiated when we receive a request and the agent is enabled to process that request. It holds
+    # the ruleset that we perform filtering operations on (currently prefilter and postfilter).
     class RequestHandler
       include Contrast::Components::Logger::InstanceMethods
 
       attr_reader :ruleset, :context
 
+      # @param context [Contrast::Agent::RequestContext] the context of the request for which this handler applies
       def initialize context
         @context = context
         @ruleset = ::Contrast::AGENT.ruleset
       end
 
+      # TODO: RUBY-1353
+      # TODO: RUBY-1355
+      # TODO: RUBY-1357
+      # TODO: RUBY-1358
       def send_activity_messages
         Contrast::Agent::Inventory::DependencyUsageAnalysis.instance.generate_library_usage(context.activity)
         [context.server_activity, context.activity, context.observed_route].each do |message|

data/lib/contrast/agent/response.rb

@@ -8,6 +8,7 @@ require 'contrast/utils/object_share'
 require 'contrast/utils/string_utils'
 require 'contrast/utils/hash_digest'
 require 'contrast/components/logger'
+require 'contrast/utils/response_utils'
 
 module Contrast
   module Agent
@@ -17,6 +18,7 @@ module Contrast
     # order to avoid repeatedly creating Strings & thrashing GC.
     class Response
       include Contrast::Components::Logger::InstanceMethods
+      include Contrast::Utils::ResponseUtils
 
       extend Forwardable
 
@@ -88,79 +90,6 @@ module Contrast
         body_content = @is_array ? rack_response[2] : rack_response.body
         extract_body(body_content)
       end
-
-      private
-
-      # From the dtm for normalized_response_headers:
-      #   Key is UPPERCASE_UNDERSCORE
-      #
-      #   Example: Content-Type: text/html; charset=utf-8
-      #   "CONTENT_TYPE" => Content-Type,["text/html; charset=utf8"]
-      def append_pair map, key, value
-        return unless key && value
-        return if value.is_a?(Hash)
-
-        safe_key = Contrast::Utils::StringUtils.force_utf8(key)
-        hash_key = Contrast::Utils::StringUtils.normalized_key(safe_key)
-        map[hash_key] ||= Contrast::Api::Dtm::Pair.new
-        map[hash_key].key = safe_key
-        map[hash_key].values << Contrast::Utils::StringUtils.force_utf8(value)
-      end
-
-      HTTP_PREFIX = /^[Hh][Tt][Tt][Pp][_-]/i.cs__freeze
-
-      # Given some holder of the content of the response's body, extract that
-      # content and return it as a String
-      #
-      # @param body [String, Rack::File, Rack::BodyProxy,
-      #   ActionDispatch::Response::RackBody, Rack::Response] Something that
-      #   holds, wraps, or is the body of the Response
-      # @return [nil, String] the content of the body
-      def extract_body body
-        return unless body
-
-        if defined?(Rack::File) && body.is_a?(Rack::File)
-          # not sure what to do in this situation, so don't do anything.
-          nil
-        elsif body.is_a?(Rack::BodyProxy)
-          handle_rack_body_proxy(body)
-        elsif (defined?(ActionDispatch::Response::RackBody) && body.is_a?(ActionDispatch::Response::RackBody)) ||
-              body.is_a?(Rack::Response)
-
-          extract_body(body.body)
-        elsif Contrast::Utils::DuckUtils.quacks_to?(body, :each)
-          acc = []
-          body.each { |tmp| acc << read_or_string(tmp) }
-          acc.compact.join(Contrast::Utils::ObjectShare::NEW_LINE)
-        elsif ActionView::OutputBuffer
-          # https://stackoverflow.com/questions/15654676/how-to-convert-activesupportsafebuffer-to-string
-          body.to_str
-        else
-          read_or_string(body)
-        end
-      end
-
-      def handle_rack_body_proxy body
-        next_body = body.instance_variable_get(:@body)
-        case next_body
-        when Array
-          extract_body(next_body[0])
-        else
-          extract_body(next_body)
-        end
-      end
-
-      def read_or_string obj
-        return unless obj
-
-        if Contrast::Utils::DuckUtils.quacks_to?(obj, :read)
-          tmp = obj.read
-          obj.rewind
-          tmp
-        else
-          obj.to_s
-        end
-      end
     end
   end
 end

data/lib/contrast/agent/rule_set.rb

@@ -16,8 +16,7 @@ module Contrast
       # terminate requests on attack detection if set to block at perimeter
       def prefilter
         context = Contrast::Agent::REQUEST_TRACKER.current
-        # TODO: RUBY-801 We shouldn't be responsible for knowing what modes are enabled
-        return unless context&.analyze_request? || ::Contrast::PROTECT.enabled?
+        return unless context&.analyze_request?
 
         logger.trace_with_time('Running prefilter...') do
           map { |rule| rule.prefilter(context) }
@@ -33,8 +32,7 @@ module Contrast
       # has been created. The main actions here are analyzing the response for unsafe state or actions.
       def postfilter
         context = Contrast::Agent::REQUEST_TRACKER.current
-        # TODO: RUBY-801 We shouldn't be responsible for knowing what modes are enabled
-        return unless context&.analyze_response? || ::Contrast::PROTECT.enabled?
+        return unless context&.analyze_response?
 
         logger.trace_with_time('Running postfilter...') do
           map { |rule| rule.postfilter(context) }