contrast-agent 7.1.0 → 7.3.0

data/lib/contrast/agent/protect/input_analyzer/worth_watching_analyzer.rb

@@ -5,7 +5,10 @@ require 'contrast/agent/thread/worker_thread'
 require 'contrast/agent/reporting/input_analysis/input_analysis_result'
 require 'contrast/agent/reporting/input_analysis/score_level'
 require 'contrast/agent/reporting/reporting_events/application_activity'
-require 'contrast/utils/input_classification_base'
+require 'contrast/agent/protect/rule/input_classification/base'
+require 'contrast/agent/telemetry/input_analysis_cache_event'
+require 'contrast/agent/telemetry/input_analysis_encoding_event'
+require 'contrast/utils/reporting/application_activity_batch_utils'
 
 module Contrast
   module Agent
@@ -15,7 +18,8 @@ module Contrast
       # Currently only includes: cmd_injection & sqli_injection rules
       class WorthWatchingInputAnalyzer < WorkerThread
         include Timeout
-        include Contrast::Agent::Protect::Rule::InputClassificationBase
+        include Contrast::Agent::Protect::Rule::InputClassification::Base
+        include Contrast::Utils::Reporting::ApplicationActivityBatchUtils
 
         QUEUE_SIZE = 1000.cs__freeze
         AGENTLIB_TIMEOUT = 5.cs__freeze
@@ -48,8 +52,10 @@ module Contrast
                 activity.attach_defend(attack_result)
                 report = true
               end
-              Contrast::Agent::Reporting::Masker.mask(activity)
-              Contrast::Agent.reporter.send_event(activity) if report
+              report_activity(activity) if report
+              # Handle reporting of IA Cache statistics:
+              enqueue_cache_event(stored_ia.request)
+              enqueue_encoding_event(stored_ia.request)
             rescue StandardError => e
               logger.error('[WorthWatchingAnalyzer] thread could not process result because of:', e)
             end
@@ -73,6 +79,27 @@ module Contrast
 
         private
 
+        # After we have finished with all IA results, we need to send the cache statistics to Telemetry.
+        # Now the request cycle is finished and we can send the cache statistics.
+        #
+        # @param request [Contrast::Agent::Request] stored request.
+        def enqueue_cache_event request
+          return unless Contrast::Agent::Telemetry::Base.enabled?
+
+          Contrast::TELEMETRY_IA_CACHE[request.__id__] = Contrast::Agent::Protect::InputAnalyzer.
+              lru_cache.statistics.to_events.dup
+          Contrast::Agent::Protect::InputAnalyzer.lru_cache.clear_statistics
+        end
+
+        def enqueue_encoding_event request
+          return unless Contrast::Agent::Telemetry::Base.enabled?
+          return unless Contrast::PROTECT.normalize_base64?
+
+          Contrast::TELEMETRY_BASE64_HASH[request.__id__] = Contrast::Agent::Protect::InputAnalyzer.
+              base64_statistic.to_events.dup
+          Contrast::Agent::Protect::InputAnalyzer.base64_statistic.clear
+        end
+
         # This method will build the attack results from the saved ia.
         #
         # @param input_analysis [Contrast::Agent::Reporting::InputAnalysis]
@@ -114,6 +141,12 @@ module Contrast
           @_queue ||= Queue.new
         end
 
+        def report_activity activity
+          logger.debug('[WorthWatchingAnalyzer] preparing to send activity batch')
+          add_activity_to_batch(activity)
+          report_batch
+        end
+
         def delete_queue!
           @_queue&.clear
           @_queue&.close

data/lib/contrast/agent/protect/rule/base.rb

@@ -63,6 +63,10 @@ module Contrast
             RULE_NAME
           end
 
+          # Should return the short name.
+          #
+          # @return [String]
+
           # Should return list of all sub_rules.
           # Extend for each main rule any sub-rules.
           #
@@ -166,17 +170,15 @@ module Contrast
           # Check if the protect rules is excluded by url from the exclusion rules for this application.
           #
           # @param rule_id [String]
-          # @param request_path [String] Current request path
-          def protect_excluded_by_url? rule_id, request_path
-            Contrast::SETTINGS.excluder.protect_excluded_by_url?(rule_id, request_path)
+          def protect_excluded_by_url? rule_id
+            Contrast::SETTINGS.excluder.protect_excluded_by_url?(rule_id)
           end
 
           # Check if the protect rules is excluded by input from the exclusion rules for this application.
           #
           # @param results [Array<Contrast::Agent::Reporting::InputAnalysis>]
-          # @param request_path [String] Current request path
-          def protect_excluded_by_input? results, request_path
-            Contrast::SETTINGS.excluder.protect_excluded_by_input?(results, request_path)
+          def protect_excluded_by_input? results
+            Contrast::SETTINGS.excluder.protect_excluded_by_input?(results)
           end
 
           # Allows for the InputAnalysis from Agent Library to be extracted early
@@ -330,7 +332,7 @@ module Contrast
           # @param context [Contrast::Agent::RequestContext]
           # @return [Array<Contrast::Agent::Reporting::InputAnalysis>]
           def gather_ia_results context
-            return Contrast::Utils::ObjectShare::EMPTY_ARRAY unless context&.agent_input_analysis&.results
+            return [] unless context&.agent_input_analysis&.results
 
             context.agent_input_analysis.results.select do |ia_result|
               ia_result.rule_id == rule_name && ia_result.score_level != Contrast::Agent::Reporting::ScoreLevel::IGNORE

data/lib/contrast/agent/protect/rule/bot_blocker/bot_blocker.rb

@@ -77,7 +77,7 @@ module Contrast
           # @return [Contrast::Agent::Reporting::RaspRuleSample]
           def build_sample context, ia_result, _candidate_string, **_kwargs
             sample = build_base_sample(context, ia_result)
-            sample.details = Contrast::Agent::Reporting::BotBlockerDetails.new
+            sample.details = Contrast::Agent::Reporting::Details::BotBlockerDetails.new
             sample.details.bot = ia_result.value
             sample.details.user_agent = context&.request&.user_agent
             sample

data/lib/contrast/agent/protect/rule/bot_blocker/bot_blocker_input_classification.rb

@@ -3,8 +3,8 @@
 
 require 'contrast/agent/reporting/input_analysis/input_type'
 require 'contrast/agent/reporting/input_analysis/score_level'
-require 'contrast/agent/reporting/input_analysis/details/bot_blocker_details'
-require 'contrast/utils/input_classification_base'
+require 'contrast/agent/reporting/details/bot_blocker_details'
+require 'contrast/agent/protect/rule/input_classification/base'
 require 'contrast/utils/object_share'
 
 module Contrast
@@ -20,7 +20,7 @@ module Contrast
           BOT_BLOCKER_MATCH = 'bot-blocker-input-tracing-v1'
 
           class << self
-            include InputClassificationBase
+            include Contrast::Agent::Protect::Rule::InputClassification::Base
 
             # Input Classification stage is done to determine if an user input is
             # DEFINITEATTACK or to be ignored.
@@ -45,6 +45,7 @@ module Contrast
               input_analysis
             rescue StandardError => e
               logger.debug("An Error was recorded in the input classification of the #{ rule_id }", error: e)
+              nil
             end
 
             private
@@ -57,29 +58,44 @@ module Contrast
             # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
             # @param value [String, Array<String>] the value of the input.
             #
-            # @return res [Contrast::Agent::Reporting::InputAnalysisResult]
+            # @return res [Contrast::Agent::Reporting::InputAnalysisResult, nil]
             def create_new_input_result request, rule_id, input_type, value
               return unless request.headers.key(value) == USER_AGENT
-              return unless Contrast::AGENT_LIB
 
-              # If there is no match this would return nil.
-              header_eval = Contrast::AGENT_LIB.eval_header(AGENT_LIB_HEADER_NAME,
-                                                            value,
-                                                            Contrast::AGENT_LIB.rule_set[rule_id],
-                                                            Contrast::AGENT_LIB.eval_option[:NONE])
+              super(request, rule_id, input_type, value)
+            end
+
+            # Creates new instance of AgentLib evaluation result with direct call to AgentLib.
+            #
+            # @param rule_id [String] The name of the Protect Rule.
+            # @param _input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+            # @param value [String, Array<String>] the value of the input.
+            def build_input_eval rule_id, _input_type, value
+              Contrast::AGENT_LIB.eval_header(AGENT_LIB_HEADER_NAME,
+                                              value,
+                                              Contrast::AGENT_LIB.rule_set[rule_id],
+                                              Contrast::AGENT_LIB.eval_option[:NONE])
+            end
 
+            # Creates specific result from the AgentLib evaluation.
+            #
+            # @param rule_id [String] The name of the Protect Rule.
+            # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+            # @param value [String, Array<String>] the value of the input.
+            # @param request [Contrast::Agent::Request] the current request context.
+            # @param input_eval [Contrast::AgentLib::EvalResult] the result of the input evaluation.
+            def build_ia_result rule_id, input_type, value, request, input_eval
               ia_result = new_ia_result(rule_id, input_type, request.path, value)
-              score = header_eval&.score || 0
+              score = input_eval&.score || 0
               if score >= THRESHOLD
                 ia_result.score_level = DEFINITEATTACK
                 ia_result.ids << BOT_BLOCKER_MATCH
-                ia_result.details = Contrast::Agent::Reporting::BotBlockerDetails.new
+                ia_result.details = Contrast::Agent::Reporting::Details::BotBlockerDetails.new
                 # details:
                 add_details(ia_result, value)
               else
                 ia_result.score_level = IGNORE
               end
-              add_needed_key(request, ia_result, input_type, value)
               ia_result
             end
 

data/lib/contrast/agent/protect/rule/cmdi/cmdi_backdoors.rb

@@ -37,7 +37,7 @@ module Contrast
           # @raise [Contrast::SecurityException] if the rule mode ise set
           # to BLOCK and valid cdmi is detected.
           def infilter context, classname, method, command
-            return if protect_excluded_by_url?(rule_name, context.request.path)
+            return if protect_excluded_by_url?(rule_name)
             return unless backdoors_match?(command)
             return unless (result = build_attack_with_match(context, nil, nil, command,
                                                             **{ classname: classname, method: method }))

data/lib/contrast/agent/protect/rule/cmdi/cmdi_base_rule.rb

@@ -43,7 +43,6 @@ module Contrast
           # to BLOCK and valid cdmi is detected.
           def infilter context, classname, method, command
             return unless infilter?(command)
-            return if protect_excluded_by_url?(rule_name, context.request.path)
             return unless (result = build_violation(context, command))
 
             append_to_activity(context, result)

data/lib/contrast/agent/protect/rule/cmdi/cmdi_input_classification.rb

@@ -4,7 +4,7 @@
 require 'contrast/agent/protect/rule/cmdi/cmd_injection'
 require 'contrast/agent/reporting/input_analysis/score_level'
 require 'contrast/agent/protect/input_analyzer/input_analyzer'
-require 'contrast/utils/input_classification_base'
+require 'contrast/agent/protect/rule/input_classification/base'
 require 'contrast/components/logger'
 
 module Contrast
@@ -17,7 +17,7 @@ module Contrast
         module CmdiInputClassification
           WORTHWATCHING_MATCH = 'cmdi-worth-watching-v2'.cs__freeze
           class << self
-            include InputClassificationBase
+            include Contrast::Agent::Protect::Rule::InputClassification::Base
             include Contrast::Components::Logger::InstanceMethods
           end
         end

data/lib/contrast/agent/protect/rule/deserialization/deserialization.rb

@@ -58,9 +58,9 @@ module Contrast
           # Per the spec, this rule applies regardless of input. Only the mode
           # of the rule and code exclusions apply at this point.
           # @return [Boolean] should the rule apply to this call.
-          def infilter? context
+          def infilter?_context
             return false unless enabled?
-            return false if protect_excluded_by_url?(rule_name, context.request.path)
+            return false if protect_excluded_by_url?(rule_name)
 
             true
           end

data/lib/contrast/agent/protect/rule/input_classification/base.rb

@@ -0,0 +1,191 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+require 'contrast/agent/protect/input_analyzer/input_analyzer'
+require 'contrast/agent/protect/rule/input_classification/extendable'
+require 'contrast/agent/protect/rule/input_classification/encoding'
+require 'contrast/components/logger'
+
+module Contrast
+  module Agent
+    module Protect
+      module Rule
+        module InputClassification
+          # This module will include all the similar information for all input classifications
+          # between different rules
+          module Base
+            UNKNOWN_KEY = 'unknown'
+            include Contrast::Components::Logger::InstanceMethods
+            include Contrast::Agent::Protect::Rule::InputClassification::Extendable
+            include Contrast::Agent::Protect::Rule::InputClassification::Encoding
+
+            KEYS_NEEDED = [
+              COOKIE_VALUE, PARAMETER_VALUE, HEADER, JSON_VALUE, MULTIPART_VALUE, XML_VALUE, DWR_VALUE
+            ].cs__freeze
+
+            BASE64_INPUT_TYPES = [BODY, COOKIE_VALUE, HEADER, PARAMETER_VALUE, MULTIPART_VALUE, XML_VALUE].cs__freeze
+
+            class << self
+              include Contrast::Components::Logger::InstanceMethods
+              include Contrast::Agent::Reporting::InputType
+
+              # Finds key value and type based on input type and value.
+              # @param request [Contrast::Agent::Request] the current request context.
+              # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+              # @param value [String, Array<String>] the value of the input.
+              # @return [Array<(String, Contrast::Agent::Reporting::InputType)>] key and key type.
+              def find_key request, input_type, value
+                # TODO: RUBY-99999 Add handling for multipart, json and if any missing types.
+                case input_type
+                when COOKIE_VALUE
+                  [request.cookies.key(value), Contrast::Agent::Reporting::InputType::COOKIE_NAME]
+                when PARAMETER_VALUE, URL_PARAMETER
+                  [request.parameters.key(value), Contrast::Agent::Reporting::InputType::PARAMETER_NAME]
+                when HEADER
+                  [request.headers.key(value), Contrast::Agent::Reporting::InputType::HEADER]
+                when UNKNOWN
+                  [UNKNOWN_KEY, Contrast::Agent::Reporting::InputType::UNKNOWN]
+                else
+                  [nil, nil]
+                end
+              rescue StandardError => e
+                logger.warn('[InputAnalyzer] Could not find proper key for input traced value', message: e)
+                [nil, nil]
+              end
+
+              # Some input types are not yet supported from the AgentLib.
+              # This will convert the type to the closet possible if viable,
+              # so that the input tracing could be done.
+              #
+              # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+              # @return [Integer<Contrast::AgentLib::Interface::INPUT_SET>]
+              def convert_input_type input_type
+                case input_type
+                when URI, URL_PARAMETER
+                  Contrast::AGENT_LIB.input_set[:URI_PATH]
+                when BODY, DWR_VALUE, SOCKET, UNDEFINED_TYPE, UNKNOWN, REQUEST, QUERYSTRING
+                  Contrast::AGENT_LIB.input_set[:PARAMETER_VALUE]
+                when HEADER
+                  Contrast::AGENT_LIB.input_set[:HEADER_VALUE]
+                when MULTIPART_VALUE, MULTIPART_FIELD_NAME
+                  Contrast::AGENT_LIB.input_set[:MULTIPART_NAME]
+                when JSON_ARRAYED_VALUE
+                  Contrast::AGENT_LIB.input_set[:JSON_KEY]
+                when PARAMETER_NAME
+                  Contrast::AGENT_LIB.input_set[:PARAMETER_KEY]
+                else
+                  Contrast::AGENT_LIB.input_set[input_type]
+                end
+              rescue StandardError => e
+                logger.debug('[InputAnalyzer] Protect Input classification could not determine input type,
+                          falling back to default',
+                             error: e)
+                Contrast::AGENT_LIB.input_set[:PARAMETER_VALUE]
+              end
+            end
+
+            # Input Classification stage is done to determine if an user input is
+            # DEFINITEATTACK or to be ignored.
+            #
+            # @param rule_id [String] Name of the protect rule.
+            # @param input_type [Symbol, Contrast::Agent::Reporting::InputType] The type of the user input.
+            # @param value [String, Array<String>] the value of the input.
+            # @param input_analysis [Contrast::Agent::Reporting::InputAnalysis] Holds all the results from the
+            #                                                       agent analysis from the current
+            #                                                       Request.
+            # @return ia [Contrast::Agent::Reporting::InputAnalysis, nil] with updated results.
+            def classify rule_id, input_type, value, input_analysis
+              return unless (rule = Contrast::PROTECT.rule(rule_id))
+              return unless rule.applicable_user_inputs.include?(input_type)
+              return unless input_analysis.request
+
+              Array(value).each do |val|
+                Array(val).each do |v|
+                  next unless v
+
+                  result = create_new_input_result(input_analysis.request, rule.rule_name, input_type, v)
+                  append_result(input_analysis, result)
+                end
+              end
+
+              input_analysis
+            rescue StandardError => e
+              logger.debug("An Error was recorded in the input classification of the #{ rule_id }", error: e)
+              nil
+            end
+
+            # This methods checks if input is value that matches a key in the input.
+            #
+            # @param request [Contrast::Agent::Request] the current request context.
+            # @param ia_result [Contrast::Agent::Reporting::InputAnalysisResult] result to be updated.
+            # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+            # @param value [String, Array<String>] the value of the input.
+            #
+            # @return result [Array<String, Symbol>] updated with key result.
+            def add_needed_key request, ia_result, input_type, value
+              ia_result.key, ia_result.key_type = Contrast::Agent::Protect::Rule::InputClassification::Base.
+                  find_key(request, input_type, value)
+            end
+
+            private
+
+            # Appends result to the InputAnalysis.
+            #
+            # @param ia_analysis [Contrast::Agent::Reporting::InputAnalysis] the current input analysis.
+            # @param result [Contrast::Agent::Reporting::InputAnalysisResult] result to be appended.
+            # @return [Contrast::Agent::Reporting::InputAnalysis] the input analysis with the appended result.
+            def append_result ia_analysis, result
+              ia_analysis.results << result if result
+              ia_analysis
+            end
+
+            # Do not override this method, it will hold base operations, instead overwrite methods called inside
+            # of this method.
+            # This methods checks if input is tagged WORTHWATCHING or IGNORE matches value with it's
+            # key if needed and Creates new instance of InputAnalysisResult.
+            #
+            # @param request [Contrast::Agent::Request] the current request context.
+            # @param rule_id [String] The name of the Protect Rule.
+            # @param input_type [Contrast::Agent::Reporting::InputType] The type of the user input.
+            # @param value [String, Array<String>] the value of the input.
+            #
+            # @return res [Contrast::Agent::Reporting::InputAnalysisResult, nil]
+            def create_new_input_result request, rule_id, input_type, value
+              return unless Contrast::AGENT_LIB
+
+              # Cache retrieve
+              cached = Contrast::Agent::Protect::InputAnalyzer.lru_cache.lookout(rule_id, value, input_type, request)
+              return cached.result if cached.cs__is_a?(Contrast::Agent::Protect::InputClassification::CachedResult)
+
+              # Input evaluation
+              input_eval = build_input_eval(rule_id, input_type, base64_decode_input(value, input_type))
+              ia_result = build_ia_result(rule_id, input_type, value, request, input_eval)
+              return unless ia_result
+
+              add_needed_key(request, ia_result, input_type, value) if KEYS_NEEDED.include?(input_type)
+              # Input evaluation end
+
+              # Cache save. Cache must be saved after the input evaluation is completed.
+              Contrast::Agent::Protect::InputAnalyzer.lru_cache.save(rule_id, ia_result, request)
+              ia_result
+            end
+
+            # Decodes the value for the given input type.
+            # Applies to BODY, COOKIE_VALUE, HEADER, PARAMETER_VALUE, MULTIPART_VALUE, XML_VALUE
+            #
+            # @param value [String]
+            # @param input_type [Symbol]
+            # @return input [String]
+            def base64_decode_input value, input_type
+              return value unless Contrast::PROTECT.normalize_base64?
+              return value unless BASE64_INPUT_TYPES.include?(input_type)
+
+              cs__decode64(value, input_type)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/protect/rule/input_classification/base64_statistic.rb

@@ -0,0 +1,71 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/agent/protect/rule/input_classification/encoding_rates'
+require 'contrast/agent/telemetry/input_analysis_encoding_event'
+require 'contrast/components/logger'
+
+module Contrast
+  module Agent
+    module Protect
+      module Rule
+        module InputClassification
+          # This class will safe all the information for the Base64 decoding matches per input type.
+          class Base64Statistic
+            include Contrast::Components::Logger::InstanceMethods
+
+            # @return [Hash<Contrast::Agent::Protect::Rule::InputClassification::EncodingRates>]
+            attr_reader :data
+
+            # Capacity for request context life cycle.
+            CAPACITY = 1000
+
+            def initialize
+              @data = {}
+            end
+
+            # Add a match for the given input type.
+            #
+            # @param input_type [Symbol]
+            def match! input_type
+              return @data[input_type]&.increase_match_base64 if @data[input_type]
+
+              @data[input_type] = Contrast::Agent::Protect::Rule::InputClassification::EncodingRates.new(input_type)
+              @data[input_type].increase_match_base64
+            end
+
+            # Add a mismatch for the given input type.
+            #
+            # @param input_type [Symbol]
+            def mismatch! input_type
+              return @data[input_type]&.increase_mismatch_base64 if @data[input_type]
+
+              @data[input_type] = Contrast::Agent::Protect::Rule::InputClassification::EncodingRates.new(input_type)
+              @data[input_type].increase_mismatch_base64
+            end
+
+            # Clears statistic data.
+            def clear
+              @data.clear
+            end
+
+            # @return [Array<Contrast::Agent::Telemetry::InputAnalysisCacheEvent>] the events to be sent.
+            def to_events
+              events = []
+              data.each do |_input_type, encoding_rate|
+                event = Contrast::Agent::Telemetry::InputAnalysisEncodingEvent.new(nil, encoding_rate)
+                next if event.empty?
+
+                events << event
+              end
+              events
+            rescue StandardError => e
+              logger.error("[Telemetry] Error while creating events: #{ e }", stacktrace: e.backtrace)
+              []
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/protect/rule/input_classification/cached_result.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/duck_utils'
+require 'contrast/agent/reporting/input_analysis/input_analysis_result'
+
+module Contrast
+  module Agent
+    module Protect
+      module InputClassification
+        # This Class with store the input classification results for a given user input.
+        class CachedResult
+          # @return [String]
+          attr_reader :result
+          # @return [Integer]
+          attr_reader :request_id
+
+          # Initialize Input Classification Cached Result
+          #
+          # @param result [Contrast::Agent::Reporting::InputAnalysisResult]
+          # @param request_id [Integer] the id of current request.
+          def initialize result, request_id
+            @result = result.dup if result&.cs__is_a?(Contrast::Agent::Reporting::InputAnalysisResult)
+            @request_id = request_id
+          end
+
+          # Check if the input classification result is empty.
+          #
+          # @return [Boolean]
+          def empty?
+            Contrast::Utils::DuckUtils.empty_duck?(@result) && Contrast::Utils::DuckUtils.empty_duck?(@request_id)
+          end
+        end
+      end
+    end
+  end
+end