contrast-agent 5.3.0 → 6.0.0

data/lib/contrast/agent/reporting/attack_result/rasp_rule_sample.rb

@@ -0,0 +1,52 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+require 'contrast/utils/timer'
+require 'contrast/agent/reporting/attack_result/user_input'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This class will hold the new RaspRuleSample.
+      # protect rules.
+      class RaspRuleSample
+        def timestamp
+          @_timestamp ||= 0
+        end
+
+        def timestamp= timestamp_ms
+          @_timestamp = timestamp_ms
+        end
+
+        def user_input
+          @_user_input ||= Contrast::Agent::Reporting::UserInput.new
+        end
+
+        def user_input= input
+          @_user_input = input if input.is_a?(Contrast::Agent::Reporting::UserInput)
+        end
+
+        def build context, ia_result
+          sample = self
+          sample.timestamp = context&.timer&.start_ms
+          sample.user_input = build_user_input_from_ia(ia_result)
+          sample.user_input.document_type = if context&.request
+                                              Contrast::Utils::StringUtils.force_utf8(context.request.document_type)
+                                            end
+          sample
+        end
+
+        def build_user_input_from_ia ia_result
+          user_input = Contrast::Agent::Reporting::UserInput.new
+          user_input.input_type = ia_result.input_type
+          user_input.matcher_ids = ia_result.ids
+          user_input.path = ia_result.path
+          user_input.key = ia_result.key if ia_result.key
+          user_input.value = ia_result.value if ia_result.value
+          user_input
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/attack_result/response_type.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This module will hold the response types used to generate
+      # attack result.
+      module ResponseType
+        BLOCKED             = :BLOCKED.cs__freeze
+        MONITORED           = :MONITORED.cs__freeze
+        PROBED              = :PROBED.cs__freeze
+        BLOCK_AT_PERIMETER  = :BLOCK_AT_PERIMETER.cs__freeze
+        SUSPICIOUS          = :SUSPICIOUS.cs__freeze
+        AGGREGATED          = :AGGREGATED.cs__freeze
+        EXPLOITED           = :EXPLOITED.cs__freeze
+        NO_ACTION           = :NO_ACTION.cs__freeze
+
+        class << self
+          def to_a
+            [NO_ACTION, BLOCKED, MONITORED, PROBED, BLOCK_AT_PERIMETER, EXPLOITED, SUSPICIOUS, AGGREGATED]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/attack_result/user_input.rb

@@ -0,0 +1,87 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+require 'contrast/agent/reporting/input_analysis/input_type'
+require 'contrast/agent/request_context'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This class will hold the new Sqli detail used by RaspRuleSample
+      class UserInput
+        INPUT_TYPE = Contrast::Agent::Reporting::InputType
+        DOCUMENT_TYPE = { XML: :XML, JSON: :JSON, NORMAL: :NORMAL }.cs__freeze
+
+        # @return @_path [String]
+        def path
+          @_path ||= Contrast::Utils::ObjectShare::EMPTY_STRING
+        end
+
+        # @param path [String]
+        # @return @_path [String]
+        def path= path
+          @_path = path if path.is_a?(String)
+        end
+
+        # @return @_key [String]
+        def key
+          @_key ||= Contrast::Utils::ObjectShare::EMPTY_STRING
+        end
+
+        # @param key [String]
+        # @return @_key [String]
+        def key= key
+          @_key = key if key.is_a?(String)
+        end
+
+        # @return value [String]
+        def value
+          @_value ||= Contrast::Utils::ObjectShare::EMPTY_STRING
+        end
+
+        # @param value [String]
+        # @return value [String]
+        def value= value
+          @_value = value if value.is_a?(String)
+        end
+
+        # @return @_input_type [
+        # Symbol<Contrast::Agent::Reporting::Settings::InputAnalysis::InputAnalysisResult::InputType>]
+        def input_type
+          @_input_type ||= INPUT_TYPE::UNDEFINED_TYPE
+        end
+
+        # @param input_type [
+        # Symbol<Contrast::Agent::Reporting::Settings::InputAnalysis::InputAnalysisResult::InputType>]
+        # @return @_input_type [
+        # Symbol<Contrast::Agent::Reporting::Settings::InputAnalysis::InputAnalysisResult::InputType>]
+        def input_type= input_type
+          @_input_type = input_type if INPUT_TYPE.to_a.include?(input_type)
+        end
+
+        # type [Symbol<:XML, :JSON, :NORMAL>]
+        def document_type
+          @_document_type ||= DOCUMENT_TYPE[:NORMAL]
+        end
+
+        def document_type= type
+          @_document_type = type if DOCUMENT_TYPE.value?(type)
+        end
+
+        # Matchers IDs
+        # @return @_ids [Array<String>]
+        def matcher_ids
+          @_matcher_ids ||= []
+        end
+
+        # Matchers IDs
+        # @param ids [Array<String>]
+        # @return @_ids [Array<String>]
+        def matcher_ids= ids
+          @_matcher_ids = ids if ids.is_a?(Array) && ids.any?(String)
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/masker/masker.rb

@@ -0,0 +1,246 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporter'
+require 'contrast/utils/object_share'
+require 'contrast/agent/reporting/attack_result/response_type'
+require 'contrast/agent/reporting/masker/masker_utils'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This module duty is to mask any activity containing sensitive information - PII masking.
+      module Masker
+        MASK = 'contrast-redacted-'
+        BODY_MASK = 'contrast-redacted-body'
+        BODY_BINARY_MASK = BODY_MASK.bytes.to_s.cs__freeze
+
+        class << self
+          include Contrast::Components::Logger::InstanceMethods
+          include Contrast::Utils::ObjectShare
+          include Contrast::Agent::Reporting::MaskerUtils
+
+          # Keyword dictionary, extracted from the TS response.
+          #
+          # @return Array<Contrast::Agent::Reporting::Settings::SensitiveDataMaskingRule>
+          def dictionary
+            @_dictionary ||= update_dictionary
+          end
+
+          # Mask sensitive data according to the contrast sensitive data rules.
+          #
+          # @param [Contrast::Api::Dtm::Activity]
+          def mask activity
+            return unless Contrast::Agent::Reporter.enabled?
+            return unless activity || activity.http_request || activity.results
+
+            logger.debug('Searching for sensitive data',
+                         activity: activity.__id__,
+                         request: activity.http_request&.uuid)
+            mask_body activity
+            mask_query_string activity
+            mask_request_params activity
+            mask_request_cookies activity
+            mask_request_headers activity
+          rescue StandardError => e
+            logger.debug('Could not mask activity!',
+                         activity: activity.__id__,
+                         request: activity.http_request&.uuid,
+                         error_msg: e.message)
+          end
+
+          private
+
+          # When called this will overwrite existing rules with those
+          # in settings. Ideally we need to call this on init and when
+          # the Agent receives new rules from TS response.This is called
+          # from Contrast::Components::Settings.
+          #
+          # To make it save this is private method because if new
+          # settings arrive and they might be empty as edge case,
+          # this will produce dictionary with empty rules and this
+          # is not desirable.
+          def update_dictionary
+            @_dictionary = Contrast::SETTINGS.sensitive_data_masking.rules
+          end
+
+          # Mask request body:
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_body [String, nil]
+          def mask_body activity
+            return unless mask_body?
+
+            body = activity.http_request.request_body
+            return if body.nil? || body.empty?
+
+            activity.http_request.request_body = BODY_MASK
+            activity.http_request.request_body_binary = BODY_BINARY_MASK
+          end
+
+          # Mask request params.
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_body [String, nil]
+          def mask_request_params activity
+            params = activity.http_request.normalized_request_params
+            return unless params
+
+            mask_with_dictionary activity.results, params
+          end
+
+          def mask_request_headers activity
+            if activity.http_request.parsed_request_headers
+              # Used normalized request_headers
+              mask_with_dictionary activity.results, activity.http_request.normalized_request_headers
+            else
+              headers = activity.http_request.request_headers
+              mask_field_hash headers, activity.results
+            end
+          end
+
+          # Mask Cookies.
+          #
+          # @param activity [Contrast::Api::Dtm::Activity] Activity to mask
+          # @return masked_values [Hash, nil]
+          def mask_request_cookies activity
+            cookies = activity.http_request.normalized_cookies
+            return unless cookies
+
+            mask_with_dictionary activity.results, cookies
+          end
+
+          # Mask request query string:
+          # exp: password => sensitive to password => contrast-redacted-password
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_query [String]
+          def mask_query_string activity
+            qs = activity.http_request.query_string
+            return if qs.nil? || qs.empty?
+
+            mask_field_hash qs, activity.results unless qs.cs__is_a?(String)
+            mask_raw_query qs, activity.results
+          end
+
+          # Mask if the value in the passed hash are matched against dictionary
+          # keyword. If the mask_attack_vector flag is set, this will also mask
+          # any attack.
+          #
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param hash [Hash] Normalized hash representing the key/val pair from
+          # the activity's http request parameters.
+          # @return nil | Protobuf::Field::FieldHash
+          def mask_with_dictionary results, hash
+            return if hash.nil? || hash.empty?
+
+            hash.each do |key, val|
+              match = dictionary_matcher key
+              next unless match
+
+              # The normalized values are paired.
+              # key => Contrast::Api::Dtm::Pair (key, val<Values>).
+              # try one level down
+              if val.cs__respond_to? :values
+                mask_values key, val, results
+              else
+                # Just assign keys.
+                mask_hash key, val, hash, results
+              end
+            end
+            hash
+          end
+
+          # Mask the values of DTM pair with attack vector condition check.
+          # if the attack vector flag is set then mask the attack value.
+          #
+          # @param key [String] current iterable key from Protobuf::Field::FieldHash
+          # pointing to Contrast::Api::Dtm::Pair<key, val>(holding the value to mask)
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param val [Contrast::Api::Dtm::Pair<Value>]
+          def mask_values key, val, results
+            val.values.each.with_index do |v, idx|
+              # Mask the attack vector only if the flag is set.
+              val.values[idx] = MASK + key.downcase if attack_vector?(results, v) && mask_attack_vector?
+              # It is not attack vector and we mask it as normal.
+              val.values[idx] = MASK + key.downcase unless attack_vector?(results, v)
+            end
+            val
+          end
+
+          # Handles the masking of Field hash with string values.
+          # this case is used when called from #mask_field_hash
+          # and #mask_raw_query helper methods. Since they dont
+          # return values containing sub-values  (key, val<Values>).
+          #
+          # @param key [String] current iterable key from Protobuf::Field::FieldHash
+          # @param val [String] normalized value to be matched against the results
+          # and masked.
+          # @param hash [Hash] Normalized hash representing the key/val pair.
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          def mask_hash key, val, hash, results
+            hash[key] = MASK + key.downcase if attack_vector?(results, val) && mask_attack_vector?
+            hash[key] = MASK + key.downcase unless attack_vector?(results, val)
+          end
+
+          # Match to see if values matches input from AttackResults array.
+          # If match is found and the attack result's response is any of
+          # [BAP(Block At Perimeter), BLOCKED, PROBED] the return is true.
+          #
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param value [String] Input to match.
+          # @return true | false
+          def attack_vector? results, value
+            return false unless value && results
+
+            results.each do |result|
+              # Check samples Contrast::Api::Dtm::RaspRuleSample
+              # is the value in sample and the response is valid?
+              result.samples.any? do |sample|
+                # Check user input Contrast::Api::Dtm::UserInput.
+                match = sample.user_input.value == value.to_s &&
+                    result.response&.name != Contrast::Agent::Reporting::ResponseType::NO_ACTION
+
+                return match if match
+              end
+            end
+            false
+          end
+
+          # Consult with our current settings state.
+          #
+          # @return true | false
+          def mask_attack_vector?
+            Contrast::SETTINGS.sensitive_data_masking.mask_attack_vector?
+          end
+
+          # Consult with our current settings state.
+          #
+          # @return true | false
+          def mask_body?
+            Contrast::SETTINGS.sensitive_data_masking.mask_http_body?
+          end
+
+          # Check to see if value is matched in the dictionary.
+          #
+          # @param value [String] Value to check.
+          # @return match [String, nil] from the Dictionary, or nil.
+          def dictionary_matcher value
+            return unless @_dictionary
+
+            @_dictionary.each do |rule|
+              idx = rule.keywords.find_index(value.downcase)
+              return rule.keywords[idx] if idx
+            end
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/masker/masker_utils.rb

@@ -0,0 +1,58 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+
+module Contrast
+  module Agent
+    module Reporting
+      # helper methods used for masking
+      module MaskerUtils
+        include Contrast::Utils::ObjectShare
+        # Helper to deal with Protobuf FieldHash.
+        #
+        # @param field_hash [Protobuf::Field::FieldHash] hash to be masked
+        # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+        # results to match against.
+        def mask_field_hash field_hash, results
+          hash = {}
+          masked = EMPTY_STRING
+          field_hash.any? do |entry|
+            # Protobuf::Field::FieldHash produces array, with the key as first param and value as second.
+            new_value = entry[1].delete(SEMICOLON).split(SPACE)
+            new_value.each do |value|
+              arr = value.split(EQUALS)
+              # Add to new hash.
+              hash[arr[0]] = arr[1]
+            end
+            # Mask the newly created hash.
+            mask_with_dictionary results, hash
+
+            # Restore to original form.
+            hash.each { |k, v| masked += "#{ k + EQUALS }#{ v + SEMICOLON + SPACE }" }
+            masked.rstrip!
+            field_hash[entry[0]] = masked
+          end
+        end
+
+        # Mask raw query as it comes from the env.
+        # exp:
+        # 'ssn=1234567&id=%272%20or%202%20=%202%27' =>
+        # 'ssn=contrast-redacted-ssn&id=contrast-redacted-id'
+        #
+        # @param query [String]
+        # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+        # results to match against.
+        def mask_raw_query query, results
+          masked = EMPTY_STRING
+          hash = URI.decode_www_form(query).to_h
+          mask_with_dictionary results, hash
+          # Restore to string form.
+          hash.each { |k, v| masked += "#{ k + EQUALS }#{ v }#{ AMPERSAND }" }
+          query = masked
+          query.chomp!(masked[-1])
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/report.rb

@@ -26,3 +26,5 @@ require 'contrast/agent/reporting/reporting_events/observed_route'
 require 'contrast/agent/reporting/reporting_events/route_coverage'
 require 'contrast/agent/reporting/reporting_events/observed_library_usage'
 require 'contrast/agent/reporting/reporting_events/poll'
+# Sensitive data masking
+require 'contrast/agent/reporting/masker/masker'

data/lib/contrast/agent/reporting/reporter.rb

@@ -4,6 +4,7 @@
 require 'contrast/agent/worker_thread'
 require 'contrast/agent/reporting/report'
 require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporting_events/agent_startup'
 
 module Contrast
   module Agent
@@ -11,8 +12,6 @@ module Contrast
     class Reporter < WorkerThread
       include Contrast::Components::Logger::InstanceMethods
       include Contrast::Utils::ObjectShare
-      # 15 min
-      TIMEOUT = 900.cs__freeze
 
       class << self
         # check if we can report to TS
@@ -58,19 +57,24 @@ module Contrast
             # to figure out why that is and lock it so that it isn't.
             next unless client && connection
 
-            event = queue.pop
-            begin
-              response = client.send_event(event, connection)
-              audit&.audit_event(event, response) if ::Contrast::API.request_audit_enable?
-            rescue StandardError => e
-              logger.error('Could not send message to service from Reporter queue.', e)
-            end
-            sleep(TIMEOUT) if client.sleep?
-            client.wake_up
+            process_event(queue.pop)
           end
         end
       end
 
+      # Suspend the Reporter and try sending the event after the timeout.
+      # The timeout is either default 15 min or received via TS response.
+      #
+      # @param event [Contrast::Agent::Reporting::FindingEvent] Freshly pop-ed event.
+      def handle_resend event
+        sleep(client.timeout) if client.sleep?
+        # Retry once than discard the event. This is trigger on too many events of
+        # same kind error.
+        client.send_event(event, connection) if client.mode.status == client.mode.resending
+        client.mode.reset_mode
+        client.wake_up
+      end
+
       def send_event event
         if ::Contrast::AGENT.disabled?
           logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
@@ -117,6 +121,14 @@ module Contrast
       def queue
         @_queue ||= Queue.new
       end
+
+      def process_event event
+        response = client.send_event(event, connection)
+        audit&.audit_event(event, response) if ::Contrast::API.request_audit_enable?
+        handle_resend event
+      rescue StandardError => e
+        logger.error('Could not send message to service from Reporter queue.', e)
+      end
     end
   end
 end

data/lib/contrast/agent/reporting/reporting_events/agent_startup.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/agent/reporting/reporting_events/reporting_event'
+require 'contrast/config'
+
+module Contrast
+  module Agent
+    module Reporting
+      #	AgentStartup Event which sends the agent data to TeamServer on the startup of a server or process,
+      # used to create a new Server entity there.
+      class AgentStartup < Contrast::Agent::Reporting::ReportingEvent
+        def initialize
+          @event_method = :PUT
+          @event_endpoint = Contrast::Agent::Reporting::Endpoints::NG_ENDPOINTS[:agent_startup]
+          @event_type = :agent_startup
+          super
+        end
+
+        def to_controlled_hash
+          {
+              environment: ::Contrast::CONFIG.root.server.environment,
+              tags: ::Contrast::CONFIG.root.server.tags,
+              version: Contrast::Agent::VERSION
+          }
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/application_inventory.rb

@@ -9,9 +9,13 @@ module Contrast
   module Agent
     module Reporting
       # This is the new ApplicationInventory class which will include all the needed information
-      # for the new reporting system to report
+      # for the new reporting system to report. Reporting those components or details discovered at
+      # startup, or as soon as possible, but not necessarily seen used by the application. Each of
+      # these messages should only be sent once per application.
       class ApplicationInventory < Contrast::Agent::Reporting::ReportingEvent
-        attr_accessor :routes, :agent_session_id_value
+        # @param [Array<Contrast::Agent::Reporting::DiscoveredRoute>] the routes registered to this application, as
+        #   discovered during first request processing.
+        attr_reader :routes
 
         class << self
           def convert app_update_dtm
@@ -31,7 +35,7 @@ module Contrast
 
         def to_controlled_hash
           {
-              session_id: agent_session_id_value,
+              session_id: @agent_session_id_value,
               routes: routes.map(&:to_controlled_hash)
           }
         end

data/lib/contrast/agent/reporting/reporting_events/application_startup.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/agent/reporting/reporting_events/application_startup_instrumentation'
+require 'contrast/agent/reporting/reporting_events/reporting_event'
+require 'contrast/config'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This is the new ApplicationStartup class which will include all the needed information for the new reporting
+      # system to report an Application has started; creating a new one or marking an existing one as online in the
+      # Contrast UI
+      class ApplicationStartup < Contrast::Agent::Reporting::ReportingEvent
+        def initialize
+          @event_method = :POST
+          @event_endpoint = Contrast::Agent::Reporting::Endpoints::NG_ENDPOINTS[:application_create]
+          super
+        end
+
+        # Convert the instance variables on the class, and other information, into the identifiers required for
+        # TeamServer to process the JSON form of this message.
+        #
+        # @return [Hash]
+        # @raise [ArgumentError]
+        def to_controlled_hash
+          {
+              code: ::Contrast::CONFIG.root.application.code,
+              group: ::Contrast::CONFIG.root.application.group,
+              instrumentation: Contrast::Agent::Reporting::ApplicationStartupInstrumentation.new.to_controlled_hash,
+              metadata: ::Contrast::CONFIG.root.application.metadata,
+              session_id: ::Contrast::CONFIG.root.application.session_id,
+              session_metadata: ::Contrast::CONFIG.root.application.session_metadata,
+              tags: ::Contrast::CONFIG.root.application.tags
+          }
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/application_startup_instrumentation.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/protect'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This is the new ApplicationStartupInstrumentation class which will include all the needed information for the
+      # system to report the instrumentation mode to which the agent was set by local configuration.
+      class ApplicationStartupInstrumentation
+        # Convert the instance variables on the class, and other information, into the identifiers required for
+        # TeamServer to process the JSON form of this message.
+        #
+        # @return [Hash]
+        # @raise [ArgumentError]
+        def to_controlled_hash
+          {
+              protect: {
+                  enable: ::Contrast::PROTECT.enabled?
+              }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/finding.rb

@@ -100,7 +100,7 @@ module Contrast
           return unless request
 
           @request = Contrast::Agent::Reporting::FindingRequest.convert(request)
-          @routes << Contrast::Agent::Reporting::RouteDiscovery.convert(request.route)
+          @routes << Contrast::Agent::Reporting::RouteDiscovery.convert(request.route) if request.route
         end
 
         # Attach the data from a Contrast::Api::Dtm::Finding required for property based findings generated during