contrast-agent 5.3.0 → 6.1.1

data/lib/contrast/agent/reporting/masker/masker.rb

@@ -0,0 +1,243 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporter'
+require 'contrast/utils/object_share'
+require 'contrast/agent/reporting/attack_result/response_type'
+require 'contrast/agent/reporting/masker/masker_utils'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This module duty is to mask any activity containing sensitive information - PII masking.
+      module Masker
+        MASK = 'contrast-redacted-'
+        BODY_MASK = 'contrast-redacted-body'
+        BODY_BINARY_MASK = BODY_MASK.bytes.to_s.cs__freeze
+
+        class << self
+          include Contrast::Components::Logger::InstanceMethods
+          include Contrast::Utils::ObjectShare
+          include Contrast::Agent::Reporting::MaskerUtils
+
+          # Keyword dictionary, extracted from the TS response.
+          #
+          # @return Array<Contrast::Agent::Reporting::Settings::SensitiveDataMaskingRule>
+          def dictionary
+            @_dictionary ||= update_dictionary
+          end
+
+          # Mask sensitive data according to the contrast sensitive data rules.
+          #
+          # @param [Contrast::Api::Dtm::Activity]
+          def mask activity
+            return unless Contrast::Agent::Reporter.enabled?
+            return unless activity
+
+            logger.debug('Searching for sensitive data',
+                         activity: activity.__id__,
+                         request: activity.http_request&.uuid)
+            mask_body(activity)
+            mask_query_string(activity)
+            mask_request_params(activity)
+            mask_request_cookies(activity)
+            mask_request_headers(activity)
+          rescue StandardError => _e
+            logger.debug('Could not mask activity!', activity: activity.__id__, request: activity.http_request&.uuid)
+          end
+
+          private
+
+          # When called this will overwrite existing rules with those
+          # in settings. Ideally we need to call this on init and when
+          # the Agent receives new rules from TS response.This is called
+          # from Contrast::Components::Settings.
+          #
+          # To make it save this is private method because if new
+          # settings arrive and they might be empty as edge case,
+          # this will produce dictionary with empty rules and this
+          # is not desirable.
+          def update_dictionary
+            @_dictionary = Contrast::SETTINGS.sensitive_data_masking.rules
+          end
+
+          # Mask request body:
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_body [String, nil]
+          def mask_body activity
+            return unless mask_body?
+
+            body = activity.http_request.request_body
+            return if body.nil? || body.empty?
+
+            activity.http_request.request_body = BODY_MASK
+            activity.http_request.request_body_binary = BODY_BINARY_MASK
+          end
+
+          # Mask request params.
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_body [String, nil]
+          def mask_request_params activity
+            params = activity.http_request.normalized_request_params
+            return unless params
+
+            mask_with_dictionary(activity.results, params)
+          end
+
+          def mask_request_headers activity
+            if activity.http_request.parsed_request_headers
+              # Used normalized request_headers
+              mask_with_dictionary(activity.results, activity.http_request.normalized_request_headers)
+            else
+              headers = activity.http_request.request_headers
+              mask_field_hash(headers, activity.results)
+            end
+          end
+
+          # Mask Cookies.
+          #
+          # @param activity [Contrast::Api::Dtm::Activity] Activity to mask
+          # @return masked_values [Hash, nil]
+          def mask_request_cookies activity
+            cookies = activity.http_request.normalized_cookies
+            return unless cookies
+
+            mask_with_dictionary(activity.results, cookies)
+          end
+
+          # Mask request query string:
+          # exp: password => sensitive to password => contrast-redacted-password
+          #
+          # @param activity [Contrast::Api::Dtm::Activity]
+          # @return masked_query [String]
+          def mask_query_string activity
+            qs = activity.http_request.query_string
+            return if qs.nil? || qs.empty?
+
+            mask_field_hash(qs, activity.results) unless qs.cs__is_a?(String)
+            mask_raw_query(qs, activity.results)
+          end
+
+          # Mask if the value in the passed hash are matched against dictionary
+          # keyword. If the mask_attack_vector flag is set, this will also mask
+          # any attack.
+          #
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param hash [Hash] Normalized hash representing the key/val pair from
+          # the activity's http request parameters.
+          # @return nil | Protobuf::Field::FieldHash
+          def mask_with_dictionary results, hash
+            return if hash.nil? || hash.empty?
+
+            hash.each do |key, val|
+              match = dictionary_matcher(key)
+              next unless match
+
+              # The normalized values are paired.
+              # key => Contrast::Api::Dtm::Pair (key, val<Values>).
+              # try one level down
+              if val.cs__respond_to?(:values)
+                mask_values(key, val, results)
+              else
+                # Just assign keys.
+                mask_hash(key, val, hash, results)
+              end
+            end
+            hash
+          end
+
+          # Mask the values of DTM pair with attack vector condition check.
+          # if the attack vector flag is set then mask the attack value.
+          #
+          # @param key [String] current iterable key from Protobuf::Field::FieldHash
+          # pointing to Contrast::Api::Dtm::Pair<key, val>(holding the value to mask)
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param val [Contrast::Api::Dtm::Pair<Value>]
+          def mask_values key, val, results
+            val.values.each.with_index do |v, idx|
+              # Mask the attack vector only if the flag is set.
+              val.values[idx] = MASK + key.downcase if attack_vector?(results, v) && mask_attack_vector?
+              # It is not attack vector and we mask it as normal.
+              val.values[idx] = MASK + key.downcase unless attack_vector?(results, v)
+            end
+            val
+          end
+
+          # Handles the masking of Field hash with string values.
+          # this case is used when called from #mask_field_hash
+          # and #mask_raw_query helper methods. Since they dont
+          # return values containing sub-values  (key, val<Values>).
+          #
+          # @param key [String] current iterable key from Protobuf::Field::FieldHash
+          # @param val [String] normalized value to be matched against the results
+          # and masked.
+          # @param hash [Hash] Normalized hash representing the key/val pair.
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          def mask_hash key, val, hash, results
+            hash[key] = MASK + key.downcase if attack_vector?(results, val) && mask_attack_vector?
+            hash[key] = MASK + key.downcase unless attack_vector?(results, val)
+          end
+
+          # Match to see if values matches input from AttackResults array.
+          # If match is found and the attack result's response is any of
+          # [BAP(Block At Perimeter), BLOCKED, PROBED] the return is true.
+          #
+          # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+          # results to match against.
+          # @param value [String] Input to match.
+          # @return true | false
+          def attack_vector? results, value
+            return false unless value && results
+
+            results.each do |result|
+              # Check samples Contrast::Api::Dtm::RaspRuleSample
+              # is the value in sample and the response is valid?
+              result.samples.any? do |sample|
+                # Check user input Contrast::Api::Dtm::UserInput.
+                match = sample.user_input.value == value.to_s &&
+                    result.response&.name != Contrast::Agent::Reporting::ResponseType::NO_ACTION
+
+                return match if match
+              end
+            end
+            false
+          end
+
+          # Consult with our current settings state.
+          #
+          # @return true | false
+          def mask_attack_vector?
+            Contrast::SETTINGS.sensitive_data_masking.mask_attack_vector?
+          end
+
+          # Consult with our current settings state.
+          #
+          # @return true | false
+          def mask_body?
+            Contrast::SETTINGS.sensitive_data_masking.mask_http_body?
+          end
+
+          # Check to see if value is matched in the dictionary.
+          #
+          # @param value [String] Value to check.
+          # @return match [String, nil] from the Dictionary, or nil.
+          def dictionary_matcher value
+            return unless @_dictionary
+
+            @_dictionary.each do |rule|
+              idx = rule.keywords.find_index(value.downcase)
+              return rule.keywords[idx] if idx
+            end
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/masker/masker_utils.rb

@@ -0,0 +1,62 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/object_share'
+
+module Contrast
+  module Agent
+    module Reporting
+      # helper methods used for masking
+      module MaskerUtils
+        include Contrast::Utils::ObjectShare
+        # Helper to deal with Protobuf FieldHash.
+        #
+        # @param field_hash [Protobuf::Field::FieldHash] hash to be masked
+        # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+        # results to match against.
+        # @return [Hash]
+        def mask_field_hash field_hash, results
+          return {} unless field_hash&.any?
+
+          hash = {}
+          # Because this is the start of a built string, we have to be sure that it is not frozen.
+          masked = +''
+          field_hash.each do |entry|
+            # Protobuf::Field::FieldHash produces array, with the key as first param and value as second.
+            new_value = entry[1].delete(SEMICOLON).split(SPACE)
+            new_value.each do |value|
+              arr = value.split(EQUALS)
+              # Add to new hash.
+              hash[arr[0]] = arr[1]
+            end
+            # Mask the newly created hash.
+            mask_with_dictionary(results, hash)
+
+            # Restore to original form.
+            hash.each { |k, v| masked += "#{ k }=#{ v }; " }
+            masked.rstrip!
+            field_hash[entry[0]] = masked
+          end
+        end
+
+        # Mask raw query as it comes from the env.
+        # exp:
+        # 'ssn=1234567&id=%272%20or%202%20=%202%27' =>
+        # 'ssn=contrast-redacted-ssn&id=contrast-redacted-id'
+        #
+        # @param query [String]
+        # @param results [Array<Contrast::Api::Dtm::AttackResults>]
+        # results to match against.
+        def mask_raw_query query, results
+          masked = EMPTY_STRING
+          hash = URI.decode_www_form(query).to_h
+          mask_with_dictionary(results, hash)
+          # Restore to string form.
+          hash.each { |k, v| masked += "#{ k }=#{ v }&" }
+          query = masked
+          query.chomp!(masked[-1])
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/report.rb

@@ -26,3 +26,5 @@ require 'contrast/agent/reporting/reporting_events/observed_route'
 require 'contrast/agent/reporting/reporting_events/route_coverage'
 require 'contrast/agent/reporting/reporting_events/observed_library_usage'
 require 'contrast/agent/reporting/reporting_events/poll'
+# Sensitive data masking
+require 'contrast/agent/reporting/masker/masker'

data/lib/contrast/agent/reporting/reporter.rb

@@ -4,6 +4,7 @@
 require 'contrast/agent/worker_thread'
 require 'contrast/agent/reporting/report'
 require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporting_events/agent_startup'
 
 module Contrast
   module Agent
@@ -11,8 +12,6 @@ module Contrast
     class Reporter < WorkerThread
       include Contrast::Components::Logger::InstanceMethods
       include Contrast::Utils::ObjectShare
-      # 15 min
-      TIMEOUT = 900.cs__freeze
 
       class << self
         # check if we can report to TS
@@ -32,10 +31,6 @@ module Contrast
         @_connection ||= client.initialize_connection
       end
 
-      def audit
-        @_audit ||= Contrast::Agent::Reporting::Audit.new
-      end
-
       def attempt_to_start?
         unless cs__class.enabled?
           logger.warn('Reporter service is disabled!')
@@ -58,19 +53,25 @@ module Contrast
             # to figure out why that is and lock it so that it isn't.
             next unless client && connection
 
-            event = queue.pop
-            begin
-              response = client.send_event(event, connection)
-              audit&.audit_event(event, response) if ::Contrast::API.request_audit_enable?
-            rescue StandardError => e
-              logger.error('Could not send message to service from Reporter queue.', e)
-            end
-            sleep(TIMEOUT) if client.sleep?
-            client.wake_up
+            process_event(queue.pop)
           end
         end
       end
 
+      # Suspend the Reporter and try sending the event after the timeout.
+      # The timeout is either default 15 min or received via TS response.
+      #
+      # @param event [Contrast::Agent::Reporting::ReportingEvent] Freshly pop-ed event.
+      def handle_resend event
+        sleep(client.timeout) if client.sleep?
+        # Retry once than discard the event. This is trigger on too many events of
+        # same kind error.
+        client.send_event(event, connection) if client.mode.status == client.mode.resending
+        client.mode.reset_mode
+        client.wake_up
+      end
+
+      # @param event [Contrast::Agent::Reporting::ReportingEvent]
       def send_event event
         if ::Contrast::AGENT.disabled?
           logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
@@ -80,23 +81,21 @@ module Contrast
         return unless cs__class.enabled?
 
         logger.debug('Enqueued event for sending', event_type: event.cs__class)
-        audit&.audit_event(event) if ::Contrast::API.request_audit_enable?
         queue << event
       end
 
       # Use this to bypass the messaging queue and leave response processing to the caller
+      #
+      # @param event [Contrast::Agent::Reporting::ReportingEvent]
+      # @return [Net::HTTPResponse, nil]
       def send_event_immediately event
         if ::Contrast::AGENT.disabled?
           logger.warn('Reporter attempted to send event immediately with Agent disabled', caller: caller, event: event)
           return
         end
-        response = client.send_event(event, connection, true)
-        return unless response
-
-        client.handle_response(event, response, connection)
-        audit&.audit_event(event, response) if ::Contrast::API.request_audit_enable?
+        client.send_event(event, connection, send_immediately: true)
       rescue StandardError => e
-        logger.error('Could not send message to service from Reporter queue.', e)
+        logger.error('Could not send message to TeamServer from Reporter queue.', e)
       end
 
       def delete_queue!
@@ -117,6 +116,14 @@ module Contrast
       def queue
         @_queue ||= Queue.new
       end
+
+      # @param event [Contrast::Agent::Reporting::ReportingEvent]
+      def process_event event
+        client.send_event(event, connection)
+        handle_resend(event) if client.mode.status == client.mode.resending
+      rescue StandardError => e
+        logger.error('Could not send message to TeamServer from Reporter queue.', e)
+      end
     end
   end
 end

data/lib/contrast/agent/reporting/reporter_heartbeat.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/agent/worker_thread'
+require 'contrast/agent/reporting/report'
+require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporting_events/agent_startup'
+
+module Contrast
+  module Agent
+    # The ReporterHeartbeat will make sure that the process remains marked alive by TeamServer and that we periodically
+    # reach out to get the latest settings for this application.
+    class ReporterHeartbeat < WorkerThread
+      include Contrast::Components::Logger::InstanceMethods
+
+      # TeamServer will mark an application offline after 5 minutes. Sending this every one should be more than enough
+      # to satisfy our goals.
+      REFRESH_INTERVAL_SEC = 60
+
+      # check if we can report to TS
+      #
+      # @return[Boolean] true if bypass is enabled, or false if bypass disabled
+      def enabled?
+        @_enabled = Contrast::CONTRAST_SERVICE.use_agent_communication? if @_enabled.nil?
+        @_enabled
+      end
+
+      def connection
+        @_connection ||= client.initialize_connection
+      end
+
+      def start_thread!
+        return if running?
+
+        @_thread = Contrast::Agent::Thread.new do
+          logger.info('Starting heartbeat thread.')
+          loop do
+            Contrast::Agent.reporter&.send_event(poll_message)
+            sleep(REFRESH_INTERVAL_SEC)
+          end
+        end
+      end
+
+      def poll_message
+        @_poll_message ||= Contrast::Agent::Reporting::Poll.new
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/agent_startup.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/agent/reporting/reporting_events/server_reporting_event'
+require 'contrast/config'
+
+module Contrast
+  module Agent
+    module Reporting
+      #	AgentStartup Event which sends the agent data to TeamServer on the startup of a server or process,
+      # used to create a new Server entity there.
+      class AgentStartup < Contrast::Agent::Reporting::ServerReportingEvent
+        def initialize
+          @event_method = :PUT
+          @event_endpoint = Contrast::Agent::Reporting::Endpoints::NG_ENDPOINTS[:agent_startup]
+          @event_type = :agent_startup
+          super
+        end
+
+        def file_name
+          'agent-startup'
+        end
+
+        def to_controlled_hash
+          {
+              environment: ::Contrast::CONFIG.root.server.environment,
+              tags: ::Contrast::CONFIG.root.server.tags,
+              version: Contrast::Agent::VERSION
+          }
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/application_activity.rb

@@ -0,0 +1,51 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporting_events/application_reporting_event'
+require 'contrast/agent/reporting/reporting_events/application_defend_activity'
+require 'contrast/agent/reporting/reporting_events/application_inventory_activity'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This is the new ApplicationActivity class which will include all the needed information for the new reporting
+      # system to report
+      class ApplicationActivity < Contrast::Agent::Reporting::ApplicationReportingEvent
+        class << self
+          # @param app_activity_dtm [Contrast::Api::Dtm::Activity]
+          # @return [Contrast::Agent::Reporting::ApplicationActivity]
+          def convert app_activity_dtm
+            app_activity = new
+            app_activity.attach_data(app_activity_dtm)
+            app_activity
+          end
+        end
+
+        def initialize
+          @event_method = :PUT
+          @event_type = :application_activity
+          @event_endpoint = Contrast::Agent::Reporting::Endpoints.application_activity
+          super
+        end
+
+        def file_name
+          'activity-application'
+        end
+
+        def to_controlled_hash
+          hsh = { lastUpdate: since_last_update }
+          hsh[:defend] = @defend&.to_controlled_hash if @defend
+          hsh[:inventory] = @inventory&.to_controlled_hash if @inventory
+          hsh
+        end
+
+        # @param activity_dtm [Contrast::Api::Dtm::ApplicationActivity]
+        def attach_data activity_dtm
+          @defend = Contrast::Agent::Reporting::ApplicationDefendActivity.convert(activity_dtm)
+          @inventory = Contrast::Agent::Reporting::ApplicationInventoryActivity.convert(activity_dtm)
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/reporting/reporting_events/application_defend_activity.rb

@@ -0,0 +1,96 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/logger'
+require 'contrast/agent/reporting/reporting_events/application_defend_attacker_activity'
+
+module Contrast
+  module Agent
+    module Reporting
+      # This is the new ApplicationDefendActivity class which includes information about the defense of the application
+      # which was discovered during exercise of the application during this activity period.
+      class ApplicationDefendActivity
+        # @return [Array<Contrast::Agent::Reporting::ApplicationDefendAttackerActivity>]
+        attr_reader :attackers
+
+        class << self
+          # @param activity_dtm [Contrast::Api::Dtm::ApplicationActivity]
+          # @return [Contrast::Agent::Reporting::ApplicationDefendActivity]
+          def convert activity_dtm
+            activity = new
+            activity.attach_data(activity_dtm.results)
+            activity
+          end
+        end
+
+        def initialize
+          @attackers = []
+          @event_type = :application_defend_activity
+          super
+        end
+
+        def to_controlled_hash
+          {
+              attackers: attackers.map(&:to_controlled_hash)
+          }
+        end
+
+        # @param attack_dtms [Array<Contrast::Api::Dtm::AttackResult>]
+        def attach_data attack_dtms
+          attack_dtms.each do |attack_result|
+            attacker_activity = Contrast::Agent::Reporting::ApplicationDefendAttackerActivity.convert(attack_result)
+            existing_attacker_activity = attackers.find do |existing|
+              existing.source_forwarded_for == attacker_activity.source_forwarded_for &&
+                  existing.source_ip == attacker_activity.source_ip
+            end
+            rule = attack_result.rule_id
+            if existing_attacker_activity
+              attach_existing(existing_attacker_activity, attacker_activity, rule)
+            else
+              attackers << attacker_activity
+            end
+          end
+        end
+
+        # @param existing_attacker_activity [Contrast::Agent::Reporting::ApplicationDefendAttackerActivity]
+        # @param attacker_activity [Contrast::Agent::Reporting::ApplicationDefendAttackerActivity]
+        # @param rule [String]
+        def attach_existing existing_attacker_activity, attacker_activity, rule
+          # TODO: RUBY-1663 figure out attack time mapping
+          new_violation = attacker_activity.protection_rules[rule]
+          if (previously_violated = existing_attacker_activity.protection_rules[rule])
+            if (new_blocked = new_violation.blocked&.samples)
+              unless previously_violated.blocked
+                previously_violated.blocked = Contrast::Agent::Reporting::ApplicationDefendAttackSampleActivity.new
+              end
+              previously_violated.blocked.samples.concat(new_blocked)
+            end
+
+            if (new_exploited = new_violation.exploited&.samples)
+              unless previously_violated.exploited
+                previously_violated.exploited = Contrast::Agent::Reporting::ApplicationDefendAttackSampleActivity.new
+              end
+              previously_violated.exploited.samples.concat(new_exploited)
+            end
+
+            if (new_ineffective = new_violation.ineffective&.samples)
+              unless previously_violated.ineffective
+                previously_violated.ineffective = Contrast::Agent::Reporting::ApplicationDefendAttackSampleActivity.new
+              end
+              previously_violated.ineffective.samples.concat(new_ineffective)
+            end
+
+            if (new_suspicious = new_violation.suspicious&.samples)
+              unless previously_violated.suspicious
+                previously_violated.suspicious = Contrast::Agent::Reporting::ApplicationDefendAttackSampleActivity.new
+              end
+              previously_violated.suspicious.samples.concat(new_suspicious)
+            end
+          else
+            existing_attacker_activity.protection_rules[rule] = new_violation
+          end
+        end
+      end
+    end
+  end
+end