contrast-agent 4.11.0 → 4.14.0

data/lib/contrast/agent/startup_metrics_telemetry_event.rb

@@ -0,0 +1,94 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+require 'contrast/agent/metric_telemetry_event'
+require 'contrast/agent/version'
+require 'contrast/utils/os'
+
+module Contrast
+  module Agent
+    # This class will hold the Startup Metrics Telemetry Event
+    # The class will include initialization of the agent version, language version
+    # os type, arch and version
+    # application framework and version and server framework
+    # It will be initialized and send in Middleware#agent_startup_routine
+    class StartupMetricsTelemetryEvent < Contrast::Agent::MetricTelemetryEvent
+      include Contrast::Utils::OS
+
+      APP_AND_SERVER_DATA = ::Contrast::APP_CONTEXT.app_and_server_information.cs__freeze
+      # Multi-tenant Production Environments
+      SAAS_DEFAULT = { addr: 'app.contrastsecurity.com', type: 'SAAS_DEFAULT' }.cs__freeze
+      SAAS_CS = { addr: /cs[[:digit:]]+\.contrastsecurity\.com/, type: 'SAAS_DEFAULT' }.cs__freeze
+      SAAS_JP = { addr: 'app.contrastsecurity.jp', type: 'SAAS_DEFAULT' }.cs__freeze
+      SAAS_CE = { addr: 'ce.contrastsecurity.com', type: 'SAAS_CE' }.cs__freeze
+      # Multi-tenant Demo Environments
+      SAAS_DEMO = { addr: 'apptwo.contrastsecurity.com', type: 'SAAS_DEMO' }.cs__freeze
+      SAAS_POV = { addr: 'eval.contrastsecurity.com', type: 'SAAS_POV' }.cs__freeze
+      # Multi-tenant Testing Environment
+      SAAS_RESEARCH = { addr: 'security-research.contrastsecurity.com', type: 'SAAS_RESEARCH' }.cs__freeze
+      SAAS_ALPHA = { addr: 'alpha.contrastsecurity.com', type: 'SAAS_ALPHA' }.cs__freeze
+      SAAS_STAGING = { addr: 'teamserver-staging.contsec.com', type: 'SAAS_TESTING' }.cs__freeze
+      SAAS_STAGING_TOKYO = { addr: 'teamserver-staging.contsec.jp', type: 'SAAS_TESTING' }.cs__freeze
+      SAAS_TESTING = { addr: 'teamserver-darpa.contsec.com', type: 'SAAS_TESTING' }.cs__freeze
+      SAAS_OPS_TESTING = { addr: 'teamserver-ops.contsec.com', type: 'SAAS_TESTING' }.cs__freeze
+      # Fallback for Single-tenant Production Environments
+      SAAS_CUSTOM = { addr: 'contrastsecurity.com', type: 'SAAS_CUSTOM' }.cs__freeze
+      SAAS_CUSTOM_JP = { addr: 'contrastsecurity.jp', type: 'SAAS_CUSTOM' }.cs__freeze
+
+      SINGLE_MAP_TENANTS = [
+        SAAS_DEFAULT, SAAS_JP, SAAS_CE, SAAS_DEMO, SAAS_POV, SAAS_RESEARCH, SAAS_ALPHA,
+        SAAS_STAGING, SAAS_STAGING_TOKYO, SAAS_TESTING, SAAS_OPS_TESTING
+      ].cs__freeze
+      REGEXP_MAP_TENANTS = [SAAS_CS].cs__freeze
+      FALLBACK_TENANTS = [SAAS_CUSTOM, SAAS_CUSTOM_JP].cs__freeze
+      # Fallback for Custom, most likely self-hosted, Environments
+      EOP = 'EOP'
+
+      def initialize
+        super
+        add_tags
+      end
+
+      def path
+        '/startup'
+      end
+
+      def add_tags
+        @tags['teamserver'] = teamserver_type
+        @tags['agent_version'] = VERSION
+        @tags['ruby_version'] = RUBY_VERSION
+        @tags['os_type'] = sys_info['os_type'] == 'Darwin' ? 'MacOS' : 'Linux'
+        @tags['os_arch'] = sys_info['os_arch']
+        @tags['os_version'] = sys_info['os_version']
+        @tags['app_framework_and_version'] = APP_AND_SERVER_DATA[:application_info].to_s
+        @tags['server_framework_and_version'] = APP_AND_SERVER_DATA[:server_info].to_s
+      end
+
+      def sys_info
+        @sys_info ||= get_system_information if @sys_info.nil?
+        @sys_info
+      end
+
+      private
+
+      # Here we extract the TeamServer url type
+      #
+      # @return [String] the type of TeamServer environment to which we're connecting
+      def teamserver_type
+        @_teamserver_type ||= begin
+          url = Contrast::API.api_url
+          if (single = SINGLE_MAP_TENANTS.find { |tenant| url.include?(tenant[:addr]) })
+            single[:type]
+          elsif (regexp = REGEXP_MAP_TENANTS.find { |tenant| tenant[:addr].match?(url) })
+            regexp[:type]
+          elsif (fallback = FALLBACK_TENANTS.find { |tenant| url.include?(tenant[:addr]) })
+            fallback[:type]
+          else
+            EOP
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/static_analysis.rb

@@ -10,12 +10,12 @@ module Contrast
     # this module handles one time static analysis tasks
     class StaticAnalysis
       include Singleton
-      include Contrast::Components::Logger::InstanceMethods
       include Contrast::Components::Scope::InstanceMethods
+      extend Contrast::Components::Logger::InstanceMethods
 
       class << self
-        # After the first request is complete, we do a one-time manual catchup to review and
-        # report the already-loaded gems.
+        # After the first request is complete, we do a one-time manual catchup to review and report the already-loaded
+        # gems.
         def catchup
           @_catchup ||= begin
             threaded_analysis!
@@ -23,6 +23,8 @@ module Contrast
           end
         end
 
+        # TODO: RUBY-1354
+        # TODO: RUBY-1356
         def send_inventory_message
           return unless ::Contrast::INVENTORY.enabled?
 

data/lib/contrast/agent/telemetry.rb

@@ -0,0 +1,137 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/env_variables'
+require 'contrast/components/logger'
+require 'contrast/utils/telemetry_client'
+require 'contrast/agent/worker_thread'
+require 'contrast/utils/telemetry'
+
+module Contrast
+  module Agent
+    # This class will initialize and hold everything needed for the telemetry
+    class Telemetry < WorkerThread
+      include Contrast::Components::Logger::InstanceMethods
+      # this is where we will send the data from the agents
+      URL = 'https://telemetry.ruby.contrastsecurity.com/'
+      # Suggested timeout after each send is to be 3 hours (10800 seconds)
+      SUGGESTED_TIMEOUT = 10_800
+
+      class << self
+        include Contrast::Components::Logger::InstanceMethods
+        include Contrast::Config::EnvVariables
+
+        def application_id
+          @_application_id ||= begin
+            id = nil
+            mac = Contrast::Utils::Telemetry::Identifier.mac
+            app_name = Contrast::Utils::Telemetry::Identifier.app_name
+            id = mac + app_name if mac && app_name
+            Digest::SHA2.new(256).hexdigest(id || ('_' + SecureRandom.uuid))
+          end
+        end
+
+        def instance_id
+          @_instance_id ||= Digest::SHA2.new(256).hexdigest(Contrast::Utils::Telemetry::Identifier.mac ||
+                                                                  ('_' + SecureRandom.uuid))
+        end
+
+        def enabled?
+          @_enabled = telemetry_enabled? if @_enabled.nil?
+          @_enabled
+        end
+
+        private
+
+        def telemetry_enabled?
+          opt_out_telemetry = return_value(:telemetry_opt_outs)
+          return false if opt_out_telemetry.to_s.casecmp('true').zero?
+          return false if opt_out_telemetry.to_s.casecmp('1').zero?
+
+          # In case of connection error, do not create the background thread or queue,
+          # as if the opt-out env var was set
+          @_client = Contrast::Utils::TelemetryClient.new
+          ip_opt_out_telemetry = @_client.initialize_connection(URL)
+          if ip_opt_out_telemetry.nil?
+            logger.warn('Connection was not established properly!!!')
+            logger.warn('THE SERVICE IS GOING TO BE TERMINATED!!')
+            return false
+          end
+
+          true
+        end
+      end
+
+      def client
+        @_client ||= Contrast::Utils::TelemetryClient.new
+      end
+
+      def connection
+        @_connection ||= client.initialize_connection(URL)
+      end
+
+      def attempt_to_start?
+        unless cs__class.enabled?
+          logger.warn('Telemetry service is disabled!')
+          return false
+        end
+
+        logger.debug('Attempting to start telemetry thread') unless running?
+        true
+      end
+
+      def start_thread!
+        return if running?
+
+        # It is recommended that implementations send a single payload of
+        # general metrics every 3 hours, starting from implementation startup.
+        @_thread = Contrast::Agent::Thread.new do
+          logger.debug('Starting background telemetry thread.')
+          event = queue.pop
+
+          begin
+            logger.debug('This is the current processed event', event)
+            res = client.send_request event, connection
+            sleep_time = client.handle_response res
+            sleep(sleep_time) unless sleep_time.nil?
+          rescue StandardError => e
+            logger.error('Could not send message to service from telemetry queue.', e)
+          end
+          sleep(SUGGESTED_TIMEOUT)
+        end
+      end
+
+      def send_event event
+        if ::Contrast::AGENT.disabled?
+          logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
+          return
+        end
+
+        return unless cs__class.enabled?
+
+        logger.debug('Enqueued event for sending', event_type: event.cs__class)
+
+        queue << event if event
+      end
+
+      def delete_queue!
+        @_queue&.clear
+        @_queue&.close
+        @_queue = nil
+      end
+
+      def stop!
+        return unless running?
+
+        super
+        delete_queue!
+      end
+
+      private
+
+      def queue
+        @_queue ||= Queue.new
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry_event.rb

@@ -0,0 +1,33 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+
+module Contrast
+  module Agent
+    # This class will hold the basic information for a Telemetry Event
+    class TelemetryEvent
+      include Contrast::Utils
+
+      attr_reader :timestamp, :tags
+
+      def initialize
+        @tags = MetricsHash.new(String)
+        @timestamp = Time.now.iso8601
+      end
+
+      def path
+        ''
+      end
+
+      def to_hash **_args
+        {
+            tags: @tags,
+            timestamp: @timestamp,
+            instance: Contrast::Agent::Telemetry.instance_id,
+            application: Contrast::Agent::Telemetry.application_id
+        }
+      end
+    end
+  end
+end

data/lib/contrast/agent/thread_watcher.rb

@@ -4,6 +4,8 @@
 require 'contrast/components/logger'
 require 'contrast/agent/service_heartbeat'
 require 'contrast/api/communication/messaging_queue'
+require 'contrast/agent/reporting/report'
+require 'contrast/agent/telemetry'
 
 module Contrast
   module Agent
@@ -22,41 +24,94 @@ module Contrast
         @heapdump_util = Contrast::Utils::HeapDumpUtil.new
         @heartbeat = Contrast::Agent::ServiceHeartbeat.new
         @messaging_queue = Contrast::Api::Communication::MessagingQueue.new
+        @telemetry = Contrast::Agent::Telemetry.new if Contrast::Agent::Telemetry.enabled?
+        @reporter = Contrast::Agent::Reporter.new if Contrast::Agent::Reporter.enabled?
       end
 
       def startup!
         return unless ::Contrast::AGENT.enabled?
 
+        telemetry_status = telemetry_thread_init
+        heartbeat_status = heartbeat_thread_init
+        messaging_status = messaging_thread_init
+        reporter_status = reporter_thread_init
+
+        logger.debug('ThreadWatcher started threads')
+
+        @pids[Process.pid] = messaging_status && heartbeat_status
+        return @pids unless Contrast::Agent::Telemetry.enabled?
+
+        @pids[Process.pid] = @pids[Process.pid] && telemetry_status
+        return @pids unless Contrast::Agent::Reporter.enabled?
+
+        @pids[Process.pid] = @pids[Process.pid] && reporter_status
+      end
+
+      def ensure_running?
+        return if @pids[Process.pid] == true
+
+        logger.debug('ThreadWatcher - threads not running')
+        startup!
+      end
+
+      def shutdown!
+        heartbeat.stop!
+        messaging_queue.stop!
+        heapdump_util.stop!
+        telemetry_queue&.stop!
+        reporter_queue&.stop!
+      end
+
+      def heartbeat_thread_init
         unless heartbeat.running?
           logger.debug('Attempting to start heartbeat thread')
           heartbeat.start_thread!
         end
         heartbeat_result = heartbeat.running?
         logger.debug('Heartbeat thread status', alive: heartbeat_result)
+        heartbeat_result
+      end
+
+      def telemetry_thread_init
+        @telemetry.start_thread! if @telemetry&.attempt_to_start?
+        telemetry_result = @telemetry&.running?
+        logger.debug('Telemetry thread status', alive: telemetry_result)
+        telemetry_result
+      end
 
+      def messaging_thread_init
         unless messaging_queue.running?
           logger.debug('Attempting to start messaging queue thread')
           messaging_queue.start_thread!
         end
         messaging_result = messaging_queue.running?
         logger.debug('Messaging thread status', alive: messaging_result)
+        messaging_result
+      end
 
-        logger.debug('ThreadWatcher started threads')
-
-        @pids[Process.pid] = messaging_result && heartbeat_result
+      def reporter_thread_init
+        @reporter.start_thread! if @reporter&.attempt_to_start?
+        unless @reporter&.running?
+          logger.debug('Attempting to start reporter thread')
+          @reporter&.start_thread!
+        end
+        reporter_result = @reporter&.running?
+        logger.debug('Reporter thread status', alive: reporter_result)
+        reporter_result
       end
 
-      def ensure_running?
-        return if @pids[Process.pid] == true
+      # @return [Contrast::Agent::Telemetry]
+      def telemetry_queue
+        return if @telemetry.nil?
 
-        logger.debug('ThreadWatcher - threads not running')
-        startup!
+        @telemetry
       end
 
-      def shutdown!
-        heartbeat.stop!
-        messaging_queue.stop!
-        heapdump_util.stop!
+      # @return [Contrast::Agent::Reporter]
+      def reporter_queue
+        return if @reporter.nil?
+
+        @reporter
       end
     end
   end

data/lib/contrast/agent/version.rb

@@ -3,6 +3,6 @@
 
 module Contrast
   module Agent
-    VERSION = '4.11.0'
+    VERSION = '4.14.0'
   end
 end

data/lib/contrast/agent.rb

@@ -20,7 +20,6 @@ require 'contrast/extension/delegator'
 require 'contrast/extension/inventory'
 require 'contrast/extension/module'
 require 'contrast/extension/protect'
-require 'contrast/extension/protect/kernel'
 
 require 'contrast/utils/object_share'
 require 'contrast/utils/string_utils'
@@ -30,6 +29,9 @@ require 'contrast/utils/os'
 require 'contrast/utils/hash_digest'
 require 'contrast/utils/invalid_configuration_util'
 
+# Collect findings
+require 'contrast/utils/findings'
+
 # scoping
 require 'contrast/agent/scope'
 
@@ -49,19 +51,37 @@ module Contrast
   module Agent
     # build a map for tracking the context of the current request
     REQUEST_TRACKER = Contrast::Utils::ThreadTracker.new
+    FINDINGS = Contrast::Utils::Findings.new
 
+    # @return [Contrast::Framework::Manager]
     def self.framework_manager
       @_framework_manager ||= Contrast::Framework::Manager.new
     end
 
+    # @return [nil, Contrast::Utils::HeapDumpUtil]
     def self.heapdump_util
       thread_watcher.heapdump_util
     end
 
+    # @return [nil, Contrast::Api::Communication::MessagingQueue]
     def self.messaging_queue
       thread_watcher.messaging_queue
     end
 
+    # @return [nil, Contrast::Agent::Telemetry]
+    def self.telemetry_queue
+      return unless thread_watcher.telemetry_queue
+
+      thread_watcher.telemetry_queue
+    end
+
+    # @return [nil, Contrast::Agent::Reporter]
+    def self.reporter_queue
+      return unless  thread_watcher.reporter_queue
+
+      thread_watcher.reporter_queue
+    end
+
     def self.thread_watcher
       @_thread_watcher ||= Contrast::Agent::ThreadWatcher.new
     end

data/lib/contrast/api/communication/connection_status.rb

@@ -4,7 +4,7 @@
 module Contrast
   module Api
     module Communication
-      # Keeps track of the state of connections to the service.
+      # Keeps track of the state of connections to SpeedRacer.
       class ConnectionStatus
         def initialize
           @last_success = nil
@@ -12,25 +12,28 @@ module Contrast
           @startup_messages_sent = false
         end
 
-        # Whether we have sent startup message to the service. True after successfully
-        # sending startup messages to service and reset to false if we lose connection
-        # to the service.
+        # Whether we have sent startup message to SpeedRacer. True after successfully sending startup messages to
+        # SpeedRacer and reset to false if we lose connection.
+        #
+        # @return [Boolean]
         def startup_messages_sent?
           @startup_messages_sent
         end
 
-        # The last message sent was successful
+        # The last message sent was successful or not
+        #
+        # @return [Boolean]
         def connected?
           @last_success && (@last_failure.nil? || @last_success > @last_failure)
         end
 
-        # The current state of the service is active with a successful message sent
+        # The current state of the SpeedRacer is active with a successful message sent
         def success!
           @startup_messages_sent = true
           @last_success = Time.now.to_f
         end
 
-        # The service may be in some sort of error state
+        # The SpeedRacer may be in some sort of error state
         def failure!
           @startup_messages_sent = false
           @last_failure = Time.now.to_f

data/lib/contrast/api/communication/messaging_queue.rb

@@ -3,6 +3,7 @@
 
 require 'contrast/components/logger'
 require 'contrast/agent/worker_thread'
+require 'contrast/agent/reporting/reporting_utilities/audit'
 
 module Contrast
   module Api
@@ -15,23 +16,46 @@ module Contrast
 
         def initialize
           @speedracer = Contrast::Api::Communication::Speedracer.new
+          @audit = Contrast::Agent::Reporting::Audit.new if ::Contrast::API.request_audit_enable
           super
         end
 
-        # Use this to bypass the messaging queue and leave response processing to the caller
+        # Use this to bypass the messaging queue and leave response processing to the caller. We use this method for
+        # those operations which are blocking, meaning they must complete for the Agent to continue operations. These
+        # types of events include initial settings/ setup on startup and analysis required to complete before handing
+        # execution from our pre-filter operations to the application code (i.e. Protect's input analysis).
+        #
+        # @param event [Contrast::Api::Dtm] One of the DTMs valid for the event field of
+        #   Contrast::Api::Dtm::Message|Contrast::Api::Dtm::Activity
+        # @return [Contrast::Api::Settings::AgentSettings,nil]
         def send_event_immediately event
           if ::Contrast::AGENT.disabled?
             logger.warn('Attempted to send event immediately with Agent disabled', caller: caller, event: event)
             return
           end
-          speedracer.return_response(event)
+          response_data = speedracer.return_response(event)
+          return response_data unless ::Contrast::API.request_audit_enable
+
+          @audit&.audit_event(event, response_data)
+          response_data
         end
 
+        # A simple Queue used to hold messages that are ready to be sent to SpeedRacer but for which we do not need an
+        # immediate response
+        #
+        # @return [Queue]
         def queue
           @_queue ||= Queue.new
         end
 
-        # Use this to add a message to the queue and process the response internally
+        # Use this to add a message to the queue and process the response internally. We use this method for those
+        # operations which are non-blocking, meaning they don't need to complete for the Agent to continue operations.
+        # These types of events include any post-request messaging that occurs after execution is returned from the
+        # application to our post-filter operations as well as those that don't alter the application's execution (i.e.
+        # Assess' vulnerability reporting).
+        #
+        # @param event [Contrast::Api::Dtm] One of the DTMs valid for the event field of
+        #   Contrast::Api::Dtm::Message|Contrast::Api::Dtm::Activity
         def send_event_eventually event
           if ::Contrast::AGENT.disabled?
             logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
@@ -39,8 +63,14 @@ module Contrast
           end
           logger.debug('Enqueued event for sending', event_type: event.cs__class)
           queue << event if event
+          return unless ::Contrast::API.request_audit_enable
+
+          @audit&.audit_event(event)
         end
 
+        # Create the reporting thread, which will pull from the queue in order to send messages to SpeedRacer. If
+        # SpeedRacer is not running and should be, meaning the Agent is configured to control it, then we will also
+        # try to start that process.
         def start_thread!
           speedracer.ensure_startup!
           return if running?
@@ -60,12 +90,16 @@ module Contrast
           logger.debug('Started background sending thread.')
         end
 
+        # When the Agent shuts down, we terminate the message sending operations. This method clears, closes, and
+        # destroys the queue.
         def delete_queue!
           @_queue&.clear
           @_queue&.close
           @_queue = nil
         end
 
+        # When the Agent shuts down, we terminate the thread responsible for reporting. This method destroys that
+        # thread and any data we would have sent with it.
         def stop!
           return unless running?
 

data/lib/contrast/api/communication/response_processor.rb

@@ -7,10 +7,13 @@ require 'contrast/components/logger'
 module Contrast
   module Api
     module Communication
-      # Handles processing deferred messages
+      # Handles processing deferred messages sent to SpeedRacer.
       class ResponseProcessor
         include Contrast::Components::Logger::InstanceMethods
 
+        # Use the given response to update the Agent's server features and application settings, allowing it to reflect
+        # the latest options configured by the user in TeamServer
+        #
         # @param response [Contrast::Api::Settings::AgentSettings]
         def process response
           logger.debug('Received a response', sent_ms: response&.sent_ms)
@@ -18,8 +21,8 @@ module Contrast
           server_features = process_server_response(response)
           app_settings = process_application_response(response)
 
-          # ReactionProcessor is a design pattern from TeamServer.
-          # Right now, there's one potential reaction, which is disabling the agent
+          # ReactionProcessor is a design pattern from TeamServer. Right now, there's one potential reaction, which is
+          # disabling the agent
           Contrast::Agent::ReactionProcessor.process(response&.application_settings)
 
           Contrast::Logger::Log.instance.update(server_features&.log_file, server_features&.log_level)
@@ -30,10 +33,10 @@ module Contrast
 
         private
 
-        # Given some protobuf messages, update server features.
-        # This is the bridge between Contrast Service <-> Settings.
+        # Given some protobuf messages, update server features. This is the bridge between SpeedRacer <-> Settings.
         #
         # @param response [Contrast::Api::Settings::AgentSettings]
+        # @return [Contrast::Api::Settings::ServerFeatures]
         def process_server_response response
           server_features = response&.server_features
           return unless server_features
@@ -45,10 +48,11 @@ module Contrast
           server_features
         end
 
-        # Given some protobuf messages, update application settings.
-        # This is the bridge between Contrast Service <-> Settings.
+        # Given some protobuf messages, update application settings. This is the bridge between SpeedRacer <->
+        # Settings.
         #
         # @param response [Contrast::Api::Settings::AgentSettings]
+        # @return [Contrast::Api::Settings::ApplicationSettings]
         def process_application_response response
           app_settings = response&.application_settings
           return unless app_settings
@@ -61,7 +65,10 @@ module Contrast
         end
 
         # This can't go in the Settings component because protect and assess depend on settings
-        # I don't think it should go into contrast_service because that only handles connection specific data
+        # I don't think it should go into contrast_service because that only handles connection specific data.
+        #
+        # @param server_features [Contrast::Api::Settings::AgentSettings]
+        # @param app_settings [Contrast::Api::Settings::ApplicationSettings]
         def update_features server_features, app_settings
           return unless !!(server_features || app_settings)
           return unless ::Contrast::AGENT.enabled?