contrast-agent 4.9.1 → 4.13.0

data/lib/contrast/agent/telemetry.rb

@@ -0,0 +1,129 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/env_variables'
+require 'contrast/components/logger'
+require 'contrast/utils/requests_client'
+require 'contrast/agent/worker_thread'
+require 'contrast/utils/telemetry'
+
+module Contrast
+  module Agent
+    # This class will initialize and hold everything needed for the telemetry
+    class Telemetry < WorkerThread
+      include Contrast::Utils::RequestsClient
+      include Contrast::Components::Logger::InstanceMethods
+      # this is where we will send the data from the agents
+      URL = 'https://telemetry.ruby.contrastsecurity.com/'
+      # Suggested timeout after each send is to be 3 hours (10800 seconds)
+      SUGGESTED_TIMEOUT = 10_800
+
+      class << self
+        include Contrast::Components::Logger::InstanceMethods
+        include Contrast::Config::EnvVariables
+
+        def application_id
+          @_application_id ||= begin
+            id = nil
+            mac = Contrast::Utils::Telemetry::Identifier.mac
+            app_name = Contrast::Utils::Telemetry::Identifier.app_name
+            id = mac + app_name if mac && app_name
+            Digest::SHA2.new(256).hexdigest(id || '_' + SecureRandom.uuid)
+          end
+        end
+
+        def instance_id
+          @_instance_id ||= Digest::SHA2.new(256).hexdigest(Contrast::Utils::Telemetry::Identifier.mac ||
+                                                                  '_' + SecureRandom.uuid)
+        end
+
+        def enabled?
+          @_enabled = telemetry_enabled? if @_enabled.nil?
+          @_enabled
+        end
+
+        private
+
+        def telemetry_enabled?
+          opt_out_telemetry = return_value(:telemetry_opt_outs)
+          return false if opt_out_telemetry.to_s.casecmp('true').zero?
+          return false if opt_out_telemetry.to_s.casecmp('1').zero?
+
+          # In case of connection error, do not create the background thread or queue,
+          # as if the opt-out env var was set
+          ip_opt_out_telemetry = Contrast::Utils::RequestsClient.initialize_connection(URL)
+          if ip_opt_out_telemetry.nil?
+            logger.warn('Connection was not established properly!!!')
+            logger.warn('THE SERVICE IS GOING TO BE TERMINATED!!')
+            return false
+          end
+
+          true
+        end
+      end
+
+      def attempt_to_start?
+        unless cs__class.enabled?
+          logger.warn('Telemetry service is disabled!')
+          return false
+        end
+
+        logger.debug('Attempting to start telemetry thread') unless running?
+        true
+      end
+
+      def start_thread!
+        return if running?
+
+        # It is recommended that implementations send a single payload of
+        # general metrics every 3 hours, starting from implementation startup.
+        @_thread = Contrast::Agent::Thread.new do
+          logger.debug('Starting background telemetry thread.')
+          event = queue.pop
+
+          begin
+            logger.debug('This is the current processed event', event)
+            res = Contrast::Utils::RequestsClient.send_request event, connection
+            sleep_time = Contrast::Utils::RequestsClient.handle_response res
+            sleep(sleep_time) unless sleep_time.nil?
+          rescue StandardError => e
+            logger.error('Could not send message to service from telemetry queue.', e)
+          end
+          sleep(SUGGESTED_TIMEOUT)
+        end
+      end
+
+      def send_event event
+        if ::Contrast::AGENT.disabled?
+          logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
+          return
+        end
+
+        return unless cs__class.enabled?
+
+        logger.debug('Enqueued event for sending', event_type: event.cs__class)
+
+        queue << event if event
+      end
+
+      def delete_queue!
+        @_queue&.clear
+        @_queue&.close
+        @_queue = nil
+      end
+
+      def stop!
+        return unless running?
+
+        super
+        delete_queue!
+      end
+
+      private
+
+      def queue
+        @_queue ||= Queue.new
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry_event.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+
+module Contrast
+  module Agent
+    # This class will hold the basic information for a Telemetry Event
+    class TelemetryEvent
+      include Contrast::Utils
+
+      attr_reader :timestamp, :tags
+
+      # The instance_id comes from the Telemetry Instance
+      def initialize
+        @tags = MetricsHash.new(String)
+        @timestamp = format_date
+        @instance_id = Contrast::Agent.telemetry_queue.__id__
+      end
+
+      def path
+        ''
+      end
+
+      def to_json **_args
+        { tags: @tags, timestamp: @timestamp, instance_id: @instance_id }
+      end
+
+      def format_date
+        Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')
+      end
+    end
+  end
+end

data/lib/contrast/agent/thread_watcher.rb

@@ -4,6 +4,7 @@
 require 'contrast/components/logger'
 require 'contrast/agent/service_heartbeat'
 require 'contrast/api/communication/messaging_queue'
+require 'contrast/agent/telemetry'
 
 module Contrast
   module Agent
@@ -22,28 +23,22 @@ module Contrast
         @heapdump_util = Contrast::Utils::HeapDumpUtil.new
         @heartbeat = Contrast::Agent::ServiceHeartbeat.new
         @messaging_queue = Contrast::Api::Communication::MessagingQueue.new
+        @telemetry = Contrast::Agent::Telemetry.new if Contrast::Agent::Telemetry.enabled?
       end
 
       def startup!
         return unless ::Contrast::AGENT.enabled?
 
-        unless heartbeat.running?
-          logger.debug('Attempting to start heartbeat thread')
-          heartbeat.start_thread!
-        end
-        heartbeat_result = heartbeat.running?
-        logger.debug('Heartbeat thread status', alive: heartbeat_result)
-
-        unless messaging_queue.running?
-          logger.debug('Attempting to start messaging queue thread')
-          messaging_queue.start_thread!
-        end
-        messaging_result = messaging_queue.running?
-        logger.debug('Messaging thread status', alive: messaging_result)
+        telemetry_thread_status = telemetry_thread_init
+        heartbeat_thread_status = heartbeat_thread_init
+        messaging_thread_status = messaging_thread_init
 
         logger.debug('ThreadWatcher started threads')
 
-        @pids[Process.pid] = messaging_result && heartbeat_result
+        @pids[Process.pid] = messaging_thread_status && heartbeat_thread_status
+        return @pids unless Contrast::Agent::Telemetry.enabled?
+
+        @pids[Process.pid] = messaging_thread_status && heartbeat_thread_status && telemetry_thread_status
       end
 
       def ensure_running?
@@ -57,6 +52,40 @@ module Contrast
         heartbeat.stop!
         messaging_queue.stop!
         heapdump_util.stop!
+        telemetry_queue&.stop!
+      end
+
+      def heartbeat_thread_init
+        unless heartbeat.running?
+          logger.debug('Attempting to start heartbeat thread')
+          heartbeat.start_thread!
+        end
+        heartbeat_result = heartbeat.running?
+        logger.debug('Heartbeat thread status', alive: heartbeat_result)
+        heartbeat_result
+      end
+
+      def telemetry_thread_init
+        @telemetry.start_thread! if @telemetry&.attempt_to_start?
+        telemetry_result = @telemetry&.running?
+        logger.debug('Telemetry thread status', alive: telemetry_result)
+        telemetry_result
+      end
+
+      def messaging_thread_init
+        unless messaging_queue.running?
+          logger.debug('Attempting to start messaging queue thread')
+          messaging_queue.start_thread!
+        end
+        messaging_result = messaging_queue.running?
+        logger.debug('Messaging thread status', alive: messaging_result)
+        messaging_result
+      end
+
+      def telemetry_queue
+        return if @telemetry.nil?
+
+        @telemetry
       end
     end
   end

data/lib/contrast/agent/tracepoint_hook.rb

@@ -27,18 +27,31 @@ module Contrast
 
         private
 
+        # Use the TracePoint from the :end event, meaning the completion of a definition of a Class or Module (or
+        # really the completion of that piece of a definition, as determined by an `end` statement since there could be
+        # definitions across multiple files) to carry out actions required on definition. This typically involves
+        # patching and usage analysis
+        #
+        # @param tracepoint_event [TracePoint] the TracePoint from the :end
         def process tracepoint_event
           with_contrast_scope do
-            logger.trace('Received TracePoint end event', module: tracepoint_event.self.to_s)
-
+            # the Module or Class that was loaded during this event
             loaded_module = tracepoint_event.self
+            # the file being loaded that contained this definition
             path = tracepoint_event.path
             return if path&.include?('contrast')
 
+            logger.trace('Received TracePoint end event', module: loaded_module, path: path)
+
+            Contrast::Agent.framework_manager.register_late_framework(loaded_module)
             Contrast::Agent::Inventory::DependencyUsageAnalysis.instance.associate_file(path) if path
             Contrast::Agent::Patching::Policy::Patcher.patch_specific_module(loaded_module)
-            Contrast::Agent::Assess::Policy::RewriterPatch.rewrite_interpolation(loaded_module) if RUBY_VERSION < '2.6.0' # TODO: RUBY-714 remove guard w/ EOL of 2.5
+            if RUBY_VERSION < '2.6.0' # TODO: RUBY-714 remove guard w/ EOL of 2.5
+              Contrast::Agent::Assess::Policy::RewriterPatch.rewrite_interpolation(loaded_module)
+            end
             Contrast::Agent::Assess::Policy::PolicyScanner.scan(tracepoint_event)
+          rescue StandardError => e
+            logger.error('Unable to complete TracePoint analysis', e, module: loaded_module, path: path)
           end
         end
       end

data/lib/contrast/agent/version.rb

@@ -3,6 +3,6 @@
 
 module Contrast
   module Agent
-    VERSION = '4.9.1'
+    VERSION = '4.13.0'
   end
 end

data/lib/contrast/agent.rb

@@ -20,7 +20,6 @@ require 'contrast/extension/delegator'
 require 'contrast/extension/inventory'
 require 'contrast/extension/module'
 require 'contrast/extension/protect'
-require 'contrast/extension/protect/kernel'
 
 require 'contrast/utils/object_share'
 require 'contrast/utils/string_utils'
@@ -62,6 +61,12 @@ module Contrast
       thread_watcher.messaging_queue
     end
 
+    def self.telemetry_queue
+      return unless thread_watcher.telemetry_queue
+
+      thread_watcher.telemetry_queue
+    end
+
     def self.thread_watcher
       @_thread_watcher ||= Contrast::Agent::ThreadWatcher.new
     end

data/lib/contrast/api/communication/messaging_queue.rb

@@ -11,10 +11,9 @@ module Contrast
       class MessagingQueue < Contrast::Agent::WorkerThread
         include Contrast::Components::Logger::InstanceMethods
 
-        attr_reader :queue, :speedracer
+        attr_reader :speedracer
 
         def initialize
-          @queue = Queue.new
           @speedracer = Contrast::Api::Communication::Speedracer.new
           super
         end
@@ -28,6 +27,10 @@ module Contrast
           speedracer.return_response(event)
         end
 
+        def queue
+          @_queue ||= Queue.new
+        end
+
         # Use this to add a message to the queue and process the response internally
         def send_event_eventually event
           if ::Contrast::AGENT.disabled?
@@ -42,7 +45,6 @@ module Contrast
           speedracer.ensure_startup!
           return if running?
 
-          @queue ||= Queue.new
           @_thread = Contrast::Agent::Thread.new do
             loop do
               event = queue.pop
@@ -58,13 +60,17 @@ module Contrast
           logger.debug('Started background sending thread.')
         end
 
+        def delete_queue!
+          @_queue&.clear
+          @_queue&.close
+          @_queue = nil
+        end
+
         def stop!
           return unless running?
 
           super
-          @queue&.clear
-          @queue&.close
-          @queue = nil
+          delete_queue!
         end
       end
     end

data/lib/contrast/api/communication/service_lifecycle.rb

@@ -6,10 +6,13 @@ require 'contrast/components/logger'
 module Contrast
   module Api
     module Communication
-      # Handles local service startup
+      # Handles local service startup. As this should only ever be invoked by the Speedracer class, which includes
+      # this, all methods here are private.
       module ServiceLifecycle
         include Contrast::Components::Logger::InstanceMethods
 
+        private
+
         def attempt_local_service_startup
           zombie_check
           service_starter_thread.join(5)

data/lib/contrast/api/communication/socket_client.rb

@@ -37,8 +37,7 @@ module Contrast
           log_connection
           if ::Contrast::CONTRAST_SERVICE.use_tcp?
             Contrast::Api::Communication::TcpSocket.new(
-              ::Contrast::CONTRAST_SERVICE.host, ::Contrast::CONTRAST_SERVICE.port
-            )
+                ::Contrast::CONTRAST_SERVICE.host, ::Contrast::CONTRAST_SERVICE.port)
           else
             Contrast::Api::Communication::UnixSocket.new(::Contrast::CONTRAST_SERVICE.socket_path)
           end
@@ -61,8 +60,9 @@ module Contrast
 
           # Or something is not set.
           logger.warn(
-            log_connection_error_msg, host: ::Contrast::CONTRAST_SERVICE.host, port: ::Contrast::CONTRAST_SERVICE.port
-          )
+              log_connection_error_msg,
+              host: ::Contrast::CONTRAST_SERVICE.host,
+              port: ::Contrast::CONTRAST_SERVICE.port)
         end
 
         # If our connection isn't built properly, we need to warn the user. This builds out the context specific

data/lib/contrast/api/decorators/agent_startup.rb

@@ -41,11 +41,11 @@ module Contrast
           #
           # @param msg [Contrast::Api::Dtm::AgentStartup]
           def config! msg
-            msg.version          = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.version
-            msg.environment      = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.environment
-            msg.server_tags      = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.tags
+            msg.version      = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.version
+            msg.server_tags  = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.tags
+            msg.library_tags = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.inventory.tags
+            msg.environment  = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.server.environment
             msg.application_tags = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.tags
-            msg.library_tags     = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.inventory.tags
           end
         end
       end

data/lib/contrast/api/decorators/application_startup.rb

@@ -24,11 +24,12 @@ module Contrast
           # @return [Contrast::Api::Dtm::ApplicationCreate]
           def build
             msg = new
-            msg.app_version = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.version.to_s
-            msg.code        = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.code
-            msg.group       = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.group
-            msg.metadata    = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.metadata
-            msg.mode        = Contrast::Api::Dtm::InstrumentationMode.build
+            msg.code     = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.code
+            msg.group    = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.group
+            msg.metadata = Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.metadata
+            msg.mode     = Contrast::Api::Dtm::InstrumentationMode.build
+            msg.app_version =
+                Contrast::Utils::StringUtils.protobuf_format ::Contrast::CONFIG.root.application.version.to_s # rubocop:disable Layout/AssignmentIndentation Layout/FirstArgumentIndentation:
             session!(msg)
             msg
           end

data/lib/contrast/api/decorators/route_coverage.rb

@@ -46,7 +46,7 @@ module Contrast
           #
           # @param controller [::Sinatra::Base] the route's final controller.
           # @param method [String] GET, PUT, POST, etc...
-          # @param method [::Mustermann::Sinatra]  the pattern that was matched in routing.
+          # @param pattern [::Mustermann::Sinatra]  the pattern that was matched in routing.
           # @param url [String, nil] use url from string instead matched pattern.
           # @return [Contrast::Api::Dtm::RouteCoverage]
           def from_sinatra_route controller, method, pattern, url = nil
@@ -59,6 +59,29 @@ module Contrast
             msg.url = Contrast::Utils::StringUtils.force_utf8(safe_url)
             msg
           end
+
+          # Convert Grape route data to dtm message.
+          #
+          # @param controller [::Grape::API] the route's final controller.
+          # @param method [String] GET, PUT, POST, etc...
+          # @param url [String, nil] use url from string instead matched pattern.
+          # @param pattern [String, Grape::Router::Route] the pattern that was matched in routing.
+          # @return [Contrast::Api::Dtm::RouteCoverage]
+          def from_grape_controller controller, method, pattern, url = nil
+            if pattern.cs__is_a?(Grape::Router::Route)
+              safe_pattern = pattern.pattern&.path&.to_s
+              safe_url = source_or_string(url || safe_pattern)
+            else
+              safe_pattern = source_or_string(pattern)
+              safe_url = source_or_string(url || pattern)
+            end
+
+            msg = new
+            msg.route = "#{ controller }##{ method } #{ safe_pattern }"
+            msg.verb = Contrast::Utils::StringUtils.force_utf8(method)
+            msg.url = Contrast::Utils::StringUtils.force_utf8(safe_url)
+            msg
+          end
         end
       end
     end

data/lib/contrast/components/agent.rb

@@ -54,7 +54,9 @@ module Contrast
         end
 
         def interpolation_enabled?
-          @_interpolation_enabled = !false?(::Contrast::CONFIG.root.agent.ruby.interpolate) if @_interpolation_enabled.nil?
+          if @_interpolation_enabled.nil?
+            @_interpolation_enabled = !false?(::Contrast::CONFIG.root.agent.ruby.interpolate)
+          end
           @_interpolation_enabled
         end
 
@@ -69,7 +71,8 @@ module Contrast
               status:
                 ::Contrast::CONFIG.root.agent.ruby.exceptions.override_status || 403,
               message:
-                ::Contrast::CONFIG.root.agent.ruby.exceptions.override_message || Contrast::Utils::ObjectShare::OVERRIDE_MESSAGE
+                ::Contrast::CONFIG.root.agent.ruby.exceptions.override_message ||
+                    Contrast::Utils::ObjectShare::OVERRIDE_MESSAGE
           }
         end
 

data/lib/contrast/components/api.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/base'
+require 'contrast/components/config'
+
+module Contrast
+  module Components
+    module Api
+      # A wrapper build around the Common Agent Configuration project to allow
+      # for Api keys to be mapped with their values contained in their
+      # parent_configuration_spec.yaml.
+      class Interface
+        include Contrast::Components::ComponentBase
+
+        def api_url
+          @_api_url ||= ::Contrast::CONFIG.root.api.url
+        end
+
+        def api_key
+          @_api_key ||= ::Contrast::CONFIG.root.api.api_key
+        end
+
+        def service_key
+          @_service_key ||= ::Contrast::CONFIG.root.api.service_key
+        end
+
+        def username
+          @_username ||= ::Contrast::CONFIG.root.api.user_name
+        end
+      end
+    end
+  end
+end

data/lib/contrast/components/app_context.rb

@@ -23,6 +23,10 @@ module Contrast
         DEFAULT_SERVER_NAME = 'localhost'
         DEFAULT_SERVER_PATH = '/'
 
+        SUPPORTED_FRAMEWORKS = %w[rails sinatra grape rack].cs__freeze
+
+        SUPPORTED_SERVERS = %w[passenger puma thin unicorn].cs__freeze
+
         def initialize
           original_pid
         end
@@ -109,6 +113,26 @@ module Contrast
           @_client_id ||= [app_name, pgid].join('-')
         end
 
+        def app_and_server_information
+          {
+              application_info: find_gem_information(SUPPORTED_FRAMEWORKS),
+              server_info: find_gem_information(SUPPORTED_SERVERS)
+          }
+        end
+
+        def find_gem_information arr
+          arr.each do |framework|
+            next unless Gem.loaded_specs.key?(framework)
+
+            loaded = Gem.loaded_specs[framework]
+            next unless loaded
+
+            name = loaded.instance_variable_get(:@name)
+            version = loaded.instance_variable_get(:@version).to_s
+            return [name, version].join(' ')
+          end
+        end
+
         def instrument_middleware_stack?
           !Contrast::Utils::JobServersRunning.job_servers_running?
         end

data/lib/contrast/components/assess.rb

@@ -5,7 +5,6 @@ require 'contrast/components/base'
 require 'contrast/components/config'
 require 'contrast/components/settings'
 
-
 module Contrast
   module Components
     module Assess
@@ -78,7 +77,9 @@ module Contrast
         end
 
         def track_frozen_sources?
-          @_track_frozen_sources = !false?(::Contrast::CONFIG.root.agent.ruby.track_frozen_sources) if @_track_frozen_sources.nil?
+          if @_track_frozen_sources.nil?
+            @_track_frozen_sources = !false?(::Contrast::CONFIG.root.agent.ruby.track_frozen_sources)
+          end
           @_track_frozen_sources
         end
 
@@ -87,13 +88,22 @@ module Contrast
           @_require_scan
         end
 
+        def require_dynamic_sources?
+          if @_require_dynamic_sources.nil?
+            @_require_dynamic_sources = !false?(::Contrast::CONFIG.root.assess.enable_dynamic_sources)
+          end
+          @_require_dynamic_sources
+        end
+
         def tags
           ::Contrast::CONFIG.root.assess&.tags
         end
 
         def disabled_rules
           # TODO: RUBY-903
-          ::Contrast::CONFIG.root.assess&.rules&.disabled_rules || ::Contrast::SETTINGS.assess_state.disabled_assess_rules || []
+          ::Contrast::CONFIG.root.assess&.rules&.disabled_rules ||
+              ::Contrast::SETTINGS.assess_state.disabled_assess_rules ||
+              []
         end
 
         private

data/lib/contrast/components/base.rb

@@ -18,7 +18,7 @@ module Contrast
         return true if config_param == false
         return false unless config_param.cs__is_a?(String)
 
-        Contrast::Utils::ObjectShare::FALSE.casecmp?(config_param)
+        config_param.downcase == Contrast::Utils::ObjectShare::FALSE
       end
 
       # use this to determine if the configuration value is literally boolean
@@ -33,7 +33,7 @@ module Contrast
         return true if config_param == true
         return false unless config_param.cs__is_a?(String)
 
-        Contrast::Utils::ObjectShare::TRUE.casecmp?(config_param)
+        config_param.downcase == Contrast::Utils::ObjectShare::TRUE
       end
     end
   end