contrast-agent 4.12.0 → 4.13.0

data/lib/contrast/agent/startup_metrics_telemetry_event.rb

@@ -0,0 +1,71 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+require 'contrast/agent/metric_telemetry_event'
+require 'contrast/agent/version'
+require 'contrast/utils/os'
+
+module Contrast
+  module Agent
+    # This class will hold the Startup Metrics Telemetry Event
+    # The class will include initialization of the agent version, language version
+    # os type, arch and version
+    # application framework and version and server framework
+    # It will be initialized and send in Middleware#agent_startup_routine
+    class StartupMetricsTelemetryEvent < Contrast::Agent::MetricTelemetryEvent
+      include Contrast::Utils::OS
+
+      APP_AND_SERVER_DATA = ::Contrast::APP_CONTEXT.app_and_server_information.cs__freeze
+      SAAS_DEFAULT = { addr: 'app.contrastsecuirty.com', type: 'SAAS_DEFAULT' }.cs__freeze
+      SAAS_CE = { addr: 'ce.contrastsecurity.com', type: 'SAAS_CE' }.cs__freeze
+      SAAS_CUSTOM = { addr: 'contrastsecurite.com', type: 'SAAS_CUSTOM' }.cs__freeze
+      SAAS_POV = { addr: 'eval.contrastsecuirty.com', type: 'SAAS_POV' }.cs__freeze
+      EOP = 'EOP'
+
+      def initialize
+        super
+        add_tags
+      end
+
+      def path
+        '/startup'
+      end
+
+      def add_tags
+        @tags['teamserver'] = teamserver_type
+        @tags['agent_version'] = VERSION
+        @tags['ruby_version'] = RUBY_VERSION
+        @tags['os_type'] = sys_info['os_type'] == 'Darwin' ? 'MacOS' : 'Linux'
+        @tags['os_arch'] = sys_info['os_arch']
+        @tags['os_version'] = sys_info['os_version']
+        @tags['app_framework_and_version'] = APP_AND_SERVER_DATA[:application_info].to_s
+        @tags['server_framework_and_version'] = APP_AND_SERVER_DATA[:server_info].to_s
+      end
+
+      def sys_info
+        @sys_info ||= get_system_information if @sys_info.nil?
+        @sys_info
+      end
+
+      private
+
+      # Here we extract the Teamserver url type
+      #
+      # @return[String] type, it could be SAAS_DEFAULT, SAAS_POV, SAAS_CE, SAAS_CUSTOM, or EOP
+      def teamserver_type
+        @_teamserver_type ||= if Contrast::API.api_url.include?(SAAS_DEFAULT[:addr])
+                                SAAS_DEFAULT[:type]
+                              elsif Contrast::API.api_url.include?(SAAS_POV[:addr])
+                                SAAS_POV[:type]
+                              elsif Contrast::API.api_url.include?(SAAS_CE[:addr])
+                                SAAS_CE[:type]
+                              elsif Contrast::API.api_url.end_with? SAAS_CUSTOM[:addr]
+                                SAAS_CUSTOM[:type]
+                              else
+                                EOP
+                              end
+      end
+    end
+  end
+end

data/lib/contrast/agent/static_analysis.rb

@@ -14,8 +14,8 @@ module Contrast
       include Contrast::Components::Scope::InstanceMethods
 
       class << self
-        # After the first request is complete, we do a one-time manual catchup to review and
-        # report the already-loaded gems.
+        # After the first request is complete, we do a one-time manual catchup to review and report the already-loaded
+        # gems.
         def catchup
           @_catchup ||= begin
             threaded_analysis!
@@ -23,6 +23,8 @@ module Contrast
           end
         end
 
+        # TODO: RUBY-1354
+        # TODO: RUBY-1356
         def send_inventory_message
           return unless ::Contrast::INVENTORY.enabled?
 

data/lib/contrast/agent/telemetry.rb

@@ -0,0 +1,129 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/env_variables'
+require 'contrast/components/logger'
+require 'contrast/utils/requests_client'
+require 'contrast/agent/worker_thread'
+require 'contrast/utils/telemetry'
+
+module Contrast
+  module Agent
+    # This class will initialize and hold everything needed for the telemetry
+    class Telemetry < WorkerThread
+      include Contrast::Utils::RequestsClient
+      include Contrast::Components::Logger::InstanceMethods
+      # this is where we will send the data from the agents
+      URL = 'https://telemetry.ruby.contrastsecurity.com/'
+      # Suggested timeout after each send is to be 3 hours (10800 seconds)
+      SUGGESTED_TIMEOUT = 10_800
+
+      class << self
+        include Contrast::Components::Logger::InstanceMethods
+        include Contrast::Config::EnvVariables
+
+        def application_id
+          @_application_id ||= begin
+            id = nil
+            mac = Contrast::Utils::Telemetry::Identifier.mac
+            app_name = Contrast::Utils::Telemetry::Identifier.app_name
+            id = mac + app_name if mac && app_name
+            Digest::SHA2.new(256).hexdigest(id || '_' + SecureRandom.uuid)
+          end
+        end
+
+        def instance_id
+          @_instance_id ||= Digest::SHA2.new(256).hexdigest(Contrast::Utils::Telemetry::Identifier.mac ||
+                                                                  '_' + SecureRandom.uuid)
+        end
+
+        def enabled?
+          @_enabled = telemetry_enabled? if @_enabled.nil?
+          @_enabled
+        end
+
+        private
+
+        def telemetry_enabled?
+          opt_out_telemetry = return_value(:telemetry_opt_outs)
+          return false if opt_out_telemetry.to_s.casecmp('true').zero?
+          return false if opt_out_telemetry.to_s.casecmp('1').zero?
+
+          # In case of connection error, do not create the background thread or queue,
+          # as if the opt-out env var was set
+          ip_opt_out_telemetry = Contrast::Utils::RequestsClient.initialize_connection(URL)
+          if ip_opt_out_telemetry.nil?
+            logger.warn('Connection was not established properly!!!')
+            logger.warn('THE SERVICE IS GOING TO BE TERMINATED!!')
+            return false
+          end
+
+          true
+        end
+      end
+
+      def attempt_to_start?
+        unless cs__class.enabled?
+          logger.warn('Telemetry service is disabled!')
+          return false
+        end
+
+        logger.debug('Attempting to start telemetry thread') unless running?
+        true
+      end
+
+      def start_thread!
+        return if running?
+
+        # It is recommended that implementations send a single payload of
+        # general metrics every 3 hours, starting from implementation startup.
+        @_thread = Contrast::Agent::Thread.new do
+          logger.debug('Starting background telemetry thread.')
+          event = queue.pop
+
+          begin
+            logger.debug('This is the current processed event', event)
+            res = Contrast::Utils::RequestsClient.send_request event, connection
+            sleep_time = Contrast::Utils::RequestsClient.handle_response res
+            sleep(sleep_time) unless sleep_time.nil?
+          rescue StandardError => e
+            logger.error('Could not send message to service from telemetry queue.', e)
+          end
+          sleep(SUGGESTED_TIMEOUT)
+        end
+      end
+
+      def send_event event
+        if ::Contrast::AGENT.disabled?
+          logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
+          return
+        end
+
+        return unless cs__class.enabled?
+
+        logger.debug('Enqueued event for sending', event_type: event.cs__class)
+
+        queue << event if event
+      end
+
+      def delete_queue!
+        @_queue&.clear
+        @_queue&.close
+        @_queue = nil
+      end
+
+      def stop!
+        return unless running?
+
+        super
+        delete_queue!
+      end
+
+      private
+
+      def queue
+        @_queue ||= Queue.new
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry_event.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+
+module Contrast
+  module Agent
+    # This class will hold the basic information for a Telemetry Event
+    class TelemetryEvent
+      include Contrast::Utils
+
+      attr_reader :timestamp, :tags
+
+      # The instance_id comes from the Telemetry Instance
+      def initialize
+        @tags = MetricsHash.new(String)
+        @timestamp = format_date
+        @instance_id = Contrast::Agent.telemetry_queue.__id__
+      end
+
+      def path
+        ''
+      end
+
+      def to_json **_args
+        { tags: @tags, timestamp: @timestamp, instance_id: @instance_id }
+      end
+
+      def format_date
+        Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')
+      end
+    end
+  end
+end

data/lib/contrast/agent/thread_watcher.rb

@@ -4,6 +4,7 @@
 require 'contrast/components/logger'
 require 'contrast/agent/service_heartbeat'
 require 'contrast/api/communication/messaging_queue'
+require 'contrast/agent/telemetry'
 
 module Contrast
   module Agent
@@ -22,28 +23,22 @@ module Contrast
         @heapdump_util = Contrast::Utils::HeapDumpUtil.new
         @heartbeat = Contrast::Agent::ServiceHeartbeat.new
         @messaging_queue = Contrast::Api::Communication::MessagingQueue.new
+        @telemetry = Contrast::Agent::Telemetry.new if Contrast::Agent::Telemetry.enabled?
       end
 
       def startup!
         return unless ::Contrast::AGENT.enabled?
 
-        unless heartbeat.running?
-          logger.debug('Attempting to start heartbeat thread')
-          heartbeat.start_thread!
-        end
-        heartbeat_result = heartbeat.running?
-        logger.debug('Heartbeat thread status', alive: heartbeat_result)
-
-        unless messaging_queue.running?
-          logger.debug('Attempting to start messaging queue thread')
-          messaging_queue.start_thread!
-        end
-        messaging_result = messaging_queue.running?
-        logger.debug('Messaging thread status', alive: messaging_result)
+        telemetry_thread_status = telemetry_thread_init
+        heartbeat_thread_status = heartbeat_thread_init
+        messaging_thread_status = messaging_thread_init
 
         logger.debug('ThreadWatcher started threads')
 
-        @pids[Process.pid] = messaging_result && heartbeat_result
+        @pids[Process.pid] = messaging_thread_status && heartbeat_thread_status
+        return @pids unless Contrast::Agent::Telemetry.enabled?
+
+        @pids[Process.pid] = messaging_thread_status && heartbeat_thread_status && telemetry_thread_status
       end
 
       def ensure_running?
@@ -57,6 +52,40 @@ module Contrast
         heartbeat.stop!
         messaging_queue.stop!
         heapdump_util.stop!
+        telemetry_queue&.stop!
+      end
+
+      def heartbeat_thread_init
+        unless heartbeat.running?
+          logger.debug('Attempting to start heartbeat thread')
+          heartbeat.start_thread!
+        end
+        heartbeat_result = heartbeat.running?
+        logger.debug('Heartbeat thread status', alive: heartbeat_result)
+        heartbeat_result
+      end
+
+      def telemetry_thread_init
+        @telemetry.start_thread! if @telemetry&.attempt_to_start?
+        telemetry_result = @telemetry&.running?
+        logger.debug('Telemetry thread status', alive: telemetry_result)
+        telemetry_result
+      end
+
+      def messaging_thread_init
+        unless messaging_queue.running?
+          logger.debug('Attempting to start messaging queue thread')
+          messaging_queue.start_thread!
+        end
+        messaging_result = messaging_queue.running?
+        logger.debug('Messaging thread status', alive: messaging_result)
+        messaging_result
+      end
+
+      def telemetry_queue
+        return if @telemetry.nil?
+
+        @telemetry
       end
     end
   end

data/lib/contrast/agent/version.rb

@@ -3,6 +3,6 @@
 
 module Contrast
   module Agent
-    VERSION = '4.12.0'
+    VERSION = '4.13.0'
   end
 end

data/lib/contrast/agent.rb

@@ -61,6 +61,12 @@ module Contrast
       thread_watcher.messaging_queue
     end
 
+    def self.telemetry_queue
+      return unless thread_watcher.telemetry_queue
+
+      thread_watcher.telemetry_queue
+    end
+
     def self.thread_watcher
       @_thread_watcher ||= Contrast::Agent::ThreadWatcher.new
     end

data/lib/contrast/components/api.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/components/base'
+require 'contrast/components/config'
+
+module Contrast
+  module Components
+    module Api
+      # A wrapper build around the Common Agent Configuration project to allow
+      # for Api keys to be mapped with their values contained in their
+      # parent_configuration_spec.yaml.
+      class Interface
+        include Contrast::Components::ComponentBase
+
+        def api_url
+          @_api_url ||= ::Contrast::CONFIG.root.api.url
+        end
+
+        def api_key
+          @_api_key ||= ::Contrast::CONFIG.root.api.api_key
+        end
+
+        def service_key
+          @_service_key ||= ::Contrast::CONFIG.root.api.service_key
+        end
+
+        def username
+          @_username ||= ::Contrast::CONFIG.root.api.user_name
+        end
+      end
+    end
+  end
+end

data/lib/contrast/components/app_context.rb

@@ -23,6 +23,10 @@ module Contrast
         DEFAULT_SERVER_NAME = 'localhost'
         DEFAULT_SERVER_PATH = '/'
 
+        SUPPORTED_FRAMEWORKS = %w[rails sinatra grape rack].cs__freeze
+
+        SUPPORTED_SERVERS = %w[passenger puma thin unicorn].cs__freeze
+
         def initialize
           original_pid
         end
@@ -109,6 +113,26 @@ module Contrast
           @_client_id ||= [app_name, pgid].join('-')
         end
 
+        def app_and_server_information
+          {
+              application_info: find_gem_information(SUPPORTED_FRAMEWORKS),
+              server_info: find_gem_information(SUPPORTED_SERVERS)
+          }
+        end
+
+        def find_gem_information arr
+          arr.each do |framework|
+            next unless Gem.loaded_specs.key?(framework)
+
+            loaded = Gem.loaded_specs[framework]
+            next unless loaded
+
+            name = loaded.instance_variable_get(:@name)
+            version = loaded.instance_variable_get(:@version).to_s
+            return [name, version].join(' ')
+          end
+        end
+
         def instrument_middleware_stack?
           !Contrast::Utils::JobServersRunning.job_servers_running?
         end

data/lib/contrast/components/config.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'contrast/utils/env_configuration_item'
+require 'ougai'
 require 'contrast/configuration'
 
 module Contrast
@@ -23,17 +24,33 @@ module Contrast
     # time than to silently fail to deliver functionality.
     module Config
       CONTRAST_ENV_MARKER = 'CONTRAST__'
+      CONTRAST_LOG = 'contrast_agent.log'
+      CONTRAST_NAME = 'Contrast Agent'
 
       class Interface # :nodoc:
         def initialize
           build
         end
 
-        def build log: true
+        # Basic logger for handling configuration validation logging
+        # the file to log is determined by the default one or set
+        # by the config file, if that configuration is found
+        def proto_logger
+          @_proto_logger ||= begin
+            @_proto_logger = ::Ougai::Logger.new(logger_path || CONTRAST_LOG)
+            @_proto_logger.progname = CONTRAST_NAME
+            @_proto_logger.level = ::Ougai::Logging::Severity::WARN
+            @_proto_logger.formatter = Contrast::Logger::Format.new
+            @_proto_logger.formatter.datetime_format = '%Y-%m-%dT%H:%M:%S.%L%z'
+            @_proto_logger
+          end
+        end
+
+        def build
           @_valid = nil
           @config = Contrast::Configuration.new
           env_overrides
-          validate(log: log)
+          validate
         end
         alias_method :rebuild, :build
 
@@ -43,7 +60,7 @@ module Contrast
         end
 
         def valid?
-          @_valid = validate(log: false) if @_valid.nil?
+          @_valid = validate if @_valid.nil?
           @_valid
         end
 
@@ -59,20 +76,28 @@ module Contrast
 
         SESSION_VARIABLES = 'Invalid configuration. '\
                             "Setting both application.session_id and application.session_metadata is not allowed.\n"
-        def validate log: false
+        API_URL = "Invalid configuration. Missing a required connection value 'url' is not set."
+        API_KEY = "Invalid configuration. Missing a required connection value 'api_key' is not set."
+        API_SERVICE_KEY = "Invalid configuration. Missing a required connection value 'service_tag' is not set."
+        API_USERNAME = "Invalid configuration. Missing a required connection value 'user_name' is not set."
+        def validate
           # The config has information about how to construct the logger.
           # If the config is invalid, and you want to know about it, then
           # you have a circular dependency if you try to log it,
-          # hence `log: false`.
+          # so we use basic proto_logger to do this job.
           if !session_id.empty? && !session_metadata.empty?
-            if log
-              cs__class.log_error(SESSION_VARIABLES)
-            else
-              puts SESSION_VARIABLES
-            end
+            proto_logger.error(SESSION_VARIABLES)
             return false
           end
-
+          if bypass
+            msg = []
+            msg << API_URL unless api_url
+            msg << API_KEY unless api_key
+            msg << API_SERVICE_KEY unless api_service_key
+            msg << API_USERNAME unless api_username
+            msg.any? { |m| proto_logger.error(m) }
+            return false unless msg.empty?
+          end
           true
         end
 
@@ -108,6 +133,60 @@ module Contrast
         def session_metadata
           @config.application.session_metadata
         end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::API, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def api_url
+          @config.api.url
+        end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::API, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def api_key
+          @config.api.api_key
+        end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::API, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def api_service_key
+          @config.api.service_key
+        end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::API, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def api_username
+          @config.api.user_name
+        end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::API, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def bypass
+          @config.root.agent.service.bypass
+        end
+
+        # Typically, the following values would be accessed through Contrast::Components::AppContext
+        # and Contrast::Components::Logger, but we're too early in the initialization of the Agent to use
+        # that mechanism, so we look it up directly for ourselves.
+        #
+        # @return [String, nil]
+        def logger_path
+          @config.root.agent.logger.path
+        end
       end
     end
   end

data/lib/contrast/components/contrast_service.rb

@@ -31,6 +31,12 @@ module Contrast
               (LOCALHOST.match?(host) || !!socket_path)
         end
 
+        def use_agent_communication?
+          return @_use_agent_communication unless @_use_agent_communication.nil?
+
+          @_use_agent_communication = true?(::Contrast::CONFIG.root.agent.service.bypass)
+        end
+
         def host
           @_host ||=
             (::Contrast::CONFIG.root.agent.service.host || Contrast::Config::ServiceConfiguration::DEFAULT_HOST).to_s

data/lib/contrast/config/api_configuration.rb

@@ -0,0 +1,22 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/default_value'
+
+module Contrast
+  module Config
+    # Api keys configuration
+    class ApiConfiguration < BaseConfiguration
+      URL = 'https://app.contrastsecurity.com/Contrast'
+      KEYS = {
+          api_key: EMPTY_VALUE,
+          url: Contrast::Config::DefaultValue.new(URL),
+          user_name: EMPTY_VALUE,
+          service_key: EMPTY_VALUE
+      }.cs__freeze
+      def initialize hsh
+        super(hsh, KEYS)
+      end
+    end
+  end
+end

data/lib/contrast/config/env_variables.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2021 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Config
+    # This module is holding all the Env Variables that we could use through the agent lifecycle
+    module EnvVariables
+      ENV_VARIABLES = {
+          telemetry_opt_outs: ENV['CONTRAST_AGENT_TELEMETRY_OPTOUT'].to_s || Contrast::Config::DefaultValue.new('false')
+      }.cs__freeze
+
+      def return_value key
+        return unless ENV_VARIABLES.key?(key.to_sym)
+
+        sym_key = key.downcase.to_sym
+        return_val = ENV_VARIABLES[sym_key]
+        if return_val.is_a?(Contrast::Config::DefaultValue)
+          return_val.value
+        else
+          return_val
+        end
+      end
+    end
+  end
+end

data/lib/contrast/config/root_configuration.rb

@@ -6,6 +6,7 @@ module Contrast
     # The base of the Common Configuration settings.
     class RootConfiguration < BaseConfiguration
       KEYS = {
+          api: Contrast::Config::ApiConfiguration,
           enable: BaseConfiguration::EMPTY_VALUE,
           agent: Contrast::Config::AgentConfiguration,
           application: Contrast::Config::ApplicationConfiguration,

data/lib/contrast/config/service_configuration.rb

@@ -19,7 +19,8 @@ module Contrast
           host: EMPTY_VALUE,
           port: EMPTY_VALUE,
           socket: EMPTY_VALUE,
-          logger: Contrast::Config::LoggerConfiguration
+          logger: Contrast::Config::LoggerConfiguration,
+          bypass: Contrast::Config::DefaultValue.new(false)
       }.cs__freeze
 
       def initialize hsh

data/lib/contrast/config.rb

@@ -24,6 +24,7 @@ require 'contrast/config/protect_rules_configuration'
 require 'contrast/config/sampling_configuration'
 
 require 'contrast/config/ruby_configuration'
+require 'contrast/config/api_configuration'
 require 'contrast/config/agent_configuration'
 require 'contrast/config/application_configuration'
 require 'contrast/config/server_configuration'