contrast-agent 5.3.0 → 6.1.1

data/lib/contrast/agent/telemetry/base.rb

@@ -0,0 +1,155 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/env_variables'
+require 'contrast/components/logger'
+require 'contrast/utils/telemetry_client'
+require 'contrast/agent/worker_thread'
+require 'contrast/utils/telemetry'
+require 'contrast/agent/telemetry/events/exceptions/telemetry_exceptions'
+require 'contrast/agent/telemetry/events/exceptions/telemetry_exceptions_report'
+
+module Contrast
+  module Agent
+    module Telemetry
+      # This class will initialize and hold everything needed for the telemetry
+      class Base < WorkerThread
+        include Contrast::Components::Logger::InstanceMethods
+        include Contrast::Agent::Telemetry::TelemetryExceptionReport
+
+        # this is where we will send the data from the agents
+        URL = 'https://telemetry.ruby.contrastsecurity.com/'
+        # Suggested timeout after each send is to be 3 hours (10800 seconds)
+        SUGGESTED_TIMEOUT = 10_800
+
+        class << self
+          include Contrast::Components::Logger::InstanceMethods
+          include Contrast::Config::EnvVariables
+
+          def application_id
+            Contrast::Utils::Telemetry::Identifier.application_id
+          end
+
+          def instance_id
+            Contrast::Utils::Telemetry::Identifier.instance_id
+          end
+
+          def enabled?
+            @_enabled = telemetry_enabled? if @_enabled.nil?
+            @_enabled
+          end
+
+          private
+
+          def telemetry_enabled?
+            opt_out_telemetry = return_value(:telemetry_opt_outs).to_s
+            return false if opt_out_telemetry.casecmp?('true') || opt_out_telemetry == '1'
+
+            # In case of connection error, do not create the background thread or queue,
+            # as if the opt-out env var was set
+            @_client = Contrast::Utils::TelemetryClient.new
+            ip_opt_out_telemetry = @_client.initialize_connection(URL)
+            if ip_opt_out_telemetry.nil?
+              logger.warn("Connection was not established properly!!! \n Telemetry reporting will be disabled!")
+              return false
+            end
+
+            true
+          end
+        end
+
+        def client
+          @_client ||= Contrast::Utils::TelemetryClient.new
+        end
+
+        def connection
+          @_connection ||= client.initialize_connection(URL)
+        end
+
+        def error_messages
+          @_error_messages ||= []
+        end
+
+        def attempt_to_start?
+          unless cs__class.enabled?
+            logger.warn('Telemetry service is disabled!')
+            return false
+          end
+
+          logger.debug('Attempting to start telemetry thread') unless running?
+          true
+        end
+
+        def start_thread!
+          return if running?
+
+          # It is recommended that implementations send a single payload of
+          # general metrics every 3 hours, starting from implementation startup.
+          @_thread = Contrast::Agent::Thread.new do
+            logger.debug('Starting background telemetry thread.')
+            loop do
+              next unless client && connection
+
+              # Start pushing exceptions to queue for reporting.
+              push_exceptions
+              until queue.empty?
+                event = queue.pop
+                begin
+                  logger.debug('This is the current processed event', event)
+                  sleep_time = request_with_response(event)
+                  if sleep_time
+                    sleep(sleep_time)
+                    logger.debug('Retrying to process event', event)
+                    retry_sleep_time = request_with_response(event)
+                    sleep(retry_sleep_time) unless retry_sleep_time.nil?
+                  end
+                rescue StandardError => e
+                  logger.error('Could not send message to service from telemetry queue.', e)
+                  stop!
+                end
+              end
+              sleep(SUGGESTED_TIMEOUT)
+            end
+          end
+        end
+
+        def send_event event
+          if ::Contrast::AGENT.disabled?
+            logger.warn('Attempted to queue event with Agent disabled', caller: caller, event: event)
+            return
+          end
+
+          return unless cs__class.enabled?
+
+          logger.debug('Enqueued event for sending', event_type: event.cs__class)
+          queue << event if event
+        end
+
+        def delete_queue!
+          @_queue&.clear
+          @_queue&.close
+          @_queue = nil
+        end
+
+        def stop!
+          return unless running?
+
+          @_enabled = false
+          delete_queue!
+          super
+        end
+
+        def request_with_response event
+          res = client.send_request(event, connection)
+          client.handle_response(res)
+        end
+
+        private
+
+        def queue
+          @_queue ||= Queue.new
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/event.rb

@@ -0,0 +1,35 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/utils/metrics_hash'
+
+module Contrast
+  module Agent
+    module Telemetry
+      # This class will hold the basic information for a Telemetry Event
+      class Event
+        include Contrast::Utils
+
+        attr_reader :tags
+
+        def initialize
+          @tags = MetricsHash.new(String)
+          @timestamp = Time.now.iso8601
+        end
+
+        def path
+          ''
+        end
+
+        def to_hash **_args
+          {
+              tags: @tags,
+              timestamp: @timestamp,
+              instance: Contrast::Agent::Telemetry::Base.instance_id,
+              application: Contrast::Agent::Telemetry::Base.application_id
+          }
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/obfuscate.rb

@@ -0,0 +1,119 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # Here will be all known gems to obfuscate during the building of TelemetryExceptions
+        module Obfuscate
+          # List of known gems to be third parties not containing any
+          # user sensitive data.
+          KNOWN_GEMS = %w[
+            activesupport redis-activesupport redis builder dry-types rails rails-html-sanitizer rails-observers sinatra
+            benchmark-ips bundler climate_control execjs factory_bot debride fake_ftp fasterer flay openssl
+            parallel_tests contrast contrast-agent ruby pry-byebug byebug pry resolv ougai protobuf async nokogiri
+            benchmark grape-order grape-activerecord newrelic newrelic-grape grape-rails-cache grape-msgpack
+            grape-batch rspec rake rubocop grape-jbuilder rack-test rack-protection minitest grape-instrumentation
+            minitest-global_expectations rack-proxy rack-attack rack-ssl grape-cancan grape-attack grape-rake-tasks
+            grape-route-helpers benchmark-memory grape-rabl grape-kaminari grape-path-helpers grape-swagger-entity
+            grape-swagger-rails grape-roar newrelic-rake grapes-wagger-representative grape-forgery_protection
+            grape-papertrail grape-app grape-middleware-logger rack-protection rake-compile rhino rspec-benchmark
+            rspec_junit_formatter rspec-rails rake-performance rubocop-rails rubocop-rake rubocop-rspec
+            ruby-debug-ide simplecov sqlite steep tilt tzinfo-data warning xpath sinatra-activerecord sinatra-param
+            sinatra-partial sinatra-flash sinatra-reloader sinatra-sinatra rake_fly rack rack-accept grape grape-entity
+            sinatra-advanced-routes sinatra-warden grape_logging grape-swagger sinatra-namespace sinatra-resource
+            sinatra-hashfix sinatra-instrumentation sinatra-helpers redis-rack sinatra-support sinatra-assetpack
+            sinatra-router sinatra-cross_origin sinatra_more sinatra-jsonp faraday-rack zlib mustermann mustermann-grape
+            rspec-expectations rspec-core rspec-mocks rspec-sidekiq
+          ].cs__freeze
+          KNOWN_ERRORS = %w[
+            ActiveModel Error Errors Resolv Rspec ActiveRecord nil NilClass NoMemoryError ScriptError
+            LoadError NotImplementedError SyntaxError SecurityError SignalException Interrupt
+            StandardError ArgumentError UncaughtThrowError FiberError IOError EOFError IndexError KeyError
+            StopIteration LocalJumpError NameError NoMethodError RangeError FloatDomainError RegexpError
+            RuntimeError FrozenError SystemCallError ThreadError TypeError ZeroDivisionError SystemExit
+            SystemStackError fatal Warning buffer OpenSSL SSL SSLError Errno Timeout Exception EOFError
+            ECONNRESET ECONNREFUSED ETIMEDOUT EINVAL ESHUTDOWN EHOSTDOWN EHOSTUNREACH EISCONN
+            ECONNABORTED ENETRESET ENETUNREACH Net HTTPBadResponse HTTPHeaderSyntaxError ProtocolError
+            Faraday BadRequestError UnauthorizedError ForbiddenError ResourceNotFound ProxyAuthError
+            ConflictError UnprocessableEntityError ClientError
+          ].cs__freeze
+          CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.cs__freeze
+          # This will generate different chars for each agent startup but will be constant
+          # for current run and will make identical obfuscation to be compared if given type
+          # is the same.
+          CYPHER = CHARS.chars.shuffle.join
+
+          class << self
+            # Returns paths for known gems.
+            #
+            # @return [Array<Regexp>] known paths
+            def known_gem_paths
+              @_known_gem_paths ||= KNOWN_GEMS.map do |gem_name|
+                to_regexp(File.join(
+                              'gems', gem_name.tr('-', ''), 'lib'))
+              end
+            end
+
+            # Returns known modules names.
+            #
+            # @return [Array<Regexp>] known modules
+            def known_modules
+              @_known_modules ||= KNOWN_ERRORS.map { |module_name| to_regexp(module_name) }
+            end
+
+            # Obfuscate a type and replace it with random characters.
+            #
+            # @param type [String] the StackFrame type to obfuscate
+            # @return [String] obfuscated type
+            def obfuscate_type type
+              return unless type
+
+              check = type.tr('[^0-9].-', '')
+              type = cypher(type) unless match_known(known_gem_paths, check)
+              type
+            end
+
+            # Obfuscate a type and replace it with random characters.
+            #
+            # @param exception_type [String] the exception type to obfuscate
+            # @return [String] obfuscated exception type
+            def obfuscate_exception_type exception_type
+              return unless exception_type
+
+              exceptions = exception_type.split('::').each do |exception|
+                cypher(exception) unless match_known(known_modules, exception)
+              end
+              exceptions.join('::')
+            end
+
+            private
+
+            # Transforms string to regexp
+            #
+            # @param string [String]
+            def to_regexp string
+              /#{ string }/
+            end
+
+            # Add cypher to path to make it obscure, but unique enough for
+            # comparisons. Mutates original or duplicate if frozen string.
+            #
+            # @param string [String] string to be transformed.
+            def cypher string
+              string = string.dup if string.cs__frozen?
+              string.to_s.tr!(CHARS, CYPHER)
+            end
+
+            # @param known [Array<Regexp>] Array of regexp to match againts
+            # @param type [String] type to check
+            def match_known known, type
+              known.any? { |regexp| type =~ regexp }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/telemetry_exception_base.rb

@@ -0,0 +1,59 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # This class will hold the all the mutual information for the Telemetry Exception
+        class Base
+          def to_controlled_hash; end
+
+          protected
+
+          # Validate required and option fields
+          #
+          # @param validations [Hash] Validation hash to use
+          # @return [nil]
+          def validate validations
+            validations.each do |k, v|
+              next if v[:required] == false
+
+              validate_field(validations[k], k)
+            end
+            nil
+          end
+
+          # This method will validate every single field passed from validate
+          #
+          # @param validation_pair [Hash] Validation hash to use
+          # @param key[String] The key to check in VALIDATIONS
+          def validate_field validation_pair, key
+            value_to_validate = send(key.to_sym)
+            validate_class(value_to_validate, validation_pair[:class], key) if validation_pair.key?(:class)
+            value_length = if value_to_validate.cs__is_a?(String) || value_to_validate.cs__is_a?(Array)
+                             value_to_validate.length
+                           else
+                             value_to_validate.entries.length
+                           end
+            unless validation_pair[:range].include?(value_length)
+              raise(ArgumentError, "The provided value for #{ key } is invalid: #{ value_to_validate }")
+            end
+
+            nil
+          end
+
+          # With the all nested classes, we still need to double check if everything passed along the way
+          # is right
+          # @param message [Object] The message we want to check the class of
+          # @param klass [Class] The klass we want to check the message with
+          # @param field [Object] The field with the error
+          def validate_class message, klass, field
+            message = message[0] if message.cs__is_a?(Array)
+            raise(ArgumentError, "The provided value for #{ field } is of wrong class") unless message.cs__is_a?(klass)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/telemetry_exception_event.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require_relative 'telemetry_exception_base'
+require_relative 'telemetry_exception_message'
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # This class will hold the basic information for a Parent Telemetry Exception Event
+        class Event < Contrast::Agent::Telemetry::TelemetryException::Base
+          # Array of Telemetry Exceptions
+          # @return [Array<Contrast::Agent::Telemetry::TelemetryException::Message>]
+          attr_reader :exceptions
+
+          # Initialization of the Parent Event requires us to require the exception
+          # to be created
+          #
+          # @param message [Contrast::Agent::Telemetry::TelemetryException::Message]
+          def initialize message
+            super()
+            validate_class(message, Contrast::Agent::Telemetry::TelemetryException::Message, 'exception_message')
+            @exceptions = Array.new(1, message)
+          end
+
+          # @param message [Contrast::Agent::Telemetry::TelemetryException::Message]
+          def push message
+            validate_class(message, Contrast::Agent::Telemetry::TelemetryException::Message, 'exception_message')
+            @exceptions << message
+          end
+
+          def self.path
+            '/ruby/runtime'
+          end
+
+          def to_controlled_hash
+            exceptions.map(&:to_controlled_hash)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/telemetry_exception_message.rb

@@ -0,0 +1,115 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require_relative 'telemetry_exception_base'
+require_relative 'telemetry_exception_message_exception'
+require 'contrast/utils/telemetry_identifier'
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # This class will hold the all the information for the specific exceptions
+        # and will be passed in the the event to be sent to TS
+        class Message < Contrast::Agent::Telemetry::TelemetryException::Base
+          VALIDATIONS = {
+              instance: { required: true, range: 12..64 },
+              tags: { required: true, range: 0..512 },
+              logger: { required: false, range: 1..128 },
+              message: { required: false, range: 1..512 },
+              exceptions: {
+                  required: true,
+                  range: 1..512,
+                  class: Contrast::Agent::Telemetry::TelemetryException::MessageException
+              }
+          }.cs__freeze
+
+          # Timestamp of creation in ISO8601 format
+          # @return [Integer]
+          attr_reader :timestamp
+
+          # An Instance ID as defined in Unique Identification // Application ID
+          # @return [String]
+          attr_reader :instance
+
+          # Tags are key-value string pairs that annotate either metrics
+          # or errors to help provide context, filtering, grouping, and deduplication.
+          # @return [Hash{String => String}]
+          attr_reader :tags
+
+          # @return [Integer] A number of the occurrences of the exception
+          attr_accessor :occurrences
+
+          # Array of exceptions, but in our case the Array will only include one exception
+          # @return [Array<Contrast::Agent::Telemetry::TelemetryException::MessageException>]
+          attr_reader :exceptions
+
+          # @return [String,nil] A string denoting the origin of this error.
+          attr_reader :logger
+
+          # @return [String | nil] A string message to provide additional context to the errors.
+          attr_reader :message
+
+          def initialize instance, tags, exceptions
+            super()
+            @tags = tags
+            @timestamp = Time.now.iso8601
+            @instance = instance
+            @occurrences = 1
+            @exceptions = exceptions
+            validate(VALIDATIONS)
+          end
+
+          # Optional parameters will be set separately from the required
+          #
+          # @param logger[String]
+          def logger= logger
+            validate_field(VALIDATIONS[:logger], 'logger')
+            @logger = logger
+          end
+
+          # Optional parameters will be set separately from the required
+          #
+          # @param message[String]
+          def message= message
+            validate_field(VALIDATIONS[:message], 'message')
+            @message = message
+          end
+
+          # Optional parameters will be set separately from the required
+          # This method is different and is regarding the way we proceed
+          # with incrementing occurrences
+          # If we keep track of them in different places and we store that value
+          # in separated variable - we may directly re-assign occurrences=
+          # But if we are not doing that - we may on same message generated
+          # to increment occurrences from here
+          def increment_occurrences
+            @occurrences += 1
+          end
+
+          def to_controlled_hash
+            super()
+            {
+                timestamp: timestamp,
+                instance: instance,
+                occurrences: occurrences,
+                tags: tags,
+                exceptions: exceptions.map(&:to_controlled_hash),
+                logger: logger,
+                message: message
+            }.compact
+          end
+
+          class << self
+            def build tags, exceptions, logger = nil, message = nil
+              inst = new(Contrast::Utils::Telemetry::Identifier.instance_id, tags, exceptions)
+              inst.logger = logger unless logger.nil?
+              inst.message = message unless message.nil?
+              inst
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/telemetry_exception_message_exception.rb

@@ -0,0 +1,83 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require_relative 'telemetry_exception_base'
+require_relative 'telemetry_exception_stack_frame'
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # This class will hold the all the information for the specific exception
+        # and will be passed in the Exception message itself
+        class MessageException < Contrast::Agent::Telemetry::TelemetryException::Base
+          VALIDATIONS = {
+              type: { required: true, range: 1..256 },
+              module_name: { required: false, range: 1..256 },
+              value: { required: false, range: 1..256 },
+              stack_frames: {
+                  required: true,
+                  range: 1..128,
+                  class: Contrast::Agent::Telemetry::TelemetryException::StackFrame
+              }
+          }.cs__freeze
+
+          # @return [String] The type of the exception itself
+          attr_reader :type
+
+          # stack frames for the message exception
+          # @return [Array<Contrast::Agent::Telemetry::TelemetryException::StackFrame>]
+          attr_reader :stack_frames
+
+          # @return [String]
+          attr_reader :module_name
+
+          # @return [String]
+          attr_reader :value
+
+          def initialize type, stack_frame
+            super()
+            @type = type
+            @stack_frames = Array.new(1, stack_frame)
+            validate(VALIDATIONS)
+          end
+
+          # @param stack_frame [Contrast::Agent::Telemetry::TelemetryException::StackFrame]
+          def push stack_frame
+            validate_class(stack_frame, Contrast::Agent::Telemetry::TelemetryException::StackFrame, 'stack_frame')
+            @stack_frames << stack_frame
+          end
+
+          def module_name= module_name
+            @module_name = module_name
+            validate_field(VALIDATIONS[:module_name], 'module_name')
+          end
+
+          def value= value
+            @value = value
+            validate_field(VALIDATIONS[:value], 'value')
+          end
+
+          def to_controlled_hash
+            super
+            {
+                type: type,
+                module: module_name,
+                stackFrames: stack_frames.map(&:to_controlled_hash),
+                value: value
+            }.compact
+          end
+
+          class << self
+            def build type, value, module_name, stackframes
+              inst = new(type, stackframes)
+              inst.module_name = module_name if module_name
+              inst.value = value
+              inst
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/agent/telemetry/events/exceptions/telemetry_exception_stack_frame.rb

@@ -0,0 +1,64 @@
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require_relative 'telemetry_exception_base'
+
+module Contrast
+  module Agent
+    module Telemetry
+      module TelemetryException
+        # This class will hold the all the information for the specific exception
+        # and will be passed in the Exception message itself
+        class StackFrame < Contrast::Agent::Telemetry::TelemetryException::Base
+          VALIDATIONS = {
+              function: { required: true, range: 1..256 },
+              type: { required: true, range: 1..256 },
+              module_name: { required: false, range: 1..256 }
+          }.cs__freeze
+
+          # @return [String] The type of the exception itself
+          attr_reader :type
+
+          # @return [String] the function of the stack trace
+          attr_reader :function
+
+          # @return [Boolean] Is it in contrast
+          attr_accessor :in_contrast
+
+          # @return [String]
+          attr_reader :module_name
+
+          def initialize function, type
+            super()
+            @function = function
+            @type = type
+            @in_contrast = false
+            validate(VALIDATIONS)
+          end
+
+          def module_name= module_name
+            @module_name = module_name
+            validate_field(VALIDATIONS[:module_name], 'module_name')
+          end
+
+          def to_controlled_hash
+            super
+            { function: function, type: type, module: module_name, inContrast: in_contrast }.compact
+          end
+
+          class << self
+            # @param method [String] method, triggered the logger on warn/error/fatal
+            # @param type [String] the type ( where it occurred )
+            # @param module_name [String, nil] Name of the module if any.
+            def build method, type, module_name = nil
+              inst = new(method, type)
+              inst.module_name = module_name if module_name
+              inst.in_contrast = type.include?('lib/contrast')
+              inst
+            end
+          end
+        end
+      end
+    end
+  end
+end