contrast-agent 6.15.3 → 7.1.0

data/lib/contrast/configuration.rb

@@ -15,6 +15,8 @@ require 'contrast/components/protect'
 require 'contrast/components/assess'
 require 'contrast/components/config/sources'
 require 'contrast/config/server_configuration'
+require 'contrast/config/configuration_files'
+require 'contrast/utils/hash_utils'
 
 module Contrast
   # This is how we read in the local settings for the Agent, both ENV/ CMD line
@@ -57,7 +59,8 @@ module Contrast
     DEFAULT_YAML_PATH  = 'contrast_security.yaml'
     MILLISECOND_MARKER = '_ms'
     CONVERSION = {}.cs__freeze
-    CONFIG_BASE_PATHS = ['', 'config/', '/etc/contrast/ruby/', '/etc/contrast/', '/etc/'].cs__freeze
+    # Precedence of paths, shift if config file values needs to go up the chain.
+    CONFIG_BASE_PATHS = %w[./ config/ /etc/contrast/ruby/ /etc/contrast/ /etc/].cs__freeze
     KEYS_TO_REDACT = %i[api_key url service_key user_name].cs__freeze
     REDACTED = '**REDACTED**'
 
@@ -65,9 +68,7 @@ module Contrast
       @default_name = default_name
 
       # Load config_kv from file
-      config_kv = deep_symbolize_all_keys(load_config)
-      config_sources = assign_source_to(config_kv, Contrast::Components::Config::Sources::YAML)
-
+      config_kv = Contrast::Utils::HashUtils.deep_symbolize_all_keys(load_config)
       unless cli_options
         cli_options = {}
         ENV.each do |key, value|
@@ -76,19 +77,22 @@ module Contrast
           cli_options[key] = value
         end
       end
+
       # Overlay CLI options - they take precedence over config file
-      cli_options = deep_symbolize_all_keys(cli_options)
+      cli_options = Contrast::Utils::HashUtils.deep_symbolize_all_keys(cli_options)
       if cli_options
-        config_kv = deep_merge(cli_options, config_kv)
-        config_sources = deep_merge(assign_source_to(cli_options, Contrast::Components::Config::Sources::CLI),
-                                    config_sources)
+        config_kv = Contrast::Utils::HashUtils.deep_merge(cli_options, config_kv)
+        @_source_file_extensions = Contrast::Utils::HashUtils.
+            deep_merge(assign_source_to(cli_options,
+                                        Contrast::Components::Config::Sources::COMMAND_LINE),
+                       @_source_file_extensions)
       end
 
       # Some in-flight rewrites to maintain backwards compatibility
       config_kv = update_prop_keys(config_kv)
       @loaded_config = config_kv
 
-      @sources = Contrast::Components::Config::Sources.new(config_sources)
+      @sources = Contrast::Components::Config::Sources.new(@_source_file_extensions)
 
       @api = Contrast::Components::Api::Interface.new(config_kv[:api])
       @enable = config_kv[:enable]
@@ -107,6 +111,11 @@ module Contrast
       convert_to_hash.to_yaml
     end
 
+    # @return [Hash] map of all extensions for each config key.
+    def source_file_extensions
+      @_source_file_extensions ||= {}
+    end
+
     # @return [Contrast::Components::Api::Interface]
     def api
       @api ||= Contrast::Components::Api::Interface.new # rubocop:disable Naming/MemoizedInstanceVariableName
@@ -142,27 +151,36 @@ module Contrast
       @protect ||= Contrast::Components::Protect::Interface.new # rubocop:disable Naming/MemoizedInstanceVariableName
     end
 
-    protected
-
-    # TODO: RUBY-546 move utility methods to auxiliary classes
+    # Base paths to check for the contrast configuration file, sorted by
+    # reverse order of precedence (first is most important).
+    def configuration_paths
+      @_configuration_paths ||= begin
+        basename = default_name.split('.').first
+        # Order of extensions comes from here:
+        extensions = Contrast::Components::Config::Sources::APP_CONFIGURATION_FILE.map(&:downcase)
 
-    def load_config
-      config = {}
-      configuration_paths.find do |path|
-        next unless File.exist?(path)
+        paths = []
+        # Environment paths takes precedence here. Look first through them.
+        paths << ENV['CONTRAST_CONFIG_PATH']     if ENV['CONTRAST_CONFIG_PATH']
+        paths << ENV['CONTRAST_SECURITY_CONFIG'] if ENV['CONTRAST_SECURITY_CONFIG']
 
-        unless File.readable?(path)
-          log_file_read_error(path)
-          next
+        extensions.each do |ext|
+          places = CONFIG_BASE_PATHS.product(["#{ basename }.#{ ext }"])
+          paths += places.map!(&:join)
         end
-        config = yaml_to_hash(path) || {}
-        @config_file = path
-        break
+        paths
       end
+    end
 
-      config
+    # List of all read configuration files.
+    #
+    # @return [Contrast::Config::ConfigurationFiles] of paths
+    def origin
+      @_origin ||= Contrast::Config::ConfigurationFiles.new
     end
 
+    protected
+
     def yaml_to_hash path
       if path && File.readable?(path)
         begin
@@ -179,6 +197,45 @@ module Contrast
       {}
     end
 
+    # TODO: RUBY-546 move utility methods to auxiliary classes
+
+    # Read through all the paths we know config may live. Merge all values from all files found.
+    # Priority is given to yaml over yml. If same keys are found on two configs with different extensions,
+    # the Agent will read first from the yaml file and ignore the values for same key on the yml file.
+    #
+    # @return config [Hash] All source configurations from files.
+    def load_config
+      config = {}
+      @_source_file_extensions = {}
+      configuration_paths.each do |path|
+        next unless File.exist?(path)
+
+        unless File.readable?(path)
+          log_file_read_error(path)
+          next
+        end
+        origin.add_source_file(path, (yaml_to_hash(path) || {}))
+      end
+
+      # Legacy usage: Assign main configuration file for reference.
+      @config_file = origin.main_file
+      # merge all settings keeping the top yaml files values as priority.
+      # If in top file a key exists it's value won't be changed if same key has different value in one
+      # of the other config files. Only unique values will be taken in consideration.w
+      # precedence of paths: see Contrast::Configuration::CONFIG_BASE_PATHS
+      extensions_maps = []
+      origin.source_files.each do |file|
+        # config.merge!(file.values) { |_key, oldval, _newval| oldval = oldval }
+        precedence_merge!(config, file.values)
+        # assign source values extentions:
+        extensions_maps << assign_source_to(Contrast::Utils::HashUtils.deep_symbolize_all_keys(file.values), file.path)
+      end
+      # merge all origin paths to be used as extension classification to preserve the precedence of config files:
+      extensions_maps.each { |path| @_source_file_extensions = precedence_merge!(@_source_file_extensions, path) }
+
+      config
+    end
+
     # We're updating properties loaded from the configuration files to match the new agreed upon standard configuration
     # names, so that one file works for all agents
     def update_prop_keys config
@@ -203,39 +260,6 @@ module Contrast
       config
     end
 
-    # Base paths to check for the contrast configuration file, sorted by
-    # reverse order of precedence (first is most important).
-    def configuration_paths
-      @_configuration_paths ||= begin
-        basename = default_name.split('.').first
-        names = %w[yml yaml].map { |suffix| "#{ basename }.#{ suffix }" }
-
-        paths = []
-        paths << ENV['CONTRAST_CONFIG_PATH']     if ENV['CONTRAST_CONFIG_PATH']
-        paths << ENV['CONTRAST_SECURITY_CONFIG'] if ENV['CONTRAST_SECURITY_CONFIG']
-
-        tmp = CONFIG_BASE_PATHS.product(names)
-        paths += tmp.map!(&:join)
-        paths
-      end
-    end
-
-    def deep_merge cli_config, file_config
-      cli_config.merge(file_config) do |_key, cli_value, file_value|
-        cli_value.is_a?(Hash) && file_value.is_a?(Hash) ? deep_merge(cli_value, file_value) : cli_value
-      end
-    end
-
-    def deep_symbolize_all_keys hash
-      return if hash.nil?
-
-      new_hash = {}
-      hash.each do |key, value|
-        new_hash[key.to_sym] = value.is_a?(Hash) ? deep_symbolize_all_keys(value) : value
-      end
-      new_hash
-    end
-
     private
 
     # We cannot use all access components at this point, unfortunately, as they
@@ -333,7 +357,7 @@ module Contrast
       KEYS_TO_REDACT.include?(key.to_sym)
     end
 
-    def assign_source_to hash, source = Contrast::Components::Config::Sources::YAML
+    def assign_source_to hash, source = Contrast::Components::Config::Sources::APP_CONFIGURATION_FILE[0]
       hash.transform_values do |value|
         if value.is_a?(Hash)
           assign_source_to(value, source)
@@ -342,5 +366,14 @@ module Contrast
         end
       end
     end
+
+    # Merges two hashes, first hash will preserve it's values and will only add unique values.
+    #
+    # @param hsh [Hash]
+    # @param other_hsh [Hash]
+    # @return [Hash]
+    def precedence_merge! hsh, other_hsh
+      hsh.merge!(other_hsh) { |_key, old_val, _new_val| old_val }
+    end
   end
 end

data/lib/contrast/utils/hash_utils.rb

@@ -0,0 +1,43 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Utils
+    # Module to hold various hash utils methods
+    module HashUtils
+      class << self
+        # Merges two hashes, first hash will preserve it's values and will only add unique values.
+        #
+        # @param hsh [Hash]
+        # @param other_hsh [Hash]
+        def deep_merge hsh, other_hsh
+          hsh.merge(other_hsh) do |_key, old_value, new_value|
+            old_value.is_a?(Hash) || new_value.is_a?(Hash) ? deep_merge(old_value, new_value) : old_value
+          end
+        end
+
+        # Deep symbolizes all keys
+        # @param value [Hash]
+        # @return [Hash]
+        def deep_symbolize_all_keys value
+          new_hash = {}
+          value.each { |key, v| new_hash[key.to_sym] = map_value(v) }
+          new_hash
+        end
+
+        private
+
+        def map_value value
+          case value
+          when Hash
+            deep_symbolize_all_keys(value)
+          when Array
+            value.map { |v| map_value(v) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/utils/json.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'json'
+require 'contrast/utils/hash_utils'
+require 'contrast/components/logger'
+
+module Contrast
+  module Utils
+    # Module to hold Agent's custom JSON.parse logic.
+    module Json
+      class << self
+        include Contrast::Components::Logger::InstanceMethods
+
+        # Add any known cases where parsing error might arise from older json parser:
+        # @return [Array<String>]
+        SPECIAL_CASES = ["\"\""].cs__freeze # rubocop:disable Style/StringLiterals
+
+        # Parses a string using JSON.parser. This method is used instead of standard JSON.parse to
+        # support older versions of json gem => not supporting key-value second parameter, which is
+        # supported after json 2.3.0.
+        #
+        # @param string [String]
+        # @param deep_symbolize [Boolean] flag to set if keys needs to be deep symbolized.
+        # @return [Hash]
+        def parse string, deep_symbolize: false
+          # The Agent receives empty responses from TS sometimes.
+          # There is a special case to handle this.
+          return {} if SPECIAL_CASES.include?(string)
+
+          symbolized_hash = {}
+          hash = JSON::Parser.new(string).parse
+          symbolized_hash = Contrast::Utils::HashUtils.deep_symbolize_all_keys(hash) if deep_symbolize
+          return hash unless deep_symbolize
+
+          symbolized_hash
+        rescue JSON::ParserError => e
+          # Any parsing error will produce empty {}. Since older JSON might pose support issues with newer Ruby,
+          # We might just log any miss-parsings and allow Agent to continue.
+          logger.warn("[JSON] parse error: #{ e }")
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/contrast/utils/middleware_utils.rb

@@ -11,12 +11,12 @@ module Contrast
     module MiddlewareUtils
       private
 
-      # TODO: RUBY-1609 update this part of the method for Ruby Version and Year
-      LANGUAGE_DEPRECATION_VERSION = '2.7'
-      LANGUAGE_DEPRECATION_YEAR = '2023'
+      LANGUAGE_DEPRECATION_VERSION = '3.0'
+      LANGUAGE_DEPRECATION_YEAR = '2024'
       LANGUAGE_DEPRECATION_WARNING =
         "[Contrast Security] [DEPRECATION] Support for Ruby #{ LANGUAGE_DEPRECATION_VERSION } will be removed in " \
-        "April #{ LANGUAGE_DEPRECATION_YEAR }. Please contact Customer Support prior if you require continued support."
+        "April #{ LANGUAGE_DEPRECATION_YEAR }. Please contact Customer Support prior if you require continued" \
+        'support.'.cs__freeze
 
       def setup_agent
         # Generate new config file if one is not already created:

data/lib/contrast/utils/net_http_base.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'net/http'
+require 'resolv'
 require 'contrast/components/logger'
 require 'contrast/utils/object_share'
 require 'socket'
@@ -10,8 +11,25 @@ module Contrast
   module Utils
     # This module creates a Net::HTTP client base to be used by different services
     # All HTTP clients reporting to Telemetry or TS should inherit from this
-    class NetHttpBase
+    class NetHttpBase # rubocop:disable Metrics/ClassLength
       include Contrast::Components::Logger::InstanceMethods
+
+      class << self
+        # Last recorded error
+        # @return [StandardError]
+        def last_error
+          @_last_error
+        end
+
+        # @param [StandardError]
+        def last_error= error
+          @_last_error = error
+        end
+      end
+
+      # @return [String]
+      attr_reader :client_name
+
       # This method initializes the Net::HTTP client we'll need. it will validate
       # the connection and make the first request. If connection is valid and response
       # is available then the open connection is returned.
@@ -23,29 +41,22 @@ module Contrast
       # self signed certificates provided by config [default = false]
       # @return [Net::HTTP, nil] Return open connection or nil
       def initialize_connection service_name, url, use_proxy: false, use_custom_cert: false
-        return unless url
-
-        addr = URI(url)
-        return if addr.host.nil? || addr.port.nil?
-        return if addr.scheme != 'https' && !addr.host.to_s.include?('localhost')
-
-        # the proxy is enabled only if there is provided url even if the enable is set to true
-        proxy_addr = URI(Contrast::API.proxy_url) if proxy_enabled?
-        net_http_client = initialize_client(addr, proxy_addr, use_proxy, use_custom_cert)
-        return if net_http_client.nil?
-
-        net_http_client.start
-        return unless net_http_client.started?
+        Contrast::Utils::NetHttpBase.last_error = nil
+        @client_name = service_name
+        return unless (addr = retrieve_address(url))
+        return unless (net_http_client = configure_new_client(addr, use_proxy, use_custom_cert))
+        return unless client_started?(net_http_client)
 
-        logger.debug("Starting #{ service_name } connection test")
+        logger.debug("Starting #{ client_name } connection test")
         return unless connection_verified?(net_http_client, url)
 
-        logger.debug('Client verified', service: service_name, url: url)
+        logger.debug('Client verified', service: client_name, url: url)
         net_http_client
       rescue StandardError => e
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug('Connection failed', e, service: service_name, url: url)
+        Contrast::Utils::NetHttpBase.last_error = e
+        return if client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+
+        logger.error('Connection failed', e, service: client_name, url: url)
         nil
       end
 
@@ -74,14 +85,51 @@ module Contrast
              Errno::ETIMEDOUT, Errno::ESHUTDOWN, Errno::EHOSTDOWN, Errno::EHOSTUNREACH, Errno::EISCONN,
              Errno::ECONNABORTED, Errno::ENETRESET, Errno::ENETUNREACH => e
 
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug("#{ service_name } connection failed", e.message)
-        false
+        Contrast::Utils::NetHttpBase.last_error = e
+        unless client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+          logger.error("#{ client_name } connection failed", e.message)
+        end
+        @_connection_verified = false
       end
 
       private
 
+      # Starts connection and return started status.
+      #
+      # @param client [Net::HTTP] client instance.
+      # @return [Boolean] status indicates whether connection has started.
+      def client_started? client
+        return false unless client
+
+        client.start
+        client.started?
+      end
+
+      # @param url
+      # @return [URI::Generic, nil]
+      def retrieve_address url
+        return unless (addr = URI(url))
+        return if addr.host.nil? || addr.port.nil?
+        return if addr.scheme != 'https' && !addr.host.to_s.include?('localhost')
+
+        addr
+      end
+
+      # Assigns proxy and custom certificates if enabled, and initializes new client.
+      #
+      # @param addr [URI::Generic, nil]
+      # @param use_proxy [Boolean] flag used to indicate proxy connections [default = false]
+      # @param use_custom_cert [Boolean] flag used to indicate whether the client is to use
+      # @return client [Net::HTTP, nil] initialized client.
+      def configure_new_client addr, use_proxy, use_custom_cert
+        # the proxy is enabled only if there is provided url even if the enable is set to true
+        proxy_addr = URI(Contrast::API.proxy_url) if proxy_enabled?
+        net_http_client = initialize_client(addr, proxy_addr, use_proxy, use_custom_cert)
+        return if net_http_client.nil?
+
+        net_http_client
+      end
+
       # Resolves the address of the assigned domain to array of corresponding IPs (if more than one)
       # and runs a matcher to see if current connection IP is in the list.
       # This is called within #verify_connection, if called on it's own there will be no
@@ -114,9 +162,10 @@ module Contrast
           client.key = OpenSSL::PKey::RSA.new(File.read(Contrast::API.certification_key_file)).to_s
         end
       rescue Errno::ENOENT => e
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug('Custom certificates failed', e.message)
+        Contrast::Utils::NetHttpBase.last_error = e
+        unless client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+          logger.error('Custom certificates failed', e.message)
+        end
       end
 
       # sets default setting for client validation of certificates and

data/lib/contrast/utils/object_share.rb

@@ -55,9 +55,9 @@ module Contrast
       EMPTY_HASH = {}.freeze
 
       # RegExps
-      DIGIT_REGEXP =       /[[:digit:]]/.freeze
-      WHITE_SPACE_REGEXP = /\s/.freeze
-      NOT_WHITE_SPACE_REGEXP = /[^\s]/.freeze
+      DIGIT_REGEXP =       /[[:digit:]]/
+      WHITE_SPACE_REGEXP = /\s/
+      NOT_WHITE_SPACE_REGEXP = /[^\s]/
 
       # Messages
       OVERRIDE_MESSAGE = 'A security filter prevented original response from being returned.'

data/lib/contrast.rb

@@ -102,19 +102,3 @@ if RUBY_VERSION >= '3.0.0' && RUBY_VERSION < '3.1.0'
   Class.alias_method(:prepend, :cs__orig_prepend)
   Class.remove_method(:cs__orig_prepend)
 end
-
-if RUBY_VERSION < '3.0.0'
-  # Better handles ancestors for older ruby versions.
-  # This is called from C, tread lightly.
-  class Module
-    @_included_in = []
-    # Returns array with modules including this instance
-    def included_in
-      @_included_in ||= [] unless cs__frozen?
-    end
-
-    def self.included_in
-      @_included_in ||= [] unless cs__frozen?
-    end
-  end
-end

data/ruby-agent.gemspec

@@ -51,7 +51,7 @@ end
 def self.add_frameworks spec
   spec.add_development_dependency 'grape', '~> 1.5', '>= 1.5.2'
   spec.add_development_dependency 'rack-protection', '>= 2'
-  spec.add_development_dependency 'rails', '~> 7'
+  spec.add_development_dependency 'rails', '>= 6', '~> 7'
   spec.add_development_dependency 'sinatra', '>= 2'
 end
 
@@ -80,7 +80,7 @@ def self.add_specs spec
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rspec-benchmark'
   spec.add_development_dependency 'rspec_junit_formatter', '0.3.0'
-  spec.add_development_dependency 'rspec-rails', '5.0'
+  spec.add_development_dependency 'rspec-rails', '6.0'
   spec.add_development_dependency 'tzinfo-data' # Alpine rspec-rails requirement.
   spec.add_development_dependency 'warning'
   spec.add_development_dependency 'typhoeus', '~> 1.4'
@@ -102,11 +102,7 @@ end
 
 # Dependencies not mocked out during RSpec that we test real code of, beyond just frameworks.
 def self.add_tested_gems spec
-  if RUBY_VERSION < '3.0.0'
-    spec.add_development_dependency 'async', '~> 1.30.3'
-  else
-    spec.add_development_dependency 'async'
-  end
+  spec.add_development_dependency 'async'
   spec.add_development_dependency 'execjs'
   spec.add_development_dependency 'rhino'
   spec.add_development_dependency 'sqlite3'
@@ -181,7 +177,7 @@ Gem::Specification.new do |spec|
                       'Testing and Protection.'
   spec.homepage     = 'https://www.contrastsecurity.com'
   spec.license      = 'CONTRAST SECURITY (see license file)'
-  spec.required_ruby_version = ['>= 2.7.0', '< 3.3.0']
+  spec.required_ruby_version = ['>= 3.0.0', '< 3.3.0']
 
   spec.bindir        = 'exe'
   # Keep cs__common first, it handles funchook.h right now.