contrast-agent 7.0.0 → 7.1.0

data/lib/contrast/config/diagnostics/user_configuration_file.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'contrast/config/diagnostics/effective_config_value'
+require 'contrast/config/diagnostics/tools'
+require 'contrast/utils/object_share'
+
+module Contrast
+  module Config
+    module Diagnostics
+      # Reads all config files
+      class UserConfigurationFile
+        # Returns all source values from all configuration files read from the Agent int {path => values} format.
+        #
+        # @return source_values [Hash<String =>Contrast::Config::Diagnostics::SourceConfigValue>]
+        def source_values
+          @_source_values ||= begin
+            sources = {}
+            Contrast::CONFIG.config.origin.source_files.each do |file|
+              config = Contrast::Utils::HashUtils.deep_symbolize_all_keys(file.values)
+              # Assign each source to it's file path
+              sources[file.path] = Contrast::Config::Diagnostics::Tools.
+                  to_config_values(Contrast::Config::Diagnostics::Tools.
+                  flatten_settings(Contrast::Config::Diagnostics::Tools.
+                    value_to_s(config), config: config), source: true)
+            end
+            sources
+          end
+        end
+
+        def to_controlled_hash
+          arr = []
+          source_values.each do |k, v|
+            hsh = {}
+            hsh[:path] = k
+            hsh[:values] = v.map(&:to_source_hash)
+            arr << hsh
+          end
+          arr
+        end
+      end
+    end
+  end
+end

data/lib/contrast/config/request_audit_configuration.rb

@@ -48,7 +48,7 @@ module Contrast
       # Converts current configuration to effective config values class and appends them to
       # EffectiveConfig class.
       #
-      # @param effective_config [Contrast::Agent::DiagnosticsConfig::EffectiveConfig]
+      # @param effective_config [Contrast::Config::Diagnostics::EffectiveConfig]
       def to_effective_config effective_config
         add_effective_config_values(effective_config, CONFIG_VALUES, CANON_NAME, "#{ CONTRAST }.#{ CANON_NAME }")
       end

data/lib/contrast/config/server_configuration.rb

@@ -51,7 +51,7 @@ module Contrast
       # Converts current configuration to effective config values class and appends them to
       # EffectiveConfig class.
       #
-      # @param effective_config [Contrast::Agent::DiagnosticsConfig::EffectiveConfig]
+      # @param effective_config [Contrast::Config::Diagnostics::EffectiveConfig]
       def to_effective_config effective_config
         super
         add_single_effective_value(effective_config,

data/lib/contrast/configuration.rb

@@ -15,6 +15,8 @@ require 'contrast/components/protect'
 require 'contrast/components/assess'
 require 'contrast/components/config/sources'
 require 'contrast/config/server_configuration'
+require 'contrast/config/configuration_files'
+require 'contrast/utils/hash_utils'
 
 module Contrast
   # This is how we read in the local settings for the Agent, both ENV/ CMD line
@@ -57,7 +59,8 @@ module Contrast
     DEFAULT_YAML_PATH  = 'contrast_security.yaml'
     MILLISECOND_MARKER = '_ms'
     CONVERSION = {}.cs__freeze
-    CONFIG_BASE_PATHS = ['', 'config/', '/etc/contrast/ruby/', '/etc/contrast/', '/etc/'].cs__freeze
+    # Precedence of paths, shift if config file values needs to go up the chain.
+    CONFIG_BASE_PATHS = %w[./ config/ /etc/contrast/ruby/ /etc/contrast/ /etc/].cs__freeze
     KEYS_TO_REDACT = %i[api_key url service_key user_name].cs__freeze
     REDACTED = '**REDACTED**'
 
@@ -65,9 +68,7 @@ module Contrast
       @default_name = default_name
 
       # Load config_kv from file
-      config_kv = deep_symbolize_all_keys(load_config)
-      config_sources = assign_source_to(config_kv, Contrast::Components::Config::Sources::YAML)
-
+      config_kv = Contrast::Utils::HashUtils.deep_symbolize_all_keys(load_config)
       unless cli_options
         cli_options = {}
         ENV.each do |key, value|
@@ -76,19 +77,22 @@ module Contrast
           cli_options[key] = value
         end
       end
+
       # Overlay CLI options - they take precedence over config file
-      cli_options = deep_symbolize_all_keys(cli_options)
+      cli_options = Contrast::Utils::HashUtils.deep_symbolize_all_keys(cli_options)
       if cli_options
-        config_kv = deep_merge(cli_options, config_kv)
-        config_sources = deep_merge(assign_source_to(cli_options, Contrast::Components::Config::Sources::CLI),
-                                    config_sources)
+        config_kv = Contrast::Utils::HashUtils.deep_merge(cli_options, config_kv)
+        @_source_file_extensions = Contrast::Utils::HashUtils.
+            deep_merge(assign_source_to(cli_options,
+                                        Contrast::Components::Config::Sources::COMMAND_LINE),
+                       @_source_file_extensions)
       end
 
       # Some in-flight rewrites to maintain backwards compatibility
       config_kv = update_prop_keys(config_kv)
       @loaded_config = config_kv
 
-      @sources = Contrast::Components::Config::Sources.new(config_sources)
+      @sources = Contrast::Components::Config::Sources.new(@_source_file_extensions)
 
       @api = Contrast::Components::Api::Interface.new(config_kv[:api])
       @enable = config_kv[:enable]
@@ -107,6 +111,11 @@ module Contrast
       convert_to_hash.to_yaml
     end
 
+    # @return [Hash] map of all extensions for each config key.
+    def source_file_extensions
+      @_source_file_extensions ||= {}
+    end
+
     # @return [Contrast::Components::Api::Interface]
     def api
       @api ||= Contrast::Components::Api::Interface.new # rubocop:disable Naming/MemoizedInstanceVariableName
@@ -142,27 +151,36 @@ module Contrast
       @protect ||= Contrast::Components::Protect::Interface.new # rubocop:disable Naming/MemoizedInstanceVariableName
     end
 
-    protected
-
-    # TODO: RUBY-546 move utility methods to auxiliary classes
+    # Base paths to check for the contrast configuration file, sorted by
+    # reverse order of precedence (first is most important).
+    def configuration_paths
+      @_configuration_paths ||= begin
+        basename = default_name.split('.').first
+        # Order of extensions comes from here:
+        extensions = Contrast::Components::Config::Sources::APP_CONFIGURATION_FILE.map(&:downcase)
 
-    def load_config
-      config = {}
-      configuration_paths.find do |path|
-        next unless File.exist?(path)
+        paths = []
+        # Environment paths takes precedence here. Look first through them.
+        paths << ENV['CONTRAST_CONFIG_PATH']     if ENV['CONTRAST_CONFIG_PATH']
+        paths << ENV['CONTRAST_SECURITY_CONFIG'] if ENV['CONTRAST_SECURITY_CONFIG']
 
-        unless File.readable?(path)
-          log_file_read_error(path)
-          next
+        extensions.each do |ext|
+          places = CONFIG_BASE_PATHS.product(["#{ basename }.#{ ext }"])
+          paths += places.map!(&:join)
         end
-        config = yaml_to_hash(path) || {}
-        @config_file = path
-        break
+        paths
       end
+    end
 
-      config
+    # List of all read configuration files.
+    #
+    # @return [Contrast::Config::ConfigurationFiles] of paths
+    def origin
+      @_origin ||= Contrast::Config::ConfigurationFiles.new
     end
 
+    protected
+
     def yaml_to_hash path
       if path && File.readable?(path)
         begin
@@ -179,6 +197,45 @@ module Contrast
       {}
     end
 
+    # TODO: RUBY-546 move utility methods to auxiliary classes
+
+    # Read through all the paths we know config may live. Merge all values from all files found.
+    # Priority is given to yaml over yml. If same keys are found on two configs with different extensions,
+    # the Agent will read first from the yaml file and ignore the values for same key on the yml file.
+    #
+    # @return config [Hash] All source configurations from files.
+    def load_config
+      config = {}
+      @_source_file_extensions = {}
+      configuration_paths.each do |path|
+        next unless File.exist?(path)
+
+        unless File.readable?(path)
+          log_file_read_error(path)
+          next
+        end
+        origin.add_source_file(path, (yaml_to_hash(path) || {}))
+      end
+
+      # Legacy usage: Assign main configuration file for reference.
+      @config_file = origin.main_file
+      # merge all settings keeping the top yaml files values as priority.
+      # If in top file a key exists it's value won't be changed if same key has different value in one
+      # of the other config files. Only unique values will be taken in consideration.w
+      # precedence of paths: see Contrast::Configuration::CONFIG_BASE_PATHS
+      extensions_maps = []
+      origin.source_files.each do |file|
+        # config.merge!(file.values) { |_key, oldval, _newval| oldval = oldval }
+        precedence_merge!(config, file.values)
+        # assign source values extentions:
+        extensions_maps << assign_source_to(Contrast::Utils::HashUtils.deep_symbolize_all_keys(file.values), file.path)
+      end
+      # merge all origin paths to be used as extension classification to preserve the precedence of config files:
+      extensions_maps.each { |path| @_source_file_extensions = precedence_merge!(@_source_file_extensions, path) }
+
+      config
+    end
+
     # We're updating properties loaded from the configuration files to match the new agreed upon standard configuration
     # names, so that one file works for all agents
     def update_prop_keys config
@@ -203,39 +260,6 @@ module Contrast
       config
     end
 
-    # Base paths to check for the contrast configuration file, sorted by
-    # reverse order of precedence (first is most important).
-    def configuration_paths
-      @_configuration_paths ||= begin
-        basename = default_name.split('.').first
-        names = %w[yml yaml].map { |suffix| "#{ basename }.#{ suffix }" }
-
-        paths = []
-        paths << ENV['CONTRAST_CONFIG_PATH']     if ENV['CONTRAST_CONFIG_PATH']
-        paths << ENV['CONTRAST_SECURITY_CONFIG'] if ENV['CONTRAST_SECURITY_CONFIG']
-
-        tmp = CONFIG_BASE_PATHS.product(names)
-        paths += tmp.map!(&:join)
-        paths
-      end
-    end
-
-    def deep_merge cli_config, file_config
-      cli_config.merge(file_config) do |_key, cli_value, file_value|
-        cli_value.is_a?(Hash) && file_value.is_a?(Hash) ? deep_merge(cli_value, file_value) : cli_value
-      end
-    end
-
-    def deep_symbolize_all_keys hash
-      return if hash.nil?
-
-      new_hash = {}
-      hash.each do |key, value|
-        new_hash[key.to_sym] = value.is_a?(Hash) ? deep_symbolize_all_keys(value) : value
-      end
-      new_hash
-    end
-
     private
 
     # We cannot use all access components at this point, unfortunately, as they
@@ -333,7 +357,7 @@ module Contrast
       KEYS_TO_REDACT.include?(key.to_sym)
     end
 
-    def assign_source_to hash, source = Contrast::Components::Config::Sources::YAML
+    def assign_source_to hash, source = Contrast::Components::Config::Sources::APP_CONFIGURATION_FILE[0]
       hash.transform_values do |value|
         if value.is_a?(Hash)
           assign_source_to(value, source)
@@ -342,5 +366,14 @@ module Contrast
         end
       end
     end
+
+    # Merges two hashes, first hash will preserve it's values and will only add unique values.
+    #
+    # @param hsh [Hash]
+    # @param other_hsh [Hash]
+    # @return [Hash]
+    def precedence_merge! hsh, other_hsh
+      hsh.merge!(other_hsh) { |_key, old_val, _new_val| old_val }
+    end
   end
 end

data/lib/contrast/utils/hash_utils.rb

@@ -0,0 +1,43 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+module Contrast
+  module Utils
+    # Module to hold various hash utils methods
+    module HashUtils
+      class << self
+        # Merges two hashes, first hash will preserve it's values and will only add unique values.
+        #
+        # @param hsh [Hash]
+        # @param other_hsh [Hash]
+        def deep_merge hsh, other_hsh
+          hsh.merge(other_hsh) do |_key, old_value, new_value|
+            old_value.is_a?(Hash) || new_value.is_a?(Hash) ? deep_merge(old_value, new_value) : old_value
+          end
+        end
+
+        # Deep symbolizes all keys
+        # @param value [Hash]
+        # @return [Hash]
+        def deep_symbolize_all_keys value
+          new_hash = {}
+          value.each { |key, v| new_hash[key.to_sym] = map_value(v) }
+          new_hash
+        end
+
+        private
+
+        def map_value value
+          case value
+          when Hash
+            deep_symbolize_all_keys(value)
+          when Array
+            value.map { |v| map_value(v) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/contrast/utils/json.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2023 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# frozen_string_literal: true
+
+require 'json'
+require 'contrast/utils/hash_utils'
+require 'contrast/components/logger'
+
+module Contrast
+  module Utils
+    # Module to hold Agent's custom JSON.parse logic.
+    module Json
+      class << self
+        include Contrast::Components::Logger::InstanceMethods
+
+        # Add any known cases where parsing error might arise from older json parser:
+        # @return [Array<String>]
+        SPECIAL_CASES = ["\"\""].cs__freeze # rubocop:disable Style/StringLiterals
+
+        # Parses a string using JSON.parser. This method is used instead of standard JSON.parse to
+        # support older versions of json gem => not supporting key-value second parameter, which is
+        # supported after json 2.3.0.
+        #
+        # @param string [String]
+        # @param deep_symbolize [Boolean] flag to set if keys needs to be deep symbolized.
+        # @return [Hash]
+        def parse string, deep_symbolize: false
+          # The Agent receives empty responses from TS sometimes.
+          # There is a special case to handle this.
+          return {} if SPECIAL_CASES.include?(string)
+
+          symbolized_hash = {}
+          hash = JSON::Parser.new(string).parse
+          symbolized_hash = Contrast::Utils::HashUtils.deep_symbolize_all_keys(hash) if deep_symbolize
+          return hash unless deep_symbolize
+
+          symbolized_hash
+        rescue JSON::ParserError => e
+          # Any parsing error will produce empty {}. Since older JSON might pose support issues with newer Ruby,
+          # We might just log any miss-parsings and allow Agent to continue.
+          logger.warn("[JSON] parse error: #{ e }")
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/contrast/utils/net_http_base.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'net/http'
+require 'resolv'
 require 'contrast/components/logger'
 require 'contrast/utils/object_share'
 require 'socket'
@@ -10,8 +11,25 @@ module Contrast
   module Utils
     # This module creates a Net::HTTP client base to be used by different services
     # All HTTP clients reporting to Telemetry or TS should inherit from this
-    class NetHttpBase
+    class NetHttpBase # rubocop:disable Metrics/ClassLength
       include Contrast::Components::Logger::InstanceMethods
+
+      class << self
+        # Last recorded error
+        # @return [StandardError]
+        def last_error
+          @_last_error
+        end
+
+        # @param [StandardError]
+        def last_error= error
+          @_last_error = error
+        end
+      end
+
+      # @return [String]
+      attr_reader :client_name
+
       # This method initializes the Net::HTTP client we'll need. it will validate
       # the connection and make the first request. If connection is valid and response
       # is available then the open connection is returned.
@@ -23,29 +41,22 @@ module Contrast
       # self signed certificates provided by config [default = false]
       # @return [Net::HTTP, nil] Return open connection or nil
       def initialize_connection service_name, url, use_proxy: false, use_custom_cert: false
-        return unless url
-
-        addr = URI(url)
-        return if addr.host.nil? || addr.port.nil?
-        return if addr.scheme != 'https' && !addr.host.to_s.include?('localhost')
-
-        # the proxy is enabled only if there is provided url even if the enable is set to true
-        proxy_addr = URI(Contrast::API.proxy_url) if proxy_enabled?
-        net_http_client = initialize_client(addr, proxy_addr, use_proxy, use_custom_cert)
-        return if net_http_client.nil?
-
-        net_http_client.start
-        return unless net_http_client.started?
+        Contrast::Utils::NetHttpBase.last_error = nil
+        @client_name = service_name
+        return unless (addr = retrieve_address(url))
+        return unless (net_http_client = configure_new_client(addr, use_proxy, use_custom_cert))
+        return unless client_started?(net_http_client)
 
-        logger.debug("Starting #{ service_name } connection test")
+        logger.debug("Starting #{ client_name } connection test")
         return unless connection_verified?(net_http_client, url)
 
-        logger.debug('Client verified', service: service_name, url: url)
+        logger.debug('Client verified', service: client_name, url: url)
         net_http_client
       rescue StandardError => e
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug('Connection failed', e, service: service_name, url: url)
+        Contrast::Utils::NetHttpBase.last_error = e
+        return if client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+
+        logger.error('Connection failed', e, service: client_name, url: url)
         nil
       end
 
@@ -74,14 +85,51 @@ module Contrast
              Errno::ETIMEDOUT, Errno::ESHUTDOWN, Errno::EHOSTDOWN, Errno::EHOSTUNREACH, Errno::EISCONN,
              Errno::ECONNABORTED, Errno::ENETRESET, Errno::ENETUNREACH => e
 
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug("#{ service_name } connection failed", e.message)
-        false
+        Contrast::Utils::NetHttpBase.last_error = e
+        unless client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+          logger.error("#{ client_name } connection failed", e.message)
+        end
+        @_connection_verified = false
       end
 
       private
 
+      # Starts connection and return started status.
+      #
+      # @param client [Net::HTTP] client instance.
+      # @return [Boolean] status indicates whether connection has started.
+      def client_started? client
+        return false unless client
+
+        client.start
+        client.started?
+      end
+
+      # @param url
+      # @return [URI::Generic, nil]
+      def retrieve_address url
+        return unless (addr = URI(url))
+        return if addr.host.nil? || addr.port.nil?
+        return if addr.scheme != 'https' && !addr.host.to_s.include?('localhost')
+
+        addr
+      end
+
+      # Assigns proxy and custom certificates if enabled, and initializes new client.
+      #
+      # @param addr [URI::Generic, nil]
+      # @param use_proxy [Boolean] flag used to indicate proxy connections [default = false]
+      # @param use_custom_cert [Boolean] flag used to indicate whether the client is to use
+      # @return client [Net::HTTP, nil] initialized client.
+      def configure_new_client addr, use_proxy, use_custom_cert
+        # the proxy is enabled only if there is provided url even if the enable is set to true
+        proxy_addr = URI(Contrast::API.proxy_url) if proxy_enabled?
+        net_http_client = initialize_client(addr, proxy_addr, use_proxy, use_custom_cert)
+        return if net_http_client.nil?
+
+        net_http_client
+      end
+
       # Resolves the address of the assigned domain to array of corresponding IPs (if more than one)
       # and runs a matcher to see if current connection IP is in the list.
       # This is called within #verify_connection, if called on it's own there will be no
@@ -114,9 +162,10 @@ module Contrast
           client.key = OpenSSL::PKey::RSA.new(File.read(Contrast::API.certification_key_file)).to_s
         end
       rescue Errno::ENOENT => e
-        # TODO: RUBY-2033 we cannot log the error above debug level here b/c it results in
-        #   an infinite loop w/ telemetry
-        logger.debug('Custom certificates failed', e.message)
+        Contrast::Utils::NetHttpBase.last_error = e
+        unless client_name == Contrast::Agent::Telemetry::Client::SERVICE_NAME
+          logger.error('Custom certificates failed', e.message)
+        end
       end
 
       # sets default setting for client validation of certificates and

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: contrast-agent
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.1.0
 platform: ruby
 authors:
 - galen.palmer@contrastsecurity.com
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-04-03 00:00:00.000000000 Z
+date: 2023-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -1079,6 +1079,7 @@ files:
 - lib/contrast/agent/reporting/masker/masker_utils.rb
 - lib/contrast/agent/reporting/report.rb
 - lib/contrast/agent/reporting/reporter.rb
+- lib/contrast/agent/reporting/reporting_events/agent_effective_config.rb
 - lib/contrast/agent/reporting/reporting_events/agent_startup.rb
 - lib/contrast/agent/reporting/reporting_events/application_activity.rb
 - lib/contrast/agent/reporting/reporting_events/application_defend_activity.rb
@@ -1224,11 +1225,17 @@ files:
 - lib/contrast/config/api_proxy_configuration.rb
 - lib/contrast/config/base_configuration.rb
 - lib/contrast/config/certification_configuration.rb
-- lib/contrast/config/config.rb
-- lib/contrast/config/diagnostics.rb
-- lib/contrast/config/diagnostics_tools.rb
-- lib/contrast/config/effective_config.rb
-- lib/contrast/config/effective_config_value.rb
+- lib/contrast/config/configuration_files.rb
+- lib/contrast/config/diagnostics/command_line.rb
+- lib/contrast/config/diagnostics/config.rb
+- lib/contrast/config/diagnostics/contrast_ui.rb
+- lib/contrast/config/diagnostics/effective_config.rb
+- lib/contrast/config/diagnostics/effective_config_value.rb
+- lib/contrast/config/diagnostics/environment_variables.rb
+- lib/contrast/config/diagnostics/monitor.rb
+- lib/contrast/config/diagnostics/source_config_value.rb
+- lib/contrast/config/diagnostics/tools.rb
+- lib/contrast/config/diagnostics/user_configuration_file.rb
 - lib/contrast/config/env_variables.rb
 - lib/contrast/config/exception_configuration.rb
 - lib/contrast/config/protect_rule_configuration.rb
@@ -1297,12 +1304,14 @@ files:
 - lib/contrast/utils/findings.rb
 - lib/contrast/utils/hash_digest.rb
 - lib/contrast/utils/hash_digest_extend.rb
+- lib/contrast/utils/hash_utils.rb
 - lib/contrast/utils/head_dump_utils_extend.rb
 - lib/contrast/utils/heap_dump_util.rb
 - lib/contrast/utils/input_classification_base.rb
 - lib/contrast/utils/invalid_configuration_util.rb
 - lib/contrast/utils/io_util.rb
 - lib/contrast/utils/job_servers_running.rb
+- lib/contrast/utils/json.rb
 - lib/contrast/utils/log_utils.rb
 - lib/contrast/utils/lru_cache.rb
 - lib/contrast/utils/metrics_hash.rb