contrast-agent 3.8.4 → 6.9.0

data/lib/contrast/agent/middleware.rb

@@ -1,351 +1,198 @@
-# Copyright (c) 2020 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
 # frozen_string_literal: true
 
-cs__scoped_require 'ipaddr'
-cs__scoped_require 'json'
-cs__scoped_require 'rack'
-cs__scoped_require 'contrast/core_extensions/object'
-cs__scoped_require 'contrast/security_exception'
-cs__scoped_require 'contrast/utils/object_share'
-cs__scoped_require 'contrast/utils/gemfile_reader'
-cs__scoped_require 'contrast/utils/operating_environment'
-cs__scoped_require 'contrast/agent/service_heartbeat'
-cs__scoped_require 'contrast/components/interface'
-cs__scoped_require 'contrast/utils/heap_dump_util'
-
-cs__scoped_require 'contrast/utils/path_util'
-cs__scoped_require 'contrast/utils/timer'
-cs__scoped_require 'contrast/utils/freeze_util'
-cs__scoped_require 'contrast/utils/service_sender_util'
-cs__scoped_require 'contrast/utils/service_response_util'
+require 'ipaddr'
+require 'json'
+require 'rack'
+
+require 'contrast/security_exception'
+require 'contrast/utils/object_share'
+require 'contrast/components/logger'
+require 'contrast/components/scope'
+require 'contrast/utils/heap_dump_util'
+require 'contrast/utils/telemetry'
+require 'contrast/agent/request_handler'
+require 'contrast/agent/static_analysis'
+require 'contrast/agent/telemetry/events/startup_metrics_event'
+require 'contrast/utils/middleware_utils'
+require 'contrast/utils/reporting/application_activity_batch_utils'
+require 'contrast/utils/timer'
 
 module Contrast
   module Agent
-    # This class allows the Agent to plug into the Rack middleware stack,
-    # providing hooks to relevant application events, such as request start and
-    # application code.
+    # This class allows the Agent to plug into the Rack middleware stack. When the application is first started, we
+    # initialize ourselves as a rack middleware inside of #initialize. Afterwards, we process each http request and
+    # response as it goes through the middleware stack inside of #call.
     class Middleware
-      include Contrast::Components::Interface
-      access_component :contrast_service, :logging, :agent, :analysis, :config, :scope
+      include Contrast::Components::Logger::InstanceMethods
+      include Contrast::Components::Scope::InstanceMethods
+      include Contrast::Utils::MiddlewareUtils
+      include Contrast::Utils::Reporting::ApplicationActivityBatchUtils
 
       attr_reader :app
 
-      LOG_DEBUG_MIDDLEWARE_ENV     = 'middleware: log environment'
-      LOG_DEBUG_MIDDLEWARE_LIB     = 'middleware: instrument shared libraries'
-      LOG_DEBUG_MIDDLEWARE_START   = 'middleware: startup agent'
-      LOG_DEBUG_MIDDLEWARE_SERVICE = 'middleware: starting service'
-
-      # allows the Agent to function as a middleware
+      # Allows the Agent to function as a middleware. We perform all our one-time whole-app routines in here since
+      # we're only going to be initialized a single time. Our initialization order is:
+      # - capture the application
+      # - setup the Agent
+      # - startup the Agent
       #
       # @param app [Rack::Application] the application to be instrumented
       # @param _legacy_param [nil] was a flag we no longer need, but Sinatra may call it
       def initialize app, _legacy_param = nil
-        @app = app
-
-        # TODO: RUBY-545 nomenclature here needs to be updated.
-        #   enabled/initialized are really only reflective of whether we were
-        #   able to parse the config, which is the bare minimum for the agent
-        #   to do anything.
-        unless AGENT.enabled?
-          logger.error('Contrast middleware initializer detected an early-stage setup failure (likely config parse).  Disabling.')
-          # ensure the agent is disabled (probably redundant)
-          AGENT.disable!
+        @app = app  # THIS MUST BE FIRST AND ALWAYS SET!
+        setup_agent # THIS MUST BE SECOND AND ALWAYS CALLED!
+        unless ::Contrast::AGENT.enabled?
+          logger.error('The Agent was unable to initialize before the application middleware was initialized. '\
+                       'Disabling permanently.')
+          ::Contrast::AGENT.disable! # ensure the agent is disabled (probably redundant)
           return
         end
-
-        logger.debug_with_time(LOG_DEBUG_MIDDLEWARE_ENV) do
-          settings.log_environment
-          settings.log_configuration
-          settings.log_specific_libraries
-          settings.log_all_libraries
-        end
-
-        # Initialization order should be:
-        # - start separate service sending threads
-        # - start heartbeat thread, which triggers service startup
-        # - start patching to achieve instrumentation
-
-        logger.debug_with_time(LOG_DEBUG_MIDDLEWARE_SERVICE) do
-          # get threads ready to poll for messages on the queue
-          run_service_sender_thread
-
-          # sends first message to service, which triggers service startup
-          run_service_heartbeat_thread
-        end
-
-        # Default instrumentation
-        logger.debug_with_time(LOG_DEBUG_MIDDLEWARE_LIB) do
-          AGENT.run_instrumentation
-        end
-
-        Contrast::Agent::AtExitHook.exit_hook
-      end
-
-      def settings
-        Contrast::Agent::FeatureState.instance
+        agent_startup_routine
+      rescue StandardError => e
+        logger.error('Unable to initialize the agent. Disabling permanently.', e)
+        ::Contrast::AGENT.disable! # ensure the agent is disabled (probably redundant)
       end
 
-      LOG_DEBUG_MIDDLEWARE_START_CALL = 'middleware: startup agent (call)'
-      # def _call env
+      # This is where we're hooked into the middleware stack. If the agent is enabled, we're ready to do some
+      # processing on a per request basis. If not, we just pass the request along to the next middleware in the stack.
+      #
+      # @param env [Hash] the various variables stored by this and other Middlewares to know the state and values of
+      #   this Request
+      # @return [Array,Rack::Response] the Response of this and subsequent Middlewares to be passed back to the user up
+      #   the Rack framework.
       def call env
-        Contrast::Utils::HeapDumpUtil.run
+        logger.trace_with_time('Elapsed time for Contrast::Agent::Middleware#call') do
+          ::Contrast::Agent::ThreadWatcher.check_before_start
+          return app.call(env) unless ::Contrast::AGENT.enabled?
 
-        if AGENT.enabled?
-          response = call_with_agent(env)
-          with_contrast_scope { do_static_analysis_catchup }
-          response
-        else
-          call_without_agent(env)
+          Contrast::Agent.heapdump_util.start_thread!
+          handle_first_request
+          call_with_agent(env)
         end
       end
 
-      def call_without_agent env
-        app.call(env)
-      end
+      private
 
-      REQUEST_PATH = 'REQUEST_PATH'
-      LOG_DEBUG_REQUEST = 'HTTP request cycle'
-      def call_with_agent env
-        rack_request = generate_request(env)
-
-        context = Contrast::Agent::RequestContext.new(rack_request, true) # 'true' here is legacy, by the time we're here, we're ready
-
-        # default response
-        streaming = false
-        # make the context available for the lifecycle of this request
-        Contrast::Agent::REQUEST_TRACKER.lifespan(context) do
-          # record entire time
-          logger.debug_with_time(LOG_DEBUG_REQUEST) do
-            # process filters that can short circuit application processing
-            with_contrast_scope do
-              context.service_extract_request
-              prefilter(context)
-            end
-
-            # run application by passing request down the Rack chain using
-            # the original env
-            response = application_code(env)
-
-            # if streaming, allow for early return with application response
-            streaming = possibly_streaming?(env)
-            if streaming
-              with_contrast_scope { postfilter(context, streaming) }
-              return response
-            end
-
-            # update context with final response information
-            context.extract_after(response)
+      # Startup the Agent as part of the initialization process:
+      # - start the TeamServer sending thread, responsible for sending and processing messages
+      # - start the heartbeat thread, which handles periodic messages to TeamServer
+      # - start instrumenting libraries and do a 'catchup' patch for everything we didn't see get loaded
+      # - enable TracePoint, which handles all class loads and required instrumentation going forward
+      def agent_startup_routine
+        logger.debug_with_time('middleware: starting reporting threads') do
+          Contrast::Agent.thread_watcher.ensure_running?
+        end
 
-            # process filters that look at response headers and body
-            with_contrast_scope { postfilter(context) }
+        if Contrast::Agent::Telemetry::Base.enabled?
+          logger.debug_with_time('middleware: sending startup metrics telemetry event') do
+            event = Contrast::Agent::Telemetry::StartupMetricsEvent.new
+            Contrast::Agent.thread_watcher.telemetry_queue.send_event(event)
           end
         end
 
-        # return the response stored in the context in case any postfilter rules
-        # updated the response data
-        context&.response&.rack_response || response
-
-      # handle security exception
-      rescue StandardError => e
-        handle_exception(e)
-      ensure
-        begin
-          handle_ensure(context, streaming)
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          logger.error(e, 'Exception raised while flushing messages to Contrast service!')
-          raise
+        logger.debug_with_time('middleware: instrument shared libraries and patch') do
+          Contrast::Agent::Patching::Policy::Patcher.patch
         end
-      end
-
-      LOG_WARN_FRAMEWORK_PARSE = 'Unable to generate framework specific request - falling back to Rack'
-      # Given the rack environment of this call, generate a framework specific
-      # request object to bind to this context. In the event that multiple
-      # supported frameworks are defined OR we cannot determine the framework
-      # currently in use or there is an exception during request generation, we
-      # will fall back on Rack::Request object
-      def generate_request env
-        rails_defined = defined?(Rails)
-        sinatra_defined = defined?(Sinatra) && defined?(Sinatra::Request)
 
-        if rails_defined && !sinatra_defined
-          # code from /lib/action_cable/connection/base
-          environment = Rails.application.env_config.merge(env) if defined?(Rails.application) && Rails.application
-          ActionDispatch::Request.new(environment || env)
-        # !defined? currently redundant, won't be if we have more frameworks
-        elsif sinatra_defined && !rails_defined
-          Sinatra::Request.new(env)
-        else # many OR none
-          Rack::Request.new(env)
+        logger.debug_with_time('middleware: enabling tracepoint') do
+          ::Contrast::AGENT.enable_tracepoint
         end
-      rescue StandardError => e
-        logger.warn(e, LOG_WARN_FRAMEWORK_PARSE)
-        Rack::Request.new(env)
-      end
-
-      LOG_ERROR_STREAM_CHECK = 'Unable to check for streaming'
-      ACTION_CONTROLLER_INSTANCE = 'action_controller.instance'
-      # First check to see if an action could potentially be streaming a response
-      def possibly_streaming? env
-        return false unless defined?(ActionController::Live)
-
-        env[ACTION_CONTROLLER_INSTANCE].cs__class.included_modules.include?(ActionController::Live)
-      rescue StandardError => e
-        logger.warn(LOG_ERROR_STREAM_CHECK, e)
+        Contrast::Agent::AtExitHook.exit_hook
       end
 
-      LOG_ERROR_ENSURE = 'Context not defined in middleware ensure.'
-      # Send a server activity message and an application activity message
-      # and return response
-      def handle_ensure context, streaming
-        if context
-          logger.debug('Middleware request lifecycle complete; flushing context activity to Contrast service.')
-          Contrast::Utils::GemfileReader.instance.generate_library_usage(context.activity)
-          [context.server_activity, context.activity, context.observed_route].each do |message|
-            CONTRAST_SERVICE.send_message message
-          end
-        else
-          logger.error(LOG_ERROR_ENSURE)
+      # Some things have to wait until first request to happen, either because resolution is not complete or because
+      # the framework will preload classes, which confuses some of our instrumentation.
+      def handle_first_request
+        @_handle_first_request ||= begin
+          Contrast::Agent::StaticAnalysis.catchup
+          true
         end
-        return unless streaming
-
-        context.reset_activity
       end
 
-      SECURITY_EXCEPTION_MARKER = 'Contrast::SecurityException'
-      LOG_RETHROW_ERROR = 'Re-throwing original error'
-      # We're only going to suppress SecurityExceptions indicating a blocked attack.
-      # And, only if the config.agent.ruby.exceptions.capture? is set
-      def handle_exception exception
-        if exception.is_a?(Contrast::SecurityException) ||
-              exception.message&.include?(SECURITY_EXCEPTION_MARKER)
-
-          exception_control = Contrast::Agent::FeatureState.instance.exception_control
-          raise exception unless exception_control[:enable]
+      # This is where we process each request we intercept as a middleware. We make the request context available
+      # globally so that it can be accessed from anywhere. A RequestHandler object is made for each request, which
+      # handles prefilter and postfilter operations.
+      # @param env [Hash] the various variables stored by this and other Middlewares to know the state and values of
+      #   this Request
+      # @return [Array,Rack::Response] the Response of this and subsequent Middlewares to be passed back to the user
+      #   up the Rack framework.
+      def call_with_agent env
+        Contrast::Agent.thread_watcher.ensure_running?
+        framework_request = Contrast::Agent.framework_manager.retrieve_request(env)
+        context = Contrast::Agent::RequestContext.new(framework_request)
+        response = nil
 
-          [exception_control[:status], {}, [exception_control[:message]]]
-        else
-          # Log original re-raise
-          logger.debug(exception, LOG_RETHROW_ERROR)
-          raise exception
+        # Make the context available for the lifecycle of this request.
+        Contrast::Agent::REQUEST_TRACKER.lifespan(context) do
+          logger.request_start
+          request_handler = Contrast::Agent::RequestHandler.new(context)
+
+          pre_call_with_agent(context, request_handler)
+          response = application_code(env)
+          post_call_with_agent(context, env, request_handler, response)
+        ensure
+          logger.request_end
         end
-      end
-
-      LOG_DEBUG_PREFILTER = 'prefilter'
-      LOG_ERROR_PREFILTER = 'Unexpected exception during prefilter'
-      # Iterate through rules that only depend upon the request object.
-      def prefilter context
-        return unless context.app_loaded?
 
-        logger.debug_with_time(LOG_DEBUG_PREFILTER) do
-          prefilter_assess(context) if ASSESS.enabled? && context.analyze_request?
-          prefilter_protect(context) if PROTECT.enabled?
-        end
-      rescue Contrast::SecurityException => e
-        logger.warn("RASP threw security exception in prefilter: exception=#{ e.message }")
-        raise e
+        response
       rescue StandardError => e
-        logger.error(LOG_ERROR_PREFILTER, e)
-      end
-
-      def prefilter_assess context
-        rules = ASSESS.rules
-        logger.debug("Assess: Running #{ rules.length } rules in prefilter.")
-        prefilter_rules(rules, context)
-      end
-
-      def prefilter_protect context
-        rules = PROTECT.rules
-        logger.debug("Protect: Running #{ rules.length } rules in prefilter.")
-        prefilter_rules(rules, context)
-      end
-
-      def prefilter_rules rules, context
-        rules.each do |_, rule|
-          rule.prefilter(context)
-        end
-      end
-
-      LOG_DEBUG_APPLICATION = 'application'
-      # Run the next level of the Rack stack as normal
-      def application_code env
-        logger.debug_with_time(LOG_DEBUG_APPLICATION) do
-          app.call(env)
-        end
-      rescue Contrast::SecurityException => e
-        logger.info("RASP threw security exception in application code: exception=#{ e.message }")
-        raise e
+        handle_exception(e)
       end
 
-      LOG_DEBUG_POSTFILTER = 'postfilter'
-      LOG_ERROR_POSTFILTER = 'Unexpected exception during postfilter'
-      # Iterate through rules that depend on the full response object
-      def postfilter context, streaming = false
-        return unless context.app_loaded?
-
-        logger.debug_with_time(LOG_DEBUG_POSTFILTER) do
-          if ASSESS.enabled? && context.analyze_response?
-            logger.debug("Assess:\tRunning #{ ASSESS.rules.length } rules in postfilter.")
-            postfilter_rules(ASSESS.rules, context, streaming)
-          end
-
-          if PROTECT.enabled?
-            logger.debug("Protect:\tRunning #{ PROTECT.rules.length } rules in postfilter.")
-            postfilter_rules(PROTECT.rules, context, streaming)
-          end
+      # Handle the operations the Agent needs to accomplish prior to the Application code executing during this
+      # request.
+      #
+      # @param context [Contrast::Agent::RequestContext]
+      # @param request_handler [Contrast::Agent::RequestHandler]
+      # @raise [StandardError] raises an error if the exception is security concern
+      #    which is being triggered when there is a failure within the pre-call with the agent
+      def pre_call_with_agent context, request_handler
+        with_contrast_scope do
+          context.protect_input_analysis
+          request_handler.ruleset.prefilter
         end
-      rescue Contrast::SecurityException => e
-        logger.warn("RASP threw security exception: exception=#{ e.message }")
-        raise e
       rescue StandardError => e
-        logger.error(LOG_ERROR_POSTFILTER, e)
+        raise(e) if security_exception?(e)
+
+        logger.error('Unable to execute agent pre_call', e)
       end
 
-      # Iterate through a list of rules with the current context and the full response body
-      def postfilter_rules rules, context, streaming
-        rules.each do |_, rule|
-          if !streaming || rule.stream_safe?
-            rule.postfilter(context)
+      # Handle the operations the Agent needs to accomplish after the Application code executes during this request.
+      #
+      # @param context [Contrast::Agent::RequestContext]
+      # @param env [Hash] the various variables stored by this and other Middlewares to know the state and values of
+      #   this Request
+      # @param request_handler [Contrast::Agent::RequestHandler]
+      # @param response [Array,Rack::Response]
+      # @raise [StandardError] raises an error if the exception is security concern
+      #    which is being triggered when there is a failure within the post-call with the agent
+      def post_call_with_agent context, env, request_handler, response
+        with_contrast_scope do
+          context.extract_after(response) # update context with final response information
+
+          # Build and report all collected findings prior response
+          Contrast::Agent::FINDINGS.report_collected_findings unless Contrast::Agent::FINDINGS.collection.empty?
+          # All protect rules, which are trigger but require response to be reported
+          Contrast::Agent::EXPLOITS.report_recorded_exploits(context) unless Contrast::Agent::EXPLOITS.collection.empty?
+          # Process Worth Watching Inputs for v2 rules
+          Contrast::Agent.worth_watching_analyzer&.add_to_queue(context.agent_input_analysis)
+
+          if Contrast::Agent.framework_manager.streaming?(env)
+            context.reset_activity
+            request_handler.stream_safe_postfilter
           else
-            logger.debug("Skipping rule: #{ rule.name } in streamed response.")
+            request_handler.ruleset.postfilter
+            request_handler.report_observed_route
+            add_activity_to_batch(context.activity)
+            report_batch
           end
         end
-      end
-
-      def run_service_heartbeat_thread
-        # Rspec stubs over this method for simplicity's sake in testing.
-        # Take care if you refactor this back into #initialize.
-        Contrast::Agent::ServiceHeartbeat.new.start
-      end
-
-      def run_service_sender_thread
-        # Rspec stubs over this method for simplicity's sake in testing.
-        # Take care if you refactor this back into #initialize.
-        Contrast::Utils::ServiceSenderUtil.start
-      end
-
-      LOG_DEBUG_MW_INV = 'middleware: send inventory'
-      LOG_WARN_STATIC_ANALYSIS = 'Unable to run post-initialization static analysis'
-      # this is memoized, should only be meaningful after the first agented
-      # HTTP request
-      def do_static_analysis_catchup
-        @_do_static_analysis_catchup ||= begin
-                                           # Everything in here should be asynchronous, as we need to return
-                                           # the HTTP response from middleware ASAP.
-
-                                           # Review already-loaded inventory
-                                           logger.debug_with_time('initializer: report loaded gemset') do
-                                             Contrast::Utils::GemfileReader.instance.map_loaded_classes
-                                           end
-
-                                           # Report already-loaded inventory
-                                           logger.debug_with_time(LOG_DEBUG_MW_INV) do
-                                             settings.send_inventory_message
-                                           end
-
-                                           true
-                                         end
+      # unsuccessful attack
       rescue StandardError => e
-        logger.warn(LOG_WARN_STATIC_ANALYSIS, e)
+        raise(e) if security_exception?(e)
+
+        logger.error('Unable to execute agent post_call', e)
       end
     end
   end

data/lib/contrast/agent/module_data.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
 # frozen_string_literal: true
 
 module Contrast
@@ -6,10 +6,11 @@ module Contrast
     # A simple wrapper around a Module and a call to its name, used to avoid
     # calling the Module#name method and generating extra Strings
     class ModuleData
-      attr_accessor :mod, :name
-      def initialize mod, name = nil
+      attr_reader :mod, :mod_name
+
+      def initialize mod, mod_name = nil
         @mod = mod
-        @name = name || mod.cs__name
+        @mod_name = mod_name || mod.cs__name
       end
     end
   end

data/lib/contrast/agent/patching/policy/after_load_patch.rb

@@ -1,35 +1,82 @@
-# Copyright (c) 2020 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
+# Copyright (c) 2022 Contrast Security, Inc. See https://www.contrastsecurity.com/enduser-terms-0317a for more details.
 # frozen_string_literal: true
 
+require 'contrast/components/scope'
+require 'contrast/extension/module'
+require 'contrast/utils/class_util'
+
 module Contrast
   module Agent
     module Patching
       module Policy
         # Used to handle tracking patches that need to apply special instrumentation when a module is loaded
         class AfterLoadPatch
-          attr_reader :applied, :module_name, :instrumentation_file_path
-          def initialize module_name, instrumentation_file_path
+          include Contrast::Components::Scope::InstanceMethods
+
+          attr_reader :applied, :module_name, :instrumentation_file_path, :method_to_instrument, :instrumenting_module
+
+          def initialize module_name, instrumentation_file_path, method_to_instrument: nil, instrumenting_module: nil
             @applied = false
             @module_name = module_name
+            @method_to_instrument = method_to_instrument
             @instrumentation_file_path = instrumentation_file_path
+            @instrumenting_module = instrumenting_module
           end
 
           def applied?
-            @applied
+            applied
+          end
+
+          # Modules can be re-opened, so the first load may not
+          # necessarily define the method we're looking for:
+          #
+          # patching MyMod#instrumentable:
+          #
+          # file1:
+          #   module MyMod
+          #     def unrelated        <-- false lead
+          #     end
+          #   end
+          #
+          # file2:
+          #   module MyMod
+          #     def instrumentable   <-- actual target
+          #     end
+          #   end
+          def blocked_by_method?
+            return true  unless target_defined? # bc no methods are loaded
+            return false unless method_to_instrument
+
+            !module_lookup.instance_methods.include?(method_to_instrument)
           end
 
           def applies? loaded_module_name
-            loaded_module_name == module_name
+            (loaded_module_name == module_name) && !blocked_by_method?
           end
 
           def target_defined?
-            Module.cs__truly_defined?(module_name)
+            Contrast::Utils::ClassUtil.truly_defined?(module_name)
           end
 
           def instrument!
-            cs__scoped_require instrumentation_file_path
+            require(instrumentation_file_path)
+
+            if instrumenting_module
+              mod = Module.cs__const_get(instrumenting_module)
+              with_contrast_scope { mod.instrument } if mod
+            end
             @applied = true
           end
+
+          private
+
+          def module_lookup
+            @_module_lookup ||= begin
+              Module.cs__const_get(module_name)
+            rescue StandardError => _e
+              nil
+            end
+          end
         end
       end
     end