logstruct 0.1.11 → 0.2.0

data/lib/log_struct/integrations/rack_error_handler/error_lines.rb

@@ -0,0 +1,160 @@
+# typed: strict
+# frozen_string_literal: true
+
+require_relative "../../params_errors"
+
+module LogStruct
+  module Integrations
+    module RackErrorHandler
+      # The structured lines for errors raised while Rails handles a request. Each error is logged once:
+      # security errors from the app by DebugExceptions' interceptor, anything else that gets past
+      # ShowExceptions by the middleware. Rails still turns each error into its usual response.
+      module ErrorLines
+        extend T::Sig
+
+        CSRF_ERROR_NAMES = T.let(
+          ["ActionController::InvalidAuthenticityToken", "ActionController::InvalidCrossOriginRequest"].freeze,
+          T::Array[String]
+        )
+
+        sig { returns(T::Boolean) }
+        def self.enabled?
+          LogStruct.enabled? && LogStruct.config.integrations.enable_rack_error_handler
+        end
+
+        # For DebugExceptions.register_interceptor: CSRF and IP spoofing errors from the app, which
+        # ShowExceptions would otherwise turn into a response before any middleware saw them.
+        # Other errors are left to Rails' own logging and error reporting.
+        sig { params(request: ::ActionDispatch::Request, error: Exception).void }
+        def self.intercept(request, error)
+          return unless enabled?
+          return unless security_error?(error)
+          return if LoggedErrors.logged?(error)
+
+          log_security_error(request, error)
+        end
+
+        # An error that got past ShowExceptions (or came from a middleware outside it)
+        sig { params(request: ::ActionDispatch::Request, error: StandardError).void }
+        def self.log_escaped(request, error)
+          return unless enabled?
+          return if LoggedErrors.logged?(error)
+          return log_security_error(request, error) if security_error?(error)
+
+          LoggedErrors.mark(error)
+          LogStruct.error(
+            Log::Error.new(
+              source: Source::Rails,
+              error_class: error.class,
+              message: ParamsErrors.messages(error).first,
+              backtrace: error.backtrace,
+              path: request.path,
+              http_method: request.method,
+              source_ip: source_ip(request),
+              user_agent: request.user_agent,
+              referer: StringScrubber.without_query(request.referer)
+            )
+          )
+        end
+
+        # An error the app didn't handle, which DebugExceptions logs before ShowExceptions renders Rails'
+        # error page: at config.action_dispatch.debug_exception_log_level (fatal by default), with the
+        # trace Rails would log and the response's status. A security error already has its own line.
+        sig { params(request: ::ActionDispatch::Request, wrapper: T.untyped).void }
+        def self.log_unhandled(request, wrapper)
+          error = T.cast(wrapper.exception, Exception)
+          return if LoggedErrors.logged?(error)
+
+          LoggedErrors.mark(error)
+          # Without a backtrace cleaner, Rails' trace is Thread::Backtrace::Locations
+          trace = Array(wrapper.exception_trace).map(&:to_s)
+          message, cause_message = ParamsErrors.messages(error)
+          line = Log::Error.new(
+            source: Source::Rails,
+            error_class: error.class,
+            message: message,
+            backtrace: trace.empty? ? error.backtrace : trace,
+            path: request.path,
+            http_method: request.method,
+            source_ip: source_ip(request),
+            user_agent: request.user_agent,
+            referer: StringScrubber.without_query(request.referer),
+            status: wrapper.status_code,
+            cause_class: error.cause&.class&.name,
+            cause_message: cause_message
+          )
+          case request.get_header("action_dispatch.debug_exception_log_level")
+          when ::Logger::DEBUG then LogStruct.debug(line)
+          when ::Logger::INFO then LogStruct.info(line)
+          when ::Logger::WARN then LogStruct.warn(line)
+          when ::Logger::ERROR then LogStruct.error(line)
+          else LogStruct.fatal(line)
+          end
+        end
+
+        # A security line, and the error handled in the security_errors mode (reported by default)
+        sig { params(request: ::ActionDispatch::Request, error: Exception).void }
+        def self.log_security_error(request, error)
+          LoggedErrors.mark(error)
+          if ip_spoof_error?(error)
+            LogStruct.warn(
+              Log::Security::IPSpoof.new(
+                path: request.path,
+                http_method: request.method,
+                user_agent: request.user_agent,
+                referer: StringScrubber.without_query(request.referer),
+                message: error.message,
+                source_ip: request.ip,
+                client_ip: request.get_header("HTTP_CLIENT_IP"),
+                x_forwarded_for: request.x_forwarded_for,
+                timestamp: Time.now
+              )
+            )
+          else
+            LogStruct.error(
+              Log::Security::CSRFViolation.new(
+                path: request.path,
+                http_method: request.method,
+                source_ip: source_ip(request),
+                user_agent: request.user_agent,
+                referer: StringScrubber.without_query(request.referer),
+                message: error.message,
+                timestamp: Time.now
+              )
+            )
+          end
+          reported = error.is_a?(StandardError) ? error : RuntimeError.new(error.message)
+          LogStruct.handle_exception(reported, source: Source::Security, context: request_context(request), log: false)
+        end
+
+        sig { params(request: ::ActionDispatch::Request).returns(T::Hash[Symbol, T.untyped]) }
+        def self.request_context(request)
+          {
+            path: request.path,
+            method: request.method,
+            user_agent: request.user_agent,
+            referer: StringScrubber.without_query(request.referer)
+          }
+        end
+
+        # The client's IP, as Rails works it out, or the connection's when Rails finds a spoofed one
+        sig { params(request: ::ActionDispatch::Request).returns(T.nilable(String)) }
+        def self.source_ip(request)
+          request.remote_ip
+        rescue ::ActionDispatch::RemoteIp::IpSpoofAttackError
+          request.ip
+        end
+
+        sig { params(error: Exception).returns(T::Boolean) }
+        def self.security_error?(error)
+          ip_spoof_error?(error) || CSRF_ERROR_NAMES.include?(error.class.name)
+        end
+
+        sig { params(error: Exception).returns(T::Boolean) }
+        def self.ip_spoof_error?(error)
+          error.is_a?(::ActionDispatch::RemoteIp::IpSpoofAttackError)
+        end
+      end
+    end
+  end
+end

data/lib/log_struct/integrations/rack_error_handler/middleware.rb

@@ -1,181 +1,26 @@
 # typed: strict
 # frozen_string_literal: true
 
+require_relative "../rack_middleware"
+require_relative "error_lines"
+
 module LogStruct
   module Integrations
     module RackErrorHandler
-      # Custom middleware to enhance Rails error logging with JSON format and request details
-      class Middleware
-        extend T::Sig
-
-        # IP Spoofing error response
-        IP_SPOOF_HTML = T.let(
-          "<html><head><title>IP Spoofing Detected</title></head><body>" \
-          "<h1>Forbidden</h1>" \
-          "<p>IP spoofing detected. This request has been blocked for security reasons.</p>" \
-          "</body></html>",
-          String
-        )
-
-        # CSRF error response
-        CSRF_HTML = T.let(
-          "<html><head><title>CSRF Error</title></head><body>" \
-          "<h1>Forbidden</h1>" \
-          "<p>Invalid authenticity token. This request has been blocked to prevent cross-site request forgery.</p>" \
-          "</body></html>",
-          String
-        )
-
-        # Response headers calculated at load time
-        IP_SPOOF_HEADERS = T.let(
-          {
-            "Content-Type" => "text/html",
-            "Content-Length" => IP_SPOOF_HTML.bytesize.to_s
-          }.freeze,
-          T::Hash[String, String]
-        )
-
-        CSRF_HEADERS = T.let(
-          {
-            "Content-Type" => "text/html",
-            "Content-Length" => CSRF_HTML.bytesize.to_s
-          }.freeze,
-          T::Hash[String, String]
-        )
-
-        # HTTP status code for forbidden responses
-        FORBIDDEN_STATUS = T.let(403, Integer)
-
-        sig { params(app: T.untyped).void }
-        def initialize(app)
-          @app = app
-        end
-
-        sig { params(env: T.untyped).returns(T.untyped) }
+      # Logs the errors that get past ShowExceptions, or come from the middleware outside it (an IP
+      # spoofing error from a middleware that reads remote_ip, for example), then raises them again
+      # for the server to handle as it would without LogStruct
+      class Middleware < RackMiddleware
+        sig { override.params(env: T.untyped).returns(T.untyped) }
         def call(env)
-          return @app.call(env) unless LogStruct.enabled?
-
-          request = ::ActionDispatch::Request.new(env)
-
-          begin
-            # Trigger the same spoofing checks that ActionDispatch::RemoteIp performs after
-            # it is initialized in the middleware stack. We run this manually because we
-            # execute before that middleware and still want spoofing attacks to surface here.
-            perform_remote_ip_check!(request)
-
-            @app.call(env)
-          rescue ::ActionDispatch::RemoteIp::IpSpoofAttackError => ip_spoof_error
-            # Create a security log for IP spoofing
-            security_log = Log::Security::IPSpoof.new(
-              path: env["PATH_INFO"],
-              http_method: env["REQUEST_METHOD"],
-              user_agent: env["HTTP_USER_AGENT"],
-              referer: env["HTTP_REFERER"],
-              message: ip_spoof_error.message,
-              client_ip: env["HTTP_CLIENT_IP"],
-              x_forwarded_for: env["HTTP_X_FORWARDED_FOR"],
-              timestamp: Time.now
-            )
-
-            ::Rails.logger.warn(security_log)
-
-            [FORBIDDEN_STATUS, IP_SPOOF_HEADERS.dup, [IP_SPOOF_HTML]]
-          rescue => error
-            if csrf_error?(error)
-              # Create a security log for CSRF error
-              security_log = Log::Security::CSRFViolation.new(
-                path: request.path,
-                http_method: request.method,
-                source_ip: request.remote_ip,
-                user_agent: request.user_agent,
-                referer: request.referer,
-                message: error.message,
-                timestamp: Time.now
-              )
-              LogStruct.error(security_log)
-
-              # Report to error reporting service and/or re-raise
-              context = extract_request_context(env, request)
-              LogStruct.handle_exception(error, source: Source::Security, context: context)
-
-              # If handle_exception raised an exception then Rails will deal with it (e.g. config.exceptions_app)
-              # If we are only logging or reporting these security errors, then return a default response
-              [FORBIDDEN_STATUS, CSRF_HEADERS.dup, [CSRF_HTML]]
-            else
-              # Extract request context for error reporting
-              context = extract_request_context(env, request)
-
-              # Create and log a structured exception with request context
-              exception_log = Log.from_exception(Source::Rails, error, context)
-              LogStruct.error(exception_log)
-
-              # Re-raise any standard errors to let Rails or error reporter handle it.
-              # Rails will also log the request details separately
-              raise error
-            end
-          end
-        end
-
-        private
-
-        sig { params(request: ::ActionDispatch::Request).void }
-        def perform_remote_ip_check!(request)
-          action_dispatch_config = ::Rails.application.config.action_dispatch
-          check_ip = action_dispatch_config.ip_spoofing_check
-          return unless check_ip
-
-          proxies = normalized_trusted_proxies(action_dispatch_config.trusted_proxies)
-
-          ::ActionDispatch::RemoteIp::GetIp
-            .new(request, check_ip, proxies)
-            .to_s
-        end
-
-        sig { params(env: T::Hash[String, T.untyped], request: T.nilable(::ActionDispatch::Request)).returns(T::Hash[Symbol, T.untyped]) }
-        def extract_request_context(env, request = nil)
-          request ||= ::ActionDispatch::Request.new(env)
-          {
-            path: request.path,
-            method: request.method,
-            user_agent: request.user_agent,
-            referer: request.referer
-          }
+          @app.call(env)
         rescue => error
-          # If we can't extract request context, return minimal info
-          {error_extracting_context: error.message}
-        end
-
-        sig { params(error: StandardError).returns(T::Boolean) }
-        def csrf_error?(error)
-          error_name = error.class.name
-          error_name == "ActionController::InvalidAuthenticityToken" ||
-            error_name == "ActionController::InvalidCrossOriginRequest"
-        end
-
-        sig { params(configured_proxies: T.untyped).returns(T.untyped) }
-        def normalized_trusted_proxies(configured_proxies)
-          if configured_proxies.nil? || (configured_proxies.respond_to?(:empty?) && configured_proxies.empty?)
-            return ::ActionDispatch::RemoteIp::TRUSTED_PROXIES
+          begin
+            ErrorLines.log_escaped(::ActionDispatch::Request.new(env), error)
+          rescue => logging_error
+            LogStruct.handle_exception(logging_error, source: Source::Internal)
           end
-
-          return configured_proxies if configured_proxies.respond_to?(:any?)
-
-          raise(
-            ArgumentError,
-            <<~EOM
-              Setting config.action_dispatch.trusted_proxies to a single value isn't
-              supported. Please set this to an enumerable instead. For
-              example, instead of:
-
-              config.action_dispatch.trusted_proxies = IPAddr.new("10.0.0.0/8")
-
-              Wrap the value in an Array:
-
-              config.action_dispatch.trusted_proxies = [IPAddr.new("10.0.0.0/8")]
-
-              Note that passing an enumerable will *replace* the default set of trusted proxies.
-            EOM
-          )
+          raise error
         end
       end
     end

data/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb

@@ -0,0 +1,36 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  module Integrations
+    module RackErrorHandler
+      # Prepended to ActionDispatch::Request. When Rails can't parse a request's body (malformed JSON), it logs the
+      # whole body at debug level, where nothing filters it. LogStruct logs the body's size and type instead. The
+      # request line and the ParseError's error line are written as usual.
+      module ParseErrorLogging
+        extend T::Sig
+        extend T::Helpers
+
+        requires_ancestor { ::ActionDispatch::Request }
+
+        private
+
+        sig { returns(T.untyped) }
+        def log_parse_error_once
+          return super unless ErrorLines.enabled?
+
+          @logstruct_parse_error_logged ||= T.let(
+            begin
+              (logger || ::ActiveSupport::Logger.new($stderr)).debug(
+                "Error occurred while parsing request parameters. LogStruct leaves out the body " \
+                "(#{raw_post.to_s.bytesize} bytes of #{media_type}), which it can't filter."
+              )
+              true
+            end,
+            T.nilable(TrueClass)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/log_struct/integrations/rack_error_handler.rb

@@ -2,8 +2,10 @@
 # frozen_string_literal: true
 
 require "rack"
-require "action_dispatch/middleware/show_exceptions"
+require "action_dispatch/middleware/debug_exceptions"
 require_relative "rack_error_handler/middleware"
+require_relative "rack_error_handler/debug_exceptions_logging"
+require_relative "rack_error_handler/parse_error_logging"
 require_relative "rack_setup"
 
 module LogStruct
@@ -13,17 +15,31 @@ module LogStruct
       extend T::Sig
       extend IntegrationInterface
 
-      # Set up Rack middleware for structured error logging
+      @interceptor_registered = T.let(false, T::Boolean)
+
+      # Set up structured lines for CSRF and IP spoofing errors, errors the app didn't handle, and
+      # errors that get past ShowExceptions
       sig { override.params(config: LogStruct::Configuration).returns(T.nilable(T::Boolean)) }
       def self.setup(config)
         return nil unless RackSetup.enabled?(config)
 
-        # Add structured logging middleware for security violations and errors
-        # Need to insert before RemoteIp to catch IP spoofing errors it raises
+        # Errors from the middleware outside ShowExceptions, and any that get past it
         RackSetup.insert_before(
           ::ActionDispatch::RemoteIp,
           Integrations::RackErrorHandler::Middleware
         )
+        # DebugExceptions calls its interceptors with each error from the app (in every environment),
+        # before ShowExceptions turns it into Rails' usual response: 422 for a CSRF error
+        unless @interceptor_registered
+          ::ActionDispatch::DebugExceptions.register_interceptor(ErrorLines.method(:intercept))
+          @interceptor_registered = true
+        end
+        # An error line instead of the text Rails logs for an error the app didn't handle
+        unless ::ActionDispatch::DebugExceptions <= DebugExceptionsLogging
+          ::ActionDispatch::DebugExceptions.prepend(DebugExceptionsLogging)
+        end
+        # The size of a body Rails can't parse, instead of the body
+        ::ActionDispatch::Request.prepend(ParseErrorLogging) unless ::ActionDispatch::Request <= ParseErrorLogging
 
         true
       end

data/lib/log_struct/integrations/rack_middleware.rb

@@ -0,0 +1,23 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  module Integrations
+    # A Rack middleware around the app: subclasses define call, and pass the request on to @app
+    class RackMiddleware
+      extend T::Sig
+      extend T::Helpers
+
+      abstract!
+
+      sig { params(app: T.untyped).void }
+      def initialize(app)
+        @app = app
+      end
+
+      sig { abstract.params(env: T.untyped).returns(T.untyped) }
+      def call(env)
+      end
+    end
+  end
+end

data/lib/log_struct/integrations/rails_log_subscribers.rb

@@ -0,0 +1,32 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  module Integrations
+    # Rails' own log subscribers, which LogStruct's lines replace
+    module RailsLogSubscribers
+      extend T::Sig
+
+      # Detaches a framework's log subscriber (ActiveStorage::LogSubscriber, :active_storage). Up to Rails 8.1 it's an
+      # ActiveSupport::LogSubscriber attached to the framework's notifications; from Rails 8.2 it's an
+      # ActiveSupport::EventReporter::LogSubscriber, subscribed to Rails.event when its file loads.
+      sig { params(subscriber_class: T.untyped, namespace: Symbol).void }
+      def self.detach(subscriber_class, namespace)
+        if subscriber_class.respond_to?(:detach_from)
+          subscriber_class.detach_from(namespace)
+        else
+          unsubscribe_from_events(subscriber_class)
+        end
+      end
+
+      # Unsubscribes a Rails 8.2 log subscriber from Rails.event. Does nothing for one that isn't subscribed there.
+      sig { params(subscriber_class: T.untyped).void }
+      def self.unsubscribe_from_events(subscriber_class)
+        return unless ::ActiveSupport.respond_to?(:event_reporter)
+
+        reporter = ::ActiveSupport.event_reporter
+        reporter.unsubscribe(subscriber_class) if reporter.respond_to?(:unsubscribe)
+      end
+    end
+  end
+end

data/lib/log_struct/integrations/request_context/middleware.rb

@@ -1,20 +1,15 @@
 # typed: strict
 # frozen_string_literal: true
 
+require_relative "../rack_middleware"
+
 module LogStruct
   module Integrations
     module RequestContext
       # Middleware that captures request_id and stores it in SemanticLogger's
       # named_tags so all logs during the request include the request_id.
-      class Middleware
-        extend T::Sig
-
-        sig { params(app: T.untyped).void }
-        def initialize(app)
-          @app = app
-        end
-
-        sig { params(env: T.untyped).returns(T.untyped) }
+      class Middleware < RackMiddleware
+        sig { override.params(env: T.untyped).returns(T.untyped) }
         def call(env)
           request = ::ActionDispatch::Request.new(env)
           request_id = request.request_id