logtail-rack 0.2.7 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f416be58cf3b8f82cdd582233d0de6e3b88653f11fbfdc76800dd60aadb9f55
-  data.tar.gz: 59cfce37f93ffbee0442c8a2d257d444e578a604f7685c3dc0371e4fee2b97c2
+  metadata.gz: 4df8d3a1e2937eb55804fc33386d4fff4225cb262e3aee05730dc8d34cb0ea3f
+  data.tar.gz: 36d88de7aeffcc49e185f2dc9149e4fc7811f113ee46e65c22d5410bbca539da
 SHA512:
-  metadata.gz: c70fddaa55215c0c43ca728cee93d7c6c77d07b2ee69e85b2923dc72cdd572e674f795cd2f2f1da543a1f0364308e94e51385fbaaa03879741a45430fd62523c
-  data.tar.gz: a074cb26984e9d0802b9aae58cd7961a0e894ae0095655ffc6f7e9d4aec7f4c7a428f7b359b13ac3fe99d079c01e917e74594d93a1afe5509d385fbefe57b96c
+  metadata.gz: 7e0e826ff231eaffa61f7a25846321cb9e39966f62d223f9f9d7bf290b80faacd504fa6230b2a314ab54f0bee4d6ac923d1b2431af81c64f47400b9eb0280fc3
+  data.tar.gz: 5ad21673f3fa6afbd3f14f839637a5fafa89ba6f7d4c87eda2461b8a58ac62de72e5fb80328a93a62e8f60d94cf14912d700597494063b5dffbe8774c74240d8

data/Gemfile

@@ -5,3 +5,5 @@ gemspec
 gem "base64" if RUBY_VERSION >= "3.4.0"
 # No longer a default gem on Ruby 4.0 and TruffleRuby 40, and logtail 0.1.17 requires it undeclared
 gem "logger"
+# Rack 3 moved the session middlewares the tests use into this gem; its 1.x releases for older Racks are empty
+gem "rack-session"

data/lib/logtail-rack/error_event.rb

@@ -17,7 +17,7 @@ module Logtail
                 error_message: exception.message,
                 backtrace: exception.backtrace
               )
-            end
+            end rescue logging_failed($!)
 
             raise exception
           end

data/lib/logtail-rack/http_context.rb

@@ -1,3 +1,4 @@
+require "logtail/config"
 require "logtail/contexts/http"
 require "logtail/current_context"
 require "logtail-rack/middleware"
@@ -10,19 +11,30 @@ module Logtail
       # A Rack middleware that is reponsible for adding the HTTP context {Logtail::Contexts::HTTP}.
       class HTTPContext < Middleware
         def call(env)
-          request = Util::Request.new(env)
-          context = Contexts::HTTP.new(
-            host: Util::Encoding.force_utf8_encoding(request.host),
-            method: Util::Encoding.force_utf8_encoding(request.request_method),
-            path: request.path,
-            remote_addr: Util::Encoding.force_utf8_encoding(request.ip),
-            request_id: request.request_id
-          )
-
-          CurrentContext.with(context.to_hash) do
+          context = get_http_context(env)
+          if context
+            CurrentContext.with(context) do
+              @app.call(env)
+            end
+          else
             @app.call(env)
           end
         end
+
+        private
+          def get_http_context(env)
+            request = Util::Request.new(env)
+            Contexts::HTTP.new(
+              host: Util::Encoding.force_utf8_encoding(request.host),
+              method: Util::Encoding.force_utf8_encoding(request.request_method),
+              path: request.path,
+              remote_addr: Util::Encoding.force_utf8_encoding(request.ip),
+              request_id: request.request_id
+            ).to_hash
+          rescue StandardError => e
+            Logtail::Config.instance.debug { "Could not build the HTTP context: #{e.inspect}\n\n#{e.backtrace}" }
+            nil
+          end
       end
     end
   end

data/lib/logtail-rack/http_events.rb

@@ -1,4 +1,5 @@
 require "set"
+require "uri"
 
 require "logtail/config"
 require "logtail/contexts/http"
@@ -15,7 +16,9 @@ module Logtail
       # response events. The {Events::HTTPRequest} and {Events::HTTPResponse} events
       # respectively.
       class HTTPEvents < Middleware
+        DEFAULT_STATUS_FOR_EXCEPTION = ->(_exception) { 500 }
         DEFAULT_HTTP_HEADER_FILTERS = ["Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"].freeze
+        DEFAULT_QUERY_STRING_FILTERS = %w[passw secret token _key crypt salt certificate otp ssn cvv cvc].freeze
 
         class << self
           # Allows you to capture the HTTP request body, default is off (false).
@@ -105,6 +108,29 @@ module Logtail
             @silence_request
           end
 
+          # When the app raises instead of returning a response, the response event is still
+          # logged, and the exception is re-raised unchanged. This setting resolves the HTTP
+          # status of that event: a callable that receives the exception and returns an Integer
+          # status. The default, {DEFAULT_STATUS_FOR_EXCEPTION}, returns 500, and so does a
+          # callable that raises itself.
+          #
+          # @example
+          #   Logtail::Integrations::Rack::HTTPEvents.status_for_exception = lambda do |exception|
+          #     exception.is_a?(MyApp::NotFound) ? 404 : 500
+          #   end
+          def status_for_exception=(callable)
+            if !callable.respond_to?(:call)
+              raise ArgumentError.new("The value passed to #status_for_exception must respond to #call")
+            end
+
+            @status_for_exception = callable
+          end
+
+          # Accessor method for {#status_for_exception=}
+          def status_for_exception
+            @status_for_exception
+          end
+
           # Filter sensitive HTTP headers (such as "Authorization: Bearer secret_token")
           #
           # Filtered HTTP header values will be sent to Better Stack as "[FILTERED]"
@@ -123,14 +149,60 @@ module Logtail
             @http_header_filters
           end
 
+          # Filter sensitive query string parameters (such as "?token=secret_token")
+          #
+          # Filtered values will be sent to Better Stack as "[FILTERED]", in the query string of
+          # the request event and in the query of the URLs in the Referer and Location headers.
+          # Like Rails' filter_parameters, a String or Symbol filters every parameter whose
+          # URL-decoded name contains it, ignoring case, and a Regexp every name it matches.
+          #
+          # {DEFAULT_QUERY_STRING_FILTERS} are filtered out of the box. Setting this replaces
+          # the whole list, pass an empty list to log query strings unfiltered.
+          #
+          # @example
+          #   Logtail::Integrations::Rack::HTTPEvents.query_string_filters = Logtail::Integrations::Rack::HTTPEvents::DEFAULT_QUERY_STRING_FILTERS + ["code", /\Aapi_/]
+          def query_string_filters=(value)
+            strings = value.select { |filter| filter.is_a?(String) || filter.is_a?(Symbol) }
+            regexps = value.select { |filter| filter.is_a?(Regexp) }
+            if !strings.empty?
+              regexps << Regexp.new(strings.map { |filter| Regexp.escape(filter.to_s) }.join("|"), Regexp::IGNORECASE)
+            end
+
+            @query_string_filters = value
+            @query_string_filter_regexps = regexps
+          end
+
+          # Accessor method for {#query_string_filters=}
+          def query_string_filters
+            @query_string_filters
+          end
+
+          # Whether {#query_string_filters} filter the parameter with this name from a query string
+          def filter_query_string_parameter?(name)
+            begin
+              name = URI.decode_www_form_component(name)
+            rescue ArgumentError
+              # Invalid %-encoding, match the name as it is
+            end
+
+            name = name.scrub
+            @query_string_filter_regexps.any? { |regexp| regexp =~ name }
+          end
+
           def normalize_header_name(name)
             name.to_s.downcase.gsub("-", "_")
           end
         end
 
+        self.status_for_exception = DEFAULT_STATUS_FOR_EXCEPTION
         self.http_header_filters = DEFAULT_HTTP_HEADER_FILTERS
+        self.query_string_filters = DEFAULT_QUERY_STRING_FILTERS
 
-        CONTENT_LENGTH_KEY = 'Content-Length'.freeze
+        CONTENT_LENGTH_KEY = 'content-length'.freeze
+        # A query string parameter name and its value, up to the next separator
+        QUERY_STRING_PARAMETER = /([^&;=]+)=([^&;]+)/
+        URL_QUERY = /\?([^#]*)/
+        URL_HEADERS = ["referer", "location"].freeze
 
         def call(env)
           request = Util::Request.new(env)
@@ -145,14 +217,14 @@ module Logtail
             end
 
           elsif collapse_into_single_event?
-            request_start = Time.now
-            status, headers, body = @app.call(env)
-            request_end = Time.now
+            request_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            status, headers, body = call_app(env, request, request_start)
+            request_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
             Config.instance.logger.info do
-              http_context = CurrentContext.fetch(:http)
-              content_length = safe_to_i(headers[CONTENT_LENGTH_KEY])
-              duration_ms = (request_end - request_start) * 1000.0
+              http_context = CurrentContext.fetch(:http, nil)
+              content_length = response_content_length(headers)
+              duration_ms = ((request_end - request_start) * 1000.0).round(1)
 
               http_response = HTTPResponse.new(
                 content_length: content_length,
@@ -177,7 +249,7 @@ module Logtail
                   }
                 }
               }
-            end
+            end rescue logging_failed($!)
 
             [status, headers, body]
           else
@@ -191,7 +263,7 @@ module Logtail
                 method: request.request_method,
                 path: request.path,
                 port: request.port,
-                query_string: Util::Encoding.force_utf8_encoding(request.query_string),
+                query_string: filter_query_string(Util::Encoding.force_utf8_encoding(request.query_string)),
                 request_id: request.request_id,
                 scheme: Util::Encoding.force_utf8_encoding(request.scheme),
               )
@@ -214,16 +286,17 @@ module Logtail
                   }
                 }
               }
-            end
+            end rescue logging_failed($!)
 
-            request_start = Time.now
-            status, headers, body = @app.call(env)
-            request_end = Time.now
+            request_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            status, headers, body = call_app(env, request, request_start)
+            request_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
             Config.instance.logger.info do
-              event_body = capture_response_body? ? body : nil
-              content_length = safe_to_i(headers[CONTENT_LENGTH_KEY])
-              duration_ms = (request_end - request_start) * 1000.0
+              # Only an Array body can be read twice, other bodies may stream to the server once
+              event_body = capture_response_body? && body.is_a?(Array) ? body.join : nil
+              content_length = response_content_length(headers)
+              duration_ms = ((request_end - request_start) * 1000.0).round(1)
 
               http_response = HTTPResponse.new(
                 body: event_body,
@@ -248,7 +321,7 @@ module Logtail
                   }
                 }
               }
-            end
+            end rescue logging_failed($!)
 
             [status, headers, body]
           end
@@ -275,14 +348,88 @@ module Logtail
             end
           end
 
+          # Calls the app. When it raises instead of returning a response, logs the response
+          # event with the status from {.status_for_exception} and re-raises the exception.
+          def call_app(env, request, request_start)
+            @app.call(env)
+          rescue Exception => exception
+            log_exception_response(request, exception, request_start)
+            raise exception
+          end
+
+          # Never raises, so that the exception of the app is the one that propagates.
+          def log_exception_response(request, exception, request_start)
+            request_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+            Config.instance.logger.info do
+              duration_ms = ((request_end - request_start) * 1000.0).round(1)
+
+              http_response = HTTPResponse.new(
+                http_context: collapse_into_single_event? ? CurrentContext.fetch(:http, nil) : nil,
+                request_id: request.request_id,
+                status: status_for_exception(exception),
+                duration_ms: duration_ms,
+              )
+
+              {
+                message: http_response.message,
+                event: {
+                  http_response_sent: {
+                    body: http_response.body,
+                    content_length: http_response.content_length,
+                    headers_json: http_response.headers_json,
+                    request_id: http_response.request_id,
+                    service_name: http_response.service_name,
+                    status: http_response.status,
+                    duration_ms: http_response.duration_ms,
+                  }
+                }
+              }
+            end
+          rescue Exception => e
+            Config.instance.debug { "Failed to log the response of a request that raised #{exception.class}: #{e.class}: #{e.message}" }
+          end
+
+          def status_for_exception(exception)
+            self.class.status_for_exception.call(exception)
+          rescue Exception => e
+            Config.instance.debug { "status_for_exception raised #{e.class}: #{e.message}, logging status 500" }
+            500
+          end
+
           def filter_http_headers(headers)
             headers.map do |name, value|
               normalized_name = self.class.normalize_header_name(name)
               is_filtered = self.class.http_header_filters.include?(normalized_name)
+              value = filter_url_query(value) if URL_HEADERS.include?(normalized_name)
               [name, is_filtered ? "[FILTERED]" : value]
             end.to_h
           end
 
+          # Replaces the value of every parameter that {.query_string_filters} match with
+          # "[FILTERED]", leaving the rest of the query string unchanged.
+          def filter_query_string(query_string)
+            return query_string if !query_string.is_a?(String) || self.class.query_string_filters.empty?
+
+            query_string.gsub(QUERY_STRING_PARAMETER) do |parameter|
+              name = Regexp.last_match(1)
+              self.class.filter_query_string_parameter?(name) ? "#{name}=[FILTERED]" : parameter
+            end
+          end
+
+          # Filters the query of the URL in a Referer or Location header.
+          def filter_url_query(url)
+            return url if !url.is_a?(String) || !url.include?("?") || self.class.query_string_filters.empty?
+
+            Util::Encoding.force_utf8_encoding(url).sub(URL_QUERY) { "?#{filter_query_string(Regexp.last_match(1))}" }
+          end
+
+          # Rack 3 applications return "content-length", older ones usually "Content-Length".
+          def response_content_length(headers)
+            _name, value = headers.find { |name, _value| name.to_s.downcase == CONTENT_LENGTH_KEY }
+            safe_to_i(value)
+          end
+
           def safe_to_i(val)
             val.nil? ? nil : val.to_i
           end

data/lib/logtail-rack/http_request.rb

@@ -1,3 +1,5 @@
+require "json"
+
 require "logtail-rack/util/encoding"
 
 module Logtail
@@ -26,7 +28,7 @@ module Logtail
           @service_name = attributes[:service_name]
 
           if @headers
-            @headers_json = Util::Encoding.force_utf8_encoding(@headers).to_json
+            @headers_json = ::JSON.generate(Util::Encoding.force_utf8_encoding(@headers))
           end
         end
 

data/lib/logtail-rack/http_response.rb

@@ -1,3 +1,5 @@
+require "json"
+
 require "logtail-rack/util/encoding"
 
 module Logtail
@@ -21,7 +23,7 @@ module Logtail
           @duration_ms = attributes[:duration_ms]
 
           if @headers
-            @headers_json = Util::Encoding.force_utf8_encoding(@headers).to_json
+            @headers_json = ::JSON.generate(Util::Encoding.force_utf8_encoding(@headers))
           end
         end
 

data/lib/logtail-rack/middleware.rb

@@ -1,3 +1,5 @@
+require "logtail/config"
+
 module Logtail
   module Integrations
     module Rack
@@ -22,6 +24,18 @@ module Logtail
         def initialize(app)
           @app = app
         end
+
+        private
+          # Logging must never fail the request, so an error raised while building or writing an event
+          # only goes to the debug logger. The middlewares call the logger themselves, so that the
+          # runtime context of the line points to them, and rescue the error with this method:
+          #
+          #   Config.instance.logger.info do
+          #     ...
+          #   end rescue logging_failed($!)
+          def logging_failed(error)
+            Config.instance.debug { "#{self.class.name} could not log an event: #{error.inspect}\n\n#{error.backtrace}" }
+          end
       end
     end
   end

data/lib/logtail-rack/session_context.rb

@@ -25,7 +25,10 @@ module Logtail
             if session = env['rack.session']
               if session.respond_to?(:id)
                 Logtail::Config.instance.debug { "Rack env session detected, using id attribute" }
-                session.id
+                id = session.id
+                # Since Rack 1.6.12 and 2.0.8 a Rack::Session::SessionId, which MessagePack can't encode. Its public id
+                # is the session cookie of server-side stores, the private id a hash of it that can't be used as one.
+                id.respond_to?(:private_id) ? id.private_id : id
               elsif session.respond_to?(:[])
                 Logtail::Config.instance.debug { "Rack env session detected, using the session_id key" }
                 session["session_id"]
@@ -39,6 +42,7 @@ module Logtail
               nil
             end
           rescue Exception => e
+            Logtail::Config.instance.debug { "Could not obtain the session id: #{e.inspect}\n\n#{e.backtrace}" }
             nil
           end
       end

data/lib/logtail-rack/user_context.rb

@@ -93,6 +93,9 @@ module Logtail
               Logtail::Config.instance.debug { "Could not locate any user data" }
               nil
             end
+          rescue StandardError => e
+            Logtail::Config.instance.debug { "Could not obtain the user context: #{e.inspect}\n\n#{e.backtrace}" }
+            nil
           end
 
           def get_user_object_hash(user)

data/lib/logtail-rack/util/request.rb

@@ -13,6 +13,10 @@ module Logtail
       REQUEST_ID_KEY_NAME2 = 'HTTP_X_REQUEST_ID'.freeze
 
       def body_content
+        # Rack 3 allows a missing or non-rewindable input, and reading one that can't be rewound takes the body
+        # from the app
+        return nil unless body.respond_to?(:rewind)
+
         content = body.read
         body.rewind
         content

data/lib/logtail-rack/version.rb

@@ -1,7 +1,7 @@
 module Logtail
   module Integrations
     module Rack
-      VERSION = "0.2.7"
+      VERSION = "0.2.9"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logtail-rack
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.9
 platform: ruby
 authors:
 - Better Stack