semantic_logger 5.0.0 → 5.2.0

data/lib/semantic_logger/base.rb

@@ -28,23 +28,38 @@ module SemanticLogger
       end
     end
 
-    # Returns the current log level if set, otherwise it returns the global
-    # default log level
+    # Returns the level set by an enclosing `#with_level` block on the current fiber,
+    # otherwise the level set for this instance, otherwise the global default level.
     def level
-      @level || SemanticLogger.default_level
+      index = @with_level_used && with_level_index
+      return Levels.level(index) if index
+
+      @level || fallback_level
     end
 
-    # Set the logging level for this logger during the execution of the given block
+    # Change the log level of this instance while the block runs, on the current fiber only.
     #
-    # Refer to the documentation for `#level=` for more information about the possible log levels.
-    def with_level(new_level)
-      old_level = level
-
-      self.level = new_level
-
-      yield
-    ensure
-      self.level = old_level
+    # Other threads and fibers that share this instance keep logging at its own level.
+    # The level set with `#level=` is left untouched, so an instance that follows the
+    # global default level keeps following it after the block. Blocks can be nested.
+    #
+    # Matches Ruby's `Logger#with_level`, which Sidekiq calls to apply a job's `log_level`.
+    # Accepts the same levels as `#level=`. Returns the result of the block.
+    def with_level(level)
+      index            = Levels.index(level)
+      overrides        = (Thread.current[:semantic_logger_with_level] ||= {}.compare_by_identity)
+      previous         = overrides[self]
+      @with_level_used = true
+      overrides[self]  = index
+      begin
+        yield
+      ensure
+        if previous
+          overrides[self] = previous
+        else
+          overrides.delete(self)
+        end
+      end
     end
 
     # Implement the log level calls
@@ -150,13 +165,11 @@ module SemanticLogger
         if thread == Thread.current
           Utils.extract_backtrace(caller)
         else
-          log.thread_name = thread.name
+          log.thread_name = Log.thread_name_for(thread)
           log.tags        = (thread[:semantic_logger_tags] || []).clone
           log.named_tags  = (thread[:semantic_logger_named_tags] || {}).clone
           thread.backtrace
         end
-      # TODO: Keep backtrace instead of transforming into a text message at this point
-      # Maybe log_backtrace: true
       if backtrace
         message += "\n"
         message << backtrace.join("\n")
@@ -321,6 +334,10 @@ module SemanticLogger
       # (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
       @instance_tags       = [].freeze
       @instance_named_tags = {}.freeze
+      # Set the first time `#with_level` is called on this instance, so that instances that
+      # never use it skip the fiber-local lookup when checking their level. Only the fiber
+      # that sets an override needs to see the flag, and that fiber set it itself.
+      @with_level_used     = false
       if level.nil?
         # Allow the global default level to determine this loggers log level
         @level_index = nil
@@ -330,11 +347,25 @@ module SemanticLogger
       end
     end
 
-    # Return the level index for fast comparisons
-    # Returns the global default level index if the level has not been explicitly
-    # set for this instance
+    # Return the level index for fast comparisons, resolved in the same order as `#level`.
     def level_index
-      @level_index || SemanticLogger.default_level_index
+      (@with_level_used && with_level_index) || @level_index || fallback_level_index
+    end
+
+    # Level to use when no level has been set for this instance: the global default level.
+    def fallback_level
+      SemanticLogger.default_level
+    end
+
+    # Level index to use when no level has been set for this instance. Unlike
+    # `#fallback_level`, it honors `SemanticLogger.silence`.
+    def fallback_level_index
+      SemanticLogger.default_level_index
+    end
+
+    # Level index set for this instance by an enclosing `#with_level` block on the current fiber.
+    def with_level_index
+      Thread.current[:semantic_logger_with_level]&.[](self)
     end
 
     # Whether to log the supplied message based on the current filter if any
@@ -446,6 +477,9 @@ module SemanticLogger
     end
 
     # Measure the supplied block and log the message
+    #
+    # The options can hold untrusted data, so an option with a value that is not valid for it is
+    # ignored (see Log.argument?) and its default is used instead.
     def measure_internal(level, index, message, params)
       exception = nil
       # Single parameter is a hash
@@ -456,9 +490,7 @@ module SemanticLogger
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       begin
         if block_given?
-          if (silence_level = params[:silence])
-            # In case someone accidentally sets `silence: true` instead of `silence: :error`
-            silence_level = :error if silence_level == true
+          if (silence_level = measure_silence_level(params))
             silence(silence_level) { yield(params) }
           else
             yield(params)
@@ -469,13 +501,15 @@ module SemanticLogger
       ensure
         # Must use ensure block otherwise a `return` in the yield above will skip the log entry
         log = new_log(level, index)
-        exception ||= params[:exception]
-        message   = params[:message] if params[:message]
-        duration  =
+        # Only an Exception, since it is re-raised below
+        exception ||= params[:exception] if params[:exception].is_a?(Exception)
+        message     = measure_option(params, :message) || message
+        duration    =
           if block_given?
             1_000.0 * (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
           else
-            params[:duration] || raise("Mandatory block missing when :duration option is not supplied")
+            measure_option(params, :duration) ||
+              raise("Mandatory block missing when :duration option is not supplied")
           end
 
         # Extract options after block completes so that block can modify any of the options
@@ -485,13 +519,13 @@ module SemanticLogger
         should_log = log.assign(
           message:            message,
           payload:            payload,
-          min_duration:       params[:min_duration] || 0.0,
+          min_duration:       measure_option(params, :min_duration) || 0.0,
           exception:          exception,
-          metric:             params[:metric],
-          metric_amount:      params[:metric_amount],
+          metric:             measure_option(params, :metric),
+          metric_amount:      measure_option(params, :metric_amount),
           duration:           duration,
-          log_exception:      params[:log_exception] || :partial,
-          on_exception_level: params[:on_exception_level]
+          log_exception:      measure_option(params, :log_exception) || :partial,
+          on_exception_level: measure_option(params, :on_exception_level)
         )
 
         # Log level may change during assign due to :on_exception_level
@@ -500,6 +534,22 @@ module SemanticLogger
       end
     end
 
+    # Returns the value of the measure option that is also a log argument, or nil when it is not
+    # supplied or its value is not valid for that argument.
+    def measure_option(params, key)
+      value = params[key]
+      value if Log.argument?(key, value)
+    end
+
+    # Returns the level to silence other log messages to within a measure block, or nil when none
+    # is supplied or it is not a valid level.
+    def measure_silence_level(params)
+      silence_level = params[:silence]
+      # In case someone accidentally sets `silence: true` instead of `silence: :error`
+      silence_level = :error if silence_level == true
+      silence_level if Levels.valid?(silence_level)
+    end
+
     # For measuring methods and logging their duration.
     def measure_method(index:,
                        level:,

data/lib/semantic_logger/exception_details.rb

@@ -0,0 +1,50 @@
+module SemanticLogger
+  # Separates the detail that Ruby's Exception#detailed_message adds to an exception's message,
+  # such as did_you_mean suggestions or error_highlight's source snippet, from the message itself.
+  #
+  # Internal: Log caches the result for each of its exceptions; see Log#exception_details.
+  module ExceptionDetails
+    # Ruby's own Exception#detailed_message: the message, decorated with the class name.
+    DETAILED_MESSAGE = Exception.instance_method(:detailed_message)
+    private_constant :DETAILED_MESSAGE
+
+    # Returns [String] the detail that #detailed_message adds to the exception's message, without
+    # the class name that Ruby also adds, or nil when there is none or it cannot be separated.
+    #
+    # Never raises, so that an exception's own #detailed_message cannot stop a log entry being
+    # written.
+    def self.extract(exception, source: false)
+      detailed = full_detailed_message(exception, source)
+      return unless detailed
+
+      base = DETAILED_MESSAGE.bind_call(exception, highlight: false)
+      # An override that does not build on Ruby's message cannot be separated from it.
+      return unless detailed.is_a?(String) && detailed.start_with?(base)
+
+      details = detailed.delete_prefix(base).sub(/\A\n+/, "").rstrip
+      details unless details.empty?
+    rescue StandardError
+      nil
+    end
+
+    # Returns [String] the exception's #detailed_message, with or without error_highlight's
+    # source snippet, or nil when the snippet cannot be left out.
+    def self.full_detailed_message(exception, source)
+      exception.detailed_message(highlight: false, error_highlight: source)
+    rescue ArgumentError
+      # An override of #detailed_message that does not accept extra keyword arguments. Calling it
+      # without `error_highlight: false` lets error_highlight add its source snippet, so only do so
+      # when the snippet was asked for, or error_highlight does not decorate this exception.
+      return if !source && error_highlighted?(exception)
+
+      exception.detailed_message(highlight: false)
+    end
+    private_class_method :full_detailed_message
+
+    # Whether error_highlight adds a source snippet to this exception's #detailed_message.
+    def self.error_highlighted?(exception)
+      defined?(ErrorHighlight::CoreExt) && exception.is_a?(ErrorHighlight::CoreExt)
+    end
+    private_class_method :error_highlighted?
+  end
+end

data/lib/semantic_logger/formatters/base.rb

@@ -3,7 +3,7 @@ module SemanticLogger
   module Formatters
     class Base
       attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
-                    :escape_control_chars
+                    :escape_control_chars, :log_exception_details
 
       # Printable escapes for the most common control characters. Any other
       # control character is escaped to its hexadecimal `\xHH` form by
@@ -50,25 +50,39 @@ module SemanticLogger
       #     Default: PRECISION (6, except on older JRuby, where 3)
       #   escape_control_chars: [Boolean]
       #     Replace control characters (newlines, the ANSI escape, etc.) in
-      #     untrusted log data (message, tags, named tags, and exception
-      #     message) with a printable escaped form, e.g. "\n".
+      #     untrusted log data (logger name, thread name, message, payload, tags,
+      #     named tags, and the message and backtrace lines of the exception and
+      #     of each exception that caused it) with a printable escaped form, e.g. "\n".
       #     This prevents log forging and terminal escape-sequence injection
       #     when logging data that may contain attacker-controlled content.
       #     Note: Has no effect on structured formatters such as :json, which
       #     already escape control characters via JSON encoding.
       #     Default: false (preserve newlines and ANSI colors in text output)
+      #   log_exception_details: [true|false|:with_source]
+      #     Whether to log the detail that Ruby's Exception#detailed_message adds to an
+      #     exception's message, such as did_you_mean suggestions ("Did you mean?  upcase") or
+      #     the detail an exception class supplies by overriding #detailed_message.
+      #     :with_source also logs error_highlight's snippet of the failing source line. It is
+      #     about 100 times slower, since it re-parses that source file, and it copies source
+      #     code into the log.
+      #     On Ruby 3.2 a NameError or NoMethodError builds its message from receiver.inspect each
+      #     time it is asked, and logging the details asks for it twice more, so turn this off when
+      #     inspecting the receivers of logged exceptions is slow.
+      #     Default: true for the Default, Color, and Pattern text formatters, false otherwise.
       def initialize(time_format: nil,
                      log_host: true,
                      log_application: true,
                      log_environment: true,
                      precision: PRECISION,
-                     escape_control_chars: false)
+                     escape_control_chars: false,
+                     log_exception_details: default_log_exception_details)
         @time_format          = time_format || self.class.build_time_format(precision)
         @log_host             = log_host
         @log_application      = log_application
         @log_environment      = log_environment
         @precision            = precision
-        @escape_control_chars = escape_control_chars
+        @escape_control_chars  = escape_control_chars
+        @log_exception_details = log_exception_details
       end
 
       # Return default time format string
@@ -101,10 +115,42 @@ module SemanticLogger
       # Note: This escapes (preserves) control characters, and is opt-in. It is
       # distinct from Log#cleansed_message, which unconditionally *strips* ANSI
       # colorization from the message for structured (JSON/Loki) output.
+      #
+      # A string with invalid byte sequences, for example an exception message that quotes
+      # untrusted binary input, or in another encoding, is first converted to valid UTF-8
+      # (dropping the invalid bytes), since matching against it would raise and the log entry
+      # would be lost.
       def escape_control_characters(value)
         return value unless escape_control_chars && value
 
-        value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
+        string = Utils.encode_utf8_string(value.to_s)
+        string.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
+      end
+
+      # Whether this formatter logs exception details unless told otherwise. See #initialize.
+      def default_log_exception_details
+        false
+      end
+
+      # Returns [String] the detail Ruby adds to the message of the supplied exception (see
+      # Log#exception_details), or nil when there is none or this formatter does not log it.
+      def exception_details_for(exception)
+        return unless log_exception_details
+
+        log.exception_details(exception, source: log_exception_details == :with_source)
+      end
+
+      # Returns [String] the backtrace of the logged exception, followed by each exception
+      # that caused it, as laid out by Log#backtrace_to_s.
+      #
+      # When `escape_control_chars` is enabled, the message of each cause and each backtrace
+      # line is passed through #escape_control_characters, the same as the message of the
+      # logged exception: causes are often parser or validation errors that quote untrusted
+      # input, and a backtrace can be set from untrusted data.
+      def exception_backtrace
+        return log.backtrace_to_s unless escape_control_chars
+
+        log.backtrace_to_s { |text| escape_control_characters(text) }
       end
 
       # Return the Time as a formatted string

data/lib/semantic_logger/formatters/color.rb

@@ -71,6 +71,8 @@ module SemanticLogger
       #
       #    Note: The option :multiline is set to false if not supplied.
       #    Note: Has no effect if Awesome Print is not installed.
+      #    Note: With `escape_control_chars: true` the payload is always rendered plain
+      #          (without colors) and on a single line, then escaped.
       #
       #  color_map: [Hash | SemanticLogger::Formatters::Color::ColorMap]
       #    ColorMaps each of the log levels to a color
@@ -97,7 +99,9 @@ module SemanticLogger
         return if named_tags.nil? || named_tags.empty?
 
         list = []
-        named_tags.each_pair { |name, value| list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}" }
+        named_tags.each_pair do |name, value|
+          list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}"
+        end
         "{#{list.join(', ')}}"
       end
 
@@ -114,7 +118,7 @@ module SemanticLogger
 
         if log.payload.respond_to?(:ai)
           begin
-            "-- #{log.payload.ai(@ai_options)}"
+            "-- #{escape_control_characters(log.payload_to_s { |payload| payload.ai(ai_options) })}"
           rescue StandardError
             super
           end
@@ -126,13 +130,28 @@ module SemanticLogger
       def exception
         return unless log.exception
 
-        "-- Exception: #{color}#{log.exception.class}: #{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
+        details = exception_details
+        "-- Exception: #{color}#{log.exception.class}: " \
+          "#{escape_control_characters(log.exception.message)}#{color_map.clear}\n" \
+          "#{"#{details}\n" if details}#{exception_backtrace}"
       end
 
       def call(log, logger)
         self.color = color_map[log.level]
         super
       end
+
+      private
+
+      # Returns [Hash] the Amazing Print options to render the payload with.
+      #
+      # When `escape_control_chars` is enabled the payload is rendered plain, on one line:
+      # the colors and line breaks Amazing Print adds cannot be told apart from control
+      # characters in the data (for example an object whose #inspect returns them), and
+      # escaping them all leaves no colors or line breaks to keep.
+      def ai_options
+        escape_control_chars ? @ai_options.merge(plain: true, multiline: false) : @ai_options
+      end
     end
   end
 end

data/lib/semantic_logger/formatters/default.rb

@@ -12,13 +12,16 @@ module SemanticLogger
 
       # Name of the thread that logged the message.
       def thread_name
-        format("%.30s", log.thread_name)
+        escape_control_characters(format("%.30s", log.thread_name))
       end
 
       # Ruby file name and line number that logged the message.
+      #
+      # The file name is escaped like the other fields, since it comes from the first line of the
+      # backtrace, which a log call can supply from untrusted data.
       def file_name_and_line
         file, line = log.file_name_and_line(true)
-        "#{file}:#{line}" if file
+        "#{escape_control_characters(file)}:#{line}" if file
       end
 
       # Returns [String] the available process info
@@ -43,7 +46,9 @@ module SemanticLogger
         return if named_tags.nil? || named_tags.empty?
 
         list = []
-        named_tags.each_pair { |name, value| list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }
+        named_tags.each_pair do |name, value|
+          list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}"
+        end
         "{#{list.join(', ')}}"
       end
 
@@ -54,7 +59,7 @@ module SemanticLogger
 
       # Class / app name
       def name
-        log.name
+        escape_control_characters(log.name)
       end
 
       # Log message
@@ -67,12 +72,22 @@ module SemanticLogger
         pl = log.payload_to_s
         return unless pl
 
-        "-- #{pl}"
+        "-- #{escape_control_characters(pl)}"
       end
 
       # Exception
       def exception
-        "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n#{log.backtrace_to_s}" if log.exception
+        return unless log.exception
+
+        details = exception_details
+        "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n" \
+          "#{"#{details}\n" if details}#{exception_backtrace}"
+      end
+
+      # The detail Ruby adds to the exception's message, such as did_you_mean suggestions.
+      # See Base#initialize for the `log_exception_details` option.
+      def exception_details
+        escape_control_characters(exception_details_for(log.exception)) if log.exception
       end
 
       # Default text log format
@@ -84,6 +99,13 @@ module SemanticLogger
 
         [time, level, process_info, tags, named_tags, duration, name, message, payload, exception].compact.join(" ")
       end
+
+      private
+
+      # Text output is read by people, who benefit most from exception details.
+      def default_log_exception_details
+        true
+      end
     end
   end
 end