semantic_logger 5.1.0 → 5.2.0

data/lib/semantic_logger/formatters/default.rb

@@ -12,13 +12,16 @@ module SemanticLogger
 
       # Name of the thread that logged the message.
       def thread_name
-        format("%.30s", log.thread_name)
+        escape_control_characters(format("%.30s", log.thread_name))
       end
 
       # Ruby file name and line number that logged the message.
+      #
+      # The file name is escaped like the other fields, since it comes from the first line of the
+      # backtrace, which a log call can supply from untrusted data.
       def file_name_and_line
         file, line = log.file_name_and_line(true)
-        "#{file}:#{line}" if file
+        "#{escape_control_characters(file)}:#{line}" if file
       end
 
       # Returns [String] the available process info
@@ -56,7 +59,7 @@ module SemanticLogger
 
       # Class / app name
       def name
-        log.name
+        escape_control_characters(log.name)
       end
 
       # Log message
@@ -69,15 +72,22 @@ module SemanticLogger
         pl = log.payload_to_s
         return unless pl
 
-        "-- #{pl}"
+        "-- #{escape_control_characters(pl)}"
       end
 
       # Exception
       def exception
         return unless log.exception
 
+        details = exception_details
         "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n" \
-          "#{log.backtrace_to_s}"
+          "#{"#{details}\n" if details}#{exception_backtrace}"
+      end
+
+      # The detail Ruby adds to the exception's message, such as did_you_mean suggestions.
+      # See Base#initialize for the `log_exception_details` option.
+      def exception_details
+        escape_control_characters(exception_details_for(log.exception)) if log.exception
       end
 
       # Default text log format
@@ -89,6 +99,13 @@ module SemanticLogger
 
         [time, level, process_info, tags, named_tags, duration, name, message, payload, exception].compact.join(" ")
       end
+
+      private
+
+      # Text output is read by people, who benefit most from exception details.
+      def default_log_exception_details
+        true
+      end
     end
   end
 end

data/lib/semantic_logger/formatters/logfmt.rb

@@ -13,15 +13,69 @@ module SemanticLogger
     #
     # All timestamps are ISO8601 formatteed
     # All user supplied values are escaped and surrounded by double quotes to avoid ambiguious message delimeters
-    # `tags` are treated as keys with boolean values. Tag names are not formatted or validated;
-    # ensure you use valid logfmt format for tag names.
-    # `named_tags` are flattened are merged into the top level message field. Any conflicting fields are overridden.
-    # `payload` values take precedence over `tags` and `named_tags`. Any conflicting fields are overridden.
+    # `tags` are treated as keys with boolean values.
+    # In every key (tags, named tags, payload), spaces, control and format characters, `=`, and `"`
+    # are replaced with `_`, so that untrusted data cannot inject fields or lines, or disguise a key.
+    # `tags`, `named_tags`, and `payload` are flattened into the top level fields. A key named like
+    # one of the entry's own fields (the time key, `level`, `name`, `message`, `duration`,
+    # `duration_ms`, `tag`) or an exception field is prefixed with its source, as in `payload.level`,
+    # so that logged data cannot replace those fields. So is a key that starts with `tags.`,
+    # `named_tags.`, or `payload.`, so that it cannot forge another source's field. Other keys that
+    # are the same once encoded, such as `:user` and `"user"`, or `"a b"` and `"a=b"`, are written
+    # once, with the value from the later source: `payload` values take precedence over
+    # `named_tags`, and `named_tags` over `tags`.
+    # An exception is emitted as `exception_name`, `exception_message`, `stack_trace`, and `cause`,
+    # and with `log_exception_details` also `exception_details`.
     #
     # Futher Reading https://brandur.org/logfmt
     class Logfmt < Raw
+      # The entry's own fields in the Raw hash, written in this order after the time key.
+      ENTRY_FIELDS = %i[level name message duration duration_ms].freeze
+
+      # The field that tells an entry with an exception from one without.
+      TAG_FIELD = "tag".freeze
+
+      # The exception's fields in the Raw hash, mapped to their logfmt names, so that the exception's
+      # name and message do not replace the entry's own.
+      EXCEPTION_FIELDS = {
+        name:        "exception_name",
+        message:     "exception_message",
+        details:     "exception_details",
+        stack_trace: "stack_trace",
+        cause:       "cause"
+      }.freeze
+
+      # Prefixes that mark a key from logged data with its source, when the key alone would name
+      # one of the entry's own fields, or look like a key marked this way.
+      TAGS_PREFIX       = "tags.".freeze
+      NAMED_TAGS_PREFIX = "named_tags.".freeze
+      PAYLOAD_PREFIX    = "payload.".freeze
+      SOURCE_PREFIXES   = [TAGS_PREFIX, NAMED_TAGS_PREFIX, PAYLOAD_PREFIX].freeze
+
+      # Characters that cannot appear in a logfmt key: spaces and control characters (including
+      # NEL), which would end the key or the line, the `=` and `"` delimiters, LINE SEPARATOR and
+      # PARAGRAPH SEPARATOR, which some readers treat as newlines, and the invisible format
+      # characters, such as ZERO WIDTH SPACE and RIGHT-TO-LEFT OVERRIDE, which would let a key pass
+      # for another one, such as "level" followed by a ZERO WIDTH SPACE for "level".
+      UNSAFE_KEY_CHARS = /[\p{Cc}\p{Cf} ="\u2028\u2029]/
+
+      # The ASCII subset of UNSAFE_KEY_CHARS, which is much faster to match against the ASCII keys
+      # that make up nearly every log event.
+      UNSAFE_ASCII_KEY_CHARS = /[\x00-\x20="\x7f]/
+
+      # The most Symbol keys to remember the names of, so that untrusted keys cannot grow the cache
+      # without bound.
+      PLAIN_SYMBOL_CACHE_SIZE = 512
+
       def initialize(time_format: :iso_8601, time_key: :timestamp, **args)
         super
+        name_entry_fields
+      end
+
+      # Also writes and reserves the new time key in place of the old one.
+      def time_key=(time_key)
+        super
+        name_entry_fields
       end
 
       def call(log, logger)
@@ -32,8 +86,12 @@ module SemanticLogger
 
       private
 
+      # @parsed maps the logfmt name of each field to its value. Logged data is added after the
+      # entry's own fields, and a field with the same name as an earlier one replaces its value, so
+      # the payload takes precedence over named tags, and named tags over tags.
       def raw_to_logfmt
-        @parsed = @raw.slice(time_key, :level, :name, :message, :duration, :duration_ms).merge(tag: "success")
+        @parsed            = @raw.slice(*@entry_keys).transform_keys(@entry_names)
+        @parsed[TAG_FIELD] = "success"
         handle_tags
         handle_payload
         handle_exception
@@ -42,37 +100,111 @@ module SemanticLogger
       end
 
       def handle_tags
-        tags = @raw.fetch(:tags) { [] }.
-               each_with_object({}) { |tag, accum| accum[tag] = true }
-
-        @parsed = @parsed.merge(tags).
-                  merge(@raw.fetch(:named_tags) { {} })
+        @raw[:tags]&.each { |tag| add_field(TAGS_PREFIX, tag, true) }
+        @raw[:named_tags]&.each_pair { |key, value| add_field(NAMED_TAGS_PREFIX, key, value) }
       end
 
       def handle_payload
         return unless @raw.key? :payload
 
-        @parsed = @parsed.merge(@raw[:payload])
+        payload = @raw[:payload]
+        # A payload is normally a Hash, but a log call can supply any value, so emit anything else
+        # as a single field rather than failing to format the whole entry.
+        if payload.is_a?(Hash)
+          payload.each_pair { |key, value| add_field(PAYLOAD_PREFIX, key, value) }
+        else
+          add_field(PAYLOAD_PREFIX, "payload", payload)
+        end
       end
 
       def handle_exception
         return unless @raw.key? :exception
 
-        @parsed[:tag] = "exception"
-        @parsed = @parsed.merge(@raw[:exception])
+        @parsed[TAG_FIELD] = "exception"
+        @raw[:exception].each_pair { |key, value| @parsed[EXCEPTION_FIELDS[key] || logfmt_key(key)] = value }
+      end
+
+      # Adds a field from logged data, under the name returned by #field_name.
+      def add_field(prefix, key, value)
+        name          = (key.is_a?(Symbol) && @plain_symbols[key]) || field_name(prefix, key)
+        @parsed[name] = value
+      end
+
+      # Returns [String] the logfmt name of a key from logged data: the key encoded by #logfmt_key,
+      # prefixed with its source, as in "payload.level", when it names one of the entry's own fields
+      # or starts with a source's prefix, so that untrusted data cannot replace or forge that field,
+      # or a field from another source.
+      def field_name(prefix, key)
+        name = key.is_a?(Symbol) ? key.name : key.to_s
+        name = logfmt_key(name) unless plain_name?(name)
+        return prefix + name if @reserved_names.key?(name) || name.start_with?(*SOURCE_PREFIXES)
+
+        remember_symbol(key, name) if key.is_a?(Symbol)
+        name
+      end
+
+      # Remembers the name of a Symbol key that is not reserved, while there is room. The cache is
+      # replaced rather than updated, since a formatter can be shared by several threads.
+      def remember_symbol(symbol, name)
+        plain_symbols  = @plain_symbols
+        @plain_symbols = plain_symbols.merge(symbol => name).freeze if plain_symbols.size < PLAIN_SYMBOL_CACHE_SIZE
       end
 
       def flatten_log
-        flattened = @parsed.map do |key, value|
-          case value
-          when Hash, Array
-            "#{key}=#{Utils.to_json(value.to_s)}"
-          else
-            "#{key}=#{Utils.to_json(value)}"
-          end
+        line = +""
+        @parsed.each_pair do |name, value|
+          line << " " unless line.empty?
+          line << name << "=" << format_value(value)
+        end
+        line
+      end
+
+      # Integers, true, false, and nil have a single JSON form, so they skip the cost of `to_json`.
+      # Strings, the most common values, are checked first.
+      def format_value(value)
+        return Utils.to_json(value) if value.is_a?(String)
+
+        case value
+        when Integer, true, false
+          value.to_s
+        when nil
+          "null"
+        else
+          # The JSON form rather than #inspect, which can print an object's instance variables
+          # and has no depth limit. A structure is quoted, since its JSON holds spaces and quotes.
+          json = Utils.to_json(value)
+          json.start_with?("{", "[") ? Utils.to_json(json) : json
         end
+      end
+
+      # Names the entry's own fields from the time key, and reserves those names and the
+      # exception's, so that logged data cannot use them.
+      def name_entry_fields
+        @entry_names    = [time_key, *ENTRY_FIELDS].to_h { |key| [key, logfmt_key(key)] }.freeze
+        @entry_keys     = @entry_names.keys.freeze
+        @reserved_names = [*@entry_names.values, TAG_FIELD, *EXCEPTION_FIELDS.values].
+                          to_h { |name| [name, true] }.freeze
+        # Symbol keys from logged data that need no prefix, mapped to their logfmt names.
+        @plain_symbols = {}.freeze
+      end
 
-        flattened.join(" ")
+      # Whether the name is already a valid logfmt key, made of ASCII characters that are all safe.
+      def plain_name?(name)
+        !name.empty? && name.ascii_only? && !name.match?(UNSAFE_ASCII_KEY_CHARS)
+      end
+
+      # Returns [String] the key as valid UTF-8, with every character that would break the logfmt
+      # structure replaced by an underscore. Keys come from tags, named tags, and payloads, which
+      # can hold untrusted data; unescaped, they could inject fields or forge an entire line.
+      def logfmt_key(key)
+        key = key.is_a?(Symbol) ? key.name : key.to_s
+        unsafe = UNSAFE_ASCII_KEY_CHARS
+        unless key.ascii_only?
+          key    = Utils.encode_utf8_string(key)
+          unsafe = UNSAFE_KEY_CHARS
+        end
+        key = key.gsub(unsafe, "_") if key.match?(unsafe)
+        key.empty? ? "_" : key
       end
     end
   end

data/lib/semantic_logger/formatters/loki.rb

@@ -81,13 +81,7 @@ module SemanticLogger
         return unless log.context && !log.context.empty?
 
         log.context.each do |key, value|
-          serialized_value = if value.is_a?(Hash)
-                               Utils.to_json(value)
-                             else
-                               value.to_s
-                             end
-
-          stream[:stream].merge!(key.to_s => serialized_value)
+          stream[:stream].merge!(key.to_s => loki_value(value))
         end
       end
 
@@ -104,11 +98,14 @@ module SemanticLogger
       end
 
       def payload
-        self.payload_value = if log.payload.respond_to?(:empty?) && !log.payload.empty?
-                               # Loki only accepts strings as key and values
+        self.payload_value = if !log.payload?
+                               {}
+                             elsif log.payload.is_a?(Hash)
                                stringify_hash(log.payload)
                              else
-                               {}
+                               # A log call can supply any value as the payload, such as a String
+                               # parsed from a request body.
+                               {"payload" => loki_value(log.payload)}
                              end
       end
 
@@ -136,21 +133,28 @@ module SemanticLogger
         )
       end
 
+      # Loki only accepts Strings as keys and values.
       def stringify_hash(hash)
-        result = {}
-
-        hash.each do |key, value|
-          string_key = key.to_s
-
-          result[string_key] = case value
-                               when Hash
-                                 Utils.to_json(stringify_hash(value))
-                               else
-                                 value.to_s
-                               end
+        hash.each_with_object({}) do |(key, value), result|
+          result[key.to_s] = loki_value(value)
         end
+      end
 
-        result
+      # Returns [String] the value as Loki accepts it.
+      #
+      # A structure is written once, as JSON, depth-limited and repaired by Utils.to_json.
+      # Encoding each nested level into a String that the next level encodes again doubled the
+      # escaping at every level, so a request parameter 30 levels deep produced gigabytes. A
+      # structure's #to_s is its #inspect, which would also expose the instance variables of any
+      # object it holds.
+      def loki_value(value)
+        if value.is_a?(String)
+          value
+        elsif Utils.structure?(value)
+          Utils.to_json(value)
+        else
+          value.to_s
+        end
       end
     end
   end

data/lib/semantic_logger/formatters/one_line.rb

@@ -3,15 +3,79 @@ module SemanticLogger
     # Only output one line for each log entry.
     #
     # Notes:
-    # * New lines are stripped from log messages.
+    # * New lines and carriage returns are stripped from every text field: the name, thread name,
+    #   file name, tags, named tags, message, payload and exception message.
+    #   With `escape_control_chars: true` they, and all other control characters, are escaped
+    #   instead, like the other text formatters.
     # * Exceptions only include the class and message, the stack trace is not shown.
+    # * Exception details, when enabled, are joined onto the same line.
     class OneLine < Default
+      def thread_name
+        format("%.30s", one_line(log.thread_name))
+      end
+
+      def tags
+        return if log.tags.nil? || log.tags.empty?
+
+        "[#{log.tags.map { |tag| one_line(tag) }.join('] [')}]"
+      end
+
+      def named_tags
+        named_tags = log.named_tags
+        return if named_tags.nil? || named_tags.empty?
+
+        list = named_tags.map { |name, value| "#{one_line(name)}: #{one_line(value)}" }
+        "{#{list.join(', ')}}"
+      end
+
+      def name
+        one_line(log.name) if log.name
+      end
+
+      # A log call can supply the backtrace that the file name comes from.
+      def file_name_and_line
+        file, line = log.file_name_and_line(true)
+        "#{one_line(file)}:#{line}" if file
+      end
+
       def message
-        "-- #{log.message.delete("\n")}" if log.message
+        "-- #{one_line(log.message)}" if log.message
+      end
+
+      def payload
+        pl = log.payload_to_s
+        "-- #{one_line(pl)}" if pl
       end
 
       def exception
-        "-- Exception: #{log.exception.class}: #{log.exception.message.delete("\n")}" if log.exception
+        return unless log.exception
+
+        details = exception_details
+        "-- Exception: #{log.exception.class}: #{one_line(log.exception.message)}#{" -- #{details}" if details}"
+      end
+
+      # The exception details, with each line trimmed and joined by a space.
+      def exception_details
+        details = exception_details_for(log.exception) if log.exception
+        return unless details
+
+        one_line(Utils.encode_utf8_string(details).split(/\s*\n\s*/).reject(&:empty?).join(" ").strip)
+      end
+
+      private
+
+      # Off by default, so that existing one line output does not change.
+      def default_log_exception_details
+        false
+      end
+
+      # Returns [String] the value without line breaks, so that the entry stays on one line.
+      #
+      # A String with invalid bytes, such as a request parameter decoded from "%FF", or in another
+      # encoding, such as a binary request body, is converted to valid UTF-8 first, since deleting
+      # from it, or joining it with UTF-8 text, would raise and lose the entry.
+      def one_line(value)
+        Utils.encode_utf8_string(escape_control_characters(value.to_s)).delete("\r\n")
       end
     end
   end

data/lib/semantic_logger/formatters/open_telemetry.rb

@@ -20,21 +20,29 @@ module SemanticLogger
         hash[:level_index] = severity_number(log.level)
       end
 
-      # Payload is submitted directly as attributes
+      # Payload is submitted directly as attributes, which must be a Hash. A log call can supply
+      # any value as the payload, such as a String parsed from a request body, so any other value
+      # is submitted as a single "payload" attribute.
       def payload
-        return unless log.payload.respond_to?(:empty?) && !log.payload.empty?
+        return unless log.payload?
 
-        hash[:payload] = coerce_map(log.payload)
+        hash[:payload] = coerce_map(log.payload.is_a?(Hash) ? log.payload : {payload: log.payload})
       end
 
       private
 
+      # An attribute value: a primitive, or an Array of them.
       def coerce_value(v)
+        v.is_a?(Array) ? v.map { |e| coerce_element(e) }.compact : coerce_element(v)
+      end
+
+      # A primitive as-is, and anything else as a String: a structure (such as a Hash, Struct, or
+      # Set) as JSON, depth-limited and repaired by Utils.to_json, rather than its #to_s, which is
+      # its #inspect and would expose the instance variables of any object it holds.
+      def coerce_element(v)
         case v
-        when *PRIMS then v
-        when Array  then v.map { |e| coerce_value(e) }.compact   # arrays of scalars only.
-        when NilClass then nil                                   # drop nils by caller.
-        else v.to_s                                              # stringify objects / hashes.
+        when *PRIMS, NilClass then v
+        else Utils.structure?(v) ? Utils.to_json(v) : v.to_s
         end
       end
 
@@ -42,16 +50,7 @@ module SemanticLogger
         h.each_with_object({}) do |(k, v), out|
           next if v.nil?
 
-          out[k.to_s] =
-            if v.is_a?(Hash)
-              # Stringify whole hash. Cleanse here too: this runs while building the
-              # hash, before the top-level call cleanse, so it must not raise on its own.
-              Utils.to_json(
-                v.transform_values { |vv| coerce_value(vv) }.transform_keys!(&:to_s)
-              )
-            else
-              coerce_value(v)
-            end
+          out[k.to_s] = coerce_value(v)
         end
       end
 

data/lib/semantic_logger/formatters/pattern.rb

@@ -30,6 +30,8 @@ module SemanticLogger
     #   payload              Payload rendered as a string.
     #   exception_class      Class of the logged exception, e.g. "RuntimeError".
     #   exception_message    Message of the logged exception.
+    #   exception_details    Detail Ruby adds to the exception's message, such as
+    #                        did_you_mean suggestions (see the log_exception_details option).
     #   backtrace            Backtrace of the logged exception.
     #   duration             Human readable duration, e.g. "1.2ms".
     #   duration_ms          Duration in milliseconds (numeric).
@@ -59,6 +61,7 @@ module SemanticLogger
         payload:           false,
         exception_class:   false,
         exception_message: false,
+        exception_details: false,
         backtrace:         false,
         duration:          false,
         duration_ms:       false,
@@ -118,7 +121,7 @@ module SemanticLogger
 
       # Raw payload rendered as a string.
       def payload
-        log.payload_to_s
+        escape_control_characters(log.payload_to_s)
       end
 
       # Class of the logged exception, e.g. "RuntimeError".
@@ -133,7 +136,7 @@ module SemanticLogger
 
       # Backtrace of the logged exception.
       def backtrace
-        log.backtrace_to_s if log.exception
+        exception_backtrace if log.exception
       end
 
       # Human readable duration (without the Default formatter's surrounding parentheses).
@@ -146,9 +149,10 @@ module SemanticLogger
         log.duration
       end
 
-      # Name of the Ruby file that logged the message, e.g. "app.rb".
+      # Name of the Ruby file that logged the message, e.g. "app.rb". Escaped like the other
+      # fields, since a log call can supply the backtrace it comes from.
       def file_name
-        log.file_name_and_line(true)&.first
+        escape_control_characters(log.file_name_and_line(true)&.first)
       end
 
       # Line number within the Ruby file that logged the message.

data/lib/semantic_logger/formatters/raw.rb

@@ -96,11 +96,10 @@ module SemanticLogger
         root = hash
         log.each_exception do |exception, i|
           name       = i.zero? ? :exception : :cause
-          root[name] = {
-            name:        exception.class.name,
-            message:     exception.message,
-            stack_trace: exception.backtrace
-          }
+          root[name] = {name: exception.class.name, message: exception.message}
+          details    = exception_details_for(exception)
+          root[name][:details]     = details if details
+          root[name][:stack_trace] = exception.backtrace
           root = root[name]
         end
       end