semantic_logger 4.18.0 → 5.0.0

data/lib/semantic_logger/formatters/base.rb

@@ -2,7 +2,21 @@ require "time"
 module SemanticLogger
   module Formatters
     class Base
-      attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision
+      attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
+                    :escape_control_chars
+
+      # Printable escapes for the most common control characters. Any other
+      # control character is escaped to its hexadecimal `\xHH` form by
+      # #escape_control_characters.
+      CONTROL_CHAR_ESCAPES = {
+        "\n" => "\\n",
+        "\r" => "\\r",
+        "\t" => "\\t",
+        "\e" => "\\e"
+      }.freeze
+
+      # Matches C0 control characters (including newlines and the ANSI escape) and DEL.
+      CONTROL_CHARS = /[\x00-\x1f\x7f]/
 
       # Time precision varies by Ruby interpreter
       # JRuby 9.1.8.0 supports microseconds
@@ -34,16 +48,27 @@ module SemanticLogger
       #   precision: [Integer]
       #     How many fractional digits to log times with.
       #     Default: PRECISION (6, except on older JRuby, where 3)
+      #   escape_control_chars: [Boolean]
+      #     Replace control characters (newlines, the ANSI escape, etc.) in
+      #     untrusted log data (message, tags, named tags, and exception
+      #     message) with a printable escaped form, e.g. "\n".
+      #     This prevents log forging and terminal escape-sequence injection
+      #     when logging data that may contain attacker-controlled content.
+      #     Note: Has no effect on structured formatters such as :json, which
+      #     already escape control characters via JSON encoding.
+      #     Default: false (preserve newlines and ANSI colors in text output)
       def initialize(time_format: nil,
                      log_host: true,
                      log_application: true,
                      log_environment: true,
-                     precision: PRECISION)
-        @time_format     = time_format || self.class.build_time_format(precision)
-        @log_host        = log_host
-        @log_application = log_application
-        @log_environment = log_environment
-        @precision       = precision
+                     precision: PRECISION,
+                     escape_control_chars: false)
+        @time_format          = time_format || self.class.build_time_format(precision)
+        @log_host             = log_host
+        @log_application      = log_application
+        @log_environment      = log_environment
+        @precision            = precision
+        @escape_control_chars = escape_control_chars
       end
 
       # Return default time format string
@@ -68,6 +93,20 @@ module SemanticLogger
 
       private
 
+      # When `escape_control_chars` is enabled, return a copy of the supplied
+      # value with any control characters replaced by a printable escaped form
+      # so that untrusted log data cannot forge log entries or inject terminal
+      # escape sequences. Otherwise the value is returned unchanged.
+      #
+      # Note: This escapes (preserves) control characters, and is opt-in. It is
+      # distinct from Log#cleansed_message, which unconditionally *strips* ANSI
+      # colorization from the message for structured (JSON/Loki) output.
+      def escape_control_characters(value)
+        return value unless escape_control_chars && value
+
+        value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
+      end
+
       # Return the Time as a formatted string
       def format_time(time)
         time = time.dup

data/lib/semantic_logger/formatters/color.rb

@@ -85,7 +85,10 @@ module SemanticLogger
       end
 
       def tags
-        "[#{color}#{log.tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]" if log.tags && !log.tags.empty?
+        return if log.tags.nil? || log.tags.empty?
+
+        tags = log.tags.map { |tag| escape_control_characters(tag) }
+        "[#{color}#{tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]"
       end
 
       # Named Tags
@@ -94,7 +97,7 @@ module SemanticLogger
         return if named_tags.nil? || named_tags.empty?
 
         list = []
-        named_tags.each_pair { |name, value| list << "#{color}#{name}: #{value}#{color_map.clear}" }
+        named_tags.each_pair { |name, value| list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}" }
         "{#{list.join(', ')}}"
       end
 
@@ -123,7 +126,7 @@ module SemanticLogger
       def exception
         return unless log.exception
 
-        "-- Exception: #{color}#{log.exception.class}: #{log.exception.message}#{color_map.clear}\n#{log.backtrace_to_s}"
+        "-- Exception: #{color}#{log.exception.class}: #{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
       end
 
       def call(log, logger)

data/lib/semantic_logger/formatters/default.rb

@@ -32,7 +32,9 @@ module SemanticLogger
 
       # Tags
       def tags
-        "[#{log.tags.join('] [')}]" if log.tags && !log.tags.empty?
+        return if log.tags.nil? || log.tags.empty?
+
+        "[#{log.tags.map { |tag| escape_control_characters(tag) }.join('] [')}]"
       end
 
       # Named Tags
@@ -41,7 +43,7 @@ module SemanticLogger
         return if named_tags.nil? || named_tags.empty?
 
         list = []
-        named_tags.each_pair { |name, value| list << "#{name}: #{value}" }
+        named_tags.each_pair { |name, value| list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }
         "{#{list.join(', ')}}"
       end
 
@@ -57,7 +59,7 @@ module SemanticLogger
 
       # Log message
       def message
-        "-- #{log.message}" if log.message
+        "-- #{escape_control_characters(log.message)}" if log.message
       end
 
       # Payload
@@ -70,7 +72,7 @@ module SemanticLogger
 
       # Exception
       def exception
-        "-- Exception: #{log.exception.class}: #{log.exception.message}\n#{log.backtrace_to_s}" if log.exception
+        "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n#{log.backtrace_to_s}" if log.exception
       end
 
       # Default text log format

data/lib/semantic_logger/formatters/ecs.rb

@@ -0,0 +1,151 @@
+require "json"
+module SemanticLogger
+  module Formatters
+    # Formatter conforming to the Elastic Common Schema (ECS).
+    #
+    # Emits log events using the nested field names defined by ECS so that they
+    # integrate cleanly with Filebeat and the Elastic stack (Elasticsearch,
+    # Kibana) without requiring an ingest pipeline to rename fields.
+    #
+    # Usage:
+    #   SemanticLogger.add_appender(io: $stdout, formatter: :ecs)
+    #
+    #   # Route the payload, metric, and other SemanticLogger-specific data into
+    #   # a custom top-level namespace (default "semantic_logger"):
+    #   SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: "my_app"}})
+    #
+    #   # Or merge the payload directly into ECS `labels` instead of a namespace:
+    #   SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: nil}})
+    #
+    # == Field mapping (SemanticLogger -> ECS 8.x)
+    #   time                  -> @timestamp        (ISO-8601)
+    #   level                 -> log.level
+    #   name                  -> log.logger
+    #   file_name / line      -> log.origin.file.name / log.origin.file.line
+    #   message               -> message
+    #   thread_name           -> process.thread.name
+    #   pid                   -> process.pid
+    #   host                  -> host.hostname
+    #   application           -> service.name
+    #   environment           -> service.environment
+    #   exception             -> error.type / error.message / error.stack_trace
+    #   duration              -> event.duration   (nanoseconds, as required by ECS)
+    #   tags                  -> tags             (ECS top-level array)
+    #   named_tags            -> labels.*         (scalar key/value pairs)
+    #   payload               -> <namespace>.*    (or labels.* when namespace is nil)
+    #   metric/metric_amount  -> <namespace>.metric / <namespace>.metric_amount
+    #
+    # == Reference
+    #   * https://www.elastic.co/docs/reference/ecs
+    #   * https://www.elastic.co/docs/reference/ecs/ecs-custom-fields-in-ecs
+    class Ecs < Raw
+      # ECS version this formatter targets.
+      ECS_VERSION = "8.11.0".freeze
+
+      # namespace: [String|Symbol|nil]
+      #   Top-level field set used to hold SemanticLogger-specific data that has
+      #   no native ECS home (payload, metric, metric_amount). A proper-noun
+      #   namespace is guaranteed never to collide with a current or future ECS
+      #   field. Set to nil to merge the payload into ECS `labels` instead.
+      #   Default: "semantic_logger"
+      attr_reader :namespace
+
+      def initialize(namespace: "semantic_logger", time_format: :iso_8601, time_key: :timestamp, **args)
+        @namespace = namespace&.to_sym
+
+        super(time_format: time_format, time_key: time_key, **args)
+      end
+
+      # Returns the log event as a single line of ECS-formatted JSON, so it can
+      # be written to stdout / a file and shipped by Filebeat or Elastic Agent.
+      def call(log, logger)
+        Utils.to_json(ecs_hash(super))
+      end
+
+      # Returns a batch of log events as a single JSON array.
+      def batch(logs, logger)
+        "[#{logs.map { |log| call(log, logger) }.join(',')}]"
+      end
+
+      private
+
+      # Remap the flat hash built by Raw#call into the nested ECS field layout.
+      def ecs_hash(hash)
+        result = base(hash)
+        result[:process] = process(hash)
+        result[:host]    = {hostname: hash[:host]} if hash[:host]
+        add_service(result, hash)
+        add_origin(result, hash)
+        add_event(result, hash)
+        result[:tags] = hash[:tags] if hash[:tags]
+        add_error(result, hash)
+        add_extras(result, hash)
+        result
+      end
+
+      def base(hash)
+        log = {level: hash[:level].to_s}
+        log[:logger] = hash[:name] if hash[:name]
+        {
+          "@timestamp": hash[:timestamp],
+          message:      hash[:message],
+          ecs:          {version: ECS_VERSION},
+          log:          log
+        }
+      end
+
+      def process(hash)
+        result = {pid: hash[:pid]}
+        result[:thread] = {name: hash[:thread].to_s} if hash[:thread]
+        result
+      end
+
+      def add_service(result, hash)
+        service = {}
+        service[:name]        = hash[:application] if hash[:application]
+        service[:environment] = hash[:environment] if hash[:environment]
+        result[:service] = service unless service.empty?
+      end
+
+      def add_origin(result, hash)
+        return unless hash[:file]
+
+        result[:log][:origin] = {file: {name: hash[:file], line: hash[:line]}.compact}
+      end
+
+      # ECS event.duration is measured in nanoseconds.
+      def add_event(result, hash)
+        return unless hash[:duration_ms]
+
+        result[:event] = {duration: (hash[:duration_ms] * 1_000_000).round}
+      end
+
+      def add_error(result, hash)
+        return unless hash[:exception]
+
+        result[:error] = {
+          type:        hash[:exception][:name],
+          message:     hash[:exception][:message],
+          stack_trace: Array(hash[:exception][:stack_trace]).join("\n")
+        }
+      end
+
+      # Place SemanticLogger-specific data (payload, metric) that has no native
+      # ECS home into the configured namespace, plus named_tags into labels.
+      def add_extras(result, hash)
+        labels = hash[:named_tags].is_a?(Hash) ? hash[:named_tags].dup : {}
+
+        extra = {}
+        extra[:payload]       = hash[:payload] if hash[:payload]
+        extra[:metric]        = hash[:metric] if hash[:metric]
+        extra[:metric_amount] = hash[:metric_amount] if hash[:metric_amount]
+
+        unless extra.empty?
+          namespace ? result[namespace] = extra : labels.merge!(extra)
+        end
+
+        result[:labels] = labels unless labels.empty?
+      end
+    end
+  end
+end

data/lib/semantic_logger/formatters/fluentd.rb

@@ -7,9 +7,9 @@ module SemanticLogger
     class Fluentd < Json
       attr_reader :need_process_info
 
-      def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
+      def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, log_host: false, **args)
         @need_process_info = need_process_info
-        super(time_format: time_format, time_key: time_key, **args)
+        super(time_format: time_format, time_key: time_key, log_host: log_host, **args)
       end
 
       def level
@@ -17,8 +17,19 @@ module SemanticLogger
         hash["severity_index"] = log.level_index
       end
 
-      def process_info
-        # Ignore fields: pid, thread, file and line by default
+      # Ignore process fields: pid, thread, file and line by default.
+      # These are rarely useful under Fluentd (e.g. containerized processes
+      # usually have pid 1), so they are only included when explicitly requested
+      # via `need_process_info: true`.
+      def pid
+        super if need_process_info
+      end
+
+      def thread_name
+        super if need_process_info
+      end
+
+      def file_name_and_line
         super if need_process_info
       end
     end

data/lib/semantic_logger/formatters/json.rb

@@ -9,7 +9,12 @@ module SemanticLogger
 
       # Returns log messages in JSON format
       def call(log, logger)
-        super.to_json
+        Utils.to_json(super)
+      end
+
+      # Returns a batch of log messages as a single JSON array.
+      def batch(logs, logger)
+        "[#{logs.map { |log| call(log, logger) }.join(',')}]"
       end
     end
   end

data/lib/semantic_logger/formatters/logfmt.rb

@@ -64,9 +64,9 @@ module SemanticLogger
         flattened = @parsed.map do |key, value|
           case value
           when Hash, Array
-            "#{key}=#{value.to_s.to_json}"
+            "#{key}=#{Utils.to_json(value.to_s)}"
           else
-            "#{key}=#{value.to_json}"
+            "#{key}=#{Utils.to_json(value)}"
           end
         end
 

data/lib/semantic_logger/formatters/loki.rb

@@ -10,7 +10,7 @@ module SemanticLogger
         self.logger = logger
         self.log = log
 
-        {streams: [build_stream]}.to_json
+        Utils.to_json({streams: [build_stream]})
       end
 
       # Returns [String] a JSON batch of logs
@@ -22,7 +22,7 @@ module SemanticLogger
           build_stream
         end
 
-        {streams: streams}.to_json
+        Utils.to_json({streams: streams})
       end
 
       private
@@ -82,7 +82,7 @@ module SemanticLogger
 
         log.context.each do |key, value|
           serialized_value = if value.is_a?(Hash)
-                               value.to_json
+                               Utils.to_json(value)
                              else
                                value.to_s
                              end
@@ -144,7 +144,7 @@ module SemanticLogger
 
           result[string_key] = case value
                                when Hash
-                                 JSON.generate(stringify_hash(value))
+                                 Utils.to_json(stringify_hash(value))
                                else
                                  value.to_s
                                end

data/lib/semantic_logger/formatters/open_telemetry.rb

@@ -5,10 +5,19 @@ module SemanticLogger
       # primitives allowed by OTLP logs in Ruby: String, Integer, Float, TrueClass, FalseClass
       PRIMS = [String, Integer, Float, TrueClass, FalseClass].freeze
 
+      # Returns the attributes hash submitted to the OpenTelemetry SDK.
+      #
+      # Unlike the JSON formatters there is no single `.to_json` boundary here:
+      # the hash is handed to the OTLP exporter, which requires valid UTF-8. Cleanse
+      # the whole structure so binary / non UTF-8 strings cannot break the export.
+      def call(log, logger)
+        Utils.encode_utf8(super)
+      end
+
       # Log level
       def level
         hash[:level]       = log.level.to_s
-        hash[:level_index] = severity_number(log.level_index)
+        hash[:level_index] = severity_number(log.level)
       end
 
       # Payload is submitted directly as attributes
@@ -35,10 +44,11 @@ module SemanticLogger
 
           out[k.to_s] =
             if v.is_a?(Hash)
-              # Stringify whole hash.
-              v.transform_values { |vv| coerce_value(vv) }.
-                transform_keys!(&:to_s).
-                to_json
+              # Stringify whole hash. Cleanse here too: this runs while building the
+              # hash, before the top-level call cleanse, so it must not raise on its own.
+              Utils.to_json(
+                v.transform_values { |vv| coerce_value(vv) }.transform_keys!(&:to_s)
+              )
             else
               coerce_value(v)
             end

data/lib/semantic_logger/formatters/pattern.rb

@@ -0,0 +1,235 @@
+require "time"
+
+module SemanticLogger
+  module Formatters
+    # Formats log messages using a configurable pattern string, so a custom
+    # log line layout can be specified directly in the configuration without
+    # having to write a new formatter class.
+    #
+    # Pattern placeholders use the form `%{directive}`, where `directive` is the
+    # name of any of the formatting methods (inherited from Default, or defined
+    # below). Named tags support a parameterized form: `%{named_tags:request_id}`
+    # returns the value of a single named tag. Use `%%{...}` to emit a literal
+    # `%{...}` without interpolation.
+    #
+    # Example:
+    #   SemanticLogger.add_appender(
+    #     io:        $stdout,
+    #     formatter: {
+    #       pattern: { pattern: "%{time} %{level} %{name} -- %{message}" }
+    #     }
+    #   )
+    #
+    # Available directives:
+    #   time                 Formatted timestamp. Optionally accepts a strftime
+    #                        format, e.g. time:%Y-%m-%dT%H:%M:%S.%6N.
+    #   level                Full level name, e.g. "debug".
+    #   level_short          Single character level, e.g. "D".
+    #   name                 Logger / class name.
+    #   message              Log message.
+    #   payload              Payload rendered as a string.
+    #   exception_class      Class of the logged exception, e.g. "RuntimeError".
+    #   exception_message    Message of the logged exception.
+    #   backtrace            Backtrace of the logged exception.
+    #   duration             Human readable duration, e.g. "1.2ms".
+    #   duration_ms          Duration in milliseconds (numeric).
+    #   thread_name          Name of the thread that logged the message.
+    #   pid                  Process id.
+    #   file_name            Ruby file name that logged the message, e.g. "app.rb".
+    #   line                 Line number within the Ruby file, e.g. 42.
+    #   tags                 Tags, comma separated.
+    #   named_tags           All named tags, or one tag with named_tags:key.
+    #   host                 Host name.
+    #   application          Application name.
+    #   environment          Environment name.
+    class Pattern < Default
+      attr_reader :pattern
+
+      # Approximates the Default formatter's output.
+      DEFAULT_PATTERN = "%{time} %{level} [%{pid}:%{thread_name}] %{name} -- %{message}".freeze
+
+      # The directives that may appear in a pattern. The value is whether the
+      # directive accepts a parameter, e.g. %{named_tags:request_id}.
+      DIRECTIVES = {
+        time:              true,
+        level:             false,
+        level_short:       false,
+        name:              false,
+        message:           false,
+        payload:           false,
+        exception_class:   false,
+        exception_message: false,
+        backtrace:         false,
+        duration:          false,
+        duration_ms:       false,
+        thread_name:       false,
+        pid:               false,
+        file_name:         false,
+        line:              false,
+        tags:              false,
+        named_tags:        true,
+        host:              false,
+        application:       false,
+        environment:       false
+      }.freeze
+
+      # A single interpolated directive within a compiled pattern.
+      Token = Struct.new(:method_name, :arguments)
+      private_constant :Token
+
+      # Parameters:
+      #   pattern: [String]
+      #     The pattern string used to format every log entry.
+      #     Default: DEFAULT_PATTERN
+      #
+      # Plus all the options supported by SemanticLogger::Formatters::Base.
+      def initialize(pattern: DEFAULT_PATTERN, **args)
+        @pattern = pattern
+        super(**args)
+        # Parse the pattern once, up front, so that formatting every log entry
+        # is just a walk over the pre-compiled tokens (no regex on the hot path).
+        # Unknown directives raise here, at configuration time, not per log.
+        @tokens = compile(pattern)
+      end
+
+      # Formatted timestamp. With a strftime format argument, e.g.
+      # %{time:%Y-%m-%dT%H:%M:%S.%6N}, the time is formatted with that string.
+      # Without an argument it uses the formatter's configured time_format.
+      def time(format = nil)
+        return super() if format.nil?
+
+        log.time.strftime(format)
+      end
+
+      # Full level name, e.g. "debug" (Default formatter uses the short "D").
+      def level
+        log.level.to_s
+      end
+
+      # Single character level, e.g. "D".
+      def level_short
+        log.level_to_s
+      end
+
+      # Log message (without the "-- " prefix the Default formatter adds).
+      def message
+        escape_control_characters(log.message)
+      end
+
+      # Raw payload rendered as a string.
+      def payload
+        log.payload_to_s
+      end
+
+      # Class of the logged exception, e.g. "RuntimeError".
+      def exception_class
+        log.exception&.class
+      end
+
+      # Message of the logged exception.
+      def exception_message
+        escape_control_characters(log.exception&.message)
+      end
+
+      # Backtrace of the logged exception.
+      def backtrace
+        log.backtrace_to_s if log.exception
+      end
+
+      # Human readable duration (without the Default formatter's surrounding parentheses).
+      def duration
+        log.duration_human
+      end
+
+      # Duration in milliseconds.
+      def duration_ms
+        log.duration
+      end
+
+      # Name of the Ruby file that logged the message, e.g. "app.rb".
+      def file_name
+        log.file_name_and_line(true)&.first
+      end
+
+      # Line number within the Ruby file that logged the message.
+      def line
+        log.file_name_and_line(true)&.last
+      end
+
+      # Tags joined by a comma (without the Default formatter's surrounding brackets).
+      def tags
+        return if log.tags.nil? || log.tags.empty?
+
+        log.tags.map { |tag| escape_control_characters(tag) }.join(", ")
+      end
+
+      # With a key: the value of a single named tag, e.g. %{named_tags:request_id}.
+      # Without a key: all named tags rendered as "key: value, ...".
+      def named_tags(key = nil)
+        named = log.named_tags
+        return if named.nil? || named.empty?
+
+        if key
+          escape_control_characters(named[key.to_sym] || named[key.to_s])
+        else
+          named.map { |name, value| "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }.join(", ")
+        end
+      end
+
+      # Host name.
+      def host
+        logger&.host if log_host
+      end
+
+      # Application name.
+      def application
+        logger&.application if log_application
+      end
+
+      # Environment name.
+      def environment
+        logger&.environment if log_environment
+      end
+
+      def call(log, logger)
+        self.log    = log
+        self.logger = logger
+
+        @tokens.each_with_object(+"") do |token, out|
+          out << (token.is_a?(Token) ? public_send(token.method_name, *token.arguments).to_s : token)
+        end
+      end
+
+      private
+
+      # Parse the pattern string into an array of tokens: frozen literal strings
+      # and Token structs for each %{directive} placeholder. %%{...} is an escape
+      # that produces a literal %{...}.
+      def compile(string)
+        tokens = []
+        pos    = 0
+
+        string.scan(/%%?\{[^}]+\}/) do |match|
+          current = Regexp.last_match
+          tokens << string[pos...current.begin(0)].freeze if current.begin(0) > pos
+
+          if match.start_with?("%%")
+            tokens << match[1..].freeze
+          else
+            name, arg = match[/\{([^}]+)\}/, 1].split(":", 2)
+            name      = name.strip.to_sym
+            raise(ArgumentError, "Invalid pattern directive: %{#{name}}") unless DIRECTIVES.key?(name)
+            raise(ArgumentError, "%{#{name}} does not accept an argument") if arg && !DIRECTIVES[name]
+
+            tokens << Token.new(name, arg ? [arg.strip] : []).freeze
+          end
+
+          pos = current.end(0)
+        end
+
+        tokens << string[pos..].freeze if pos < string.length
+        tokens
+      end
+    end
+  end
+end

data/lib/semantic_logger/formatters/raw.rb

@@ -18,12 +18,12 @@ module SemanticLogger
 
       # Application name
       def application
-        hash[:application] = logger.application if log_application && logger && logger.application
+        hash[:application] = logger.application if log_application && logger&.application
       end
 
       # Environment
       def environment
-        hash[:environment] = logger.environment if log_environment && logger && logger.environment
+        hash[:environment] = logger.environment if log_environment && logger&.environment
       end
 
       # Date & time