semantic_logger 4.17.0 → 5.0.0

data/lib/semantic_logger/formatters/new_relic_logs.rb

@@ -47,7 +47,7 @@ module SemanticLogger
         hash = super
 
         result = {
-          **newrelic_metadata,
+          **linking_metadata(log),
           message:       hash[:message].to_s,
           tags:          hash[:tags],
           metric:        hash[:metric],
@@ -115,11 +115,9 @@ module SemanticLogger
 
       private
 
-      # NOTE: This function will already include trace.id and span.id if they
-      # are available so I believe the previous implementation of this is redundant
-      # https://rubydoc.info/gems/newrelic_rpm/NewRelic/Agent#linking_metadata-instance_method
-      def newrelic_metadata
-        NewRelic::Agent.linking_metadata.transform_keys(&:to_sym)
+      # returns "entity.name", "trace.id" etc from NewRelic::Agent.linking_metadata
+      def linking_metadata(log)
+        (log.context && log.context[:new_relic_metadata]) || {}
       end
     end
   end

data/lib/semantic_logger/formatters/open_telemetry.rb

@@ -2,21 +2,59 @@ require "json"
 module SemanticLogger
   module Formatters
     class OpenTelemetry < Raw
+      # primitives allowed by OTLP logs in Ruby: String, Integer, Float, TrueClass, FalseClass
+      PRIMS = [String, Integer, Float, TrueClass, FalseClass].freeze
+
+      # Returns the attributes hash submitted to the OpenTelemetry SDK.
+      #
+      # Unlike the JSON formatters there is no single `.to_json` boundary here:
+      # the hash is handed to the OTLP exporter, which requires valid UTF-8. Cleanse
+      # the whole structure so binary / non UTF-8 strings cannot break the export.
+      def call(log, logger)
+        Utils.encode_utf8(super)
+      end
+
       # Log level
       def level
         hash[:level]       = log.level.to_s
-        hash[:level_index] = severity_number(log.level_index)
+        hash[:level_index] = severity_number(log.level)
       end
 
       # Payload is submitted directly as attributes
       def payload
         return unless log.payload.respond_to?(:empty?) && !log.payload.empty?
 
-        hash[:payload] = log.payload.transform_keys!(&:to_s)
+        hash[:payload] = coerce_map(log.payload)
       end
 
       private
 
+      def coerce_value(v)
+        case v
+        when *PRIMS then v
+        when Array  then v.map { |e| coerce_value(e) }.compact   # arrays of scalars only.
+        when NilClass then nil                                   # drop nils by caller.
+        else v.to_s                                              # stringify objects / hashes.
+        end
+      end
+
+      def coerce_map(h)
+        h.each_with_object({}) do |(k, v), out|
+          next if v.nil?
+
+          out[k.to_s] =
+            if v.is_a?(Hash)
+              # Stringify whole hash. Cleanse here too: this runs while building the
+              # hash, before the top-level call cleanse, so it must not raise on its own.
+              Utils.to_json(
+                v.transform_values { |vv| coerce_value(vv) }.transform_keys!(&:to_s)
+              )
+            else
+              coerce_value(v)
+            end
+        end
+      end
+
       def severity_number(severity)
         case severity
         when :trace

data/lib/semantic_logger/formatters/pattern.rb

@@ -0,0 +1,235 @@
+require "time"
+
+module SemanticLogger
+  module Formatters
+    # Formats log messages using a configurable pattern string, so a custom
+    # log line layout can be specified directly in the configuration without
+    # having to write a new formatter class.
+    #
+    # Pattern placeholders use the form `%{directive}`, where `directive` is the
+    # name of any of the formatting methods (inherited from Default, or defined
+    # below). Named tags support a parameterized form: `%{named_tags:request_id}`
+    # returns the value of a single named tag. Use `%%{...}` to emit a literal
+    # `%{...}` without interpolation.
+    #
+    # Example:
+    #   SemanticLogger.add_appender(
+    #     io:        $stdout,
+    #     formatter: {
+    #       pattern: { pattern: "%{time} %{level} %{name} -- %{message}" }
+    #     }
+    #   )
+    #
+    # Available directives:
+    #   time                 Formatted timestamp. Optionally accepts a strftime
+    #                        format, e.g. time:%Y-%m-%dT%H:%M:%S.%6N.
+    #   level                Full level name, e.g. "debug".
+    #   level_short          Single character level, e.g. "D".
+    #   name                 Logger / class name.
+    #   message              Log message.
+    #   payload              Payload rendered as a string.
+    #   exception_class      Class of the logged exception, e.g. "RuntimeError".
+    #   exception_message    Message of the logged exception.
+    #   backtrace            Backtrace of the logged exception.
+    #   duration             Human readable duration, e.g. "1.2ms".
+    #   duration_ms          Duration in milliseconds (numeric).
+    #   thread_name          Name of the thread that logged the message.
+    #   pid                  Process id.
+    #   file_name            Ruby file name that logged the message, e.g. "app.rb".
+    #   line                 Line number within the Ruby file, e.g. 42.
+    #   tags                 Tags, comma separated.
+    #   named_tags           All named tags, or one tag with named_tags:key.
+    #   host                 Host name.
+    #   application          Application name.
+    #   environment          Environment name.
+    class Pattern < Default
+      attr_reader :pattern
+
+      # Approximates the Default formatter's output.
+      DEFAULT_PATTERN = "%{time} %{level} [%{pid}:%{thread_name}] %{name} -- %{message}".freeze
+
+      # The directives that may appear in a pattern. The value is whether the
+      # directive accepts a parameter, e.g. %{named_tags:request_id}.
+      DIRECTIVES = {
+        time:              true,
+        level:             false,
+        level_short:       false,
+        name:              false,
+        message:           false,
+        payload:           false,
+        exception_class:   false,
+        exception_message: false,
+        backtrace:         false,
+        duration:          false,
+        duration_ms:       false,
+        thread_name:       false,
+        pid:               false,
+        file_name:         false,
+        line:              false,
+        tags:              false,
+        named_tags:        true,
+        host:              false,
+        application:       false,
+        environment:       false
+      }.freeze
+
+      # A single interpolated directive within a compiled pattern.
+      Token = Struct.new(:method_name, :arguments)
+      private_constant :Token
+
+      # Parameters:
+      #   pattern: [String]
+      #     The pattern string used to format every log entry.
+      #     Default: DEFAULT_PATTERN
+      #
+      # Plus all the options supported by SemanticLogger::Formatters::Base.
+      def initialize(pattern: DEFAULT_PATTERN, **args)
+        @pattern = pattern
+        super(**args)
+        # Parse the pattern once, up front, so that formatting every log entry
+        # is just a walk over the pre-compiled tokens (no regex on the hot path).
+        # Unknown directives raise here, at configuration time, not per log.
+        @tokens = compile(pattern)
+      end
+
+      # Formatted timestamp. With a strftime format argument, e.g.
+      # %{time:%Y-%m-%dT%H:%M:%S.%6N}, the time is formatted with that string.
+      # Without an argument it uses the formatter's configured time_format.
+      def time(format = nil)
+        return super() if format.nil?
+
+        log.time.strftime(format)
+      end
+
+      # Full level name, e.g. "debug" (Default formatter uses the short "D").
+      def level
+        log.level.to_s
+      end
+
+      # Single character level, e.g. "D".
+      def level_short
+        log.level_to_s
+      end
+
+      # Log message (without the "-- " prefix the Default formatter adds).
+      def message
+        escape_control_characters(log.message)
+      end
+
+      # Raw payload rendered as a string.
+      def payload
+        log.payload_to_s
+      end
+
+      # Class of the logged exception, e.g. "RuntimeError".
+      def exception_class
+        log.exception&.class
+      end
+
+      # Message of the logged exception.
+      def exception_message
+        escape_control_characters(log.exception&.message)
+      end
+
+      # Backtrace of the logged exception.
+      def backtrace
+        log.backtrace_to_s if log.exception
+      end
+
+      # Human readable duration (without the Default formatter's surrounding parentheses).
+      def duration
+        log.duration_human
+      end
+
+      # Duration in milliseconds.
+      def duration_ms
+        log.duration
+      end
+
+      # Name of the Ruby file that logged the message, e.g. "app.rb".
+      def file_name
+        log.file_name_and_line(true)&.first
+      end
+
+      # Line number within the Ruby file that logged the message.
+      def line
+        log.file_name_and_line(true)&.last
+      end
+
+      # Tags joined by a comma (without the Default formatter's surrounding brackets).
+      def tags
+        return if log.tags.nil? || log.tags.empty?
+
+        log.tags.map { |tag| escape_control_characters(tag) }.join(", ")
+      end
+
+      # With a key: the value of a single named tag, e.g. %{named_tags:request_id}.
+      # Without a key: all named tags rendered as "key: value, ...".
+      def named_tags(key = nil)
+        named = log.named_tags
+        return if named.nil? || named.empty?
+
+        if key
+          escape_control_characters(named[key.to_sym] || named[key.to_s])
+        else
+          named.map { |name, value| "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }.join(", ")
+        end
+      end
+
+      # Host name.
+      def host
+        logger&.host if log_host
+      end
+
+      # Application name.
+      def application
+        logger&.application if log_application
+      end
+
+      # Environment name.
+      def environment
+        logger&.environment if log_environment
+      end
+
+      def call(log, logger)
+        self.log    = log
+        self.logger = logger
+
+        @tokens.each_with_object(+"") do |token, out|
+          out << (token.is_a?(Token) ? public_send(token.method_name, *token.arguments).to_s : token)
+        end
+      end
+
+      private
+
+      # Parse the pattern string into an array of tokens: frozen literal strings
+      # and Token structs for each %{directive} placeholder. %%{...} is an escape
+      # that produces a literal %{...}.
+      def compile(string)
+        tokens = []
+        pos    = 0
+
+        string.scan(/%%?\{[^}]+\}/) do |match|
+          current = Regexp.last_match
+          tokens << string[pos...current.begin(0)].freeze if current.begin(0) > pos
+
+          if match.start_with?("%%")
+            tokens << match[1..].freeze
+          else
+            name, arg = match[/\{([^}]+)\}/, 1].split(":", 2)
+            name      = name.strip.to_sym
+            raise(ArgumentError, "Invalid pattern directive: %{#{name}}") unless DIRECTIVES.key?(name)
+            raise(ArgumentError, "%{#{name}} does not accept an argument") if arg && !DIRECTIVES[name]
+
+            tokens << Token.new(name, arg ? [arg.strip] : []).freeze
+          end
+
+          pos = current.end(0)
+        end
+
+        tokens << string[pos..].freeze if pos < string.length
+        tokens
+      end
+    end
+  end
+end

data/lib/semantic_logger/formatters/raw.rb

@@ -18,12 +18,12 @@ module SemanticLogger
 
       # Application name
       def application
-        hash[:application] = logger.application if log_application && logger && logger.application
+        hash[:application] = logger.application if log_application && logger&.application
       end
 
       # Environment
       def environment
-        hash[:environment] = logger.environment if log_environment && logger && logger.environment
+        hash[:environment] = logger.environment if log_environment && logger&.environment
       end
 
       # Date & time

data/lib/semantic_logger/formatters/signalfx.rb

@@ -101,7 +101,7 @@ module SemanticLogger
           data[:counter] = [hash]
         end
 
-        data.to_json
+        Utils.to_json(data)
       end
 
       # Returns [Hash] a batch of log messages.
@@ -138,7 +138,7 @@ module SemanticLogger
           end
         end
 
-        data.to_json
+        Utils.to_json(data)
       end
 
       private

data/lib/semantic_logger/formatters/syslog.rb

@@ -49,14 +49,25 @@ module SemanticLogger
       #   level_map: [Hash | SemanticLogger::Formatters::Syslog::LevelMap]
       #     Supply a custom map of SemanticLogger levels to syslog levels.
       #
+      #   escape_control_chars: [Boolean]
+      #     Replace control characters (newlines, the ANSI escape, etc.) in
+      #     untrusted log data with a printable escaped form so that they cannot
+      #     forge or split syslog records.
+      #     Default: true (unlike other formatters, since syslog frames records
+      #     with a separator)
+      #
       #   Example:
       #     # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
       #     SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
-      def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer)
+      def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer,
+                     escape_control_chars: true, **args)
         @facility  = facility
-        @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(level_map)
+        @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(**level_map)
         @max_size = max_size
-        super()
+        # Syslog frames records with a separator, so embedded newlines or other
+        # control characters in untrusted log data can forge or split records.
+        # Default to escaping them, overridable for backwards compatibility.
+        super(escape_control_chars: escape_control_chars, **args)
       end
 
       # Time is part of the syslog packet and is not included in the formatted message.

data/lib/semantic_logger/formatters/syslog_cee.rb

@@ -38,7 +38,7 @@ module SemanticLogger
 
       def call(log, logger)
         hash = super
-        create_syslog_packet("@cee: #{hash.to_json}")
+        create_syslog_packet("@cee: #{Utils.to_json(hash)}")
       end
 
       private

data/lib/semantic_logger/formatters.rb

@@ -3,6 +3,7 @@ module SemanticLogger
     autoload :Base,          "semantic_logger/formatters/base"
     autoload :Color,         "semantic_logger/formatters/color"
     autoload :Default,       "semantic_logger/formatters/default"
+    autoload :Ecs,           "semantic_logger/formatters/ecs"
     autoload :Json,          "semantic_logger/formatters/json"
     autoload :Raw,           "semantic_logger/formatters/raw"
     autoload :OneLine,       "semantic_logger/formatters/one_line"
@@ -11,6 +12,7 @@ module SemanticLogger
     autoload :Syslog,        "semantic_logger/formatters/syslog"
     autoload :Fluentd,       "semantic_logger/formatters/fluentd"
     autoload :Logfmt,        "semantic_logger/formatters/logfmt"
+    autoload :Pattern,       "semantic_logger/formatters/pattern"
     autoload :SyslogCee,     "semantic_logger/formatters/syslog_cee"
     autoload :NewRelicLogs,  "semantic_logger/formatters/new_relic_logs"
     autoload :Loki,          "semantic_logger/formatters/loki"

data/lib/semantic_logger/log.rb

@@ -93,6 +93,11 @@ module SemanticLogger
       self.dimensions    = dimensions
 
       if exception
+        unless exception.is_a?(Exception)
+          exception = ArgumentError.new("Invalid value for logger exception: #{exception.inspect}")
+          exception.set_backtrace(Utils.extract_backtrace(caller))
+        end
+
         case log_exception
         when :full
           self.exception = exception
@@ -124,11 +129,17 @@ module SemanticLogger
       true
     end
 
+    # Keys that #assign_hash may assign directly to a log attribute. Every other
+    # key is folded into the payload, preventing a supplied hash from overwriting
+    # sensitive fields such as :level, :name, or :time, and keeping this path
+    # consistent with #extract_arguments.
+    ASSIGNABLE_KEYS = (NON_PAYLOAD_KEYS + %i[payload]).freeze
+
     # Assign known keys to self, all other keys to the payload.
     def assign_hash(hash)
       self.payload ||= {}
       hash.each_pair do |key, value|
-        if respond_to?(:"#{key}=")
+        if ASSIGNABLE_KEYS.include?(key) && respond_to?(:"#{key}=")
           public_send(:"#{key}=", value)
         else
           payload[key] = value
@@ -245,7 +256,7 @@ module SemanticLogger
       "#{$$}:#{format("%.#{thread_name_length}s", thread_name)}#{file_name}"
     end
 
-    CALLER_REGEXP = /^(.*):(\d+).*/.freeze
+    CALLER_REGEXP = /^(.*):(\d+).*/
 
     # Extract the filename and line number from the last entry in the supplied backtrace
     def extract_file_and_line(stack, short_name = false)
@@ -264,9 +275,17 @@ module SemanticLogger
       extract_file_and_line(stack, short_name)
     end
 
-    # Strip the standard Rails colorizing from the logged message
+    # Strip the standard Rails colorizing from the logged message.
+    #
+    # Note: This unconditionally *strips* ANSI colorization, and is used to keep
+    # terminal escape codes out of structured (JSON/Loki) output. It is distinct
+    # from Formatters::Base#escape_control_characters, which instead *escapes*
+    # (preserves) control characters and is opt-in for the text formatters.
     def cleansed_message
-      message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, "").strip
+      msg = message.to_s
+      return msg.strip unless msg.include?("\e")
+
+      msg.gsub(/\e\[[\d;]*[mz]?|\e/, "").strip
     end
 
     # Return the payload in text form

data/lib/semantic_logger/logger.rb

@@ -65,9 +65,9 @@ module SemanticLogger
     #
     # Subscribers are called inline before handing off to the queue so that
     # they can capture additional context information as needed.
-    def log(log, message = nil, progname = nil, &block)
+    def log(log, message = nil, progname = nil, &)
       # Compatibility with ::Logger
-      return add(log, message, progname, &block) unless log.is_a?(SemanticLogger::Log)
+      return add(log, message, progname, &) unless log.is_a?(SemanticLogger::Log)
 
       Logger.call_subscribers(log)
 

data/lib/semantic_logger/metric/new_relic.rb

@@ -38,9 +38,9 @@ module SemanticLogger
       #     regular expression. All other messages will be ignored.
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
-      def initialize(prefix: "Custom", **args, &block)
+      def initialize(prefix: "Custom", **args, &)
         @prefix = prefix
-        super(**args, &block)
+        super(**args, &)
       end
 
       # Returns metric name to use.

data/lib/semantic_logger/metric/signalfx.rb

@@ -78,10 +78,10 @@ module SemanticLogger
                      url: "https://ingest.signalfx.com",
                      formatter: nil,
                      **args,
-                     &block)
+                     &)
         formatter ||= SemanticLogger::Formatters::Signalfx.new(token: token, dimensions: dimensions)
 
-        super(url: url, formatter: formatter, **args, &block)
+        super(url: url, formatter: formatter, **args, &)
 
         @header["X-SF-TOKEN"] = token
         @full_url             = "#{url}/#{END_POINT}"

data/lib/semantic_logger/metric/statsd.rb

@@ -24,7 +24,7 @@ module SemanticLogger
       # Example:
       #   SemanticLogger.add_appender(
       #     metric: :statsd,
-      #     url:    'localhost:8125'
+      #     url:    'udp://localhost:8125'
       #   )
       def initialize(url: "udp://localhost:8125")
         @url = url
@@ -47,7 +47,7 @@ module SemanticLogger
         else
           amount = (log.metric_amount || 1).round
           if amount.negative?
-            amount.times { @statsd.decrement(metric) }
+            amount.abs.times { @statsd.decrement(metric) }
           else
             amount.times { @statsd.increment(metric) }
           end

data/lib/semantic_logger/processor.rb

@@ -23,7 +23,7 @@ module SemanticLogger
 
     attr_reader :appenders
 
-    def initialize(max_queue_size: -1)
+    def initialize(max_queue_size: 10_000)
       @appenders = Appenders.new(self.class.logger.dup)
       super(appender: @appenders, max_queue_size: max_queue_size)
     end
@@ -35,5 +35,26 @@ module SemanticLogger
       thread
       true
     end
+
+    # Returns [Hash] operational statistics for the logging pipeline.
+    #
+    #   queue_size:     [Integer] Number of log messages waiting on the main pipeline queue.
+    #   capped:         [Boolean] Whether the main queue has a maximum size.
+    #   max_queue_size: [Integer] Maximum queue size, or nil when uncapped.
+    #   thread_active:  [Boolean] Whether the main pipeline thread is running.
+    #   processed:      [Integer] Cumulative number of log messages processed since startup.
+    #   dropped:        [Integer] Cumulative number of log messages dropped at the main queue.
+    #   appenders:      [Array<Hash>] Per-appender statistics, see Appenders#stats.
+    def stats
+      {
+        queue_size:     queue.size,
+        capped:         capped?,
+        max_queue_size: capped? ? max_queue_size : nil,
+        thread_active:  active? || false,
+        processed:      processed_count,
+        dropped:        dropped_count,
+        appenders:      appenders.stats
+      }
+    end
   end
 end