semantic_logger 4.18.0 → 5.0.0

data/lib/semantic_logger/formatters/signalfx.rb

@@ -101,7 +101,7 @@ module SemanticLogger
           data[:counter] = [hash]
         end
 
-        data.to_json
+        Utils.to_json(data)
       end
 
       # Returns [Hash] a batch of log messages.
@@ -138,7 +138,7 @@ module SemanticLogger
           end
         end
 
-        data.to_json
+        Utils.to_json(data)
       end
 
       private

data/lib/semantic_logger/formatters/syslog.rb

@@ -49,14 +49,25 @@ module SemanticLogger
       #   level_map: [Hash | SemanticLogger::Formatters::Syslog::LevelMap]
       #     Supply a custom map of SemanticLogger levels to syslog levels.
       #
+      #   escape_control_chars: [Boolean]
+      #     Replace control characters (newlines, the ANSI escape, etc.) in
+      #     untrusted log data with a printable escaped form so that they cannot
+      #     forge or split syslog records.
+      #     Default: true (unlike other formatters, since syslog frames records
+      #     with a separator)
+      #
       #   Example:
       #     # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
       #     SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
-      def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer)
+      def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer,
+                     escape_control_chars: true, **args)
         @facility  = facility
-        @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(level_map)
+        @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(**level_map)
         @max_size = max_size
-        super()
+        # Syslog frames records with a separator, so embedded newlines or other
+        # control characters in untrusted log data can forge or split records.
+        # Default to escaping them, overridable for backwards compatibility.
+        super(escape_control_chars: escape_control_chars, **args)
       end
 
       # Time is part of the syslog packet and is not included in the formatted message.

data/lib/semantic_logger/formatters/syslog_cee.rb

@@ -38,7 +38,7 @@ module SemanticLogger
 
       def call(log, logger)
         hash = super
-        create_syslog_packet("@cee: #{hash.to_json}")
+        create_syslog_packet("@cee: #{Utils.to_json(hash)}")
       end
 
       private

data/lib/semantic_logger/formatters.rb

@@ -3,6 +3,7 @@ module SemanticLogger
     autoload :Base,          "semantic_logger/formatters/base"
     autoload :Color,         "semantic_logger/formatters/color"
     autoload :Default,       "semantic_logger/formatters/default"
+    autoload :Ecs,           "semantic_logger/formatters/ecs"
     autoload :Json,          "semantic_logger/formatters/json"
     autoload :Raw,           "semantic_logger/formatters/raw"
     autoload :OneLine,       "semantic_logger/formatters/one_line"
@@ -11,6 +12,7 @@ module SemanticLogger
     autoload :Syslog,        "semantic_logger/formatters/syslog"
     autoload :Fluentd,       "semantic_logger/formatters/fluentd"
     autoload :Logfmt,        "semantic_logger/formatters/logfmt"
+    autoload :Pattern,       "semantic_logger/formatters/pattern"
     autoload :SyslogCee,     "semantic_logger/formatters/syslog_cee"
     autoload :NewRelicLogs,  "semantic_logger/formatters/new_relic_logs"
     autoload :Loki,          "semantic_logger/formatters/loki"

data/lib/semantic_logger/log.rb

@@ -129,11 +129,17 @@ module SemanticLogger
       true
     end
 
+    # Keys that #assign_hash may assign directly to a log attribute. Every other
+    # key is folded into the payload, preventing a supplied hash from overwriting
+    # sensitive fields such as :level, :name, or :time, and keeping this path
+    # consistent with #extract_arguments.
+    ASSIGNABLE_KEYS = (NON_PAYLOAD_KEYS + %i[payload]).freeze
+
     # Assign known keys to self, all other keys to the payload.
     def assign_hash(hash)
       self.payload ||= {}
       hash.each_pair do |key, value|
-        if respond_to?(:"#{key}=")
+        if ASSIGNABLE_KEYS.include?(key) && respond_to?(:"#{key}=")
           public_send(:"#{key}=", value)
         else
           payload[key] = value
@@ -250,7 +256,7 @@ module SemanticLogger
       "#{$$}:#{format("%.#{thread_name_length}s", thread_name)}#{file_name}"
     end
 
-    CALLER_REGEXP = /^(.*):(\d+).*/.freeze
+    CALLER_REGEXP = /^(.*):(\d+).*/
 
     # Extract the filename and line number from the last entry in the supplied backtrace
     def extract_file_and_line(stack, short_name = false)
@@ -269,9 +275,17 @@ module SemanticLogger
       extract_file_and_line(stack, short_name)
     end
 
-    # Strip the standard Rails colorizing from the logged message
+    # Strip the standard Rails colorizing from the logged message.
+    #
+    # Note: This unconditionally *strips* ANSI colorization, and is used to keep
+    # terminal escape codes out of structured (JSON/Loki) output. It is distinct
+    # from Formatters::Base#escape_control_characters, which instead *escapes*
+    # (preserves) control characters and is opt-in for the text formatters.
     def cleansed_message
-      message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, "").strip
+      msg = message.to_s
+      return msg.strip unless msg.include?("\e")
+
+      msg.gsub(/\e\[[\d;]*[mz]?|\e/, "").strip
     end
 
     # Return the payload in text form

data/lib/semantic_logger/logger.rb

@@ -65,9 +65,9 @@ module SemanticLogger
     #
     # Subscribers are called inline before handing off to the queue so that
     # they can capture additional context information as needed.
-    def log(log, message = nil, progname = nil, &block)
+    def log(log, message = nil, progname = nil, &)
       # Compatibility with ::Logger
-      return add(log, message, progname, &block) unless log.is_a?(SemanticLogger::Log)
+      return add(log, message, progname, &) unless log.is_a?(SemanticLogger::Log)
 
       Logger.call_subscribers(log)
 

data/lib/semantic_logger/metric/new_relic.rb

@@ -38,9 +38,9 @@ module SemanticLogger
       #     regular expression. All other messages will be ignored.
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
-      def initialize(prefix: "Custom", **args, &block)
+      def initialize(prefix: "Custom", **args, &)
         @prefix = prefix
-        super(**args, &block)
+        super(**args, &)
       end
 
       # Returns metric name to use.

data/lib/semantic_logger/metric/signalfx.rb

@@ -78,10 +78,10 @@ module SemanticLogger
                      url: "https://ingest.signalfx.com",
                      formatter: nil,
                      **args,
-                     &block)
+                     &)
         formatter ||= SemanticLogger::Formatters::Signalfx.new(token: token, dimensions: dimensions)
 
-        super(url: url, formatter: formatter, **args, &block)
+        super(url: url, formatter: formatter, **args, &)
 
         @header["X-SF-TOKEN"] = token
         @full_url             = "#{url}/#{END_POINT}"

data/lib/semantic_logger/metric/statsd.rb

@@ -24,7 +24,7 @@ module SemanticLogger
       # Example:
       #   SemanticLogger.add_appender(
       #     metric: :statsd,
-      #     url:    'localhost:8125'
+      #     url:    'udp://localhost:8125'
       #   )
       def initialize(url: "udp://localhost:8125")
         @url = url
@@ -47,7 +47,7 @@ module SemanticLogger
         else
           amount = (log.metric_amount || 1).round
           if amount.negative?
-            amount.times { @statsd.decrement(metric) }
+            amount.abs.times { @statsd.decrement(metric) }
           else
             amount.times { @statsd.increment(metric) }
           end

data/lib/semantic_logger/processor.rb

@@ -35,5 +35,26 @@ module SemanticLogger
       thread
       true
     end
+
+    # Returns [Hash] operational statistics for the logging pipeline.
+    #
+    #   queue_size:     [Integer] Number of log messages waiting on the main pipeline queue.
+    #   capped:         [Boolean] Whether the main queue has a maximum size.
+    #   max_queue_size: [Integer] Maximum queue size, or nil when uncapped.
+    #   thread_active:  [Boolean] Whether the main pipeline thread is running.
+    #   processed:      [Integer] Cumulative number of log messages processed since startup.
+    #   dropped:        [Integer] Cumulative number of log messages dropped at the main queue.
+    #   appenders:      [Array<Hash>] Per-appender statistics, see Appenders#stats.
+    def stats
+      {
+        queue_size:     queue.size,
+        capped:         capped?,
+        max_queue_size: capped? ? max_queue_size : nil,
+        thread_active:  active? || false,
+        processed:      processed_count,
+        dropped:        dropped_count,
+        appenders:      appenders.stats
+      }
+    end
   end
 end

data/lib/semantic_logger/queue_processor.rb

@@ -0,0 +1,369 @@
+module SemanticLogger
+  # Internal class that processes log messages from a queue on a separate thread.
+  #
+  # Internal use only: it owns the worker thread and the in-memory queue that back the
+  # asynchronous appender proxy (SemanticLogger::Appender::Async). It is never returned to
+  # application code.
+  #
+  # Supports two processing modes, selected by the `batch:` option:
+  #   * Streaming (batch: false): each log message is written to the appender as it is dequeued.
+  #   * Batching  (batch: true):  log messages are grouped and written via the appender's #batch
+  #                               method, either once batch_size messages have accumulated, or
+  #                               batch_seconds have elapsed since the previous batch.
+  class QueueProcessor
+    attr_accessor :lag_check_interval, :lag_threshold_s, :dropped_message_report_seconds,
+                  :batch_size, :batch_seconds
+    attr_reader :appender, :queue, :max_queue_size, :non_blocking, :signal,
+                :processed_count, :dropped_count, :async_max_retries, :retry_count
+
+    # Create a new processor and start its worker thread.
+    def self.start(**args)
+      processor = new(**args)
+      processor.thread
+      processor
+    end
+
+    # Parameters:
+    #   appender: [SemanticLogger::Subscriber]
+    #     The appender to forward log messages to from the worker thread.
+    #
+    #   max_queue_size: [Integer]
+    #     The maximum number of log messages to hold on the queue before blocking attempts to add to the queue.
+    #     -1: The queue size is uncapped and will never block no matter how long the queue is.
+    #     Default: 10,000
+    #
+    #   lag_threshold_s: [Float]
+    #     Log a warning when a log message has been on the queue for longer than this period in seconds.
+    #     Default: 30
+    #
+    #   lag_check_interval: [Integer]
+    #     Number of messages to process before checking for slow logging.
+    #     Default: 1,000
+    #     Note: Not applicable when batch: true.
+    #
+    #   batch: [true|false]
+    #     Process log messages in batches via the appender's #batch method.
+    #     Default: false
+    #     Note: The appender must implement #batch.
+    #
+    #   batch_size: [Integer]
+    #     Maximum number of messages to batch up before sending.
+    #     Default: 300
+    #     Note: Only applicable when batch: true.
+    #
+    #   batch_seconds: [Integer]
+    #     Maximum number of seconds between sending batches.
+    #     Default: 5
+    #     Note: Only applicable when batch: true.
+    #
+    #   non_blocking: [true|false]
+    #     Whether to drop log messages instead of blocking the calling thread when the queue is full.
+    #     Only applies to a capped queue.
+    #     Default: false
+    #
+    #   dropped_message_report_seconds: [Integer]
+    #     When non_blocking is enabled, log the count of dropped messages to the internal logger
+    #     at most once every this number of seconds.
+    #     Default: 30
+    #
+    #   async_max_retries: [Integer]
+    #     Maximum number of consecutive times to restart the worker thread after it raises an
+    #     exception while processing messages. Each restart first sleeps for `retry_count` seconds
+    #     (1s, then 2s, ...) as a back-off. Once this many consecutive retries are exhausted the
+    #     thread stops instead of restarting. The counter resets to zero whenever a message is
+    #     processed successfully.
+    #     -1: Retry indefinitely and never stop the thread (the pre-v5 behaviour). The back-off
+    #         still applies, and still resets after any successful message.
+    #     Default: 100
+    def initialize(appender:,
+                   max_queue_size: 10_000,
+                   lag_check_interval: 1_000,
+                   lag_threshold_s: 30,
+                   batch: false,
+                   batch_size: 300,
+                   batch_seconds: 5,
+                   non_blocking: false,
+                   dropped_message_report_seconds: 30,
+                   async_max_retries: 100)
+      @appender                       = appender
+      @max_queue_size                 = max_queue_size
+      @lag_check_interval             = lag_check_interval
+      @lag_threshold_s                = lag_threshold_s
+      @batch                          = batch
+      @batch_size                     = batch_size
+      @batch_seconds                  = batch_seconds
+      @non_blocking                   = non_blocking
+      @dropped_message_report_seconds = dropped_message_report_seconds
+      @async_max_retries              = async_max_retries
+      @retry_count                    = 0
+      @thread                         = nil
+      # Only batch mode parks the worker on the signal; streaming mode never touches it.
+      @signal                         = Concurrent::Event.new if batch
+      @dropped_message_count          = 0
+      @dropped_message_reported_at    = Time.now
+      @dropped_message_mutex          = Mutex.new
+      @processed_count                = 0
+      @dropped_count                  = 0
+      create_queue
+
+      return unless batch? && !appender.respond_to?(:batch)
+
+      raise(ArgumentError, "#{appender.class.name} does not support batching. It must implement #batch")
+    end
+
+    # Internal logger used to report problems encountered while processing log messages.
+    def logger
+      appender.logger
+    end
+
+    # Returns [Thread] the worker thread.
+    #
+    # Starts the worker thread if it is not currently running.
+    def thread
+      return @thread if @thread&.alive?
+
+      @thread = spawn_worker
+    end
+
+    # Returns [true|false] whether the worker thread is running.
+    def active?
+      @thread&.alive?
+    end
+
+    # Returns [true|false] whether the queue has a capped size.
+    def capped?
+      @capped
+    end
+
+    # Returns [true|false] whether messages are dropped instead of blocking when the queue is full.
+    # Only a capped queue can drop messages.
+    def non_blocking?
+      @non_blocking && capped?
+    end
+
+    # Returns [true|false] whether messages are processed in batches.
+    def batch?
+      @batch
+    end
+
+    # Add a log message to the queue for processing.
+    #
+    # When non-blocking and the queue is full, the message is dropped instead of blocking the
+    # calling thread, and the count of dropped messages is reported periodically.
+    def log(log)
+      enqueued = true
+      if non_blocking?
+        begin
+          queue.push(log, true)
+        rescue ThreadError
+          message_dropped
+          enqueued = false
+        end
+      else
+        queue << log
+      end
+
+      # For batches wake up the processing thread once the number of queued messages has been
+      # exceeded. Also wake it on a drop so it drains the full queue rather than waiting out the
+      # batch interval.
+      signal.set if batch? && (queue.size >= batch_size)
+
+      enqueued
+    end
+
+    # Flush all queued log entries to the appender.
+    #  All queued log messages are written and then the appender is flushed.
+    def flush
+      submit_request(:flush)
+    end
+
+    # Flush any outstanding messages and close the appender.
+    def close
+      submit_request(:close)
+    end
+
+    # Re-open the queue and worker thread after a fork.
+    def reopen
+      # Workaround CRuby crash on fork by recreating queue on reopen
+      #   https://github.com/reidmorrison/semantic_logger/issues/103
+      @queue&.close
+      create_queue
+
+      @thread&.kill if @thread&.alive?
+      @thread = spawn_worker
+    end
+
+    private
+
+    # Start the worker thread, naming it after the internal logger.
+    def spawn_worker
+      Thread.new do
+        Thread.current.name = logger.name
+        process
+      end
+    end
+
+    def create_queue
+      if max_queue_size == -1
+        @queue  = Queue.new
+        @capped = false
+      else
+        @queue  = SizedQueue.new(max_queue_size)
+        @capped = true
+      end
+    end
+
+    # Separate thread for processing log messages.
+    def process
+      # This thread is designed to never go down unless the main thread terminates
+      # or the appender is closed.
+      logger.trace "Async: Appender thread active"
+      begin
+        batch? ? process_messages_in_batches : process_messages
+      rescue StandardError => e
+        if async_max_retries == -1 || retry_count < async_max_retries
+          @retry_count += 1
+          limit = async_max_retries == -1 ? "unlimited" : async_max_retries
+          safe_log(:warn, "Async: Restarting due to exception, retry #{retry_count} of #{limit}, sleeping #{retry_count}s", e)
+          sleep(retry_count)
+          retry
+        else
+          safe_log(:error, "Async: Stopping after #{retry_count} failed retries", e)
+        end
+      rescue Exception => e
+        safe_log(:error, "Async: Stopping due to fatal exception", e)
+      ensure
+        @thread = nil
+        safe_log(:trace, "Async: Thread has stopped")
+      end
+    end
+
+    # Log to the internal logger, ignoring any error.
+    # These calls may run after Ruby has released file handles during shutdown.
+    def safe_log(level, message, exception = nil)
+      logger.public_send(level, message, exception)
+    rescue StandardError
+      nil
+    end
+
+    # Streaming: write each log message to the appender as it is dequeued.
+    def process_messages
+      count = 0
+      while (message = queue.pop)
+        if message.is_a?(Log)
+          appender.log(message)
+          @processed_count += 1
+          @retry_count = 0 unless retry_count.zero?
+          count += 1
+          # Check every few log messages whether this appender thread is falling behind
+          if count > lag_check_interval
+            check_lag(message)
+            count = 0
+          end
+        else
+          break unless process_message(message)
+        end
+      end
+      logger.trace "Async: Queue Closed"
+    end
+
+    # Batching: group up log messages before writing them via the appender's #batch method.
+    def process_messages_in_batches
+      loop do
+        # Wait for batch interval or number of messages to be exceeded.
+        signal.wait(batch_seconds)
+
+        logs          = []
+        messages      = []
+        first         = true
+        message_count = queue.length
+        message_count.times do
+          # Queue#pop(true) raises an exception when there are no more messages, which is considered expensive.
+          message = queue.pop
+          if message.is_a?(Log)
+            logs << message
+            if first
+              check_lag(message)
+              first = false
+            end
+          else
+            messages << message
+          end
+        end
+        if logs.size.positive?
+          appender.batch(logs)
+          @processed_count += logs.size
+          @retry_count = 0 unless retry_count.zero?
+        end
+        # Stop processing once a command (e.g. :close) signals the thread to terminate.
+        break if messages.any? { |message| !process_message(message) }
+
+        signal.reset unless queue.size >= batch_size
+      end
+    end
+
+    # Returns false when message processing should be stopped.
+    def process_message(message)
+      case message[:command]
+      when :flush
+        appender.flush
+        message[:reply_queue] << true if message[:reply_queue]
+      when :close
+        appender.close
+        message[:reply_queue] << true if message[:reply_queue]
+        return false
+      else
+        logger.warn "Async: Appender thread: Ignoring unknown command: #{message[:command]}"
+      end
+      true
+    end
+
+    # Record a dropped message, reporting the running count to the internal logger at most
+    # once every dropped_message_report_seconds.
+    def message_dropped
+      @dropped_message_mutex.synchronize do
+        @dropped_message_count += 1
+        @dropped_count         += 1
+        diff = Time.now - @dropped_message_reported_at
+        return if diff < dropped_message_report_seconds
+
+        logger.warn(
+          "Async: Dropped #{@dropped_message_count} log messages in the last #{diff.round} seconds. " \
+          "The queue is full (max_queue_size: #{max_queue_size}). " \
+          "Consider reducing the log level, increasing max_queue_size, or changing the appenders"
+        )
+        @dropped_message_count       = 0
+        @dropped_message_reported_at = Time.now
+      end
+    end
+
+    def check_lag(log)
+      diff = Time.now - log.time
+      return unless diff > lag_threshold_s
+
+      logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with #{queue.size} messages queued up. Consider reducing the log level or changing the appenders"
+    end
+
+    # Submit a command to the worker thread and wait for the reply.
+    def submit_request(command)
+      return false unless active?
+
+      queue_size = queue.size
+      msg        = "Async: Queued log messages: #{queue_size}, running command: #{command}"
+      if queue_size > 1_000
+        logger.warn msg
+      elsif queue_size > 100
+        logger.info msg
+      elsif queue_size.positive?
+        logger.trace msg
+      end
+
+      # Wake up the processing thread to process this command immediately.
+      signal.set if batch?
+
+      reply_queue = Queue.new
+      queue << {command: command, reply_queue: reply_queue}
+      reply_queue.pop
+    end
+  end
+end

data/lib/semantic_logger/reporters/minitest.rb

@@ -28,18 +28,18 @@ module SemanticLogger
       attr_accessor :io
 
       def before_test(test)
-        logger.info("START #{test.class_name} #{test.name}")
+        logger.info("START #{test.class.name} #{test.name}")
       end
 
       def after_test(test)
         if test.error?
-          logger.benchmark_error("FAIL #{test.class_name} #{test.name}", duration: test.time * 1_000,
+          logger.benchmark_error("FAIL #{test.class.name} #{test.name}", duration: test.time * 1_000,
                                                                          metric:   "minitest/fail")
         elsif test.skipped?
-          logger.benchmark_warn("SKIP #{test.class_name} #{test.name}", duration: test.time * 1_000,
+          logger.benchmark_warn("SKIP #{test.class.name} #{test.name}", duration: test.time * 1_000,
                                                                         metric:   "minitest/skip")
         else
-          logger.benchmark_info("PASS #{test.class_name} #{test.name}", duration: test.time * 1_000,
+          logger.benchmark_info("PASS #{test.class.name} #{test.name}", duration: test.time * 1_000,
                                                                         metric:   "minitest/pass")
         end
       end