semantic_logger 5.0.0 → 5.2.0

data/lib/semantic_logger/metric/signalfx.rb

@@ -8,7 +8,7 @@
 module SemanticLogger
   module Metric
     class Signalfx < SemanticLogger::Appender::Http
-      attr_reader :full_url
+      attr_reader :full_path
 
       END_POINT = "v2/datapoint".freeze
 
@@ -66,8 +66,10 @@ module SemanticLogger
       #   logger.info(metric: 'Common/User/authorize', duration: 1.4)
       #
       # Then it is translated into the following 2 log entries under the covers:
-      #   logger.info(metric: 'Application.average', metric_amount: 1.4, dimensions: {class: 'Common::User', action: 'authorize'})
-      #   logger.info(metric: 'Application.counter', metric_amount: 1, dimensions: {class: 'Common::User', action: 'authorize'})
+      #   logger.info(metric: 'Application.average', metric_amount: 1.4,
+      #               dimensions: {class: 'Common::User', action: 'authorize'})
+      #   logger.info(metric: 'Application.counter', metric_amount: 1,
+      #               dimensions: {class: 'Common::User', action: 'authorize'})
       #
       # Similarly with a measure block which automatically supplies the duration:
       #   logger.measure_info(metric: 'Common/User/authorize') do
@@ -84,20 +86,22 @@ module SemanticLogger
         super(url: url, formatter: formatter, **args, &)
 
         @header["X-SF-TOKEN"] = token
-        @full_url             = "#{url}/#{END_POINT}"
+        # Built from the parsed path, not the supplied url, so that a query string in the
+        # url stays at the end of the request uri. See Appender::Http#uri_with_query.
+        @full_path            = "#{@path.end_with?('/') ? @path : "#{@path}/"}#{END_POINT}"
       end
 
       def log(log)
-        message = formatter.call(log, self)
+        message = format_log(log)
         logger.trace(message)
-        post(message, full_url)
+        post(message, full_path)
       end
 
       # Logs in batches
       def batch(logs)
-        message = formatter.batch(logs, self)
+        message = format_batch(logs)
         logger.trace(message)
-        post(message, full_url)
+        post(message, full_path)
       end
 
       # Only forward log entries that contain metrics.

data/lib/semantic_logger/metric/statsd.rb

@@ -32,7 +32,7 @@ module SemanticLogger
       end
 
       def reopen
-        uri = URI.parse(@url)
+        uri = Utils.parse_url(@url)
         raise('Statsd only supports udp. Example: "udp://localhost:8125"') if uri.scheme != "udp"
 
         @statsd           = ::Statsd.new(uri.host, uri.port)
@@ -45,12 +45,15 @@ module SemanticLogger
         if (duration = log.duration)
           @statsd.timing(metric, duration)
         else
-          amount = (log.metric_amount || 1).round
-          if amount.negative?
-            amount.abs.times { @statsd.decrement(metric) }
-          else
-            amount.times { @statsd.increment(metric) }
-          end
+          amount = log.metric_amount || 1
+          # Infinity and NaN cannot be counted, whether a Float or a BigDecimal.
+          return if amount.respond_to?(:finite?) && !amount.finite?
+
+          amount = amount.round
+          # A single counter packet, whatever the size of the amount. Incrementing once per unit
+          # would let a large amount, such as one supplied by a user, flood the network and
+          # stall the logging thread.
+          @statsd.count(metric, amount) unless amount.zero?
         end
       end
 
@@ -59,6 +62,11 @@ module SemanticLogger
         # Does not support metrics with dimensions.
         log.metric && !log.dimensions && meets_log_level?(log) && !filtered?(log)
       end
+
+      # The url without any credentials. See Subscriber#inspect.
+      def inspect_details
+        "url=#{Utils.redacted_url(url).inspect}"
+      end
     end
   end
 end

data/lib/semantic_logger/queue_processor/request.rb

@@ -0,0 +1,55 @@
+module SemanticLogger
+  class QueueProcessor
+    # Internal: a command, such as :flush or :close, submitted on the queue for the worker thread
+    # to run, while the submitting thread waits for the reply.
+    #
+    # It knows its command and how its reply gets back to the submitting thread, and makes sure
+    # that the submitting thread is never left waiting on a reply that cannot come.
+    class Request
+      attr_reader :command
+
+      def initialize(command)
+        @command = command
+        @replies = Queue.new
+      end
+
+      # Called by the worker thread: run the block for this request, then reply with true, or
+      # with false when the block raised. Replies either way.
+      def reply
+        done = false
+        yield
+        done = true
+      ensure
+        @replies.push(done)
+      end
+
+      # Called by the worker thread: run this command on the appender, then reply. Returns false
+      # once message processing must stop (:close).
+      def run(appender)
+        case command
+        when :flush
+          reply { appender.flush }
+        when :close
+          reply { appender.close }
+          return false
+        else
+          reply { appender.logger.warn "Async: Appender thread: Ignoring unknown command: #{command}" }
+        end
+        true
+      end
+
+      # Called by the submitting thread: wait for the reply from the worker thread.
+      #
+      # The block returns whether a worker thread that can still reply is running. Returns the
+      # reply, or false once none is, for example once the worker gave up after
+      # async_max_retries, or was replaced after a fork.
+      def wait
+        loop do
+          reply = @replies.pop(timeout: 1)
+          return reply unless reply.nil?
+          return @replies.pop(timeout: 0) || false unless yield
+        end
+      end
+    end
+  end
+end

data/lib/semantic_logger/queue_processor/throttled_counter.rb

@@ -0,0 +1,46 @@
+module SemanticLogger
+  class QueueProcessor
+    # Internal: counts occurrences of a problem, such as a dropped log message, and has it
+    # reported at most once per interval instead of once per occurrence.
+    #
+    # It knows the total number of occurrences, how many there have been since the previous
+    # report, and when that report was made. What to report, and where, is up to the caller.
+    #
+    # Thread-safe: occurrences may be recorded from any thread.
+    class ThrottledCounter
+      # Returns [Integer] the total number of occurrences recorded.
+      attr_reader :total
+
+      # Parameters:
+      #   reported_at: [Time|nil]
+      #     When counting starts, so that the first report is due one interval later.
+      #     nil: Report the first occurrence straight away.
+      #     Default: Time.now
+      def initialize(reported_at: Time.now)
+        @reported_at = reported_at
+        @total       = 0
+        @unreported  = 0
+        @mutex       = Mutex.new
+      end
+
+      # Record one or more occurrences.
+      #
+      # When the previous report was at least `interval` seconds ago, yields the number of
+      # occurrences since then (including these) and the seconds elapsed since that report (nil
+      # when there has been no report yet), then starts counting afresh for the next report.
+      def record(count = 1, interval:)
+        @mutex.synchronize do
+          @total      += count
+          @unreported += count
+          now          = Time.now
+          elapsed      = @reported_at && (now - @reported_at)
+          return if elapsed && elapsed < interval
+
+          yield(@unreported, elapsed)
+          @unreported  = 0
+          @reported_at = now
+        end
+      end
+    end
+  end
+end

data/lib/semantic_logger/queue_processor.rb

@@ -1,3 +1,6 @@
+require "semantic_logger/queue_processor/request"
+require "semantic_logger/queue_processor/throttled_counter"
+
 module SemanticLogger
   # Internal class that processes log messages from a queue on a separate thread.
   #
@@ -12,9 +15,9 @@ module SemanticLogger
   #                               batch_seconds have elapsed since the previous batch.
   class QueueProcessor
     attr_accessor :lag_check_interval, :lag_threshold_s, :dropped_message_report_seconds,
-                  :batch_size, :batch_seconds
-    attr_reader :appender, :queue, :max_queue_size, :non_blocking, :signal,
-                :processed_count, :dropped_count, :async_max_retries, :retry_count
+                  :batch_size, :batch_seconds, :non_blocking
+    attr_reader :appender, :queue, :max_queue_size, :signal,
+                :processed_count, :async_max_retries, :retry_count
 
     # Create a new processor and start its worker thread.
     def self.start(**args)
@@ -63,7 +66,8 @@ module SemanticLogger
     #
     #   dropped_message_report_seconds: [Integer]
     #     When non_blocking is enabled, log the count of dropped messages to the internal logger
-    #     at most once every this number of seconds.
+    #     at most once every this number of seconds. Also limits how often messages discarded
+    #     because their content could not be written are reported, after the first.
     #     Default: 30
     #
     #   async_max_retries: [Integer]
@@ -99,11 +103,9 @@ module SemanticLogger
       @thread                         = nil
       # Only batch mode parks the worker on the signal; streaming mode never touches it.
       @signal                         = Concurrent::Event.new if batch
-      @dropped_message_count          = 0
-      @dropped_message_reported_at    = Time.now
-      @dropped_message_mutex          = Mutex.new
+      @dropped                        = ThrottledCounter.new
+      @failed                         = ThrottledCounter.new(reported_at: nil)
       @processed_count                = 0
-      @dropped_count                  = 0
       create_queue
 
       return unless batch? && !appender.respond_to?(:batch)
@@ -112,9 +114,7 @@ module SemanticLogger
     end
 
     # Internal logger used to report problems encountered while processing log messages.
-    def logger
-      appender.logger
-    end
+    def logger = appender.logger
 
     # Returns [Thread] the worker thread.
     #
@@ -126,33 +126,39 @@ module SemanticLogger
     end
 
     # Returns [true|false] whether the worker thread is running.
-    def active?
-      @thread&.alive?
-    end
+    def active? = @thread&.alive?
 
     # Returns [true|false] whether the queue has a capped size.
-    def capped?
-      @capped
-    end
+    def capped? = @capped
 
     # Returns [true|false] whether messages are dropped instead of blocking when the queue is full.
     # Only a capped queue can drop messages.
-    def non_blocking?
-      @non_blocking && capped?
-    end
+    def non_blocking? = @non_blocking && capped?
+
+    # Returns [Integer] the number of log messages dropped because the queue was full.
+    def dropped_count = @dropped.total
+
+    # Returns [Integer] the number of log messages discarded because their content could not be
+    # written (see Subscriber#message_error?).
+    def failed_count = @failed.total
 
     # Returns [true|false] whether messages are processed in batches.
-    def batch?
-      @batch
-    end
+    def batch? = @batch
+
+    # Returns [true|false] whether the worker thread gave up after an error, so that queued
+    # messages are no longer being processed.
+    def stopped? = @stopped == true
 
     # Add a log message to the queue for processing.
     #
     # When non-blocking and the queue is full, the message is dropped instead of blocking the
     # calling thread, and the count of dropped messages is reported periodically.
+    #
+    # Once the worker thread has stopped for good (see async_max_retries), nothing will empty the
+    # queue, so messages are also dropped instead of blocking the calling thread forever.
     def log(log)
       enqueued = true
-      if non_blocking?
+      if non_blocking? || stopped?
         begin
           queue.push(log, true)
         rescue ThreadError
@@ -195,14 +201,26 @@ module SemanticLogger
 
     private
 
-    # Start the worker thread, naming it after the internal logger.
+    # Start the worker thread.
     def spawn_worker
+      @stopped        = false
+      @stack_overflow = nil
       Thread.new do
-        Thread.current.name = logger.name
+        name_worker
         process
       end
     end
 
+    # Name the worker thread after its appender, such as "SL File", short enough for tools that
+    # list threads, such as `top -H`, to tell the workers apart: Linux keeps only the first 15 bytes.
+    # The name is only a label, so when Ruby rejects it, for example for a class given a temporary
+    # name with a NUL byte, the thread carries on without one.
+    def name_worker
+      Thread.current.name = ["SL", appender.class.name&.split("::")&.last].compact.join(" ")
+    rescue StandardError
+      nil
+    end
+
     def create_queue
       if max_queue_size == -1
         @queue  = Queue.new
@@ -219,22 +237,26 @@ module SemanticLogger
       # or the appender is closed.
       logger.trace "Async: Appender thread active"
       begin
-        batch? ? process_messages_in_batches : process_messages
+        carry_on = batch? ? process_messages_in_batches : process_messages
       rescue StandardError => e
         if async_max_retries == -1 || retry_count < async_max_retries
           @retry_count += 1
           limit = async_max_retries == -1 ? "unlimited" : async_max_retries
-          safe_log(:warn, "Async: Restarting due to exception, retry #{retry_count} of #{limit}, sleeping #{retry_count}s", e)
+          safe_log(:warn,
+                   "Async: Restarting due to exception, retry #{retry_count} of #{limit}, sleeping #{retry_count}s", e)
           sleep(retry_count)
           retry
         else
-          safe_log(:error, "Async: Stopping after #{retry_count} failed retries", e)
+          @stopped = true
+          safe_log(:error, "Async: Stopping after #{retry_count} failed retries, dropping new messages", e)
         end
       rescue Exception => e
+        @stopped = true
         safe_log(:error, "Async: Stopping due to fatal exception", e)
       ensure
-        @thread = nil
-        safe_log(:trace, "Async: Thread has stopped")
+        # A thread that overflowed its stack hands the queue over to a new one (see #stack_overflowed).
+        @thread = carry_on ? spawn_worker : nil
+        safe_log(:trace, "Async: Thread has stopped") unless carry_on
       end
     end
 
@@ -247,12 +269,15 @@ module SemanticLogger
     end
 
     # Streaming: write each log message to the appender as it is dequeued.
+    #
+    # Returns true when processing must carry on in a new thread (see #stack_overflowed).
     def process_messages
       count = 0
       while (message = queue.pop)
         if message.is_a?(Log)
-          appender.log(message)
-          @processed_count += 1
+          write_message(message)
+          return true if @stack_overflow
+
           @retry_count = 0 unless retry_count.zero?
           count += 1
           # Check every few log messages whether this appender thread is falling behind
@@ -261,20 +286,23 @@ module SemanticLogger
             count = 0
           end
         else
-          break unless process_message(message)
+          break unless message.run(appender)
         end
       end
       logger.trace "Async: Queue Closed"
+      false
     end
 
     # Batching: group up log messages before writing them via the appender's #batch method.
+    #
+    # Returns true when processing must carry on in a new thread (see #stack_overflowed).
     def process_messages_in_batches
       loop do
         # Wait for batch interval or number of messages to be exceeded.
         signal.wait(batch_seconds)
 
         logs          = []
-        messages      = []
+        commands      = []
         first         = true
         message_count = queue.length
         message_count.times do
@@ -287,53 +315,122 @@ module SemanticLogger
               first = false
             end
           else
-            messages << message
+            commands << message
           end
         end
-        if logs.size.positive?
-          appender.batch(logs)
-          @processed_count += logs.size
-          @retry_count = 0 unless retry_count.zero?
-        end
         # Stop processing once a command (e.g. :close) signals the thread to terminate.
-        break if messages.any? { |message| !process_message(message) }
+        return false unless write_batch_then_run(logs, commands)
+        return true if @stack_overflow
 
         signal.reset unless queue.size >= batch_size
       end
     end
 
-    # Returns false when message processing should be stopped.
-    def process_message(message)
-      case message[:command]
-      when :flush
-        appender.flush
-        message[:reply_queue] << true if message[:reply_queue]
-      when :close
-        appender.close
-        message[:reply_queue] << true if message[:reply_queue]
-        return false
-      else
-        logger.warn "Async: Appender thread: Ignoring unknown command: #{message[:command]}"
+    # Write the log messages taken off the queue as a batch, then run the commands taken off the
+    # queue with them. Returns false once a command (:close) stops processing.
+    #
+    # When the batch cannot be written, for example because the destination is unavailable, the
+    # commands still run before the error propagates to #process to back off. Otherwise they
+    # would be lost, and a caller waiting on #flush or #close, such as the flush at exit, would
+    # wait forever.
+    def write_batch_then_run(logs, commands)
+      unless logs.empty?
+        write_batch(logs)
+        @retry_count = 0 unless retry_count.zero?
+      end
+      commands.all? { |command| command.run(appender) }
+    rescue StandardError => e
+      raise e if commands.all? { |command| process_message_after_failure(command) }
+
+      safe_log(:warn, "Async: Closed without writing #{logs.size} log messages", e)
+      false
+    end
+
+    # Run a command after a batch could not be written. Returns false once a command (:close)
+    # stops processing, even when the command itself fails.
+    def process_message_after_failure(command)
+      command.run(appender)
+    rescue StandardError => e
+      safe_log(:warn, "Async: Failed to #{command.command} the appender", e)
+      command.command != :close
+    end
+
+    # Write one log message to the appender.
+    #
+    # A message whose content cannot be written (see #message_error?) is discarded and reported,
+    # without restarting the thread. Any other error, such as the destination being unavailable,
+    # propagates to #process, which backs off before retrying. A stack overflow discards the
+    # message too (see #stack_overflowed).
+    def write_message(log)
+      appender.log(log)
+      @processed_count += 1
+    rescue SystemStackError => e
+      stack_overflowed([log], e)
+    rescue StandardError => e
+      raise unless message_error?(e)
+
+      messages_failed([log], e)
+    end
+
+    # Write the log messages to the appender as a batch.
+    #
+    # When the content of a message cannot be written (see #message_error?), and the appender's
+    # #batch is all or nothing, the batch is split in half and each half written separately, so
+    # that only the offending message is discarded rather than every message in the batch.
+    # Otherwise the batch is discarded, since writing it again could duplicate the messages that
+    # the appender had already written. A stack overflow discards the batch, and the rest of the
+    # batch being split, without splitting it further (see #stack_overflowed).
+    def write_batch(logs)
+      return messages_failed(logs, @stack_overflow) if @stack_overflow
+
+      appender.batch(logs)
+      @processed_count += logs.size
+    rescue SystemStackError => e
+      stack_overflowed(logs, e)
+    rescue StandardError => e
+      raise unless message_error?(e)
+      return messages_failed(logs, e) if logs.size == 1 || !appender.batch_all_or_nothing?
+
+      logs.each_slice((logs.size + 1) / 2) { |half| write_batch(half) }
+    end
+
+    # Discard log messages whose writing overflowed the stack, for example a value nested too
+    # deeply for a custom formatter. Retrying cannot write them, as for #message_error?.
+    #
+    # This thread must not overflow its stack again: on macOS a second stack overflow on the same
+    # thread crashes Ruby. So nothing else is written on it, including the rest of a batch, which
+    # is discarded rather than split up further to find the message at fault. Once any commands
+    # taken off the queue with the messages have run, a new thread carries on (see #process).
+    def stack_overflowed(logs, exception)
+      @stack_overflow = exception
+      messages_failed(logs, exception)
+    end
+
+    # Whether the appender says the exception was caused by the content of a log message, rather
+    # than by the destination. An appender that cannot tell (it does not implement
+    # #message_error?) has every error retried with a back-off.
+    def message_error?(exception) = appender.respond_to?(:message_error?) && appender.message_error?(exception)
+
+    # Record log messages discarded because their content could not be written. The first is
+    # reported to the internal logger straight away, with the error. After that the count is
+    # reported, with the latest error, at most once every dropped_message_report_seconds.
+    def messages_failed(logs, exception)
+      @failed.record(logs.size, interval: dropped_message_report_seconds) do |count, seconds|
+        period = seconds ? " in the last #{seconds.round} seconds" : ""
+        safe_log(:error, "Async: Discarded #{count} log message(s)#{period} that could not be written, " \
+                         "most recently from #{logs.first.name}", exception)
       end
-      true
     end
 
     # Record a dropped message, reporting the running count to the internal logger at most
     # once every dropped_message_report_seconds.
     def message_dropped
-      @dropped_message_mutex.synchronize do
-        @dropped_message_count += 1
-        @dropped_count         += 1
-        diff = Time.now - @dropped_message_reported_at
-        return if diff < dropped_message_report_seconds
-
+      @dropped.record(interval: dropped_message_report_seconds) do |count, seconds|
         logger.warn(
-          "Async: Dropped #{@dropped_message_count} log messages in the last #{diff.round} seconds. " \
+          "Async: Dropped #{count} log messages in the last #{seconds.round} seconds. " \
           "The queue is full (max_queue_size: #{max_queue_size}). " \
           "Consider reducing the log level, increasing max_queue_size, or changing the appenders"
         )
-        @dropped_message_count       = 0
-        @dropped_message_reported_at = Time.now
       end
     end
 
@@ -341,7 +438,9 @@ module SemanticLogger
       diff = Time.now - log.time
       return unless diff > lag_threshold_s
 
-      logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with #{queue.size} messages queued up. Consider reducing the log level or changing the appenders"
+      logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with " \
+                  "#{queue.size} messages queued up. Consider reducing the log level or " \
+                  "changing the appenders"
     end
 
     # Submit a command to the worker thread and wait for the reply.
@@ -361,9 +460,12 @@ module SemanticLogger
       # Wake up the processing thread to process this command immediately.
       signal.set if batch?
 
-      reply_queue = Queue.new
-      queue << {command: command, reply_queue: reply_queue}
-      reply_queue.pop
+      submitted_to = queue
+      request      = Request.new(command)
+      submitted_to << request
+      # A reply can come while the request is on the current queue, which #reopen replaces, and a
+      # worker thread is running it, which may be a new one after a stack overflow.
+      request.wait { queue.equal?(submitted_to) && active? }
     end
   end
 end

data/lib/semantic_logger/reporters/minitest.rb

@@ -16,10 +16,10 @@ module SemanticLogger
     #
     # Log entries similar to the following should show up in the log file:
     #
-    # 2019-02-06 18:58:17.522467 I [84730:70256441962000] Minitest -- START RocketJob::DirmonEntry::with valid entry::#archive_file test_0001_moves file to archive dir
-    # 2019-02-06 18:58:17.527492 I [84730:70256441962000] (4.980ms) Minitest -- PASS RocketJob::DirmonEntry::with valid entry::#archive_file test_0001_moves file to archive dir
-    # 2019-02-06 18:58:17.527835 I [84730:70256441962000] Minitest -- START RocketJob::DirmonEntry::#job_class::with a valid job_class_name test_0001_return job class
-    # 2019-02-06 18:58:17.529761 I [84730:70256441962000] (1.882ms) Minitest -- PASS RocketJob::DirmonEntry::#job_class::with a valid job_class_name test_0001_return job class
+    # 2019-02-06 18:58:17.522467 I [84730:70256] Minitest -- START MyJob test_0001_archives file
+    # 2019-02-06 18:58:17.527492 I [84730:70256] (4.980ms) Minitest -- PASS MyJob test_0001_archives file
+    # 2019-02-06 18:58:17.527835 I [84730:70256] Minitest -- START MyJob test_0002_returns job class
+    # 2019-02-06 18:58:17.529761 I [84730:70256] (1.882ms) Minitest -- PASS MyJob test_0002_returns job class
     class Minitest < ::Minitest::AbstractReporter
       include SemanticLogger::Loggable
 

data/lib/semantic_logger/semantic_logger.rb

@@ -549,8 +549,9 @@ module SemanticLogger
   #   processed:      [Integer] Cumulative number of log messages processed since startup.
   #   dropped:        [Integer] Cumulative number of log messages dropped at the main queue.
   #   appenders:      [Array<Hash>] Per-appender statistics. Appenders that run their own
-  #                                 async thread report their queue_size and processed/dropped
-  #                                 counts; appenders that log inline report `async: false`.
+  #                                 async thread report their queue_size and
+  #                                 processed/dropped/failed counts; appenders that log
+  #                                 inline report `async: false`.
   #
   # All counters are cumulative since process startup. They are thread-safe to read and
   # are maintained without adding any locking to the logging hot path.
@@ -576,6 +577,18 @@ module SemanticLogger
     Logger.processor.lag_threshold_s
   end
 
+  # Returns [true|false] whether log messages are dropped instead of blocking
+  # when the main pipeline queue is full.
+  def self.non_blocking?
+    Logger.processor.non_blocking?
+  end
+
+  # Drop log messages instead of blocking the calling thread when the main
+  # pipeline queue is full.
+  def self.non_blocking=(non_blocking)
+    Logger.processor.non_blocking = non_blocking
+  end
+
   def self.default_level_index
     Thread.current[:semantic_logger_silence] || @default_level_index
   end