semantic_logger 4.18.0 → 5.0.0

data/lib/semantic_logger/semantic_logger.rb

@@ -4,9 +4,42 @@ module SemanticLogger
   # Logging levels in order of most detailed to most severe
   LEVELS = Levels::LEVELS
 
-  # Return a logger for the supplied class or class_name
+  # Return a logger for the supplied class or class_name.
+  #
+  # When `SemanticLogger.cache_loggers` is enabled (opt-in, default off) and a
+  # Class or Module is supplied, the same Logger instance is returned for every
+  # call with that class. This makes it possible to obtain a logger once and
+  # later change its level (or filter) and have every holder of that logger see
+  # the change.
+  #
+  # A String is always given its own new Logger instance, even when caching is
+  # enabled: callers that pass a string typically want an independent logger
+  # (for example to set a different level per call site). Anonymous classes
+  # (those with no `name`) are never cached, to avoid pinning short-lived
+  # dynamically created classes in memory.
   def self.[](klass)
-    Logger.new(klass)
+    return Logger.new(klass) if !@cache_loggers || klass.is_a?(String) || klass.name.nil?
+
+    logger_cache.compute_if_absent(klass) { Logger.new(klass) }
+  end
+
+  # Whether `SemanticLogger[Class]` returns a shared, cached Logger instance per
+  # class. Disabled by default. Strings are never cached (see #[]).
+  def self.cache_loggers=(cache_loggers)
+    @cache_loggers = cache_loggers
+    clear_logger_cache unless cache_loggers
+  end
+
+  # Returns whether logger caching is enabled.
+  def self.cache_loggers?
+    @cache_loggers
+  end
+
+  # Discard all cached loggers so that subsequent `SemanticLogger[Class]` calls
+  # build fresh instances. Primarily useful in tests, or after redefining a
+  # class that was previously cached.
+  def self.clear_logger_cache
+    @logger_cache&.clear
   end
 
   # Sets the global default log level
@@ -162,8 +195,8 @@ module SemanticLogger
   #   logger = SemanticLogger['Example']
   #   logger.info "Hello World"
   #   logger.debug("Login time", user: 'Joe', duration: 100, ip_address: '127.0.0.1')
-  def self.add_appender(**args, &block)
-    appender = appenders.add(**args, &block)
+  def self.add_appender(**args, &)
+    appender = appenders.add(**args, &)
     # Start appender thread if it is not already running
     Logger.processor.start
     appender
@@ -202,14 +235,43 @@ module SemanticLogger
     Logger.processor.close
   end
 
-  # After forking an active process call SemanticLogger.reopen to re-open
-  # any open file handles etc to resources.
+  # Re-open any open file handles etc. to resources.
+  #
+  # Called automatically in the child process after a fork (see reopen_on_fork?),
+  # and may also be called manually.
+  #
+  # To avoid reopening twice after a single fork (for example when the automatic
+  # fork hook and a framework's `after_fork` callback both fire), reopen is a no-op
+  # if it has already run in the current process. Pass `force: true` to override
+  # this guard and reopen unconditionally, for example when re-opening file handles
+  # in the same process after an external log rotation.
   #
   # Note:
   #   Not all appender's implement reopen.
   #   Check the code for each appender you are using before relying on this behavior.
-  def self.reopen
+  def self.reopen(force: false)
+    return if !force && @reopened_pid == ::Process.pid
+
     Logger.processor.reopen
+    @reopened_pid = ::Process.pid
+  end
+
+  # Whether appenders are automatically reopened in the child process after a fork.
+  #
+  # Enabled by default. A `Process._fork` hook (Ruby 3.1+) calls SemanticLogger.reopen
+  # in the child after `fork`, `Process.daemon`, `IO.popen`, `Kernel#system`, and
+  # backticks, so framework specific `after_fork` hooks (Puma, Unicorn, Resque,
+  # Spring, etc.) are no longer required.
+  def self.reopen_on_fork?
+    @reopen_on_fork != false
+  end
+
+  # Enable or disable automatic reopening of appenders after a fork.
+  #
+  #   # Opt out of the automatic behavior and manage reopen manually:
+  #   SemanticLogger.reopen_on_fork = false
+  def self.reopen_on_fork=(reopen_on_fork)
+    @reopen_on_fork = reopen_on_fork
   end
 
   # Supply a callback to be called whenever a log entry is created.
@@ -236,8 +298,8 @@ module SemanticLogger
   # Note:
   # * This callback is called within the thread of the application making the logging call.
   # * If these callbacks are slow they will slow down the application.
-  def self.on_log(object = nil, &block)
-    Logger.subscribe(object, &block)
+  def self.on_log(object = nil, &)
+    Logger.subscribe(object, &)
   end
 
   # Add signal handlers for Semantic Logger
@@ -342,13 +404,13 @@ module SemanticLogger
   #     `logger.tagged([['first', nil], nil, ['more'], 'other'])`
   #   to the equivalent of:
   #     `logger.tagged('first', 'more', 'other')`
-  def self.tagged(*tags, &block)
+  def self.tagged(*tags, &)
     return yield if tags.empty?
 
     # Allow named tags to be passed into the logger
     if tags.size == 1
       tag = tags[0]
-      return tag.is_a?(Hash) ? named_tagged(tag, &block) : fast_tag(tag, &block)
+      return tag.is_a?(Hash) ? named_tagged(tag, &) : fast_tag(tag, &)
     end
 
     begin
@@ -475,6 +537,27 @@ module SemanticLogger
     Logger.processor.queue.size
   end
 
+  # Returns [Hash] operational statistics for the logging pipeline.
+  #
+  # Useful for exporting Semantic Logger's own health to a monitoring system such as
+  # Prometheus, statsd, etc. The returned Hash contains:
+  #
+  #   queue_size:     [Integer] Number of log messages waiting on the main pipeline queue.
+  #   capped:         [Boolean] Whether the main queue has a maximum size.
+  #   max_queue_size: [Integer] Maximum queue size, or nil when uncapped.
+  #   thread_active:  [Boolean] Whether the main pipeline thread is running.
+  #   processed:      [Integer] Cumulative number of log messages processed since startup.
+  #   dropped:        [Integer] Cumulative number of log messages dropped at the main queue.
+  #   appenders:      [Array<Hash>] Per-appender statistics. Appenders that run their own
+  #                                 async thread report their queue_size and processed/dropped
+  #                                 counts; appenders that log inline report `async: false`.
+  #
+  # All counters are cumulative since process startup. They are thread-safe to read and
+  # are maintained without adding any locking to the logging hot path.
+  def self.stats
+    Logger.processor.stats
+  end
+
   # Returns the check_interval which is the number of messages between checks
   # to determine if the appender thread is falling behind.
   def self.lag_check_interval
@@ -516,6 +599,14 @@ module SemanticLogger
   @backtrace_level       = :error
   @backtrace_level_index = Levels.index(@backtrace_level)
   @sync                  = false
+  @cache_loggers         = false
+  @logger_cache          = nil
+
+  # Lazily initialized thread-safe cache of one Logger per Class/Module.
+  def self.logger_cache
+    @logger_cache ||= Concurrent::Map.new
+  end
+  private_class_method :logger_cache
 
   # @formatter:off
   module Metric
@@ -531,6 +622,7 @@ module SemanticLogger
   module Test
     autoload :CaptureLogEvents,  "semantic_logger/test/capture_log_events"
     autoload :Minitest,          "semantic_logger/test/minitest"
+    autoload :RSpec,             "semantic_logger/test/rspec"
   end
 
   if defined?(JRuby)

data/lib/semantic_logger/subscriber.rb

@@ -73,9 +73,22 @@ module SemanticLogger
       super && (log.metric_only? ? metrics? : true)
     end
 
-    # Whether this appender is logging to stdout or stderror
+    # The console stream this appender writes to, if any.
+    # Returns one of :stdout, :stderr, or nil when not writing to a console stream.
+    def console_stream
+      nil
+    end
+
+    # Whether this appender is logging to stdout or stderr.
     def console_output?
-      false
+      !console_stream.nil?
+    end
+
+    # Whether an appender that implements #batch should be wrapped in a batch
+    # proxy automatically. Appenders that support batching but should only batch
+    # when explicitly requested (via `batch: true`) override this to return false.
+    def batch_by_default?
+      true
     end
 
     private

data/lib/semantic_logger/sync_processor.rb

@@ -9,7 +9,28 @@ module SemanticLogger
     end
 
     def log(...)
-      @monitor.synchronize { @appenders.log(...) }
+      @monitor.synchronize do
+        @processed_count += 1
+        @appenders.log(...)
+      end
+    end
+
+    # Returns [Hash] operational statistics for the logging pipeline.
+    #
+    # In synchronous mode there is no queue: messages are written inline on the calling
+    # thread, so queue_size is always 0 and no messages can be dropped.
+    def stats
+      @monitor.synchronize do
+        {
+          queue_size:     0,
+          capped:         false,
+          max_queue_size: nil,
+          thread_active:  false,
+          processed:      @processed_count,
+          dropped:        0,
+          appenders:      @appenders.stats
+        }
+      end
     end
 
     def flush
@@ -47,8 +68,9 @@ module SemanticLogger
     attr_reader :appenders
 
     def initialize(appenders = nil)
-      @monitor   = Monitor.new
-      @appenders = appenders || Appenders.new(self.class.logger.dup)
+      @monitor         = Monitor.new
+      @appenders       = appenders || Appenders.new(self.class.logger.dup)
+      @processed_count = 0
     end
 
     def start

data/lib/semantic_logger/test/capture_log_events.rb

@@ -34,8 +34,8 @@ module SemanticLogger
       end
 
       # Supports batching of log events
-      def batch(_logs)
-        @events += log
+      def batch(logs)
+        logs.each { |log| self.log(log) }
       end
 
       def clear

data/lib/semantic_logger/test/minitest.rb

@@ -57,15 +57,17 @@ module SemanticLogger
         if payload_includes
           payload_includes.each_pair do |key, expected|
             actual = event.payload[key]
-            assert_semantic_logger_entry(event, "payload #{name}", expected, actual)
+
+            assert_semantic_logger_entry(event, "payload #{key}", expected, actual)
           end
         end
 
         return unless exception_includes
 
-        payload_includes.each_pair do |key, expected|
+        exception_includes.each_pair do |key, expected|
           actual = event.exception.send(key)
-          assert_semantic_logger_entry(event, "Exception #{name}", expected, actual)
+
+          assert_semantic_logger_entry(event, "exception #{key}", expected, actual)
         end
       end
 
@@ -76,9 +78,11 @@ module SemanticLogger
 
         case expected
         when :nil
+
           assert_nil actual, "Expected nil #{name} for log event: #{event.to_h.inspect}"
         when Class
-          assert actual.is_a?(expected), lambda {
+
+          assert_kind_of expected, actual, lambda {
             "Type #{expected} expected for #{name} in log event: #{event.to_h.inspect}"
           }
         else

data/lib/semantic_logger/test/rspec.rb

@@ -0,0 +1,249 @@
+require "rspec/expectations"
+
+module SemanticLogger
+  module Test
+    # RSpec matchers and helpers for asserting on Semantic Logger events.
+    #
+    # These mirror the Minitest helpers in SemanticLogger::Test::Minitest.
+    #
+    # Enable them once, in spec_helper.rb:
+    #
+    #   require "semantic_logger/test/rspec"
+    #
+    #   RSpec.configure do |config|
+    #     config.include SemanticLogger::Test::RSpec
+    #   end
+    #
+    # Capture the events emitted whilst running a block, then assert on them:
+    #
+    #   events = capture_semantic_logger_events { User.new.enable! }
+    #   expect(events.first).to be_a_semantic_logger_event(level: :info, message: "User enabled")
+    #
+    # Or assert directly that a block logs a matching event:
+    #
+    #   expect { User.new.enable! }.to(
+    #     log_semantic_logger_event(level: :info, message: "User enabled")
+    #   )
+    module RSpec
+      # Attributes that may be asserted on a single log event. The keys map
+      # directly onto SemanticLogger::Log attributes, except for the
+      # *_includes variants which assert a partial (substring / subset) match.
+      EVENT_ATTRIBUTES = %i[
+        level name message thread_name tags named_tags context
+        metric metric_amount dimensions level_index duration time
+        exception backtrace payload
+      ].freeze
+
+      # Matches a single SemanticLogger::Log against a set of expectations.
+      #
+      # An expected value may be:
+      #   * a Class    - the actual value must be a kind_of that class
+      #   * :nil       - the actual value must be nil
+      #   * any value  - compared with ==
+      class EventMatcher
+        include ::RSpec::Matchers::Composable
+
+        def initialize(message_includes: nil, payload_includes: nil, exception_includes: nil, **expected)
+          unknown = expected.keys - EVENT_ATTRIBUTES
+          raise ArgumentError, "Unknown log event attribute(s): #{unknown.join(', ')}" unless unknown.empty?
+
+          @expected           = expected
+          @message_includes   = message_includes
+          @payload_includes   = payload_includes
+          @exception_includes = exception_includes
+        end
+
+        def matches?(event)
+          @event = event
+          return failed("no log event") unless event
+
+          @expected.each_pair do |name, expected|
+            actual = event.public_send(name)
+            return failed(mismatch(name, expected, actual)) unless attribute_matches?(expected, actual)
+          end
+
+          return failed(includes_failure(:message, @message_includes)) unless message_includes_matches?
+          return failed(includes_failure(:payload, @payload_includes)) unless payload_includes_matches?
+          return failed(includes_failure(:exception, @exception_includes)) unless exception_includes_matches?
+
+          true
+        end
+
+        def description
+          "be a semantic logger event matching #{full_expectations.inspect}"
+        end
+
+        def failure_message
+          "expected log event to #{description}, but #{@failure}.\n" \
+            "Log event: #{event_inspect}"
+        end
+
+        def failure_message_when_negated
+          "expected log event not to #{description}.\nLog event: #{event_inspect}"
+        end
+
+        private
+
+        def failed(reason)
+          @failure = reason
+          false
+        end
+
+        def attribute_matches?(expected, actual)
+          case expected
+          when :nil
+            actual.nil?
+          when Class
+            actual.is_a?(expected)
+          else
+            expected == actual
+          end
+        end
+
+        def message_includes_matches?
+          return true unless @message_includes
+
+          @event.message&.include?(@message_includes)
+        end
+
+        def payload_includes_matches?
+          return true unless @payload_includes
+
+          payload = @event.payload || {}
+          @payload_includes.all? { |key, value| payload[key] == value }
+        end
+
+        def exception_includes_matches?
+          return true unless @exception_includes
+
+          exception = @event.exception
+          return false unless exception
+
+          @exception_includes.all? { |key, value| exception.public_send(key) == value }
+        end
+
+        def mismatch(name, expected, actual)
+          "#{name} was #{actual.inspect} (expected #{expected.inspect})"
+        end
+
+        def includes_failure(name, expected)
+          "#{name} did not include #{expected.inspect}"
+        end
+
+        def full_expectations
+          @expected.merge(
+            {
+              message_includes:   @message_includes,
+              payload_includes:   @payload_includes,
+              exception_includes: @exception_includes
+            }.compact
+          )
+        end
+
+        def event_inspect
+          @event.respond_to?(:to_h) ? @event.to_h.inspect : @event.inspect
+        end
+      end
+
+      # Matches a block, asserting that it emits at least one log event that
+      # matches the supplied expectations.
+      class LogEventMatcher
+        include ::RSpec::Matchers::Composable
+
+        def initialize(capture, on:, expected:)
+          @capture  = capture
+          @on       = on
+          @matcher  = EventMatcher.new(**expected)
+        end
+
+        def matches?(block)
+          @events = @capture.call(@on, &block)
+          @events.any? { |event| @matcher.matches?(event) }
+        end
+
+        def supports_block_expectations?
+          true
+        end
+
+        def description
+          "log a semantic logger event matching #{@matcher.description}"
+        end
+
+        def failure_message
+          "expected the block to #{description}.\n" \
+            "Captured #{@events.size} event(s):\n#{captured_inspect}"
+        end
+
+        def failure_message_when_negated
+          "expected the block not to #{description}, but it did.\n" \
+            "Captured #{@events.size} event(s):\n#{captured_inspect}"
+        end
+
+        private
+
+        def captured_inspect
+          @events.map { |event| "  #{event.to_h.inspect}" }.join("\n")
+        end
+      end
+
+      # Returns [Array<SemanticLogger::Log>] the log events captured whilst
+      # running the supplied block.
+      #
+      # Notes:
+      # - All log events are captured regardless of the global default log level.
+      # - Pass a class to capture only events logged through that class's logger.
+      #   Otherwise every log event in the process is captured for the duration
+      #   of the block.
+      def capture_semantic_logger_events(klass = nil, silence: :trace, &block)
+        logger = SemanticLogger::Test::CaptureLogEvents.new
+
+        if klass
+          allow(klass).to receive(:logger).and_return(logger)
+          block.call
+        elsif silence
+          SemanticLogger.silence(silence) do
+            stub_processor(logger, &block)
+          end
+        else
+          stub_processor(logger, &block)
+        end
+
+        logger.events
+      end
+
+      # Matcher for a single captured log event.
+      #
+      #   expect(events.first).to be_a_semantic_logger_event(level: :info, message: "Hi")
+      def be_a_semantic_logger_event(**expected)
+        EventMatcher.new(**expected)
+      end
+
+      # Composable alias, for use inside other matchers:
+      #
+      #   expect(events).to include(a_semantic_logger_event(message: "Hi"))
+      def a_semantic_logger_event(**expected)
+        EventMatcher.new(**expected)
+      end
+
+      # Block matcher asserting that the block logs a matching event.
+      #
+      #   expect { User.new.enable! }.to(
+      #     log_semantic_logger_event(level: :info, message: "User enabled")
+      #   )
+      #
+      # Pass `on:` to capture only one class's events.
+      def log_semantic_logger_event(on: nil, **expected)
+        LogEventMatcher.new(method(:capture_semantic_logger_events), on: on, expected: expected)
+      end
+
+      private
+
+      def stub_processor(logger)
+        allow(SemanticLogger::Logger).to receive(:processor).and_return(logger)
+        yield
+      ensure
+        allow(SemanticLogger::Logger).to receive(:processor).and_call_original
+      end
+    end
+  end
+end

data/lib/semantic_logger/utils.rb

@@ -1,15 +1,28 @@
+require "json"
+
 module SemanticLogger
   # Internal-use only utility functions for Semantic Logger.
   # Not intended for public use.
   module Utils
     def self.constantize_symbol(symbol, namespace = "SemanticLogger::Appender")
       klass = "#{namespace}::#{camelize(symbol.to_s)}"
-      begin
-        Object.const_get(klass)
-      rescue NameError
+      constant =
+        begin
+          Object.const_get(klass)
+        rescue NameError
+          raise(ArgumentError,
+                "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. Looking for: #{klass}")
+        end
+
+      # The resolved constant is instantiated by the caller, so ensure it is
+      # actually a class within the expected namespace rather than some other
+      # constant that happens to share the name.
+      unless constant.is_a?(Class)
         raise(ArgumentError,
-              "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. Looking for: #{klass}")
+              "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. #{klass} is not a class.")
       end
+
+      constant
     end
 
     # Borrow from Rails, when not running Rails
@@ -76,5 +89,71 @@ module SemanticLogger
     def self.strip_path?(path)
       strip_paths.any? { |exclude| path.start_with?(exclude) }
     end
+
+    # Serializes the value to JSON, repairing invalid UTF-8 only when necessary.
+    #
+    # Non UTF-8 data appears in well under 1% of log events, so it is wasteful to
+    # walk and reallocate the entire structure (see .encode_utf8) on every call.
+    # Instead this attempts `.to_json` directly and only falls back to cleansing
+    # when serialization fails because of an encoding problem.
+    #
+    # The exception raised for non UTF-8 data depends on the json gem version:
+    # older versions raise Encoding::UndefinedConversionError (an EncodingError),
+    # newer versions wrap it as JSON::GeneratorError, so both are rescued. The
+    # retry is attempted only once: if it still fails (for example a
+    # JSON::GeneratorError caused by something other than encoding, such as NaN),
+    # the error propagates unchanged rather than being swallowed.
+    def self.to_json(value)
+      value.to_json
+    rescue JSON::GeneratorError, EncodingError
+      encode_utf8(value).to_json
+    end
+
+    # Returns a copy of the supplied value with every String converted to valid UTF-8.
+    #
+    # Recurses through Hash and Array structures, cleansing both keys and values.
+    # Strings that are already valid UTF-8 are returned unchanged (the common case),
+    # so the fast path allocates nothing. Any other value (Symbol, Numeric, Time, nil,
+    # ...) is returned as-is.
+    #
+    # Used by .to_json on the rare failing path, and directly by formatters that
+    # serialize per value or emit to a non-JSON sink (where a single `.to_json`
+    # rescue boundary cannot catch an intermediate failure).
+    def self.encode_utf8(value)
+      case value
+      when String
+        encode_utf8_string(value)
+      when Hash
+        value.each_with_object({}) do |(key, val), hash|
+          hash[encode_utf8(key)] = encode_utf8(val)
+        end
+      when Array
+        value.map { |element| encode_utf8(element) }
+      else
+        value
+      end
+    end
+
+    # Options used when transcoding a string to UTF-8.
+    # Invalid byte sequences and characters that cannot be represented in UTF-8 are
+    # dropped rather than substituted, matching the preference in issue #180.
+    ENCODE_UTF8_OPTIONS = {invalid: :replace, undef: :replace, replace: "".freeze}.freeze
+
+    # Returns the string converted to valid UTF-8, dropping any invalid bytes.
+    def self.encode_utf8_string(string)
+      return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
+
+      if string.encoding == Encoding::UTF_8
+        # Correctly tagged as UTF-8 but contains invalid byte sequences.
+        string.scrub("")
+      else
+        # Different encoding (e.g. ASCII-8BIT / Latin-1): transcode into UTF-8.
+        string.encode(Encoding::UTF_8, **ENCODE_UTF8_OPTIONS)
+      end
+    rescue EncodingError
+      # Last resort for encodings without a converter to UTF-8: reinterpret the
+      # raw bytes as UTF-8 and drop anything invalid. Logging must never raise.
+      string.dup.force_encoding(Encoding::UTF_8).scrub("")
+    end
   end
 end

data/lib/semantic_logger/version.rb

@@ -1,3 +1,3 @@
 module SemanticLogger
-  VERSION = "4.18.0".freeze
+  VERSION = "5.0.0".freeze
 end

data/lib/semantic_logger.rb

@@ -12,12 +12,21 @@ require "semantic_logger/loggable"
 require "semantic_logger/concerns/compatibility"
 require "semantic_logger/appender"
 require "semantic_logger/appenders"
+require "semantic_logger/queue_processor"
 require "semantic_logger/processor"
 require "semantic_logger/sync_processor"
 require "semantic_logger/logger"
 require "semantic_logger/debug_as_trace_logger"
 require "semantic_logger/semantic_logger"
 
+# Automatically reopen appenders in the child process after a fork.
+# Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
+# Skipped on platforms without `Process._fork` (e.g. JRuby), which cannot fork.
+if Process.respond_to?(:_fork)
+  require "semantic_logger/core_ext/process"
+  Process.singleton_class.prepend(SemanticLogger::CoreExt::Process)
+end
+
 # Flush all appenders at exit, waiting for outstanding messages on the queue
 # to be written first.
 at_exit do