semantic_logger 4.17.0 → 5.0.0

data/lib/semantic_logger/test/rspec.rb

@@ -0,0 +1,249 @@
+require "rspec/expectations"
+
+module SemanticLogger
+  module Test
+    # RSpec matchers and helpers for asserting on Semantic Logger events.
+    #
+    # These mirror the Minitest helpers in SemanticLogger::Test::Minitest.
+    #
+    # Enable them once, in spec_helper.rb:
+    #
+    #   require "semantic_logger/test/rspec"
+    #
+    #   RSpec.configure do |config|
+    #     config.include SemanticLogger::Test::RSpec
+    #   end
+    #
+    # Capture the events emitted whilst running a block, then assert on them:
+    #
+    #   events = capture_semantic_logger_events { User.new.enable! }
+    #   expect(events.first).to be_a_semantic_logger_event(level: :info, message: "User enabled")
+    #
+    # Or assert directly that a block logs a matching event:
+    #
+    #   expect { User.new.enable! }.to(
+    #     log_semantic_logger_event(level: :info, message: "User enabled")
+    #   )
+    module RSpec
+      # Attributes that may be asserted on a single log event. The keys map
+      # directly onto SemanticLogger::Log attributes, except for the
+      # *_includes variants which assert a partial (substring / subset) match.
+      EVENT_ATTRIBUTES = %i[
+        level name message thread_name tags named_tags context
+        metric metric_amount dimensions level_index duration time
+        exception backtrace payload
+      ].freeze
+
+      # Matches a single SemanticLogger::Log against a set of expectations.
+      #
+      # An expected value may be:
+      #   * a Class    - the actual value must be a kind_of that class
+      #   * :nil       - the actual value must be nil
+      #   * any value  - compared with ==
+      class EventMatcher
+        include ::RSpec::Matchers::Composable
+
+        def initialize(message_includes: nil, payload_includes: nil, exception_includes: nil, **expected)
+          unknown = expected.keys - EVENT_ATTRIBUTES
+          raise ArgumentError, "Unknown log event attribute(s): #{unknown.join(', ')}" unless unknown.empty?
+
+          @expected           = expected
+          @message_includes   = message_includes
+          @payload_includes   = payload_includes
+          @exception_includes = exception_includes
+        end
+
+        def matches?(event)
+          @event = event
+          return failed("no log event") unless event
+
+          @expected.each_pair do |name, expected|
+            actual = event.public_send(name)
+            return failed(mismatch(name, expected, actual)) unless attribute_matches?(expected, actual)
+          end
+
+          return failed(includes_failure(:message, @message_includes)) unless message_includes_matches?
+          return failed(includes_failure(:payload, @payload_includes)) unless payload_includes_matches?
+          return failed(includes_failure(:exception, @exception_includes)) unless exception_includes_matches?
+
+          true
+        end
+
+        def description
+          "be a semantic logger event matching #{full_expectations.inspect}"
+        end
+
+        def failure_message
+          "expected log event to #{description}, but #{@failure}.\n" \
+            "Log event: #{event_inspect}"
+        end
+
+        def failure_message_when_negated
+          "expected log event not to #{description}.\nLog event: #{event_inspect}"
+        end
+
+        private
+
+        def failed(reason)
+          @failure = reason
+          false
+        end
+
+        def attribute_matches?(expected, actual)
+          case expected
+          when :nil
+            actual.nil?
+          when Class
+            actual.is_a?(expected)
+          else
+            expected == actual
+          end
+        end
+
+        def message_includes_matches?
+          return true unless @message_includes
+
+          @event.message&.include?(@message_includes)
+        end
+
+        def payload_includes_matches?
+          return true unless @payload_includes
+
+          payload = @event.payload || {}
+          @payload_includes.all? { |key, value| payload[key] == value }
+        end
+
+        def exception_includes_matches?
+          return true unless @exception_includes
+
+          exception = @event.exception
+          return false unless exception
+
+          @exception_includes.all? { |key, value| exception.public_send(key) == value }
+        end
+
+        def mismatch(name, expected, actual)
+          "#{name} was #{actual.inspect} (expected #{expected.inspect})"
+        end
+
+        def includes_failure(name, expected)
+          "#{name} did not include #{expected.inspect}"
+        end
+
+        def full_expectations
+          @expected.merge(
+            {
+              message_includes:   @message_includes,
+              payload_includes:   @payload_includes,
+              exception_includes: @exception_includes
+            }.compact
+          )
+        end
+
+        def event_inspect
+          @event.respond_to?(:to_h) ? @event.to_h.inspect : @event.inspect
+        end
+      end
+
+      # Matches a block, asserting that it emits at least one log event that
+      # matches the supplied expectations.
+      class LogEventMatcher
+        include ::RSpec::Matchers::Composable
+
+        def initialize(capture, on:, expected:)
+          @capture  = capture
+          @on       = on
+          @matcher  = EventMatcher.new(**expected)
+        end
+
+        def matches?(block)
+          @events = @capture.call(@on, &block)
+          @events.any? { |event| @matcher.matches?(event) }
+        end
+
+        def supports_block_expectations?
+          true
+        end
+
+        def description
+          "log a semantic logger event matching #{@matcher.description}"
+        end
+
+        def failure_message
+          "expected the block to #{description}.\n" \
+            "Captured #{@events.size} event(s):\n#{captured_inspect}"
+        end
+
+        def failure_message_when_negated
+          "expected the block not to #{description}, but it did.\n" \
+            "Captured #{@events.size} event(s):\n#{captured_inspect}"
+        end
+
+        private
+
+        def captured_inspect
+          @events.map { |event| "  #{event.to_h.inspect}" }.join("\n")
+        end
+      end
+
+      # Returns [Array<SemanticLogger::Log>] the log events captured whilst
+      # running the supplied block.
+      #
+      # Notes:
+      # - All log events are captured regardless of the global default log level.
+      # - Pass a class to capture only events logged through that class's logger.
+      #   Otherwise every log event in the process is captured for the duration
+      #   of the block.
+      def capture_semantic_logger_events(klass = nil, silence: :trace, &block)
+        logger = SemanticLogger::Test::CaptureLogEvents.new
+
+        if klass
+          allow(klass).to receive(:logger).and_return(logger)
+          block.call
+        elsif silence
+          SemanticLogger.silence(silence) do
+            stub_processor(logger, &block)
+          end
+        else
+          stub_processor(logger, &block)
+        end
+
+        logger.events
+      end
+
+      # Matcher for a single captured log event.
+      #
+      #   expect(events.first).to be_a_semantic_logger_event(level: :info, message: "Hi")
+      def be_a_semantic_logger_event(**expected)
+        EventMatcher.new(**expected)
+      end
+
+      # Composable alias, for use inside other matchers:
+      #
+      #   expect(events).to include(a_semantic_logger_event(message: "Hi"))
+      def a_semantic_logger_event(**expected)
+        EventMatcher.new(**expected)
+      end
+
+      # Block matcher asserting that the block logs a matching event.
+      #
+      #   expect { User.new.enable! }.to(
+      #     log_semantic_logger_event(level: :info, message: "User enabled")
+      #   )
+      #
+      # Pass `on:` to capture only one class's events.
+      def log_semantic_logger_event(on: nil, **expected)
+        LogEventMatcher.new(method(:capture_semantic_logger_events), on: on, expected: expected)
+      end
+
+      private
+
+      def stub_processor(logger)
+        allow(SemanticLogger::Logger).to receive(:processor).and_return(logger)
+        yield
+      ensure
+        allow(SemanticLogger::Logger).to receive(:processor).and_call_original
+      end
+    end
+  end
+end

data/lib/semantic_logger/utils.rb

@@ -1,15 +1,28 @@
+require "json"
+
 module SemanticLogger
   # Internal-use only utility functions for Semantic Logger.
   # Not intended for public use.
   module Utils
     def self.constantize_symbol(symbol, namespace = "SemanticLogger::Appender")
       klass = "#{namespace}::#{camelize(symbol.to_s)}"
-      begin
-        Object.const_get(klass)
-      rescue NameError
+      constant =
+        begin
+          Object.const_get(klass)
+        rescue NameError
+          raise(ArgumentError,
+                "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. Looking for: #{klass}")
+        end
+
+      # The resolved constant is instantiated by the caller, so ensure it is
+      # actually a class within the expected namespace rather than some other
+      # constant that happens to share the name.
+      unless constant.is_a?(Class)
         raise(ArgumentError,
-              "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. Looking for: #{klass}")
+              "Could not convert symbol: #{symbol.inspect} to a class in: #{namespace}. #{klass} is not a class.")
       end
+
+      constant
     end
 
     # Borrow from Rails, when not running Rails
@@ -76,5 +89,71 @@ module SemanticLogger
     def self.strip_path?(path)
       strip_paths.any? { |exclude| path.start_with?(exclude) }
     end
+
+    # Serializes the value to JSON, repairing invalid UTF-8 only when necessary.
+    #
+    # Non UTF-8 data appears in well under 1% of log events, so it is wasteful to
+    # walk and reallocate the entire structure (see .encode_utf8) on every call.
+    # Instead this attempts `.to_json` directly and only falls back to cleansing
+    # when serialization fails because of an encoding problem.
+    #
+    # The exception raised for non UTF-8 data depends on the json gem version:
+    # older versions raise Encoding::UndefinedConversionError (an EncodingError),
+    # newer versions wrap it as JSON::GeneratorError, so both are rescued. The
+    # retry is attempted only once: if it still fails (for example a
+    # JSON::GeneratorError caused by something other than encoding, such as NaN),
+    # the error propagates unchanged rather than being swallowed.
+    def self.to_json(value)
+      value.to_json
+    rescue JSON::GeneratorError, EncodingError
+      encode_utf8(value).to_json
+    end
+
+    # Returns a copy of the supplied value with every String converted to valid UTF-8.
+    #
+    # Recurses through Hash and Array structures, cleansing both keys and values.
+    # Strings that are already valid UTF-8 are returned unchanged (the common case),
+    # so the fast path allocates nothing. Any other value (Symbol, Numeric, Time, nil,
+    # ...) is returned as-is.
+    #
+    # Used by .to_json on the rare failing path, and directly by formatters that
+    # serialize per value or emit to a non-JSON sink (where a single `.to_json`
+    # rescue boundary cannot catch an intermediate failure).
+    def self.encode_utf8(value)
+      case value
+      when String
+        encode_utf8_string(value)
+      when Hash
+        value.each_with_object({}) do |(key, val), hash|
+          hash[encode_utf8(key)] = encode_utf8(val)
+        end
+      when Array
+        value.map { |element| encode_utf8(element) }
+      else
+        value
+      end
+    end
+
+    # Options used when transcoding a string to UTF-8.
+    # Invalid byte sequences and characters that cannot be represented in UTF-8 are
+    # dropped rather than substituted, matching the preference in issue #180.
+    ENCODE_UTF8_OPTIONS = {invalid: :replace, undef: :replace, replace: "".freeze}.freeze
+
+    # Returns the string converted to valid UTF-8, dropping any invalid bytes.
+    def self.encode_utf8_string(string)
+      return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
+
+      if string.encoding == Encoding::UTF_8
+        # Correctly tagged as UTF-8 but contains invalid byte sequences.
+        string.scrub("")
+      else
+        # Different encoding (e.g. ASCII-8BIT / Latin-1): transcode into UTF-8.
+        string.encode(Encoding::UTF_8, **ENCODE_UTF8_OPTIONS)
+      end
+    rescue EncodingError
+      # Last resort for encodings without a converter to UTF-8: reinterpret the
+      # raw bytes as UTF-8 and drop anything invalid. Logging must never raise.
+      string.dup.force_encoding(Encoding::UTF_8).scrub("")
+    end
   end
 end

data/lib/semantic_logger/version.rb

@@ -1,3 +1,3 @@
 module SemanticLogger
-  VERSION = "4.17.0".freeze
+  VERSION = "5.0.0".freeze
 end

data/lib/semantic_logger.rb

@@ -12,12 +12,21 @@ require "semantic_logger/loggable"
 require "semantic_logger/concerns/compatibility"
 require "semantic_logger/appender"
 require "semantic_logger/appenders"
+require "semantic_logger/queue_processor"
 require "semantic_logger/processor"
 require "semantic_logger/sync_processor"
 require "semantic_logger/logger"
 require "semantic_logger/debug_as_trace_logger"
 require "semantic_logger/semantic_logger"
 
+# Automatically reopen appenders in the child process after a fork.
+# Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
+# Skipped on platforms without `Process._fork` (e.g. JRuby), which cannot fork.
+if Process.respond_to?(:_fork)
+  require "semantic_logger/core_ext/process"
+  Process.singleton_class.prepend(SemanticLogger::CoreExt::Process)
+end
+
 # Flush all appenders at exit, waiting for outstanding messages on the queue
 # to be written first.
 at_exit do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: semantic_logger
 version: !ruby/object:Gem::Version
-  version: 4.17.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Reid Morrison
@@ -23,6 +23,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+description: Semantic Logger is a high-performance, asynchronous structured logging
+  framework for Ruby & Rails. It logs to multiple destinations via a background thread,
+  preserving structured (semantic) payloads.
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -34,10 +37,10 @@ files:
 - lib/semantic_logger/ansi_colors.rb
 - lib/semantic_logger/appender.rb
 - lib/semantic_logger/appender/async.rb
-- lib/semantic_logger/appender/async_batch.rb
 - lib/semantic_logger/appender/bugsnag.rb
 - lib/semantic_logger/appender/cloudwatch_logs.rb
 - lib/semantic_logger/appender/elasticsearch.rb
+- lib/semantic_logger/appender/elasticsearch_base.rb
 - lib/semantic_logger/appender/elasticsearch_http.rb
 - lib/semantic_logger/appender/file.rb
 - lib/semantic_logger/appender/graylog.rb
@@ -51,6 +54,7 @@ files:
 - lib/semantic_logger/appender/new_relic.rb
 - lib/semantic_logger/appender/new_relic_logs.rb
 - lib/semantic_logger/appender/open_telemetry.rb
+- lib/semantic_logger/appender/opensearch.rb
 - lib/semantic_logger/appender/rabbitmq.rb
 - lib/semantic_logger/appender/sentry.rb
 - lib/semantic_logger/appender/sentry_ruby.rb
@@ -63,12 +67,14 @@ files:
 - lib/semantic_logger/appenders.rb
 - lib/semantic_logger/base.rb
 - lib/semantic_logger/concerns/compatibility.rb
+- lib/semantic_logger/core_ext/process.rb
 - lib/semantic_logger/core_ext/thread.rb
 - lib/semantic_logger/debug_as_trace_logger.rb
 - lib/semantic_logger/formatters.rb
 - lib/semantic_logger/formatters/base.rb
 - lib/semantic_logger/formatters/color.rb
 - lib/semantic_logger/formatters/default.rb
+- lib/semantic_logger/formatters/ecs.rb
 - lib/semantic_logger/formatters/fluentd.rb
 - lib/semantic_logger/formatters/json.rb
 - lib/semantic_logger/formatters/logfmt.rb
@@ -76,6 +82,7 @@ files:
 - lib/semantic_logger/formatters/new_relic_logs.rb
 - lib/semantic_logger/formatters/one_line.rb
 - lib/semantic_logger/formatters/open_telemetry.rb
+- lib/semantic_logger/formatters/pattern.rb
 - lib/semantic_logger/formatters/raw.rb
 - lib/semantic_logger/formatters/signalfx.rb
 - lib/semantic_logger/formatters/syslog.rb
@@ -89,6 +96,7 @@ files:
 - lib/semantic_logger/metric/signalfx.rb
 - lib/semantic_logger/metric/statsd.rb
 - lib/semantic_logger/processor.rb
+- lib/semantic_logger/queue_processor.rb
 - lib/semantic_logger/reporters/minitest.rb
 - lib/semantic_logger/semantic_logger.rb
 - lib/semantic_logger/subscriber.rb
@@ -96,6 +104,7 @@ files:
 - lib/semantic_logger/sync_processor.rb
 - lib/semantic_logger/test/capture_log_events.rb
 - lib/semantic_logger/test/minitest.rb
+- lib/semantic_logger/test/rspec.rb
 - lib/semantic_logger/utils.rb
 - lib/semantic_logger/version.rb
 homepage: https://logger.rocketjob.io
@@ -103,8 +112,9 @@ licenses:
 - Apache-2.0
 metadata:
   bug_tracker_uri: https://github.com/reidmorrison/semantic_logger/issues
+  changelog_uri: https://github.com/reidmorrison/semantic_logger/releases
   documentation_uri: https://logger.rocketjob.io
-  source_code_uri: https://github.com/reidmorrison/semantic_logger/tree/v4.17.0
+  source_code_uri: https://github.com/reidmorrison/semantic_logger/tree/v5.0.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -113,7 +123,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -122,6 +132,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.9
 specification_version: 4
-summary: Feature rich logging framework, and replacement for existing Ruby & Rails
-  loggers.
+summary: High-performance, asynchronous structured logging framework for Ruby & Rails.
 test_files: []

data/lib/semantic_logger/appender/async_batch.rb

@@ -1,93 +0,0 @@
-module SemanticLogger
-  module Appender
-    # Log asynchronously in batches using a separate thread.
-    #
-    # Log messages are grouped up and only logged when:
-    # * The number of queued messages is exceeded.
-    # * Or, the appropriate amount of time has passed since the last batch was sent.
-    class AsyncBatch < Async
-      attr_accessor :batch_size, :batch_seconds
-      attr_reader :signal
-
-      # Batching Appender proxy for appenders that support batches.
-      #
-      # Parameters:
-      #   batch_size: [Integer]
-      #     Maximum number of messages to batch up before sending.
-      #     Default: 300
-      #
-      #   batch_seconds: [Integer]
-      #     Maximum number of seconds between sending batches.
-      #     Default: 5
-      #
-      # See SemanticLogger::Appender::Async for other paramaters
-      #
-      # Note:
-      # * `lag_check_interval` is not applicable to batches, since the first message of every batch
-      #   is the oldest and is always checked to see if the lag interval has been exceeded.
-      def initialize(appender:,
-                     max_queue_size: 10_000,
-                     lag_threshold_s: 30,
-                     batch_size: 300,
-                     batch_seconds: 5)
-        @batch_size    = batch_size
-        @batch_seconds = batch_seconds
-        @signal        = Concurrent::Event.new
-        super(
-          appender:        appender,
-          max_queue_size:  max_queue_size,
-          lag_threshold_s: lag_threshold_s
-        )
-
-        return if appender.respond_to?(:batch)
-
-        raise(ArgumentError, "#{appender.class.name} does not support batching. It must implement #batch")
-      end
-
-      # Add log message for processing.
-      def log(log)
-        result = super
-        # Wake up the processing thread since the number of queued messages has been exceeded.
-        signal.set if queue.size >= batch_size
-        result
-      end
-
-      private
-
-      # Separate thread for batching up log messages before writing.
-      def process_messages
-        loop do
-          # Wait for batch interval or number of messages to be exceeded.
-          signal.wait(batch_seconds)
-
-          logs          = []
-          messages      = []
-          first         = true
-          message_count = queue.length
-          message_count.times do
-            # Queue#pop(true) raises an exception when there are no more messages, which is considered expensive.
-            message = queue.pop
-            if message.is_a?(Log)
-              logs << message
-              if first
-                check_lag(message)
-                first = false
-              end
-            else
-              messages << message
-            end
-          end
-          appender.batch(logs) if logs.size.positive?
-          messages.each { |message| process_message(message) }
-          signal.reset unless queue.size >= batch_size
-        end
-      end
-
-      def submit_request(command)
-        # Wake up the processing thread to process this command immediately.
-        signal.set
-        super
-      end
-    end
-  end
-end