semantic_logger 4.8.2 → 4.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0bd2e2a218bdbf0bfa67ca85da999b352de9b214383890278c0710cd3f84e91
-  data.tar.gz: dd3b393e8f0a031ddb25c2ca3cdeaa7d761617586f366006e79d7fcf3de8a9a5
+  metadata.gz: ef345374458ca40a1c31ab82b43fa040974c586fd089a3a5a2a5ab6fe0393c77
+  data.tar.gz: a7d13c0f42e7571b3ddcc47303f83a4b28a2b46929134be7d02e9c5af23d2963
 SHA512:
-  metadata.gz: dba969cf8d70cbb63a6542a692b68773f395eda22515267c572630aa5180a3bbfe490f2096e83cc366f06e86bbf06bafc87d2e5800810110107ac5080b7a1046
-  data.tar.gz: 35d69548ee2a91fb4d7c0575ea2d1da48648a7352d52304e8f270a6fe095c42e7ad1367594cef0b14941e14fc47906816df2360899c37fc577828b368c71c21a
+  metadata.gz: 0c4913a0f50f89e9b92b03b89c01b8a02f71f29efd6c06ae0c3f59b49c290b5878569cc6e5a946a7cc29943974d822a29481e6bd83d33c0de66f87f4b524e05c
+  data.tar.gz: ffe8ca5c2ce97b4bfe67bd79b4ae562c509c577cc70daca9128fe2c28cb7e92f1f6e567e4fb5199f89927653b6a1a3d32d53d6287e5f96ec935207534027746f

data/README.md

@@ -9,16 +9,6 @@ Semantic Logger is a feature rich logging framework, and replacement for existin
 
 [Semantic Logger Guide](https://logger.rocketjob.io/)
 
-## Upgrading to Semantic Logger v4.4
-
-With some forking frameworks it is necessary to call `reopen` after the fork. With v4.4 the
-workaround for Ruby 2.5 crashes is no longer needed. 
-I.e. Please remove the following line if being called anywhere:
-
-~~~ruby
-SemanticLogger::Processor.instance.instance_variable_set(:@queue, Queue.new)
-~~~
-
 ## Logging Destinations
 
 Logging to the following destinations are all supported "out-of-the-box":
@@ -32,7 +22,7 @@ Logging to the following destinations are all supported "out-of-the-box":
 * Splunk
 * MongoDB
 * Honeybadger
-* Sentry
+* Sentry (both with legacy `sentry-raven` and modern `sentry-ruby` gem)
 * HTTP
 * TCP
 * UDP
@@ -70,8 +60,50 @@ and are therefore not automatically included by this gem:
 - Splunk Appender: gem 'splunk-sdk-ruby'
 - Elasticsearch Appender: gem 'elasticsearch'
 - Kafka Appender: gem 'ruby-kafka'
+- Legacy Sentry Appender: gem 'sentry-raven' (deprecated)
+- Sentry Appender: gem 'sentry-ruby'
+
+## Upgrading to Semantic Logger v4.9
+
+These changes should not be noticeable by the majority of users of Semantic Logger, since
+they are to the internal API. It is possible that advanced users may be using these internal
+API's directly.
+
+This does not affect any calls to the public api `SemanticLogger.add_appender`.
+
+File and IO are now separate appenders. When creating the File appender explicitly, its arguments
+have changed. For example, when requesting an IO stream, it needs to be changed from:
+
+~~~ruby
+SemanticLogger::Appender::File.new(io: $stderr)
+~~~
+to:
+~~~ruby
+SemanticLogger::Appender::IO.new($stderr)
+~~~
+
+Additionally, this needs to be changed from:
+~~~ruby
+SemanticLogger::Appender::File.new(file_name: "file.log")
+~~~
+to:
+~~~ruby
+SemanticLogger::Appender::File.new("file.log")
+~~~
+
+Rails Semantic Logger, if used, needs to be upgraded to v4.9 when upgrading to Semantic Logger v4.9.
+
+## Upgrading to Semantic Logger v4.4
+
+With some forking frameworks it is necessary to call `reopen` after the fork. With v4.4 the
+workaround for Ruby 2.5 crashes is no longer needed.
+I.e. Please remove the following line if being called anywhere:
+
+~~~ruby
+SemanticLogger::Processor.instance.instance_variable_set(:@queue, Queue.new)
+~~~
 
-## V4 Upgrade notes
+## Upgrading to Semantic Logger v4.0
 
 The following changes need to be made when upgrading to V4:
 - Ruby V2.3 / JRuby V9.1 is now the minimum runtime version.

data/lib/semantic_logger/appender/async_batch.rb

@@ -64,6 +64,7 @@ module SemanticLogger
           signal.wait(batch_seconds)
 
           logs          = []
+          messages      = []
           first         = true
           message_count = queue.length
           message_count.times do
@@ -76,10 +77,11 @@ module SemanticLogger
                 first = false
               end
             else
-              process_message(message)
+              messages << message
             end
           end
           appender.batch(logs) if logs.size.positive?
+          messages.each { |message| process_message(message) }
           signal.reset unless queue.size >= batch_size
         end
       end

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -135,7 +135,7 @@ module SemanticLogger
                      application: nil,
                      environment: nil,
                      host: nil,
-                     metrics: false,
+                     data_stream: false,
                      **elasticsearch_args,
                      &block)
 
@@ -146,6 +146,7 @@ module SemanticLogger
         @elasticsearch_args          = elasticsearch_args.dup
         @elasticsearch_args[:url]    = url if url && !elasticsearch_args[:hosts]
         @elasticsearch_args[:logger] = logger
+        @data_stream = data_stream
 
         super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: false, &block)
         reopen
@@ -175,7 +176,12 @@ module SemanticLogger
       private
 
       def write_to_elasticsearch(messages)
-        bulk_result = @client.bulk(body: messages)
+        bulk_result = if @data_stream
+                        @client.bulk(index: index, body: messages)
+                      else
+                        @client.bulk(body: messages)
+                      end
+
         return unless bulk_result["errors"]
 
         failed = bulk_result["items"].reject { |x| x["status"] == 201 }
@@ -184,11 +190,21 @@ module SemanticLogger
 
       def bulk_index(log)
         expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
-        {"index" => {"_index" => expanded_index_name, "_type" => type}}
+        if @data_stream
+          {"create" => {}}
+        else
+          {"index" => {"_index" => expanded_index_name, "_type" => type}}
+        end
       end
 
       def default_formatter
-        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: :timestamp)
+        time_key = if @data_stream
+                     "@timestamp"
+                   else
+                     :timestamp
+                   end
+
+        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
       end
     end
   end

data/lib/semantic_logger/appender/file.rb

@@ -1,3 +1,4 @@
+require "date"
 # File appender
 #
 #   Writes log messages to a file or open iostream
@@ -5,114 +6,285 @@
 module SemanticLogger
   module Appender
     class File < SemanticLogger::Subscriber
-      # Create a File Logger appender instance.
+      attr_accessor :file_name, :retry_count, :append, :exclusive_lock, :encoding,
+                    :reopen_period, :reopen_count, :reopen_size
+      attr_reader :log_count, :log_size, :current_file_name, :reopen_at
+
+      # Create an appender to log to a named file.
       #
       # Parameters
-      #  :file_name [String]
-      #    Name of file to write to.
-      #  Or,
-      #  :io [IO]
-      #    An IO stream to which to write the log messages to.
-      #
-      #  :level [:trace | :debug | :info | :warn | :error | :fatal]
-      #    Override the log level for this appender.
-      #    Default: SemanticLogger.default_level
-      #
-      #  :formatter: [Object|Proc]
-      #    An instance of a class that implements #call, or a Proc to be used to format
-      #    the output from this appender
-      #    Default: Use the built-in formatter (See: #call)
-      #
-      #  :filter [Regexp|Proc]
-      #    RegExp: Only include log messages where the class name matches the supplied
-      #    regular expression. All other messages will be ignored.
-      #    Proc: Only include log messages where the supplied Proc returns true
-      #          The Proc must return true or false.
+      #   file_name [String]
+      #     Name of the file to write to.
       #
-      # Example
-      #    require 'semantic_logger'
+      #     File name format directives:
+      #       %p - Process Id
+      #       %n - Short hostname (SemanticLogger.host). Everything before the first period in the hostname.
+      #       %N - Full hostname (SemanticLogger.host)
+      #       %a - Application name (SemanticLogger.application)
+      #       %e - Environment name (SemanticLogger.environment)
+      #       %D - Current Date. Equivalent to "%Y%m%d"
+      #       %T - Current Time. Equivalent to "%H%M%S"
+      #       %% - Literal `%` character
       #
-      #    # Enable trace level logging
-      #    SemanticLogger.default_level = :info
+      #     Date:
+      #       %Y - Year with century
+      #       %C - year / 100 (round down.  20 in 2009)
+      #       %y - year % 100 (00..99)
+      #       %m - Month of the year, zero-padded (01..12)
+      #       %d - Day of the month, zero-padded (01..31)
+      #       %j - Day of the year (001..366)
+      #       %U - Week number of the year.  The week starts with Sunday.  (00..53)
+      #       %W - Week number of the year.  The week starts with Monday.  (00..53)
       #
-      #    # Log to screen
-      #    SemanticLogger.add_appender(io: $stdout, formatter: :color)
+      #     Time:
+      #       %H - 24 Hour of the day, zero-padded (00..23)
+      #       %M - Minute of the hour (00..59)
+      #       %S - Second of the minute (00..60)
       #
-      #    # And log to a file at the same time
-      #    SemanticLogger.add_appender(file_name: 'application.log', formatter: :color)
+      #     Examples:
+      #       Create a log file name consisting of the short host name, process id, date, and time.
+      #         "log/production-%n-%p-%D-%T.log"
       #
-      #    logger = SemanticLogger['test']
-      #    logger.info 'Hello World'
+      #   :level [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
       #
-      # Example 2. To log all levels to file and only :info and above to screen:
+      #   :formatter: [Object|Proc]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: Use the built-in formatter (See: #call)
       #
-      #    require 'semantic_logger'
+      #   :filter [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
       #
-      #    # Enable trace level logging
-      #    SemanticLogger.default_level = :trace
+      #   :append [true|false]
+      #     Append to the log file if already present?
+      #     Default: true
       #
-      #    # Log to screen but only display :info and above
-      #    SemanticLogger.add_appender(io: $stdout, level: :info)
+      #   :exclusive_lock [true|false]
+      #     Obtain an exclusive lock on the file, for operating systems that support it.
+      #     Prevents multiple processes from trying to write to the same log file.
+      #     Default: false
       #
-      #    # And log to a file at the same time, including all :trace level data
-      #    SemanticLogger.add_appender(file_name: 'application.log')
+      #   :encoding ["UTF-8", "UTF-16", etc.]
+      #     Encoding to use when writing to the file.
+      #     Default: Encoding::BINARY
       #
-      #    logger =  SemanticLogger['test']
-      #    logger.info 'Hello World'
-      def initialize(io: nil, file_name: nil, **args, &block)
-        if io
-          @log = io
-          unless @log.respond_to?(:write)
-            raise(ArgumentError, "SemanticLogging::Appender::File :io is not a valid IO instance: #{io.inspect}")
-          end
-        else
-          @file_name = file_name
-          unless file_name
-            raise(ArgumentError, "SemanticLogging::Appender::File missing mandatory parameter :file_name or :io")
-          end
-
-          reopen
+      #   :retry_count [Integer]
+      #     Number of times to attempt to re-open the file name when an error occurs trying to
+      #     write to the file.
+      #     Note: Set to 0 to disable retries.
+      #     Default: 1
+      #
+      #   :reopen_period [String]
+      #     Specify a period after which to re-open the log file, specified in minutes, hours, or days.
+      #     The format of the duration must start with an Integer or Float number,
+      #     followed by the duration specified as:
+      #       "m" : minutes
+      #       "h" : hours
+      #       "d" : days
+      #     The time is rounded down to the specified time interval, so that:
+      #     - "1h" will re-open every hour at the beginning of the hour.
+      #     - "30m" will re-open every 30 minutes at the beginning of the 30th minute.
+      #     - "1d" will re-open every day at midnight.
+      #     Examples:
+      #       "60m" : Every 60 minutes at the beginning of the minute: 10:24:00, 11:24:00, 12:24:00, ...
+      #       "1h"  : Every hour at the beginning of the hour: 10:00:00, 11:00:00, 12:00:00, ...
+      #       "1d"  : Every day at the beginning of the day: "20211008 00:00:00", "20211009 00:00:00", ...
+      #     Default: nil (Disabled)
+      #
+      #   :reopen_count [Integer]
+      #     Close and re-open the log file after every `reopen_count` number of logged entries.
+      #     Default: 0 (Disabled)
+      #
+      #   :reopen_size [Integer]
+      #     Approximate number of bytes to write to a log file by this process before closing and re-opening it.
+      #     Notes:
+      #     - When `append: true` and the file already exists, it reads the size of the current log file
+      #       and starts with that size.
+      #       - If the current log file size already exceeds the `reopen_size`, its current size is ignored.
+      #     - The `reopen_size` is only the amount of bytes written by this process, it excludes data
+      #       written by other processes. Use a unique filename to prevent multiple processes from writing to
+      #       the same log file at the same time.
+      #     Default: 0 (Disabled)
+      #
+      # Example
+      #    require "semantic_logger"
+      #
+      #    # Enable trace level logging
+      #    SemanticLogger.default_level = :info
+      #
+      #    # Log to a file
+      #    SemanticLogger.add_appender(file_name: "application.log", formatter: :color)
+      #
+      #    logger = SemanticLogger["test"]
+      #    logger.info "Hello World"
+      def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0, encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
+        if !file_name.is_a?(String) || file_name.empty?
+          raise(ArgumentError, "SemanticLogging::Appender::File file_name must be a non-empty string")
         end
 
+        @file_name      = file_name
+        @retry_count    = retry_count
+        @file           = nil
+        @append         = append
+        @reopen_period  = reopen_period
+        @reopen_count   = reopen_count
+        @reopen_size    = reopen_size
+        @encoding       = encoding
+        @exclusive_lock = exclusive_lock
+        @log_count      = 0
+        @log_size       = 0
+        @reopen_at      = nil
+
         super(**args, &block)
       end
 
       # After forking an active process call #reopen to re-open
-      # open the file handles etc to resources
-      #
-      # Note: This method will only work if :file_name was supplied
-      #       on the initializer.
-      #       If :io was supplied, it will need to be re-opened manually.
+      # open the file handles etc to resources.
       def reopen
-        return unless @file_name
+        begin
+          @file&.close
+        rescue StandardError
+          nil
+        end
+
+        self.current_file_name = apply_format_directives(file_name)
+        if ::File.directory?(file_name)
+          raise(ArgumentError, "The supplied log file_name: #{current_file_name} is already a directory.")
+        end
+
+        self.log_count = 0
+        if append && reopen_size && ::File.exist?(current_file_name)
+          self.log_size = ::File.size(current_file_name)
+          self.log_size = 0 if log_size >= reopen_size
+        else
+          self.log_size = 0
+        end
 
-        @log = ::File.open(@file_name, ::File::WRONLY | ::File::APPEND | ::File::CREAT)
+        self.reopen_at = reopen_period ? next_reopen_period(reopen_period) : nil
+
+        options = ::File::WRONLY | ::File::CREAT
+        options |= ::File::APPEND if append
+        @file = ::File.open(current_file_name, options)
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
-        @log.sync = true
-        @log.set_encoding(Encoding::BINARY) if @log.respond_to?(:set_encoding)
-        @log
+        @file.sync = true
+        @file.set_encoding(encoding) if @file.respond_to?(:set_encoding)
+        @file.flock(::File::LOCK_EX) if exclusive_lock
+        @file
       end
 
-      # Pass log calls to the underlying Rails, log4j or Ruby logger
-      #  trace entries are mapped to debug since :trace is not supported by the
-      #  Ruby or Rails Loggers
+      # Since only one appender thread will be writing to the file at a time
+      # it is not necessary to protect access to the file with a semaphore.
       def log(log)
-        # Since only one appender thread will be writing to the file at a time
-        # it is not necessary to protect access to the file with a semaphore
-        # Allow this logger to filter out log levels lower than it's own
-        @log.write(formatter.call(log, self) << "\n")
+        reopen if time_to_reopen?
+
+        count = 0
+        begin
+          message = formatter.call(log, self) << "\n"
+          @file.write(message)
+          @log_count += 1
+          @log_size  += message.size
+        rescue StandardError => e
+          if count < retry_count
+            count += 1
+            reopen
+            retry
+          end
+          raise(e)
+        end
         true
       end
 
       # Flush all pending logs to disk.
-      #  Waits for all sent documents to be writted to disk
+      #  Waits for all sent documents to be written to disk
       def flush
-        @log.flush if @log.respond_to?(:flush)
+        @file&.flush
+      end
+
+      private
+
+      attr_writer :log_count, :log_size, :current_file_name, :reopen_at
+
+      def time_to_reopen?
+        return true unless @file
+
+        (reopen_count.positive? && (log_count >= reopen_count)) ||
+          (reopen_size.positive? && (log_size >= reopen_size)) ||
+          (reopen_at && (Time.now > reopen_at))
+      end
+
+      def apply_format_directives(file_name)
+        return file_name unless file_name.include?("%")
+
+        file_name.gsub(/%(.)/) { format_directive(Regexp.last_match(1)) }
+      end
+
+      def format_directive(directive)
+        case directive
+        when "p"
+          $$
+        when "n"
+          SemanticLogger.host.split(".")[0]
+        when "N"
+          SemanticLogger.host
+        when "a"
+          SemanticLogger.application
+        when "e"
+          SemanticLogger.environment
+        when "D"
+          Date.today.strftime("%Y%m%d")
+        when "Y", "C", "y", "m", "d", "j", "U", "W"
+          Date.today.strftime("%#{directive}")
+        when "T"
+          Time.now.strftime("%H%M%S")
+        when "H", "M", "S"
+          Time.now.strftime("%#{directive}")
+        when "%"
+          "%"
+        else
+          raise(ArgumentError, "Format Directive '#{directive}' in file_name: #{file_name} is not supported.")
+        end
       end
 
-      def console_output?
-        [$stderr, $stdout].include?(@log)
+      def next_reopen_period(period_string)
+        return unless period_string
+
+        duration, period = parse_period(period_string)
+        calculate_reopen_at(duration, period)
+      end
+
+      def parse_period(period_string)
+        match = period_string.to_s.downcase.gsub(/\s+/, "").match(/([\d.]+)([mhd])/)
+        unless match
+          raise(ArgumentError,
+                "Invalid period definition: #{period_string}, must begin with an integer, followed by m,h, or d.")
+        end
+
+        duration = match[1]
+        period   = match[2]
+        raise(ArgumentError, "Invalid or missing duration in: #{period_string}, must begin with an integer.") unless duration
+        raise(ArgumentError, "Invalid or missing period in: #{period_string}, must end with m,h, or d.") unless period
+
+        [duration.to_i, period]
+      end
+
+      # Round down the current time based on the period, then add on the duration for that period
+      def calculate_reopen_at(duration, period, time = Time.now)
+        case period
+        when "m"
+          Time.new(time.year, time.month, time.day, time.hour, time.min, 0) + (duration * 60)
+        when "h"
+          Time.new(time.year, time.month, time.day, time.hour, 0, 0) + (duration * 60 * 60)
+        when "d"
+          Time.new(time.year, time.month, time.day, 0, 0, 0) + (duration * 24 * 60 * 60)
+        else
+          raise(ArgumentError, "Invalid or missing period in: #{reopen_period}, must end with m,h, or d.")
+        end
       end
     end
   end

data/lib/semantic_logger/appender/io.rb

@@ -0,0 +1,68 @@
+# File appender
+#
+#   Writes log messages to a file or open iostream
+#
+module SemanticLogger
+  module Appender
+    class IO < SemanticLogger::Subscriber
+      # Create a Stream Logger appender instance.
+      #
+      # Parameters
+      #  io [IO]
+      #    An IO stream to which to write the log messages to.
+      #
+      #  :level [:trace | :debug | :info | :warn | :error | :fatal]
+      #    Override the log level for this appender.
+      #    Default: SemanticLogger.default_level
+      #
+      #  :formatter: [Object|Proc]
+      #    An instance of a class that implements #call, or a Proc to be used to format
+      #    the output from this appender
+      #    Default: Use the built-in formatter (See: #call)
+      #
+      #  :filter [Regexp|Proc]
+      #    RegExp: Only include log messages where the class name matches the supplied
+      #    regular expression. All other messages will be ignored.
+      #    Proc: Only include log messages where the supplied Proc returns true
+      #          The Proc must return true or false.
+      #
+      # Example
+      #    require "semantic_logger"
+      #
+      #    # Enable trace level logging
+      #    SemanticLogger.default_level = :info
+      #
+      #    # Log to screen
+      #    SemanticLogger.add_appender(io: $stdout, formatter: :color)
+      #
+      #    logger = SemanticLogger['test']
+      #    logger.info 'Hello World'
+      def initialize(io, **args, &block)
+        @io = io
+        unless @io.respond_to?(:write)
+          raise(ArgumentError, "SemanticLogging::Appender::IO io is not a valid IO instance: #{io.inspect}")
+        end
+
+        super(**args, &block)
+      end
+
+      def log(log)
+        # Since only one appender thread will be writing to the file at a time
+        # it is not necessary to protect access to the file with a semaphore
+        # Allow this logger to filter out log levels lower than it's own
+        @io.write(formatter.call(log, self) << "\n")
+        true
+      end
+
+      # Flush all pending logs to disk.
+      #  Waits for all sent documents to be written to disk
+      def flush
+        @io.flush if @io.respond_to?(:flush)
+      end
+
+      def console_output?
+        [$stderr, $stdout].include?(@io)
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/kafka.rb

@@ -23,7 +23,7 @@ module SemanticLogger
   module Appender
     class Kafka < SemanticLogger::Subscriber
       attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
-                    :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key,
+                    :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
                     :delivery_threshold, :delivery_interval,
                     :topic, :partition, :partition_key, :key
 
@@ -79,6 +79,9 @@ module SemanticLogger
       #     Must be used in combination with ssl_client_cert.
       #     Default: nil
       #
+      #   ssl_ca_certs_from_system: [boolean]
+      #     Delegate SSL CA cert to the system certs
+      #
       #    delivery_threshold: [Integer]
       #      Number of messages between triggering a delivery of messages to Apache Kafka.
       #      Default: 100
@@ -116,24 +119,25 @@ module SemanticLogger
       #     Send metrics only events to kafka.
       #     Default: true
       def initialize(seed_brokers:, client_id: "semantic-logger", connect_timeout: nil, socket_timeout: nil,
-                     ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
+                     ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
                      topic: "log_messages", partition: nil, partition_key: nil, key: nil,
                      delivery_threshold: 100, delivery_interval: 10,
                      metrics: true, **args, &block)
 
-        @seed_brokers        = seed_brokers
-        @client_id           = client_id
-        @connect_timeout     = connect_timeout
-        @socket_timeout      = socket_timeout
-        @ssl_ca_cert         = ssl_ca_cert
-        @ssl_client_cert     = ssl_client_cert
-        @ssl_client_cert_key = ssl_client_cert_key
-        @topic               = topic
-        @partition           = partition
-        @partition_key       = partition_key
-        @key                 = key
-        @delivery_threshold  = delivery_threshold
-        @delivery_interval   = delivery_interval
+        @seed_brokers             = seed_brokers
+        @client_id                = client_id
+        @connect_timeout          = connect_timeout
+        @socket_timeout           = socket_timeout
+        @ssl_ca_cert              = ssl_ca_cert
+        @ssl_client_cert          = ssl_client_cert
+        @ssl_client_cert_key      = ssl_client_cert_key
+        @ssl_ca_certs_from_system = ssl_ca_certs_from_system
+        @topic                    = topic
+        @partition                = partition
+        @partition_key            = partition_key
+        @key                      = key
+        @delivery_threshold       = delivery_threshold
+        @delivery_interval        = delivery_interval
 
         super(metrics: metrics, **args, &block)
         reopen
@@ -141,14 +145,15 @@ module SemanticLogger
 
       def reopen
         @kafka = ::Kafka.new(
-          seed_brokers:        seed_brokers,
-          client_id:           client_id,
-          connect_timeout:     connect_timeout,
-          socket_timeout:      socket_timeout,
-          ssl_ca_cert:         ssl_ca_cert,
-          ssl_client_cert:     ssl_client_cert,
-          ssl_client_cert_key: ssl_client_cert_key,
-          logger:              logger
+          seed_brokers:             seed_brokers,
+          client_id:                client_id,
+          connect_timeout:          connect_timeout,
+          socket_timeout:           socket_timeout,
+          ssl_ca_cert:              ssl_ca_cert,
+          ssl_client_cert:          ssl_client_cert,
+          ssl_client_cert_key:      ssl_client_cert_key,
+          ssl_ca_certs_from_system: ssl_ca_certs_from_system,
+          logger:                   logger
         )
 
         @producer = @kafka.async_producer(

data/lib/semantic_logger/appender/sentry_ruby.rb

@@ -0,0 +1,138 @@
+begin
+  require "sentry-ruby"
+rescue LoadError
+  raise LoadError, 'Gem sentry-ruby is required for logging purposes. Please add the gem "sentry-ruby" to your Gemfile.'
+end
+
+# Send log messages to sentry
+#
+# Example:
+#   SemanticLogger.add_appender(appender: :sentry_ruby)
+#
+module SemanticLogger
+  module Appender
+    class SentryRuby < SemanticLogger::Subscriber
+      # Create Appender
+      #
+      # Parameters
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: :error
+      #
+      #   formatter: [Object|Proc|Symbol|Hash]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: Use the built-in formatter (See: #call)
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      #
+      #   host: [String]
+      #     Name of this host to appear in log messages.
+      #     Default: SemanticLogger.host
+      #
+      #   application: [String]
+      #     Name of this application to appear in log messages.
+      #     Default: SemanticLogger.application
+      def initialize(level: :error, **args, &block)
+        # Replace the Sentry Ruby logger so that we can identify its log
+        # messages and not forward them to Sentry
+        ::Sentry.init { |config| config.logger = SemanticLogger[::Sentry] }
+        super(level: level, **args, &block)
+      end
+
+      # Send an error notification to sentry
+      def log(log)
+        # Ignore logs coming from Sentry itself
+        return false if log.name == "Sentry"
+
+        context = formatter.call(log, self)
+        payload = context.delete(:payload) || {}
+        named_tags = context[:named_tags] || {}
+        transaction_name = named_tags.delete(:transaction_name)
+
+        user = extract_user!(named_tags, payload)
+        tags = extract_tags!(context)
+
+        fingerprint = payload.delete(:fingerprint)
+
+        ::Sentry.with_scope do |scope|
+          scope.set_user(user) if user
+          scope.set_level(context.delete(:level)) if context[:level]
+          scope.set_fingerprint(fingerprint) if fingerprint
+          scope.set_transaction_name(transaction_name) if transaction_name
+          scope.set_tags(tags)
+          scope.set_extras(context)
+          scope.set_extras(payload)
+
+          if log.exception
+            ::Sentry.capture_exception(log.exception)
+          elsif log.backtrace
+            ::Sentry.capture_message(context[:message], backtrace: log.backtrace)
+          else
+            ::Sentry.capture_message(context[:message])
+          end
+        end
+
+        true
+      end
+
+      private
+
+      # Use Raw Formatter by default
+      def default_formatter
+        SemanticLogger::Formatters::Raw.new
+      end
+
+      # Extract user data from named tags or payload.
+      #
+      # Keys :user_id and :user_email will be used as :id and :email respectively.
+      # Keys :username and :ip_address will be used verbatim.
+      #
+      # Any additional value nested in a :user key will be added, provided any of
+      # the above keys is already present.
+      #
+      def extract_user!(*sources)
+        keys = {user_id: :id, username: :username, user_email: :email, ip_address: :ip_address}
+
+        user = {}
+
+        sources.each do |source|
+          keys.each do |source_key, target_key|
+            value = source.delete(source_key)
+            user[target_key] = value if value
+          end
+        end
+
+        return if user.empty?
+
+        sources.each do |source|
+          extras = source.delete(:user)
+          user.merge!(extras) if extras.is_a?(Hash)
+        end
+
+        user
+      end
+
+      # Extract tags.
+      #
+      # Named tags will be stringified (both key and value).
+      # Unnamed tags will be stringified and joined with a comma. Then they will
+      # be used as a "tag" named tag. If such a tag already exists, it is also
+      # joined with a comma.
+      #
+      # Finally, the tag names are limited to 32 characters and the tag values to 256.
+      #
+      def extract_tags!(context)
+        named_tags = context.delete(:named_tags) || {}
+        named_tags = named_tags.map { |k, v| [k.to_s, v.to_s] }.to_h
+        tags = context.delete(:tags)
+        named_tags.merge!("tag" => tags.join(", ")) { |_, v1, v2| "#{v1}, #{v2}" } if tags
+        named_tags.map { |k, v| [k[0...32], v[0...256]] }.to_h
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender.rb

@@ -9,6 +9,7 @@ module SemanticLogger
     autoload :File,              "semantic_logger/appender/file"
     autoload :Graylog,           "semantic_logger/appender/graylog"
     autoload :Honeybadger,       "semantic_logger/appender/honeybadger"
+    autoload :IO,                "semantic_logger/appender/io"
     autoload :Kafka,             "semantic_logger/appender/kafka"
     autoload :Sentry,            "semantic_logger/appender/sentry"
     autoload :Http,              "semantic_logger/appender/http"
@@ -21,6 +22,7 @@ module SemanticLogger
     autoload :Tcp,               "semantic_logger/appender/tcp"
     autoload :Udp,               "semantic_logger/appender/udp"
     autoload :Wrapper,           "semantic_logger/appender/wrapper"
+    autoload :SentryRuby,        "semantic_logger/appender/sentry_ruby"
     # @formatter:on
 
     # Returns [SemanticLogger::Subscriber] appender for the supplied options
@@ -32,7 +34,7 @@ module SemanticLogger
       appender = build(**args, &block)
 
       # If appender implements #batch, then it should use the batch proxy by default.
-      batch = true if batch.nil? && appender.respond_to?(:batch)
+      batch    = true if batch.nil? && appender.respond_to?(:batch)
 
       if batch == true
         Appender::AsyncBatch.new(
@@ -56,8 +58,10 @@ module SemanticLogger
 
     # Returns [Subscriber] instance from the supplied options.
     def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &block)
-      if io || file_name
-        SemanticLogger::Appender::File.new(io: io, file_name: file_name, **args, &block)
+      if file_name
+        SemanticLogger::Appender::File.new(file_name, **args, &block)
+      elsif io
+        SemanticLogger::Appender::IO.new(io, **args, &block)
       elsif logger
         SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &block)
       elsif appender

data/lib/semantic_logger/base.rb

@@ -186,9 +186,11 @@ module SemanticLogger
     #   to:
     #     `logger.tagged('first', 'more', 'other')`
     # - For better performance with clean tags, see `SemanticLogger.tagged`.
-    def tagged(*tags, &block)
+    def tagged(*tags)
+      block = -> { yield(self) }
       # Allow named tags to be passed into the logger
-      if tags.size == 1
+      # Rails::Rack::Logger passes logs as an array with a single argument
+      if tags.size == 1 && !tags.first.is_a?(Array)
         tag = tags[0]
         return yield if tag.nil? || tag == ""
 
@@ -326,14 +328,14 @@ module SemanticLogger
 
       log = Log.new(name, level, index)
       should_log =
-        if payload.nil? && exception.nil? && message.is_a?(Hash)
-          # Everything as keyword arguments.
+        if exception.nil? && payload.nil? && message.is_a?(Hash)
+          # All arguments as a hash in the message.
           log.assign(**log.extract_arguments(message))
         elsif exception.nil? && message && payload && payload.is_a?(Hash)
-          # Message with keyword arguments as the rest.
-          log.assign(message: message, **log.extract_arguments(payload))
+          # Message supplied along with a hash with the remaining arguments.
+          log.assign(**log.extract_arguments(payload, message))
         else
-          # No keyword arguments.
+          # All fields supplied directly.
           log.assign(message: message, payload: payload, exception: exception)
         end
 

data/lib/semantic_logger/formatters/base.rb

@@ -70,6 +70,7 @@ module SemanticLogger
 
       # Return the Time as a formatted string
       def format_time(time)
+        time = time.dup
         case time_format
         when :rfc_3339
           time.utc.to_datetime.rfc3339

data/lib/semantic_logger/formatters/logfmt.rb

@@ -2,6 +2,21 @@ require "json"
 
 module SemanticLogger
   module Formatters
+    # Produces logfmt formatted messages
+    #
+    # The following fields are extracted from the raw log and included in the formatted message:
+    #   :timestamp, :level, :name, :message, :duration, :tags, :named_tags
+    #
+    # E.g.
+    #   timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast" spaces="second breakfast" double_quotes="\"elevensies\"" single_quotes="'lunch'" tag="success"
+    #
+    # All timestamps are ISO8601 formatteed
+    # All user supplied values are escaped and surrounded by double quotes to avoid ambiguious message delimeters
+    # `tags` are treated as keys with boolean values. Tag names are not formatted or validated, ensure you use valid logfmt format for tag names.
+    # `named_tags` are flattened are merged into the top level message field. Any conflicting fields are overridden.
+    # `payload` values take precedence over `tags` and `named_tags`. Any conflicting fields are overridden.
+    #
+    # Futher Reading https://brandur.org/logfmt
     class Logfmt < Raw
       def initialize(time_format: :iso_8601, time_key: :timestamp, **args)
         super(time_format: time_format, time_key: time_key, **args)
@@ -16,13 +31,22 @@ module SemanticLogger
       private
 
       def raw_to_logfmt
-        @parsed = @raw.slice(:timestamp, :level, :name, :message, :duration).merge tag: "success"
+        @parsed = @raw.slice(time_key, :level, :name, :message, :duration).merge(tag: "success")
+        handle_tags
         handle_payload
         handle_exception
 
         flatten_log
       end
 
+      def handle_tags
+        tags = @raw.fetch(:tags){ [] }
+                    .each_with_object({}){ |tag, accum| accum[tag] = true }
+
+        @parsed = @parsed.merge(tags)
+                         .merge(@raw.fetch(:named_tags){ {} })
+      end
+
       def handle_payload
         return unless @raw.key? :payload
 
@@ -38,17 +62,11 @@ module SemanticLogger
 
       def flatten_log
         flattened = @parsed.map do |key, value|
-          "#{key}=#{parse_value(value)}"
+          "#{key}=#{value.to_json}"
         end
 
         flattened.join(" ")
       end
-
-      def parse_value(value)
-        return value.to_json if value.instance_of? String
-
-        value
-      end
     end
   end
 end

data/lib/semantic_logger/log.rb

@@ -140,14 +140,25 @@ module SemanticLogger
     end
 
     # Extract the arguments from a Hash Payload
-    def extract_arguments(payload)
+    def extract_arguments(payload, message = nil)
       raise(ArgumentError, "payload must be a Hash") unless payload.is_a?(Hash)
 
+      message = nil if message == ""
       return payload if payload.key?(:payload)
 
-      args = {}
-      payload.each_key { |key| args[key] = payload.delete(key) if NON_PAYLOAD_KEYS.include?(key) }
-      args[:payload] = payload unless payload.empty?
+      new_payload = {}
+      args        = {}
+      payload.each_pair do |key, value|
+        # Supplied message takes precedence
+        if (key == :message) && !message.nil?
+          new_payload[key] = value
+          next
+        end
+
+        NON_PAYLOAD_KEYS.include?(key) ? args[key] = value : new_payload[key] = value
+      end
+      args[:payload] = new_payload unless new_payload.empty?
+      args[:message] = message if message
       args
     end
 

data/lib/semantic_logger/processor.rb

@@ -15,7 +15,7 @@ module SemanticLogger
     def self.logger
       @logger ||=
         begin
-          l      = SemanticLogger::Appender::File.new(io: $stderr, level: :warn)
+          l      = SemanticLogger::Appender::IO.new($stderr, level: :warn)
           l.name = name
           l
         end

data/lib/semantic_logger/semantic_logger.rb

@@ -122,7 +122,7 @@ module SemanticLogger
   #     Default: SemanticLogger.default_level
   #
   #   formatter: [Symbol|Object|Proc]
-  #     Any of the following symbol values: :default, :color, :json
+  #     Any of the following symbol values: :default, :color, :json, :logfmt, etc...
   #       Or,
   #     An instance of a class that implements #call
   #       Or,
@@ -164,7 +164,7 @@ module SemanticLogger
   #   logger.info "Hello World"
   #   logger.debug("Login time", user: 'Joe', duration: 100, ip_address: '127.0.0.1')
   def self.add_appender(**args, &block)
-    appender = Logger.processor.appenders.add(**args, &block)
+    appender = appenders.add(**args, &block)
     # Start appender thread if it is not already running
     Logger.processor.start
     appender
@@ -175,7 +175,7 @@ module SemanticLogger
   def self.remove_appender(appender)
     return unless appender
 
-    Logger.processor.appenders.delete(appender)
+    appenders.delete(appender)
     appender.close
   end
 
@@ -189,7 +189,7 @@ module SemanticLogger
   # Use SemanticLogger.add_appender and SemanticLogger.remove_appender
   # to manipulate the active appenders list
   def self.appenders
-    Logger.processor.appenders.to_a
+    Logger.processor.appenders
   end
 
   # Flush all queued log entries disk, database, etc.

data/lib/semantic_logger/subscriber.rb

@@ -22,6 +22,11 @@ module SemanticLogger
       # NOOP
     end
 
+    # Method called to log an event
+    def log(log)
+      raise NotImplementedError
+    end
+
     # Returns [SemanticLogger::Formatters::Default] default formatter for this subscriber.
     def default_formatter
       SemanticLogger::Formatters::Default.new
@@ -68,6 +73,11 @@ module SemanticLogger
       super(log) && (log.metric_only? ? metrics? : true)
     end
 
+    # Whether this appender is logging to stdout or stderror
+    def console_output?
+      false
+    end
+
     private
 
     # Initializer for Abstract Class SemanticLogger::Subscriber

data/lib/semantic_logger/sync_processor.rb

@@ -24,7 +24,7 @@ module SemanticLogger
     def self.logger
       @logger ||=
         begin
-          l      = SemanticLogger::Appender::File.new(io: $stderr, level: :warn)
+          l      = SemanticLogger::Appender::IO.new($stderr, level: :warn)
           l.name = name
           l
         end

data/lib/semantic_logger/test/capture_log_events.rb

@@ -0,0 +1,34 @@
+module SemanticLogger
+  module Test
+    # Logging class to captures all logging events in memory.
+    #
+    # Example:
+    #
+    # class UserTest < ActiveSupport::TestCase
+    #   describe User do
+    #     let(:capture_logger) { SemanticLogger::Test::CaptureLogEvents.new }
+    #     let(:user) { User.new }
+    #
+    #     it "logs message" do
+    #       user.stub(:logger, capture_logger) do
+    #         user.enable!
+    #       end
+    #       assert_equal "Hello World", capture_logger.events.last.message
+    #       assert_equal :info, capture_logger.events.last.level
+    #     end
+    #   end
+    # end
+    class CaptureLogEvents < SemanticLogger::Subscriber
+      attr_accessor :events
+
+      # By default collect all log levels, and collect metric only log events.
+      def initialize(level: :trace, metrics: true)
+        super(level: level, metrics: true)
+      end
+
+      def log(log)
+        (@events ||= []) << log
+      end
+    end
+  end
+end

data/lib/semantic_logger/version.rb

@@ -1,3 +1,3 @@
 module SemanticLogger
-  VERSION = "4.8.2".freeze
+  VERSION = "4.11.0".freeze
 end

data/lib/semantic_logger.rb

@@ -32,6 +32,10 @@ module SemanticLogger
     autoload :Minitest,          "semantic_logger/reporters/minitest"
   end
 
+  module Test
+    autoload :CaptureLogEvents,  "semantic_logger/test/capture_log_events"
+  end
+
   if defined?(JRuby)
     module JRuby
       autoload :GarbageCollectionLogger, "semantic_logger/jruby/garbage_collection_logger"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: semantic_logger
 version: !ruby/object:Gem::Version
-  version: 4.8.2
+  version: 4.11.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2022-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -45,11 +45,13 @@ files:
 - lib/semantic_logger/appender/graylog.rb
 - lib/semantic_logger/appender/honeybadger.rb
 - lib/semantic_logger/appender/http.rb
+- lib/semantic_logger/appender/io.rb
 - lib/semantic_logger/appender/kafka.rb
 - lib/semantic_logger/appender/mongodb.rb
 - lib/semantic_logger/appender/new_relic.rb
 - lib/semantic_logger/appender/rabbitmq.rb
 - lib/semantic_logger/appender/sentry.rb
+- lib/semantic_logger/appender/sentry_ruby.rb
 - lib/semantic_logger/appender/splunk.rb
 - lib/semantic_logger/appender/splunk_http.rb
 - lib/semantic_logger/appender/syslog.rb
@@ -87,6 +89,7 @@ files:
 - lib/semantic_logger/subscriber.rb
 - lib/semantic_logger/sync.rb
 - lib/semantic_logger/sync_processor.rb
+- lib/semantic_logger/test/capture_log_events.rb
 - lib/semantic_logger/utils.rb
 - lib/semantic_logger/version.rb
 homepage: https://logger.rocketjob.io
@@ -108,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Feature rich logging framework, and replacement for existing Ruby & Rails