semantic_logger 4.6.1 → 4.10.0

data/lib/semantic_logger/appender/file.rb

@@ -1,3 +1,4 @@
+require "date"
 # File appender
 #
 #   Writes log messages to a file or open iostream
@@ -5,104 +6,285 @@
 module SemanticLogger
   module Appender
     class File < SemanticLogger::Subscriber
-      # Create a File Logger appender instance.
+      attr_accessor :file_name, :retry_count, :append, :exclusive_lock, :encoding,
+                    :reopen_period, :reopen_count, :reopen_size
+      attr_reader :log_count, :log_size, :current_file_name, :reopen_at
+
+      # Create an appender to log to a named file.
       #
       # Parameters
-      #  :file_name [String]
-      #    Name of file to write to.
-      #  Or,
-      #  :io [IO]
-      #    An IO stream to which to write the log messages to.
-      #
-      #  :level [:trace | :debug | :info | :warn | :error | :fatal]
-      #    Override the log level for this appender.
-      #    Default: SemanticLogger.default_level
-      #
-      #  :formatter: [Object|Proc]
-      #    An instance of a class that implements #call, or a Proc to be used to format
-      #    the output from this appender
-      #    Default: Use the built-in formatter (See: #call)
-      #
-      #  :filter [Regexp|Proc]
-      #    RegExp: Only include log messages where the class name matches the supplied
-      #    regular expression. All other messages will be ignored.
-      #    Proc: Only include log messages where the supplied Proc returns true
-      #          The Proc must return true or false.
+      #   file_name [String]
+      #     Name of the file to write to.
       #
-      # Example
-      #    require 'semantic_logger'
+      #     File name format directives:
+      #       %p - Process Id
+      #       %n - Short hostname (SemanticLogger.host). Everything before the first period in the hostname.
+      #       %N - Full hostname (SemanticLogger.host)
+      #       %a - Application name (SemanticLogger.application)
+      #       %e - Environment name (SemanticLogger.environment)
+      #       %D - Current Date. Equivalent to "%Y%m%d"
+      #       %T - Current Time. Equivalent to "%H%M%S"
+      #       %% - Literal `%` character
       #
-      #    # Enable trace level logging
-      #    SemanticLogger.default_level = :info
+      #     Date:
+      #       %Y - Year with century
+      #       %C - year / 100 (round down.  20 in 2009)
+      #       %y - year % 100 (00..99)
+      #       %m - Month of the year, zero-padded (01..12)
+      #       %d - Day of the month, zero-padded (01..31)
+      #       %j - Day of the year (001..366)
+      #       %U - Week number of the year.  The week starts with Sunday.  (00..53)
+      #       %W - Week number of the year.  The week starts with Monday.  (00..53)
       #
-      #    # Log to screen
-      #    SemanticLogger.add_appender(io: STDOUT, formatter: :color)
+      #     Time:
+      #       %H - 24 Hour of the day, zero-padded (00..23)
+      #       %M - Minute of the hour (00..59)
+      #       %S - Second of the minute (00..60)
       #
-      #    # And log to a file at the same time
-      #    SemanticLogger.add_appender(file_name: 'application.log', formatter: :color)
+      #     Examples:
+      #       Create a log file name consisting of the short host name, process id, date, and time.
+      #         "log/production-%n-%p-%D-%T.log"
       #
-      #    logger = SemanticLogger['test']
-      #    logger.info 'Hello World'
+      #   :level [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
       #
-      # Example 2. To log all levels to file and only :info and above to screen:
+      #   :formatter: [Object|Proc]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: Use the built-in formatter (See: #call)
       #
-      #    require 'semantic_logger'
+      #   :filter [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
       #
-      #    # Enable trace level logging
-      #    SemanticLogger.default_level = :trace
+      #   :append [true|false]
+      #     Append to the log file if already present?
+      #     Default: true
       #
-      #    # Log to screen but only display :info and above
-      #    SemanticLogger.add_appender(io: STDOUT, level: :info)
+      #   :exclusive_lock [true|false]
+      #     Obtain an exclusive lock on the file, for operating systems that support it.
+      #     Prevents multiple processes from trying to write to the same log file.
+      #     Default: false
       #
-      #    # And log to a file at the same time, including all :trace level data
-      #    SemanticLogger.add_appender(file_name: 'application.log')
+      #   :encoding ["UTF-8", "UTF-16", etc.]
+      #     Encoding to use when writing to the file.
+      #     Default: Encoding::BINARY
       #
-      #    logger =  SemanticLogger['test']
-      #    logger.info 'Hello World'
-      def initialize(io: nil, file_name: nil, **args, &block)
-        if io
-          @log = io
-        else
-          @file_name = file_name
-          raise 'SemanticLogging::Appender::File missing mandatory parameter :file_name or :io' unless file_name
-          reopen
+      #   :retry_count [Integer]
+      #     Number of times to attempt to re-open the file name when an error occurs trying to
+      #     write to the file.
+      #     Note: Set to 0 to disable retries.
+      #     Default: 1
+      #
+      #   :reopen_period [String]
+      #     Specify a period after which to re-open the log file, specified in minutes, hours, or days.
+      #     The format of the duration must start with an Integer or Float number,
+      #     followed by the duration specified as:
+      #       "m" : minutes
+      #       "h" : hours
+      #       "d" : days
+      #     The time is rounded down to the specified time interval, so that:
+      #     - "1h" will re-open every hour at the beginning of the hour.
+      #     - "30m" will re-open every 30 minutes at the beginning of the 30th minute.
+      #     - "1d" will re-open every day at midnight.
+      #     Examples:
+      #       "60m" : Every 60 minutes at the beginning of the minute: 10:24:00, 11:24:00, 12:24:00, ...
+      #       "1h"  : Every hour at the beginning of the hour: 10:00:00, 11:00:00, 12:00:00, ...
+      #       "1d"  : Every day at the beginning of the day: "20211008 00:00:00", "20211009 00:00:00", ...
+      #     Default: nil (Disabled)
+      #
+      #   :reopen_count [Integer]
+      #     Close and re-open the log file after every `reopen_count` number of logged entries.
+      #     Default: 0 (Disabled)
+      #
+      #   :reopen_size [Integer]
+      #     Approximate number of bytes to write to a log file by this process before closing and re-opening it.
+      #     Notes:
+      #     - When `append: true` and the file already exists, it reads the size of the current log file
+      #       and starts with that size.
+      #       - If the current log file size already exceeds the `reopen_size`, its current size is ignored.
+      #     - The `reopen_size` is only the amount of bytes written by this process, it excludes data
+      #       written by other processes. Use a unique filename to prevent multiple processes from writing to
+      #       the same log file at the same time.
+      #     Default: 0 (Disabled)
+      #
+      # Example
+      #    require "semantic_logger"
+      #
+      #    # Enable trace level logging
+      #    SemanticLogger.default_level = :info
+      #
+      #    # Log to a file
+      #    SemanticLogger.add_appender(file_name: "application.log", formatter: :color)
+      #
+      #    logger = SemanticLogger["test"]
+      #    logger.info "Hello World"
+      def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0, encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
+        if !file_name.is_a?(String) || file_name.empty?
+          raise(ArgumentError, "SemanticLogging::Appender::File file_name must be a non-empty string")
         end
 
+        @file_name      = file_name
+        @retry_count    = retry_count
+        @file           = nil
+        @append         = append
+        @reopen_period  = reopen_period
+        @reopen_count   = reopen_count
+        @reopen_size    = reopen_size
+        @encoding       = encoding
+        @exclusive_lock = exclusive_lock
+        @log_count      = 0
+        @log_size       = 0
+        @reopen_at      = nil
+
         super(**args, &block)
       end
 
       # After forking an active process call #reopen to re-open
-      # open the file handles etc to resources
-      #
-      # Note: This method will only work if :file_name was supplied
-      #       on the initializer.
-      #       If :io was supplied, it will need to be re-opened manually.
+      # open the file handles etc to resources.
       def reopen
-        return unless @file_name
+        begin
+          @file&.close
+        rescue StandardError
+          nil
+        end
 
-        @log = ::File.open(@file_name, ::File::WRONLY | ::File::APPEND | ::File::CREAT)
+        self.current_file_name = apply_format_directives(file_name)
+        if ::File.directory?(file_name)
+          raise(ArgumentError, "The supplied log file_name: #{current_file_name} is already a directory.")
+        end
+
+        self.log_count = 0
+        if append && reopen_size && ::File.exist?(current_file_name)
+          self.log_size = ::File.size(current_file_name)
+          self.log_size = 0 if log_size >= reopen_size
+        else
+          self.log_size = 0
+        end
+
+        self.reopen_at = reopen_period ? next_reopen_period(reopen_period) : nil
+
+        options = ::File::WRONLY | ::File::CREAT
+        options |= ::File::APPEND if append
+        @file = ::File.open(current_file_name, options)
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
-        @log.sync = true
-        @log.set_encoding(Encoding::BINARY) if @log.respond_to?(:set_encoding)
-        @log
+        @file.sync = true
+        @file.set_encoding(encoding) if @file.respond_to?(:set_encoding)
+        @file.flock(::File::LOCK_EX) if exclusive_lock
+        @file
       end
 
-      # Pass log calls to the underlying Rails, log4j or Ruby logger
-      #  trace entries are mapped to debug since :trace is not supported by the
-      #  Ruby or Rails Loggers
+      # Since only one appender thread will be writing to the file at a time
+      # it is not necessary to protect access to the file with a semaphore.
       def log(log)
-        # Since only one appender thread will be writing to the file at a time
-        # it is not necessary to protect access to the file with a semaphore
-        # Allow this logger to filter out log levels lower than it's own
-        @log.write(formatter.call(log, self) << "\n")
+        reopen if time_to_reopen?
+
+        count = 0
+        begin
+          message = formatter.call(log, self) << "\n"
+          @file.write(message)
+          @log_count += 1
+          @log_size  += message.size
+        rescue StandardError => e
+          if count < retry_count
+            count += 1
+            reopen
+            retry
+          end
+          raise(e)
+        end
         true
       end
 
       # Flush all pending logs to disk.
-      #  Waits for all sent documents to be writted to disk
+      #  Waits for all sent documents to be written to disk
       def flush
-        @log.flush if @log.respond_to?(:flush)
+        @file&.flush
+      end
+
+      private
+
+      attr_writer :log_count, :log_size, :current_file_name, :reopen_at
+
+      def time_to_reopen?
+        return true unless @file
+
+        (reopen_count.positive? && (log_count >= reopen_count)) ||
+          (reopen_size.positive? && (log_size >= reopen_size)) ||
+          (reopen_at && (Time.now > reopen_at))
+      end
+
+      def apply_format_directives(file_name)
+        return file_name unless file_name.include?("%")
+
+        file_name.gsub(/%(.)/) { format_directive(Regexp.last_match(1)) }
+      end
+
+      def format_directive(directive)
+        case directive
+        when "p"
+          $$
+        when "n"
+          SemanticLogger.host.split(".")[0]
+        when "N"
+          SemanticLogger.host
+        when "a"
+          SemanticLogger.application
+        when "e"
+          SemanticLogger.environment
+        when "D"
+          Date.today.strftime("%Y%m%d")
+        when "Y", "C", "y", "m", "d", "j", "U", "W"
+          Date.today.strftime("%#{directive}")
+        when "T"
+          Time.now.strftime("%H%M%S")
+        when "H", "M", "S"
+          Time.now.strftime("%#{directive}")
+        when "%"
+          "%"
+        else
+          raise(ArgumentError, "Format Directive '#{directive}' in file_name: #{file_name} is not supported.")
+        end
+      end
+
+      def next_reopen_period(period_string)
+        return unless period_string
+
+        duration, period = parse_period(period_string)
+        calculate_reopen_at(duration, period)
+      end
+
+      def parse_period(period_string)
+        match = period_string.to_s.downcase.gsub(/\s+/, "").match(/([\d.]+)([mhd])/)
+        unless match
+          raise(ArgumentError,
+                "Invalid period definition: #{period_string}, must begin with an integer, followed by m,h, or d.")
+        end
+
+        duration = match[1]
+        period   = match[2]
+        raise(ArgumentError, "Invalid or missing duration in: #{period_string}, must begin with an integer.") unless duration
+        raise(ArgumentError, "Invalid or missing period in: #{period_string}, must end with m,h, or d.") unless period
+
+        [duration.to_i, period]
+      end
+
+      # Round down the current time based on the period, then add on the duration for that period
+      def calculate_reopen_at(duration, period, time = Time.now)
+        case period
+        when "m"
+          Time.new(time.year, time.month, time.day, time.hour, time.min, 0) + (duration * 60)
+        when "h"
+          Time.new(time.year, time.month, time.day, time.hour, 0, 0) + (duration * 60 * 60)
+        when "d"
+          Time.new(time.year, time.month, time.day, 0, 0, 0) + (duration * 24 * 60 * 60)
+        else
+          raise(ArgumentError, "Invalid or missing period in: #{reopen_period}, must end with m,h, or d.")
+        end
       end
     end
   end

data/lib/semantic_logger/appender/graylog.rb

@@ -1,8 +1,8 @@
-require 'uri'
+require "uri"
 begin
-  require 'gelf'
+  require "gelf"
 rescue LoadError
-  raise LoadError.new('Gem gelf is required for logging to Graylog. Please add the gem "gelf" to your Gemfile.')
+  raise LoadError, 'Gem gelf is required for logging to Graylog. Please add the gem "gelf" to your Gemfile.'
 end
 
 # Forward log entries to a Graylog server.
@@ -82,8 +82,8 @@ module SemanticLogger
       #   application: [String]
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
-      def initialize(url: 'udp://localhost:12201',
-                     max_size: 'WAN',
+      def initialize(url: "udp://localhost:12201",
+                     max_size: "WAN",
                      gelf_options: {},
                      level_map: LevelMap.new,
                      **args,
@@ -105,7 +105,9 @@ module SemanticLogger
         @port     = uri.port
         @protocol = uri.scheme.to_sym
 
-        raise(ArgumentError, "Invalid protocol value: #{@protocol}. Must be :udp or :tcp") unless %i[udp tcp].include?(@protocol)
+        unless %i[udp tcp].include?(@protocol)
+          raise(ArgumentError, "Invalid protocol value: #{@protocol}. Must be :udp or :tcp")
+        end
 
         gelf_options[:protocol] ||= (@protocol == :tcp ? GELF::Protocol::TCP : GELF::Protocol::UDP)
         gelf_options[:facility] ||= application
@@ -120,11 +122,11 @@ module SemanticLogger
 
         h[:short_message] = h.delete(:message)
         if h[:short_message].nil?
-          h[:short_message] = log.exception.nil?  ? '<no-exception-message>' : log.exception.message
+          h[:short_message] = log.exception.nil? ? "<no-exception-message>" : log.exception.message
         end
-        h[:level]         = logger.level_map[log.level]
-        h[:level_str]     = log.level.to_s
-        h[:duration_str]  = h.delete(:duration)
+        h[:level]        = logger.level_map[log.level]
+        h[:level_str]    = log.level.to_s
+        h[:duration_str] = h.delete(:duration)
         h
       end
 

data/lib/semantic_logger/appender/honeybadger.rb

@@ -1,7 +1,7 @@
 begin
-  require 'honeybadger'
+  require "honeybadger"
 rescue LoadError
-  raise LoadError.new('Gem honeybadger is required for logging purposes. Please add the gem "honeybadger" to your Gemfile.')
+  raise LoadError, 'Gem honeybadger is required for logging purposes. Please add the gem "honeybadger" to your Gemfile.'
 end
 
 # Send log messages to honeybadger
@@ -48,7 +48,7 @@ module SemanticLogger
           context.delete(:exception)
           ::Honeybadger.notify(log.exception, context)
         else
-          message             = {
+          message = {
             error_class:   context.delete(:name),
             error_message: context.delete(:message),
             context:       context

data/lib/semantic_logger/appender/http.rb

@@ -1,8 +1,8 @@
-require 'net/http'
-require 'uri'
-require 'socket'
-require 'json'
-require 'openssl'
+require "net/http"
+require "uri"
+require "socket"
+require "json"
+require "openssl"
 
 # Log to any HTTP(S) server that accepts log messages in JSON form
 #
@@ -101,27 +101,29 @@ module SemanticLogger
         @continue_timeout = continue_timeout
 
         # On Ruby v2.0 and greater, Net::HTTP.new already uses a persistent connection if the server allows it
-        @header                     = {
-          'Accept'       => 'application/json',
-          'Content-Type' => 'application/json',
-          'Connection'   => 'keep-alive',
-          'Keep-Alive'   => '300'
+        @header = {
+          "Accept"       => "application/json",
+          "Content-Type" => "application/json",
+          "Connection"   => "keep-alive",
+          "Keep-Alive"   => "300"
         }
-        @header['Content-Encoding'] = 'gzip' if @compress
+        @header["Content-Encoding"] = "gzip" if @compress
 
         uri     = URI.parse(@url)
         @server = uri.host
-        raise(ArgumentError, "Invalid format for :url: #{@url.inspect}. Should be similar to: 'http://hostname:port/path'") unless @server
+        unless @server
+          raise(ArgumentError, "Invalid format for :url: #{@url.inspect}. Should be similar to: 'http://hostname:port/path'")
+        end
 
         @port     = uri.port
         @username = uri.user if !@username && uri.user
         @password = uri.password if !@password && uri.password
         @path     = uri.path
         # Path cannot be empty
-        @path     = '/' if @path == ''
+        @path     = "/" if @path == ""
 
-        if uri.scheme == 'https'
-          @ssl_options[:use_ssl]     = true
+        if uri.scheme == "https"
+          @ssl_options[:use_ssl] = true
           @ssl_options[:verify_mode] ||= OpenSSL::SSL::VERIFY_PEER
           @port                      ||= HTTP.https_default_port
         else
@@ -205,16 +207,16 @@ module SemanticLogger
         end
         request.basic_auth(@username, @password) if @username
         response = @http.request(request)
-        if response.code == '200' || response.code == '201'
+        if response.code == "200" || response.code == "201"
           true
         else
           # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
           logger.error("Bad HTTP response from: #{url} code: #{response.code}, #{response.body}")
           false
         end
-      rescue RuntimeError => exc
+      rescue RuntimeError => e
         reopen
-        raise exc
+        raise e
       end
     end
   end

data/lib/semantic_logger/appender/io.rb

@@ -0,0 +1,68 @@
+# File appender
+#
+#   Writes log messages to a file or open iostream
+#
+module SemanticLogger
+  module Appender
+    class IO < SemanticLogger::Subscriber
+      # Create a Stream Logger appender instance.
+      #
+      # Parameters
+      #  io [IO]
+      #    An IO stream to which to write the log messages to.
+      #
+      #  :level [:trace | :debug | :info | :warn | :error | :fatal]
+      #    Override the log level for this appender.
+      #    Default: SemanticLogger.default_level
+      #
+      #  :formatter: [Object|Proc]
+      #    An instance of a class that implements #call, or a Proc to be used to format
+      #    the output from this appender
+      #    Default: Use the built-in formatter (See: #call)
+      #
+      #  :filter [Regexp|Proc]
+      #    RegExp: Only include log messages where the class name matches the supplied
+      #    regular expression. All other messages will be ignored.
+      #    Proc: Only include log messages where the supplied Proc returns true
+      #          The Proc must return true or false.
+      #
+      # Example
+      #    require "semantic_logger"
+      #
+      #    # Enable trace level logging
+      #    SemanticLogger.default_level = :info
+      #
+      #    # Log to screen
+      #    SemanticLogger.add_appender(io: $stdout, formatter: :color)
+      #
+      #    logger = SemanticLogger['test']
+      #    logger.info 'Hello World'
+      def initialize(io, **args, &block)
+        @io = io
+        unless @io.respond_to?(:write)
+          raise(ArgumentError, "SemanticLogging::Appender::IO io is not a valid IO instance: #{io.inspect}")
+        end
+
+        super(**args, &block)
+      end
+
+      def log(log)
+        # Since only one appender thread will be writing to the file at a time
+        # it is not necessary to protect access to the file with a semaphore
+        # Allow this logger to filter out log levels lower than it's own
+        @io.write(formatter.call(log, self) << "\n")
+        true
+      end
+
+      # Flush all pending logs to disk.
+      #  Waits for all sent documents to be written to disk
+      def flush
+        @io.flush if @io.respond_to?(:flush)
+      end
+
+      def console_output?
+        [$stderr, $stdout].include?(@io)
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/kafka.rb

@@ -1,10 +1,10 @@
 begin
-  require 'kafka'
+  require "kafka"
 rescue LoadError
-  raise LoadError.new('Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.')
+  raise LoadError, 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
 end
 
-require 'date'
+require "date"
 
 # Forward all log messages to Apache Kafka.
 #
@@ -115,9 +115,9 @@ module SemanticLogger
       #   metrics: [Boolean]
       #     Send metrics only events to kafka.
       #     Default: true
-      def initialize(seed_brokers:, client_id: 'semantic-logger', connect_timeout: nil, socket_timeout: nil,
+      def initialize(seed_brokers:, client_id: "semantic-logger", connect_timeout: nil, socket_timeout: nil,
                      ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil,
-                     topic: 'log_messages', partition: nil, partition_key: nil, key: nil,
+                     topic: "log_messages", partition: nil, partition_key: nil, key: nil,
                      delivery_threshold: 100, delivery_interval: 10,
                      metrics: true, **args, &block)
 
@@ -183,6 +183,10 @@ module SemanticLogger
           delivery_interval:  delivery_interval
         )
       end
+
+      private
+
+      attr_reader :producer
     end
   end
 end

data/lib/semantic_logger/appender/mongodb.rb

@@ -1,8 +1,8 @@
-require 'socket'
+require "socket"
 begin
-  require 'mongo'
+  require "mongo"
 rescue LoadError
-  raise LoadError.new('Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.')
+  raise LoadError, 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
 end
 
 module SemanticLogger
@@ -104,9 +104,9 @@ module SemanticLogger
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
       def initialize(uri:,
-                     collection_name: 'semantic_logger',
+                     collection_name: "semantic_logger",
                      write_concern: 0,
-                     collection_size: 1024 ** 3,
+                     collection_size: 1024**3,
                      collection_max: nil,
                      **args,
                      &block)
@@ -118,7 +118,7 @@ module SemanticLogger
           size:   collection_size,
           write:  {w: write_concern}
         }
-        @options[:max]   = collection_max if collection_max
+        @options[:max] = collection_max if collection_max
 
         reopen
 

data/lib/semantic_logger/appender/new_relic.rb

@@ -1,7 +1,7 @@
 begin
-  require 'newrelic_rpm'
+  require "newrelic_rpm"
 rescue LoadError
-  raise LoadError.new('Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.')
+  raise LoadError, 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
 end
 
 # Send log messages to NewRelic