semantic_logger 4.1.1 → 4.2.0

data/lib/semantic_logger/processor.rb

@@ -1,17 +1,24 @@
 module SemanticLogger
   # Thread that submits and processes log requests
   class Processor
+    # Returns [Appender::Async => SemanticLogger::Processor] the global instance of this processor
+    # wrapped in the Async proxy so that all logging is asynchronous in a thread of its own.
+    #
+    # More than one instance can be created if needed.
+    def self.instance
+      @processor
+    end
+
     # Start the appender thread
     def self.start
-      return false if active?
-      @thread = Thread.new { process_requests }
-      raise 'Failed to start Appender Thread' unless @thread
+      return false if instance.active?
+      instance.thread
       true
     end
 
     # Returns true if the appender_thread is active
     def self.active?
-      @thread && @thread.alive?
+      instance.alive?
     end
 
     # Returns [Integer] the number of log entries waiting to be written to the appenders.
@@ -21,105 +28,40 @@ module SemanticLogger
     # logging, increase the log level, reduce the number of appenders, or
     # look into speeding up the appenders themselves
     def self.queue_size
-      queue.size
-    end
-
-    # Flush all queued log entries disk, database, etc.
-    #  All queued log messages are written and then each appender is flushed in turn.
-    def self.flush
-      submit_request(:flush)
-    end
-
-    # Close all appenders and flush any outstanding messages.
-    def self.close
-      submit_request(:close)
+      instance.queue.size
     end
 
     # Add log request to the queue for processing.
+    # Log subscribers are called inline before handing off to the queue.
     def self.<<(log)
-      return unless active?
-
-      call_log_subscribers(log)
-      queue << log
-    end
-
-    # Submit command and wait for reply
-    def self.submit_request(command)
-      return false unless active?
-
-      msg = "Too many queued log messages: #{queue_size}, running command: #{command}"
-      if queue_size > 1_000
-        logger.warn msg
-      elsif queue_size > 100
-        logger.info msg
-      elsif queue_size > 0
-        logger.trace msg
-      end
-
-      reply_queue = Queue.new
-      queue << {command: command, reply_queue: reply_queue}
-      reply_queue.pop
-    end
-
-    # Allow the internal logger to be overridden from its default of STDERR
-    #   Can be replaced with another Ruby logger or Rails logger, but never to
-    #   SemanticLogger::Logger itself since it is for reporting problems
-    #   while trying to log to the various appenders
-    def self.logger=(logger)
-      @logger = logger
+      instance.appender.send(:call_log_subscribers, log)
+      instance.log(log)
     end
 
     # Returns the check_interval which is the number of messages between checks
-    # to determine if the appender thread is falling behind
+    # to determine if the appender thread is falling behind.
     def self.lag_check_interval
-      @lag_check_interval
+      instance.lag_check_interval
     end
 
     # Set the check_interval which is the number of messages between checks
-    # to determine if the appender thread is falling behind
+    # to determine if the appender thread is falling behind.
     def self.lag_check_interval=(lag_check_interval)
-      @lag_check_interval = lag_check_interval
+      instance.lag_check_interval = lag_check_interval
     end
 
     # Returns the amount of time in seconds
-    # to determine if the appender thread is falling behind
+    # to determine if the appender thread is falling behind.
     def self.lag_threshold_s
-      @lag_threshold_s
+      instance.lag_threshold_s
     end
 
-    def self.on_metric(options = {}, &block)
-      # Backward compatibility
-      options    = options.is_a?(Hash) ? options.dup : {appender: options}
-      subscriber = block || options.delete(:appender)
-
-      # Convert symbolized metrics appender to an actual object
-      subscriber = SemanticLogger::Appender.constantize_symbol(subscriber, 'SemanticLogger::Metrics').new(options) if subscriber.is_a?(Symbol)
-
-      raise('When supplying a metrics subscriber, it must support the #call method') unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
-      subscribers = (@metric_subscribers ||= Concurrent::Array.new)
-      subscribers << subscriber unless subscribers.include?(subscriber)
-    end
-
-    def self.on_log(object = nil, &block)
-      subscriber = block || object
-
-      raise('When supplying an on_log subscriber, it must support the #call method') unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
-      subscribers = (@log_subscribers ||= Concurrent::Array.new)
-      subscribers << subscriber unless subscribers.include?(subscriber)
-    end
-
-    private
-
-    @thread             = nil
-    @queue              = Queue.new
-    @lag_check_interval = 5000
-    @lag_threshold_s    = 30
-    @metric_subscribers = nil
-    @log_subscribers    = nil
-
-    # Queue to hold messages that need to be logged to the various appenders
-    def self.queue
-      @queue
+    # Allow the internal logger to be overridden from its default of STDERR
+    #   Can be replaced with another Ruby logger or Rails logger, but never to
+    #   SemanticLogger::Logger itself since it is for reporting problems
+    #   while trying to log to the various appenders
+    def self.logger=(logger)
+      @logger = logger
     end
 
     # Internal logger for SemanticLogger
@@ -133,103 +75,33 @@ module SemanticLogger
       end
     end
 
-    # Separate appender thread responsible for reading log messages and
-    # calling the appenders in it's thread
-    def self.process_requests
-      # This thread is designed to never go down unless the main thread terminates
-      # Before terminating at_exit is used to flush all the appenders
-      #
-      # Should any appender fail to log or flush, the exception is logged and
-      # other appenders will still be called
-      Thread.current.name = 'SemanticLogger::Processor'
-      logger.trace "V#{VERSION} Processor thread active"
-      begin
-        count = 0
-        while message = queue.pop
-          if message.is_a?(Log)
-            call_appenders(message)
-            call_metric_subscribers(message) if message.metric
-            count += 1
-            # Check every few log messages whether this appender thread is falling behind
-            if count > lag_check_interval
-              if (diff = Time.now - message.time) > lag_threshold_s
-                logger.warn "Appender thread has fallen behind by #{diff} seconds with #{queue_size} messages queued up. Consider reducing the log level or changing the appenders"
-              end
-              count = 0
-            end
-          else
-            case message[:command]
-            when :flush
-              flush_appenders
-              message[:reply_queue] << true if message[:reply_queue]
-            when :close
-              close_appenders
-              message[:reply_queue] << true if message[:reply_queue]
-              break
-            else
-              logger.warn "Appender thread: Ignoring unknown command: #{message[:command]}"
-            end
-          end
-        end
-      rescue Exception => exception
-        # This block may be called after the file handles have been released by Ruby
-        begin
-          logger.error 'Appender thread restarting due to exception', exception
-        rescue Exception
-          nil
-        end
-        retry
-      ensure
-        @thread = nil
-        # This block may be called after the file handles have been released by Ruby
-        begin
-          logger.trace 'Appender thread has stopped'
-        rescue Exception
-          nil
-        end
-      end
-    end
-
-    # Call Metric subscribers
-    def self.call_metric_subscribers(log)
-      # If no subscribers registered, then return immediately
-      return unless @metric_subscribers
+    attr_accessor :logger, :log_subscribers
 
-      @metric_subscribers.each do |subscriber|
-        begin
-          subscriber.call(log)
-        rescue Exception => exc
-          logger.error 'Exception calling metrics subscriber', exc
-        end
-      end
+    def initialize
+      @log_subscribers = nil
+      @logger          = self.class.logger.dup
+      @logger.name     = self.class.name
     end
 
-    # Call on_log subscribers
-    def self.call_log_subscribers(log)
-      # If no subscribers registered, then return immediately
-      return unless @log_subscribers
+    def on_log(object = nil, &block)
+      subscriber = block || object
 
-      @log_subscribers.each do |subscriber|
-        begin
-          subscriber.call(log)
-        rescue Exception => exc
-          logger.error 'Exception calling :on_log subscriber', exc
-        end
-      end
+      raise('When supplying an on_log subscriber, it must support the #call method') unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
+      subscribers = (@log_subscribers ||= Concurrent::Array.new)
+      subscribers << subscriber unless subscribers.include?(subscriber)
     end
 
-    # Call Appenders
-    def self.call_appenders(log)
+    def log(log)
       SemanticLogger.appenders.each do |appender|
         begin
-          appender.log(log)
+          appender.log(log) if appender.should_log?(log)
         rescue Exception => exc
           logger.error "Appender thread: Failed to log to appender: #{appender.inspect}", exc
         end
       end
     end
 
-    def self.flush_appenders
+    def flush
       SemanticLogger.appenders.each do |appender|
         begin
           logger.trace "Appender thread: Flushing appender: #{appender.name}"
@@ -241,7 +113,7 @@ module SemanticLogger
       logger.trace 'Appender thread: All appenders flushed'
     end
 
-    def self.close_appenders
+    def close
       SemanticLogger.appenders.each do |appender|
         begin
           logger.trace "Appender thread: Closing appender: #{appender.name}"
@@ -255,5 +127,31 @@ module SemanticLogger
       logger.trace 'Appender thread: All appenders closed and removed from appender list'
     end
 
+    private
+
+    def self.create_instance
+      SemanticLogger::Appender::Async.new(
+        name:           'SemanticLogger::Processor',
+        appender:       new,
+        max_queue_size: -1
+      )
+    end
+
+    @processor = create_instance
+
+    # Call on_log subscribers
+    def call_log_subscribers(log)
+      # If no subscribers registered, then return immediately
+      return unless log_subscribers
+
+      log_subscribers.each do |subscriber|
+        begin
+          subscriber.call(log)
+        rescue Exception => exc
+          logger.error 'Exception calling :on_log subscriber', exc
+        end
+      end
+    end
+
   end
 end

data/lib/semantic_logger/semantic_logger.rb

@@ -12,7 +12,7 @@ module SemanticLogger
 
   # Sets the global default log level
   def self.default_level=(level)
-    @default_level       = level
+    @default_level = level
     # For performance reasons pre-calculate the level index
     @default_level_index = level_to_index(level)
   end
@@ -33,7 +33,7 @@ module SemanticLogger
   #   Capturing backtraces is very expensive and should not be done all
   #   the time. It is recommended to run it at :error level in production.
   def self.backtrace_level=(level)
-    @backtrace_level       = level
+    @backtrace_level = level
     # For performance reasons pre-calculate the level index
     @backtrace_level_index = level.nil? ? 65535 : level_to_index(level)
   end
@@ -153,7 +153,7 @@ module SemanticLogger
   #   logger.debug("Login time", user: 'Joe', duration: 100, ip_address: '127.0.0.1')
   def self.add_appender(options, deprecated_level = nil, &block)
     options  = options.is_a?(Hash) ? options.dup : convert_old_appender_args(options, deprecated_level)
-    appender = SemanticLogger::Appender.create(options, &block)
+    appender = SemanticLogger::Appender.factory(options, &block)
     @appenders << appender
 
     # Start appender thread if it is not already running
@@ -178,12 +178,12 @@ module SemanticLogger
   # Flush all queued log entries disk, database, etc.
   #  All queued log messages are written and then each appender is flushed in turn.
   def self.flush
-    SemanticLogger::Processor.flush
+    SemanticLogger::Processor.instance.flush
   end
 
   # Close all appenders and flush any outstanding messages.
   def self.close
-    SemanticLogger::Processor.close
+    SemanticLogger::Processor.instance.close
   end
 
   # After forking an active process call SemanticLogger.reopen to re-open
@@ -198,30 +198,6 @@ module SemanticLogger
     SemanticLogger::Processor.start
   end
 
-  # Supply a metrics appender to be called whenever a logging metric is encountered
-  #
-  #  Parameters
-  #    appender: [Symbol | Object | Proc]
-  #      [Proc] the block to call.
-  #      [Object] the block on which to call #call.
-  #      [Symbol] :new_relic, or :statsd to forward metrics to
-  #
-  #    block
-  #      The block to be called
-  #
-  # Example:
-  #   SemanticLogger.on_metric do |log|
-  #     puts "#{log.metric} was received. Log: #{log.inspect}"
-  #   end
-  #
-  # Note:
-  # * This callback is called in the separate logging thread.
-  # * Does not slow down the application.
-  # * Only context is what is passed in the log struct, the original thread context is not available.
-  def self.on_metric(options = {}, &block)
-    SemanticLogger::Processor.on_metric(options, &block)
-  end
-
   # Supply a callback to be called whenever a log entry is created.
   # Useful for capturing appender specific context information.
   #
@@ -247,7 +223,7 @@ module SemanticLogger
   # * This callback is called within the thread of the application making the logging call.
   # * If these callbacks are slow they will slow down the application.
   def self.on_log(object = nil, &block)
-    Processor.on_log(object, &block)
+    Processor.instance.on_log(object, &block)
   end
 
   # Add signal handlers for Semantic Logger
@@ -513,7 +489,7 @@ module SemanticLogger
     else
       options[:logger] = appender
     end
-    warn "[DEPRECATED] SemanticLogger.add_appender parameters have changed. Please use: #{options.inspect}" if $VERBOSE
+    warn "[DEPRECATED] SemanticLogger.add_appender parameters have changed. Please use: #{options.inspect}"
     options
   end
 

data/lib/semantic_logger/subscriber.rb

@@ -1,11 +1,11 @@
 # Abstract Subscriber
 #
-#   Abstract base class for appender and metrics subscribers.
+#   Abstract base class for all appenders.
 module SemanticLogger
   class Subscriber < SemanticLogger::Base
-    # Every logger has its own formatter
-    attr_accessor :formatter
-    attr_writer :application, :host
+    # Every appender has its own formatter
+    attr_reader :formatter
+    attr_writer :application, :host, :logger
 
     # Returns the current log level if set, otherwise it logs everything it receives.
     def level
@@ -37,6 +37,31 @@ module SemanticLogger
       @host || SemanticLogger.host
     end
 
+    # Give each appender its own logger for logging.
+    # For example trace messages sent to services or errors when something fails.
+    def logger
+      @logger ||= begin
+        logger      = SemanticLogger::Processor.logger.clone
+        logger.name = self.class.name
+        logger
+      end
+    end
+
+    # Set the formatter from Symbol|Hash|Block
+    def formatter=(formatter)
+      @formatter =
+        if formatter.nil?
+          respond_to?(:call) ? self : default_formatter
+        else
+          Formatters.factory(formatter)
+        end
+    end
+
+    # Whether this log entry meets the criteria to be logged by this appender.
+    def should_log?(log)
+      super(log) && !log.metric_only?
+    end
+
     private
 
     # Initializer for Abstract Class SemanticLogger::Subscriber
@@ -44,7 +69,6 @@ module SemanticLogger
     # Parameters
     #   level: [:trace | :debug | :info | :warn | :error | :fatal]
     #     Override the log level for this subscriber.
-    #     Default: :error
     #
     #   formatter: [Object|Proc]
     #     An instance of a class that implements #call, or a Proc to be used to format
@@ -65,9 +89,9 @@ module SemanticLogger
     #     Name of this host to appear in log messages.
     #     Default: SemanticLogger.host
     def initialize(level: nil, formatter: nil, filter: nil, application: nil, host: nil, &block)
-      @formatter   = extract_formatter(formatter, &block)
-      @application = application
-      @host        = host
+      self.formatter = block || formatter
+      @application   = application
+      @host          = host
 
       # Subscribers don't take a class name, so use this class name if a subscriber
       # is logged to directly.
@@ -81,33 +105,5 @@ module SemanticLogger
       @level_index || 0
     end
 
-    # Return formatter that responds to call.
-    #
-    # Supports formatter supplied as:
-    # - Symbol
-    # - Hash ( Symbol => { options })
-    # - Instance of any of SemanticLogger::Formatters
-    # - Proc
-    # - Any object that responds to :call
-    # - If none of the above apply, then the supplied block is returned as the formatter.
-    # - Otherwise an instance of the default formatter is returned.
-    def extract_formatter(formatter, &block)
-      case
-      when formatter.is_a?(Symbol)
-        SemanticLogger::Appender.constantize_symbol(formatter, 'SemanticLogger::Formatters').new
-      when formatter.is_a?(Hash) && formatter.size > 0
-        fmt, options = formatter.first
-        SemanticLogger::Appender.constantize_symbol(fmt.to_sym, 'SemanticLogger::Formatters').new(options)
-      when formatter.respond_to?(:call)
-        formatter
-      when block
-        block
-      when respond_to?(:call)
-        self
-      else
-        default_formatter
-      end
-    end
-
   end
 end