semantic_logger 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cf47326afecad1dcc0ecd8cffa732b51c023356
-  data.tar.gz: d8d72838b88abb13b94e1433d1676fff824eda26
+  metadata.gz: b67fae09283837194e489b469dace9f3b5103901
+  data.tar.gz: fc112ededd82585ff40307585e65769ab92601ce
 SHA512:
-  metadata.gz: fbefeb4f49ea32b347f09997baf3c9ca7138177c1ae9ea307088ca900cc74ad0fd8d1af8fba0479fcfbd16c37ec9df24aeb1e0fbe988eef90b3fce0e92933652
-  data.tar.gz: 0e168803566680fa7b20843df409e55bc4767b90a23c89249a6d7d7237f1b683d135088fc250a192d37726930ebee972844cb5b002e39c68b244eec8ad271d25
+  metadata.gz: a548305f79d04f28a1c2989132eb148f29da9415a5a3bc06b7879d7d95cabe94627533ad56c50a925137f3dfeca1dec62c43707472ade54eb331fd9f764f91a4
+  data.tar.gz: 7bf65721bfb19e865601ef929774877ab514607d86f150e86bb59be969cbecf080d2330e83c8ae2ba4aa2c44ea9b428773e5a791e281a142aeb20b4028036492

data/lib/semantic_logger.rb

@@ -8,29 +8,22 @@ module SemanticLogger
   autoload :Appender,           'semantic_logger/appender'
   autoload :Base,               'semantic_logger/base'
   autoload :DebugAsTraceLogger, 'semantic_logger/debug_as_trace_logger'
+  autoload :Formatters,         'semantic_logger/formatters'
   autoload :Log,                'semantic_logger/log'
   autoload :Logger,             'semantic_logger/logger'
   autoload :Loggable,           'semantic_logger/loggable'
   autoload :Processor,          'semantic_logger/processor'
   autoload :Subscriber,         'semantic_logger/subscriber'
+  autoload :Utils,              'semantic_logger/utils'
 
   module Concerns
     autoload :Compatibility,    'semantic_logger/concerns/compatibility'
   end
 
-  module Formatters
-    autoload :Base,             'semantic_logger/formatters/base'
-    autoload :Color,            'semantic_logger/formatters/color'
-    autoload :Default,          'semantic_logger/formatters/default'
-    autoload :Json,             'semantic_logger/formatters/json'
-    autoload :Raw,              'semantic_logger/formatters/raw'
-    autoload :Syslog,           'semantic_logger/formatters/syslog'
-  end
-
-  module Metrics
-    autoload :NewRelic,         'semantic_logger/metrics/new_relic'
-    autoload :Statsd,           'semantic_logger/metrics/statsd'
-    autoload :Udp,              'semantic_logger/metrics/udp'
+  module Metric
+    autoload :NewRelic,          'semantic_logger/metric/new_relic'
+    autoload :Signalfx,          'semantic_logger/metric/signalfx'
+    autoload :Statsd,            'semantic_logger/metric/statsd'
   end
 
   if defined?(JRuby)

data/lib/semantic_logger/ansi_colors.rb

@@ -1,16 +1,16 @@
 module SemanticLogger
   # Formatting & colors used by optional color formatter
   module AnsiColors
-    CLEAR     = "\e[0m"
-    BOLD      = "\e[1m"
-    BLACK     = "\e[30m"
-    RED       = "\e[31m"
-    GREEN     = "\e[32m"
-    YELLOW    = "\e[33m"
-    BLUE      = "\e[34m"
-    MAGENTA   = "\e[35m"
-    CYAN      = "\e[36m"
-    WHITE     = "\e[37m"
+    CLEAR   = "\e[0m"
+    BOLD    = "\e[1m"
+    BLACK   = "\e[30m"
+    RED     = "\e[31m"
+    GREEN   = "\e[32m"
+    YELLOW  = "\e[33m"
+    BLUE    = "\e[34m"
+    MAGENTA = "\e[35m"
+    CYAN    = "\e[36m"
+    WHITE   = "\e[37m"
 
     # DEPRECATED - NOT USED
     LEVEL_MAP = {

data/lib/semantic_logger/appender.rb

@@ -1,6 +1,8 @@
 module SemanticLogger
   module Appender
     # @formatter:off
+    autoload :Async,             'semantic_logger/appender/async'
+    autoload :AsyncBatch,        'semantic_logger/appender/async_batch'
     autoload :Bugsnag,           'semantic_logger/appender/bugsnag'
     autoload :Elasticsearch,     'semantic_logger/appender/elasticsearch'
     autoload :ElasticsearchHttp, 'semantic_logger/appender/elasticsearch_http'
@@ -34,45 +36,59 @@ module SemanticLogger
     end
 
     # Returns [SemanticLogger::Subscriber] appender for the supplied options
-    def self.create(options, &block)
+    def self.factory(options, &block)
+      options = options.dup
+      async   = options.delete(:async)
+      batch   = options.delete(:batch)
+
+      # Extract batch and async options
+      proxy_options = {}
+      ASYNC_OPTION_KEYS.each { |key| proxy_options[key] = options.delete(key) if options.key?(key) }
+
+      appender = build(options, &block)
+
+      # If appender implements #batch, then it should use the batch proxy by default.
+      batch    = true if batch.nil? && appender.respond_to?(:batch)
+
+      if batch == true
+        proxy_options[:appender] = appender
+        Appender::AsyncBatch.new(proxy_options)
+      elsif async == true
+        proxy_options[:appender] = appender
+        Appender::Async.new(proxy_options)
+      else
+        appender
+      end
+    end
+
+    private
+
+    ASYNC_OPTION_KEYS = [:max_queue_size, :lag_threshold_s, :batch_size, :batch_seconds, :lag_check_interval]
+
+    # Returns [Subscriber] instance from the supplied options.
+    def self.build(options, &block)
       if options[:io] || options[:file_name]
         SemanticLogger::Appender::File.new(options, &block)
       elsif appender = options.delete(:appender)
         if appender.is_a?(Symbol)
-          constantize_symbol(appender).new(options)
+          SemanticLogger::Utils.constantize_symbol(appender).new(options)
         elsif appender.is_a?(Subscriber)
           appender
         else
           raise(ArgumentError, "Parameter :appender must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
         end
-      elsif options[:logger]
-        SemanticLogger::Appender::Wrapper.new(options, &block)
-      end
-    end
-
-    def self.constantize_symbol(symbol, namespace = 'SemanticLogger::Appender')
-      klass = "#{namespace}::#{camelize(symbol.to_s)}"
-      begin
-        if RUBY_VERSION.to_i >= 2
-          Object.const_get(klass)
+      elsif appender = options.delete(:metric)
+        if appender.is_a?(Symbol)
+          SemanticLogger::Utils.constantize_symbol(appender, 'SemanticLogger::Metric').new(options)
+        elsif appender.is_a?(Subscriber)
+          appender
         else
-          klass.split('::').inject(Object) { |o, name| o.const_get(name) }
+          raise(ArgumentError, "Parameter :metric must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
         end
-      rescue NameError
-        raise(ArgumentError, "Could not convert symbol: #{symbol} to a class in: #{namespace}. Looking for: #{klass}")
+      elsif options[:logger]
+        SemanticLogger::Appender::Wrapper.new(options, &block)
       end
     end
 
-    private
-
-    # Borrow from Rails, when not running Rails
-    def self.camelize(term)
-      string = term.to_s
-      string = string.sub(/^[a-z\d]*/) { |match| match.capitalize }
-      string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
-      string.gsub!('/'.freeze, '::'.freeze)
-      string
-    end
-
   end
 end

data/lib/semantic_logger/appender/async.rb

@@ -0,0 +1,179 @@
+require 'forwardable'
+
+module SemanticLogger
+  module Appender
+    # Allow any appender to run asynchronously in a separate thread.
+    class Async
+      extend Forwardable
+
+      attr_accessor :logger, :lag_check_interval, :lag_threshold_s
+      attr_reader :queue, :appender
+
+      # Forward methods that can be called directly
+      def_delegator :@appender, :name
+      def_delegator :@appender, :should_log?
+      def_delegator :@appender, :filter
+      def_delegator :@appender, :host
+      def_delegator :@appender, :application
+      def_delegator :@appender, :level
+      def_delegator :@appender, :level=
+      def_delegator :@appender, :logger
+
+      # Appender proxy to allow an existing appender to run asynchronously in a separate thread.
+      #
+      # Parameters:
+      #   name: [String]
+      #     Name to use for the log thread and the log name when logging any errors from this appender.
+      #
+      #   max_queue_size: [Integer]
+      #     The maximum number of log messages to hold on the queue before blocking attempts to add to the queue.
+      #     -1: The queue size is uncapped and will never block no matter how long the queue is.
+      #     Default: 10,000
+      #
+      #   lag_threshold_s [Float]
+      #     Log a warning when a log message has been on the queue for longer than this period in seconds.
+      #     Default: 30
+      #
+      #   lag_check_interval: [Integer]
+      #     Number of messages to process before checking for slow logging.
+      #     Default: 1,000
+      def initialize(appender:,
+                     name: appender.class.name,
+                     max_queue_size: 10_000,
+                     lag_check_interval: 1_000,
+                     lag_threshold_s: 30)
+
+        @appender           = appender
+        @lag_check_interval = lag_check_interval
+        @lag_threshold_s    = lag_threshold_s
+
+        if max_queue_size == -1
+          @queue  = Queue.new
+          @capped = false
+        else
+          @queue  = SizedQueue.new(max_queue_size)
+          @capped = true
+        end
+        thread
+      end
+
+      # Returns [true|false] if the queue has a capped size.
+      def capped?
+        @capped
+      end
+
+      # Returns [Thread] the worker thread.
+      #
+      # Starts the worker thread if not running.
+      def thread
+        return @thread if @thread && @thread.alive?
+        @thread = Thread.new { process }
+      end
+
+      # Returns true if the worker thread is active
+      def active?
+        @thread && @thread.alive?
+      end
+
+      # Add log message for processing.
+      def log(log)
+        queue << log
+      end
+
+      # Flush all queued log entries disk, database, etc.
+      #  All queued log messages are written and then each appender is flushed in turn.
+      def flush
+        submit_request(:flush)
+      end
+
+      # Close all appenders and flush any outstanding messages.
+      def close
+        # TODO: Prevent new close requests once this appender has been closed.
+        submit_request(:close)
+      end
+
+      private
+
+      # Separate thread for batching up log messages before writing.
+      def process
+        # This thread is designed to never go down unless the main thread terminates
+        # or the appender is closed.
+        Thread.current.name = logger.name
+        logger.trace "Async: Appender thread active"
+        begin
+          process_messages
+        rescue StandardError => exception
+          # This block may be called after the file handles have been released by Ruby
+          logger.error('Async: Restarting due to exception', exception) rescue nil
+          retry
+        rescue Exception => exception
+          # This block may be called after the file handles have been released by Ruby
+          logger.error('Async: Stopping due to fatal exception', exception) rescue nil
+        ensure
+          @thread = nil
+          # This block may be called after the file handles have been released by Ruby
+          logger.trace('Async: Thread has stopped') rescue nil
+        end
+      end
+
+      def process_messages
+        count = 0
+        while message = queue.pop
+          if message.is_a?(Log)
+            appender.log(message)
+            count += 1
+            # Check every few log messages whether this appender thread is falling behind
+            if count > lag_check_interval
+              check_lag(message)
+              count = 0
+            end
+          else
+            break unless process_message(message)
+          end
+        end
+      end
+
+      # Returns false when message processing should be stopped
+      def process_message(message)
+        case message[:command]
+        when :flush
+          appender.flush
+          message[:reply_queue] << true if message[:reply_queue]
+        when :close
+          appender.close
+          message[:reply_queue] << true if message[:reply_queue]
+          return false
+        else
+          logger.warn "Async: Appender thread: Ignoring unknown command: #{message[:command]}"
+        end
+        true
+      end
+
+      def check_lag(log)
+        if (diff = Time.now - log.time) > lag_threshold_s
+          logger.warn "Async: Appender thread has fallen behind by #{diff} seconds with #{queue.size} messages queued up. Consider reducing the log level or changing the appenders"
+        end
+      end
+
+      # Submit command and wait for reply
+      def submit_request(command)
+        return false unless active?
+
+        queue_size = queue.size
+        msg        = "Async: Queued log messages: #{queue_size}, running command: #{command}"
+        if queue_size > 1_000
+          logger.warn msg
+        elsif queue_size > 100
+          logger.info msg
+        elsif queue_size > 0
+          logger.trace msg
+        end
+
+        reply_queue = Queue.new
+        queue << {command: command, reply_queue: reply_queue}
+        reply_queue.pop
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/appender/async_batch.rb

@@ -0,0 +1,95 @@
+require 'concurrent'
+
+module SemanticLogger
+  module Appender
+    # Log asynchronously in batches using a separate thread.
+    #
+    # Log messages are grouped up and only logged when:
+    # * The number of queued messages is exceeded.
+    # * Or, the appropriate amount of time has passed since the last batch was sent.
+    class AsyncBatch < Async
+      attr_accessor :batch_size, :batch_seconds
+      attr_reader :signal
+
+      # Batching Appender proxy for appenders that support batches.
+      #
+      # Parameters:
+      #   batch_size: [Integer]
+      #     Maximum number of messages to batch up before sending.
+      #     Default: 300
+      #
+      #   batch_seconds: [Integer]
+      #     Maximum number of seconds between sending batches.
+      #     Default: 5
+      #
+      # See SemanticLogger::Appender::Async for other paramaters
+      #
+      # Note:
+      # * `lag_check_interval` is not applicable to batches, since the first message of every batch
+      #   is the oldest and is always checked to see if the lag interval has been exceeded.
+      def initialize(appender:,
+                     name: appender.class.name,
+                     max_queue_size: 10_000,
+                     lag_threshold_s: 30,
+                     batch_size: 300,
+                     batch_seconds: 5)
+
+        @batch_size    = batch_size
+        @batch_seconds = batch_seconds
+        @signal        = Concurrent::Event.new
+        super(
+          appender:        appender,
+          name:            name,
+          max_queue_size:  max_queue_size,
+          lag_threshold_s: lag_threshold_s
+        )
+
+        raise(ArgumentError, "#{appender.class.name} does not support batching. It must implement #batch") unless appender.respond_to?(:batch)
+      end
+
+      # Add log message for processing.
+      def log(log)
+        result = super(log)
+        # Wake up the processing thread since the number of queued messages has been exceeded.
+        signal.set if queue.size >= batch_size
+        result
+      end
+
+      private
+
+      # Separate thread for batching up log messages before writing.
+      def process_messages
+        loop do
+          # Wait for batch interval or number of messages to be exceeded.
+          signal.wait(batch_seconds)
+
+          logs          = []
+          first         = true
+          message_count = queue.length
+          message_count.times do
+            # Queue#pop(true) raises an exception when there are no more messages, which is considered expensive.
+            message = queue.pop
+            if message.is_a?(Log)
+              logs << message
+              if first
+                check_lag(message)
+                first = false
+              end
+            else
+              process_message(message)
+            end
+          end
+          appender.batch(logs) if logs.size > 0
+          signal.reset unless queue.size >= batch_size
+        end
+      end
+
+      def submit_request(command)
+        # Wake up the processing thread to process this command immediately.
+        signal.set
+        super
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/appender/bugsnag.rb

@@ -48,13 +48,14 @@ class SemanticLogger::Appender::Bugsnag < SemanticLogger::Subscriber
 
   # Send an error notification to Bugsnag
   def log(log)
-    return false unless should_log?(log)
     # Ignore logs coming from Bugsnag itself
     return false if log.name == 'Bugsnag'
 
     # Send error messages as Runtime exceptions
     exception =
       if log.exception
+        # Manually constructed Exception, without a backtrace.
+        log.exception.set_backtrace(log.backtrace) if !log.exception.backtrace && log.backtrace
         log.exception
       else
         error = RuntimeError.new(log.message)