semantic_logger 4.3.1 → 4.4.0

data/lib/semantic_logger/formatters/color.rb

@@ -60,12 +60,13 @@ module SemanticLogger
       #    ColorMaps each of the log levels to a color
       def initialize(ap: {multiline: false},
                      color_map: ColorMap.new,
-                     time_format: TIME_FORMAT,
+                     time_format: nil,
                      log_host: false,
-                     log_application: false)
+                     log_application: false,
+                     precision: PRECISION)
         @ai_options = ap
         @color_map  = color_map.is_a?(ColorMap) ? color_map : ColorMap.new(color_map)
-        super(time_format: time_format, log_host: log_host, log_application: log_application)
+        super(time_format: time_format, log_host: log_host, log_application: log_application, precision: precision)
       end
 
       def level

data/lib/semantic_logger/formatters/fluentd.rb

@@ -0,0 +1,37 @@
+require 'json'
+
+module SemanticLogger
+  module Formatters
+    # Fluentd is similar to SemanticLogger::Formatters::Json but with log level that are recongnized
+    # by kubernetes fluentd.
+    class Fluentd < Json
+      attr_reader :need_process_info
+
+      def initialize(log_host: true, log_application: true, need_process_info: false)
+        @need_process_info = need_process_info
+        super(log_host: log_host, log_application: log_application, time_key: 'time', time_format: :rfc_3339)
+      end
+
+      def severity
+        hash['severity'] = log.level
+        hash['severity_index'] = log.level_index
+      end
+
+      def process_info
+        # Ignore fields: pid, thread, file and line by default
+        super() if need_process_info
+      end
+
+      def call(log, logger)
+        self.hash   = {}
+        self.log    = log
+        self.logger = logger
+
+        host; application; time; severity; process_info; duration; tags; named_tags; name; message; payload; exception; metric
+        hash
+
+        hash.to_json
+      end
+    end
+  end
+end

data/lib/semantic_logger/formatters/json.rb

@@ -3,8 +3,10 @@ module SemanticLogger
   module Formatters
     class Json < Raw
       # Default JSON time format is ISO8601
-      def initialize(time_format: :iso_8601, log_host: true, log_application: true, time_key: :timestamp)
-        super(time_format: time_format, log_host: log_host, log_application: log_application, time_key: time_key)
+      def initialize(time_format: :iso_8601, log_host: true, log_application: true, time_key: :timestamp,
+                     precision: PRECISION)
+        super(time_format: time_format, log_host: log_host, log_application: log_application, time_key: time_key,
+              precision: precision)
       end
 
       # Returns log messages in JSON format

data/lib/semantic_logger/formatters/raw.rb

@@ -6,9 +6,9 @@ module SemanticLogger
       attr_accessor :hash, :log, :logger, :time_key
 
       # By default Raw formatter does not reformat the time
-      def initialize(time_format: :none, log_host: true, log_application: true, time_key: :time)
+      def initialize(time_format: :none, log_host: true, log_application: true, time_key: :time, precision: PRECISION)
         @time_key = time_key
-        super(time_format: time_format, log_host: log_host, log_application: log_application)
+        super(time_format: time_format, log_host: log_host, log_application: log_application, precision: precision)
       end
 
       # Host name

data/lib/semantic_logger/formatters/signalfx.rb

@@ -10,7 +10,8 @@ module SemanticLogger
                      log_application: true,
                      gauge_name: 'Application.average',
                      counter_name: 'Application.counter',
-                     environment: true)
+                     environment: true,
+                     precision: PRECISION)
 
         @token        = token
         @dimensions   = dimensions.map(&:to_sym) if dimensions
@@ -18,12 +19,12 @@ module SemanticLogger
         @counter_name = counter_name
 
         if environment == true
-          @environment = defined?(Rails) ? Rails.env : ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+          @environment = defined?(Rails) && Rails.respond_to?(:env) ? Rails.env : ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
         elsif environment
           @environment = environment
         end
 
-        super(time_format: :ms, log_host: log_host, log_application: log_application)
+        super(time_format: :ms, log_host: log_host, log_application: log_application, precision: precision)
       end
 
       # Create SignalFx friendly metric.

data/lib/semantic_logger/levels.rb

@@ -0,0 +1,38 @@
+module SemanticLogger
+  module Levels
+    # Logging levels in order of most detailed to most severe
+    LEVELS = %i[trace debug info warn error fatal].freeze
+
+    # Internal method to return the log level as an internal index
+    # Also supports mapping the ::Logger levels to SemanticLogger levels
+    def self.index(level)
+      return if level.nil?
+
+      index =
+        if level.is_a?(Symbol)
+          LEVELS.index(level)
+        elsif level.is_a?(String)
+          level = level.downcase.to_sym
+          LEVELS.index(level)
+        elsif level.is_a?(Integer) && defined?(::Logger::Severity)
+          # Mapping of Rails and Ruby Logger levels to SemanticLogger levels
+          @map_levels ||= begin
+            levels = []
+            ::Logger::Severity.constants.each do |constant|
+              levels[::Logger::Severity.const_get(constant)] =
+                LEVELS.find_index(constant.downcase.to_sym) || LEVELS.find_index(:error)
+            end
+            levels
+          end
+          @map_levels[level]
+        end
+      raise "Invalid level:#{level.inspect} being requested. Must be one of #{LEVELS.inspect}" unless index
+      index
+    end
+
+    # Returns the symbolic level for the supplied level index
+    def self.level(level_index)
+      LEVELS[level_index]
+    end
+  end
+end

data/lib/semantic_logger/log.rb

@@ -59,7 +59,7 @@ module SemanticLogger
       @time        = Time.now
       @tags        = SemanticLogger.tags
       @named_tags  = SemanticLogger.named_tags
-      @level_index = index.nil? ? SemanticLogger.level_to_index(level) : index
+      @level_index = index.nil? ? Levels.index(level) : index
     end
 
     # Assign named arguments to this log entry, supplying defaults where applicable
@@ -87,7 +87,12 @@ module SemanticLogger
       end
 
       self.message = message
-      self.payload = payload
+      if payload && payload.is_a?(Hash)
+        self.payload = payload
+      elsif payload
+        self.message = message.nil? ? payload.to_s : "#{message} -- #{payload}"
+        self.payload = nil
+      end
 
       if exception
         case log_exception
@@ -104,7 +109,7 @@ module SemanticLogger
         # On exception change the log level
         if on_exception_level
           self.level       = on_exception_level
-          self.level_index = SemanticLogger.level_to_index(level)
+          self.level_index = Levels.index(level)
         end
       end
 
@@ -120,7 +125,6 @@ module SemanticLogger
         self.dimensions    = dimensions
       end
 
-      self.payload = payload if payload&.size&.positive?
       true
     end
 
@@ -140,8 +144,9 @@ module SemanticLogger
       elsif exception.nil? && payload && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
         exception = payload
         payload   = nil
-      elsif payload.is_a?(String)
+      elsif payload && !payload.is_a?(Hash)
         message = message.nil? ? payload : "#{message} -- #{payload}"
+        payload = nil
       end
 
       # Add result of block as message or payload if not nil
@@ -279,7 +284,7 @@ module SemanticLogger
 
     # DEPRECATED
     def formatted_time
-      time.strftime(Formatters::Base::TIME_FORMAT)
+      time.strftime(Formatters::Base.build_time_format)
     end
 
     DeprecatedLogger = Struct.new(:host, :application)

data/lib/semantic_logger/loggable.rb

@@ -82,7 +82,7 @@ module SemanticLogger
         #   return false
         # end
 
-        index = SemanticLogger.level_to_index(level)
+        index = Levels.index(level)
 
         logger_measure_module.module_eval(<<~MEASURE_METHOD, __FILE__, __LINE__ + 1)
           def #{method_name}(*args, &block)

data/lib/semantic_logger/logger.rb

@@ -5,6 +5,25 @@ module SemanticLogger
   class Logger < Base
     include SemanticLogger::Concerns::Compatibility
 
+    def self.subscribe(object = nil, &block)
+      subscriber = block || object
+
+      unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
+        raise('When supplying an on_log subscriber, it must support the #call method')
+      end
+
+      subscribers = (@subscribers ||= Concurrent::Array.new)
+      subscribers << subscriber unless subscribers.include?(subscriber)
+    end
+
+    class << self
+      attr_reader :subscribers
+    end
+
+    def self.processor
+      @processor
+    end
+
     # Returns a Logger instance
     #
     # Return the logger for a specific class, supports class specific log levels
@@ -32,10 +51,33 @@ module SemanticLogger
 
     # Place log request on the queue for the Appender thread to write to each
     # appender in the order that they were registered
+    #
+    # Subscribers are called inline before handing off to the queue so that
+    # they can capture additional context information as needed.
     def log(log, message = nil, progname = nil, &block)
       # Compatibility with ::Logger
       return add(log, message, progname, &block) unless log.is_a?(SemanticLogger::Log)
-      Processor << log
+
+      Logger.call_subscribers(log)
+
+      Logger.processor.log(log)
+    end
+
+    private
+
+    @processor   = Processor.new
+    @subscribers = nil
+
+    def self.call_subscribers(log)
+      return unless @subscribers
+
+      @subscribers.each do |subscriber|
+        begin
+          subscriber.call(log)
+        rescue Exception => exc
+          self.class.processor.logger.error('Exception calling :on_log subscriber', exc)
+        end
+      end
     end
   end
 end

data/lib/semantic_logger/processor.rb

@@ -1,61 +1,6 @@
 module SemanticLogger
   # Thread that submits and processes log requests
-  class Processor
-    # Returns [Appender::Async => SemanticLogger::Processor] the global instance of this processor
-    # wrapped in the Async proxy so that all logging is asynchronous in a thread of its own.
-    #
-    # More than one instance can be created if needed.
-    def self.instance
-      @processor
-    end
-
-    # Start the appender thread
-    def self.start
-      return false if instance.active?
-      instance.thread
-      true
-    end
-
-    # Returns true if the appender_thread is active
-    def self.active?
-      instance.alive?
-    end
-
-    # Returns [Integer] the number of log entries waiting to be written to the appenders.
-    #
-    # When this number grows it is because the logging appender thread is not
-    # able to write to the appenders fast enough. Either reduce the amount of
-    # logging, increase the log level, reduce the number of appenders, or
-    # look into speeding up the appenders themselves
-    def self.queue_size
-      instance.queue.size
-    end
-
-    # Add log request to the queue for processing.
-    # Log subscribers are called inline before handing off to the queue.
-    def self.<<(log)
-      instance.appender.send(:call_log_subscribers, log)
-      instance.log(log)
-    end
-
-    # Returns the check_interval which is the number of messages between checks
-    # to determine if the appender thread is falling behind.
-    def self.lag_check_interval
-      instance.lag_check_interval
-    end
-
-    # Set the check_interval which is the number of messages between checks
-    # to determine if the appender thread is falling behind.
-    def self.lag_check_interval=(lag_check_interval)
-      instance.lag_check_interval = lag_check_interval
-    end
-
-    # Returns the amount of time in seconds
-    # to determine if the appender thread is falling behind.
-    def self.lag_threshold_s
-      instance.lag_threshold_s
-    end
-
+  class Processor < Appender::Async
     # Allow the internal logger to be overridden from its default of STDERR
     #   Can be replaced with another Ruby logger or Rails logger, but never to
     #   SemanticLogger::Logger itself since it is for reporting problems
@@ -75,83 +20,18 @@ module SemanticLogger
       end
     end
 
-    attr_accessor :logger, :log_subscribers
+    attr_reader :appenders
 
-    def initialize
-      @log_subscribers = nil
-      @logger          = self.class.logger.dup
-      @logger.name     = self.class.name
+    def initialize(max_queue_size: -1)
+      @appenders = Appenders.new(self.class.logger.dup)
+      super(appender: @appenders, max_queue_size: max_queue_size)
     end
 
-    def on_log(object = nil, &block)
-      subscriber = block || object
-
-      unless subscriber.is_a?(Proc) || subscriber.respond_to?(:call)
-        raise('When supplying an on_log subscriber, it must support the #call method')
-      end
-
-      subscribers = (@log_subscribers ||= Concurrent::Array.new)
-      subscribers << subscriber unless subscribers.include?(subscriber)
-    end
-
-    def log(log)
-      SemanticLogger.appenders.each do |appender|
-        begin
-          appender.log(log) if appender.should_log?(log)
-        rescue Exception => exc
-          logger.error "Appender thread: Failed to log to appender: #{appender.inspect}", exc
-        end
-      end
-    end
-
-    def flush
-      SemanticLogger.appenders.each do |appender|
-        begin
-          logger.trace "Appender thread: Flushing appender: #{appender.name}"
-          appender.flush
-        rescue Exception => exc
-          logger.error "Appender thread: Failed to flush appender: #{appender.inspect}", exc
-        end
-      end
-      logger.trace 'Appender thread: All appenders flushed'
-    end
-
-    def close
-      SemanticLogger.appenders.each do |appender|
-        begin
-          logger.trace "Appender thread: Closing appender: #{appender.name}"
-          appender.flush
-          appender.close
-          SemanticLogger.remove_appender(appender)
-        rescue Exception => exc
-          logger.error "Appender thread: Failed to close appender: #{appender.inspect}", exc
-        end
-      end
-      logger.trace 'Appender thread: All appenders closed and removed from appender list'
-    end
-
-    private
-
-    def self.create_instance
-      SemanticLogger::Appender::Async.new(appender: new, max_queue_size: -1)
-    end
-
-    private_class_method :create_instance
-
-    @processor = create_instance
-
-    # Call on_log subscribers
-    def call_log_subscribers(log)
-      # If no subscribers registered, then return immediately
-      return unless log_subscribers
-
-      log_subscribers.each do |subscriber|
-        begin
-          subscriber.call(log)
-        rescue Exception => exc
-          logger.error 'Exception calling :on_log subscriber', exc
-        end
-      end
+    # Start the appender thread
+    def start
+      return false if active?
+      thread
+      true
     end
   end
 end

data/lib/semantic_logger/reporters/minitest.rb

@@ -0,0 +1,49 @@
+module SemanticLogger
+  module Reporters
+    # When using Minitest to run tests, log start and end messages for every test to the log file.
+    # On completion the time it took to run the test is also logged.
+    #
+    # For example, add the following lines to `test_helper.rb`:
+    #   reporters = [
+    #     Minitest::Reporters::ProgressReporter.new,
+    #     SemanticLogger::Reporters::Minitest.new
+    #   ]
+    #   Minitest::Reporters.use!(reporters)
+    #
+    # Log entries similar to the following should show up in the log file:
+    #
+    # 2019-01-30 14:41:21.590383 I [9989:70268303433760] (9.958ms) Minitest -- Passed: test_0002_infinite timeout
+    # 2019-01-30 14:41:21.590951 I [9989:70268303433760] Minitest -- Started: test_0002_must return the servers in the supplied order
+    # 2019-01-30 14:41:21.592012 I [9989:70268303433760] (1.019ms) Minitest -- Passed: test_0002_must return the servers in the supplied order
+    # 2019-01-30 14:41:21.592054 I [9989:70268303433760] Minitest -- Started: test_0003_must handle an empty list of servers
+    # 2019-01-30 14:41:21.592094 I [9989:70268303433760] (0.014ms) Minitest -- Passed: test_0003_must handle an empty list of servers
+    # 2019-01-30 14:41:21.592118 I [9989:70268303433760] Minitest -- Started: test_0001_must return one server, once
+    # 2019-01-30 14:41:21.592510 I [9989:70268303433760] (0.361ms) Minitest -- Passed: test_0001_must return one server, once
+    #
+    # Note:
+    # - To use `Minitest::Reporters::ProgressReporter` the gem `minitest-reporters` is required, as well as the
+    #   following line in `test_helper.rb`:
+    #     `require 'minitest/reporters'`
+    class Minitest < ::Minitest::AbstractReporter
+      include SemanticLogger::Loggable
+
+      logger.name = 'Minitest'
+
+      attr_accessor :io
+
+      def before_test(test)
+        logger.info('START', name: test.name)
+      end
+
+      def after_test(test)
+        if test.error?
+          logger.benchmark_error('FAIL', payload: {name: test.name}, duration: test.time * 1_000, metric: 'minitest/fail')
+        elsif test.skipped?
+          logger.benchmark_warn('SKIP', payload: {name: test.name}, duration: test.time * 1_000, metric: 'minitest/skip')
+        else
+          logger.benchmark_info('PASS', payload: {name: test.name}, duration: test.time * 1_000, metric: 'minitest/pass')
+        end
+      end
+    end
+  end
+end