sapience 0.1.1

data/lib/sapience/formatters/json.rb

@@ -0,0 +1,22 @@
+require "json"
+module Sapience
+  module Formatters
+    class Json < Raw
+      # Default JSON time format is ISO8601
+      def initialize(options = {})
+        options               = options.dup
+        options[:time_format] = :iso_8601 unless options.key?(:time_format)
+        super(options)
+      end
+
+      # Returns log messages in JSON format
+      def call(log, logger)
+        h = super(log, logger)
+        h.delete(:time)
+        h[:timestamp] = format_time(log.time)
+        h.to_json
+      end
+
+    end
+  end
+end

data/lib/sapience/formatters/raw.rb

@@ -0,0 +1,12 @@
+require "json"
+module Sapience
+  module Formatters
+    class Raw < Base
+      # Returns log messages in Hash format
+      def call(log, logger)
+        log.to_h(log_host ? logger.host : nil, log_application ? logger.application : nil)
+      end
+
+    end
+  end
+end

data/lib/sapience/log.rb

@@ -0,0 +1,221 @@
+module Sapience
+  # Log Struct
+  #
+  #   Structure for holding all log entries
+  #
+  # level
+  #   Log level of the supplied log call
+  #   :trace, :debug, :info, :warn, :error, :fatal
+  #
+  # thread_name
+  #   Name of the thread in which the logging call was called
+  #
+  # name
+  #   Class name supplied to the logging instance
+  #
+  # message
+  #   Text message to be logged
+  #
+  # payload
+  #   Optional Hash or Ruby Exception object to be logged
+  #
+  # time
+  #   The time at which the log entry was created
+  #
+  # duration
+  #   The time taken to complete a measure call
+  #
+  # tags
+  #   Any tags active on the thread when the log call was made
+  #
+  # level_index
+  #   Internal index of the log level
+  #
+  # exception
+  #   Ruby Exception object to log
+  #
+  # metric [Object]
+  #   Object supplied when measure_x was called
+  #
+  # backtrace [Array<String>]
+  #   The backtrace captured at source when the log level >= Sapience.config.backtrace_level
+  #
+  # metric_amount [Numeric]
+  #   Used for numeric or counter metrics.
+  #   For example, the number of inquiries or, the amount purchased etc.
+  Log = Struct.new(:level, :thread_name, :name, :message, :payload, :time, :duration, :tags, :level_index, :exception, :metric, :backtrace, :metric_amount) do
+    MAX_EXCEPTIONS_TO_UNWRAP = 5
+    # Call the block for exception and any nested exception
+    def each_exception # rubocop:disable AbcSize, PerceivedComplexity, CyclomaticComplexity
+      # With thanks to https://github.com/bugsnag/bugsnag-ruby/blob/6348306e44323eee347896843d16c690cd7c4362/lib/bugsnag/notification.rb#L81
+      depth      = 0
+      exceptions = []
+      ex         = exception
+      while !ex.nil? && !exceptions.include?(ex) && exceptions.length < MAX_EXCEPTIONS_TO_UNWRAP
+        exceptions << ex
+        yield(ex, depth)
+
+        depth += 1
+        ex    =
+          if ex.respond_to?(:cause) && ex.cause
+            ex.cause
+          elsif ex.respond_to?(:continued_exception) && ex.continued_exception
+            ex.continued_exception
+          elsif ex.respond_to?(:original_exception) && ex.original_exception
+            ex.original_exception
+          end
+      end
+    end
+
+    # Returns [String] the exception backtrace including all of the child / caused by exceptions
+    def backtrace_to_s
+      trace = ""
+      each_exception do |exception, i|
+        if i == 0
+          trace = (exception.backtrace || []).join("\n")
+        else
+          trace << "\nCause: #{exception.class.name}: #{exception.message}\n#{(exception.backtrace || []).join("\n")}"
+        end
+      end
+      trace
+    end
+
+    # Returns [String] duration of the log entry as a string
+    # Returns nil if their is no duration
+    def duration_to_s
+      return unless duration
+      duration < 10.0 ? "#{"%.3f" % duration}ms" : "#{"%.1f" % duration}ms"
+    end
+
+    # Returns [String] the duration in human readable form
+    def duration_human # rubocop:disable AbcSize
+      return nil unless duration
+      seconds = duration / 1000
+      if seconds >= 86_400.0 # 1 day
+        "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime("%-Hh %-Mm")}"
+      elsif seconds >= 3600.0 # 1 hour
+        Time.at(seconds).strftime("%-Hh %-Mm")
+      elsif seconds >= 60.0 # 1 minute
+        Time.at(seconds).strftime("%-Mm %-Ss")
+      elsif seconds >= 1.0 # 1 second
+        "#{"%.3f" % seconds}s"
+      else
+        duration_to_s
+      end
+    end
+
+    # Returns [String] single character upper case log level
+    def level_to_s
+      level.to_s[0..0].upcase
+    end
+
+    # Returns [String] the available process info
+    # Example:
+    #    18934:thread 23 test_logging.rb:51
+    def process_info(thread_name_length = 30)
+      file, line = file_name_and_line(true)
+      file_name  = " #{file}:#{line}" if file
+
+      "#{$PROCESS_ID}:#{"%.#{thread_name_length}s" % thread_name}#{file_name}"
+    end
+
+    CALLER_REGEXP = /^(.*):(\d+).*/
+
+    # Extract the filename and line number from the last entry in the supplied backtrace
+    def extract_file_and_line(stack, short_name = false)
+      match = CALLER_REGEXP.match(stack.first)
+      [short_name ? File.basename(match[1]) : match[1], match[2].to_i]
+    end
+
+    # Returns [String, String] the file_name and line_number from the backtrace supplied
+    # in either the backtrace or exception
+    def file_name_and_line(short_name = false)
+      return unless backtrace || (exception && exception.backtrace)
+      stack = backtrace || exception.backtrace
+      extract_file_and_line(stack, short_name) if stack && stack.size > 0
+    end
+
+    # Strip the standard Rails colorizing from the logged message
+    def cleansed_message
+      message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, "").strip
+    end
+
+    # Return the payload in text form
+    # Returns nil if payload is missing or empty
+    def payload_to_s
+      payload.inspect if payload?
+    end
+
+    # Returns [true|false] whether the log entry has a payload
+    def payload?
+      !(payload.nil? || (payload.respond_to?(:empty?) && payload.empty?))
+    end
+
+    # Return the Time as a formatted string
+    # Ruby MRI supports micro seconds
+    # DEPRECATED
+    def formatted_time
+      "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%06d" % (time.usec)}"
+    end
+
+    # Returns [Hash] representation of this log entry
+    def to_h(host = Sapience.config.host, application = Sapience.config.application) # rubocop:disable AbcSize, CyclomaticComplexity, PerceivedComplexity, LineLength
+      # Header
+      h               = {
+        name:        name,
+        pid:         $PROCESS_ID,
+        thread:      thread_name,
+        time:        time,
+        level:       level,
+        level_index: level_index,
+      }
+      h[:host]        = host if host
+      h[:application] = application if application
+      file, line      = file_name_and_line
+      if file
+        h[:file] = file
+        h[:line] = line.to_i
+      end
+
+      # Tags
+      h[:tags] = tags if tags && (tags.size > 0)
+
+      # Duration
+      if duration
+        h[:duration_ms] = duration
+        h[:duration]    = duration_human
+      end
+
+      # Log message
+      h[:message] = cleansed_message if message
+
+      # Payload
+      if payload
+        if payload.is_a?(Hash)
+          h.merge!(payload)
+        else
+          h[:payload] = payload
+        end
+      end
+
+      # Exceptions
+      if exception
+        root = h
+        each_exception do |exception, i|
+          name       = i == 0 ? :exception : :cause
+          root[name] = {
+            name:        exception.class.name,
+            message:     exception.message,
+            stack_trace: exception.backtrace,
+          }
+          root       = root[name]
+        end
+      end
+
+      # Metric
+      h[:metric] = metric if metric
+      h
+    end
+  end
+
+end

data/lib/sapience/loggable.rb

@@ -0,0 +1,29 @@
+module Sapience
+  # rubocop:disable TrivialAccessors
+  module Loggable
+    def self.included(base)
+      base.class_eval do
+        # Returns [Sapience::Logger] class level logger
+        def self.logger
+          @sapience ||= Sapience[self]
+        end
+
+        # Replace instance class level logger
+        def self.logger=(logger)
+          @sapience = logger
+        end
+
+        # Returns [Sapience::Logger] instance level logger
+        def logger
+          @sapience ||= self.class.logger
+        end
+
+        # Replace instance level logger
+        def logger=(logger)
+          @sapience = logger
+        end
+      end
+    end
+  end
+  # rubocop:enable TrivialAccessors
+end

data/lib/sapience/logger.rb

@@ -0,0 +1,236 @@
+require "concurrent"
+
+# rubocop:disable ClassVars
+module Sapience
+  # Logger stores the class name to be used for all log messages so that every
+  # log message written by this instance will include the class name
+  class Logger < Base # rubocop:disable ClassLength, ClassVars
+    include Sapience::Concerns::Compatibility
+
+    # Returns a Logger instance
+    #
+    # Return the logger for a specific class, supports class specific log levels
+    #   logger = Sapience::Logger.new(self)
+    # OR
+    #   logger = Sapience::Logger.new('MyClass')
+    #
+    # Parameters:
+    #  application
+    #    A class, module or a string with the application/class name
+    #    to be used in the logger
+    #
+    #  level
+    #    The initial log level to start with for this logger instance
+    #    Default: Sapience.config.default_level
+    #
+    #  filter [Regexp|Proc]
+    #    RegExp: Only include log messages where the class name matches the supplied
+    #    regular expression. All other messages will be ignored
+    #    Proc: Only include log messages where the supplied Proc returns true
+    #          The Proc must return true or false
+    def initialize(klass, level = nil, filter = nil)
+      super
+    end
+
+    # Returns [Integer] the number of log entries that have not been written
+    # to the appenders
+    #
+    # When this number grows it is because the logging appender thread is not
+    # able to write to the appenders fast enough. Either reduce the amount of
+    # logging, increase the log level, reduce the number of appenders, or
+    # look into speeding up the appenders themselves
+    def self.queue_size
+      queue.size
+    end
+
+    # Flush all queued log entries disk, database, etc.
+    #  All queued log messages are written and then each appender is flushed in turn
+    def self.flush
+      msg = "Flushing appenders with #{queue_size} log messages on the queue"
+      if queue_size > 1_000
+        logger.warn msg
+      elsif queue_size > 100
+        logger.info msg
+      elsif queue_size > 0
+        logger.trace msg
+      end
+      process_request(:flush)
+    end
+
+    # Close all appenders and flush any outstanding messages
+    def self.close
+      msg = "Closing appenders with #{queue_size} log messages on the queue"
+      if queue_size > 1_000
+        logger.warn msg
+      elsif queue_size > 100
+        logger.info msg
+      elsif queue_size > 0
+        logger.trace msg
+      end
+      process_request(:close)
+    end
+
+    @@lag_check_interval = 5000
+    @@lag_threshold_s    = 30
+
+    # Returns the check_interval which is the number of messages between checks
+    # to determine if the appender thread is falling behind
+    def self.lag_check_interval
+      @@lag_check_interval
+    end
+
+    # Set the check_interval which is the number of messages between checks
+    # to determine if the appender thread is falling behind
+    def self.lag_check_interval=(lag_check_interval)
+      @@lag_check_interval = lag_check_interval
+    end
+
+    # Returns the amount of time in seconds
+    # to determine if the appender thread is falling behind
+    def self.lag_threshold_s
+      @@lag_threshold_s
+    end
+
+    # Allow the internal logger to be overridden from its default to STDERR
+    #   Can be replaced with another Ruby logger or Rails logger, but never to
+    #   Sapience::Logger itself since it is for reporting problems
+    #   while trying to log to the various appenders
+    def self.logger=(logger)
+      @@logger = logger
+    end
+
+    # Place log request on the queue for the Appender thread to write to each
+    # appender in the order that they were registered
+    def log(log, message = nil, progname = nil, &block)
+      # Compatibility with ::Logger
+      return add(log, message, progname, &block) unless log.is_a?(Sapience::Log)
+      self.class.queue << log if @@appender_thread
+    end
+
+    private
+
+    @@appender_thread    = nil
+    @@queue              = Queue.new
+
+    # Queue to hold messages that need to be logged to the various appenders
+    def self.queue
+      @@queue
+    end
+
+    # Internal logger for Sapience
+    #   For example when an appender is not working etc..
+    #   By default logs to STDERR
+    def self.logger
+      @@logger ||= begin
+        l      = Sapience::Appender::File.new(STDERR, :warn)
+        l.name = name
+        l
+      end
+    end
+
+    # Start the appender thread
+    def self.start_appender_thread
+      return false if appender_thread_active?
+      @@appender_thread = Thread.new { appender_thread }
+      fail "Failed to start Appender Thread" unless @@appender_thread
+      true
+    end
+
+    # Returns true if the appender_thread is active
+    def self.appender_thread_active?
+      @@appender_thread && @@appender_thread.alive?
+    end
+
+    # Separate appender thread responsible for reading log messages and
+    # calling the appenders in it's thread
+    # rubocop:disable BlockNesting, AssignmentInCondition, PerceivedComplexity, CyclomaticComplexity, AbcSize, LineLength, RescueException
+    def self.appender_thread
+      # This thread is designed to never go down unless the main thread terminates
+      # Before terminating at_exit is used to flush all the appenders
+      #
+      # Should any appender fail to log or flush, the exception is logged and
+      # other appenders will still be called
+      Thread.current.name = "Sapience::AppenderThread"
+      logger.trace "V#{VERSION} Appender thread active"
+      begin
+        count = 0
+        while message = queue.pop
+          if message.is_a?(Log)
+            Sapience.appenders.each do |appender|
+              begin
+                appender.log(message)
+              rescue Exception => exc
+                logger.error "Appender thread: Failed to log to appender: #{appender.inspect}", exc
+              end
+            end
+            count += 1
+            # Check every few log messages whether this appender thread is falling behind
+            if count > lag_check_interval
+              if (diff = Time.now - message.time) > lag_threshold_s
+                logger.warn "Appender thread has fallen behind by #{diff} seconds with #{queue_size} messages queued up. Consider reducing the log level or changing the appenders"
+              end
+              count = 0
+            end
+          else
+            case message[:command]
+            when :flush
+              Sapience.appenders.each do |appender|
+                begin
+                  logger.trace "Appender thread: Flushing appender: #{appender.name}"
+                  appender.flush
+                rescue Exception => exc
+                  logger.error "Appender thread: Failed to flush appender: #{appender.inspect}", exc
+                end
+              end
+
+              message[:reply_queue] << true if message[:reply_queue]
+              logger.trace "Appender thread: All appenders flushed"
+            when :close
+              Sapience.appenders.each do |appender|
+                begin
+                  logger.trace "Appender thread: Closing appender: #{appender.name}"
+                  appender.flush
+                  appender.close
+                  Sapience.remove_appender(appender)
+                rescue Exception => exc
+                  logger.error "Appender thread: Failed to close appender: #{appender.inspect}", exc
+                end
+              end
+
+              message[:reply_queue] << true if message[:reply_queue]
+              logger.trace "Appender thread: All appenders flushed"
+            else
+              logger.warn "Appender thread: Ignoring unknown command: #{message[:command]}"
+            end
+          end
+        end
+      rescue Exception => exception
+        # This block may be called after the file handles have been released by Ruby
+        begin
+          logger.error "Appender thread restarting due to exception", exception
+        rescue Exception
+          nil
+        end
+        retry
+      ensure
+        @@appender_thread = nil
+        # This block may be called after the file handles have been released by Ruby
+        begin
+          logger.trace "Appender thread has stopped"
+        rescue Exception
+          nil
+        end
+      end
+    end
+    # rubocop:enable BlockNesting, AssignmentInCondition, PerceivedComplexity, CyclomaticComplexity, AbcSize, LineLength, RescueException
+
+    # Close all appenders and flush any outstanding messages
+    def self.process_request(command)
+      return false unless appender_thread_active?
+
+      reply_queue = Queue.new
+      queue << { command: command, reply_queue: reply_queue }
+      reply_queue.pop
+    end
+  end
+end

data/lib/sapience/rails.rb

@@ -0,0 +1,15 @@
+module Sapience
+  class Rails < ::Rails::Engine
+    ::Rails::Application::Bootstrap.initializers.delete_if { |i| i.name == :initialize_logger }
+    initializer :initialize_logger, group: :all do
+      # TODO: Is this really needed?
+      # Existing loggers are ignored because servers like trinidad supply their
+      # own file loggers which would result in duplicate logging to the same log file
+      # ::Rails.logger = config.logger = begin
+      #   Sapience[::Rails]
+      # end
+
+      # Replace Rails loggers
+    end
+  end
+end