semantic_logger 3.4.1 → 4.0.0.beta1

data/lib/semantic_logger/metrics/new_relic.rb

@@ -1,6 +1,6 @@
 module SemanticLogger
   module Metrics
-    class NewRelic
+    class NewRelic < Subscriber
       attr_accessor :prefix
 
       # Parameters:

data/lib/semantic_logger/metrics/statsd.rb

@@ -7,7 +7,7 @@ end
 
 module SemanticLogger
   module Metrics
-    class Statsd
+    class Statsd < Subscriber
       # Create Statsd metrics subscriber
       #
       # Parameters:
@@ -18,6 +18,10 @@ module SemanticLogger
       #     Example, send all metrics to a particular namespace:
       #       udp://localhost:8125/namespace
       #     Default: udp://localhost:8125
+      #
+      # Example:
+      #   subscriber = SemanticLogger::Metrics::Statsd.new(url: 'udp://localhost:8125')
+      #   SemanticLogger.on_metric(subscriber)
       def initialize(options = {})
         options = options.dup
         @url    = options.delete(:url) || 'udp://localhost:8125'

data/lib/semantic_logger/metrics/udp.rb

@@ -0,0 +1,80 @@
+require 'socket'
+module SemanticLogger
+  module Metrics
+    class Udp < Subscriber
+      attr_accessor :server, :separator, :udp_flags
+      attr_reader :socket
+
+      # Write metrics to in JSON format to Udp
+      #
+      # Parameters:
+      #   server: [String]
+      #     Host name and port to write UDP messages to
+      #     Example:
+      #       localhost:8125
+      #
+      #   udp_flags: [Integer]
+      #     Should be a bitwise OR of Socket::MSG_* constants.
+      #     Default: 0
+      #
+      # Limitations:
+      # * UDP packet size is limited by the connected network and any routers etc
+      #   that the message has to traverse. See https://en.wikipedia.org/wiki/Maximum_transmission_unit
+      #
+      # Example:
+      #   subscriber = SemanticLogger::Metrics::Udp.new(server: 'localhost:8125')
+      #   SemanticLogger.on_metric(subscriber)
+      def initialize(options = {}, &block)
+        options    = options.dup
+        @server    = options.delete(:server)
+        @udp_flags = options.delete(:udp_flags) || 0
+        raise(ArgumentError, 'Missing mandatory argument: :server') unless @server
+
+        super(options, &block)
+        reopen
+      end
+
+      # After forking an active process call #reopen to re-open
+      # open the handles to resources
+      def reopen
+        close
+        @socket    = UDPSocket.new
+        host, port = server.split(':')
+        @socket.connect(host, port)
+      end
+
+      def call(log)
+        metric = log.metric
+        if duration = log.duration
+          @statsd.timing(metric, duration)
+        else
+          amount = (log.metric_amount || 1).round
+          if amount < 0
+            amount.times { @statsd.decrement(metric) }
+          else
+            amount.times { @statsd.increment(metric) }
+          end
+        end
+        @socket.send(data, udp_flags)
+      end
+
+      # Flush is called by the semantic_logger during shutdown.
+      def flush
+        @socket.flush if @socket
+      end
+
+      # Close is called during shutdown, or with reopen
+      def close
+        @socket.close if @socket
+      end
+
+      private
+
+      # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
+      def default_formatter
+        SemanticLogger::Formatters::Json.new
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/processor.rb

@@ -0,0 +1,235 @@
+module SemanticLogger
+  # Thread that submits and processes log requests
+  class Processor
+    # Start the appender thread
+    def self.start
+      return false if active?
+      @@thread = Thread.new { process_requests }
+      raise 'Failed to start Appender Thread' unless @@thread
+      true
+    end
+
+    # Returns true if the appender_thread is active
+    def self.active?
+      @@thread && @@thread.alive?
+    end
+
+    # Returns [Integer] the number of log entries waiting to be written to the appenders
+    def self.queue_size
+      queue.size
+    end
+
+    # Add log request to the queue for processing
+    def self.<<(log)
+      queue << log if active?
+    end
+
+    # Submit command and wait for reply
+    def self.submit_request(command)
+      return false unless active?
+
+      msg = "Too many queued log messages: #{queue_size}, running command: #{command}"
+      if queue_size > 1_000
+        logger.warn msg
+      elsif queue_size > 100
+        logger.info msg
+      elsif queue_size > 0
+        logger.trace msg
+      end
+
+      reply_queue = Queue.new
+      queue << {command: command, reply_queue: reply_queue}
+      reply_queue.pop
+    end
+
+    # Allow the internal logger to be overridden from its default of STDERR
+    #   Can be replaced with another Ruby logger or Rails logger, but never to
+    #   SemanticLogger::Logger itself since it is for reporting problems
+    #   while trying to log to the various appenders
+    def self.logger=(logger)
+      @@logger = logger
+    end
+
+    # Returns the check_interval which is the number of messages between checks
+    # to determine if the appender thread is falling behind
+    def self.lag_check_interval
+      @@lag_check_interval
+    end
+
+    # Set the check_interval which is the number of messages between checks
+    # to determine if the appender thread is falling behind
+    def self.lag_check_interval=(lag_check_interval)
+      @@lag_check_interval = lag_check_interval
+    end
+
+    # Returns the amount of time in seconds
+    # to determine if the appender thread is falling behind
+    def self.lag_threshold_s
+      @@lag_threshold_s
+    end
+
+    # Supply a metrics appender to be called whenever a logging metric is encountered
+    #
+    #  Parameters
+    #    appender: [Symbol | Object | Proc]
+    #      [Proc] the block to call.
+    #      [Object] the block on which to call #call.
+    #      [Symbol] :new_relic, or :statsd to forward metrics to
+    #
+    #    block
+    #      The block to be called
+    #
+    # Example:
+    #   SemanticLogger.on_metric do |log|
+    #     puts "#{log.metric} was received. Log: #{log.inspect}"
+    #   end
+    #
+    # Note:
+    # * This callback is called in the logging thread.
+    # * Does not slow down the application.
+    # * Only context is what is passed in the log struct, the original thread context is not available.
+    def self.on_metric(options = {}, &block)
+      # Backward compatibility
+      options  = options.is_a?(Hash) ? options.dup : {appender: options}
+      appender = block || options.delete(:appender)
+
+      # Convert symbolized metrics appender to an actual object
+      appender = SemanticLogger::Appender.constantize_symbol(appender, 'SemanticLogger::Metrics').new(options) if appender.is_a?(Symbol)
+
+      raise('When supplying a metrics appender, it must support the #call method') unless appender.is_a?(Proc) || appender.respond_to?(:call)
+      (@@metric_subscribers ||= Concurrent::Array.new) << appender
+    end
+
+    private
+
+    @@thread             = nil
+    @@queue              = Queue.new
+    @@lag_check_interval = 5000
+    @@lag_threshold_s    = 30
+    @@metric_subscribers = nil
+
+    # Queue to hold messages that need to be logged to the various appenders
+    def self.queue
+      @@queue
+    end
+
+    # Internal logger for SemanticLogger
+    #   For example when an appender is not working etc..
+    #   By default logs to STDERR
+    def self.logger
+      @@logger ||= begin
+        l      = SemanticLogger::Appender::File.new(STDERR, :warn)
+        l.name = name
+        l
+      end
+    end
+
+    # Separate appender thread responsible for reading log messages and
+    # calling the appenders in it's thread
+    def self.process_requests
+      # This thread is designed to never go down unless the main thread terminates
+      # Before terminating at_exit is used to flush all the appenders
+      #
+      # Should any appender fail to log or flush, the exception is logged and
+      # other appenders will still be called
+      Thread.current.name = 'SemanticLogger::Processor'
+      logger.trace "V#{VERSION} Processor thread active"
+      begin
+        count = 0
+        while message = queue.pop
+          if message.is_a?(Log)
+            call_appenders(message)
+            call_metric_subscribers(message) if message.metric
+            count += 1
+            # Check every few log messages whether this appender thread is falling behind
+            if count > lag_check_interval
+              if (diff = Time.now - message.time) > lag_threshold_s
+                logger.warn "Appender thread has fallen behind by #{diff} seconds with #{queue_size} messages queued up. Consider reducing the log level or changing the appenders"
+              end
+              count = 0
+            end
+          else
+            case message[:command]
+            when :flush
+              flush_appenders
+              message[:reply_queue] << true if message[:reply_queue]
+            when :close
+              close_appenders
+              message[:reply_queue] << true if message[:reply_queue]
+            else
+              logger.warn "Appender thread: Ignoring unknown command: #{message[:command]}"
+            end
+          end
+        end
+      rescue Exception => exception
+        # This block may be called after the file handles have been released by Ruby
+        begin
+          logger.error 'Appender thread restarting due to exception', exception
+        rescue Exception
+          nil
+        end
+        retry
+      ensure
+        @@thread = nil
+        # This block may be called after the file handles have been released by Ruby
+        begin
+          logger.trace 'Appender thread has stopped'
+        rescue Exception
+          nil
+        end
+      end
+    end
+
+    # Call Metric subscribers
+    def self.call_metric_subscribers(log)
+      # If no subscribers registered, then return immediately
+      return unless @@metric_subscribers
+
+      @@metric_subscribers.each do |subscriber|
+        begin
+          subscriber.call(log)
+        rescue Exception => exc
+          logger.error 'Exception calling metrics subscriber', exc
+        end
+      end
+    end
+
+    # Call Appenders
+    def self.call_appenders(log)
+      SemanticLogger.appenders.each do |appender|
+        begin
+          appender.log(log)
+        rescue Exception => exc
+          logger.error "Appender thread: Failed to log to appender: #{appender.inspect}", exc
+        end
+      end
+    end
+
+    def self.flush_appenders
+      SemanticLogger.appenders.each do |appender|
+        begin
+          logger.trace "Appender thread: Flushing appender: #{appender.name}"
+          appender.flush
+        rescue Exception => exc
+          logger.error "Appender thread: Failed to flush appender: #{appender.inspect}", exc
+        end
+      end
+      logger.trace 'Appender thread: All appenders flushed'
+    end
+
+    def self.close_appenders
+      SemanticLogger.appenders.each do |appender|
+        begin
+          logger.trace "Appender thread: Closing appender: #{appender.name}"
+          appender.flush
+          appender.close
+          SemanticLogger.remove_appender(appender)
+        rescue Exception => exc
+          logger.error "Appender thread: Failed to close appender: #{appender.inspect}", exc
+        end
+      end
+      logger.trace 'Appender thread: All appenders closed and removed from appender list'
+    end
+
+  end
+end

data/lib/semantic_logger/semantic_logger.rb

@@ -153,11 +153,11 @@ module SemanticLogger
   #   logger.debug("Login time", user: 'Joe', duration: 100, ip_address: '127.0.0.1')
   def self.add_appender(options, deprecated_level = nil, &block)
     options  = options.is_a?(Hash) ? options.dup : convert_old_appender_args(options, deprecated_level)
-    appender = appender_from_options(options, &block)
+    appender = SemanticLogger::Appender.create(options, &block)
     @@appenders << appender
 
     # Start appender thread if it is not already running
-    SemanticLogger::Logger.start_appender_thread
+    SemanticLogger::Processor.start
     appender
   end
 
@@ -193,7 +193,7 @@ module SemanticLogger
   def self.reopen
     @@appenders.each { |appender| appender.reopen if appender.respond_to?(:reopen) }
     # After a fork the appender thread is not running, start it if it is not running
-    SemanticLogger::Logger.start_appender_thread
+    SemanticLogger::Processor.start
   end
 
   # Supply a block to be called whenever a metric is seen during measure logging
@@ -209,7 +209,7 @@ module SemanticLogger
   #
   # Example:
   #   SemanticLogger.on_metric do |log|
-  #     puts "#{log.metric} was received. Log Struct: #{log.inspect}"
+  #     puts "#{log.metric} was received. Log: #{log.inspect}"
   #   end
   #
   # Note:
@@ -217,7 +217,7 @@ module SemanticLogger
   # * Does not slow down the application.
   # * Only context is what is passed in the log struct, the original thread context is not available.
   def self.on_metric(options = {}, &block)
-    SemanticLogger::Logger.on_metric(options, &block)
+    SemanticLogger::Processor.on_metric(options, &block)
   end
 
   # Add signal handlers for Semantic Logger
@@ -267,14 +267,7 @@ module SemanticLogger
       logger = SemanticLogger['Thread Dump']
       Thread.list.each do |thread|
         next if thread == Thread.current
-        message = thread.name
-        if backtrace = thread.backtrace
-          message += "\n"
-          message << backtrace.join("\n")
-        end
-        tags = thread[:semantic_logger_tags]
-        tags = tags.nil? ? [] : tags.clone
-        logger.tagged(tags) { logger.warn(message) }
+        logger.backtrace(thread: thread)
       end
     end if thread_dump_signal
 
@@ -297,19 +290,6 @@ module SemanticLogger
     Thread.current[:semantic_logger_tags].pop
   end
 
-  # Add the supplied named tags to the list of tags to log for this thread whilst
-  # the supplied block is active.
-  #
-  # Returns result of block
-  #
-  # Example:
-  def self.named_tags(tag)
-    (Thread.current[:semantic_logger_tags] ||= []) << tag
-    yield
-  ensure
-    Thread.current[:semantic_logger_tags].pop
-  end
-
   # Add the supplied tags to the list of tags to log for this thread whilst
   # the supplied block is active.
   # Returns result of block
@@ -344,6 +324,36 @@ module SemanticLogger
     t.pop(quantity) unless t.nil?
   end
 
+  # Add the supplied named tags to the list of tags to log for this thread whilst
+  # the supplied block is active.
+  #
+  # Add a payload to all log calls on This Thread within the supplied block
+  #
+  #   SemanticLogger.named_tagged(tracking_number: 12345) do
+  #     logger.debug('Hello World')
+  #   end
+  #
+  # Returns result of block
+  def self.named_tagged(hash)
+    raise(ArgumentError, '#named_tagged only accepts named parameters (Hash)') unless hash.is_a?(Hash)
+    (Thread.current[:semantic_logger_named_tags] ||= []) << hash
+    yield
+  ensure
+    Thread.current[:semantic_logger_named_tags].pop
+  end
+
+  # Returns a copy of the [Hash] of named tags currently active for this thread
+  # Returns nil if no named tags are set
+  def self.named_tags
+    if (list = Thread.current[:semantic_logger_named_tags]) && !list.empty?
+      if list.size > 1
+        list.inject({}) { |h, sum| sum.merge(h) }
+      else
+        list.first.clone
+      end
+    end
+  end
+
   # Silence noisy log levels by changing the default_level within the block
   #
   # This setting is thread-safe and only applies to the current thread
@@ -445,45 +455,6 @@ module SemanticLogger
     options
   end
 
-  # Returns [SemanticLogger::Subscriber] appender for the supplied options
-  def self.appender_from_options(options, &block)
-    if options[:io] || options[:file_name]
-      SemanticLogger::Appender::File.new(options, &block)
-    elsif appender = options.delete(:appender)
-      if appender.is_a?(Symbol)
-        constantize_symbol(appender).new(options)
-      elsif appender.is_a?(Subscriber)
-        appender
-      else
-        raise(ArgumentError, "Parameter :appender must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
-      end
-    elsif options[:logger]
-      SemanticLogger::Appender::Wrapper.new(options, &block)
-    end
-  end
-
-  def self.constantize_symbol(symbol, namespace = 'SemanticLogger::Appender')
-    klass = "#{namespace}::#{camelize(symbol.to_s)}"
-    begin
-      if RUBY_VERSION.to_i >= 2
-        Object.const_get(klass)
-      else
-        klass.split('::').inject(Object) { |o, name| o.const_get(name) }
-      end
-    rescue NameError
-      raise(ArgumentError, "Could not convert symbol: #{symbol} to a class in: #{namespace}. Looking for: #{klass}")
-    end
-  end
-
-  # Borrow from Rails, when not running Rails
-  def self.camelize(term)
-    string = term.to_s
-    string = string.sub(/^[a-z\d]*/) { |match| match.capitalize }
-    string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
-    string.gsub!('/'.freeze, '::'.freeze)
-    string
-  end
-
   # Initial default Level for all new instances of SemanticLogger::Logger
   @@default_level         = :info
   @@default_level_index   = level_to_index(@@default_level)