semantic_logger 3.2.1 → 3.3.0

data/lib/semantic_logger/appender/new_relic.rb

@@ -11,7 +11,7 @@ end
 #
 # Example:
 #   SemanticLogger.add_appender(appender: :new_relic)
-class SemanticLogger::Appender::NewRelic < SemanticLogger::Appender::Base
+class SemanticLogger::Appender::NewRelic < SemanticLogger::Subscriber
   # Create Appender
   #
   # Parameters
@@ -39,7 +39,7 @@ class SemanticLogger::Appender::NewRelic < SemanticLogger::Appender::Base
 
   # Returns [Hash] of parameters to send to New Relic.
   def call(log, logger)
-    h = log.to_h
+    h = log.to_h(host, application)
     h.delete(:time)
     h.delete(:exception)
     {metric: log.metric, custom_params: h}

data/lib/semantic_logger/appender/splunk.rb

@@ -18,8 +18,8 @@ end
 #     scheme:   :https,
 #     index:    'main'
 #   )
-class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
-  attr_reader :config, :index, :service, :service_index
+class SemanticLogger::Appender::Splunk < SemanticLogger::Subscriber
+  attr_reader :config, :index, :service, :service_index, :source_type
 
   # Write to Splunk.
   #
@@ -37,7 +37,7 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
   #     Not required if username and password are supplied.
   #
   #   :host [String]
-  #      Splunk host name.
+  #      Splunk server host name.
   #      Default: 'localhost'
   #
   #   :port [Integer]
@@ -61,6 +61,17 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
   #   :ssl_client_key [OpenSSL::PKey::RSA | OpenSSL::PKey::DSA]
   #     Client key.
   #
+  #   source_type: [String]
+  #     Optional: Source type to display in Splunk
+  #
+  #   application: [String]
+  #     The :source forwarded to Splunk
+  #     Default: SemanticLogger.application
+  #
+  #   host: [String]
+  #     Name of this host to appear in log messages.
+  #     Default: SemanticLogger.host
+  #
   #   level: [:trace | :debug | :info | :warn | :error | :fatal]
   #     Override the log level for this appender.
   #     Default: SemanticLogger.default_level
@@ -75,21 +86,16 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
   #     regular expression. All other messages will be ignored.
   #     Proc: Only include log messages where the supplied Proc returns true
   #           The Proc must return true or false.
-  def initialize(options, _deprecated_level = nil, &block)
+  def initialize(options = {}, _deprecated_level = nil, &block)
     @config         = options.dup
     @config[:level] = _deprecated_level if _deprecated_level
     @index          = @config.delete(:index) || 'main'
+    @source_type    = options.delete(:source_type)
 
-    options = {
-      level:     @config.delete(:level) || :error,
-      formatter: @config.delete(:formatter),
-      filter:    @config.delete(:filter)
-    }
-
-    reopen
-
+    options = extract_subscriber_options!(@config)
     # Pass on the level and custom formatter if supplied
     super(options, &block)
+    reopen
   end
 
   # After forking an active process call #reopen to re-open
@@ -105,20 +111,25 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
   # Log the message to Splunk
   def log(log)
     return false unless should_log?(log)
-
-    service_index.submit(log.message, formatter.call(log, self))
+    event = formatter.call(log, self)
+    service_index.submit(event.delete(:message), event)
     true
   end
 
-  # Returns [String] JSON to send to Splunk
+  # Returns [Hash] To send to Splunk
   # For splunk format requirements see:
   #   http://dev.splunk.com/view/event-collector/SP-CAAAE6P
-  def call(log, _logger)
-    h = log.to_h
-    h.delete(:message)
-    h.delete(:application)
-    h.delete(:host)
+  def call(log, logger)
+    h = log.to_h(nil, nil)
     h.delete(:time)
-    h
+    message               = {
+      source:  logger.application,
+      host:    logger.host,
+      time:    log.time.utc.to_f,
+      message: h.delete(:message),
+      event:   h
+    }
+    message[:source_type] = source_type if source_type
+    message
   end
 end

data/lib/semantic_logger/appender/splunk_http.rb

@@ -83,9 +83,7 @@ class SemanticLogger::Appender::SplunkHttp < SemanticLogger::Appender::Http
   # For splunk format requirements see:
   #   http://dev.splunk.com/view/event-collector/SP-CAAAE6P
   def call(log, logger)
-    h = log.to_h
-    h.delete(:application)
-    h.delete(:host)
+    h = log.to_h(nil, nil)
     h.delete(:time)
     message               = {
       source: logger.application,

data/lib/semantic_logger/appender/syslog.rb

@@ -20,9 +20,7 @@ require 'socket'
 #   )
 module SemanticLogger
   module Appender
-    class Syslog < SemanticLogger::Appender::Base
-
-      attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :host, :application
+    class Syslog < SemanticLogger::Subscriber
 
       # Default mapping of ruby log levels to syslog log levels
       #
@@ -42,6 +40,7 @@ module SemanticLogger
         debug: ::Syslog::LOG_INFO,
         trace: ::Syslog::LOG_DEBUG
       }
+      attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility
 
       # Create a Syslog appender instance.
       #
@@ -149,7 +148,6 @@ module SemanticLogger
       #     Default: :syslog
       def initialize(options = {}, &block)
         options             = options.dup
-        @application        = options.delete(:application) || options.delete(:ident) || 'ruby'
         @options            = options.delete(:options) || (::Syslog::LOG_PID | ::Syslog::LOG_CONS)
         @facility           = options.delete(:facility) || ::Syslog::LOG_USER
         level_map           = options.delete(:level_map)
@@ -159,7 +157,6 @@ module SemanticLogger
         @protocol           = (uri.scheme || :syslog).to_sym
         @port               = uri.port || 514
         @server             = 'localhost' if @protocol == :syslog
-        @host               = options.delete(:host) || options.delete(:local_hostname) || SemanticLogger.host
         @tcp_client_options = options.delete(:tcp_client)
 
         raise "Unknown protocol #{@protocol}!" unless [:syslog, :tcp, :udp].include?(@protocol)
@@ -187,9 +184,8 @@ module SemanticLogger
           end
         end
 
-        reopen
-
         super(options, &block)
+        reopen
       end
 
       # After forking an active process call #reopen to re-open
@@ -197,7 +193,7 @@ module SemanticLogger
       def reopen
         case @protocol
         when :syslog
-          ::Syslog.open(@application, @options, @facility)
+          ::Syslog.open(application, @options, @facility)
         when :tcp
           # Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
           @tcp_client_options[:logger] = SemanticLogger::Logger.logger
@@ -264,13 +260,32 @@ module SemanticLogger
         message
       end
 
+      private
+
+      # Extract Syslog formatter options
+      def format_options(options, protocol, &block)
+        opts      = options.delete(:options)
+        facility  = options.delete(:facility)
+        level_map = options.delete(:level_map)
+        if formatter = options.delete(:formatter)
+          extract_formatter(formatter)
+        else
+          case protocol
+          when :syslog
+            extract_formatter(syslog: {options: opts, facility: facility, level_map: level_map})
+          when :tcp, :udp
+            extract_formatter(syslog: {options: opts, facility: facility, level_map: level_map})
+          end
+        end
+      end
+
       # Format the syslog packet so it can be sent over TCP or UDP
       def syslog_packet_formatter(log)
         packet          = SyslogProtocol::Packet.new
-        packet.hostname = @host
+        packet.hostname = host
         packet.facility = @facility
         packet.severity = @level_map[log.level]
-        packet.tag      = @application
+        packet.tag      = application.gsub(' ', '')
         packet.content  = formatter.call(log, self)
         packet.time     = log.time
         packet.to_s

data/lib/semantic_logger/appender/tcp.rb

@@ -0,0 +1,231 @@
+begin
+  require 'net/tcp_client'
+rescue LoadError
+  raise 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
+end
+
+raise 'Net::TCPClient v2.0 or greater is required to log over TCP' unless Net::TCPClient::VERSION.to_f >= 2.0
+
+module SemanticLogger
+  module Appender
+    # TCP log appender.
+    #
+    # Log to a server over a TCP Socket.
+    # By default messages are in JSON format.
+    #
+    # Features:
+    # * JSON Formatted messages.
+    # * SSL encryption.
+    # * Transparently reconnect when a connection is lost.
+    #
+    # Example:
+    #   SemanticLogger.add_appender(
+    #     appender: :tcp,
+    #     server:   'server:3300',
+    #   )
+    #
+    # Example, with connection retry options:
+    #   SemanticLogger.add_appender(
+    #     appender:               :tcp,
+    #     server:                 'server:3300',
+    #     connect_retry_interval: 0.1,
+    #     connect_retry_count:    5
+    #   )
+    #
+    # Example, with SSL enabled:
+    #   SemanticLogger.add_appender(
+    #     appender: :tcp,
+    #     server:   'server:3300',
+    #     ssl:      true
+    #   )
+    #
+    class Tcp < SemanticLogger::Subscriber
+      attr_accessor :separator
+      attr_reader :tcp_client
+
+      # Create TCP log appender.
+      #
+      # Net::TCPClient Parameters:
+      #   :server [String]
+      #     URL of the server to connect to with port number
+      #     'localhost:2000'
+      #     '192.168.1.10:80'
+      #
+      #   :servers [Array of String]
+      #     Array of URL's of servers to connect to with port numbers
+      #     ['server1:2000', 'server2:2000']
+      #
+      #     The second server will only be attempted once the first server
+      #     cannot be connected to or has timed out on connect
+      #     A read failure or timeout will not result in switching to the second
+      #     server, only a connection failure or during an automatic reconnect
+      #
+      #   :connect_timeout [Float]
+      #     Time in seconds to timeout when trying to connect to the server
+      #     A value of -1 will cause the connect wait time to be infinite
+      #     Default: Half of the :read_timeout ( 30 seconds )
+      #
+      #   :read_timeout [Float]
+      #     Time in seconds to timeout on read
+      #     Can be overridden by supplying a timeout in the read call
+      #     Default: 60
+      #
+      #   :write_timeout [Float]
+      #     Time in seconds to timeout on write
+      #     Can be overridden by supplying a timeout in the write call
+      #     Default: 60
+      #
+      #   :log_level [Symbol]
+      #     Optional: Set the logging level for the TCPClient
+      #     Any valid SemanticLogger log level:
+      #       :trace, :debug, :info, :warn, :error, :fatal
+      #     Default: SemanticLogger.default_level
+      #
+      #   :buffered [Boolean]
+      #     Whether to use Nagle's Buffering algorithm (http://en.wikipedia.org/wiki/Nagle's_algorithm)
+      #     Recommend disabling for RPC style invocations where we don't want to wait for an
+      #     ACK from the server before sending the last partial segment
+      #     Buffering is recommended in a browser or file transfer style environment
+      #     where multiple sends are expected during a single response
+      #     Default: true
+      #
+      #   :connect_retry_count [Fixnum]
+      #     Number of times to retry connecting when a connection fails
+      #     Default: 10
+      #
+      #   :connect_retry_interval [Float]
+      #     Number of seconds between connection retry attempts after the first failed attempt
+      #     Default: 0.5
+      #
+      #   :retry_count [Fixnum]
+      #     Number of times to retry when calling #retry_on_connection_failure
+      #     This is independent of :connect_retry_count which still applies with
+      #     connection failures. This retry controls upto how many times to retry the
+      #     supplied block should a connection failure occurr during the block
+      #     Default: 3
+      #
+      #   :on_connect [Proc]
+      #     Directly after a connection is established and before it is made available
+      #     for use this Block is invoked.
+      #     Typical Use Cases:
+      #     - Initialize per connection session sequence numbers
+      #     - Pass any authentication information to the server
+      #     - Perform a handshake with the server
+      #
+      #   :policy [Symbol|Proc]
+      #     Specify the policy to use when connecting to servers.
+      #       :ordered
+      #         Select a server in the order supplied in the array, with the first
+      #         having the highest priority. The second server will only be connected
+      #         to if the first server is unreachable
+      #       :random
+      #         Randomly select a server from the list every time a connection
+      #         is established, including during automatic connection recovery.
+      #       :ping_time
+      #         FUTURE - Not implemented yet - Pull request anyone?
+      #         The server with the lowest ping time will be tried first
+      #       Proc:
+      #         When a Proc is supplied, it will be called passing in the list
+      #         of servers. The Proc must return one server name
+      #           Example:
+      #             :policy => Proc.new do |servers|
+      #               servers.last
+      #             end
+      #       Default: :ordered
+      #
+      #   :close_on_error [True|False]
+      #     To prevent the connection from going into an inconsistent state
+      #     automatically close the connection if an error occurs
+      #     This includes a Read Timeout
+      #     Default: true
+      #
+      # Appender Parameters:
+      #   separator: [String]
+      #     Separator between every message
+      #     Default: "\n"
+      #     Note: The separator should not be something that could be output in the formatted log message.
+      #
+      # Common Appender Parameters:
+      #   application: [String]
+      #     Name of this application to appear in log messages.
+      #     Default: SemanticLogger.application
+      #
+      #   host: [String]
+      #     Name of this host to appear in log messages.
+      #     Default: SemanticLogger.host
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   formatter: [Object|Proc]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: Use the built-in formatter (See: #call)
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      # Example:
+      #   SemanticLogger.add_appender(
+      #     appender: :tcp,
+      #     server:   'server:3300'
+      #   )
+      #
+      # Example, with connection retry options:
+      #   SemanticLogger.add_appender(
+      #     appender:               :tcp,
+      #     server:                 'server:3300',
+      #     connect_retry_interval: 0.1,
+      #     connect_retry_count:    5
+      #   )
+      def initialize(options = {}, &block)
+        @options                           = options.dup
+        @separator                         = @options.delete(:separator) || "\n"
+
+        # Use the internal logger so that errors with remote logging are only written locally.
+        Net::TCPClient.logger              = SemanticLogger::Logger.logger.dup
+        Net::TCPClient.logger.name         = 'Net::TCPClient'
+
+        options = extract_subscriber_options!(@options)
+        super(options, &block)
+        reopen
+      end
+
+      # After forking an active process call #reopen to re-open
+      # open the handles to resources
+      def reopen
+        close
+        @tcp_client = Net::TCPClient.new(@options)
+      end
+
+      # Write the log using the specified protocol and server.
+      def log(log)
+        return false unless should_log?(log)
+
+        @tcp_client.retry_on_connection_failure { @tcp_client.write("#{formatter.call(log, self)}#{separator}") }
+        true
+      end
+
+      # Flush is called by the semantic_logger during shutdown.
+      def flush
+        @tcp_client.flush if @tcp_client && @tcp_client.respond_to?(:flush)
+      end
+
+      # Close is called during shutdown, or with reopen
+      def close
+        @tcp_client.close if @tcp_client
+      end
+
+      private
+
+      # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
+      def default_formatter
+        SemanticLogger::Formatters::Json.new
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/appender/udp.rb

@@ -0,0 +1,106 @@
+require 'socket'
+module SemanticLogger
+  module Appender
+    # UDP log appender.
+    #
+    # Write log messages to UDP.
+    # By default messages are in JSON format.
+    #
+    # Example:
+    #   SemanticLogger.add_appender(
+    #     appender: :udp,
+    #     server:   'server:3300',
+    #   )
+    class Udp < SemanticLogger::Subscriber
+      attr_accessor :server, :udp_flags
+      attr_reader :socket
+
+      # Create UDP log appender.
+      #
+      #   server: [String]
+      #     URL of the server to write UDP messages to.
+      #
+      #   udp_flags: [Integer]
+      #     Should be a bitwise OR of Socket::MSG_* constants.
+      #     Default: 0
+      #
+      # Common Appender Parameters:
+      #   application: [String]
+      #     Name of this application to appear in log messages.
+      #     Default: SemanticLogger.application
+      #
+      #   host: [String]
+      #     Name of this host to appear in log messages.
+      #     Default: SemanticLogger.host
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   formatter: [Object|Proc]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: Use the built-in formatter (See: #call)
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      #
+      # Limitations:
+      # * UDP packet size is limited by the connected network and any routers etc
+      #   that the message has to traverse. See https://en.wikipedia.org/wiki/Maximum_transmission_unit
+      #
+      # Example:
+      #   SemanticLogger.add_appender(
+      #     appender: :udp,
+      #     server:   'server:3300'
+      #   )
+      def initialize(options = {}, &block)
+        options    = options.dup
+        @server    = options.delete(:server)
+        @udp_flags = options.delete(:udp_flags) || 0
+        raise(ArgumentError, 'Missing mandatory argument: :server') unless @server
+
+        super(options, &block)
+        reopen
+      end
+
+      # After forking an active process call #reopen to re-open
+      # open the handles to resources
+      def reopen
+        close
+        @socket    = UDPSocket.new
+        host, port = server.split(':')
+        @socket.connect(host, port.to_i)
+      end
+
+      # Write the log using the specified protocol and server.
+      def log(log)
+        return false unless should_log?(log)
+
+        @socket.send(formatter.call(log, self), udp_flags)
+        true
+      end
+
+      # Flush is called by the semantic_logger during shutdown.
+      def flush
+        @socket.flush if @socket
+      end
+
+      # Close is called during shutdown, or with reopen
+      def close
+        @socket.close if @socket
+      end
+
+      private
+
+      # Returns [SemanticLogger::Formatters::Default] formatter default for this Appender
+      def default_formatter
+        SemanticLogger::Formatters::Json.new
+      end
+
+    end
+  end
+end