semantic_logger 2.21.0 → 3.0.0

data/lib/semantic_logger/appender/splunk.rb

@@ -1,12 +1,17 @@
-require 'splunk-sdk-ruby'
+begin
+  require 'splunk-sdk-ruby'
+rescue LoadError
+  raise 'Gem splunk-sdk-ruby is required for logging to Splunk. Please add the gem "splunk-sdk-ruby" to your Gemfile.'
+end
 
-# Note: Not recommended to use the colorized formatter.
+# Note: This appender is Deprecated. Use: SemanticLogger::Appender::SplunkHttp
 class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
   attr_reader :config, :index, :service, :service_index
 
-  # Allow the level for this appender to be overwritten
-  #   Default: :error
+  # DEPRECATED, Please use SemanticLogger::Appender::SplunkHttp
   def initialize(options, level=:error, &block)
+    Kernel.warn('Splunk Appender is deprecated, please use SemanticLogger::Appender::SplunkHttp')
+
     # Parse input options for setting up splunk connection
     parse_options(options)
 
@@ -31,7 +36,7 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Appender::Base
     # Ensure minimum log level is met, and check filter
     return false if (level_index > (log.level_index || 0)) || !include_message?(log)
     # Submit the log message
-    @service_index.submit(formatter.call(log))
+    @service_index.submit(formatter.call(log, self))
     true
   end
 

data/lib/semantic_logger/appender/splunk_http.rb

@@ -0,0 +1,99 @@
+# Splunk log appender.
+#
+# Use the newer, faster and more complete JSON over HTTP interface for Splunk.
+#
+# To configure Splunk to receive log messages via this appender:
+#   http://dev.splunk.com/view/event-collector/SP-CAAAE7F
+#
+# Example
+#   appender = SemanticLogger::Appender::SplunkHttp.new(
+#     url:   'http://localhost:8080',
+#     token: '70CA900C-3D7E-42A4-9C79-7975D1C422A8'
+#   )
+#   SemanticLogger.add_appender(appender)
+class SemanticLogger::Appender::SplunkHttp < SemanticLogger::Appender::Http
+  # Create Splunk appender over persistent HTTP(S)
+  #
+  # Parameters:
+  #   token: [String]
+  #     Token created in Splunk for this HTTP Appender
+  #     Mandatory.
+  #
+  #   source_type: [String]
+  #     Optional: Source type to display in Splunk
+  #
+  #   index: [String]
+  #     Optional: Name of a valid index for this message in Splunk.
+  #
+  #   url: [String]
+  #     Valid URL to post to.
+  #       Example: http://example.com
+  #     To enable SSL include https in the URL.
+  #       Example: https://example.com
+  #       verify_mode will default: OpenSSL::SSL::VERIFY_PEER
+  #
+  #   application: [String]
+  #     Name of this application to appear in log messages.
+  #     Default: SemanticLogger.application
+  #
+  #   host: [String]
+  #     Name of this host to appear in log messages.
+  #     Default: SemanticLogger.host
+  #
+  #   compress: [true|false]
+  #     Whether to compress the JSON string with GZip.
+  #     Default: true
+  #
+  #   ssl: [Hash]
+  #     Specific SSL options: For more details see NET::HTTP.start
+  #       ca_file, ca_path, cert, cert_store, ciphers, key, open_timeout, read_timeout, ssl_timeout,
+  #       ssl_version, use_ssl, verify_callback, verify_depth and verify_mode.
+  #
+  #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+  #     Override the log level for this appender.
+  #     Default: SemanticLogger.default_level
+  #
+  #   filter: [Regexp|Proc]
+  #     RegExp: Only include log messages where the class name matches the supplied.
+  #     regular expression. All other messages will be ignored.
+  #     Proc: Only include log messages where the supplied Proc returns true
+  #           The Proc must return true or false.
+  def initialize(options, &block)
+    options      = options.dup
+    @source_type = options.delete(:source_type)
+    @index       = options.delete(:index)
+    token        = options.delete(:token)
+    raise(ArgumentError, 'Missing mandatory parameter :token') unless token
+
+    # Splunk supports HTTP Compression, enable by default
+    options[:compress] = true unless options.key?(:compress)
+
+    super(options, &block)
+
+    @header['Authorization'] = "Splunk #{token}"
+  end
+
+  # Returns [String] JSON to send to Splunk
+  # For splunk format requirements see:
+  #   http://dev.splunk.com/view/event-collector/SP-CAAAE6P
+  def default_formatter
+    Proc.new do |log, logger|
+      h = log.to_h
+      h.delete(:application)
+      h.delete(:host)
+      h.delete(:time)
+      message               = {
+        source: logger.application,
+        host:   logger.host,
+        time:   log.time.utc.to_f,
+        event:  h
+      }
+      message[:source_type] = @source_type if @source_type
+      message[:index]       = @index if @index
+
+      # Render to JSON
+      message.to_json
+    end
+  end
+
+end

data/lib/semantic_logger/appender/syslog.rb

@@ -1,41 +1,36 @@
-# syslog appender for SemanticLogger - Supports local and remote syslog (over TCP or UDP)
-#
-# Example 1
-# Log to the local syslog.
-#
-# require 'semantic_logger'
-# SemanticLogger.default_level = :trace
+require 'syslog'
+require 'uri'
+require 'socket'
+
+# Send log messages to local syslog, or remote syslog servers over TCP or UDP.
 #
-# syslog_appender = SemanticLogger::Appender::Syslog.new
-# SemanticLogger.add_appender(syslog_appender)
+# Example: Log to a local Syslog daemon
+#   SemanticLogger.add_appender(SemanticLogger::Appender::Syslog.new)
 #
-# logger = SemanticLogger['SyslogAppenderExample']
-# logger.info  "Info Hello! - This message should appear in the local syslog!"
+# Example: Log to a remote Syslog server using TCP:
+#   appender        = SemanticLogger::Appender::Syslog.new(
+#     url: 'tcp://myloghost:514'
+#   )
 #
+#   # Optional: Add filter to exclude health_check, or other log entries
+#   appender.filter = Proc.new { |log| log.message !~ /(health_check|Not logged in)/ }
 #
-# Example 2
-# Send to a remote syslog appender - myloghost - over TCP on port 514.
-# Tested with syslog-ng as part of an ELSA installation.
-# https://code.google.com/p/enterprise-log-search-and-archive/
+#   SemanticLogger.add_appender(appender)
 #
-# require 'semantic_logger'
-# # Only log warn and above messages to the remote syslog.
-# syslog_appender = SemanticLogger::Appender::Syslog.new(level: :warn, server: 'tcp://myloghost:514')
-# SemanticLogger.add_appender(syslog_appender)
+# Example: Log to a remote Syslog server using UDP:
+#   appender        = SemanticLogger::Appender::Syslog.new(
+#     url: 'udp://myloghost:514'
+#   )
 #
-# logger = SemanticLogger['SyslogAppenderExample']
-# logger.info  "Info Hello!   - The log level is too low and will not be logged."
-# logger.error "Error! Error! - This message should appear in the remote syslog!"
+#   # Optional: Add filter to exclude health_check, or other log entries
+#   appender.filter = Proc.new { |log| log.message !~ /(health_check|Not logged in)/ }
 #
-require 'syslog'
-require 'uri'
-require 'socket'
-
+#   SemanticLogger.add_appender(appender)
 module SemanticLogger
   module Appender
     class Syslog < SemanticLogger::Appender::Base
 
-      attr_reader :remote_syslog, :server, :host, :port, :protocol, :facility, :local_hostname
+      attr_reader :remote_syslog, :url, :server, :port, :protocol, :facility, :host, :application
 
       # Default mapping of ruby log levels to syslog log levels
       #
@@ -56,14 +51,47 @@ module SemanticLogger
         trace: ::Syslog::LOG_DEBUG
       }
 
-      # For more information on the Syslog constants used below see http://ruby-doc.org/stdlib-2.0.0/libdoc/syslog/rdoc/Syslog.html
+      # Create a Syslog appender instance.
+      #
       # Parameters
+      #   url: [String]
+      #     Default: 'syslog://localhost'
+      #     For writing logs to a remote syslog server
+      #     URL of server: protocol://host:port
+      #     Uses port 514 by default for TCP and UDP.
+      #     local syslog example:          'syslog://localhost'
+      #     TCP example with default port: 'tcp://logger'
+      #     TCP example with custom port:  'tcp://logger:8514'
+      #     UDP example with default port: 'udp://logger'
+      #     UDP example with custom port:  'udp://logger:8514'
+      #     When using the :syslog protocol, logs will always be sent to the localhost syslog
       #
-      #   :ident [String]
-      #     Identity of the program
-      #     Default: 'ruby'
+      #   host: [String]
+      #     Host name to provide to the remote syslog.
+      #     Default: SemanticLogger.host
       #
-      #   :options [Integer]
+      #   tcp_client: [Hash]
+      #     Default: {}
+      #     Only used with the TCP protocol.
+      #     Specify custom parameters to pass into Net::TCPClient.new
+      #     For a list of options see the net_tcp_client documentation:
+      #       https://www.omniref.com/ruby/gems/net_tcp_client/1.0.0/symbols/Net::TCPClient/initialize
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      #
+      #   application: [String]
+      #     Identity of the program.
+      #     Default: SemanticLogger.application
+      #
+      #   options: [Integer]
       #     Default: ::Syslog::LOG_PID | ::Syslog::LOG_CONS
       #     Any of the following (options can be logically OR'd together)
       #       ::Syslog::LOG_CONS
@@ -73,7 +101,7 @@ module SemanticLogger
       #       ::Syslog::LOG_PERROR
       #       ::Syslog::LOG_PID
       #
-      #   :facility [Integer]
+      #   facility: [Integer]
       #     Default: ::Syslog::LOG_USER
       #     Type of program (can be logically OR'd together)
       #       ::Syslog::LOG_AUTH
@@ -100,11 +128,7 @@ module SemanticLogger
       #       ::Syslog::LOG_LOCAL6
       #       ::Syslog::LOG_LOCAL7
       #
-      #   :level [Symbol]
-      #     Default: SemanticLogger's log level.
-      #     The minimum level at which this appender will write logs. Any log messages below this level will be ignored.
-      #
-      #   :level_map [Hash]
+      #   level_map: [Hash]
       #     Supply a custom map of SemanticLogger levels to syslog levels.
       #     For example, passing in { warn: ::Syslog::LOG_NOTICE }
       #       would result in a log mapping that matches the default level map,
@@ -119,51 +143,28 @@ module SemanticLogger
       #         debug:   ::Syslog::LOG_INFO,
       #         trace:   ::Syslog::LOG_DEBUG
       #       }
-      #
-      #   :local_hostname [String]
-      #     Default: Socket.gethostname || `hostname`.strip
-      #     Hostname to provide to the remote syslog.
-      #
-      #   :server [String]
-      #     Default: 'syslog://localhost'
-      #     For writing logs to a remote syslog server
-      #     URI of server: protocol://host:port
-      #     Uses port 514 by default for TCP and UDP.
-      #     local syslog example:          'syslog://localhost'
-      #     TCP example with default port: 'tcp://logger'
-      #     TCP example with custom port:  'tcp://logger:8514'
-      #     UDP example with default port: 'udp://logger'
-      #     UDP example with custom port:  'udp://logger:8514'
-      #     When using the :syslog protocol, logs will always be sent to the localhost syslog
-      #
-      #   :tcp_client [Hash]
-      #     Default: {}
-      #     Only used with the TCP protocol.
-      #     Specify custom parameters to pass into Net::TCPClient.new
-      #     For a list of options see the net_tcp_client documentation:
-      #       https://www.omniref.com/ruby/gems/net_tcp_client/1.0.0/symbols/Net::TCPClient/initialize
-      def initialize(params = {}, &block)
-        params     = params.dup
-        @ident     = params.delete(:ident) || 'ruby'
-        @options   = params.delete(:options) || (::Syslog::LOG_PID | ::Syslog::LOG_CONS)
-        @facility  = params.delete(:facility) || ::Syslog::LOG_USER
-        filter     = params.delete(:filter)
-        level      = params.delete(:level)
-        level_map  = params.delete(:level_map)
-        @level_map = DEFAULT_LEVEL_MAP.dup
-        @level_map.update(level_map) if level_map
-        @server   = params.delete(:server) || 'syslog://localhost'
-        uri       = URI(@server)
-        @host     = uri.host || 'localhost'
-        @protocol = (uri.scheme || :syslog).to_sym
+      def initialize(options = {}, &block)
+        options             = options.dup
+        level               = options.delete(:level)
+        filter              = options.delete(:filter)
+        @application        = options.delete(:application) || options.delete(:ident) || 'ruby'
+        @options            = options.delete(:options) || (::Syslog::LOG_PID | ::Syslog::LOG_CONS)
+        @facility           = options.delete(:facility) || ::Syslog::LOG_USER
+        level_map           = options.delete(:level_map)
+        @url                = options.delete(:url) || options.delete(:server) || 'syslog://localhost'
+        uri                 = URI(@url)
+        @server             = uri.host || 'localhost'
+        @protocol           = (uri.scheme || :syslog).to_sym
+        @port               = uri.port || 514
+        @server             = 'localhost' if @protocol == :syslog
+        @host               = options.delete(:host) || options.delete(:local_hostname) || SemanticLogger.host
+        @tcp_client_options = options.delete(:tcp_client)
+
         raise "Unknown protocol #{@protocol}!" unless [:syslog, :tcp, :udp].include?(@protocol)
-        @host               = 'localhost' if @protocol == :syslog
-        @port               = URI(@server).port || 514
-        @local_hostname     = params.delete(:local_hostname) || Socket.gethostname || `hostname`.strip
-        @tcp_client_options = params.delete(:tcp_client)
+        raise(ArgumentError, "Unknown options: #{options.inspect}") if options.size > 0
 
-        # Warn about any unknown configuration options.
-        params.each_pair { |key, val| SemanticLogger::Logger.logger.warn "Ignoring unknown configuration option: #{key.inspect} => #{val.inspect}" }
+        @level_map = DEFAULT_LEVEL_MAP.dup
+        @level_map.update(level_map) if level_map
 
         # The syslog_protocol gem is required when logging over TCP or UDP.
         if [:tcp, :udp].include?(@protocol)
@@ -176,7 +177,7 @@ module SemanticLogger
           # The net_tcp_client gem is required when logging over TCP.
           if protocol == :tcp
             @tcp_client_options          ||= {}
-            @tcp_client_options[:server] = "#{@host}:#{@port}"
+            @tcp_client_options[:server] = "#{@server}:#{@port}"
             begin
               require 'net/tcp_client'
             rescue LoadError
@@ -195,7 +196,7 @@ module SemanticLogger
       def reopen
         case @protocol
         when :syslog
-          ::Syslog.open(@ident, @options, @facility)
+          ::Syslog.open(@application, @options, @facility)
         when :tcp
           # Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
           @tcp_client_options[:logger] = SemanticLogger::Logger.logger
@@ -207,7 +208,7 @@ module SemanticLogger
         end
       end
 
-      # Write the log using the specified protocol and host.
+      # Write the log using the specified protocol and server.
       def log(log)
         # Ensure minimum log level is met, and check filter
         return false if (level_index > (log.level_index || 0)) || !include_message?(log)
@@ -215,12 +216,12 @@ module SemanticLogger
         case @protocol
         when :syslog
           # Since the Ruby Syslog API supports sprintf format strings, double up all existing '%'
-          message = formatter.call(log).gsub '%', '%%'
+          message = formatter.call(log, self).gsub '%', '%%'
           ::Syslog.log @level_map[log.level], message
         when :tcp
           @remote_syslog.retry_on_connection_failure { @remote_syslog.write("#{syslog_packet_formatter(log)}\r\n") }
         when :udp
-          @remote_syslog.send syslog_packet_formatter(log), 0, @host, @port
+          @remote_syslog.send syslog_packet_formatter(log), 0, @server, @port
         else
           raise "Unsupported protocol: #{protocol}"
         end
@@ -232,36 +233,48 @@ module SemanticLogger
         @remote_syslog.flush if @remote_syslog && @remote_syslog.respond_to?(:flush)
       end
 
-      # Custom log formatter for syslog
+      # Custom log formatter for syslog.
+      # Only difference is the removal of the timestamp string since it is in the syslog packet.
       def default_formatter
         Proc.new do |log|
-          tags = log.tags.collect { |tag| "[#{tag}]" }.join(" ") + " " if log.tags && (log.tags.size > 0)
+          # Header with date, time, log level and process info
+          entry = "#{log.level_to_s} [#{log.process_info}]"
 
-          message = log.message.to_s
-          message << ' -- ' << log.payload.inspect if log.payload
-          log.each_exception do |exception, i|
-            if i == 0
-              message << ' -- '
-            else
-              message << "\nCause: "
-            end
-            message << "#{exception.class}: #{exception.message}\n#{(exception.backtrace || []).join("\n")}"
-          end
+          # Tags
+          entry << ' ' << log.tags.collect { |tag| "[#{tag}]" }.join(' ') if log.tags && (log.tags.size > 0)
+
+          # Duration
+          entry << " (#{log.duration_human})" if log.duration
+
+          # Class / app name
+          entry << " #{log.name}"
 
-          duration_str = log.duration ? "(#{'%.1f' % log.duration}ms) " : ''
+          # Log message
+          entry << " -- #{log.message}" if log.message
 
-          "#{log.level.to_s[0..0].upcase} [#{$$}:#{log.thread_name}] #{tags}#{duration_str}#{log.name} -- #{message}"
+          # Payload
+          if payload = log.payload_to_s(false)
+            entry << ' -- ' << payload
+          end
+
+          # Exceptions
+          if log.exception
+            entry << " -- Exception: #{log.exception.class}: #{log.exception.message}\n"
+            entry << log.backtrace_to_s
+          end
+          entry
         end
       end
 
       # Format the syslog packet so it can be sent over TCP or UDP
       def syslog_packet_formatter(log)
         packet          = SyslogProtocol::Packet.new
-        packet.hostname = @local_hostname
+        packet.hostname = @host
         packet.facility = @facility
         packet.severity = @level_map[log.level]
-        packet.tag      = @ident
-        packet.content  = default_formatter.call(log)
+        packet.tag      = @application
+        packet.content  = default_formatter.call(log, self)
+        packet.time     = log.time
         packet.to_s
       end
     end

data/lib/semantic_logger/appender/wrapper.rb

@@ -1,19 +1,35 @@
-# Wrapper appender
+# Send log messages to any standard Ruby logging class.
 #
-#   Wraps the Rails log, log4r, or Ruby Logger with the SemanticLogger API's
+#  Forwards logging call to loggers such as Logger, log4r, etc.
 #
 module SemanticLogger
   module Appender
     class Wrapper < SemanticLogger::Appender::Base
       attr_reader :logger
 
-      # Create a Logger or Rails Logger appender instance
+      # Forward all logging calls to the supplied logging instance.
+      #
+      # Parameters
+      #   logger: [Object]
+      #     Instance of an existing logger conforming to the Ruby Logger methods.
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
       #
       # Ruby Logger
       #    require 'logger'
       #    require 'semantic_logger'
+      #
       #    ruby_logger = Logger.new(STDOUT)
       #    SemanticLogger.add_appender(ruby_logger)
+      #
       #    logger =  SemanticLogger['test']
       #    logger.info('Hello World', some: :payload)
       #
@@ -25,26 +41,25 @@ module SemanticLogger
       #    # Make ActiveRecord logging include its class name in every log entry
       #    ActiveRecord::Base.logger = SemanticLogger['ActiveRecord']
       #
-      # Note: Since the log level is controlled by setting the Ruby or Rails logger directly
-      #   the level is ignored for this appender
-      def initialize(logger, filter=nil, &block)
-        raise 'logger cannot be null when initiailizing the SemanticLogging::Appender::Wrapper' unless logger
+      # Install the `rails_semantic_logger` gem to replace the Rails logger with Semantic Logger.
+      def initialize(logger, level = nil, filter = nil, &block)
+        raise 'logger cannot be null when initializing the SemanticLogging::Appender::Wrapper' unless logger
         @logger    = logger
 
         # Set the formatter to the supplied block
         @formatter = block || self.default_formatter
-        super(nil, filter, &block)
+        super(level, filter, &block)
       end
 
       # Pass log calls to the underlying Rails, log4j or Ruby logger
       #  trace entries are mapped to debug since :trace is not supported by the
       #  Ruby or Rails Loggers
       def log(log)
-        # Check filter
-        return false unless include_message?(log)
+        # Ensure minimum log level is met, and check filter
+        return false if (level_index > (log.level_index || 0)) || !include_message?(log)
 
         # Underlying wrapper logger implements log level, so don't check here
-        @logger.send(log.level == :trace ? :debug : log.level, @formatter.call(log))
+        @logger.send(log.level == :trace ? :debug : log.level, @formatter.call(log, self))
         true
       end