semantic_logger 2.21.0 → 3.0.0

data/lib/semantic_logger/appender/http.rb

@@ -0,0 +1,169 @@
+require 'net/http'
+require 'uri'
+require 'socket'
+require 'json'
+
+# Log to any HTTP(S) server that accepts log messages in JSON form
+#
+# Features:
+# * JSON Formatted messages.
+# * Uses a persistent http connection, if the server supports it.
+# * SSL encryption (https).
+#
+# Example:
+#   appender = SemanticLogger::Appender::Http.new(
+#     url: 'http://localhost:8088/path'
+#   )
+#
+#   # Optional: Exclude health_check log entries, etc.
+#   appender.filter = Proc.new { |log| log.message !~ /(health_check|Not logged in)/}
+#
+#   SemanticLogger.add_appender(appender)
+class SemanticLogger::Appender::Http < SemanticLogger::Appender::Base
+  attr_accessor :username, :application, :host, :compress, :header
+  attr_reader :http, :url, :server, :port, :request_uri, :ssl_options
+
+  # Create HTTP(S) log appender
+  #
+  # Parameters:
+  #   url: [String]
+  #     Valid URL to post to.
+  #       Example: http://example.com/some_path
+  #     To enable SSL include https in the URL.
+  #       Example: https://example.com/some_path
+  #       verify_mode will default: OpenSSL::SSL::VERIFY_PEER
+  #
+  #   application: [String]
+  #     Name of this application to appear in log messages.
+  #     Default: SemanticLogger.application
+  #
+  #   host: [String]
+  #     Name of this host to appear in log messages.
+  #     Default: SemanticLogger.host
+  #
+  #   username: [String]
+  #     User name for basic Authentication.
+  #     Default: nil ( do not use basic auth )
+  #
+  #   password: [String]
+  #     Password for basic Authentication.
+  #
+  #   compress: [true|false]
+  #     Whether to compress the JSON string with GZip.
+  #     Default: false
+  #
+  #   ssl: [Hash]
+  #     Specific SSL options: For more details see NET::HTTP.start
+  #       ca_file, ca_path, cert, cert_store, ciphers, key, open_timeout, read_timeout, ssl_timeout,
+  #       ssl_version, use_ssl, verify_callback, verify_depth and verify_mode.
+  #
+  #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+  #     Override the log level for this appender.
+  #     Default: SemanticLogger.default_level
+  #
+  #   filter: [Regexp|Proc]
+  #     RegExp: Only include log messages where the class name matches the supplied.
+  #     regular expression. All other messages will be ignored.
+  #     Proc: Only include log messages where the supplied Proc returns true
+  #           The Proc must return true or false.
+  def initialize(options, &block)
+    @options     = options.dup
+    level        = @options.delete(:level)
+    filter       = @options.delete(:filter)
+    @url         = options.delete(:url)
+    @ssl_options = options.delete(:ssl)
+    @username    = options.delete(:username)
+    @password    = options.delete(:password)
+    @application = options.delete(:application) || 'Semantic Logger'
+    @host        = options.delete(:host) || SemanticLogger.host
+    @compress    = options.delete(:compress) || false
+    raise(ArgumentError, "Unknown options: #{options.inspect}") if options.size > 0
+
+    raise(ArgumentError, 'Missing mandatory parameter :url') unless @url
+
+    @header                     = {
+      'Accept'       => 'application/json',
+      'Content-Type' => 'application/json'
+    }
+    @header['Content-Encoding'] = 'gzip' if @compress
+
+    uri                             = URI.parse(@url)
+    (@ssl_options ||= {})[:use_ssl] = true if uri.scheme == 'https'
+
+    @server      = uri.host
+    @port        = uri.port
+    @username    = uri.user if !@username && uri.user
+    @password    = uri.password if !@password && uri.password
+    @request_uri = uri.request_uri
+
+    reopen
+
+    # Pass on the level and custom formatter if supplied
+    super(level, filter, &block)
+  end
+
+  # Re-open after process fork
+  def reopen
+    # On Ruby v2.0 and greater, Net::HTTP.new uses a persistent connection if the server allows it
+    @http = @ssl_options ? Net::HTTP.new(server, port, @ssl_options) : Net::HTTP.new(server, port)
+  end
+
+  # Forward log messages to HTTP Server
+  def log(log)
+    return false if (level_index > (log.level_index || 0)) ||
+      !include_message?(log) # Filtered out?
+
+    post(formatter.call(log, self))
+  end
+
+  # Use the JSON formatter
+  def default_formatter
+    self.class.json_formatter
+  end
+
+  private
+
+  def compress_data(data)
+    str = StringIO.new
+    gz  = Zlib::GzipWriter.new(str)
+    gz << data
+    gz.close
+    str.string
+  end
+
+  # HTTP Post
+  def post(body, request_uri = nil)
+    request = Net::HTTP::Post.new(request_uri || @request_uri, @header)
+    process_request(request, body)
+  end
+
+  # HTTP Put
+  def put(body, request_uri = nil)
+    request = Net::HTTP::Put.new(request_uri || @request_uri, @header)
+    process_request(request, body)
+  end
+
+  # HTTP Delete
+  def delete(request_uri = nil)
+    request = Net::HTTP::Delete.new(request_uri || @request_uri, @header)
+    process_request(request)
+  end
+
+  # Process HTTP Request
+  def process_request(request, body = nil)
+    if body
+      body         = compress_data(body) if compress
+      request.body = body
+    end
+    request.basic_auth(@username, @password) if @username
+    response = @http.request(request)
+    if response.code == '200' || response.code == '201'
+      true
+    else
+      # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
+      SemanticLogger::Logger.logger.error("Bad HTTP response code: #{response.code}, #{response.body}")
+      false
+    end
+  end
+
+end

data/lib/semantic_logger/appender/mongodb.rb

@@ -1,98 +1,115 @@
 require 'socket'
-require 'mongo'
+begin
+  require 'mongo'
+rescue LoadError
+  raise 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" to your Gemfile.'
+end
+
 module SemanticLogger
   module Appender
     # The Mongo Appender for the SemanticLogger
     #
     # Mongo Document Schema:
-    #    _id        : ObjectId("4d9cbcbf7abb3abdaf9679cd"),
-    #    time       : ISODate("2011-04-06T19:19:27.006Z"),
-    #    host_name  : "Name of the host on which this log entry originated",
-    #    application: 'Name of application or service logging the data - clarity_base, nginx, tomcat',
-    #    pid        : process id
-    #    thread_name: "name or id of thread",
-    #    name       : "com.clarity.MyClass",
-    #    level      : 'trace|debug|info|warn|error|fatal'
+    #    _id:         ObjectId("4d9cbcbf7abb3abdaf9679cd"),
+    #    time:        ISODate("2011-04-06T19:19:27.006Z"),
+    #    host:        'Name of the host on which this log entry originated',
+    #    application  'Name of application or service logging the data - clarity_base, nginx, tomcat',
+    #    pid:         process id
+    #    thread:      "name or id of thread",
+    #    name:        "com.clarity.MyClass",
+    #    level:       'trace|debug|info|warn|error|fatal'
     #    level_index: 0|1|2|3|4|5
-    #    message    : "Message supplied to the logging call",
-    #    duration   : ms,  # Set by Logger#benchmark
-    #    tags       : "Some tracking id" | ["id1", "id2"]
-    #    payload : {
-    #      Optional. Any user supplied data, including any thread specific context variables
-    #      values supplied on a per log entry will override any thread context values
-    #    }
-    #    # When an exception is supplied as the first or second parameter
-    #    # If supplied as the first parameter, message='exception name'
+    #    message:     "Message supplied to the logging call",
+    #    duration:    'human readable duration',
+    #    duration_ms: ms,
+    #    tags:        ["id1", "id2"]
     #    exception: {
     #      name:        'MyException',
     #      message:     'Invalid value',
     #      stack_trace: []
     #    }
     #    # When a backtrace is captured
-    #    file_name: 'my_class.rb'
+    #    file_name:   'my_class.rb'
     #    line_number: 42
     #
+    # Example:
+    #   require 'semantic_logger'
+    #
+    #   client   = Mongo::MongoClient.new
+    #   database = client['test']
+    #
+    #   appender = SemanticLogger::Appender::MongoDB.new(
+    #     db:              database,
+    #     collection_size: 1024**3 # 1.gigabyte
+    #   )
+    #   SemanticLogger.add_appender(appender)
+    #
+    #   logger = SemanticLogger['Example']
+    #
+    #   # Log some messages
+    #   logger.info 'This message is written to mongo as a document'
     class MongoDB < SemanticLogger::Appender::Base
       attr_reader :db, :collection_name, :collection
-      attr_accessor :host_name, :write_concern, :application
+      attr_accessor :host, :write_concern, :application
 
       # Create a MongoDB Appender instance
       #
-      #   SemanticLogger::Appender::MongoDB.new(db: Mongo::Connection.new['database'])
-      #
       # Parameters:
-      # :db [Mongo::Database]
-      #   The MongoDB database connection to use, not the database name
+      #   db: [Mongo::Database]
+      #     The MongoDB database connection to use, not the database name
       #
-      # :collection_name [String]
-      #   Name of the collection to store log data in
-      #   Default: semantic_logger
+      #   collection_name: [String]
+      #     Name of the collection to store log data in
+      #     Default: semantic_logger
       #
-      # :host_name [String]
-      #   host_name to include in the document logged to Mongo
-      #   Default: first part of host name extracted from Socket
+      #   write_concern: [Integer]
+      #     Write concern to use
+      #     see: http://docs.mongodb.org/manual/reference/write-concern/
+      #     Default: 0
       #
-      # :write_concern [Integer]
-      #   Write concern to use
-      #   see: http://docs.mongodb.org/manual/reference/write-concern/
-      #   Default: 1
+      #   host: [String]
+      #     host name to include in the document logged to Mongo
+      #     Default: SemanticLogger.host_name
       #
-      # :application [String]
-      #   Name of the application to include in the document written to mongo
-      #   Default: nil (None)
+      #   application: [String]
+      #     Name of the application to include in the document written to mongo
+      #     Default: nil (None)
       #
-      # :collection_size [Integer]
-      #   The size of the MongoDB capped collection to create in bytes
-      #   Default: 1 GB
+      #   collection_size: [Integer]
+      #     The size of the MongoDB capped collection to create in bytes
+      #     Default: 1 GB
+      #     Examples:
+      #       Prod: 25GB (.5GB per day across 4 servers over 10 days)
+      #       Dev: .5GB
+      #       Test: File
+      #       Release: 4GB
       #
-      #   Some examples
-      #     Prod: 25GB (.5GB per day across 4 servers over 10 days)
-      #     Dev: .5GB
-      #     Test: File
-      #     Release: 4GB
+      #   collection_max: [Integer]
+      #     Maximum number of log entries that the capped collection will hold.
       #
-      #  :collection_max [Integer]
-      #    Maximum number of log entries that the capped collection will hold
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
       #
-      #  :level [Symbol]
-      #    Only allow log entries of this level or higher to be written to MongoDB
-      #
-      #  :filter [Regexp|Proc]
-      #    RegExp: Only include log messages where the class name matches the supplied
-      #    regular expression. All other messages will be ignored
-      #    Proc: Only include log messages where the supplied Proc returns true
-      #          The Proc must return true or false
-      def initialize(params={}, &block)
-        @db              = params[:db] || raise('Missing mandatory parameter :db')
-        @collection_name = params[:collection_name] || 'semantic_logger'
-        @host_name       = params[:host_name] || Socket.gethostname.split('.').first
-        @write_concern   = params[:write_concern] || 1
-        @application     = params[:application]
-        filter           = params[:filter]
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      def initialize(options = {}, &block)
+        options          = options.dup
+        level            = options.delete(:level)
+        filter           = options.delete(:filter)
+        @db              = options.delete(:db) || raise('Missing mandatory parameter :db')
+        @collection_name = options.delete(:collection_name) || 'semantic_logger'
+        @host            = options.delete(:host) || options.delete(:host_name) || SemanticLogger.host
+        @write_concern   = options.delete(:write_concern) || 0
+        @application     = options.delete(:application) || SemanticLogger.application
 
         # Create a collection that will hold the lesser of 1GB space or 10K documents
-        @collection_size = params[:collection_size] || 1024**3
-        @collection_max  = params[:collection_max]
+        @collection_size = options.delete(:collection_size) || 1024**3
+        @collection_max  = options.delete(:collection_max)
+        raise(ArgumentError, "Unknown options: #{options.inspect}") if options.size > 0
 
         reopen
 
@@ -100,7 +117,7 @@ module SemanticLogger
         create_indexes
 
         # Set the log level and formatter
-        super(params[:level], filter, &block)
+        super(level, filter, &block)
       end
 
       # After forking an active process call #reopen to re-open
@@ -109,7 +126,8 @@ module SemanticLogger
         @collection = db[@collection_name]
       end
 
-      # Create the required capped collection
+      # Create the required capped collection.
+      #
       # Features of capped collection:
       # * No indexes by default (not even on _id)
       # * Documents cannot be deleted,
@@ -117,7 +135,7 @@ module SemanticLogger
       # * Documents are always stored in insertion order
       #   * A find will always return the documents in their insertion order
       #
-      # Creates an index based on tags to support faster lookups
+      # Creates an index based on tags to support faster searches.
       def create_indexes
         options       = {capped: true, size: @collection_size}
         options[:max] = @collection_max if @collection_max
@@ -144,59 +162,20 @@ module SemanticLogger
       #  Replace this formatter by supplying a Block to the initializer
       def default_formatter
         Proc.new do |log|
-          document               = {
-            time:        log.time,
-            host_name:   host_name,
-            pid:         $$,
-            thread_name: log.thread_name,
-            name:        log.name,
-            level:       log.level,
-            level_index: log.level_index,
-          }
-          document[:application] = application if application
-          document[:message]     = log.cleansed_message if log.message
-          document[:duration]    = log.duration if log.duration
-          document[:tags]        = log.tags if log.tags && (log.tags.size > 0)
-          document[:payload]     = log.payload if log.payload
-          if log.exception
-            root = document
-            log.each_exception do |exception, i|
-              name       = i == 0 ? :exception : :cause
-              root[name] = {
-                name:        exception.class.name,
-                message:     exception.message,
-                stack_trace: exception.backtrace
-              }
-              root       = root[name]
-            end
-          end
-
-          file, line = log.file_name_and_line
-          if file
-            document[:file_name]   = file
-            document[:line_number] = line.to_i
-          end
-          document
+          h               = log.to_h
+          h[:host]        = host
+          h[:application] = application
+          h
         end
       end
 
-      # Default host_name to use if none is supplied to the appenders initializer
-      def self.host_name
-        @@host_name ||= Socket.gethostname.split('.').first
-      end
-
-      # Override the default host_name
-      def self.host_name=(host_name)
-        @@host_name = host_name
-      end
-
       # Log the message to MongoDB
       def log(log)
         # Ensure minimum log level is met, and check filter
         return false if (level_index > (log.level_index || 0)) || !include_message?(log)
 
         # Insert log entry into Mongo
-        collection.insert(formatter.call(log), w: @write_concern)
+        collection.insert(formatter.call(log, self), w: @write_concern)
         true
       end
 

data/lib/semantic_logger/appender/new_relic.rb

@@ -1,81 +1,47 @@
-=begin
-New Relic appender for SemanticLogger
-
-The :error and :fatal log entries will show up under Applications > "Application Name" > Events > Errors in New Relic
-
-Note: Payload information is not filtered, so take care not to push any sensitive information when logging with tags or a payload.
-
-
-Example 1
-
-Adding the New Relic appender will send :error and :fatal log entries to New Relic as error events.
-
-For a Rails application already configured to use SemanticLogger and New Relic, create a file called <Rails Root>/config/initializers/newrelic_appender.rb with the following contents and restart the application:
-
-# Send :error and :fatal log messages to New Relic
-SemanticLogger.add_appender(SemanticLogger::Appender::NewRelic.new)
-Rails.logger.info 'SemanticLogger New Relic Appender added.'
-
-
-Example 2
-
-For a non-Rails application, send :info and more severe log entries to a file called application.log and also send :error and :fatal log entries to New Relic.
-
-# ./newrelic.yml needs to be set up -- see https://docs.newrelic.com/docs/ruby/ruby-agent-installation for more information.
-
-require 'semantic_logger'
-require 'newrelic_rpm'
-
-# New Relic setup
-NewRelic::Agent.manual_start
-
-# SemanticLogger setup
-SemanticLogger.default_level = :info
-SemanticLogger.add_appender('application.log')
-SemanticLogger.add_appender(SemanticLogger::Appender::NewRelic.new)
-logger = SemanticLogger['Example']
-
-# Log some messages
-logger.info  'This is only written to application.log'
-logger.error 'This is written to application.log and will also be sent to New Relic as an error event'
-
-# The appender will send tags, payloads and benchmark duration to New Relic
-logger.tagged('test') do
-  logger.with_payload( {key1: 123, key2: 'abc'} ) do
-    logger.benchmark_error(@message) do
-      sleep 0.001
-    end
-  end
+begin
+  require 'newrelic_rpm'
+rescue LoadError
+  raise 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
 end
 
-# New Relic does not seem to receive any errors if the application exits too soon after sending error alerts.
-sleep 10
-
-# New Relic shutdown - should send any queued data before exiting
-::NewRelic::Agent.shutdown
-=end
-
-
-require 'newrelic_rpm'
-
+# Send log messages to NewRelic
+#
+# The :error and :fatal log entries will show up under
+# "Applications" > "Application Name" > "Events" > "Errors" in New Relic.
+#
+# Example:
+#   SemanticLogger.add_appender(SemanticLogger::Appender::NewRelic.new)
+#
 class SemanticLogger::Appender::NewRelic < SemanticLogger::Appender::Base
-  # Allow the level for this appender to be overwritten
-  #   Default: :error
-  #   Note: Not recommended to set the log level to :info, :debug, or :trace as that would flood NewRelic with Error notices
-  def initialize(level=:error, &block)
-    super(level, &block)
+  # Create Appender
+  #
+  # Parameters
+  #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+  #     Override the log level for this appender.
+  #     Default: :error
+  #
+  #   filter: [Regexp|Proc]
+  #     RegExp: Only include log messages where the class name matches the supplied.
+  #     regular expression. All other messages will be ignored.
+  #     Proc: Only include log messages where the supplied Proc returns true
+  #           The Proc must return true or false.
+  def initialize(options = {}, &block)
+    # Backward compatibility
+    options  = {level: options} unless options.is_a?(Hash)
+    @options = options.dup
+    level    = @options.delete(:level) || :error
+    filter   = @options.delete(:filter)
+
+    super(level, filter, &block)
   end
 
   # Returns [Hash] of parameters to send to New Relic.
   def default_formatter
-    Proc.new do |log|
-      custom_params            = {thread_name: log.thread_name}
-      custom_params[:duration] = "#{log.duration} ms" if log.duration
-      custom_params[:payload]  = log.payload if log.payload
-      custom_params[:tags]     = log.tags if log.tags && (log.tags.size > 0)
-      custom_params[:message]  = log.message if log.exception
-
-      {metric: log.metric, custom_params: custom_params}
+    Proc.new do |log, logger|
+      h = log.to_h
+      h.delete(:time)
+      h.delete(:exception)
+      {metric: log.metric, custom_params: h}
     end
   end
 
@@ -96,7 +62,7 @@ class SemanticLogger::Appender::NewRelic < SemanticLogger::Appender::Base
     # For more documentation on the NewRelic::Agent.notice_error method see:
     # http://rubydoc.info/github/newrelic/rpm/NewRelic/Agent#notice_error-instance_method
     # and https://docs.newrelic.com/docs/ruby/ruby-agent-api
-    NewRelic::Agent.notice_error(exception, formatter.call(log))
+    NewRelic::Agent.notice_error(exception, formatter.call(log, self))
     true
   end