semantic_logger 4.1.1 → 4.2.0

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -15,15 +15,11 @@ require 'date'
 #     url:      'http://localhost:9200'
 #   )
 class SemanticLogger::Appender::Elasticsearch < SemanticLogger::Subscriber
-  attr_accessor :url, :index, :type, :client, :flush_interval, :timeout_interval, :batch_size
+  attr_accessor :url, :index, :type, :client, :flush_interval, :timeout_interval, :batch_size, :elasticsearch_args
 
   # Create Elasticsearch appender over persistent HTTP(S)
   #
   # Parameters:
-  #   url: [String]
-  #     Fully qualified address to the Elasticsearch service.
-  #     Default: 'http://localhost:9200'
-  #
   #   index: [String]
   #     Prefix of the index to store the logs in Elasticsearch.
   #     The final index appends the date so that indexes are used per day.
@@ -34,18 +30,6 @@ class SemanticLogger::Appender::Elasticsearch < SemanticLogger::Subscriber
   #     Document type to associate with logs when they are written.
   #     Default: 'log'
   #
-  #   batch_size: [Fixnum]
-  #     Size of list when sending to Elasticsearch. May be smaller if flush is triggered early.
-  #     Default: 500
-  #
-  #   flush_interval: [Fixnum]
-  #     Seconds to wait before attempting a flush to Elasticsearch. If no messages queued it's a NOOP.
-  #     Default: 1
-  #
-  #   timeout_interval: [Fixnum]
-  #     Seconds to allow the Elasticsearch client to flush the bulk message.
-  #     Default: 10
-  #
   #   level: [:trace | :debug | :info | :warn | :error | :fatal]
   #     Override the log level for this appender.
   #     Default: SemanticLogger.default_level
@@ -68,88 +52,136 @@ class SemanticLogger::Appender::Elasticsearch < SemanticLogger::Subscriber
   #   application: [String]
   #     Name of this application to appear in log messages.
   #     Default: SemanticLogger.application
-  def initialize(url: 'http://localhost:9200', index: 'semantic_logger', type: 'log', flush_interval: 1, timeout_interval: 10, batch_size: 500,
-                 level: nil, formatter: nil, filter: nil, application: nil, host: nil, &block)
-
-    @url              = url
-    @index            = index
-    @type             = type
-    @flush_interval   = flush_interval
-    @timeout_interval = timeout_interval
-    @batch_size       = batch_size
-
-    @messages_mutex = Mutex.new
-    @messages       = Array.new
+  #
+  # Elasticsearch Parameters:
+  #   url: [String]
+  #     Fully qualified address to the Elasticsearch service.
+  #     Default: 'http://localhost:9200'
+  #
+  #   hosts: [String|Hash|Array]
+  #     Single host passed as a String or Hash, or multiple hosts
+  #     passed as an Array; `host` or `url` keys are also valid.
+  #     Note:
+  #       :url above is ignored when supplying this option.
+  #
+  #   resurrect_after [Float]
+  #     After how many seconds a dead connection should be tried again.
+  #
+  #   reload_connections [true|false|Integer]
+  #     Reload connections after X requests.
+  #     Default: false
+  #
+  #   randomize_hosts [true|false]
+  #     Shuffle connections on initialization and reload.
+  #     Default: false
+  #
+  #   sniffer_timeout [Integer]
+  #     Timeout for reloading connections in seconds.
+  #     Default: 1
+  #
+  #   retry_on_failure [true|false|Integer]
+  #     Retry X times when request fails before raising and exception.
+  #     Default: false
+  #
+  #   retry_on_status [Array<Number>]
+  #     Retry when specific status codes are returned.
+  #
+  #   reload_on_failure [true|false]
+  #     Reload connections after failure.
+  #     Default: false
+  #
+  #   request_timeout [Integer]
+  #     The request timeout to be passed to transport in options.
+  #
+  #   adapter [Symbol]
+  #     A specific adapter for Faraday (e.g. `:patron`)
+  #
+  #   transport_options [Hash]
+  #     Options to be passed to the `Faraday::Connection` constructor.
+  #
+  #   transport_class [Constant]
+  #     A specific transport class to use, will be initialized by
+  #     the client and passed hosts and all arguments.
+  #
+  #   transport [Object]
+  #     A specific transport instance.
+  #
+  #   serializer_class [Constant]
+  #     A specific serializer class to use, will be initialized by
+  #     the transport and passed the transport instance.
+  #
+  #   selector [Elasticsearch::Transport::Transport::Connections::Selector::Base]
+  #     An instance of selector strategy derived from `Elasticsearch::Transport::Transport::Connections::Selector::Base`.
+  #
+  #   send_get_body_as [String]
+  #     Specify the HTTP method to use for GET requests with a body.
+  #     Default: 'GET'
+  def initialize(url: 'http://localhost:9200',
+                 index: 'semantic_logger',
+                 type: 'log',
+                 level: nil,
+                 formatter: nil,
+                 filter: nil,
+                 application: nil,
+                 host: nil,
+                 **elasticsearch_args,
+                 &block)
+
+    @url                         = url
+    @index                       = index
+    @type                        = type
+    @elasticsearch_args          = elasticsearch_args.dup
+    @elasticsearch_args[:url]    = url if url && !elasticsearch_args[:hosts]
+    @elasticsearch_args[:logger] = logger
 
     super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
     reopen
   end
 
   def reopen
-    @client = Elasticsearch::Client.new(url: url, logger: SemanticLogger::Processor.logger.clone)
-
-    @messages_mutex.synchronize { @messages = [] }
-
-    @flush_task = Concurrent::TimerTask.new(execution_interval: flush_interval, timeout_interval: timeout_interval) do
-      flush
-    end.execute
-  end
-
-  def close
-    @flush_task.shutdown if @flush_task
-    @flush_task = nil
-    # No api to close connections in the elasticsearch client!
-    #@client.close if @client
-    #@client = nil
+    @client = Elasticsearch::Client.new(@elasticsearch_args)
   end
 
-  def call(log, logger)
-    h = SemanticLogger::Formatters::Raw.new.call(log, logger)
-    h.delete(:time)
-    h[:timestamp] = log.time.utc.iso8601(SemanticLogger::Formatters::Base::PRECISION)
-    h
+  # Log to the index for today
+  def log(log)
+    bulk_payload = formatter.call(log, self)
+    write_to_elasticsearch([bulk_index(log), bulk_payload])
+    true
   end
 
-  def flush
-    collected_messages = nil
-    @messages_mutex.synchronize do
-      if @messages.length > 0
-        collected_messages = @messages
-        @messages          = []
+  def batch(logs)
+    messages = []
+    day      = nil
+    logs.each do |log|
+      # Only write the bulk index once per day per batch. Supports mixed dates in a batch.
+      if log.time.day != day
+        messages << bulk_index(log)
+        day = log.time.day
       end
+      messages << formatter.call(log, self)
     end
 
-    if collected_messages
-      bulk_result = @client.bulk(body: collected_messages)
-      if bulk_result["errors"]
-        failed = bulk_result["items"].select { |x| x["status"] != 201 }
-        SemanticLogger::Processor.logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
-      end
-    end
-  rescue Exception => exc
-    SemanticLogger::Processor.logger.error('ElasticSearch: Failed to bulk insert log messages', exc)
+    write_to_elasticsearch(messages)
+    true
   end
 
-  # Log to the index for today
-  def log(log)
-    return false unless should_log?(log)
-
-    daily_index = log.time.strftime("#{@index}-%Y.%m.%d")
+  private
 
-    bulk_index   = {'index' => {'_index' => daily_index, '_type' => @type}}
-    bulk_payload = formatter.call(log, self)
-
-    enqueue(bulk_index, bulk_payload)
+  def write_to_elasticsearch(messages)
+    bulk_result = @client.bulk(body: messages)
+    if bulk_result["errors"]
+      failed = bulk_result["items"].select { |x| x["status"] != 201 }
+      logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
+    end
   end
 
-  def enqueue(bulk_index, bulk_payload)
-    messages_len =
-      @messages_mutex.synchronize do
-        @messages.push(bulk_index)
-        @messages.push(bulk_payload)
-        @messages.length
-      end
+  def bulk_index(log)
+    daily_index = log.time.strftime("#{index}-%Y.%m.%d")
+    {'index' => {'_index' => daily_index, '_type' => type}}
+  end
 
-    flush if messages_len >= batch_size
+  def default_formatter
+    SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: :timestamp)
   end
+
 end

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -54,7 +54,7 @@ class SemanticLogger::Appender::ElasticsearchHttp < SemanticLogger::Appender::Ht
 
     @index = index
     @type  = type
-    super(url:   url, compress: compress, ssl: ssl, open_timeout: 2.0, read_timeout: open_timeout, continue_timeout: continue_timeout,
+    super(url:   url, compress: compress, ssl: ssl, read_timeout: read_timeout, open_timeout: open_timeout, continue_timeout: continue_timeout,
           level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
 
     @request_path = "#{@path.end_with?('/') ? @path : "#{@path}/"}#{@index}-%Y.%m.%d"
@@ -63,8 +63,6 @@ class SemanticLogger::Appender::ElasticsearchHttp < SemanticLogger::Appender::Ht
 
   # Log to the index for today.
   def log(log)
-    return false unless should_log?(log)
-
     post(formatter.call(log, self), log.time.strftime(@logging_path))
   end
 

data/lib/semantic_logger/appender/file.rb

@@ -81,7 +81,7 @@ module SemanticLogger
       def reopen
         return unless @file_name
 
-        @log      = open(@file_name, (::File::WRONLY | ::File::APPEND | ::File::CREAT))
+        @log = open(@file_name, (::File::WRONLY | ::File::APPEND | ::File::CREAT))
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
         @log.sync = true
@@ -93,8 +93,6 @@ module SemanticLogger
       #  trace entries are mapped to debug since :trace is not supported by the
       #  Ruby or Rails Loggers
       def log(log)
-        return false unless should_log?(log)
-
         # Since only one appender thread will be writing to the file at a time
         # it is not necessary to protect access to the file with a semaphore
         # Allow this logger to filter out log levels lower than it's own

data/lib/semantic_logger/appender/graylog.rb

@@ -110,11 +110,9 @@ class SemanticLogger::Appender::Graylog < SemanticLogger::Subscriber
 
   # Returns [Hash] of parameters to send
   def call(log, logger)
-    h = SemanticLogger::Formatters::Raw.new.call(log, logger)
-    h.delete(:time)
+    h = default_formatter.call(log, logger)
 
     h[:short_message] = h.delete(:message) || log.exception.message
-    h[:timestamp]     = log.time.utc.to_f
     h[:level]         = logger.level_map[log.level]
     h[:level_str]     = log.level.to_s
     h[:duration_str]  = h.delete(:duration)
@@ -123,10 +121,13 @@ class SemanticLogger::Appender::Graylog < SemanticLogger::Subscriber
 
   # Forward log messages
   def log(log)
-    return false unless should_log?(log)
-
     notifier.notify!(formatter.call(log, self))
     true
   end
 
+  private
+
+  def default_formatter
+    SemanticLogger::Formatters::Raw.new(time_format: :seconds, time_key: :timestamp)
+  end
 end

data/lib/semantic_logger/appender/honeybadger.rb

@@ -41,8 +41,6 @@ class SemanticLogger::Appender::Honeybadger < SemanticLogger::Subscriber
 
   # Send an error notification to honeybadger
   def log(log)
-    return false unless should_log?(log)
-
     context = formatter.call(log, self)
     if log.exception
       context.delete(:exception)

data/lib/semantic_logger/appender/http.rb

@@ -2,6 +2,7 @@ require 'net/http'
 require 'uri'
 require 'socket'
 require 'json'
+require 'openssl'
 
 # Log to any HTTP(S) server that accepts log messages in JSON form
 #
@@ -16,7 +17,7 @@ require 'json'
 #     url:      'http://localhost:8088/path'
 #   )
 class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
-  attr_accessor :username, :application, :host, :compress, :header,
+  attr_accessor :username, :compress, :header,
                 :open_timeout, :read_timeout, :continue_timeout
   attr_reader :http, :url, :server, :port, :path, :ssl_options
 
@@ -77,8 +78,20 @@ class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
   #
   #   continue_timeout: [Float]
   #     Default: 1.0
-  def initialize(url:, compress: false, ssl: {}, username: nil, password: nil, open_timeout: 2.0, read_timeout: 1.0, continue_timeout: 1.0,
-                 level: nil, formatter: nil, filter: nil, application: nil, host: nil, &block)
+  def initialize(url:,
+                 compress: false,
+                 ssl: {},
+                 username: nil,
+                 password: nil,
+                 open_timeout: 2.0,
+                 read_timeout: 1.0,
+                 continue_timeout: 1.0,
+                 level: nil,
+                 formatter: nil,
+                 filter: nil,
+                 application: nil,
+                 host: nil,
+                 &block)
 
     @url              = url
     @ssl_options      = ssl
@@ -89,10 +102,10 @@ class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
     @read_timeout     = read_timeout
     @continue_timeout = continue_timeout
 
+    # On Ruby v2.0 and greater, Net::HTTP.new already uses a persistent connection if the server allows it
     @header                     = {
       'Accept'       => 'application/json',
       'Content-Type' => 'application/json',
-      # On Ruby v2.0 and greater, Net::HTTP.new already uses a persistent connection if the server allows it
       'Connection'   => 'keep-alive',
       'Keep-Alive'   => '300'
     }
@@ -148,9 +161,9 @@ class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
 
   # Forward log messages to HTTP Server
   def log(log)
-    return false unless should_log?(log)
-
-    post(formatter.call(log, self))
+    message = formatter.call(log, self)
+    logger.trace(message)
+    post(message)
   end
 
   private
@@ -189,8 +202,7 @@ class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
   # Process HTTP Request
   def process_request(request, body = nil)
     if body
-      body         = compress_data(body) if compress
-      request.body = body
+      request.body = compress ? compress_data(body) : body
     end
     request.basic_auth(@username, @password) if @username
     response = @http.request(request)
@@ -198,9 +210,12 @@ class SemanticLogger::Appender::Http < SemanticLogger::Subscriber
       true
     else
       # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
-      SemanticLogger::Processor.logger.error("Bad HTTP response from: #{url} code: #{response.code}, #{response.body}")
+      logger.error("Bad HTTP response from: #{url} code: #{response.code}, #{response.body}")
       false
     end
+  rescue RuntimeError => exc
+    reopen
+    raise exc
   end
 
 end

data/lib/semantic_logger/appender/kafka.rb

@@ -142,7 +142,7 @@ class SemanticLogger::Appender::Kafka < SemanticLogger::Subscriber
       ssl_ca_cert:         ssl_ca_cert,
       ssl_client_cert:     ssl_client_cert,
       ssl_client_cert_key: ssl_client_cert_key,
-      logger:              SemanticLogger::Processor.logger.clone
+      logger:              logger
     )
 
     @producer = @kafka.async_producer(
@@ -160,8 +160,6 @@ class SemanticLogger::Appender::Kafka < SemanticLogger::Subscriber
 
   # Forward log messages to Kafka producer thread.
   def log(log)
-    return false unless should_log?(log)
-
     json = formatter.call(log, self)
     @producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
   end

data/lib/semantic_logger/appender/mongodb.rb

@@ -106,7 +106,7 @@ module SemanticLogger
       def initialize(uri:, collection_name: 'semantic_logger', write_concern: 0, collection_size: 1024**3, collection_max: nil,
                      level: nil, formatter: nil, filter: nil, host: nil, application: nil, &block)
 
-        @client          = Mongo::Client.new(uri, logger: SemanticLogger::Processor.logger.clone)
+        @client          = Mongo::Client.new(uri, logger: logger)
         @collection_name = collection_name
         @options         = {
           capped: true,
@@ -161,8 +161,6 @@ module SemanticLogger
 
       # Log the message to MongoDB
       def log(log)
-        return false unless should_log?(log)
-
         # Insert log entry into Mongo
         collection.insert_one(formatter.call(log, self))
         true

data/lib/semantic_logger/appender/new_relic.rb

@@ -29,7 +29,13 @@ class SemanticLogger::Appender::NewRelic < SemanticLogger::Subscriber
   #     regular expression. All other messages will be ignored.
   #     Proc: Only include log messages where the supplied Proc returns true
   #           The Proc must return true or false.
-  def initialize(level: :error, formatter: nil, filter: nil, application: nil, host: nil, &block)
+  def initialize(level: :error,
+                 formatter: nil,
+                 filter: nil,
+                 application: nil,
+                 host: nil,
+                 &block)
+
     super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
   end
 
@@ -43,8 +49,6 @@ class SemanticLogger::Appender::NewRelic < SemanticLogger::Subscriber
 
   # Send an error notification to New Relic
   def log(log)
-    return false unless should_log?(log)
-
     # Send error messages as Runtime exceptions
     exception =
       if log.exception