logstash-input-beats 2.0.3 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86b68287fce7667ece60ab81196daf597a34bf0f
-  data.tar.gz: eba32adca86f4b4fa7c37c7755ab524498e87b8f
+  metadata.gz: dcc82edf516cc926764c9bf592b6b4240d2b250a
+  data.tar.gz: afd96ca6287561a27ef83d1e5070785cc507da00
 SHA512:
-  metadata.gz: 2199de568cb60a38bf73f0fdf61a24ff75a5596e2aaf984f1c468f6c1d8ac505e4d7717e7bf77ff4af2579678b57c1fc48125f3a72a02e86d302d0184ab3d514
-  data.tar.gz: 86bae83fb7c1767eec85021c47c7de6b003ed725fa5dfb6e8b2f64e2fa3fe1597e70d04e356145b836c8d991545c7733949bd420d41a300b39898d7a587cbe4d
+  metadata.gz: 9a4d8d3e817adaad0e49659c713ba864e2d32837ca39a751cb78d026bc71b215564048e7db7912934a551ff60c0aa2dc3ae52ae06b5f9ee4d4bd25814e4277f2
+  data.tar.gz: 0416a16c50b648351929119370f8ea513ff2dfd1657afd47b7332e6ef3758c3381505ce639ba122790024a638272dd91e766f43fd0cd277143b32e66382f7578

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 2.1.1
+  - Release a new version of the gem that doesn't included any other gems, 2.1.0 is yanked from rubygems 
+# 2.1.0
+  - Refactor of the code to make it easier to unit test
+  - Fix a conncurrency error on high load on the SizeQueue #37
+  - Drop the internal SizeQueue to rely on Java Synchronous Queue
+  - Remove the majority of the nested blocks
+  - Move the CircuitBreaker inside an internal namespace so it doesn't conflict with the input lumberjack
+  - Add more debugging log statement
+  - Flush the codec when a disconnect happen
+  - Tag/Decorate the event when a shutdown occur.
+  - The name of the threads managed by the input beat are now meaningful.
 # 2.0.3
   - Reduce the size of the gem by removing vendor jars
 # 2.0.2

data/lib/logstash/inputs/beats.rb

@@ -5,6 +5,15 @@ require "logstash/timestamp"
 require "lumberjack/beats"
 require "lumberjack/beats/server"
 require "logstash/codecs/identity_map_codec"
+require "logstash/inputs/beats_support/circuit_breaker"
+require "logstash/inputs/beats_support/codec_callback_listener"
+require "logstash/inputs/beats_support/connection_handler"
+require "logstash/inputs/beats_support/event_transform_common"
+require "logstash/inputs/beats_support/decoded_event_transform"
+require "logstash/inputs/beats_support/raw_event_transform"
+require "logstash/inputs/beats_support/synchronous_queue_with_offer"
+require "logstash/util"
+require "thread_safe"
 
 # use Logstash provided json decoder
 Lumberjack::Beats::json = LogStash::Json
@@ -13,7 +22,28 @@ Lumberjack::Beats::json = LogStash::Json
 #
 # https://github.com/elastic/filebeat[filebeat]
 #
+
+class LogStash::Codecs::Base
+  # This monkey patch add callback based
+  # flow to the codec until its shipped with core.
+  # This give greater flexibility to the implementation by
+  # sending more data to the actual block.
+  if !method_defined?(:accept)
+    def accept(listener)
+      decode(listener.data) do |event|
+        listener.process_event(event)
+      end
+    end
+  end
+  if !method_defined?(:auto_flush)
+    def auto_flush
+    end
+  end
+end
+
 class LogStash::Inputs::Beats < LogStash::Inputs::Base
+  class InsertingToQueueTakeTooLong < Exception; end
+
   config_name "beats"
 
   default :codec, "plain"
@@ -46,41 +76,37 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   config :target_field_for_codec, :validate => :string, :default => "message"
 
   # TODO(sissel): Add CA to authenticate clients with.
-  BUFFERED_QUEUE_SIZE = 1
   RECONNECT_BACKOFF_SLEEP = 0.5
 
   def register
-    require "concurrent"
-    require "logstash/circuit_breaker"
-    require "logstash/sized_queue_timeout"
-
     if !@ssl
-      @logger.warn("Beats: SSL Certificate will not be used") unless @ssl_certificate.nil?
-      @logger.warn("Beats: SSL Key will not be used") unless @ssl_key.nil?
+      @logger.warn("Beats input: SSL Certificate will not be used") unless @ssl_certificate.nil?
+      @logger.warn("Beats input: SSL Key will not be used") unless @ssl_key.nil?
     elsif !ssl_configured?
       raise LogStash::ConfigurationError, "Certificate or Certificate Key not configured"
     end
 
-    @logger.info("Starting Beats input listener", :address => "#{@host}:#{@port}")
+    @logger.info("Beats inputs: Starting input listener", :address => "#{@host}:#{@port}")
     @lumberjack = Lumberjack::Beats::Server.new(:address => @host, :port => @port,
       :ssl => @ssl, :ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
       :ssl_key_passphrase => @ssl_key_passphrase)
 
-    # Create a reusable threadpool, we do not limit the number of connections
-    # to the input, the circuit breaker with the timeout should take care
-    # of `blocked` threads and prevent logstash to go oom.
-    @threadpool = Concurrent::CachedThreadPool.new(:idletime => 15)
-
     # in 1.5 the main SizeQueue doesnt have the concept of timeout
     # We are using a small plugin buffer to move events to the internal queue
-    @buffered_queue = LogStash::SizedQueueTimeout.new(BUFFERED_QUEUE_SIZE)
+    @buffered_queue = LogStash::Inputs::BeatsSupport::SynchronousQueueWithOffer.new(@congestion_threshold)
 
-    @circuit_breaker = LogStash::CircuitBreaker.new("Beats input",
-                            :exceptions => [LogStash::SizedQueueTimeout::TimeoutError])
+    @circuit_breaker = LogStash::Inputs::BeatsSupport::CircuitBreaker.new("Beats input",
+                            :exceptions => [InsertingToQueueTakeTooLong])
 
     # wrap the configured codec to support identity stream
     # from the producers
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
+
+    # Keep a list of active connections so we can flush their codec on shutdown
+    
+    # Use threadsafe gem, since we have a strict dependency on concurrent-ruby 0.9.2 
+    # in the core
+    @connections_list = ThreadSafe::Hash.new
   end # def register
 
   def ssl_configured?
@@ -99,104 +125,82 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
       # Wrapping the accept call into a CircuitBreaker
       if @circuit_breaker.closed?
         connection = @lumberjack.accept # call that creates a new connection
-        next if connection.nil? # if the connection is nil the connection was close.
-
-        invoke(connection) do |event|
-          if stop?
-            connection.close
-            break
-          end
-
-          begin
-            @circuit_breaker.execute {
-              @buffered_queue.push(event, @congestion_threshold)
-            }
-          rescue => e
-            raise e
-          end
+        # if the connection is nil the connection was closed upstream,
+        # so we will try in another iteration to recover or stop.
+        next if connection.nil? 
+
+        Thread.new do 
+          handle_new_connection(connection)
         end
       else
-        @logger.warn("Beats input: the pipeline is blocked, temporary refusing new connection.")
+        @logger.warn("Beats input: the pipeline is blocked, temporary refusing new connection.", 
+                     :reconnect_backoff_sleep => RECONNECT_BACKOFF_SLEEP)
         sleep(RECONNECT_BACKOFF_SLEEP)
       end
     end
   end # def run
 
-  public
   def stop
-    # we may have some stuff in the buffer
-    @codec.flush { |event| @output_queue << event }
+    @logger.debug("Beats input: stopping the plugin")
+
     @lumberjack.close rescue nil
-  end
 
-  public
-  def create_event(map, identity_stream, &block)
-    # Filebeats uses the `message` key and LSF `line`
-    target_field = target_field_for_codec ? map.delete(target_field_for_codec) : nil
-
-    if target_field.nil?
-      event = LogStash::Event.new(map)
-      copy_beat_hostname(event)
-      decorate(event)
-      block.call(event)
-    else
-      # All codecs expects to work on string
-      @codec.decode(target_field.to_s, identity_stream) do |decoded|
-        ts = coerce_ts(map.delete("@timestamp"))
-        decoded["@timestamp"] = ts unless ts.nil?
-        map.each { |k, v| decoded[k] = v }
-        copy_beat_hostname(decoded)
-        decorate(decoded)
-        block.call(decoded)
+    # we may have some stuff in the codec buffer
+    transformer = LogStash::Inputs::BeatsSupport::EventTransformCommon.new(self)
+
+    # Go through all the active connection and flush their
+    # codec content, some context data could be lost in this case
+    # but at least the events main data would be persisted.
+    @connections_list.each do |_, connection_handler|
+      connection_handler.flush do |event|
+        # We might loose some context of the
+        transformer.transform(event)
+        event.tag("beats_input_flushed_by_logtash_shutdown")
+        @output_queue << event
       end
     end
-  end
 
-  # Copies the beat.hostname field into the host field unless
-  # the host field is already defined
-  private
-  def copy_beat_hostname(event)
-    host = event["beat"] ? event["beat"]["hostname"] : nil
-    if host && event["host"].nil?
-      event["host"] = host
-    end
+    @logger.debug("Beats input: stopped")
   end
 
-  private
-  def coerce_ts(ts)
-    return nil if ts.nil?
-    timestamp = LogStash::Timestamp.coerce(ts)
-    return timestamp if timestamp
-
-    @logger.warn("Unrecognized @timestamp value, setting current time to @timestamp",
-      :value => ts.inspect)
-  rescue LogStash::TimestampParserError => e
-    @logger.warn("Error parsing @timestamp string, setting current time to @timestamp",
-      :value => ts.inspect, :exception => e.message)
-  end
-
-  private
-  def invoke(connection, &block)
-    @threadpool.post do
-      begin
-        # If any errors occur in from the events the connection should be closed in the
-        # library ensure block and the exception will be handled here
-        connection.run do |map, identity_stream|
-          create_event(map, identity_stream, &block)
-        end
-
-        # When too many errors happen inside the circuit breaker it will throw
-        # this exception and start refusing connection. The bubbling of theses
-        # exceptions make sure that the lumberjack library will close the current
-        # connection which will force the client to reconnect and restransmit
-        # his payload.
-      rescue LogStash::CircuitBreaker::OpenBreaker,
-             LogStash::CircuitBreaker::HalfOpenBreaker => e
-        logger.warn("Beats input: The circuit breaker has detected a slowdown or stall in the pipeline, the input is closing the current connection and rejecting new connection until the pipeline recover.", :exception => e.class)
-      rescue => e # If we have a malformed packet we should handle that so the input doesn't crash completely.
-        @logger.error("Beats input: unhandled exception", :exception => e, :backtrace => e.backtrace)
-      end
+  # This Method is called inside a new thread
+  def handle_new_connection(connection)
+    logger.debug? && logger.debug("Beats inputs: accepting a new connection",
+                                  :peer => connection.peer)
+
+    LogStash::Util.set_thread_name("[beats-input]>connection-#{connection.peer}")
+
+    connection_handler = LogStash::Inputs::BeatsSupport::ConnectionHandler.new(connection, self, @buffered_queue)
+    @connections_list[connection] = connection_handler
+
+    # All the errors handling is done here
+    @circuit_breaker.execute { connection_handler.accept }
+  rescue Lumberjack::Beats::Connection::ConnectionClosed => e
+    logger.warn("Beats Input: Remote connection closed",
+                :peer => connection.peer,
+                :exception => e)
+  rescue LogStash::Inputs::BeatsSupport::CircuitBreaker::OpenBreaker,
+    LogStash::Inputs::BeatsSupport::CircuitBreaker::HalfOpenBreaker => e
+    logger.warn("Beats input: The circuit breaker has detected a slowdown or stall in the pipeline, the input is closing the current connection and rejecting new connection until the pipeline recover.",
+                :exception => e.class)
+  rescue Exception => e # If we have a malformed packet we should handle that so the input doesn't crash completely.
+    @logger.error("Beats input: unhandled exception", 
+                  :exception => e,
+                  :backtrace => e.backtrace) 
+  ensure
+    transformer = LogStash::Inputs::BeatsSupport::EventTransformCommon.new(self)
+
+    connection_handler.flush do |event|
+      # handle the basic event enrichment with tags
+      # since at that time we lose all the context
+      transformer.transform(event)
+      event.tag("beats_input_flushed_by_end_of_connection")
+      @output_queue << event
     end
+
+    @connections_list.delete(connection)
+    @logger.debug? && @logger.debug("Beats input: clearing the connection from the known clients",
+                                    :peer => connection.peer)
   end
 
   # The default Logstash Sizequeue doesn't support timeouts.
@@ -206,9 +210,19 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   # We are using a proxy queue supporting blocking with a timeout and
   # this thread take the element from one queue into another one.
   def start_buffer_broker
-    @threadpool.post do
-      while !stop?
-        @output_queue << @buffered_queue.pop_no_timeout
+    Thread.new do
+      LogStash::Util.set_thread_name("[beats-input]-buffered-queue-broker")
+
+      begin
+        while !stop?
+          @output_queue << @buffered_queue.take
+        end
+      rescue InterruptionException => e
+        # If we are shutting down without waiting the queue to unblock
+        # we will get an `InterruptionException` in that context we will not log it.
+        @logger.error("Beats input: bufferered queue exception", :exception => e) unless stop?
+      rescue => e
+        @logger.error("Beats input: unexpected exception", :exception => e)
       end
     end
   end

data/lib/logstash/{circuit_breaker.rb → inputs/beats_support/circuit_breaker.rb}

@@ -1,7 +1,7 @@
 require "thread"
 require "cabin"
 
-module LogStash
+module LogStash::Inputs::BeatsSupport
   # Largely inspired by Martin's fowler circuit breaker
   class CircuitBreaker
     # Raised when too many errors has occured and we refuse to execute the block
@@ -57,7 +57,9 @@ module LogStash
     end
 
     def closed?
-      state == :close || state == :half_open
+      current_state = state
+
+      current_state == :close || current_state == :half_open
     end
 
     private
@@ -73,22 +75,26 @@ module LogStash
     end
 
     def increment_errors(exception)
+      t = Time.now
+
       @mutex.synchronize do
         @errors_count += 1
-        @last_failure_time = Time.now
-
-        logger.debug("CircuitBreaker increment errors", 
-                     :errors_count => @errors_count, 
-                     :error_threshold => @error_threshold,
-                     :exception => exception.class,
-                     :message => exception.message) if logger.debug?
+        @last_failure_time = t
       end
+
+      logger.debug("CircuitBreaker increment errors", 
+                   :errors_count => @errors_count, 
+                   :error_threshold => @error_threshold,
+                   :exception => exception.class,
+                   :message => exception.message) if logger.debug?
     end
 
     def state
+      t = Time.now
+
       @mutex.synchronize do
         if @errors_count >= @error_threshold
-          if Time.now - @last_failure_time > @time_before_retry
+          if t - @last_failure_time > @time_before_retry
             :half_open
           else
             :open

data/lib/logstash/inputs/beats_support/codec_callback_listener.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+
+module LogStash::Inputs::BeatsSupport
+  # Use the new callback based approch instead of using blocks
+  # so we can retain some context of the execution, and make it easier to test
+  class CodecCallbackListener
+    attr_accessor :data
+    # The path acts as the `stream_identity`, 
+    # usefull when the clients is reading multiples files
+    attr_accessor :path
+
+    def initialize(data, hash, path, transformer, queue)
+      @data = data
+      @hash = hash 
+      @path = path
+      @queue = queue
+      @transformer = transformer
+    end
+
+    def process_event(event)
+      @transformer.transform(event, @hash)
+      raise LogStash::Inputs::Beats::InsertingToQueueTakeTooLong if !@queue.offer(event)
+    end
+  end
+end

data/lib/logstash/inputs/beats_support/connection_handler.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/inputs/beats_support/decoded_event_transform"
+require "logstash/inputs/beats_support/raw_event_transform"
+
+module LogStash::Inputs::BeatsSupport
+  # Handle the data coming from a connection
+  # Decide which Process should be used to decode the data coming
+  # from the beat library.
+  #
+  # - Should we use a codec on specific field?
+  # - Should we just take the raw content of the parsed json frame
+  class ConnectionHandler
+    def initialize(connection, input, queue)
+      @connection = connection
+
+      @input = input
+      @queue = queue
+      @logger = input.logger
+
+      # We need to clone the codec per connection, so we can flush a specific 
+      # codec when a connection is closed.
+      @codec = input.codec.dup
+
+      @nocodec_transformer = RawEventTransform.new(@input)
+      @codec_transformer = DecodedEventTransform.new(@input)
+    end
+
+    def accept
+      @logger.debug("Beats input: waiting from new events from remote host",
+                    :peer => @connection.peer)
+
+      @connection.run { |hash, identity_stream| process(hash, identity_stream) }
+    end
+
+    def process(hash, identity_stream)
+      @logger.debug? && @logger.debug("Beats input: new event received",
+                                      :event_hash => hash,
+                                      :identity_stream => identity_stream,
+                                      :peer => @connection.peer)
+
+      # Filebeats uses the `message` key and LSF `line`
+      target_field = @input.target_field_for_codec ? hash.delete(@input.target_field_for_codec) : nil
+
+      if target_field.nil?
+        @logger.debug? && @logger.debug("Beats input: not using the codec for this event, can't find the codec target field",
+                                        :target_field_for_codec => @input.target_field_for_codec,
+                                        :event_hash => hash)
+
+        event = LogStash::Event.new(hash)
+        @nocodec_transformer.transform(event)
+
+        raise LogStash::Inputs::Beats::InsertingToQueueTakeTooLong if !@queue.offer(event)
+      else
+        @logger.debug? && @logger.debug("Beats input: decoding this event with the codec",
+                                        :target_field_value =>  target_field)
+
+        @codec.accept(CodecCallbackListener.new(target_field,
+                                                hash,
+                                                identity_stream,
+                                                @codec_transformer,
+                                                @queue))
+      end
+    end
+
+    # OOB call to flush the codec buffer, 
+    #
+    # This method is a bit tricky to decide when to be called, in the current case,
+    # this will be call on any exception raised, either is a circuit breaker or the
+    # remote host closed the connection, its better to make sure we clear their
+    # data and create duplicates then losing the data.
+    def flush(&block)
+      @logger.debug? && @logger.debug("Beats input, out of band call for flushing the content of this connection",
+                                      :peer => @connection.peer)
+
+      @codec.flush(&block)
+    end
+  end
+end