logstash-input-beats 2.0.3 → 2.1.1

data/lib/logstash/inputs/beats_support/decoded_event_transform.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require "logstash/inputs/beats_support/event_transform_common"
+module LogStash::Inputs::BeatsSupport
+  # Take the extracted content from the codec, merged with the other data coming
+  # from beats, apply the configured tags, normalize the host and try to coerce
+  # the timestamp if it was provided in the hash.
+  class DecodedEventTransform < EventTransformCommon
+    def transform(event, hash)
+      ts = coerce_ts(hash.delete("@timestamp"))
+
+      event["@timestamp"] = ts unless ts.nil?
+      hash.each { |k, v| event[k] = v }
+      super(event)
+      event.tag("beats_input_codec_#{@input.codec.base_codec.class.config_name}_applied")
+      event
+    end
+
+    private
+    def coerce_ts(ts)
+      return nil if ts.nil?
+      timestamp = LogStash::Timestamp.coerce(ts)
+
+      return timestamp if timestamp
+
+      @logger.warn("Unrecognized @timestamp value, setting current time to @timestamp",
+                   :value => ts.inspect)
+      return nil
+    rescue LogStash::TimestampParserError => e
+      @logger.warn("Error parsing @timestamp string, setting current time to @timestamp",
+                   :value => ts.inspect, :exception => e.message)
+      return nil
+    end
+  end
+end

data/lib/logstash/inputs/beats_support/event_transform_common.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+module LogStash::Inputs::BeatsSupport
+  # Base Transform class, expose the plugin decorate method,
+  # apply the tags and make sure we copy the beat hostname into `host`
+  # for backward compatibility.
+  class EventTransformCommon
+    def initialize(input)
+      @input = input
+      @logger = input.logger
+    end
+
+    # Copies the beat.hostname field into the host field unless
+    # the host field is already defined
+    def copy_beat_hostname(event)
+      host = event["[beat][hostname]"]
+
+      if host && event["host"].nil?
+        event["host"] = host
+      end
+    end
+
+    # This break the `#decorate` method visibility of the plugin base
+    # class, the method is protected and we cannot access it, but well ruby
+    # can let you do all the wrong thing.
+    #
+    # I think the correct behavior would be to allow the plugin to return a 
+    # `Decorator` object that we can pass to other objects, since only the 
+    # plugin know the data used to decorate. This would allow a more component
+    # based workflow.
+    def decorate(event)
+      @input.send(:decorate, event)
+    end
+
+    def transform(event)
+      copy_beat_hostname(event)
+      decorate(event)
+      event
+    end
+  end
+end

data/lib/logstash/inputs/beats_support/raw_event_transform.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/inputs/beats_support/event_transform_common"
+module LogStash::Inputs::BeatsSupport
+  # Take the the raw output from the library, decorate it with
+  # the configured tags in the plugins and normalize the hostname
+  # for backward compatibility
+  #
+  #
+  # @see [Lumberjack::Beats::Parser]
+  #
+  class RawEventTransform < EventTransformCommon
+    def transform(event)
+      super(event)
+      event.tag("beats_input_raw_event")
+      event
+    end
+  end
+end

data/lib/logstash/inputs/beats_support/synchronous_queue_with_offer.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+module LogStash::Inputs::BeatsSupport
+  # Wrap the `Java SynchronousQueue` to acts as the synchronization mechanism
+  # this queue can block for a maximum amount of time logstash's queue 
+  # doesn't implement that feature.
+  #  
+  # See proposal for core: https://github.com/elastic/logstash/pull/4408
+  #
+  # See https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/SynchronousQueue.html
+  java_import "java.util.concurrent.SynchronousQueue"
+  java_import "java.util.concurrent.TimeUnit"
+  class SynchronousQueueWithOffer
+    def initialize(timeout, fairness_policy = true)
+      # set Fairness policy to `FIFO`
+      #
+      # In the context of the input it makes sense to
+      # try to deal with the older connection before
+      # the newer one, since the older will be closer to
+      # reach the connection timeout.
+      #
+      @timeout = timeout
+      @queue = java.util.concurrent.SynchronousQueue.new(fairness_policy)
+    end
+
+    # This method will return true if it successfully added the element to the queue.
+    # If the timeout is reached and it wasn't inserted successfully to 
+    # the queue it will return false.
+    def offer(element, timeout = nil)
+      @queue.offer(element, timeout || @timeout, java.util.concurrent.TimeUnit::SECONDS)
+    end
+
+    def take
+      @queue.take
+    end
+  end
+end

data/lib/lumberjack/beats/server.rb

@@ -57,6 +57,13 @@ module Lumberjack module Beats
       end
     end # def initialize
 
+    # Server#run method, allow the library to manage all the connection 
+    # threads, this handing is quite minimal and don't handler
+    # all the possible cases deconnection/connection.
+    #
+    # To have a more granular control over the connection you should manage
+    # them yourself, see Server#accept method which return a Connection
+    # instance.
     def run(&block)
       while !closed?
         connection = accept
@@ -67,7 +74,14 @@ module Lumberjack module Beats
         next unless connection
 
         Thread.new(connection) do |connection|
-          connection.run(&block)
+          begin
+            connection.run(&block)
+          rescue Lumberjack::Beats::Connection::ConnectionClosed 
+            # Connection will raise a wrapped exception upstream, 
+            # but if the threads are managed by the library we can simply ignore it.
+            #
+            # Note: This follow the previous behavior of the perfect silence.
+          end
         end
       end
     end # def run
@@ -132,6 +146,7 @@ module Lumberjack module Beats
     PROTOCOL_VERSION_2 = "2".ord
 
     SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
+    class UnsupportedProtocol < StandardError; end
 
     def initialize
       @buffer_offset = 0
@@ -222,7 +237,7 @@ module Lumberjack module Beats
       if supported_protocol?(version)
         yield :version, version
       else
-        raise "unsupported protocol #{version}"
+        raise UnsupportedProtocol, "unsupported protocol #{version}"
       end
     end
 
@@ -298,9 +313,37 @@ module Lumberjack module Beats
   end # class Parser
 
   class Connection
+    # Wrap the original exception into a common one, 
+    # to make upstream managing and reporting easier
+    # But lets make sure we keep the meaning of the original exception.
+    class ConnectionClosed < StandardError
+      attr_reader :original_exception
+
+      def initialize(original_exception)
+        super(original_exception)
+
+        @original_exception = original_exception
+        set_backtrace(original_exception.backtrace) if original_exception
+      end
+
+      def to_s
+        "#{self.class.name} wrapping: #{original_exception.class.name}, #{super.to_s}"
+      end
+    end
+
     READ_SIZE = 16384
+    PEER_INFORMATION_NOT_AVAILABLE = "<PEER INFORMATION NOT AVAILABLE>"
+    RESCUED_CONNECTION_EXCEPTIONS = [
+      EOFError,
+      OpenSSL::SSL::SSLError,
+      IOError,
+      Errno::ECONNRESET,
+      Errno::EPIPE,
+      Lumberjack::Beats::Parser::UnsupportedProtocol
+    ]
 
     attr_accessor :server
+    attr_reader :peer
 
     def initialize(fd, server)
       @parser = Parser.new
@@ -308,23 +351,27 @@ module Lumberjack module Beats
 
       @server = server
       @ack_handler = nil
-    end
-
-    def peer
-      "#{@fd.peeraddr[3]}:#{@fd.peeraddr[1]}"
+      
+      # Fetch the details of the host before reading anything from the socket
+      # se we can use that information when debugging connection issues with
+      # remote hosts.
+      begin
+        @peer = "#{@fd.peeraddr[3]}:#{@fd.peeraddr[1]}"
+      rescue IOError
+        # This can happen if the connection is drop or close before
+        # fetching the host details, lets return a generic string.
+        @peer = PEER_INFORMATION_NOT_AVAILABLE
+      end
     end
 
     def run(&block)
       while !server.closed?
         read_socket(&block)
       end
-    rescue EOFError,
-      OpenSSL::SSL::SSLError,
-      IOError,
-      Errno::ECONNRESET,
-      Errno::EPIPE
+    rescue *RESCUED_CONNECTION_EXCEPTIONS => e
       # EOF or other read errors, only action is to shutdown which we'll do in
       # 'ensure'
+      raise ConnectionClosed.new(e)
     rescue
       # when the server is shutting down we can safely ignore any exceptions
       # On windows, we can get a `SystemCallErr`

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = "2.0.3"
+  s.version         = "2.1.1"
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","vendor/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -23,7 +23,8 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "concurrent-ruby", "~> 0.9.2"
-  s.add_runtime_dependency "logstash-codec-multiline", "~> 2.0.3"
+  s.add_runtime_dependency "thread_safe", "~> 0.3.5"
+  s.add_runtime_dependency "logstash-codec-multiline", "~> 2.0.5"
 
   s.add_development_dependency "flores", "~>0.0.6"
   s.add_development_dependency "rspec"

data/spec/inputs/beats_spec.rb

@@ -70,140 +70,49 @@ describe LogStash::Inputs::Beats do
     end
   end
 
-  describe "#processing of events" do
-    subject(:beats) { LogStash::Inputs::Beats.new(config) }
-    let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '\n', "what" => "previous") }
+  context "#handle_new_connection" do
+    let(:config) {{ "ssl" => false, "port" => 0, "type" => "example", "tags" => "beats" }}
+    let(:plugin) { LogStash::Inputs::Beats.new(config) }
+    let(:connection) { DummyConnection.new(events) }
+    let(:buffer_queue) { DummyNeverBlockedQueue.new }
+    let(:pipeline_queue)  { [] }
+    let(:events) {
+      [
+        { :map => { "id" => 1 }, :identity_stream => "/var/log/message" },
+        { :map => { "id" => 2 }, :identity_stream => "/var/log/message_2" }
+      ]
+    }
+
+    before :each do
+      plugin.register
+
+      # Event if we dont mock the actual socket work
+      # we have to call run because it will correctly setup the queues
+      # instances variables
+      t = Thread.new do
+        plugin.run(pipeline_queue)
+      end
 
-    let(:config) do
-      { "port" => port, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key,
-        "type" => "example", "codec" => codec }
+      sleep(0.1) until t.status == "run"
     end
 
-    before do
-      beats.register
+    after :each do
+      plugin.stop
     end
 
-    context "#create_event" do
-      let(:config) { super.merge({ "add_field" => { "foo" => "bar", "[@metadata][hidden]" => "secret"}, "tags" => ["bonjour"]}) }
-      let(:event_map) { { "hello" => "world" } }
-      let(:codec) { LogStash::Codecs::Plain.new }
-      let(:identity_stream) { "custom-type-input_type-source" }
-
-      context "without a `target_field` defined" do
-        it "decorates the event" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["foo"]).to eq("bar")
-            expect(event["[@metadata][hidden]"]).to eq("secret")
-            expect(event["tags"]).to include("bonjour")
-          end
-        end
-      end
-
-      context "with a `target_field` defined" do
-        let(:event_map) { super.merge({"message" => "with a field"}) }
-
-        it "decorates the event" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["foo"]).to eq("bar")
-            expect(event["[@metadata][hidden]"]).to eq("secret")
-            expect(event["tags"]).to include("bonjour")
-          end
-        end
-      end
-
-      context "when data is buffered in the codec" do
-        let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^\s', "what" => "previous") }
-        let(:event_map) { {"message" => "hello?", "tags" => ["syslog"]} }
-
-        it "returns nil" do
-          expect { |b| beats.create_event(event_map, identity_stream, &b) }.not_to yield_control
-        end
-      end
-
-      context "multiline" do
-        let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015', "what" => "previous", "negate" => true) }
-        let(:events_map) do
-          [
-            { "beat" => { "id" => "main", "resource_id" => "md5"},  "message" => "2015-11-10 10:14:38,907 line 1" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 1.1" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "2015-11-10 10:16:38,907 line 2" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.1" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.2" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "line 2.3" },
-            { "beat" => { "id" => "main", "resource_id" => "md5"}, "message" => "2015-11-10 10:18:38,907 line 3" }
-          ]
-        end
-
-        let(:queue) { [] }
-        before do
-          Thread.new { beats.run(queue) }
-          sleep(0.1)
-        end
-
-        it "should correctly merge multiple events" do
-          events_map.each { |map| beats.create_event(map, identity_stream) { |e| queue << e } }
-          # This cannot currently work without explicitely call a flush
-          # the flush is never timebased, if no new data is coming in we wont flush the buffer
-          # https://github.com/logstash-plugins/logstash-codec-multiline/issues/11
-          beats.stop
-          expect(queue.size).to eq(3)
-          
-          expect(queue.collect { |e| e["message"] }).to include("2015-11-10 10:14:38,907 line 1\nline 1.1",
-                                                           "2015-11-10 10:16:38,907 line 2\nline 2.1\nline 2.2\nline 2.3",
-                                                           "2015-11-10 10:18:38,907 line 3")
-        end
-      end
-
-      context "with a beat.hostname field" do
-        let(:event_map) { {"message" => "hello", "beat" => {"hostname" => "linux01"} } }
-
-        it "copies it to the host field" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["host"]).to eq("linux01")
-          end
-        end
+    context "when an exception occur" do
+      let(:connection_handler) { LogStash::Inputs::BeatsSupport::ConnectionHandler.new(connection, plugin, buffer_queue) }
+      before do 
+        expect(LogStash::Inputs::BeatsSupport::ConnectionHandler).to receive(:new).with(any_args).and_return(connection_handler)
       end
 
-      context "with a beat.hostname field but without the message" do
-        let(:event_map) { {"beat" => {"hostname" => "linux01"} } }
+      it "calls flush on the handler and tag the events" do
+        expect(connection_handler).to receive(:accept) { raise LogStash::Inputs::Beats::InsertingToQueueTakeTooLong }
+        expect(connection_handler).to receive(:flush).and_yield(LogStash::Event.new)
+        plugin.handle_new_connection(connection)
 
-        it "copies it to the host field" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["host"]).to eq("linux01")
-          end
-        end
-      end
-
-      context "without a beat.hostname field" do
-        let(:event_map) { {"message" => "hello", "beat" => {"name" => "linux01"} } }
-
-        it "should not add a host field" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["beat"]["name"]).to eq("linux01")
-            expect(event["host"]).to be_nil
-          end
-        end
-      end
-
-      context "with a beat.hostname and host fields" do
-        let(:event_map) { {"message" => "hello", "host" => "linux02", "beat" => {"hostname" => "linux01"} } }
-
-        it "should not overwrite host" do
-          beats.create_event(event_map, identity_stream) do |event|
-            expect(event["host"]).to eq("linux02")
-          end
-        end
-      end
-
-      context "with a host field in the message" do
-        let(:codec) { LogStash::Codecs::JSON.new }
-        let(:event_map) { {"message" => '{"host": "linux02"}', "beat" => {"hostname" => "linux01"} } }
-
-        it "should take the host from the JSON message" do
-          beats.create_event(event_map, identity_stream) do
-            expect(event["host"]).to eq("linux02")
-          end
-        end
+        event = pipeline_queue.shift
+        expect(event["tags"]).to include("beats_input_flushed_by_end_of_connection")
       end
     end
   end