logstash-input-beats 2.0.3 → 2.1.1

data/spec/{logstash → inputs/beats_support}/circuit_breaker_spec.rb

@@ -1,9 +1,9 @@
-require_relative "../spec_helper"
-require "logstash/circuit_breaker"
+require_relative "../../spec_helper"
+require "logstash/inputs/beats_support/circuit_breaker"
 
 class DummyErrorTest < StandardError; end
 
-describe LogStash::CircuitBreaker do
+describe LogStash::Inputs::BeatsSupport::CircuitBreaker do
   let(:error_threshold) { 1 }
   let(:options) do 
     {
@@ -12,7 +12,7 @@ describe LogStash::CircuitBreaker do
     }
   end
 
-  subject { LogStash::CircuitBreaker.new("testing", options) }
+  subject { described_class.new("testing", options) }
 
   context "when the breaker is closed" do
     it "closed by default" do
@@ -24,7 +24,7 @@ describe LogStash::CircuitBreaker do
         subject.execute do
           raise DummyErrorTest
         end
-      }.to raise_error(LogStash::CircuitBreaker::HalfOpenBreaker)
+      }.to raise_error(LogStash::Inputs::BeatsSupport::CircuitBreaker::HalfOpenBreaker)
     end
 
     it "open if we pass the errors threadshold" do
@@ -32,18 +32,19 @@ describe LogStash::CircuitBreaker do
         subject.execute do
           raise DummyErrorTest
         end
-      }.to raise_error(LogStash::CircuitBreaker::HalfOpenBreaker)
+      }.to raise_error(LogStash::Inputs::BeatsSupport::CircuitBreaker::HalfOpenBreaker)
 
       expect {
         subject.execute do
           raise DummyErrorTest
         end
-      }.to raise_error(LogStash::CircuitBreaker::OpenBreaker)
+      }.to raise_error(LogStash::Inputs::BeatsSupport::CircuitBreaker::OpenBreaker)
     end
   end
 
   context "When the breaker is open" do
-    let(:future_time) { Time.now + 3600 }
+    let(:retry_time) { 2 }
+    let(:options) { super.merge(:time_before_retry => retry_time) }
 
     before do
       # trip the breaker
@@ -62,7 +63,7 @@ describe LogStash::CircuitBreaker do
     end
 
     it "resets the breaker after the time before retry" do
-      expect(Time).to receive(:now).at_least(2).and_return(future_time)
+      sleep(retry_time + 1)
       expect(subject.closed?).to eq(true)
     end
 
@@ -73,7 +74,7 @@ describe LogStash::CircuitBreaker do
         subject.execute do
           runned = true
         end
-      rescue LogStash::CircuitBreaker::OpenBreaker 
+      rescue LogStash::Inputs::BeatsSupport::CircuitBreaker::OpenBreaker 
       end
 
       expect(runned).to eq(false)

data/spec/inputs/beats_support/codec_callback_listener_spec.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/event"
+require "spec_helper"
+require "thread"
+
+Thread.abort_on_exception = true
+describe LogStash::Inputs::BeatsSupport::CodecCallbackListener do
+  let(:data) { "Hello world" }
+  let(:map) do
+    {
+      "beat" => { "hostname" => "newhost" }
+    }
+  end
+  let(:path) { "/var/log/message" }
+  let(:transformer) { double("codec_transformer") }
+  let(:queue_timeout) { 1 }
+  let(:queue) { LogStash::Inputs::BeatsSupport::SynchronousQueueWithOffer.new(queue_timeout) }
+  let(:event) { LogStash::Event.new }
+
+  before do
+    allow(transformer).to receive(:transform).with(event, map).and_return(event)
+  end
+
+  subject { described_class.new(data, map, path, transformer, queue) }
+
+  it "expose the data" do
+    expect(subject.data).to eq(data)
+  end
+
+  it "expose the path" do
+    expect(subject.path).to eq(path)
+  end
+  
+  context "when the queue is not blocked for too long" do
+    let(:queue_timeout) { 5 }
+
+    it "doesnt raise an exception" do
+      Thread.new do
+        loop { queue.take }
+      end
+      sleep(0.1)
+      expect { subject.process_event(event) }.not_to raise_error
+    end
+  end
+
+  context "when the queue is blocked for too long" do
+    it "raises an exception" do
+      expect { subject.process_event(event) }.to raise_error(LogStash::Inputs::Beats::InsertingToQueueTakeTooLong)
+    end
+  end
+end

data/spec/inputs/beats_support/connection_handler_spec.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require_relative "../../support/logstash_test"
+require "spec_helper"
+
+describe LogStash::Inputs::BeatsSupport::ConnectionHandler do
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => "beats"
+    }
+  end
+
+  let(:logger) { DummyLogger.new  }
+  let(:input) do
+    LogStash::Inputs::Beats.new(config).tap do |i|
+      i.register
+    end
+  end
+  let(:connection) { double("connection") }
+  let(:queue) { DummyNeverBlockedQueue.new }
+
+  subject { described_class.new(connection, input, queue) }
+  
+  context "#accept" do
+    let(:connection) { DummyConnection.new(events) }
+    let(:events) {
+      [
+        { :map => { "id" => 1 }, :identity_stream => "/var/log/message" },
+        { :map => { "id" => 2 }, :identity_stream => "/var/log/message_2" }
+      ]
+    }
+    it "should delegate work to `#process` for each items" do
+      events.each do |element|
+        expect(subject).to receive(:process).with(element[:map], element[:identity_stream]).and_return(true)
+      end
+
+      subject.accept
+    end
+  end
+
+  context "#process" do
+    let(:identity_stream) { "/var/log/message" }
+
+    context "without a `target_field_for_codec`" do
+      let(:map) { { "hello" => "world" } }
+
+      context "queue is not blocked" do
+        it "insert the event into the queue" do
+          subject.process(map, identity_stream)
+          event = queue.take
+
+          expect(event["hello"]).to eq(map["hello"])
+          expect(event["tags"]).to include("beats_input_raw_event")
+        end
+      end
+
+      context "queue is blocked" do
+        let(:queue_timeout) { 1 }
+        let(:queue) { LogStash::Inputs::BeatsSupport::SynchronousQueueWithOffer.new(queue_timeout) }
+      
+        it "raise an exception" do
+          expect { subject.process(map, identity_stream) }.to raise_error(LogStash::Inputs::Beats::InsertingToQueueTakeTooLong)
+        end
+      end
+    end
+
+    context "with a codec" do
+      let(:message) { "Hello world" }
+      let(:map) { { "message" => message } }
+
+      context "queue is not blocked" do
+        it "insert the event into the queue" do
+          subject.process(map, identity_stream)
+          event = queue.take
+
+          expect(event["message"]).to eq(message)
+          expect(event["tags"]).to include("beats_input_codec_plain_applied")
+        end
+      end
+
+      context "queue is blocked" do
+        let(:queue_timeout) { 1 }
+        let(:queue) { LogStash::Inputs::BeatsSupport::SynchronousQueueWithOffer.new(queue_timeout) }
+
+        it "raise an exception" do
+          expect { subject.process(map, identity_stream) }.to raise_error(LogStash::Inputs::Beats::InsertingToQueueTakeTooLong)
+        end
+      end
+    end
+  end
+end

data/spec/inputs/beats_support/decoded_event_transform_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/timestamp"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::BeatsSupport::DecodedEventTransform do
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => "beats"
+    }
+  end
+
+  let(:input) do
+    LogStash::Inputs::Beats.new(config).tap do |i|
+      i.register
+    end
+  end
+  let(:event) { LogStash::Event.new }
+  let(:map) do
+    {
+      "@metadata" =>  { "hello" => "world"},
+      "super" => "mario"
+    }
+  end
+
+  subject { described_class.new(input).transform(event, map) }
+
+  include_examples "Common Event Transformation"
+
+  it "tags the event" do
+    expect(subject["tags"]).to include("beats_input_codec_plain_applied")
+  end
+
+  it "merges the other data from the map to the event" do
+    expect(subject["super"]).to eq(map["super"])
+    expect(subject["@metadata"]).to include(map["@metadata"])
+  end
+
+  context "map contains a timestamp" do
+    context "when its valid" do
+      let(:timestamp) { Time.now }
+      let(:map) { super.merge({"@timestamp" => timestamp }) }
+     
+      it "uses as the event timestamp" do
+        expect(subject["@timestamp"]).to eq(LogStash::Timestamp.coerce(timestamp)) 
+      end
+    end
+
+    context "when its not valid" do
+      let(:map) { super.merge({"@timestamp" => "invalid" }) }
+
+      it "fallback the current time" do
+        expect(subject["@timestamp"]).to be_kind_of(LogStash::Timestamp)
+      end
+    end
+  end
+
+  context "when the map doesn't provide a timestamp" do
+    it "fallback the current time" do
+      expect(subject["@timestamp"]).to be_kind_of(LogStash::Timestamp)
+    end
+  end
+end

data/spec/inputs/beats_support/event_transform_common_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::BeatsSupport::EventTransformCommon do
+  subject { described_class.new(input).transform(event) }
+
+  include_examples "Common Event Transformation"
+end

data/spec/inputs/beats_support/raw_event_transform_spec.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::BeatsSupport::RawEventTransform do
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => "beats"
+    }
+  end
+
+  let(:input) { LogStash::Inputs::Beats.new(config) }
+  let(:event) { LogStash::Event.new }
+
+  subject { described_class.new(input).transform(event) }
+
+  include_examples "Common Event Transformation"
+
+  it "tags the event" do
+    expect(subject["tags"]).to include("beats_input_raw_event")
+  end
+end

data/spec/integration_spec.rb

@@ -17,7 +17,7 @@ describe "A client" do
   let(:host) { "127.0.0.1" }
   let(:queue) { [] }
 
-  before do
+  before :each do
     expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
     expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
 
@@ -29,23 +29,33 @@ describe "A client" do
                                         :ssl_key => certificate_file_key)
 
     @tcp_server = Thread.new do
+      tcp_server.run { |data, identity_stream| queue << [data, identity_stream] }
       while true
-        tcp_server.accept do |socket|
-          con = Lumberjack::Beats::Connection.new(socket, tcp_server)
-          begin
-            con.run { |data, identity_stream| queue << [data, identity_stream] }
-          rescue
-            # Close connection on failure. For example SSL client will make
-            # parser for TCP based server trip.
-            # Connection is closed by Server connection object
-          end
-        end
+          tcp_server.accept do |socket|
+            next if socket.nil?
+
+            begin
+              con = Lumberjack::Beats::Connection.new(socket, tcp_server)
+              con.run { |data, identity_stream| queue << [data, identity_stream] }
+            rescue
+              # Close connection on failure. For example SSL client will make
+              # parser for TCP based server trip.
+              # Connection is closed by Server connection object
+            end
+          end 
       end
     end
 
     @ssl_server = Thread.new do
       ssl_server.run { |data, identity_stream| queue << [data, identity_stream] }
     end
+
+    sleep(0.1) while @ssl_server.status != "run" && @tcp_server != "run"
+  end
+
+  after :each do
+    @tcp_server.kill
+    @ssl_server.kill
   end
 
   shared_examples "send payload" do
@@ -132,7 +142,7 @@ describe "A client" do
                                                :addresses => host,
                                                :ssl => false)
         client.write({ "line" => "foobar" })
-      }.to raise_error(RuntimeError)
+      }.to raise_error
     end
 
     context "When transmitting a payload" do

data/spec/lumberjack/beats/server_spec.rb

@@ -32,9 +32,9 @@ describe "Server" do
       end
     end
 
-    sleep(1) while thread.status != "run"
+    sleep(0.1) while thread.status != "run"
     server.close
-    sleep(2)
-    expect(thread.status).to be_falsey
+
+    wait_for { thread.status }.to be_falsey
   end
 end

data/spec/support/logstash_test.rb

@@ -21,3 +21,28 @@ module LogStashTest
     end
   end
 end
+
+class DummyNeverBlockedQueue < Array
+  def offer(element, timeout = nil)
+    push(element)
+  end
+
+  alias_method :take, :shift
+end
+
+class DummyConnection
+  def initialize(events)
+    @events = events
+  end
+
+  def run
+    @events.each do |element|
+      yield element[:map], element[:identity_stream] 
+    end
+  end
+
+  def peer
+    "localhost:5555"
+  end
+end
+

data/spec/support/shared_examples.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+shared_examples "Common Event Transformation" do
+  let(:tag) { "140-rpm-beats" }
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => tag
+    }
+  end
+
+  let(:input) do
+    LogStash::Inputs::Beats.new(config).tap do |i|
+      i.register
+    end
+  end
+  let(:event) { LogStash::Event.new(event_map) }
+  let(:event_map) do
+    {
+      "message" => "Hello world",
+    }
+  end
+
+  it "adds configured tags to the event" do
+    expect(subject["tags"]).to include(tag)
+  end
+
+  context "when the `beast.hotname` doesnt exist on the event" do
+    let(:already_exist) { "already_exist" }
+    let(:event_map) { super.merge({ "host" => already_exist }) }
+
+    it "doesnt change the value" do
+      expect(subject["host"]).to eq(already_exist)
+    end
+  end
+
+  context "when the `beat.hostname` exist in the event" do
+    let(:producer_host) { "newhost01" }
+    let(:event_map) { super.merge({ "beat" => { "hostname" => producer_host }}) }
+
+    context "when `host` key doesn't exist on the event" do
+      it "copy the `beat.hostname` to `host` or backward compatibility" do
+        expect(subject["host"]).to eq(producer_host)
+      end
+    end
+
+    context "when `host` key exists on the event" do
+      let(:already_exist) { "already_exist" }
+      let(:event_map) { super.merge({ "host" => already_exist }) }
+
+      it "doesn't override it" do
+        expect(subject["host"]).to eq(already_exist)
+      end
+    end
+  end
+end