logstash-core 7.0.0.alpha2-java → 7.0.0.beta1-java

data/spec/logstash/agent/converge_spec.rb

@@ -49,7 +49,7 @@ describe LogStash::Agent do
 
     context "system pipeline" do
       
-      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { generator { } } output { null {} }", { "pipeline.system" => true }) }
+      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { dummyblockinginput { } } output { null {} }", { "pipeline.system" => true }) }
 
       context "when we have a finite pipeline and a system pipeline running" do
 
@@ -65,40 +65,40 @@ describe LogStash::Agent do
       end
 
       context "when we have an infinite pipeline and a system pipeline running" do
-        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { generator { } } output { null {} }") }
+        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { } } output { null {} }") }
 
         let(:source_loader) do
           TestSourceLoader.new(infinite_pipeline_config, system_pipeline_config)
         end
 
         before(:each) do
-            @agent_task = start_agent(subject)
+          @agent_task = start_agent(subject)
         end
 
         after(:each) do
-            @agent_task.stop!
+          @agent_task.stop!
+          @agent_task.wait
+          subject.shutdown
         end
 
         describe "#running_user_defined_pipelines" do
           it "returns the user defined pipelines" do
-            wait_for do
-              subject.with_running_user_defined_pipelines {|pipelines| pipelines.keys }
-            end.to eq([:main])
-          end
+            # wait is necessary to accommodate for pipelines startup time
+            wait(60).for {subject.running_user_defined_pipelines.keys}.to eq([:main])
+           end
         end
 
         describe "#running_user_defined_pipelines?" do
           it "returns true" do
-            wait_for do
-              subject.running_user_defined_pipelines?
-            end.to be_truthy
+            # wait is necessary to accommodate for pipelines startup time
+            wait(60).for {subject.running_user_defined_pipelines?}.to be_truthy
           end
         end
       end
     end
 
     context "when `config.reload.automatic`" do
-      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
 
       let(:source_loader) do
         TestSourceLoader.new(pipeline_config)
@@ -114,14 +114,14 @@ describe LogStash::Agent do
 
           after(:each) do
             @agent_task.stop!
+            @agent_task.wait
+            subject.shutdown
           end
 
           it "converge only once" do
             wait(60).for { source_loader.fetch_count }.to eq(1)
-
+            # no need to wait here because have_running_pipeline? does the wait
             expect(subject).to have_running_pipeline?(pipeline_config)
-
-            subject.shutdown
           end
         end
 
@@ -135,8 +135,6 @@ describe LogStash::Agent do
 
             expect(source_loader.fetch_count).to eq(1)
             expect(subject.pipelines_count).to eq(0)
-
-            subject.shutdown
           end
         end
       end
@@ -149,26 +147,25 @@ describe LogStash::Agent do
             "config.reload.interval" =>  interval
           )
         end
+
         before(:each) do
           @agent_task = start_agent(subject)
         end
 
         after(:each) do
           @agent_task.stop!
+          @agent_task.wait
+          subject.shutdown
         end
 
         context "and successfully load the config" do
           it "converges periodically the pipelines from the configs source" do
-            sleep(2) # let the interval reload a few times
+            # no need to wait here because have_running_pipeline? does the wait
             expect(subject).to have_running_pipeline?(pipeline_config)
 
             # we rely on a periodic thread to call fetch count, we have seen unreliable run on
             # travis, so lets add a few retries
-            try do
-              expect(source_loader.fetch_count).to be > 1
-            end
-
-            subject.shutdown
+            try { expect(source_loader.fetch_count).to be > 1 }
           end
         end
 
@@ -178,12 +175,9 @@ describe LogStash::Agent do
           end
 
           it "it will keep trying to converge" do
-
             sleep(agent_settings.get("config.reload.interval") / 1_000_000_000.0 * 20) # let the interval reload a few times
             expect(subject.pipelines_count).to eq(0)
             expect(source_loader.fetch_count).to be > 1
-
-            subject.shutdown
           end
         end
       end
@@ -191,8 +185,8 @@ describe LogStash::Agent do
   end
 
   context "when shutting down the agent" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator { id => 'new' } } output { null {} }") }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput { id => 'new' } } output { null {} }") }
 
     let(:source_loader) do
       TestSourceLoader.new([pipeline_config, new_pipeline_config])
@@ -205,8 +199,8 @@ describe LogStash::Agent do
   end
 
   context "Configuration converge scenario" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
 
     before do
       # Set the Agent to an initial state of pipelines
@@ -263,7 +257,7 @@ describe LogStash::Agent do
     end
 
     context "when the source return a modified pipeline" do
-      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { generator { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
+      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
 
       let(:source_loader) do
         TestSequenceSourceLoader.new(

data/spec/logstash/agent_spec.rb

@@ -119,7 +119,7 @@ describe LogStash::Agent do
         context "if state is clean" do
           before :each do
             allow(subject).to receive(:running_user_defined_pipelines?).and_return(true)
-            allow(subject).to receive(:clean_state?).and_return(false)
+            allow(subject).to receive(:no_pipeline?).and_return(false)
           end
 
           it "should not converge state more than once" do
@@ -142,7 +142,7 @@ describe LogStash::Agent do
             it "does not upgrade the new config" do
               t = Thread.new { subject.execute }
               wait(timeout)
-                  .for { subject.running_pipelines? && subject.pipelines.values.first.ready? }
+                  .for { subject.running_pipelines? && subject.running_pipelines.values.first.ready? }
                   .to eq(true)
               expect(subject.converge_state_and_update).not_to be_a_successful_converge
               expect(subject).to have_running_pipeline?(mock_config_pipeline)
@@ -162,7 +162,7 @@ describe LogStash::Agent do
             it "does upgrade the new config" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.pipelines_count > 0 && subject.pipelines.values.first.ready?
+                sleep(0.1) until subject.running_pipelines_count > 0 && subject.running_pipelines.values.first.ready?
               end
 
               expect(subject.converge_state_and_update).to be_a_successful_converge
@@ -186,7 +186,7 @@ describe LogStash::Agent do
             it "does not try to reload the pipeline" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
+                sleep(0.1) until subject.running_pipelines? && subject.running_pipelines.values.first.running?
               end
               expect(subject.converge_state_and_update).not_to be_a_successful_converge
               expect(subject).to have_running_pipeline?(mock_config_pipeline)
@@ -206,7 +206,7 @@ describe LogStash::Agent do
             it "tries to reload the pipeline" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
+                sleep(0.1) until subject.running_pipelines? && subject.running_pipelines.values.first.running?
               end
 
               expect(subject.converge_state_and_update).to be_a_successful_converge

data/spec/logstash/event_spec.rb

@@ -141,10 +141,10 @@ describe LogStash::Event do
       expect(e.get("foo")).to eq(BigDecimal.new(1))
     end
 
-    it "should set RubyBignum" do
+    it "should set RubyInteger" do
       e = LogStash::Event.new()
       e.set("[foo]", -9223372036854776000)
-      expect(e.get("foo")).to be_kind_of(Bignum)
+      expect(e.get("foo")).to be_kind_of(Integer)
       expect(e.get("foo")).to eq(-9223372036854776000)
     end
 

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -110,7 +110,7 @@ describe LogStash::WrappedWriteClient do
 
   context "WrappedAckedQueue" do
     let(:path) { Stud::Temporary.directory }
-    let(:queue) { LogStash::WrappedAckedQueue.new(path, 1024, 10, 1024, 1024, 1024, 4096) }
+    let(:queue) { LogStash::WrappedAckedQueue.new(path, 1024, 10, 1024, 1024, 1024, false, 4096) }
 
     before do
       read_client.set_events_metric(metric.namespace([:stats, :events]))

data/spec/logstash/legacy_ruby_event_spec.rb

@@ -82,7 +82,7 @@ describe LogStash::Event do
         expect(event.get("reference_test")).not_to eq(data)
       end
 
-      it "should not return a Fixnum reference" do
+      it "should not return an Integer reference" do
         data = 1
         event = LogStash::Event.new({ "reference" => data })
         LogStash::Util::Decorators.add_fields({"reference_test" => "%{reference}"}, event, "dummy-plugin")
@@ -98,11 +98,12 @@ describe LogStash::Event do
         expect(subject.sprintf("bonjour")).to eq("bonjour")
       end
 
-      it "should raise error when formatting %{+%s} when @timestamp field is missing" do
+      it "should not raise error and should format as empty string when @timestamp field is missing" do
         str = "hello-%{+%s}"
         subj = subject.clone
         subj.remove("[@timestamp]")
-        expect{ subj.sprintf(str) }.to raise_error(LogStash::Error)
+        expect{ subj.sprintf(str) }.not_to raise_error(LogStash::Error)
+        expect(subj.sprintf(str)).to eq("hello-")
       end
 
       it "should report a time with %{+format} syntax", :if => RUBY_ENGINE == "jruby" do
@@ -115,11 +116,11 @@ describe LogStash::Event do
         expect(subject.sprintf("foo %{+YYYY-MM-dd} %{type}")).to eq("foo 2013-01-01 sprintf")
       end
 
-      it "should raise error with %{+format} syntax when @timestamp field is missing", :if => RUBY_ENGINE == "jruby" do
+      it "should not raise error with %{+format} syntax when @timestamp field is missing", :if => RUBY_ENGINE == "jruby" do
         str = "logstash-%{+YYYY}"
         subj = subject.clone
         subj.remove("[@timestamp]")
-        expect{ subj.sprintf(str) }.to raise_error(LogStash::Error)
+        expect{ subj.sprintf(str) }.not_to raise_error(LogStash::Error)
       end
 
       it "should report fields with %{field} syntax" do

data/spec/logstash/pipeline_action/create_spec.rb

@@ -2,13 +2,14 @@
 require "spec_helper"
 require_relative "../../support/helpers"
 require_relative "../../support/matchers"
+require "logstash/pipelines_registry"
 require "logstash/pipeline_action/create"
 require "logstash/inputs/generator"
 
 describe LogStash::PipelineAction::Create do
   let(:metric) { LogStash::Instrument::NullMetric.new(LogStash::Instrument::Collector.new) }
-  let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { null {} }") }
-  let(:pipelines) { java.util.concurrent.ConcurrentHashMap.new }
+  let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => '123' } } output { null {} }") }
+  let(:pipelines) { LogStash::PipelinesRegistry.new }
   let(:agent) { double("agent") }
 
   before do
@@ -18,7 +19,7 @@ describe LogStash::PipelineAction::Create do
   subject { described_class.new(pipeline_config, metric) }
 
   after do
-    pipelines.each do |_, pipeline|
+    pipelines.running_pipelines do |_, pipeline|
       pipeline.shutdown
       pipeline.thread.join
     end
@@ -44,7 +45,7 @@ describe LogStash::PipelineAction::Create do
 
     it "starts the pipeline" do
       subject.execute(agent, pipelines)
-      expect(pipelines[:main].running?).to be_truthy
+      expect(pipelines.get_pipeline(:main).running?).to be_truthy
     end
 
     it "returns a successful execution status" do
@@ -54,7 +55,7 @@ describe LogStash::PipelineAction::Create do
 
   context  "when the pipeline doesn't start" do
     context "with a syntax error" do
-      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { stdout ") } # bad syntax
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => '123' } } output { stdout ") } # bad syntax
 
       it "raises the exception upstream" do
         expect { subject.execute(agent, pipelines) }.to raise_error
@@ -62,7 +63,7 @@ describe LogStash::PipelineAction::Create do
     end
 
     context "with an error raised during `#register`" do
-      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } filter { ruby { init => '1/0' code => '1+2' } } output { null {} }") }
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => '123' } } filter { ruby { init => '1/0' code => '1+2' } } output { null {} }") }
 
       it "returns false" do
         expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
@@ -71,8 +72,8 @@ describe LogStash::PipelineAction::Create do
   end
 
   context "when sorting create action" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { null {} }") }
-    let(:system_pipeline_config) { mock_pipeline_config(:main_2, "input { generator { id => '123' } } output { null {} }", { "pipeline.system" => true }) }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => '123' } } output { null {} }") }
+    let(:system_pipeline_config) { mock_pipeline_config(:main_2, "input { dummyblockinginput { id => '123' } } output { null {} }", { "pipeline.system" => true }) }
 
     it "should give higher priority to system pipeline" do
       action_user_pipeline = described_class.new(pipeline_config, metric)

data/spec/logstash/pipeline_action/reload_spec.rb

@@ -2,15 +2,16 @@
 require "spec_helper"
 require_relative "../../support/helpers"
 require_relative "../../support/matchers"
+require "logstash/pipelines_registry"
 require "logstash/pipeline_action/reload"
 
 describe LogStash::PipelineAction::Reload do
   let(:metric) { LogStash::Instrument::NullMetric.new(LogStash::Instrument::Collector.new) }
   let(:pipeline_id) { :main }
-  let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => true}) }
-  let(:pipeline_config) { "input { generator {} } output { null {} }" }
+  let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { dummyblockinginput { id => 'new' } } output { null {} }", { "pipeline.reloadable" => true}) }
+  let(:pipeline_config) { "input { dummyblockinginput {} } output { null {} }" }
   let(:pipeline) { mock_pipeline_from_string(pipeline_config, mock_settings("pipeline.reloadable" => true)) }
-  let(:pipelines) { chm = java.util.concurrent.ConcurrentHashMap.new; chm[pipeline_id] = pipeline; chm }
+  let(:pipelines) { r = LogStash::PipelinesRegistry.new; r.create_pipeline(pipeline_id, pipeline) { true }; r }
   let(:agent) { double("agent") }
 
   subject { described_class.new(new_pipeline_config, metric) }
@@ -21,7 +22,7 @@ describe LogStash::PipelineAction::Reload do
   end
 
   after do
-    pipelines.each do |_, pipeline|
+    pipelines.running_pipelines do |_, pipeline|
       pipeline.shutdown
       pipeline.thread.join
     end
@@ -38,12 +39,12 @@ describe LogStash::PipelineAction::Reload do
 
     it "start the new pipeline" do
       subject.execute(agent, pipelines)
-      expect(pipelines[pipeline_id].running?).to be_truthy
+      expect(pipelines.get_pipeline(pipeline_id).running?).to be_truthy
     end
 
     it "run the new pipeline code" do
       subject.execute(agent, pipelines)
-      expect(pipelines[pipeline_id].config_hash).to eq(new_pipeline_config.config_hash)
+      expect(pipelines.get_pipeline(pipeline_id).config_hash).to eq(new_pipeline_config.config_hash)
     end
   end
 
@@ -58,7 +59,7 @@ describe LogStash::PipelineAction::Reload do
   end
 
   context "when the new pipeline is not reloadable" do
-    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
+    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { dummyblockinginput { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
 
     it "cannot successfully execute the action" do
       expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
@@ -66,7 +67,7 @@ describe LogStash::PipelineAction::Reload do
   end
 
   context "when the new pipeline has syntax errors" do
-    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
+    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input dummyblockinginput { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
 
     it "cannot successfully execute the action" do
       expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
@@ -75,7 +76,7 @@ describe LogStash::PipelineAction::Reload do
 
   context "when there is an error in the register" do
     before do
-      allow_any_instance_of(LogStash::Inputs::Generator).to receive(:register).and_raise("Bad value")
+      allow_any_instance_of(LogStash::Inputs::DummyBlockingInput).to receive(:register).and_raise("Bad value")
     end
 
     it "cannot successfully execute the action" do

data/spec/logstash/pipeline_action/stop_spec.rb

@@ -1,14 +1,15 @@
 # encoding: utf-8
 require "spec_helper"
 require_relative "../../support/helpers"
+require "logstash/pipelines_registry"
 require "logstash/pipeline_action/stop"
 require "logstash/pipeline"
 
 describe LogStash::PipelineAction::Stop do
-  let(:pipeline_config) { "input { generator {} } output { null {} }" }
+  let(:pipeline_config) { "input { dummyblockinginput {} } output { null {} }" }
   let(:pipeline_id) { :main }
   let(:pipeline) { mock_pipeline_from_string(pipeline_config) }
-  let(:pipelines) { chm = java.util.concurrent.ConcurrentHashMap.new; chm[:main] = pipeline; chm }
+  let(:pipelines) { chm = LogStash::PipelinesRegistry.new; chm.create_pipeline(pipeline_id, pipeline) { true }; chm }
   let(:agent) { double("agent") }
 
   subject { described_class.new(pipeline_id) }
@@ -31,6 +32,6 @@ describe LogStash::PipelineAction::Stop do
   end
 
   it "removes the pipeline from the running pipelines" do
-    expect { subject.execute(agent, pipelines) }.to change { pipelines.include?(pipeline_id) }.from(true).to(false)
+    expect { subject.execute(agent, pipelines) }.to change { pipelines.running_pipelines.keys }.from([:main]).to([])
   end
 end

data/spec/logstash/pipelines_registry_spec.rb

@@ -0,0 +1,220 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/pipelines_registry"
+
+describe LogStash::PipelinesRegistry do
+
+  let(:pipeline_id) { "test" }
+  let(:pipeline) { double("Pipeline") }
+  let (:logger) { double("Logger") }
+
+  context "at object creation" do
+    it "should be empty" do
+      expect(subject.size).to eq(0)
+      expect(subject.empty?).to be_truthy
+      expect(subject.running_pipelines).to be_empty
+      expect(subject.non_running_pipelines).to be_empty
+      expect(subject.running_user_defined_pipelines).to be_empty
+    end
+  end
+
+  context "creating a pipeline" do
+    context "without existing same pipeline id" do
+      it "registry should not have a state for pipeline_id" do
+        expect(subject.get_pipeline(pipeline_id)).to be_nil
+      end
+
+      it "should return block return value" do
+        expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to eq("dummy")
+      end
+
+      it "should register the new pipeline upon successful create block" do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+        expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
+      end
+
+      it "should not register the new pipeline upon unsuccessful create block" do
+        subject.create_pipeline(pipeline_id, pipeline) { false }
+        expect(subject.get_pipeline(pipeline_id)).to be_nil
+      end
+    end
+
+    context "with existing pipeline id" do
+      before :each do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+      end
+
+      it "registry should have a state for pipeline_id" do
+        expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
+      end
+
+      context "when existing pipeline is not terminated" do
+        before :each do
+          expect(pipeline).to receive(:finished_execution?).and_return(false)
+        end
+
+        it "should return false" do
+          expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to be_falsey
+        end
+
+        it "should not call block and log error if pipeline is not terminated" do
+          expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
+          expect(logger).to receive(:error)
+          expect { |b| subject.create_pipeline(pipeline_id, pipeline, &b) }.not_to yield_control
+        end
+      end
+
+      context "when existing pipeline is terminated" do
+        let (:new_pipeline) { double("New Pipeline") }
+
+        before :each do
+          expect(pipeline).to receive(:finished_execution?).and_return(true)
+        end
+
+        it "should return block value" do
+          expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
+        end
+
+        it "should return block value" do
+          expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
+        end
+
+        it "should register new pipeline" do
+          subject.create_pipeline(pipeline_id, new_pipeline) { true }
+          expect(subject.get_pipeline(pipeline_id)).to eq(new_pipeline)
+        end
+      end
+    end
+  end
+
+  context "terminating a pipeline" do
+    context "without existing pipeline id" do
+      it "should log error" do
+        expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
+        expect(logger).to receive(:error)
+        subject.terminate_pipeline(pipeline_id) { "dummy" }
+      end
+
+      it "should not yield to block" do
+        expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.not_to yield_control
+      end
+    end
+
+    context "with existing pipeline id" do
+      before :each do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+      end
+
+      it "should yield to block" do
+        expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.to yield_control
+      end
+
+      it "should keep pipeline id" do
+        subject.terminate_pipeline(pipeline_id) { "dummy" }
+        expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
+      end
+    end
+  end
+
+  context "reloading a pipeline" do
+    it "should log error with inexistent pipeline id" do
+      expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
+      expect(logger).to receive(:error)
+      subject.reload_pipeline(pipeline_id) { }
+    end
+
+    context "with existing pipeline id" do
+      before :each do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+      end
+
+      it "should return block value" do
+        expect(subject.reload_pipeline(pipeline_id) { ["dummy", pipeline] }).to eq("dummy")
+      end
+
+      it "should not be terminated while reloading" do
+        expect(pipeline).to receive(:finished_execution?).and_return(false, true, true)
+
+        # 1st call: finished_execution? is false
+        expect(subject.running_pipelines).not_to be_empty
+
+        # 2nd call: finished_execution? is true
+        expect(subject.running_pipelines).to be_empty
+
+
+        queue = Queue.new # threadsafe queue
+        in_block = Concurrent::AtomicBoolean.new(false)
+
+        thread = Thread.new(subject, pipeline_id, pipeline, queue, in_block) do |subject, pipeline_id, pipeline, queue, in_block|
+          subject.reload_pipeline(pipeline_id) do
+            in_block.make_true
+            queue.pop
+            [true, pipeline]
+          end
+        end
+
+        # make sure we entered the block executioin
+        wait(10).for {in_block.true?}.to be_truthy
+
+        # at this point the thread is suspended waiting on queue
+
+        # since in reloading state, running_pipelines is not empty
+        expect(subject.running_pipelines).not_to be_empty
+
+        # unblock thread
+        queue.push(:dummy)
+        thread.join
+
+        # 3rd call: finished_execution? is true
+        expect(subject.running_pipelines).to be_empty
+      end
+    end
+  end
+
+  context "pipelines collections" do
+    context "with a non terminated pipelines" do
+      before :each do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+        expect(pipeline).to receive(:finished_execution?).and_return(false)
+      end
+
+      it "should find running pipelines" do
+        expect(subject.running_pipelines).not_to be_empty
+      end
+
+      it "should not find non_running pipelines" do
+        expect(subject.non_running_pipelines).to be_empty
+      end
+
+      it "should find running_user_defined_pipelines" do
+        expect(pipeline).to receive(:system?).and_return(false)
+        expect(subject.running_user_defined_pipelines).not_to be_empty
+      end
+
+      it "should not find running_user_defined_pipelines" do
+        expect(pipeline).to receive(:system?).and_return(true)
+        expect(subject.running_user_defined_pipelines).to be_empty
+      end
+    end
+
+    context "with a terminated pipelines" do
+      before :each do
+        subject.create_pipeline(pipeline_id, pipeline) { true }
+        expect(pipeline).to receive(:finished_execution?).and_return(true)
+      end
+
+      it "should not find running pipelines" do
+        expect(subject.running_pipelines).to be_empty
+      end
+
+      it "should find non_running pipelines" do
+        expect(subject.non_running_pipelines).not_to be_empty
+      end
+
+      it "should not find running_user_defined_pipelines" do
+        expect(subject.running_user_defined_pipelines).to be_empty
+      end
+    end
+
+  end
+end

data/spec/logstash/queue_factory_spec.rb

@@ -14,6 +14,7 @@ describe LogStash::QueueFactory do
       LogStash::Setting::Numeric.new("queue.checkpoint.acks", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.writes", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.interval", 1000),
+      LogStash::Setting::Boolean.new("queue.checkpoint.retry", false),
       LogStash::Setting::String.new("pipeline.id", pipeline_id),
       LogStash::Setting::PositiveInteger.new("pipeline.batch.size", 125),
       LogStash::Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum)
@@ -46,7 +47,7 @@ describe LogStash::QueueFactory do
       let(:queue_path) { ::File.join(settings.get("path.queue"), pipeline_id) }
 
       after :each do
-        FileUtils.rmdir(queue_path)
+        FileUtils.rm_rf(queue_path)
       end
 
       it "creates a queue directory based on the pipeline id" do