logstash-core 6.0.0.alpha1-java → 6.0.0.alpha2-java

data/spec/logstash/execution_context_spec.rb

@@ -1,28 +1,38 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/util/dead_letter_queue_manager"
 require "logstash/execution_context"
 
 describe LogStash::ExecutionContext do
   let(:pipeline) { double("pipeline") }
   let(:pipeline_id) { :main }
   let(:agent) { double("agent") }
-  
+  let(:plugin_id) { "plugin_id" }
+  let(:plugin_type) { "plugin_type" }
+  let(:dlq_writer) { LogStash::Util::DummyDeadLetterQueueWriter.new }
+
   before do
     allow(pipeline).to receive(:agent).and_return(agent)
     allow(pipeline).to receive(:pipeline_id).and_return(pipeline_id)
   end
 
-  subject { described_class.new(pipeline, agent) }
+  subject { described_class.new(pipeline, agent, plugin_id, plugin_type, dlq_writer) }
 
   it "returns the `pipeline_id`" do
     expect(subject.pipeline_id).to eq(pipeline_id)
   end
-  
+
   it "returns the pipeline" do
     expect(subject.pipeline).to eq(pipeline)
   end
-  
+
   it "returns the agent" do
     expect(subject.agent).to eq(agent)
   end
+
+  it "returns the plugin-specific dlq writer" do
+    expect(subject.dlq_writer.plugin_type).to eq(plugin_type)
+    expect(subject.dlq_writer.plugin_id).to eq(plugin_id)
+    expect(subject.dlq_writer.inner_writer).to eq(dlq_writer)
+  end
 end

data/spec/logstash/inputs/base_spec.rb

@@ -66,7 +66,7 @@ describe "LogStash::Inputs::Base#decorate" do
 
   context "execution context" do
     include_context "execution_context"
-    
+
     let(:klass) { LogStash::Inputs::NOOP }
 
     subject(:instance) { klass.new({}) }

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -7,8 +7,8 @@ require_relative "../../support/mocks_classes"
 require "spec_helper"
 
 describe LogStash::Instrument::WrappedWriteClient do
-  let(:write_client) { queue.write_client }
-  let(:read_client) { queue.read_client }
+  let!(:write_client) { queue.write_client }
+  let!(:read_client) { queue.read_client }
   let(:pipeline) { double("pipeline", :pipeline_id => :main) }
   let(:collector)   { LogStash::Instrument::Collector.new }
   let(:metric) { LogStash::Instrument::Metric.new(collector) }
@@ -18,39 +18,54 @@ describe LogStash::Instrument::WrappedWriteClient do
 
   subject { described_class.new(write_client, pipeline, metric, plugin) }
 
+  def threaded_read_client
+    Thread.new do
+      started_at = Time.now
+
+      batch_size = 0
+      loop {
+        if Time.now - started_at > 60
+          raise "Took too much time to read from the queue"
+        end
+        batch_size = read_client.read_batch.size
+
+        break if batch_size > 0
+      }
+      expect(batch_size).to eq(1)
+    end
+  end
 
   shared_examples "queue tests" do
     it "pushes single event to the `WriteClient`" do
-      t = Thread.new do
-        subject.push(event)
+      pusher_thread = Thread.new(subject, event) do |_subject, _event|
+        _subject.push(_event)
       end
-      sleep(0.01) while !t.status
-      expect(read_client.read_batch.size).to eq(1)
-      t.kill rescue nil
+
+      reader_thread = threaded_read_client
+
+      [pusher_thread, reader_thread].collect(&:join)
     end
 
     it "pushes batch to the `WriteClient`" do
       batch = write_client.get_new_batch
       batch << event
 
-      t = Thread.new do
-        subject.push_batch(batch)
+      pusher_thread = Thread.new(subject, batch) do |_subject, _batch|
+        _subject.push_batch(_batch)
       end
 
-      sleep(0.01) while !t.status
-      expect(read_client.read_batch.size).to eq(1)
-      t.kill rescue nil
+      reader_thread = threaded_read_client
+      [pusher_thread, reader_thread].collect(&:join)
     end
 
     context "recorded metrics" do
       before do
-        t = Thread.new do
-          subject.push(event)
+        pusher_thread = Thread.new(subject, event) do |_subject, _event|
+          _subject.push(_event)
         end
-        sleep(0.01) while !t.status
-        sleep(0.250) # make it block for some time, so duration isn't 0
-        read_client.read_batch.size
-        t.kill rescue nil
+
+        reader_thread = threaded_read_client
+        [pusher_thread, reader_thread].collect(&:join)
       end
 
       let(:snapshot_store) { collector.snapshot_metric.metric_store }
@@ -97,7 +112,7 @@ describe LogStash::Instrument::WrappedWriteClient do
   end
 
   context "AckedMemoryQueue" do
-    let(:queue) { LogStash::Util::WrappedAckedQueue.create_memory_based("", 1024, 10, 1024) }
+    let(:queue) { LogStash::Util::WrappedAckedQueue.create_memory_based("", 1024, 10, 4096) }
 
     before do
       read_client.set_events_metric(metric.namespace([:stats, :events]))

data/spec/logstash/output_delegator_spec.rb

@@ -10,7 +10,7 @@ describe LogStash::OutputDelegator do
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
   let(:collector) { [] }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
-  
+
   include_context "execution_context"
 
   subject { described_class.new(logger, out_klass, metric, execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }

data/spec/logstash/outputs/base_spec.rb

@@ -82,7 +82,7 @@ describe "LogStash::Outputs::Base#new" do
 
   context "execution context" do
     include_context "execution_context"
-    
+
     let(:klass) { LogStash::Outputs::NOOPSingle }
 
     subject(:instance) { klass.new(params.dup) }

data/spec/logstash/pipeline_action/reload_spec.rb

@@ -10,7 +10,7 @@ describe LogStash::PipelineAction::Reload do
   let(:pipeline_id) { :main }
   let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => true}) }
   let(:pipeline_config) { "input { generator {} } output { null {} }" }
-  let(:pipeline) { LogStash::Pipeline.new(pipeline_config, mock_settings("pipeline.reloadable" => true)) }
+  let(:pipeline) { mock_pipeline_from_string(pipeline_config, mock_settings("pipeline.reloadable" => true)) }
   let(:pipelines) { { pipeline_id => pipeline } }
   let(:agent) { double("agent") }
 

data/spec/logstash/pipeline_action/stop_spec.rb

@@ -8,7 +8,7 @@ require "logstash/instrument/null_metric"
 describe LogStash::PipelineAction::Stop do
   let(:pipeline_config) { "input { generator {} } output { null {} }" }
   let(:pipeline_id) { :main }
-  let(:pipeline) { LogStash::Pipeline.new(pipeline_config) }
+  let(:pipeline) { mock_pipeline_from_string(pipeline_config) }
   let(:pipelines) { { :main => pipeline } }
   let(:agent) { double("agent") }
 

data/spec/logstash/pipeline_dlq_commit_spec.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "tmpdir"
+require "spec_helper"
+require "logstash/codecs/plain"
+require_relative "../support/mocks_classes"
+
+class SingleGeneratorInput < LogStash::Inputs::Base
+  config_name "singlegenerator"
+  milestone 2
+
+  def register
+  end
+
+  def run(queue)
+    queue << LogStash::Event.new
+  end
+
+  def close
+  end
+end
+
+class DLQCommittingFilter < LogStash::Filters::Base
+  config_name "dlq_commit"
+  milestone 2
+
+  def register()
+  end
+
+  def filter(event)
+    execution_context.dlq_writer.write(event, "my reason")
+  end
+
+  def threadsafe?() true; end
+
+  def close() end
+end
+
+describe LogStash::Pipeline do
+  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings) do
+    {
+      "pipeline.workers" => 2,
+      "pipeline.id" => pipeline_id,
+      "dead_letter_queue.enable" => enable_dlq,
+      "path.dead_letter_queue" => Dir.mktmpdir
+    }
+  end
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:test_config) {
+    <<-eos
+        input { singlegenerator { id => input_id } }
+
+        filter { dlq_commit { id => filter_id } }
+
+        output { dummyoutput { id => output_id } }
+    eos
+  }
+
+  subject { mock_pipeline_from_string(test_config, pipeline_settings_obj, metric) }
+
+  before(:each) do
+    pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
+    allow(LogStash::Plugin).to receive(:lookup).with("input", "singlegenerator").and_return(SingleGeneratorInput)
+    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+    allow(LogStash::Plugin).to receive(:lookup).with("filter", "dlq_commit").and_return(DLQCommittingFilter)
+    allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
+  end
+
+  after(:each) do
+    FileUtils.remove_entry pipeline_settings["path.dead_letter_queue"]
+  end
+
+  context "dlq is enabled" do
+    let(:enable_dlq) { true }
+    let(:pipeline_id) { "test-dlq" }
+
+    it "retrieves proper pipeline-level DLQ writer" do
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
+      expect_any_instance_of(org.logstash.common.io.DeadLetterQueueWriter).to receive(:close).and_call_original
+      subject.run
+      dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
+      dlq_reader = org.logstash.common.io.DeadLetterQueueReader.new(dlq_path)
+      entry = dlq_reader.pollEntry(40)
+      expect(entry).to_not be_nil
+      expect(entry.reason).to eq("my reason")
+    end
+  end
+
+  context "dlq is disabled" do
+    let(:enable_dlq) { false }
+    let(:pipeline_id) { "test-without-dlq" }
+
+    it "does not write to the DLQ" do
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
+      expect(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:new).and_call_original
+      expect_any_instance_of(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:close).and_call_original
+      subject.run
+      dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
+      expect(java.nio.file.Files.exists(dlq_path)).to eq(false)
+    end
+  end
+
+end

data/spec/logstash/pipeline_pq_file_spec.rb

@@ -2,6 +2,7 @@
 require "spec_helper"
 require "logstash/inputs/generator"
 require "logstash/filters/multiline"
+require_relative "../support/helpers"
 
 class PipelinePqFileOutput < LogStash::Outputs::Base
   config_name "pipelinepqfileoutput"
@@ -67,7 +68,8 @@ describe LogStash::Pipeline do
 
    let(:pipeline_settings) { { "queue.type" => queue_type, "pipeline.workers" => worker_thread_count, "pipeline.id" => pipeline_id} }
 
-  subject { described_class.new(config, pipeline_settings_obj, metric) }
+  let(:pipeline_config) { mock_pipeline_config(pipeline_id, config, pipeline_settings_obj) }
+  subject { described_class.new(pipeline_config, metric) }
 
   let(:counting_output) { PipelinePqFileOutput.new({ "id" => output_id }) }
   let(:metric_store) { subject.metric.collector.snapshot_metric.metric_store }

data/spec/logstash/pipeline_reporter_spec.rb

@@ -2,6 +2,7 @@
 require "spec_helper"
 require "logstash/pipeline"
 require "logstash/pipeline_reporter"
+require_relative "../support/helpers"
 require_relative "../support/mocks_classes"
 
 #TODO: Figure out how to add more tests that actually cover inflight events
@@ -11,7 +12,7 @@ describe LogStash::PipelineReporter do
   let(:config) do
     "input { generator { count => #{generator_count} } } output { dummyoutput {} } "
   end
-  let(:pipeline) { LogStash::Pipeline.new(config)}
+  let(:pipeline) { mock_pipeline_from_string(config)}
   let(:reporter) { pipeline.reporter }
 
   before do

data/spec/logstash/pipeline_spec.rb

@@ -81,6 +81,21 @@ class DummySafeFilter < LogStash::Filters::Base
   def close() end
 end
 
+class DummyFlushingFilter < LogStash::Filters::Base
+  config_name "dummyflushingfilter"
+  milestone 2
+
+  def register() end
+  def filter(event) end
+  def periodic_flush
+    true
+  end
+  def flush(options)
+    return [::LogStash::Event.new("message" => "dummy_flush")]
+  end
+  def close() end
+end
+
 class TestPipeline < LogStash::Pipeline
   attr_reader :outputs, :settings
 end
@@ -139,7 +154,7 @@ describe LogStash::Pipeline do
       abort_on_exception_state = Thread.abort_on_exception
       Thread.abort_on_exception = true
 
-      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
+      pipeline = mock_pipeline_from_string(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
       sleep(0.1) until pipeline.ready?
       wait(3).for do
@@ -187,21 +202,21 @@ describe LogStash::Pipeline do
         let(:logger) { double("pipeline logger").as_null_object }
 
         before do
-          expect(TestPipeline).to receive(:logger).and_return(logger)
+          expect(::LogStash::Pipeline).to receive(:logger).and_return(logger)
           allow(logger).to receive(:debug?).and_return(true)
         end
 
         it "should not receive a debug message with the compiled code" do
           pipeline_settings_obj.set("config.debug", false)
           expect(logger).not_to receive(:debug).with(/Compiled pipeline/, anything)
-          pipeline = TestPipeline.new(test_config_with_filters)
+          pipeline = mock_pipeline_from_string(test_config_with_filters)
           pipeline.close
         end
 
         it "should print the compiled code if config.debug is set to true" do
           pipeline_settings_obj.set("config.debug", true)
           expect(logger).to receive(:debug).with(/Compiled pipeline/, anything)
-          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
+          pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
           pipeline.close
         end
       end
@@ -209,7 +224,7 @@ describe LogStash::Pipeline do
       context "when there is no command line -w N set" do
         it "starts one filter thread" do
           msg = "Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads"
-          pipeline = TestPipeline.new(test_config_with_filters)
+          pipeline = mock_pipeline_from_string(test_config_with_filters)
           expect(pipeline.logger).to receive(:warn).with(msg,
             hash_including({:count_was=>worker_thread_count, :filters=>["dummyfilter"]}))
           pipeline.run
@@ -223,7 +238,7 @@ describe LogStash::Pipeline do
         it "starts multiple filter thread" do
           msg = "Warning: Manual override - there are filters that might" +
                 " not work with multiple worker threads"
-          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
+          pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
           expect(pipeline.logger).to receive(:warn).with(msg, hash_including({:worker_threads=> override_thread_count, :filters=>["dummyfilter"]}))
           pipeline.run
           expect(pipeline.worker_threads.size).to eq(override_thread_count)
@@ -250,7 +265,8 @@ describe LogStash::Pipeline do
       }
 
       it "starts multiple filter threads" do
-        pipeline = TestPipeline.new(test_config_with_filters)
+        skip("This test has been failing periodically since November 2016. Tracked as https://github.com/elastic/logstash/issues/6245")
+        pipeline = mock_pipeline_from_string(test_config_with_filters)
         pipeline.run
         expect(pipeline.worker_threads.size).to eq(worker_thread_count)
         pipeline.shutdown
@@ -293,7 +309,7 @@ describe LogStash::Pipeline do
     }
 
     context "output close" do
-      let(:pipeline) { TestPipeline.new(test_config_without_output_workers) }
+      let(:pipeline) { mock_pipeline_from_string(test_config_without_output_workers) }
       let(:output) { pipeline.outputs.first }
 
       before do
@@ -323,7 +339,7 @@ describe LogStash::Pipeline do
       let(:config) { "input { dummyinput {} } output { dummyoutput {} }"}
 
       it "should start the flusher thread only after the pipeline is running" do
-        pipeline = TestPipeline.new(config)
+        pipeline = mock_pipeline_from_string(config)
 
         expect(pipeline).to receive(:transition_to_running).ordered.and_call_original
         expect(pipeline).to receive(:start_flusher).ordered.and_call_original
@@ -379,7 +395,7 @@ describe LogStash::Pipeline do
     let(:config) { "input { dummyinput {} } output { dummyoutput {} }" }
     let(:batch_size) { 1 }
     let(:pipeline_settings) { { "pipeline.batch.size" => batch_size, "pipeline.workers" => 1 } }
-    let(:pipeline) { LogStash::Pipeline.new(config, pipeline_settings_obj) }
+    let(:pipeline) { mock_pipeline_from_string(config, pipeline_settings_obj) }
     let(:logger) { pipeline.logger }
     let(:warning_prefix) { Regexp.new("CAUTION: Recommended inflight events max exceeded!") }
 
@@ -446,7 +462,7 @@ describe LogStash::Pipeline do
     config = "input { } filter { } output { }"
 
     let(:settings) { LogStash::SETTINGS.clone }
-    subject { LogStash::Pipeline.new(config, settings, metric) }
+    subject { mock_pipeline_from_string(config, settings, metric) }
 
     after :each do
       subject.close
@@ -547,8 +563,8 @@ describe LogStash::Pipeline do
       pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
     end
 
-    let(:pipeline1) { LogStash::Pipeline.new("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
-    let(:pipeline2) { LogStash::Pipeline.new("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutputmore {}}") }
+    let(:pipeline1) { mock_pipeline_from_string("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { mock_pipeline_from_string("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutputmore {}}") }
 
     after  do
       pipeline1.close
@@ -564,23 +580,16 @@ describe LogStash::Pipeline do
   end
 
   context "Periodic Flush" do
-    let(:number_of_events) { 100 }
     let(:config) do
       <<-EOS
       input {
-        generator {
-          count => #{number_of_events}
-        }
+        dummy_input {}
       }
       filter {
-        multiline {
-          pattern => "^NeverMatch"
-          negate => true
-          what => "previous"
-        }
+        dummy_flushing_filter {}
       }
       output {
-        dummyoutput {}
+        dummy_output {}
       }
       EOS
     end
@@ -588,24 +597,26 @@ describe LogStash::Pipeline do
 
     before do
       allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output)
-      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummy_input").and_return(DummyInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummy_flushing_filter").and_return(DummyFlushingFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
-      allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
-      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
     end
 
-    it "flushes the buffered contents of the filter" do
+    it "flush periodically" do
       Thread.abort_on_exception = true
-      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
+      pipeline = mock_pipeline_from_string(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
       sleep(0.1) until pipeline.ready?
-      wait(3).for do
+      wait(10).for do
         # give us a bit of time to flush the events
         output.events.empty?
       end.to be_falsey
-      event = output.events.pop
-      expect(event.get("message").count("\n")).to eq(99)
+
+      expect(output.events.any? {|e| e.get("message") == "dummy_flush"}).to eq(true)
+
       pipeline.shutdown
+
       t.join
     end
   end
@@ -618,8 +629,8 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
     end
 
-    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
-    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline1) { mock_pipeline_from_string("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { mock_pipeline_from_string("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
 
     # multiple pipelines cannot be instantiated using the same PQ settings, force memory queue
     before :each do
@@ -656,7 +667,7 @@ describe LogStash::Pipeline do
       EOS
     end
 
-    subject { described_class.new(config) }
+    subject { mock_pipeline_from_string(config) }
 
     context "when the pipeline is not started" do
       after :each do
@@ -683,7 +694,7 @@ describe LogStash::Pipeline do
       }
       EOS
     end
-    subject { described_class.new(config) }
+    subject { mock_pipeline_from_string(config) }
 
     context "when the pipeline is not started" do
       after :each do
@@ -713,7 +724,7 @@ describe LogStash::Pipeline do
   context "when collecting metrics in the pipeline" do
     let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
 
-    subject { described_class.new(config, pipeline_settings_obj, metric) }
+    subject { mock_pipeline_from_string(config, pipeline_settings_obj, metric) }
 
     let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
     let(:pipeline_id) { "main" }
@@ -840,8 +851,8 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
     end
 
-    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
-    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline1) { mock_pipeline_from_string("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { mock_pipeline_from_string("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
 
     # multiple pipelines cannot be instantiated using the same PQ settings, force memory queue
     before :each do
@@ -861,14 +872,14 @@ describe LogStash::Pipeline do
     end
 
     context "when the pipeline is a system pipeline" do
-      let(:pipeline) { LogStash::Pipeline.new("input { generator {} } output { null {} }", mock_settings("pipeline.system" => true)) }
+      let(:pipeline) { mock_pipeline_from_string("input { generator {} } output { null {} }", mock_settings("pipeline.system" => true)) }
       it "returns true" do
         expect(pipeline.system?).to be_truthy
       end
     end
 
     context "when the pipeline is not a system pipeline" do
-      let(:pipeline) { LogStash::Pipeline.new("input { generator {} } output { null {} }", mock_settings("pipeline.system" => false)) }
+      let(:pipeline) { mock_pipeline_from_string("input { generator {} } output { null {} }", mock_settings("pipeline.system" => false)) }
       it "returns true" do
         expect(pipeline.system?).to be_falsey
       end
@@ -881,7 +892,7 @@ describe LogStash::Pipeline do
     end
 
     context "when all plugins are reloadable and pipeline is configured as reloadable" do
-      let(:pipeline) { LogStash::Pipeline.new("input { generator {} } output { null {} }", mock_settings("pipeline.reloadable" => true)) }
+      let(:pipeline) { mock_pipeline_from_string("input { generator {} } output { null {} }", mock_settings("pipeline.reloadable" => true)) }
 
       it "returns true" do
         expect(pipeline.reloadable?).to be_truthy
@@ -889,7 +900,7 @@ describe LogStash::Pipeline do
     end
 
     context "when the plugins are not reloadable and pipeline is configured as reloadable" do
-      let(:pipeline) { LogStash::Pipeline.new("input { stdin {} } output { null {} }", mock_settings("pipeline.reloadable" => true)) }
+      let(:pipeline) { mock_pipeline_from_string("input { stdin {} } output { null {} }", mock_settings("pipeline.reloadable" => true)) }
 
       it "returns true" do
         expect(pipeline.reloadable?).to be_falsey
@@ -897,7 +908,7 @@ describe LogStash::Pipeline do
     end
 
     context "when all plugins are reloadable and pipeline is configured as non-reloadable" do
-      let(:pipeline) { LogStash::Pipeline.new("input { generator {} } output { null {} }", mock_settings("pipeline.reloadable" => false)) }
+      let(:pipeline) { mock_pipeline_from_string("input { generator {} } output { null {} }", mock_settings("pipeline.reloadable" => false)) }
 
       it "returns true" do
         expect(pipeline.reloadable?).to be_falsey