logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/spec/logstash/agent/metrics_spec.rb

@@ -0,0 +1,244 @@
+# encoding: utf-8
+#
+require "logstash/agent"
+require_relative "../../support/helpers"
+require_relative "../../support/matchers"
+require_relative "../../support/mocks_classes"
+require "spec_helper"
+
+# Just make the tests a bit shorter to write and
+# assert, I will keep theses methods here for easier understanding.
+def mval(*path_elements)
+  mhash(*path_elements).value
+end
+
+def mhash(*path_elements)
+  metric.get_shallow(*path_elements)
+end
+
+describe LogStash::Agent do
+  # by default no tests uses the auto reload logic
+  let(:agent_settings) { mock_settings("config.reload.automatic" => false) }
+  let(:pipeline_settings) { { "pipeline.reloadable" => true } }
+
+  let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } filter { mutate { add_tag => 'hello world' }} output { null {} }", pipeline_settings) }
+  let(:update_pipeline_config) { mock_pipeline_config(:main, "input { generator { id => 'new' } } output { null {} }", pipeline_settings) }
+  let(:bad_update_pipeline_config) { mock_pipeline_config(:main, "hooo }", pipeline_settings) }
+
+  let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator {} } output { null {} }", pipeline_settings) }
+  let(:bad_pipeline_config) { mock_pipeline_config(:bad, "hooo }", pipeline_settings) }
+  let(:second_bad_pipeline_config) { mock_pipeline_config(:second_bad, "hooo }", pipeline_settings) }
+
+  let(:source_loader) do
+    TestSourceLoader.new([])
+  end
+
+  subject { described_class.new(agent_settings, source_loader) }
+
+  before :each do
+    # This MUST run first, before `subject` is invoked to ensure clean state
+    clear_data_dir
+
+    # TODO(ph) until we decouple the webserver from the agent
+    # we just disable these calls
+    allow(subject).to receive(:start_webserver).and_return(false)
+    allow(subject).to receive(:stop_webserver).and_return(false)
+  end
+
+  # Lets make sure we stop all the running pipeline for every example
+  # so we don't have any rogue pipeline in the background
+  after :each do
+    subject.shutdown
+  end
+
+  let(:metric) { subject.metric.collector.snapshot_metric.metric_store }
+
+  context "when starting the agent" do
+    it "initialize the instance reload metrics" do
+      expect(mval(:stats, :reloads, :successes)).to eq(0)
+      expect(mval(:stats, :reloads, :failures)).to eq(0)
+    end
+  end
+
+  context "when we try to start one pipeline" do
+    context "and it succeed" do
+      let(:source_loader) do
+        TestSourceLoader.new(pipeline_config)
+      end
+
+      let(:pipeline_name) { :main }
+
+      it "doesnt changes the global successes" do
+        expect { subject.converge_state_and_update }.not_to change { mval(:stats, :reloads, :successes) }
+      end
+
+      it "doesn't change the failures" do
+        expect { subject.converge_state_and_update }.not_to change { mval(:stats, :reloads, :failures) }
+      end
+
+      it "sets the failures to 0" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :failures)).to eq(0)
+      end
+
+      it "sets the successes to 0" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :successes)).to eq(0)
+      end
+
+      it "sets the `last_error` to nil" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_error)).to be_nil
+      end
+
+      it "sets the `last_failure_timestamp` to nil" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, :main, :reloads, :last_failure_timestamp)).to be_nil
+      end
+
+      it "sets the `last_success_timestamp` to nil" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_success_timestamp)).to be_nil
+      end
+    end
+
+    context "and it fails" do
+      let(:source_loader) do
+        TestSourceLoader.new(bad_pipeline_config)
+      end
+
+      let(:pipeline_name) { :bad }
+
+      before do
+        subject.converge_state_and_update
+      end
+
+      it "doesnt changes the global successes" do
+        expect { subject.converge_state_and_update }.not_to change { mval(:stats, :reloads, :successes)}
+      end
+
+      it "doesn't change the failures" do
+        expect { subject.converge_state_and_update }.to change { mval(:stats, :reloads, :failures) }.by(1)
+      end
+
+      it "increments the pipeline failures" do
+        expect { subject.converge_state_and_update }.to change { mval(:stats, :pipelines, pipeline_name, :reloads, :failures) }.by(1)
+      end
+
+      it "sets the successes to 0" do
+        subject.converge_state_and_update
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :successes)).to eq(0)
+      end
+
+      it "increase the global failures" do
+        expect { subject.converge_state_and_update }.to change { mval(:stats, :reloads, :failures) }
+      end
+
+      it "records the `last_error`" do
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_error)).to_not be_nil
+      end
+
+      it "records the `message` and the `backtrace`" do
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_error)[:message]).to_not be_nil
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_error)[:backtrace]).to_not be_nil
+      end
+
+      it "records the time of the last failure" do
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_failure_timestamp)).to_not be_nil
+      end
+
+      it "initializes the `last_success_timestamp`" do
+        expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_success_timestamp)).to be_nil
+      end
+    end
+
+    context "when we try to reload a pipeline" do
+      before do
+        # Running Pipeline to -> reload pipeline
+        expect(subject.converge_state_and_update.success?).to be_truthy
+      end
+
+      let(:pipeline_name) { :main }
+
+      context "and it succeed" do
+        let(:source_loader) do
+          TestSequenceSourceLoader.new(pipeline_config, update_pipeline_config)
+        end
+
+        it "increments the global successes" do
+          expect { subject.converge_state_and_update }.to change { mval(:stats, :reloads, :successes) }.by(1)
+        end
+
+        it "increment the pipeline successes" do
+          expect{ subject.converge_state_and_update }.to change { mval(:stats, :pipelines, pipeline_name, :reloads, :successes) }.by(1)
+        end
+
+        it "record the `last_success_timestamp`" do
+          expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_success_timestamp)).to be_nil
+          subject.converge_state_and_update
+          expect(mval(:stats, :pipelines, pipeline_name, :reloads, :last_success_timestamp)).not_to be_nil
+        end
+      end
+
+      context "and it fails" do
+        let(:source_loader) do
+          TestSequenceSourceLoader.new(pipeline_config, bad_update_pipeline_config)
+        end
+
+        it "increments the global failures" do
+          expect { subject.converge_state_and_update }.to change { mval(:stats, :reloads, :failures) }.by(1)
+        end
+
+        it "increment the pipeline failures" do
+          expect{ subject.converge_state_and_update }.to change { mval(:stats, :pipelines, pipeline_name, :reloads, :failures) }.by(1)
+        end
+      end
+    end
+
+    context "when we successfully reload a pipeline" do
+      let(:source_loader) do
+        TestSequenceSourceLoader.new(pipeline_config, update_pipeline_config)
+      end
+
+      before do
+        expect(subject.converge_state_and_update.success?).to be_truthy
+      end
+
+      it "it clear previous metrics" do
+        try(20) do
+          expect { mhash(:stats, :pipelines, :main, :plugins, :filters) }.not_to raise_error, "Filters stats should exist"
+        end
+        expect(subject.converge_state_and_update.success?).to be_truthy
+
+        # We do have to retry here, since stopping a pipeline is a blocking operation
+        expect { mhash(:stats, :pipelines, :main, :plugins, :filters) }.to raise_error
+      end
+    end
+
+    context "when we stop a pipeline" do
+      let(:source_loader) do
+        TestSequenceSourceLoader.new(pipeline_config, [])
+      end
+
+      before do
+        # Running Pipeline to -> reload pipeline
+        expect(subject.converge_state_and_update.success?).to be_truthy
+      end
+
+      it "clear pipeline specific metric" do
+        # since the pipeline is async, it can actually take some time to have metrics recordings
+        # so we try a few times
+        try(20) do
+          expect { mhash(:stats, :pipelines, :main, :events) }.not_to raise_error , "Events pipelien stats should exist"
+          expect { mhash(:stats, :pipelines, :main, :plugins) }.not_to raise_error, "Plugins pipeline stats should exist"
+        end
+
+        expect(subject.converge_state_and_update.success?).to be_truthy
+
+        # We do have to retry here, since stopping a pipeline is a blocking operation
+        expect { mhash(:stats, :pipelines, :main, :plugins) }.to raise_error
+        expect { mhash(:stats, :pipelines, :main, :events) }.to raise_error
+      end
+    end
+  end
+end

data/spec/logstash/agent_spec.rb

@@ -2,30 +2,35 @@
 require "spec_helper"
 require "stud/temporary"
 require "logstash/inputs/generator"
+require "logstash/config/pipeline_config"
+require "logstash/config/source/local"
 require_relative "../support/mocks_classes"
 require "fileutils"
 require_relative "../support/helpers"
-require 'timeout'
+require_relative "../support/matchers"
 
 describe LogStash::Agent do
-
-  let(:agent_settings) { LogStash::SETTINGS }
+  let(:agent_settings) { mock_settings("config.string" => "input {}") }
   let(:default_pipeline_id) { LogStash::SETTINGS.get("pipeline.id") }
   let(:agent_args) { {} }
   let(:pipeline_settings) { agent_settings.clone }
   let(:pipeline_args) { {} }
   let(:config_file) { Stud::Temporary.pathname }
-  let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
-  let(:logger) { double("logger") }
-  let(:timeout) {120} #seconds
+  let(:config_file_txt) { "input { generator { id => 'initial' } } output { }" }
+  let(:default_source_loader) do
+    sl = LogStash::Config::SourceLoader.new
+    sl.add_source(LogStash::Config::Source::Local.new(agent_settings))
+    sl
+  end
 
-  subject { LogStash::Agent.new(agent_settings) }
+  subject { LogStash::Agent.new(agent_settings, default_source_loader) }
 
   before :each do
     # This MUST run first, before `subject` is invoked to ensure clean state
     clear_data_dir
 
     File.open(config_file, "w") { |f| f.puts config_file_txt }
+
     agent_args.each do |key, value|
       agent_settings.set(key, value)
       pipeline_settings.set(key, value)
@@ -33,9 +38,6 @@ describe LogStash::Agent do
     pipeline_args.each do |key, value|
       pipeline_settings.set(key, value)
     end
-    allow(described_class).to receive(:logger).and_return(logger)
-    [:debug, :info, :error, :warn, :fatal, :trace].each {|level| allow(logger).to receive(level) }
-    [:debug?, :info?, :error?, :warn?, :fatal?, :trace?].each {|level| allow(logger).to receive(level) }
   end
 
   after :each do
@@ -44,10 +46,14 @@ describe LogStash::Agent do
   end
 
   it "fallback to hostname when no name is provided" do
-    expect(LogStash::Agent.new.name).to eq(Socket.gethostname)
+    expect(LogStash::Agent.new(agent_settings, default_source_loader).name).to eq(Socket.gethostname)
+  end
+
+  after(:each) do
+    subject.shutdown # shutdown/close the pipelines
   end
 
-  describe "register_pipeline" do
+  describe "adding a new pipeline" do
     let(:config_string) { "input { } filter { } output { }" }
     let(:agent_args) do
       {
@@ -58,16 +64,12 @@ describe LogStash::Agent do
       }
     end
 
-    after(:each) do
-      subject.close_pipelines
-    end
-
     it "should delegate settings to new pipeline" do
       expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
         expect(arg1).to eq(config_string)
         expect(arg2.to_hash).to include(agent_args)
       end
-      subject.register_pipeline(agent_settings)
+      subject.converge_state_and_update
     end
   end
 
@@ -86,7 +88,11 @@ describe LogStash::Agent do
   end
 
   describe "#execute" do
-    let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
+    let(:config_file_txt) { "input { generator { id => 'old'} } output { }" }
+    let(:mock_config_pipeline) { mock_pipeline_config(:main, config_file_txt, pipeline_settings) }
+
+    let(:source_loader) { TestSourceLoader.new(mock_config_pipeline) }
+    subject { described_class.new(agent_settings, source_loader) }
 
     before :each do
       allow(subject).to receive(:start_webserver).and_return(false)
@@ -101,99 +107,112 @@ describe LogStash::Agent do
         }
       end
 
-      before(:each) do
-        subject.register_pipeline(pipeline_settings)
-      end
-
-      context "when a system pipeline is running" do
-        context "when one pipeline is finite" do
-          let(:pipeline_args) {
-            {
-              "path.config" => "a",
-              "config.string" => "input { generator { count => 1000 }} output { null {} }"
-            }
-          }
-          let(:system_pipeline_settings) do
-            s = agent_settings.clone
-            s.set("path.config", "")
-            s.set("config.string", "input { generator {}} output { null {} }")
-            s.set("pipeline.id", ".monitoring")
-            s.set("pipeline.system", true)
-            s
-          end
-
-          it "stops logstash at the end of the execution of the finite pipeline" do
-            subject.register_pipeline(system_pipeline_settings)
-            expect(subject.execute).to be_nil
-          end
-        end
-      end
 
       context "if state is clean" do
         before :each do
-          allow(subject).to receive(:running_pipelines?).and_return(true)
-          allow(subject).to receive(:sleep)
+          allow(subject).to receive(:running_user_defined_pipelines?).and_return(true)
           allow(subject).to receive(:clean_state?).and_return(false)
         end
 
-        it "should not reload_state!" do
-          expect(subject).to_not receive(:reload_state!)
+        it "should not converge state more than once" do
+          expect(subject).to receive(:converge_state_and_update).once
           t = Thread.new { subject.execute }
 
+          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+          # a bad test design or missing class functionality.
+          sleep(0.1)
           Stud.stop!(t)
           t.join
           subject.shutdown
         end
       end
 
-      context "when calling reload_state!" do
-        context "with a pipeline with auto reloading turned off" do
+      context "when calling reloading a pipeline" do
+        context "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
+
+          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+
+          it "does not upgrade the new config" do
+            t = Thread.new { subject.execute }
+            sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.ready?
+
+            expect(subject.converge_state_and_update).not_to be_a_successful_converge
+            expect(subject).to have_running_pipeline?(mock_config_pipeline)
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
+            Stud.stop!(t)
+            t.join
+            subject.shutdown
+          end
+        end
+
+        context "with a config that does not contain reload incompatible plugins" do
           let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-          let(:pipeline_args) { { "config.reload.automatic" => false } }
+          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
 
-          it "does not try to reload the pipeline" do
+          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+
+          it "does upgrade the new config" do
             t = Thread.new { subject.execute }
-            Timeout.timeout(timeout) do
-              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-            end
-            expect(subject).to_not receive(:reload_pipeline!)
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            subject.reload_state!
+            sleep(0.1) until subject.pipelines_count > 0 && subject.pipelines.values.first.ready?
+
+            expect(subject.converge_state_and_update).to be_a_successful_converge
+            expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
 
             # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
             # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
           end
+        end
+
+      end
+      context "when calling reload_state!" do
+        context "with a pipeline with auto reloading turned off" do
+          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+          let(:pipeline_args) { { "pipeline.reloadable" => false } }
+          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
 
-          context "and force autoreload" do
-            it "reloads the pipeline" do
-              t = Thread.new { subject.execute }
-              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-              expect(subject).to receive(:reload_pipeline!).with("main", true)
-              subject.reload_state!(true)
+          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
 
-              Stud.stop!(t)
-              t.join
-              subject.shutdown
-            end
+          it "does not try to reload the pipeline" do
+            t = Thread.new { subject.execute }
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+
+            expect(subject.converge_state_and_update).not_to be_a_successful_converge
+            expect(subject).to have_running_pipeline?(mock_config_pipeline)
+
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
+            Stud.stop!(t)
+            t.join
+            subject.shutdown
           end
         end
 
         context "with a pipeline with auto reloading turned on" do
-          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-          let(:pipeline_args) { { "config.reload.automatic" => true } }
+          let(:second_pipeline_config) { "input { generator { id => 'second' } } filter { } output { }" }
+          let(:pipeline_args) { { "pipeline.reloadable" => true } }
+          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
+          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
 
           it "tries to reload the pipeline" do
             t = Thread.new { subject.execute }
-            Timeout.timeout(timeout) do
-              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-            end
-            expect(subject).to receive(:reload_pipeline!).once.and_call_original
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            subject.reload_state!
+            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+
+            expect(subject.converge_state_and_update).to be_a_successful_converge
+            expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
 
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -203,91 +222,102 @@ describe LogStash::Agent do
     end
 
     context "when auto_reload is true" do
+      subject { described_class.new(agent_settings, default_source_loader) }
+
       let(:agent_args) do
         {
+          "config.string" => "",
           "config.reload.automatic" => true,
           "config.reload.interval" => 0.01,
-          "path.config" => config_file,
+          "path.config" => config_file
         }
       end
 
-      before(:each) do
-        subject.register_pipeline(pipeline_settings)
-      end
-
       context "if state is clean" do
         it "should periodically reload_state" do
           allow(subject).to receive(:clean_state?).and_return(false)
           t = Thread.new { subject.execute }
-          Timeout.timeout(timeout) do
-            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-          end
-          expect(subject).to receive(:reload_state!).at_least(2).times
-
-          sleep 1
+          sleep(0.05) until subject.running_pipelines? && subject.pipelines.values.first.running?
+          expect(subject).to receive(:converge_state_and_update).at_least(2).times
 
+          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+          # a bad test design or missing class functionality.
+          sleep(0.1)
           Stud.stop!(t)
           t.join
           subject.shutdown
         end
       end
-    end
 
-    context "when calling reload_state!" do
-    end
-  end
+      context "when calling reload_state!" do
+        xcontext "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin { id => '123' } } filter { } output { }" }
 
-  describe "#reload_state!" do
-    let(:first_pipeline_config) { "input { } filter { } output { }" }
-    let(:second_pipeline_config) { "input { generator { count => 10000 } } filter { } output { }" }
-    let(:pipeline_args) { {
-      "config.string" => first_pipeline_config,
-      "pipeline.workers" => 4,
-      "config.reload.automatic" => true
-    } }
+          it "does not upgrade the new config" do
+            t = Thread.new { subject.execute }
+            sleep(0.05) until subject.running_pipelines? && subject.pipelines.values.first.running?
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep(0.2) # lets us catch the new file
 
-    before(:each) do
-      subject.register_pipeline(pipeline_settings)
-    end
+            try do
+              expect(subject.pipelines[default_pipeline_id.to_sym].config_str).not_to eq(second_pipeline_config)
+            end
 
-    after(:each) do
-      subject.close_pipelines
-    end
+            Stud.stop!(t)
+            t.join
+            subject.shutdown
+          end
+        end
 
-    context "when fetching a new state" do
-      it "upgrades the state" do
-        expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
-        expect(subject).to receive(:upgrade_pipeline).with(default_pipeline_id, kind_of(LogStash::Settings), second_pipeline_config)
-        subject.reload_state!
-      end
-    end
-    context "when fetching the same state" do
-      it "doesn't upgrade the state" do
-        expect(subject).to receive(:fetch_config).and_return(first_pipeline_config)
-        expect(subject).to_not receive(:upgrade_pipeline)
-        subject.reload_state!
-      end
-    end
+        context "with a config that does not contain reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { generator { id => 'new' } } filter { } output { }" }
+
+          it "does upgrade the new config" do
+            t = Thread.new { subject.execute }
+
+            sleep(0.05) until subject.running_pipelines? && subject.pipelines.values.first.running?
+
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep(5) # lets us catch the new file
 
-    context "with a config that contains reload incompatible plugins" do
-      let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+            try do
+              expect(subject.pipelines[default_pipeline_id.to_sym]).not_to be_nil
+              expect(subject.pipelines[default_pipeline_id.to_sym].config_str).to match(second_pipeline_config)
+            end
 
-      it "does not upgrade the new config" do
-        expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
-        expect(subject).to_not receive(:upgrade_pipeline)
-        subject.reload_state!
+            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
+            # a bad test design or missing class functionality.
+            sleep(0.1)
+            Stud.stop!(t)
+            t.join
+            expect(subject.get_pipeline(:main).config_str).to match(second_pipeline_config)
+            subject.shutdown
+          end
+        end
       end
     end
   end
 
   describe "Environment Variables In Configs" do
-    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' } } filter { } output { }" }
+    let(:temporary_file) { Stud::Temporary.file.path }
+
+    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' count => 1 } } filter { } output { file { path => '#{temporary_file}' } }" }
     let(:agent_args) { {
       "config.reload.automatic" => false,
       "config.reload.interval" => 0.01,
       "config.string" => pipeline_config
     } }
 
+    let(:source_loader) {
+      TestSourceLoader.new(mock_pipeline_config(default_pipeline_id, pipeline_config))
+    }
+
+    subject { described_class.new(mock_settings(agent_args), source_loader) }
+
+    after do
+      subject.shutdown
+    end
+
     context "environment variable templating" do
       before :each do
         @foo_content = ENV["FOO"]
@@ -296,52 +326,50 @@ describe LogStash::Agent do
 
       after :each do
         ENV["FOO"] = @foo_content
-        subject.close_pipelines
       end
 
-      it "doesn't upgrade the state" do
-        allow(subject).to receive(:fetch_config).and_return(pipeline_config)
-        subject.register_pipeline(pipeline_settings)
-        expect(subject.pipelines[default_pipeline_id].inputs.first.message).to eq("foo-bar")
+      it "are evaluated at plugins creation" do
+        expect(subject.converge_state_and_update).to be_a_successful_converge
+
+        # Since the pipeline is running in another threads
+        # the content of the file wont be instant.
+        sleep(0.1) until ::File.size(temporary_file) > 0
+        json_document = LogStash::Json.load(File.read(temporary_file).chomp)
+        expect(json_document["message"]).to eq("foo-bar")
       end
     end
   end
 
   describe "#upgrade_pipeline" do
-    let(:pipeline_config) { "input { } filter { } output { }" }
-    let(:pipeline_args) { {
-      "config.string" => pipeline_config,
-      "pipeline.workers" => 4
-    } }
-    let(:new_pipeline_config) { "input { generator {} } output { }" }
+    let(:pipeline_config) { "input { generator {} } filter { } output { }" }
+    let(:pipeline_args) { { "pipeline.workers" => 4 } }
+    let(:mocked_pipeline_config) { mock_pipeline_config(default_pipeline_id, pipeline_config, mock_settings(pipeline_args))}
+
+    let(:new_pipeline_config) { "input generator {} } output { }" }
+    let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_pipeline_config, mock_settings(pipeline_args))}
+    let(:source_loader) { TestSequenceSourceLoader.new(mocked_pipeline_config, mocked_new_pipeline_config)}
+
+    subject { described_class.new(agent_settings, source_loader) }
 
     before(:each) do
-      subject.register_pipeline(pipeline_settings)
+      # Run the initial config
+      expect(subject.converge_state_and_update).to be_a_successful_converge
     end
 
     after(:each) do
       # new pipelines will be created part of the upgrade process so we need
       # to close any initialized pipelines
-      subject.close_pipelines
+      subject.shutdown
     end
 
     context "when the upgrade fails" do
-      before :each do
-        allow(subject).to receive(:fetch_config).and_return(new_pipeline_config)
-        allow(subject).to receive(:create_pipeline).and_return(nil)
-        allow(subject).to receive(:stop_pipeline) do |id|
-          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
-          # not call Pipeline#shutdown but Pipeline#close
-          subject.close_pipeline(id)
-        end
-      end
-
       it "leaves the state untouched" do
-        subject.send(:"reload_pipeline!", default_pipeline_id)
+        expect(subject.converge_state_and_update).not_to be_a_successful_converge
         expect(subject.pipelines[default_pipeline_id].config_str).to eq(pipeline_config)
       end
 
-      context "and current state is empty" do
+      # TODO(ph): This valid?
+      xcontext "and current state is empty" do
         it "should not start a pipeline" do
           expect(subject).to_not receive(:start_pipeline)
           subject.send(:"reload_pipeline!", default_pipeline_id)
@@ -350,45 +378,21 @@ describe LogStash::Agent do
     end
 
     context "when the upgrade succeeds" do
-      let(:new_config) { "input { generator { count => 1 } } output { }" }
-
-      before :each do
-        allow(subject).to receive(:fetch_config).and_return(new_config)
-        allow(subject).to receive(:start_pipeline).and_return(true)
-        allow(subject).to receive(:stop_pipeline) do |id|
-          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
-          # not call Pipeline#shutdown but Pipeline#close
-          subject.close_pipeline(id)
-        end
-      end
+      let(:new_config) { "input { generator { id => 'abc' count => 1000000 } } output { }" }
+      let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_config, mock_settings(pipeline_args)) }
 
       it "updates the state" do
-        subject.send(:"reload_pipeline!", default_pipeline_id)
+        expect(subject.converge_state_and_update).to be_a_successful_converge
         expect(subject.pipelines[default_pipeline_id].config_str).to eq(new_config)
       end
 
       it "starts the pipeline" do
-        expect(subject).to receive(:start_pipeline).and_return(true)
-        expect(subject).to receive(:stop_pipeline) do |id|
-          # we register_pipeline but we never execute them so we have to mock #stop_pipeline to
-          # not call Pipeline#shutdown but Pipeline#close
-          subject.close_pipeline(id)
-        end
-        subject.send(:"reload_pipeline!", default_pipeline_id)
+        expect(subject.converge_state_and_update).to be_a_successful_converge
+        expect(subject.pipelines[default_pipeline_id].running?).to be_truthy
       end
     end
   end
 
-  describe "#fetch_config" do
-    let(:cli_config) { "filter { drop { } } " }
-    let(:agent_args) { { "config.string" => cli_config, "path.config" => config_file } }
-
-    it "should join the config string and config path content" do
-      fetched_config = subject.send(:fetch_config, agent_settings)
-      expect(fetched_config.strip).to eq(cli_config + IO.read(config_file).strip)
-    end
-  end
-
   context "#started_at" do
     it "return the start time when the agent is started" do
       expect(described_class::STARTED_AT).to be_kind_of(Time)
@@ -402,9 +406,10 @@ describe LogStash::Agent do
   end
 
   context "metrics after config reloading" do
-    let!(:config) { "input { generator { } } output { dummyoutput { } }" }
+    let(:temporary_file) { Stud::Temporary.file.path }
+    let(:config) { "input { generator { } } output { file { path => '#{temporary_file}' } }" }
 
-    let!(:config_path) do
+    let(:config_path) do
       f = Stud::Temporary.file
       f.write(config)
       f.fsync
@@ -412,55 +417,35 @@ describe LogStash::Agent do
       f.path
     end
 
-    let(:pipeline_args) do
-      {
-        "pipeline.workers" => 2,
-        "path.config" => config_path
-      }
-    end
-
     let(:agent_args) do
       {
-        "config.reload.automatic" => false,
+        "config.reload.automatic" => true,
+        "config.reload.interval" => 0.01,
         "pipeline.batch.size" => 1,
-        "metric.collect" => true
+        "metric.collect" => true,
+        "path.config" => config_path
       }
     end
 
-    # We need to create theses dummy classes to know how many
-    # events where actually generated by the pipeline and successfully send to the output.
-    # Theses values are compared with what we store in the metric store.
-    class DummyOutput2 < LogStash::Outputs::DroppingDummyOutput; end
-
-    let!(:dummy_output) { LogStash::Outputs::DroppingDummyOutput.new }
-    let!(:dummy_output2) { DummyOutput2.new }
     let(:initial_generator_threshold) { 1000 }
     let(:pipeline_thread) do
       Thread.new do
-        subject.register_pipeline(pipeline_settings)
         subject.execute
       end
     end
 
+    subject { described_class.new(agent_settings, default_source_loader) }
 
     before :each do
-      allow(LogStash::Outputs::DroppingDummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
-      allow(DummyOutput2).to receive(:new).at_least(:once).with(anything).and_return(dummy_output2)
-
-      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
-      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
-      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(LogStash::Outputs::DroppingDummyOutput)
-      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput2").and_return(DummyOutput2)
-
       @abort_on_exception = Thread.abort_on_exception
       Thread.abort_on_exception = true
 
-      pipeline_thread
+      @t = Thread.new do
+        subject.execute
+      end
 
       # wait for some events to reach the dummy_output
-      Timeout.timeout(timeout) do
-        sleep(0.1) until dummy_output.events_received > initial_generator_threshold
-      end
+      sleep(0.1) until IO.readlines(temporary_file).size > initial_generator_threshold
     end
 
     after :each do
@@ -468,8 +453,6 @@ describe LogStash::Agent do
         subject.shutdown
         Stud.stop!(pipeline_thread)
         pipeline_thread.join
-      rescue
-          #don't care about errors here.
       ensure
         Thread.abort_on_exception = @abort_on_exception
       end
@@ -477,7 +460,8 @@ describe LogStash::Agent do
 
     context "when reloading a good config" do
       let(:new_config_generator_counter) { 500 }
-      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
+      let(:output_file) { Stud::Temporary.file.path }
+      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { file { path => '#{output_file}'} }" }
 
       before :each do
         File.open(config_path, "w") do |f|
@@ -485,12 +469,8 @@ describe LogStash::Agent do
           f.fsync
         end
 
-        subject.send(:"reload_pipeline!", "main")
-
         # wait until pipeline restarts
-        Timeout.timeout(timeout) do
-          sleep(0.01) until dummy_output2.events_received > 0
-        end
+        sleep(1) if ::File.read(output_file).empty?
       end
 
       it "resets the pipeline metric collector" do
@@ -509,8 +489,8 @@ describe LogStash::Agent do
         snapshot = subject.metric.collector.snapshot_metric
         value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
         instance_value = snapshot.metric_store.get_with_path("/stats")[:stats][:reloads][:successes].value
-        expect(value).to eq(1)
         expect(instance_value).to eq(1)
+        expect(value).to eq(1)
       end
 
       it "does not set the failure reload timestamp" do
@@ -541,7 +521,7 @@ describe LogStash::Agent do
           f.fsync
         end
 
-        subject.send(:"reload_pipeline!", "main")
+        sleep(1)
       end
 
       it "does not increase the successful reload count" do
@@ -577,7 +557,15 @@ describe LogStash::Agent do
     end
 
     context "when reloading a config that raises exception on pipeline.run" do
-      let(:new_config) { "input { generator { count => 10000 } }" }
+      let(:new_config) { "input { generator { count => 10000 } } output { null {} }" }
+      let(:agent_args) do
+        {
+          "config.reload.automatic" => false,
+          "pipeline.batch.size" => 1,
+          "metric.collect" => true,
+          "path.config" => config_path
+        }
+      end
 
       class BrokenGenerator < LogStash::Inputs::Generator
         def register
@@ -595,7 +583,7 @@ describe LogStash::Agent do
       end
 
       it "does not increase the successful reload count" do
-        expect { subject.send(:"reload_pipeline!", "main") }.to_not change {
+        expect { subject.converge_state_and_update }.to_not change {
           snapshot = subject.metric.collector.snapshot_metric
           reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
           reload_metrics[:successes].value
@@ -603,7 +591,7 @@ describe LogStash::Agent do
       end
 
       it "increases the failures reload count" do
-        expect { subject.send(:"reload_pipeline!", "main") }.to change {
+        expect { subject.converge_state_and_update }.to change {
           snapshot = subject.metric.collector.snapshot_metric
           reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
           reload_metrics[:failures].value