logstash-core 6.4.3-java → 6.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0535700835dd16374433f064d770af9b96e7f226fce537eb5ea81fc802cf56de'
-  data.tar.gz: aaa23c13f0cf82109a1e05a527e28505c387558f7730d1e054ad2a48cfcae898
+  metadata.gz: 47897fa20536fa113fdf69a3f7f738ecf49e0571d0d087eab504b9ad8a1fb6ca
+  data.tar.gz: 72d006015291ab365aea0676cfd024bae8c04ee7e989c3c91d5e5cbca5ede6af
 SHA512:
-  metadata.gz: cc82b0baeecec5bfaad991e94b7c72203a6b4cbebe5cdfa6c954fb7551b7814eb08b5fd3c8d6b6aa92ab5afa40fbeab351f027d33a4193f5a018e84128ed4d9e
-  data.tar.gz: 532c63a9bc508a8bdec9f2e50203e1c13938204529b5c167a51018f06219652c1a3edf78322dba1e46e6b2f20b500f8098cbb3dea8f07834ef36c575f290b3fb
+  metadata.gz: c3ddc6a8206cdc4b44e03fd247f13e77a5b8033932f92812f0d71f209fb125f1a2cc550c152f5ee11e0675a27c513a22b6283f0581e8f9388df0ad124a4c74de
+  data.tar.gz: 12daf679b7934532e7d456b23d368a76e6207e9601b4ee1482d942fb8a3e9772604e1bdd9e2b4eb1eea3c05705849ebc0968a9020ce053ee100e2aa91ab81183

data/lib/logstash/agent.rb

@@ -33,6 +33,10 @@ class LogStash::Agent
     @auto_reload = setting("config.reload.automatic")
     @ephemeral_id = SecureRandom.uuid
 
+    # Mutex to synchonize in the exclusive method
+    # Initial usage for the Ruby pipeline initialization which is not thread safe
+    @exclusive_lock = Mutex.new
+
     # Special bus object for inter-pipelines communications. Used by the `pipeline` input/output
     @pipeline_bus = org.logstash.plugins.pipeline.PipelineBus.new
 
@@ -84,6 +88,10 @@ class LogStash::Agent
     @running = Concurrent::AtomicBoolean.new(false)
   end
 
+  def exclusive(&block)
+    @exclusive_lock.synchronize { block.call }
+  end
+
   def execute
     @thread = Thread.current # this var is implicitly used by Stud.stop?
     logger.debug("Starting agent")

data/lib/logstash/modules/logstash_config.rb

@@ -56,11 +56,25 @@ module LogStash module Modules class LogStashConfig
         get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
       when Numeric
         get_setting(LogStash::Setting::Numeric.new(name, default))
+      when true, false
+        get_setting(LogStash::Setting::Boolean.new(name, default))
       else
         get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
       end
   end
 
+  def has_setting?(name)
+    @settings.key?(name)
+  end
+
+  def raw_setting(name)
+    @settings[name]
+  end
+
+  def fetch_raw_setting(name, default)
+    @settings.fetch(name, default)
+  end
+
   def elasticsearch_output_config(type_string = nil)
     hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])))
     index = "#{@name}-#{setting("var.elasticsearch.index_suffix", "%{+YYYY.MM.dd}")}"

data/lib/logstash/pipeline_action/create.rb

@@ -35,7 +35,12 @@ module LogStash module PipelineAction
         if @pipeline_config.settings.get_value("pipeline.java_execution")
           LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
         else
-          LogStash::Pipeline.new(@pipeline_config, @metric, agent)
+          agent.exclusive do
+            # The Ruby pipeline initialization is not thread safe because of the module level
+            # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
+            # executed simultaneously in different threads and we need to synchonize this initialization.
+            LogStash::Pipeline.new(@pipeline_config, @metric, agent)
+          end
         end
 
       status = nil

data/lib/logstash/pipeline_action/reload.rb

@@ -32,7 +32,12 @@ module LogStash module PipelineAction
           if @pipeline_config.settings.get_value("pipeline.java_execution")
             LogStash::JavaBasePipeline.new(@pipeline_config, nil, logger, nil)
           else
-            LogStash::BasePipeline.new(@pipeline_config)
+            agent.exclusive do
+              # The Ruby pipeline initialization is not thread safe because of the module level
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
+              # executed simultaneously in different threads and we need to synchonize this initialization.
+              LogStash::BasePipeline.new(@pipeline_config)
+            end
           end
       rescue => e
         return LogStash::ConvergeResult::FailedAction.from_exception(e)

data/lib/logstash/runner.rb

@@ -108,8 +108,8 @@ class LogStash::Runner < Clamp::StrictCommand
     :attribute_name => "pipeline.workers",
     :default => LogStash::SETTINGS.get_default("pipeline.workers")
 
-  option ["--experimental-java-execution"], :flag,
-         I18n.t("logstash.runner.flag.experimental-java-execution"),
+  option ["--java-execution"], :flag,
+         I18n.t("logstash.runner.flag.java-execution"),
          :attribute_name => "pipeline.java_execution",
          :default => LogStash::SETTINGS.get_default("pipeline.java_execution")
 
@@ -348,6 +348,8 @@ class LogStash::Runner < Clamp::StrictCommand
     # lock path.data before starting the agent
     @data_path_lock = FileLockFactory.obtainLock(java.nio.file.Paths.get(setting("path.data")).to_absolute_path, ".lock")
 
+    logger.info("Starting Logstash", "logstash.version" => LOGSTASH_VERSION)
+
     @dispatcher.fire(:before_agent)
     @agent = create_agent(@settings, @source_loader)
     @dispatcher.fire(:after_agent)
@@ -357,8 +359,6 @@ class LogStash::Runner < Clamp::StrictCommand
     sigint_id = trap_sigint()
     sigterm_id = trap_sigterm()
 
-    logger.info("Starting Logstash", "logstash.version" => LOGSTASH_VERSION)
-
     @agent_task = Stud::Task.new { @agent.execute }
 
     # no point in enabling config reloading before the agent starts

data/locales/en.yml

@@ -297,8 +297,8 @@ en:
           Sets the ID of the pipeline.
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
-        experimental-java-execution: |+
-          (Experimental) Use new Java execution engine.
+        java-execution: |+
+          (Beta) Use new Java execution engine.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+

data/spec/logstash/agent_spec.rb

@@ -12,131 +12,118 @@ require_relative "../support/matchers"
 java_import org.logstash.Timestamp
 
 describe LogStash::Agent do
-  let(:agent_settings) { mock_settings({}) }
-  let(:agent_args) { {} }
-  let(:pipeline_settings) { agent_settings.clone }
-  let(:pipeline_args) { {} }
-  let(:default_pipeline_id) { agent_settings.get("pipeline.id") }
-  let(:config_string) { "input { } filter { } output { }" }
-  let(:config_file) { Stud::Temporary.pathname }
-  let(:config_file_txt) { config_string }
-  let(:default_source_loader) do
-    sl = LogStash::Config::SourceLoader.new
-    sl.add_source(LogStash::Config::Source::Local.new(agent_settings))
-    sl
-  end
-  let(:logger) { double("logger") }
-  let(:timeout) {160} #seconds
-
-  subject { LogStash::Agent.new(agent_settings, default_source_loader) }
-
-  before :each do
-    # This MUST run first, before `subject` is invoked to ensure clean state
-    clear_data_dir
 
-    File.open(config_file, "w") { |f| f.puts(config_file_txt) }
-
-    agent_args.each do |key, value|
-      agent_settings.set(key, value)
-      pipeline_settings.set(key, value)
-    end
-    pipeline_args.each do |key, value|
-      pipeline_settings.set(key, value)
+  shared_examples "all Agent tests" do
+
+    let(:agent_settings) { mock_settings({}) }
+    let(:agent_args) { {} }
+    let(:pipeline_settings) { agent_settings.clone }
+    let(:pipeline_args) { {} }
+    let(:default_pipeline_id) { agent_settings.get("pipeline.id") }
+    let(:config_string) { "input { } filter { } output { }" }
+    let(:config_file) { Stud::Temporary.pathname }
+    let(:config_file_txt) { config_string }
+    let(:default_source_loader) do
+      sl = LogStash::Config::SourceLoader.new
+      sl.add_source(LogStash::Config::Source::Local.new(agent_settings))
+      sl
     end
-    allow(described_class).to receive(:logger).and_return(logger)
-    [:debug, :info, :error, :fatal, :trace].each {|level| allow(logger).to receive(level) }
-    [:debug?, :info?, :error?, :fatal?, :trace?].each {|level| allow(logger).to receive(level) }
-  end
+    let(:logger) { double("logger") }
+    let(:timeout) { 160 } #seconds
 
-  after :each do
-    subject.shutdown
-    LogStash::SETTINGS.reset
+    subject { LogStash::Agent.new(agent_settings, default_source_loader) }
 
-    FileUtils.rm(config_file)
-    FileUtils.rm_rf(subject.id_path)
-  end
-
-  it "fallback to hostname when no name is provided" do
-    expect(LogStash::Agent.new(agent_settings, default_source_loader).name).to eq(Socket.gethostname)
-  end
+    before :each do
+      # This MUST run first, before `subject` is invoked to ensure clean state
+      clear_data_dir
 
-  describe "adding a new pipeline" do
-    let(:agent_args) { { "config.string" => config_string } }
+      File.open(config_file, "w") { |f| f.puts(config_file_txt) }
 
-    it "should delegate settings to new pipeline" do
-      expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
-        expect(arg1).to eq(config_string)
-        expect(arg2.to_hash).to include(agent_args)
+      agent_args.each do |key, value|
+        agent_settings.set(key, value)
+        pipeline_settings.set(key, value)
+      end
+      pipeline_args.each do |key, value|
+        pipeline_settings.set(key, value)
       end
-      subject.converge_state_and_update
+      allow(described_class).to receive(:logger).and_return(logger)
+      [:debug, :info, :error, :fatal, :trace].each {|level| allow(logger).to receive(level) }
+      [:debug?, :info?, :error?, :fatal?, :trace?].each {|level| allow(logger).to receive(level) }
     end
-  end
 
-  describe "#id" do
-    let(:id_file_data) { File.open(subject.id_path) {|f| f.read } }
+    after :each do
+      subject.shutdown
+      LogStash::SETTINGS.reset
 
-    it "should return a UUID" do
-      expect(subject.id).to be_a(String)
-      expect(subject.id.size).to be > 0
+      FileUtils.rm(config_file)
+      FileUtils.rm_rf(subject.id_path)
     end
 
-    it "should write out the persistent UUID" do
-      expect(id_file_data).to eql(subject.id)
+    it "fallback to hostname when no name is provided" do
+      expect(LogStash::Agent.new(agent_settings, default_source_loader).name).to eq(Socket.gethostname)
     end
-  end
 
-  describe "ephemeral_id" do
-    it "create a ephemeral id at creation time" do
-      expect(subject.ephemeral_id).to_not be_nil
+    describe "adding a new pipeline" do
+      let(:agent_args) { { "config.string" => config_string } }
+
+      it "should delegate settings to new pipeline" do
+        expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
+          expect(arg1).to eq(config_string)
+          expect(arg2.to_hash).to include(agent_args)
+        end
+        subject.converge_state_and_update
+      end
     end
-  end
 
-  describe "#execute" do
-    let(:config_string) { "input { generator { id => 'old'} } output { }" }
-    let(:mock_config_pipeline) { mock_pipeline_config(:main, config_string, pipeline_settings) }
+    describe "#id" do
+      let(:id_file_data) { File.open(subject.id_path) {|f| f.read } }
 
-    let(:source_loader) { TestSourceLoader.new(mock_config_pipeline) }
-    subject { described_class.new(agent_settings, source_loader) }
+      it "should return a UUID" do
+        expect(subject.id).to be_a(String)
+        expect(subject.id.size).to be > 0
+      end
 
-    before :each do
-      allow(subject).to receive(:start_webserver).and_return(false)
-      allow(subject).to receive(:stop_webserver).and_return(false)
+      it "should write out the persistent UUID" do
+        expect(id_file_data).to eql(subject.id)
+      end
     end
 
-    context "when auto_reload is false" do
-      let(:agent_settings) { mock_settings("config.reload.automatic" => false) }
-      let(:agent_args) { { "path.config" => config_file } }
+    describe "ephemeral_id" do
+      it "create a ephemeral id at creation time" do
+        expect(subject.ephemeral_id).to_not be_nil
+      end
+    end
 
-      context "if state is clean" do
-        before :each do
-          allow(subject).to receive(:running_user_defined_pipelines?).and_return(true)
-          allow(subject).to receive(:clean_state?).and_return(false)
-        end
+    describe "#execute" do
+      let(:config_string) { "input { generator { id => 'old'} } output { }" }
+      let(:mock_config_pipeline) { mock_pipeline_config(:main, config_string, pipeline_settings) }
 
-        it "should not converge state more than once" do
-          expect(subject).to receive(:converge_state_and_update).once
-          t = Thread.new { subject.execute }
+      let(:source_loader) { TestSourceLoader.new(mock_config_pipeline) }
+      subject { described_class.new(agent_settings, source_loader) }
 
-          Stud.stop!(t)
-          t.join
-          subject.shutdown
-        end
+      before :each do
+        allow(subject).to receive(:start_webserver).and_return(false)
+        allow(subject).to receive(:stop_webserver).and_return(false)
       end
 
-      context "when calling reloading a pipeline" do
-        context "with a config that contains reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
-          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
+      context "when auto_reload is false" do
+        let(:agent_args) { { "config.reload.automatic" => false, "path.config" => config_file } }
 
-          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+        context "verify settings" do
+          it "should not auto reload" do
+            expect(subject.settings.get("config.reload.automatic")).to eq(false)
+          end
+        end
+
+        context "if state is clean" do
+          before :each do
+            allow(subject).to receive(:running_user_defined_pipelines?).and_return(true)
+            allow(subject).to receive(:clean_state?).and_return(false)
+          end
 
-          it "does not upgrade the new config" do
+          it "should not converge state more than once" do
+            expect(subject).to receive(:converge_state_and_update).once
             t = Thread.new { subject.execute }
-            wait(timeout)
-                .for { subject.running_pipelines? && subject.pipelines.values.first.ready? }
-                .to eq(true)
-            expect(subject.converge_state_and_update).not_to be_a_successful_converge
-            expect(subject).to have_running_pipeline?(mock_config_pipeline)
 
             Stud.stop!(t)
             t.join
@@ -144,364 +131,405 @@ describe LogStash::Agent do
           end
         end
 
-        context "with a config that does not contain reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
+        context "when calling reloading a pipeline" do
+          context "with a config that contains reload incompatible plugins" do
+            let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+            let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
 
-          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+            let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
 
-          it "does upgrade the new config" do
-            t = Thread.new { subject.execute }
-            Timeout.timeout(timeout) do
-              sleep(0.01) until subject.pipelines_count > 0 && subject.pipelines.values.first.ready?
+            it "does not upgrade the new config" do
+              t = Thread.new { subject.execute }
+              wait(timeout)
+                  .for { subject.running_pipelines? && subject.pipelines.values.first.ready? }
+                  .to eq(true)
+              expect(subject.converge_state_and_update).not_to be_a_successful_converge
+              expect(subject).to have_running_pipeline?(mock_config_pipeline)
+
+              Stud.stop!(t)
+              t.join
+              subject.shutdown
             end
+          end
 
-            expect(subject.converge_state_and_update).to be_a_successful_converge
-            expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
+          context "with a config that does not contain reload incompatible plugins" do
+            let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+            let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, pipeline_settings) }
 
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
-          end
-        end
+            let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
 
-      end
-      context "when calling reload_state!" do
-        context "with a pipeline with auto reloading turned off" do
-          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-          let(:pipeline_args) { { "pipeline.reloadable" => false } }
-          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
+            it "does upgrade the new config" do
+              t = Thread.new { subject.execute }
+              Timeout.timeout(timeout) do
+                sleep(0.1) until subject.pipelines_count > 0 && subject.pipelines.values.first.ready?
+              end
 
-          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+              expect(subject.converge_state_and_update).to be_a_successful_converge
+              expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
 
-          it "does not try to reload the pipeline" do
-            t = Thread.new { subject.execute }
-            Timeout.timeout(timeout) do
-              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+              Stud.stop!(t)
+              t.join
+              subject.shutdown
             end
-            expect(subject.converge_state_and_update).not_to be_a_successful_converge
-            expect(subject).to have_running_pipeline?(mock_config_pipeline)
-
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
           end
+
         end
+        context "when calling reload_state!" do
+          context "with a pipeline with auto reloading turned off" do
+            let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+            let(:pipeline_args) { { "pipeline.reloadable" => false } }
+            let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
+
+            let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+
+            it "does not try to reload the pipeline" do
+              t = Thread.new { subject.execute }
+              Timeout.timeout(timeout) do
+                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
+              end
+              expect(subject.converge_state_and_update).not_to be_a_successful_converge
+              expect(subject).to have_running_pipeline?(mock_config_pipeline)
+
+              Stud.stop!(t)
+              t.join
+              subject.shutdown
+            end
+          end
 
-        context "with a pipeline with auto reloading turned on" do
-          let(:second_pipeline_config) { "input { generator { id => 'second' } } filter { } output { }" }
-          let(:pipeline_args) { { "pipeline.reloadable" => true } }
-          let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
-          let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
+          context "with a pipeline with auto reloading turned on" do
+            let(:second_pipeline_config) { "input { generator { id => 'second' } } filter { } output { }" }
+            let(:pipeline_args) { { "pipeline.reloadable" => true } }
+            let(:mock_second_pipeline_config) { mock_pipeline_config(:main, second_pipeline_config, mock_settings(pipeline_args)) }
+            let(:source_loader) { TestSequenceSourceLoader.new(mock_config_pipeline, mock_second_pipeline_config)}
 
-          it "tries to reload the pipeline" do
-            t = Thread.new { subject.execute }
-            Timeout.timeout(timeout) do
-              sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-            end
+            it "tries to reload the pipeline" do
+              t = Thread.new { subject.execute }
+              Timeout.timeout(timeout) do
+                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
+              end
 
-            expect(subject.converge_state_and_update).to be_a_successful_converge
-            expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
+              expect(subject.converge_state_and_update).to be_a_successful_converge
+              expect(subject).to have_running_pipeline?(mock_second_pipeline_config)
 
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
+              Stud.stop!(t)
+              t.join
+              subject.shutdown
+            end
           end
         end
       end
     end
-  end
-
-  describe "Environment Variables In Configs" do
-    let(:temporary_file) { Stud::Temporary.file.path }
 
-    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' count => 1 } } filter { } output { file { path => '#{temporary_file}' } }" }
-    let(:agent_args) { {
-      "config.reload.automatic" => false,
-      "config.reload.interval" => "10ms",
-      "config.string" => pipeline_config
-    } }
+    describe "Environment Variables In Configs" do
+      let(:temporary_file) { Stud::Temporary.file.path }
 
-    let(:source_loader) {
-      TestSourceLoader.new(mock_pipeline_config(default_pipeline_id, pipeline_config))
-    }
+      let(:pipeline_config) { "input { generator { message => '${FOO}-bar' count => 1 } } filter { } output { file { path => '#{temporary_file}' } }" }
+      let(:agent_args) { {
+        "config.reload.automatic" => false,
+        "config.reload.interval" => "10ms",
+        "config.string" => pipeline_config
+      } }
 
-    subject { described_class.new(mock_settings(agent_args), source_loader) }
+      let(:source_loader) {
+        TestSourceLoader.new(mock_pipeline_config(default_pipeline_id, pipeline_config))
+      }
 
-    after do
-      subject.shutdown
-    end
+      subject { described_class.new(mock_settings(agent_args), source_loader) }
 
-    context "environment variable templating" do
-      before :each do
-        @foo_content = ENV["FOO"]
-        ENV["FOO"] = "foo"
+      after do
+        subject.shutdown
       end
 
-      after :each do
-        ENV["FOO"] = @foo_content
-      end
+      context "environment variable templating" do
+        before :each do
+          @foo_content = ENV["FOO"]
+          ENV["FOO"] = "foo"
+        end
 
-      it "are evaluated at plugins creation" do
-        expect(subject.converge_state_and_update).to be_a_successful_converge
+        after :each do
+          ENV["FOO"] = @foo_content
+        end
 
-        # Since the pipeline is running in another threads
-        # the content of the file wont be instant.
-        sleep(0.01) until ::File.size(temporary_file) > 0
-        json_document = LogStash::Json.load(File.read(temporary_file).chomp)
-        expect(json_document["message"]).to eq("foo-bar")
+        it "are evaluated at plugins creation" do
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+
+          # Since the pipeline is running in another threads
+          # the content of the file wont be instant.
+          Timeout.timeout(timeout) do
+            sleep(0.1) until ::File.size(temporary_file) > 0
+          end
+          json_document = LogStash::Json.load(File.read(temporary_file).chomp)
+          expect(json_document["message"]).to eq("foo-bar")
+        end
       end
     end
-  end
-
-  describe "#upgrade_pipeline" do
-    let(:pipeline_config) { "input { generator {} } filter { } output { }" }
-    let(:pipeline_args) { { "pipeline.workers" => 4 } }
-    let(:mocked_pipeline_config) { mock_pipeline_config(default_pipeline_id, pipeline_config, mock_settings(pipeline_args))}
 
-    let(:new_pipeline_config) { "input generator {} } output { }" }
-    let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_pipeline_config, mock_settings(pipeline_args))}
-    let(:source_loader) { TestSequenceSourceLoader.new(mocked_pipeline_config, mocked_new_pipeline_config)}
+    describe "#upgrade_pipeline" do
+      let(:pipeline_config) { "input { generator {} } filter { } output { }" }
+      let(:pipeline_args) { { "pipeline.workers" => 4 } }
+      let(:mocked_pipeline_config) { mock_pipeline_config(default_pipeline_id, pipeline_config, mock_settings(pipeline_args))}
 
-    subject { described_class.new(agent_settings, source_loader) }
+      let(:new_pipeline_config) { "input generator {} } output { }" }
+      let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_pipeline_config, mock_settings(pipeline_args))}
+      let(:source_loader) { TestSequenceSourceLoader.new(mocked_pipeline_config, mocked_new_pipeline_config)}
 
-    before(:each) do
-      # Run the initial config
-      expect(subject.converge_state_and_update).to be_a_successful_converge
-    end
+      subject { described_class.new(agent_settings, source_loader) }
 
-    after(:each) do
-      # new pipelines will be created part of the upgrade process so we need
-      # to close any initialized pipelines
-      subject.shutdown
-    end
+      before(:each) do
+        # Run the initial config
+        expect(subject.converge_state_and_update).to be_a_successful_converge
+      end
 
-    context "when the upgrade fails" do
-      it "leaves the state untouched" do
-        expect(subject.converge_state_and_update).not_to be_a_successful_converge
-        expect(subject.get_pipeline(default_pipeline_id).config_str).to eq(pipeline_config)
+      after(:each) do
+        # new pipelines will be created part of the upgrade process so we need
+        # to close any initialized pipelines
+        subject.shutdown
       end
 
-      # TODO(ph): This valid?
-      xcontext "and current state is empty" do
-        it "should not start a pipeline" do
-          expect(subject).to_not receive(:start_pipeline)
-          subject.send(:"reload_pipeline!", default_pipeline_id)
+      context "when the upgrade fails" do
+        it "leaves the state untouched" do
+          expect(subject.converge_state_and_update).not_to be_a_successful_converge
+          expect(subject.get_pipeline(default_pipeline_id).config_str).to eq(pipeline_config)
+        end
+
+        # TODO(ph): This valid?
+        xcontext "and current state is empty" do
+          it "should not start a pipeline" do
+            expect(subject).to_not receive(:start_pipeline)
+            subject.send(:"reload_pipeline!", default_pipeline_id)
+          end
         end
       end
-    end
 
-    context "when the upgrade succeeds" do
-      let(:new_config) { "input { generator { id => 'abc' count => 1000000 } } output { }" }
-      let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_config, mock_settings(pipeline_args)) }
+      context "when the upgrade succeeds" do
+        let(:new_config) { "input { generator { id => 'abc' count => 1000000 } } output { }" }
+        let(:mocked_new_pipeline_config) { mock_pipeline_config(default_pipeline_id, new_config, mock_settings(pipeline_args)) }
 
-      it "updates the state" do
-        expect(subject.converge_state_and_update).to be_a_successful_converge
-        expect(subject.get_pipeline(default_pipeline_id).config_str).to eq(new_config)
-      end
+        it "updates the state" do
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+          expect(subject.get_pipeline(default_pipeline_id).config_str).to eq(new_config)
+        end
 
-      it "starts the pipeline" do
-        expect(subject.converge_state_and_update).to be_a_successful_converge
-        expect(subject.get_pipeline(default_pipeline_id).running?).to be_truthy
+        it "starts the pipeline" do
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+          expect(subject.get_pipeline(default_pipeline_id).running?).to be_truthy
+        end
       end
     end
-  end
 
-  context "#started_at" do
-    it "return the start time when the agent is started" do
-      expect(described_class::STARTED_AT).to be_kind_of(Time)
+    context "#started_at" do
+      it "return the start time when the agent is started" do
+        expect(described_class::STARTED_AT).to be_kind_of(Time)
+      end
     end
-  end
 
-  context "#uptime" do
-    it "return the number of milliseconds since start time" do
-      expect(subject.uptime).to be >= 0
+    context "#uptime" do
+      it "return the number of milliseconds since start time" do
+        expect(subject.uptime).to be >= 0
+      end
     end
-  end
 
-  context "metrics after config reloading" do
+    context "metrics after config reloading" do
 
-    let(:initial_generator_threshold) { 1000 }
-    let(:original_config_output) { Stud::Temporary.pathname }
-    let(:new_config_output) { Stud::Temporary.pathname }
+      let(:initial_generator_threshold) { 1000 }
+      let(:original_config_output) { Stud::Temporary.pathname }
+      let(:new_config_output) { Stud::Temporary.pathname }
 
-    let(:config_file_txt) { "input { generator { count => #{initial_generator_threshold*2} } } output { file { path => '#{original_config_output}'} }" }
+      let(:config_file_txt) { "input { generator { count => #{initial_generator_threshold*2} } } output { file { path => '#{original_config_output}'} }" }
 
-    let(:agent_args) do
-      {
-        "metric.collect" => true,
-        "path.config" => config_file
-      }
-    end
+      let(:agent_args) do
+        {
+          "metric.collect" => true,
+          "path.config" => config_file
+        }
+      end
 
-    subject { described_class.new(agent_settings, default_source_loader) }
+      subject { described_class.new(agent_settings, default_source_loader) }
 
-    let(:agent_thread) do
-      # subject has to be called for the first time outside the thread because it could create a race condition
-      # with subsequent subject calls
-      s = subject
-      Thread.new { s.execute }
-    end
+      let(:agent_thread) do
+        # subject has to be called for the first time outside the thread because it could create a race condition
+        # with subsequent subject calls
+        s = subject
+        Thread.new { s.execute }
+      end
 
-    before(:each) do
-      @abort_on_exception = Thread.abort_on_exception
-      Thread.abort_on_exception = true
+      before(:each) do
+        @abort_on_exception = Thread.abort_on_exception
+        Thread.abort_on_exception = true
 
-      agent_thread
+        agent_thread
 
-      # wait for some events to reach the dummy_output
-      Timeout.timeout(timeout) do
-        # wait for file existence otherwise it will raise exception on Windows
-        sleep(0.1) until ::File.exist?(original_config_output)
-        sleep(0.1) until IO.readlines(original_config_output).size > initial_generator_threshold
-      end
+        # wait for some events to reach the dummy_output
+        Timeout.timeout(timeout) do
+          # wait for file existence otherwise it will raise exception on Windows
+          sleep(0.3) until ::File.exist?(original_config_output)
+          sleep(0.3) until IO.readlines(original_config_output).size > initial_generator_threshold
+        end
 
-      # write new config
-      File.open(config_file, "w") { |f| f.write(new_config) }
-    end
+        # write new config
+        File.open(config_file, "w") { |f| f.write(new_config) }
+      end
 
-    after :each do
-      begin
-        Stud.stop!(agent_thread) rescue nil # it may be dead already
-        agent_thread.join
-        subject.shutdown
+      after :each do
+        begin
+          Stud.stop!(agent_thread) rescue nil # it may be dead already
+          agent_thread.join
+          subject.shutdown
 
-        FileUtils.rm(original_config_output)
-        FileUtils.rm(new_config_output) if File.exist?(new_config_output)
-      rescue
-          #don't care about errors here.
-      ensure
-        Thread.abort_on_exception = @abort_on_exception
+          FileUtils.rm(original_config_output)
+          FileUtils.rm(new_config_output) if File.exist?(new_config_output)
+        rescue
+            #don't care about errors here.
+        ensure
+          Thread.abort_on_exception = @abort_on_exception
+        end
       end
-    end
 
-    context "when reloading a good config" do
-      let(:new_config_generator_counter) { 500 }
-      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { file { path => '#{new_config_output}'} }" }
+      context "when reloading a good config" do
+        let(:new_config_generator_counter) { 500 }
+        let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { file { path => '#{new_config_output}'} }" }
 
-      before :each do
-        subject.converge_state_and_update
+        before :each do
+          subject.converge_state_and_update
+
+          # wait for file existence otherwise it will raise exception on Windows
+          wait(timeout)
+            .for { ::File.exists?(new_config_output) && !::File.read(new_config_output).chomp.empty? }
+            .to eq(true)
+          # ensure the converge_state_and_update method has updated metrics by
+          # invoking the mutex
+          subject.running_pipelines?
+        end
 
-        # wait for file existence otherwise it will raise exception on Windows
-        wait(timeout)
-          .for { ::File.exists?(new_config_output) && !::File.read(new_config_output).chomp.empty? }
-          .to eq(true)
-        # ensure the converge_state_and_update method has updated metrics by
-        # invoking the mutex
-        subject.running_pipelines?
-      end
+        it "resets the pipeline metric collector" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value
+          expect(value).to be <= new_config_generator_counter
+        end
 
-      it "resets the pipeline metric collector" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value
-        expect(value).to be <= new_config_generator_counter
-      end
+        it "does not reset the global event count" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
+          expect(value).to be > initial_generator_threshold
+        end
 
-      it "does not reset the global event count" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
-        expect(value).to be > initial_generator_threshold
-      end
+        it "increases the successful reload count" do
+          skip("This test fails randomly, tracked in https://github.com/elastic/logstash/issues/8005")
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
+          expect(value).to eq(1)
+          instance_value = snapshot.metric_store.get_with_path("/stats")[:stats][:reloads][:successes].value
+          expect(instance_value).to eq(1)
+        end
 
-      it "increases the successful reload count" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
-        expect(value).to eq(1)
-        instance_value = snapshot.metric_store.get_with_path("/stats")[:stats][:reloads][:successes].value
-        expect(instance_value).to eq(1)
-      end
+        it "does not set the failure reload timestamp" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
+          expect(value).to be(nil)
+        end
 
-      it "does not set the failure reload timestamp" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
-        expect(value).to be(nil)
-      end
+        it "sets the success reload timestamp" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
+          expect(value).to be_a(Timestamp)
+        end
 
-      it "sets the success reload timestamp" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
-        expect(value).to be_a(Timestamp)
+        it "does not set the last reload error" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
+          expect(value).to be(nil)
+        end
       end
 
-      it "does not set the last reload error" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
-        expect(value).to be(nil)
-      end
-    end
+      context "when reloading a bad config" do
+        let(:new_config) { "input { generator { count => " }
+        before(:each) { subject.converge_state_and_update }
 
-    context "when reloading a bad config" do
-      let(:new_config) { "input { generator { count => " }
-      before(:each) { subject.converge_state_and_update }
+        it "does not increase the successful reload count" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
+          expect(value).to eq(0)
+        end
 
-      it "does not increase the successful reload count" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
-        expect(value).to eq(0)
-      end
+        it "does not set the successful reload timestamp" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
+          expect(value).to be(nil)
+        end
 
-      it "does not set the successful reload timestamp" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
-        expect(value).to be(nil)
-      end
+        it "sets the failure reload timestamp" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
+          expect(value).to be_a(Timestamp)
+        end
 
-      it "sets the failure reload timestamp" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
-        expect(value).to be_a(Timestamp)
-      end
+        it "sets the last reload error" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
+          expect(value).to be_a(Hash)
+          expect(value).to include(:message, :backtrace)
+        end
 
-      it "sets the last reload error" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
-        expect(value).to be_a(Hash)
-        expect(value).to include(:message, :backtrace)
+        it "increases the failed reload count" do
+          snapshot = subject.metric.collector.snapshot_metric
+          value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:failures].value
+          expect(value).to be > 0
+        end
       end
 
-      it "increases the failed reload count" do
-        snapshot = subject.metric.collector.snapshot_metric
-        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:failures].value
-        expect(value).to be > 0
-      end
-    end
+      context "when reloading a config that raises exception on pipeline.run" do
+        let(:new_config) { "input { generator { count => 10000 } } output { null {} }" }
+        let(:agent_args) do
+          {
+            "config.reload.automatic" => false,
+            "pipeline.batch.size" => 1,
+            "metric.collect" => true,
+            "path.config" => config_file
+          }
+        end
 
-    context "when reloading a config that raises exception on pipeline.run" do
-      let(:new_config) { "input { generator { count => 10000 } } output { null {} }" }
-      let(:agent_args) do
-        {
-          "config.reload.automatic" => false,
-          "pipeline.batch.size" => 1,
-          "metric.collect" => true,
-          "path.config" => config_file
-        }
-      end
+        class BrokenGenerator < LogStash::Inputs::Generator
+          def register
+            raise ArgumentError
+          end
+        end
 
-      class BrokenGenerator < LogStash::Inputs::Generator
-        def register
-          raise ArgumentError
+        before :each do
+          allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(BrokenGenerator)
         end
-      end
 
-      before :each do
-        allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(BrokenGenerator)
-      end
+        it "does not increase the successful reload count" do
+          expect { subject.converge_state_and_update }.to_not change {
+            snapshot = subject.metric.collector.snapshot_metric
+            reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
+            reload_metrics[:successes].value
+          }
+        end
 
-      it "does not increase the successful reload count" do
-        expect { subject.converge_state_and_update }.to_not change {
-          snapshot = subject.metric.collector.snapshot_metric
-          reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
-          reload_metrics[:successes].value
-        }
+        it "increases the failures reload count" do
+          expect { subject.converge_state_and_update }.to change {
+            snapshot = subject.metric.collector.snapshot_metric
+            reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
+            reload_metrics[:failures].value
+          }.by(1)
+        end
       end
+    end
+  end
 
-      it "increases the failures reload count" do
-        expect { subject.converge_state_and_update }.to change {
-          snapshot = subject.metric.collector.snapshot_metric
-          reload_metrics = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads]
-          reload_metrics[:failures].value
-        }.by(1)
-      end
+  # running all agent tests both using memory and persisted queue is important to make sure we
+  # don't introduce regressions in the queue/pipeline initialization sequence which typically surface
+  # in agent tests and in particular around config reloading
+
+  describe "using memory queue" do
+    it_behaves_like "all Agent tests" do
+      let(:agent_settings) { mock_settings("queue.type" => "memory") }
+    end
+  end
+
+  describe "using persisted queue" do
+    it_behaves_like "all Agent tests" do
+      let(:agent_settings) { mock_settings("queue.type" => "persisted", "queue.drain" => true) }
     end
   end
 end

data/spec/logstash/pipeline_action/create_spec.rb

@@ -33,21 +33,25 @@ describe LogStash::PipelineAction::Create do
     let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 1 } } output { null {} }") }
 
     it "returns a successful execution status" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect(subject.execute(agent, pipelines)).to be_truthy
     end
   end
 
   context "when the pipeline successfully start" do
     it "adds the pipeline to the current pipelines" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect { subject.execute(agent, pipelines) }.to change(pipelines, :size).by(1)
     end
 
     it "starts the pipeline" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       subject.execute(agent, pipelines)
       expect(pipelines[:main].running?).to be_truthy
     end
 
     it "returns a successful execution status" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect(subject.execute(agent, pipelines)).to be_truthy
     end
   end
@@ -65,6 +69,7 @@ describe LogStash::PipelineAction::Create do
       let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } filter { ruby { init => '1/0' code => '1+2' } } output { null {} }") }
 
       it "returns false" do
+        allow(agent).to receive(:exclusive) { |&arg| arg.call }
         expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
       end
     end

data/spec/logstash/pipeline_action/reload_spec.rb

@@ -33,15 +33,18 @@ describe LogStash::PipelineAction::Reload do
 
   context "when existing pipeline and new pipeline are both reloadable" do
     it "stop the previous pipeline" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect { subject.execute(agent, pipelines) }.to change(pipeline, :running?).from(true).to(false)
     end
 
     it "start the new pipeline" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       subject.execute(agent, pipelines)
       expect(pipelines[pipeline_id].running?).to be_truthy
     end
 
     it "run the new pipeline code" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       subject.execute(agent, pipelines)
       expect(pipelines[pipeline_id].config_hash).to eq(new_pipeline_config.config_hash)
     end
@@ -61,6 +64,7 @@ describe LogStash::PipelineAction::Reload do
     let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
 
     it "cannot successfully execute the action" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
     end
   end
@@ -69,6 +73,7 @@ describe LogStash::PipelineAction::Reload do
     let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
 
     it "cannot successfully execute the action" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
     end
   end
@@ -79,6 +84,7 @@ describe LogStash::PipelineAction::Reload do
     end
 
     it "cannot successfully execute the action" do
+      allow(agent).to receive(:exclusive) { |&arg| arg.call }
       expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
     end
   end

data/versions-gem-copy.yml

@@ -1,6 +1,6 @@
 ---
-logstash: 6.4.3
-logstash-core: 6.4.3
+logstash: 6.5.0
+logstash-core: 6.5.0
 logstash-core-plugin-api: 2.1.16
 
 # jruby must reference a *released* version of jruby which can be downloaded from the official download url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 6.4.3
+  version: 6.5.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2018-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement