logstash-core 6.6.0-java → 6.6.1-java

data/lib/logstash/pipeline_action/create.rb

@@ -3,6 +3,7 @@ require "logstash/pipeline_action/base"
 require "logstash/pipeline"
 require "logstash/java_pipeline"
 
+
 module LogStash module PipelineAction
   class Create < Base
     include LogStash::Util::Loggable
@@ -30,33 +31,26 @@ module LogStash module PipelineAction
 
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
-    def execute(agent, pipelines)
-      pipeline =
+    def execute(agent, pipelines_registry)
+      new_pipeline =
         if @pipeline_config.settings.get_value("pipeline.java_execution")
           LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
         else
           agent.exclusive do
             # The Ruby pipeline initialization is not thread safe because of the module level
             # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
-            # executed simultaneously in different threads and we need to synchonize this initialization.
+            # executed simultaneously in different threads and we need to synchronize this initialization.
             LogStash::Pipeline.new(@pipeline_config, @metric, agent)
           end
         end
 
-      status = nil
-      pipelines.compute(pipeline_id) do |id,value|
-        if value
-          LogStash::ConvergeResult::ActionResult.create(self, true)
-        end
-        status = pipeline.start # block until the pipeline is correctly started or crashed
-        pipeline # The pipeline is successfully started we can add it to the map
+      success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
+        new_pipeline.start # block until the pipeline is correctly started or crashed
       end
 
-
-      LogStash::ConvergeResult::ActionResult.create(self, status)
+      LogStash::ConvergeResult::ActionResult.create(self, success)
     end
 
-
     def to_s
       "PipelineAction::Create<#{pipeline_id}>"
     end

data/lib/logstash/pipeline_action/reload.rb

@@ -20,8 +20,12 @@ module LogStash module PipelineAction
       "PipelineAction::Reload<#{pipeline_id}>"
     end
 
-    def execute(agent, pipelines)
-      old_pipeline = pipelines[pipeline_id]
+    def execute(agent, pipelines_registry)
+      old_pipeline = pipelines_registry.get_pipeline(pipeline_id)
+
+      if old_pipeline.nil?
+        return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the pipeline does not exist")
+      end
 
       if !old_pipeline.reloadable?
         return LogStash::ConvergeResult::FailedAction.new("Cannot reload pipeline, because the existing pipeline is not reloadable")
@@ -34,8 +38,8 @@ module LogStash module PipelineAction
           else
             agent.exclusive do
               # The Ruby pipeline initialization is not thread safe because of the module level
-              # shared state in LogsStash::Config::AST. When using multiple pipelines this can gets
-              # executed simultaneously in different threads and we need to synchonize this initialization.
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
+              # executed simultaneously in different threads and we need to synchronize this initialization.
               LogStash::BasePipeline.new(@pipeline_config)
             end
           end
@@ -49,16 +53,35 @@ module LogStash module PipelineAction
 
       logger.info("Reloading pipeline", "pipeline.id" => pipeline_id)
 
-      pipelines.compute(pipeline_id) do |_,pipeline|
-        status = Stop.new(pipeline_id).execute(agent, pipelines)
+      success = pipelines_registry.reload_pipeline(pipeline_id) do
+        # important NOT to explicitly return from block here
+        # the block must emit a success boolean value
 
-        if status
-          return Create.new(@pipeline_config, @metric).execute(agent, pipelines)
-        else
-          return status
-        end
-        pipeline
+        # First shutdown old pipeline
+        old_pipeline.shutdown { LogStash::ShutdownWatcher.start(old_pipeline) }
+        old_pipeline.thread.join
+
+        # Then create a new pipeline
+        new_pipeline =
+          if @pipeline_config.settings.get_value("pipeline.java_execution")
+            LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
+          else
+            agent.exclusive do
+              # The Ruby pipeline initialization is not thread safe because of the module level
+              # shared state in LogsStash::Config::AST. When using multiple pipelines this gets
+              # executed simultaneously in different threads and we need to synchronize this initialization.
+              LogStash::Pipeline.new(@pipeline_config, @metric, agent)
+            end
+          end
+
+        success = new_pipeline.start # block until the pipeline is correctly started or crashed
+
+        # return success and new_pipeline to registry reload_pipeline
+        [success, new_pipeline]
       end
+
+      LogStash::ConvergeResult::ActionResult.create(self, success)
     end
+
   end
 end end

data/lib/logstash/pipeline_action/stop.rb

@@ -9,15 +9,13 @@ module LogStash module PipelineAction
       @pipeline_id = pipeline_id
     end
 
-    def execute(agent, pipelines)
-      pipelines.compute(pipeline_id) do |_,pipeline|
+    def execute(agent, pipelines_registry)
+      pipelines_registry.terminate_pipeline(pipeline_id) do |pipeline|
         pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
         pipeline.thread.join
-        nil # delete the pipeline
       end
-      # If we reach this part of the code we have succeeded because
-      # the shutdown call will block.
-      return LogStash::ConvergeResult::SuccessfulAction.new
+
+      LogStash::ConvergeResult::SuccessfulAction.new
     end
 
     def to_s

data/lib/logstash/pipelines_registry.rb

@@ -0,0 +1,166 @@
+# encoding: utf-8
+
+module LogStash
+  class PipelineState
+    attr_reader :pipeline_id, :pipeline
+
+    def initialize(pipeline_id, pipeline)
+      @pipeline_id = pipeline_id
+      @pipeline = pipeline
+      @reloading = Concurrent::AtomicBoolean.new(false)
+    end
+
+    def terminated?
+      # a reloading pipeline is never considered terminated
+      @reloading.false? && @pipeline.finished_execution?
+    end
+
+    def set_reloading(is_reloading)
+      @reloading.value = is_reloading
+    end
+
+    def set_pipeline(pipeline)
+      raise(ArgumentError, "invalid nil pipeline") if pipeline.nil?
+      @pipeline = pipeline
+    end
+  end
+
+  class PipelinesRegistry
+    attr_reader :states
+    include LogStash::Util::Loggable
+
+    def initialize
+      # we leverage the semantic of the Java ConcurrentHashMap for the
+      # compute() method which is atomic; calling compute() concurrently
+      # will block until the other compute finishes so no mutex is necessary
+      # for synchronizing compute calls
+      @states = java.util.concurrent.ConcurrentHashMap.new
+    end
+
+    # Execute the passed creation logic block and create a new state upon success
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param pipeline [Pipeline] the new pipeline to create
+    # @param create_block [Block] the creation execution logic
+    #
+    # @yieldreturn [Boolean] the new pipeline creation success
+    #
+    # @return [Boolean] new pipeline creation success
+    def create_pipeline(pipeline_id, pipeline, &create_block)
+      success = false
+
+      @states.compute(pipeline_id) do |_, state|
+        if state
+          if state.terminated?
+            success = yield
+            state.set_pipeline(pipeline)
+          else
+            logger.error("Attempted to create a pipeline that already exists", :pipeline_id => pipeline_id)
+          end
+          state
+        else
+          success = yield
+          success ? PipelineState.new(pipeline_id, pipeline) : nil
+        end
+      end
+
+      success
+    end
+
+    # Execute the passed termination logic block
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param stop_block [Block] the termination execution logic
+    #
+    # @yieldparam [Pipeline] the pipeline to terminate
+    def terminate_pipeline(pipeline_id, &stop_block)
+      @states.compute(pipeline_id) do |_, state|
+        if state.nil?
+          logger.error("Attempted to terminate a pipeline that does not exists", :pipeline_id => pipeline_id)
+          nil
+        else
+          yield(state.pipeline)
+          state
+        end
+      end
+    end
+
+    # Execute the passed reloading logic block in the context of the reloading state and set new pipeline in state
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @param reload_block [Block] the reloading execution logic
+    #
+    # @yieldreturn [Array<Boolean, Pipeline>] the new pipeline creation success and new pipeline object
+    #
+    # @return [Boolean] new pipeline creation success
+    def reload_pipeline(pipeline_id, &reload_block)
+      success = false
+
+      @states.compute(pipeline_id) do |_, state|
+        if state.nil?
+          logger.error("Attempted to reload a pipeline that does not exists", :pipeline_id => pipeline_id)
+          nil
+        else
+          state.set_reloading(true)
+          begin
+            success, new_pipeline = yield
+            state.set_pipeline(new_pipeline)
+          ensure
+            state.set_reloading(false)
+          end
+          state
+        end
+      end
+
+      success
+    end
+
+    # @param pipeline_id [String, Symbol] the pipeline id
+    # @return [Pipeline] the pipeline object or nil if none for pipeline_id
+    def get_pipeline(pipeline_id)
+      state = @states.get(pipeline_id)
+      state.nil? ? nil : state.pipeline
+    end
+
+    # @return [Fixnum] number of items in the states collection
+    def size
+      @states.size
+    end
+
+    # @return [Boolean] true if the states collection is empty.
+    def empty?
+      @states.isEmpty
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def running_pipelines
+      select_pipelines { |state| !state.terminated? }
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def non_running_pipelines
+      select_pipelines { |state| state.terminated? }
+    end
+
+    # @return [Hash{String=>Pipeline}]
+    def running_user_defined_pipelines
+      select_pipelines { |state | !state.terminated? && !state.pipeline.system? }
+    end
+
+    private
+
+    # Returns a mapping of pipelines by their ids.
+    # Pipelines can optionally be filtered by their `PipelineState` by passing
+    # a block that returns truthy when a pipeline should be included in the
+    # result.
+    #
+    # @yieldparam [PipelineState]
+    # @yieldreturn [Boolean]
+    #
+    # @return [Hash{String=>Pipeline}]
+    def select_pipelines(&optional_state_filter)
+      @states.each_with_object({}) do |(id, state), memo|
+        if state && (!block_given? || yield(state))
+          memo[id] = state.pipeline
+        end
+      end
+    end
+  end
+end

data/lib/logstash/state_resolver.rb

@@ -10,11 +10,11 @@ module LogStash
       @metric = metric
     end
 
-    def resolve(pipelines, pipeline_configs)
+    def resolve(pipelines_registry, pipeline_configs)
       actions = []
 
       pipeline_configs.each do |pipeline_config|
-        pipeline = pipelines[pipeline_config.pipeline_id]
+        pipeline = pipelines_registry.get_pipeline(pipeline_config.pipeline_id)
 
         if pipeline.nil?
           actions << LogStash::PipelineAction::Create.new(pipeline_config, @metric)
@@ -25,12 +25,12 @@ module LogStash
         end
       end
 
-      running_pipelines = pipeline_configs.collect(&:pipeline_id)
+      configured_pipelines = pipeline_configs.collect(&:pipeline_id)
 
       # If one of the running pipeline is not in the pipeline_configs, we assume that we need to
       # stop it.
-      pipelines.keys
-        .select { |pipeline_id| !running_pipelines.include?(pipeline_id) }
+      pipelines_registry.running_pipelines.keys
+        .select { |pipeline_id| !configured_pipelines.include?(pipeline_id) }
         .each { |pipeline_id| actions << LogStash::PipelineAction::Stop.new(pipeline_id) }
 
       actions.sort # See logstash/pipeline_action.rb

data/lib/logstash/util/safe_uri.rb

@@ -26,6 +26,7 @@ class LogStash::Util::SafeURI
            else
              raise ArgumentError, "Expected a string, java.net.URI, or URI, got a #{arg.class} creating a URL"
            end
+    raise ArgumentError, "URI is not valid - host is not specified" if @uri.host.nil?
   end
 
   def to_s

data/spec/logstash/agent/converge_spec.rb

@@ -49,7 +49,7 @@ describe LogStash::Agent do
 
     context "system pipeline" do
       
-      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { generator { } } output { null {} }", { "pipeline.system" => true }) }
+      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { dummyblockinginput { } } output { null {} }", { "pipeline.system" => true }) }
 
       context "when we have a finite pipeline and a system pipeline running" do
 
@@ -65,40 +65,40 @@ describe LogStash::Agent do
       end
 
       context "when we have an infinite pipeline and a system pipeline running" do
-        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { generator { } } output { null {} }") }
+        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { } } output { null {} }") }
 
         let(:source_loader) do
           TestSourceLoader.new(infinite_pipeline_config, system_pipeline_config)
         end
 
         before(:each) do
-            @agent_task = start_agent(subject)
+          @agent_task = start_agent(subject)
         end
 
         after(:each) do
-            @agent_task.stop!
+          @agent_task.stop!
+          @agent_task.wait
+          subject.shutdown
         end
 
         describe "#running_user_defined_pipelines" do
           it "returns the user defined pipelines" do
-            wait_for do
-              subject.with_running_user_defined_pipelines {|pipelines| pipelines.keys }
-            end.to eq([:main])
-          end
+            # wait is necessary to accommodate for pipelines startup time
+            wait(60).for {subject.running_user_defined_pipelines.keys}.to eq([:main])
+           end
         end
 
         describe "#running_user_defined_pipelines?" do
           it "returns true" do
-            wait_for do
-              subject.running_user_defined_pipelines?
-            end.to be_truthy
+            # wait is necessary to accommodate for pipelines startup time
+            wait(60).for {subject.running_user_defined_pipelines?}.to be_truthy
           end
         end
       end
     end
 
     context "when `config.reload.automatic`" do
-      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
 
       let(:source_loader) do
         TestSourceLoader.new(pipeline_config)
@@ -114,14 +114,14 @@ describe LogStash::Agent do
 
           after(:each) do
             @agent_task.stop!
+            @agent_task.wait
+            subject.shutdown
           end
 
           it "converge only once" do
             wait(60).for { source_loader.fetch_count }.to eq(1)
-
+            # no need to wait here because have_running_pipeline? does the wait
             expect(subject).to have_running_pipeline?(pipeline_config)
-
-            subject.shutdown
           end
         end
 
@@ -135,8 +135,6 @@ describe LogStash::Agent do
 
             expect(source_loader.fetch_count).to eq(1)
             expect(subject.pipelines_count).to eq(0)
-
-            subject.shutdown
           end
         end
       end
@@ -149,26 +147,25 @@ describe LogStash::Agent do
             "config.reload.interval" =>  interval
           )
         end
+
         before(:each) do
           @agent_task = start_agent(subject)
         end
 
         after(:each) do
           @agent_task.stop!
+          @agent_task.wait
+          subject.shutdown
         end
 
         context "and successfully load the config" do
           it "converges periodically the pipelines from the configs source" do
-            sleep(2) # let the interval reload a few times
+            # no need to wait here because have_running_pipeline? does the wait
             expect(subject).to have_running_pipeline?(pipeline_config)
 
             # we rely on a periodic thread to call fetch count, we have seen unreliable run on
             # travis, so lets add a few retries
-            try do
-              expect(source_loader.fetch_count).to be > 1
-            end
-
-            subject.shutdown
+            try { expect(source_loader.fetch_count).to be > 1 }
           end
         end
 
@@ -178,12 +175,9 @@ describe LogStash::Agent do
           end
 
           it "it will keep trying to converge" do
-
             sleep(agent_settings.get("config.reload.interval") / 1_000_000_000.0 * 20) # let the interval reload a few times
             expect(subject.pipelines_count).to eq(0)
             expect(source_loader.fetch_count).to be > 1
-
-            subject.shutdown
           end
         end
       end
@@ -191,8 +185,8 @@ describe LogStash::Agent do
   end
 
   context "when shutting down the agent" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator { id => 'new' } } output { null {} }") }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput { id => 'new' } } output { null {} }") }
 
     let(:source_loader) do
       TestSourceLoader.new([pipeline_config, new_pipeline_config])
@@ -205,8 +199,8 @@ describe LogStash::Agent do
   end
 
   context "Configuration converge scenario" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
 
     before do
       # Set the Agent to an initial state of pipelines
@@ -263,7 +257,7 @@ describe LogStash::Agent do
     end
 
     context "when the source return a modified pipeline" do
-      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { generator { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
+      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
 
       let(:source_loader) do
         TestSequenceSourceLoader.new(