logstash-core 6.6.0-java → 6.6.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbe096068df3326925926019e2137eb56527f92c91c9d14861616d85e19ac46b
-  data.tar.gz: 8a5a795fbb82efc19a01232f8ef173d668948a33f75aaa749df73e0cc075e206
+  metadata.gz: 6ae11e5a947442d4fe74bd284d674355a78112f5aff290403930bb35665ad724
+  data.tar.gz: 30c09cc64470baa5bb29f806cff27614fe64f83a205b6704740db4be569dd3a9
 SHA512:
-  metadata.gz: 99c5e85d59c293bd4479ca1d4805cb0dff72b225ed97aa7351060c452c1e73e495a24bf3fbc4142df0e67f01bb1eaa90893af1cd8c30d22f6a61d4fed2a0cfd0
-  data.tar.gz: 977df2140b2d9596ad4d08e784bd03cbb063c2c40b9a158771eaf91b6b4f09f80e0804dce33f801c002dfe2cf4664b8a544f588565c4aa492fa2f83c7c964339
+  metadata.gz: 81f35a278ce95f4cb063524c5333f4ce09f1de1a14fbefdc09a4b72f62cea73ec519a90b5b2b27cd720bd802bf40001656e6f90a294440c9898738567104dbdd
+  data.tar.gz: 74fdec603be67ac988e3b786e890733fd8254caa78c7d03b2c99afc1a8bb25779b0fefc9cb5675fdfec1fa06b3e124c6af880c0ee0197320d14e176d1370c742

data/lib/logstash/agent.rb

@@ -8,6 +8,7 @@ require "logstash/webserver"
 require "logstash/config/source_loader"
 require "logstash/pipeline_action"
 require "logstash/state_resolver"
+require "logstash/pipelines_registry"
 require "stud/trap"
 require "uri"
 require "socket"
@@ -19,7 +20,7 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :name, :settings, :webserver, :dispatcher, :ephemeral_id, :pipelines, :pipeline_bus
+  attr_reader :metric, :name, :settings, :dispatcher, :ephemeral_id, :pipeline_bus
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -36,11 +37,12 @@ class LogStash::Agent
     # Mutex to synchonize in the exclusive method
     # Initial usage for the Ruby pipeline initialization which is not thread safe
     @exclusive_lock = Mutex.new
+    @webserver_control_lock = Mutex.new
 
     # Special bus object for inter-pipelines communications. Used by the `pipeline` input/output
     @pipeline_bus = org.logstash.plugins.pipeline.PipelineBus.new
 
-    @pipelines = java.util.concurrent.ConcurrentHashMap.new();
+    @pipelines_registry = LogStash::PipelinesRegistry.new
 
     @name = setting("node.name")
     @http_host = setting("http.host")
@@ -118,14 +120,17 @@ class LogStash::Agent
         converge_state_and_update unless stopped?
       end
     else
-      return 1 if clean_state?
+      # exit with error status if the initial converge_state_and_update did not create any pipeline
+      return 1 if @pipelines_registry.empty?
 
       while !Stud.stop?
-        if clean_state? || running_user_defined_pipelines?
-          sleep(0.5)
-        else
-          break
-        end
+        # exit if all pipelines are terminated and none are reloading
+        break if no_pipeline?
+
+        # exit if there are no user defined pipelines (not system pipeline) and none are reloading
+        break if !running_user_defined_pipelines?
+
+        sleep(0.5)
       end
     end
 
@@ -139,11 +144,11 @@ class LogStash::Agent
   end
 
   def running?
-    @running.value
+    @running.true?
   end
 
   def stopped?
-    !@running.value
+    @running.false?
   end
 
   def converge_state_and_update
@@ -237,43 +242,48 @@ class LogStash::Agent
     @id_path ||= ::File.join(settings.get("path.data"), "uuid")
   end
 
+  #
+  # Backward compatibility proxies to the PipelineRegistry
+  #
+
   def get_pipeline(pipeline_id)
-    pipelines.get(pipeline_id)
+    @pipelines_registry.get_pipeline(pipeline_id)
   end
 
   def pipelines_count
-    pipelines.size
+    @pipelines_registry.size
   end
 
   def running_pipelines
-    pipelines.select {|id,pipeline| running_pipeline?(id) }
-  end
+    @pipelines_registry.running_pipelines
+   end
 
   def non_running_pipelines
-    pipelines.select {|id,pipeline| !running_pipeline?(id) }
+    @pipelines_registry.non_running_pipelines
   end
 
   def running_pipelines?
-    running_pipelines_count > 0
+    @pipelines_registry.running_pipelines.any?
   end
 
   def running_pipelines_count
-    running_pipelines.size
+    @pipelines_registry.running_pipelines.size
   end
 
   def running_user_defined_pipelines?
-    !running_user_defined_pipelines.empty?
+    @pipelines_registry.running_user_defined_pipelines.any?
   end
 
   def running_user_defined_pipelines
-    pipelines.select {|id, pipeline| running_pipeline?(id) && !pipeline.system? }
+    @pipelines_registry.running_user_defined_pipelines
   end
 
-  def with_running_user_defined_pipelines
-    yield running_user_defined_pipelines
+  def no_pipeline?
+    @pipelines_registry.running_pipelines.empty?
   end
 
   private
+
   def transition_to_stopped
     @running.make_false
   end
@@ -298,7 +308,7 @@ class LogStash::Agent
     converge_result = LogStash::ConvergeResult.new(pipeline_actions.size)
 
     pipeline_actions.map do |action|
-      Thread.new do
+      Thread.new(action, converge_result) do |action, converge_result|
         java.lang.Thread.currentThread().setName("Converge #{action}");
         # We execute every task we need to converge the current state of pipelines
         # for every task we will record the action result, that will help us
@@ -314,34 +324,35 @@ class LogStash::Agent
         # that we currently have.
         begin
           logger.debug("Executing action", :action => action)
-          action_result = action.execute(self, pipelines)
+          action_result = action.execute(self, @pipelines_registry)
           converge_result.add(action, action_result)
 
           unless action_result.successful?
-            logger.error("Failed to execute action", :id => action.pipeline_id,
-                         :action_type => action_result.class, :message => action_result.message,
-                         :backtrace => action_result.backtrace)
+            logger.error("Failed to execute action",
+              :id => action.pipeline_id,
+              :action_type => action_result.class,
+              :message => action_result.message,
+              :backtrace => action_result.backtrace
+            )
           end
-        rescue SystemExit => e
-          converge_result.add(action, e)
-        rescue Exception => e
+        rescue SystemExit, Exception => e
           logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
           converge_result.add(action, e)
         end
       end
     end.each(&:join)
 
-    if logger.trace?
-      logger.trace("Converge results", :success => converge_result.success?,
-                   :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
-                   :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" })
-    end
+    logger.trace? && logger.trace("Converge results",
+      :success => converge_result.success?,
+      :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
+      :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" }
+    )
 
     converge_result
   end
 
   def resolve_actions(pipeline_configs)
-    @state_resolver.resolve(@pipelines, pipeline_configs)
+    @state_resolver.resolve(@pipelines_registry, pipeline_configs)
   end
 
   def dispatch_events(converge_results)
@@ -358,20 +369,24 @@ class LogStash::Agent
   end
 
   def start_webserver
-    options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
-    @webserver = LogStash::WebServer.new(@logger, self, options)
-    @webserver_thread = Thread.new(@webserver) do |webserver|
-      LogStash::Util.set_thread_name("Api Webserver")
-      webserver.run
+    @webserver_control_lock.synchronize do
+      options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
+      @webserver = LogStash::WebServer.new(@logger, self, options)
+      @webserver_thread = Thread.new(@webserver) do |webserver|
+        LogStash::Util.set_thread_name("Api Webserver")
+        webserver.run
+      end
     end
   end
 
   def stop_webserver
-    if @webserver
-      @webserver.stop
-      if @webserver_thread.join(5).nil?
-        @webserver_thread.kill
-        @webserver_thread.join
+    @webserver_control_lock.synchronize do
+      if @webserver
+        @webserver.stop
+        if @webserver_thread.join(5).nil?
+          @webserver_thread.kill
+          @webserver_thread.join
+        end
       end
     end
   end
@@ -399,7 +414,7 @@ class LogStash::Agent
   end
 
   def shutdown_pipelines
-    logger.debug("Shutting down all pipelines", :pipelines_count => pipelines_count)
+    logger.debug("Shutting down all pipelines", :pipelines_count => running_pipelines_count)
 
     # In this context I could just call shutdown, but I've decided to
     # use the stop action implementation for that so we have the same code.
@@ -408,16 +423,6 @@ class LogStash::Agent
     converge_state(pipeline_actions)
   end
 
-  def running_pipeline?(pipeline_id)
-    pipeline = get_pipeline(pipeline_id)
-    return false unless pipeline
-    thread = pipeline.thread
-    thread.is_a?(Thread) && thread.alive?
-  end
-
-  def clean_state?
-    pipelines.empty?
-  end
 
   def setting(key)
     @settings.get(key)

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -10,13 +10,11 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
-      unless pipelines.nil?
-        pipelines.each {|_, pipeline|
-          unless pipeline.nil?
-            pipeline.collect_dlq_stats
-          end
-        }
+      pipelines = @agent.running_user_defined_pipelines
+      pipelines.each do |_, pipeline|
+        unless pipeline.nil?
+          pipeline.collect_dlq_stats
+        end
       end
     end
   end

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,14 +11,12 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
-      unless pipelines.nil?
-        pipelines.each {|_, pipeline|
-          unless pipeline.nil?
-            pipeline.collect_stats
-          end
-        }
+      pipelines = @agent.running_user_defined_pipelines
+      pipelines.each do |_, pipeline|
+        unless pipeline.nil?
+          pipeline.collect_stats
+        end
       end
     end
   end
-end; end; end
+end end end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -11,12 +11,12 @@ module LogStash module Instrument
   class PeriodicPollers
     attr_reader :metric
 
-    def initialize(metric, queue_type, pipelines)
+    def initialize(metric, queue_type, agent)
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
-                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
+                           PeriodicPoller::DeadLetterQueue.new(metric, agent)]
     end
 
     def start

data/lib/logstash/java_pipeline.rb

@@ -41,9 +41,23 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_execution signals that the pipeline thread has finished its execution
+    # regardless of any exceptions; it will always be true when the thread completes
     @finished_execution = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_run signals that the run methods called in the pipeline thread was completed
+    # without errors and it will NOT be set if the run method exits from an exception; this
+    # is by design and necessary for the wait_until_started semantic
+    @finished_run = Concurrent::AtomicBoolean.new(false)
+
+    @thread = nil
   end # def initialize
 
+  def finished_execution?
+    @finished_execution.true?
+  end
+
   def ready?
     @ready.value
   end
@@ -85,15 +99,18 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @logger.debug("Starting pipeline", default_logging_keys)
 
     @finished_execution.make_false
+    @finished_run.make_false
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_execution.make_true
+        @finished_run.make_true
       rescue => e
         close
         logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+      ensure
+        @finished_execution.make_true
       end
     end
 
@@ -108,15 +125,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def wait_until_started
     while true do
-      # This should be changed with an appropriate FSM
-      # It's an edge case, if we have a pipeline with
-      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
-      # because the execution of the thread was successful and complete
-      if @finished_execution.true?
+      if @finished_run.true?
+        # it completed run without exception
         return true
       elsif thread.nil? || !thread.alive?
+        # some exception occurred and the thread is dead
         return false
       elsif running?
+        # fully initialized and running
         return true
       else
         sleep 0.01

data/lib/logstash/pipeline.rb

@@ -107,8 +107,23 @@ module LogStash; class Pipeline < BasePipeline
     @flushing = Concurrent::AtomicReference.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
     @worker_shutdown = java.util.concurrent.atomic.AtomicBoolean.new(false)
+
+    # @finished_execution signals that the pipeline thread has finished its execution
+    # regardless of any exceptions; it will always be true when the thread completes
+    @finished_execution = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_run signals that the run methods called in the pipeline thread was completed
+    # without errors and it will NOT be set if the run method exits from an exception; this
+    # is by design and necessary for the wait_until_started semantic
+    @finished_run = Concurrent::AtomicBoolean.new(false)
+
+    @thread = nil
   end # def initialize
 
+  def finished_execution?
+    @finished_execution.true?
+  end
+
   def ready?
     @ready.value
   end
@@ -152,16 +167,19 @@ module LogStash; class Pipeline < BasePipeline
       "pipeline.batch.size" => settings.get("pipeline.batch.size"),
       "pipeline.batch.delay" => settings.get("pipeline.batch.delay")))
 
-    @finished_execution = Concurrent::AtomicBoolean.new(false)
+    @finished_execution.make_false
+    @finished_run.make_false
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_execution.make_true
+        @finished_run.make_true
       rescue => e
         close
         @logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+      ensure
+        @finished_execution.make_true
       end
     end
 
@@ -176,15 +194,14 @@ module LogStash; class Pipeline < BasePipeline
 
   def wait_until_started
     while true do
-      # This should be changed with an appropriate FSM
-      # It's an edge case, if we have a pipeline with
-      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
-      # because the execution of the thread was successful and complete
-      if @finished_execution.true?
+      if @finished_run.true?
+        # it completed run without exception
         return true
-      elsif !thread.alive?
+      elsif thread.nil? || !thread.alive?
+        # some exception occured and the thread is dead
         return false
       elsif running?
+        # fully initialized and running
         return true
       else
         sleep 0.01

data/lib/logstash/pipeline_action/base.rb

@@ -12,7 +12,7 @@ module LogStash module PipelineAction
     end
     alias_method :to_s, :inspect
 
-    def execute(agent, pipelines)
+    def execute(agent, pipelines_registry)
       raise "`#execute` Not implemented!"
     end