logstash-core 7.0.0.alpha2-java → 7.0.0.beta1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e824ba00a89915dc5806c980c92edc6fcdb117f7c92eba1ce76bdee8a6730b6
-  data.tar.gz: 9bc5a09b8af714e11b2ac6390d05698a85d5523d0b2eff30ef3d197f2fad8f5c
+  metadata.gz: 650eff422fce2ce58c826a22c759238b5cb14bb407232b3cb0ffe2119635df29
+  data.tar.gz: 44a83cccdc93c56fcc4952746675702b7593e0f494b167efcfc4fe002566dfb6
 SHA512:
-  metadata.gz: 31f101ef4e58029d9353da647dd5cef4c22d96b41ab905be18fe469ad6028cc67cfaa863a809e352f8cb887cf728f517500040a730e1c0d5664945f2c7977e7b
-  data.tar.gz: 99f257a1da0b6c3a30aba429edffca4b0981ab7700a95c41f8ba90701415c98153f5ed927f4b50e2da3c50783bddf8ba52eaf82d6ce2337de97bc26c3dda9a5d
+  metadata.gz: 94c5e143d649f75006330dfc9f77d3551d011aabef1e18017cb0eff8f96ae16d6690b5487acbabec3e9adae5eb0d3a4cb8bfb974a174cbb8d4073f659ab821cf
+  data.tar.gz: 1500544458de19cb215a0f45e6b83a00b4d8ef0b05f951536a080f2fad359502eaf847792b20719e1781727caddc11349d7c504c82d0cfe47aa5ff8e158f66e8

data/lib/logstash/agent.rb

@@ -8,6 +8,7 @@ require "logstash/webserver"
 require "logstash/config/source_loader"
 require "logstash/pipeline_action"
 require "logstash/state_resolver"
+require "logstash/pipelines_registry"
 require "stud/trap"
 require "uri"
 require "socket"
@@ -19,7 +20,7 @@ class LogStash::Agent
   include LogStash::Util::Loggable
   STARTED_AT = Time.now.freeze
 
-  attr_reader :metric, :name, :settings, :webserver, :dispatcher, :ephemeral_id, :pipelines, :pipeline_bus
+  attr_reader :metric, :name, :settings, :dispatcher, :ephemeral_id, :pipeline_bus
   attr_accessor :logger
 
   # initialize method for LogStash::Agent
@@ -36,11 +37,12 @@ class LogStash::Agent
     # Mutex to synchonize in the exclusive method
     # Initial usage for the Ruby pipeline initialization which is not thread safe
     @exclusive_lock = Mutex.new
+    @webserver_control_lock = Mutex.new
 
     # Special bus object for inter-pipelines communications. Used by the `pipeline` input/output
     @pipeline_bus = org.logstash.plugins.pipeline.PipelineBus.new
 
-    @pipelines = java.util.concurrent.ConcurrentHashMap.new();
+    @pipelines_registry = LogStash::PipelinesRegistry.new
 
     @name = setting("node.name")
     @http_host = setting("http.host")
@@ -114,14 +116,17 @@ class LogStash::Agent
         converge_state_and_update unless stopped?
       end
     else
-      return 1 if clean_state?
+      # exit with error status if the initial converge_state_and_update did not create any pipeline
+      return 1 if @pipelines_registry.empty?
 
       while !Stud.stop?
-        if clean_state? || running_user_defined_pipelines?
-          sleep(0.5)
-        else
-          break
-        end
+        # exit if all pipelines are terminated and none are reloading
+        break if no_pipeline?
+
+        # exit if there are no user defined pipelines (not system pipeline) and none are reloading
+        break if !running_user_defined_pipelines?
+
+        sleep(0.5)
       end
     end
 
@@ -135,11 +140,11 @@ class LogStash::Agent
   end
 
   def running?
-    @running.value
+    @running.true?
   end
 
   def stopped?
-    !@running.value
+    @running.false?
   end
 
   def converge_state_and_update
@@ -178,7 +183,7 @@ class LogStash::Agent
 
   # Calculate the Logstash uptime in milliseconds
   #
-  # @return [Fixnum] Uptime in milliseconds
+  # @return [Integer] Uptime in milliseconds
   def uptime
     ((Time.now.to_f - STARTED_AT.to_f) * 1000.0).to_i
   end
@@ -233,43 +238,48 @@ class LogStash::Agent
     @id_path ||= ::File.join(settings.get("path.data"), "uuid")
   end
 
+  #
+  # Backward compatibility proxies to the PipelineRegistry
+  #
+
   def get_pipeline(pipeline_id)
-    pipelines.get(pipeline_id)
+    @pipelines_registry.get_pipeline(pipeline_id)
   end
 
   def pipelines_count
-    pipelines.size
+    @pipelines_registry.size
   end
 
   def running_pipelines
-    pipelines.select {|id,pipeline| running_pipeline?(id) }
-  end
+    @pipelines_registry.running_pipelines
+   end
 
   def non_running_pipelines
-    pipelines.select {|id,pipeline| !running_pipeline?(id) }
+    @pipelines_registry.non_running_pipelines
   end
 
   def running_pipelines?
-    running_pipelines_count > 0
+    @pipelines_registry.running_pipelines.any?
   end
 
   def running_pipelines_count
-    running_pipelines.size
+    @pipelines_registry.running_pipelines.size
   end
 
   def running_user_defined_pipelines?
-    !running_user_defined_pipelines.empty?
+    @pipelines_registry.running_user_defined_pipelines.any?
   end
 
   def running_user_defined_pipelines
-    pipelines.select {|id, pipeline| running_pipeline?(id) && !pipeline.system? }
+    @pipelines_registry.running_user_defined_pipelines
   end
 
-  def with_running_user_defined_pipelines
-    yield running_user_defined_pipelines
+  def no_pipeline?
+    @pipelines_registry.running_pipelines.empty?
   end
 
   private
+
   def transition_to_stopped
     @running.make_false
   end
@@ -294,7 +304,7 @@ class LogStash::Agent
     converge_result = LogStash::ConvergeResult.new(pipeline_actions.size)
 
     pipeline_actions.map do |action|
-      Thread.new do
+      Thread.new(action, converge_result) do |action, converge_result|
         java.lang.Thread.currentThread().setName("Converge #{action}");
         # We execute every task we need to converge the current state of pipelines
         # for every task we will record the action result, that will help us
@@ -310,34 +320,35 @@ class LogStash::Agent
         # that we currently have.
         begin
           logger.debug("Executing action", :action => action)
-          action_result = action.execute(self, pipelines)
+          action_result = action.execute(self, @pipelines_registry)
           converge_result.add(action, action_result)
 
           unless action_result.successful?
-            logger.error("Failed to execute action", :id => action.pipeline_id,
-                         :action_type => action_result.class, :message => action_result.message,
-                         :backtrace => action_result.backtrace)
+            logger.error("Failed to execute action",
+              :id => action.pipeline_id,
+              :action_type => action_result.class,
+              :message => action_result.message,
+              :backtrace => action_result.backtrace
+            )
           end
-        rescue SystemExit => e
-          converge_result.add(action, e)
-        rescue Exception => e
+        rescue SystemExit, Exception => e
           logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
           converge_result.add(action, e)
         end
       end
     end.each(&:join)
 
-    if logger.trace?
-      logger.trace("Converge results", :success => converge_result.success?,
-                   :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
-                   :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" })
-    end
+    logger.trace? && logger.trace("Converge results",
+      :success => converge_result.success?,
+      :failed_actions => converge_result.failed_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}, message: #{r.message}" },
+      :successful_actions => converge_result.successful_actions.collect { |a, r| "id: #{a.pipeline_id}, action_type: #{a.class}" }
+    )
 
     converge_result
   end
 
   def resolve_actions(pipeline_configs)
-    @state_resolver.resolve(@pipelines, pipeline_configs)
+    @state_resolver.resolve(@pipelines_registry, pipeline_configs)
   end
 
   def dispatch_events(converge_results)
@@ -354,20 +365,24 @@ class LogStash::Agent
   end
 
   def start_webserver
-    options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
-    @webserver = LogStash::WebServer.new(@logger, self, options)
-    @webserver_thread = Thread.new(@webserver) do |webserver|
-      LogStash::Util.set_thread_name("Api Webserver")
-      webserver.run
+    @webserver_control_lock.synchronize do
+      options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
+      @webserver = LogStash::WebServer.new(@logger, self, options)
+      @webserver_thread = Thread.new(@webserver) do |webserver|
+        LogStash::Util.set_thread_name("Api Webserver")
+        webserver.run
+      end
     end
   end
 
   def stop_webserver
-    if @webserver
-      @webserver.stop
-      if @webserver_thread.join(5).nil?
-        @webserver_thread.kill
-        @webserver_thread.join
+    @webserver_control_lock.synchronize do
+      if @webserver
+        @webserver.stop
+        if @webserver_thread.join(5).nil?
+          @webserver_thread.kill
+          @webserver_thread.join
+        end
       end
     end
   end
@@ -395,7 +410,7 @@ class LogStash::Agent
   end
 
   def shutdown_pipelines
-    logger.debug("Shutting down all pipelines", :pipelines_count => pipelines_count)
+    logger.debug("Shutting down all pipelines", :pipelines_count => running_pipelines_count)
 
     # In this context I could just call shutdown, but I've decided to
     # use the stop action implementation for that so we have the same code.
@@ -404,16 +419,6 @@ class LogStash::Agent
     converge_state(pipeline_actions)
   end
 
-  def running_pipeline?(pipeline_id)
-    pipeline = get_pipeline(pipeline_id)
-    return false unless pipeline
-    thread = pipeline.thread
-    thread.is_a?(Thread) && thread.alive?
-  end
-
-  def clean_state?
-    pipelines.empty?
-  end
 
   def setting(key)
     @settings.get(key)

data/lib/logstash/compiler/lscl.rb

@@ -317,10 +317,9 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
 
     def precedence(op)
       #  Believe this is right for logstash?
-      case op
-      when AND_METHOD
+      if op == AND_METHOD
         2
-      when OR_METHOD
+      elsif op == OR_METHOD
         1
       else
         raise ArgumentError, "Unexpected operator #{op}"

data/lib/logstash/config/config_ast.rb

@@ -7,28 +7,60 @@ require "logstash/compiler/treetop_monkeypatches"
 module LogStash; module Config; module AST
   PROCESS_ESCAPE_SEQUENCES = :process_escape_sequences
 
-  def self.deferred_conditionals=(val)
-    @deferred_conditionals = val
-  end
+  class << self
+    # @api private
+    MUTEX = Mutex.new
 
-  def self.deferred_conditionals
-    @deferred_conditionals
-  end
+    # Executes the given block with exclusive access to the AST global variables
+    #
+    # @yieldreturn [Object]: the object that is returned from the block is returned by this method
+    #
+    # @return [Object]
+    def exclusive
+      MUTEX.synchronize { yield }
+    end
 
-  def self.deferred_conditionals_index
-    @deferred_conditionals_index
-  end
+    def deferred_conditionals=(val)
+      ensure_exclusive!
+      @deferred_conditionals = val
+    end
 
-  def self.deferred_conditionals_index=(val)
-    @deferred_conditionals_index = val
-  end
+    def deferred_conditionals
+      ensure_exclusive!
+      @deferred_conditionals
+    end
 
-  def self.plugin_instance_index
-    @plugin_instance_index
-  end
+    def deferred_conditionals_index
+      ensure_exclusive!
+      @deferred_conditionals_index
+    end
+
+    def deferred_conditionals_index=(val)
+      ensure_exclusive!
+      @deferred_conditionals_index = val
+    end
 
-  def self.plugin_instance_index=(val)
-    @plugin_instance_index = val
+    def plugin_instance_index
+      ensure_exclusive!
+      @plugin_instance_index
+    end
+
+    def plugin_instance_index=(val)
+      ensure_exclusive!
+      @plugin_instance_index = val
+    end
+
+    private
+
+    # Raises a descriptive error if the thread in which it is invoked does
+    # not have exclusive access.
+    #
+    # @raise [RuntimeError]
+    def ensure_exclusive!
+      return if MUTEX.owned?
+
+      raise "Illegal access without exclusive lock at `#{caller[1]}`"
+    end
   end
 
   class Node < Treetop::Runtime::SyntaxNode
@@ -46,6 +78,15 @@ module LogStash; module Config; module AST
 
 
     def compile
+      LogStash::Config::AST.exclusive { do_compile }
+    end
+
+    private
+
+    # NON-threadsafe method compiles an AST into executable Ruby code.
+    # @see Config#compile, which is a threadsafe wrapper around this method.
+    # @api private
+    def do_compile
       LogStash::Config::AST.deferred_conditionals = []
       LogStash::Config::AST.deferred_conditionals_index = 0
       LogStash::Config::AST.plugin_instance_index = 0
@@ -491,6 +532,7 @@ module LogStash; module Config; module AST
 end; end; end
 
 
+
 # Monkeypatch Treetop::Runtime::SyntaxNode's inspect method to skip
 # any Whitespace or SyntaxNodes with no children.
 class Treetop::Runtime::SyntaxNode

data/lib/logstash/environment.rb

@@ -38,7 +38,6 @@ module LogStash
             Setting::String.new("pipeline.id", "main"),
            Setting::Boolean.new("pipeline.system", false),
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
-   Setting::PositiveInteger.new("pipeline.output.workers", 1),
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
            Setting::Numeric.new("pipeline.batch.delay", 50), # in milliseconds
            Setting::Boolean.new("pipeline.unsafe_shutdown", false),
@@ -62,6 +61,7 @@ module LogStash
             Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
             Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::Boolean.new("queue.checkpoint.retry", false),
             Setting::Boolean.new("dead_letter_queue.enable", false),
             Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
             Setting::TimeValue.new("slowlog.threshold.warn", "-1"),

data/lib/logstash/instrument/metric_store.rb

@@ -302,7 +302,7 @@ module LogStash module Instrument
     #
     # @param [Concurrent::Map] Map to search for the key
     # @param [Array] List of path to create
-    # @param [Fixnum] Which part from the list to create
+    # @param [Integer] Which part from the list to create
     #
     def fetch_or_store_namespace_recursively(map, namespaces_path, idx = 0)
       current = namespaces_path[idx]

data/lib/logstash/instrument/periodic_poller/dlq.rb

@@ -10,13 +10,11 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
-      unless pipelines.nil?
-        pipelines.each {|_, pipeline|
-          unless pipeline.nil?
-            pipeline.collect_dlq_stats
-          end
-        }
+      pipelines = @agent.running_user_defined_pipelines
+      pipelines.each do |_, pipeline|
+        unless pipeline.nil?
+          pipeline.collect_dlq_stats
+        end
       end
     end
   end

data/lib/logstash/instrument/periodic_poller/pq.rb

@@ -11,14 +11,12 @@ module LogStash module Instrument module PeriodicPoller
     end
 
     def collect
-      pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
-      unless pipelines.nil?
-        pipelines.each {|_, pipeline|
-          unless pipeline.nil?
-            pipeline.collect_stats
-          end
-        }
+      pipelines = @agent.running_user_defined_pipelines
+      pipelines.each do |_, pipeline|
+        unless pipeline.nil?
+          pipeline.collect_stats
+        end
       end
     end
   end
-end; end; end
+end end end

data/lib/logstash/instrument/periodic_pollers.rb

@@ -11,12 +11,12 @@ module LogStash module Instrument
   class PeriodicPollers
     attr_reader :metric
 
-    def initialize(metric, queue_type, pipelines)
+    def initialize(metric, queue_type, agent)
       @metric = metric
       @periodic_pollers = [PeriodicPoller::Os.new(metric),
                            PeriodicPoller::JVM.new(metric),
-                           PeriodicPoller::PersistentQueue.new(metric, queue_type, pipelines),
-                           PeriodicPoller::DeadLetterQueue.new(metric, pipelines)]
+                           PeriodicPoller::PersistentQueue.new(metric, queue_type, agent),
+                           PeriodicPoller::DeadLetterQueue.new(metric, agent)]
     end
 
     def start

data/lib/logstash/java_pipeline.rb

@@ -25,8 +25,6 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     @worker_threads = []
 
-    @java_inputs_controller = org.logstash.execution.InputsController.new(lir_execution.javaInputs)
-
     @drain_queue =  settings.get_value("queue.drain") || settings.get("queue.type") == "memory"
 
     @events_filtered = java.util.concurrent.atomic.LongAdder.new
@@ -40,9 +38,23 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_execution signals that the pipeline thread has finished its execution
+    # regardless of any exceptions; it will always be true when the thread completes
     @finished_execution = Concurrent::AtomicBoolean.new(false)
+
+    # @finished_run signals that the run methods called in the pipeline thread was completed
+    # without errors and it will NOT be set if the run method exits from an exception; this
+    # is by design and necessary for the wait_until_started semantic
+    @finished_run = Concurrent::AtomicBoolean.new(false)
+
+    @thread = nil
   end # def initialize
 
+  def finished_execution?
+    @finished_execution.true?
+  end
+
   def ready?
     @ready.value
   end
@@ -84,15 +96,18 @@ module LogStash; class JavaPipeline < JavaBasePipeline
     @logger.debug("Starting pipeline", default_logging_keys)
 
     @finished_execution.make_false
+    @finished_run.make_false
 
     @thread = Thread.new do
       begin
         LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
         run
-        @finished_execution.make_true
+        @finished_run.make_true
       rescue => e
         close
         logger.error("Pipeline aborted due to error", default_logging_keys(:exception => e, :backtrace => e.backtrace))
+      ensure
+        @finished_execution.make_true
       end
     end
 
@@ -107,15 +122,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
   def wait_until_started
     while true do
-      # This should be changed with an appropriate FSM
-      # It's an edge case, if we have a pipeline with
-      # a generator { count => 1 } its possible that `Thread#alive?` doesn't return true
-      # because the execution of the thread was successful and complete
-      if @finished_execution.true?
+      if @finished_run.true?
+        # it completed run without exception
         return true
       elsif thread.nil? || !thread.alive?
+        # some exception occurred and the thread is dead
         return false
       elsif running?
+        # fully initialized and running
         return true
       else
         sleep 0.01
@@ -217,11 +231,11 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
       pipeline_workers.times do |t|
         thread = Thread.new do
+          Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
           org.logstash.execution.WorkerLoop.new(
               lir_execution, filter_queue_client, @events_filtered, @events_consumed,
               @flushRequested, @flushing, @shutdownRequested, @drain_queue).run
         end
-        Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
         @worker_threads << thread
       end
 
@@ -242,8 +256,13 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def wait_inputs
-    @input_threads.each(&:join)
-    @java_inputs_controller.awaitStop
+    @input_threads.each do |thread|
+      if thread.class == Java::JavaObject
+        thread.to_java.join
+      else
+        thread.join
+      end
+    end
   end
 
   def start_inputs
@@ -262,11 +281,14 @@ module LogStash; class JavaPipeline < JavaBasePipeline
 
     # then after all input plugins are successfully registered, start them
     inputs.each { |input| start_input(input) }
-    @java_inputs_controller.startInputs(self)
   end
 
   def start_input(plugin)
-    @input_threads << Thread.new { inputworker(plugin) }
+    if plugin.class == LogStash::JavaInputDelegator
+      @input_threads << plugin.start
+    else
+      @input_threads << Thread.new { inputworker(plugin) }
+    end
   end
 
   def inputworker(plugin)
@@ -328,7 +350,6 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   def stop_inputs
     @logger.debug("Closing inputs", default_logging_keys)
     inputs.each(&:do_stop)
-    @java_inputs_controller.stopInputs
     @logger.debug("Closed inputs", default_logging_keys)
   end
 
@@ -377,7 +398,7 @@ module LogStash; class JavaPipeline < JavaBasePipeline
   end
 
   def plugin_threads_info
-    input_threads = @input_threads.select {|t| t.alive? }
+    input_threads = @input_threads.select {|t| t.class == Thread && t.alive? }
     worker_threads = @worker_threads.select {|t| t.alive? }
     (input_threads + worker_threads).map {|t| Util.thread_info(t) }
   end

data/lib/logstash/patches/resolv.rb

@@ -10,7 +10,6 @@ require "resolv"
 
 # make sure we abort if a known correct JRuby version is installed 
 # to avoid having an unnecessary legacy patch being applied in the future.
-raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("9.1.16.0")
 
 # The code below is copied from JRuby 9.1.16.0 resolv.rb:
 # https://github.com/jruby/jruby/blob/9.1.16.0/lib/ruby/stdlib/resolv.rb#L775-L784
@@ -18,23 +17,3 @@ raise("Unnecessary patch on resolv.rb for JRuby version 9.1.16+") if Gem::Versio
 # JRuby is Copyright (c) 2007-2017 The JRuby project, and is released
 # under a tri EPL/GPL/LGPL license.
 # Full license available at https://github.com/jruby/jruby/blob/9.1.16.0/COPYING
-
-class Resolv
-  class DNS
-    class Requester
-      class UnconnectedUDP
-        def sender(msg, data, host, port=Port)
-          sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"]
-          return nil if !sock
-          service = [IPAddr.new(host), port]
-          id = DNS.allocate_request_id(service[0], service[1])
-          request = msg.encode
-          request[0,2] = [id].pack('n')
-          return @senders[[service, id]] =
-              Sender.new(request, data, sock, host, port)
-        end
-      end
-    end
-  end
-end
-