logstash-core 2.0.0.beta1-java → 2.0.0.beta2-java

data/lib/logstash/patches/stronger_openssl_defaults.rb

@@ -61,7 +61,7 @@ class OpenSSL::SSL::SSLContext
   # For more details see: https://github.com/elastic/logstash/issues/3657
   remove_const(:DEFAULT_PARAMS) if const_defined?(:DEFAULT_PARAMS)
   DEFAULT_PARAMS = {
-    :ssl_version => "SSLv23",
+    :ssl_version => "TLS",
     :ciphers => MOZILLA_INTERMEDIATE_CIPHERS,
     :options => __default_options # Not a constant because it's computed at start-time.
   }

data/lib/logstash/pipeline.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
-require "thread" #
+require "thread"
 require "stud/interval"
+require "concurrent"
 require "logstash/namespace"
 require "logstash/errors"
 require "logstash/event"
@@ -9,17 +10,24 @@ require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
 require "logstash/util/reporter"
+require "logstash/config/cpu_core_strategy"
+require "logstash/util/defaults_printer"
 
 class LogStash::Pipeline
+  attr_reader :inputs, :filters, :outputs, :input_to_filter, :filter_to_output
 
   def initialize(configstr)
     @logger = Cabin::Channel.get(LogStash)
+
+    @inputs = nil
+    @filters = nil
+    @outputs = nil
+
     grammar = LogStashConfigParser.new
     @config = grammar.parse(configstr)
     if @config.nil?
       raise LogStash::ConfigurationError, grammar.failure_reason
     end
-
     # This will compile the config to ruby and evaluate the resulting code.
     # The code will initialize all the plugins and define the
     # filter and output methods.
@@ -34,36 +42,27 @@ class LogStash::Pipeline
     end
 
     @input_to_filter = SizedQueue.new(20)
+    # if no filters, pipe inputs directly to outputs
+    @filter_to_output = filters? ? SizedQueue.new(20) : @input_to_filter
 
-    # If no filters, pipe inputs directly to outputs
-    if !filters?
-      @filter_to_output = @input_to_filter
-    else
-      @filter_to_output = SizedQueue.new(20)
-    end
     @settings = {
-      "filter-workers" => 1,
+      "filter-workers" => LogStash::Config::CpuCoreStrategy.fifty_percent
     }
 
-    @run_mutex = Mutex.new
-    @ready = false
-    @started = false
+    # @ready requires thread safety since it is typically polled from outside the pipeline thread
+    @ready = Concurrent::AtomicBoolean.new(false)
     @input_threads = []
   end # def initialize
 
   def ready?
-    @run_mutex.synchronize{@ready}
-  end
-
-  def started?
-    @run_mutex.synchronize{@started}
+    @ready.value
   end
 
   def configure(setting, value)
-    if setting == "filter-workers"
+    if setting == "filter-workers" && value > 1
       # Abort if we have any filters that aren't threadsafe
-      if value > 1 && @filters.any? { |f| !f.threadsafe? }
-        plugins = @filters.select { |f| !f.threadsafe? }.collect { |f| f.class.config_name }
+      plugins = @filters.select { |f| !f.threadsafe? }.collect { |f| f.class.config_name }
+      if !plugins.size.zero?
         raise LogStash::ConfigurationError, "Cannot use more than 1 filter worker because the following plugins don't work with more than one worker: #{plugins.join(", ")}"
       end
     end
@@ -75,14 +74,17 @@ class LogStash::Pipeline
   end
 
   def run
-    @run_mutex.synchronize{@started = true}
+    @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
 
-    # synchronize @input_threads between run and shutdown
-    @run_mutex.synchronize{start_inputs}
-    start_filters if filters?
-    start_outputs
-
-    @run_mutex.synchronize{@ready = true}
+    begin
+      start_inputs
+      start_filters if filters?
+      start_outputs
+    ensure
+      # it is important to garantee @ready to be true after the startup sequence has been completed
+      # to potentially unblock the shutdown method which may be waiting on @ready to proceed
+      @ready.make_true
+    end
 
     @logger.info("Pipeline started")
     @logger.terminal("Logstash startup completed")
@@ -107,12 +109,6 @@ class LogStash::Pipeline
 
   def wait_inputs
     @input_threads.each(&:join)
-  rescue Interrupt
-    # rbx does weird things during do SIGINT that I haven't debugged
-    # so we catch Interrupt here and signal a shutdown. For some reason the
-    # signal handler isn't invoked it seems? I dunno, haven't looked much into
-    # it.
-    shutdown
   end
 
   def shutdown_filters
@@ -153,10 +149,17 @@ class LogStash::Pipeline
 
   def start_filters
     @filters.each(&:register)
-    @filter_threads = @settings["filter-workers"].times.collect do
+    to_start = @settings["filter-workers"]
+    @filter_threads = to_start.times.collect do
       Thread.new { filterworker }
     end
-
+    actually_started = @filter_threads.select(&:alive?).size
+    msg = "Worker threads expected: #{to_start}, worker threads started: #{actually_started}"
+    if actually_started < to_start
+      @logger.warn(msg)
+    else
+      @logger.info(msg)
+    end
     @flusher_thread = Thread.new { Stud.interval(5) { @input_to_filter.push(LogStash::FLUSH) } }
   end
 
@@ -175,9 +178,16 @@ class LogStash::Pipeline
     LogStash::Util::set_thread_name("<#{plugin.class.config_name}")
     begin
       plugin.run(@input_to_filter)
-    rescue LogStash::ShutdownSignal
-      # ignore and quit
     rescue => e
+      # if plugin is stopping, ignore uncatched exceptions and exit worker
+      if plugin.stop?
+        @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
+                      :plugin => plugin.class.config_name, :exception => e,
+                      :backtrace => e.backtrace)
+        return
+      end
+
+      # otherwise, report error and restart
       if @logger.debug?
         @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
                              :plugin => plugin.inspect, :error => e.to_s,
@@ -187,28 +197,18 @@ class LogStash::Pipeline
         @logger.error(I18n.t("logstash.pipeline.worker-error",
                              :plugin => plugin.inspect, :error => e))
       end
-      puts e.backtrace if @logger.debug?
-      # input teardown must be synchronized since is can be called concurrently by
-      # the input worker thread and from the pipeline thread shutdown method.
-      # this means that input teardown methods must support multiple calls.
-      @run_mutex.synchronize{plugin.teardown}
-      sleep 1
-      retry
-    end
-  ensure
-    begin
-      # input teardown must be synchronized since is can be called concurrently by
-      # the input worker thread and from the pipeline thread shutdown method.
-      # this means that input teardown methods must support multiple calls.
-      @run_mutex.synchronize{plugin.teardown}
-    rescue LogStash::ShutdownSignal
-      # teardown could receive the ShutdownSignal, retry it
+
+      # Assuming the failure that caused this exception is transient,
+      # let's sleep for a bit and execute #run again
+      sleep(1)
       retry
+    ensure
+      plugin.do_close
     end
   end # def inputworker
 
   def filterworker
-    LogStash::Util::set_thread_name("|worker")
+    LogStash::Util.set_thread_name("|worker")
     begin
       while true
         event = @input_to_filter.pop
@@ -227,65 +227,50 @@ class LogStash::Pipeline
           break
         end
       end
-    rescue => e
-      @logger.error("Exception in filterworker", "exception" => e, "backtrace" => e.backtrace)
+    rescue Exception => e
+      # Plugins authors should manage their own exceptions in the plugin code
+      # but if an exception is raised up to the worker thread they are considered
+      # fatal and logstash will not recover from this situation.
+      #
+      # Users need to check their configuration or see if there is a bug in the
+      # plugin.
+      @logger.error("Exception in filterworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.",
+                    "exception" => e, "backtrace" => e.backtrace)
+      raise
+    ensure
+      @filters.each(&:do_close)
     end
-
-    @filters.each(&:teardown)
   end # def filterworker
 
   def outputworker
-    LogStash::Util::set_thread_name(">output")
+    LogStash::Util.set_thread_name(">output")
     @outputs.each(&:worker_setup)
 
     while true
       event = @filter_to_output.pop
       break if event == LogStash::SHUTDOWN
       output_func(event)
-    end # while true
-
+    end
+  ensure
     @outputs.each do |output|
-      output.worker_plugins.each(&:teardown)
+      output.worker_plugins.each(&:do_close)
     end
   end # def outputworker
 
-  # Shutdown this pipeline.
-  #
-  # This method is intended to be called from another thread
-  def shutdown
-    InflightEventsReporter.logger = @logger
-    InflightEventsReporter.start(@input_to_filter, @filter_to_output, @outputs)
-    @input_threads.each do |thread|
-      # Interrupt all inputs
-      @logger.info("Sending shutdown signal to input thread", :thread => thread)
-
-      # synchronize both ShutdownSignal and teardown below. by synchronizing both
-      # we will avoid potentially sending a shutdown signal when the inputworker is
-      # executing the teardown method.
-      @run_mutex.synchronize do
-        thread.raise(LogStash::ShutdownSignal)
-        begin
-          thread.wakeup # in case it's in blocked IO or sleeping
-        rescue ThreadError
-        end
-      end
-    end
+  # initiate the pipeline shutdown sequence
+  # this method is intended to be called from outside the pipeline thread
+  # @param before_stop [Proc] code block called before performing stop operation on input plugins
+  def shutdown(&before_stop)
+    # shutdown can only start once the pipeline has completed its startup.
+    # avoid potential race conditoon between the startup sequence and this
+    # shutdown method which can be called from another thread at any time
+    sleep(0.1) while !ready?
 
-    # sometimes an input is stuck in a blocking I/O so we need to tell it to teardown directly
-    @inputs.each do |input|
-      begin
-        # input teardown must be synchronized since is can be called concurrently by
-        # the input worker thread and from the pipeline thread shutdown method.
-        # this means that input teardown methods must support multiple calls.
-        @run_mutex.synchronize{input.teardown}
-      rescue LogStash::ShutdownSignal
-        # teardown could receive the ShutdownSignal, retry it
-        retry
-      end
-    end
+    # TODO: should we also check against calling shutdown multiple times concurently?
+
+    before_stop.call if block_given?
 
-    # No need to send the ShutdownEvent to the filters/outputs nor to wait for
-    # the inputs to finish, because in the #run method we wait for that anyway.
+    @inputs.each(&:do_stop)
   end # def shutdown
 
   def plugin(plugin_type, name, *args)

data/lib/logstash/plugin.rb

@@ -3,6 +3,7 @@ require "logstash/namespace"
 require "logstash/logging"
 require "logstash/config/mixin"
 require "cabin"
+require "concurrent"
 
 class LogStash::Plugin
   attr_accessor :params
@@ -27,72 +28,21 @@ class LogStash::Plugin
     @logger = Cabin::Channel.get(LogStash)
   end
 
-  # This method is called when someone or something wants this plugin to shut
-  # down. When you successfully shutdown, you must call 'finished'
-  # You must also call 'super' in any subclasses.
+  # close is called during shutdown, after the plugin worker
+  # main task terminates
   public
-  def shutdown(queue)
-    # By default, shutdown is assumed a no-op for all plugins.
-    # If you need to take special efforts to shutdown (like waiting for
-    # an operation to complete, etc)
-    teardown
-    @logger.info("Received shutdown signal", :plugin => self)
-
-    @shutdown_queue = queue
-    if @plugin_state == :finished
-      finished
-    else
-      @plugin_state = :terminating
-    end
-  end # def shutdown
-
-  # You should call this method when you (the plugin) are done with work
-  # forever.
-  public
-  def finished
-    # TODO(sissel): I'm not sure what I had planned for this shutdown_queue
-    # thing
-    if @shutdown_queue
-      @logger.info("Sending shutdown event to agent queue", :plugin => self)
-      @shutdown_queue << self
-    end
-
-    if @plugin_state != :finished
-      @logger.info("Plugin is finished", :plugin => self)
-      @plugin_state = :finished
-    end
-  end # def finished
-
-  # Subclasses should implement this teardown method if you need to perform any
-  # special tasks during shutdown (like flushing, etc.)
-  public
-  def teardown
-    # nothing by default
-    finished
+  def do_close
+    @logger.debug("closing", :plugin => self)
+    close
   end
 
-  # This method is called when a SIGHUP triggers a reload operation
+  # Subclasses should implement this close method if you need to perform any
+  # special tasks during shutdown (like flushing, etc.)
   public
-  def reload
-    # Do nothing by default
+  def close
+    # ..
   end
 
-  public
-  def finished?
-    return @plugin_state == :finished
-  end # def finished?
-
-  public
-  def running?
-    return @plugin_state != :finished
-  end # def finished?
-
-  public
-  def terminating?
-    return @plugin_state == :terminating
-  end # def terminating?
-
-  public
   def to_s
     return "#{self.class.name}: #{@params}"
   end

data/lib/logstash/string_interpolation.rb

@@ -46,7 +46,7 @@ module LogStash
         position = match.offset(0).last
       end
 
-      if position < template.size - 1
+      if position < template.size
         nodes << StaticNode.new(template[position..-1])
       end
 

data/lib/logstash/util/defaults_printer.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+require "logstash/util/worker_threads_default_printer"
+
+
+# This class exists to format the settings for defaults used
+module LogStash module Util class DefaultsPrinter
+  def self.print(settings)
+    new(settings).print
+  end
+
+  def initialize(settings)
+    @settings = settings
+    @printers = [workers]
+  end
+
+  def print
+    collector = []
+    @printers.each do |printer|
+      printer.visit(collector)
+    end
+    "Default settings used: " + collector.join(', ')
+  end
+
+  private
+
+  def workers
+    WorkerThreadsDefaultPrinter.new(@settings)
+  end
+end end end

data/lib/logstash/util/worker_threads_default_printer.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+# This class exists to format the settings for default worker threads
+module LogStash module Util class WorkerThreadsDefaultPrinter
+
+  def initialize(settings)
+    @setting = settings.fetch('filter-workers', 1)
+  end
+
+  def visit(collector)
+    collector.push "Filter workers: #{@setting}"
+  end
+
+end end end
+

data/lib/logstash/version.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # The version of logstash.
-LOGSTASH_VERSION = "2.0.0-beta1"
+LOGSTASH_VERSION = "2.0.0-beta2"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.