logstash-core 2.0.1.snapshot1-java → 2.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfcc3785015f1606582277fcaa83f916d71e1fc6
-  data.tar.gz: 62a9a1dce8d852d7fd8c7baa6c9467cb4bc948b5
+  metadata.gz: 79705663d762e3aa65d23f6c98c0fb39923546ef
+  data.tar.gz: 66c3fc910d9d9290189c55843bdfbf7ed4d3ea65
 SHA512:
-  metadata.gz: c7f4ebb46dbece030b609f4151ae1b989227c177f659dd1a26fcafff23ce0848eaafaf938df47d6db4a3a222713704bb7d842f2ec7c7a064dce0422acf22b4e6
-  data.tar.gz: 29dcfff2b13d39a41568368b71b5711f0a489643fc44db8aff2477db1bd86a53e997b52c66461370c6ea3527cd6343bca80f9b72adcc90b6f7321d2bcb920e37
+  metadata.gz: 455434efcd9788628c5350cb6f593eaad237a10269b68627d32e084aaa284c9a76b41d305b71ec4e0ee4d34e09f51a8dcaa2e03985f2be8f8e03ac0bef7fc3b0
+  data.tar.gz: b64af5a363fcd0096d06f4163b82595735acb58a3a958b39c6258a8cb08eb415342be4206f7d6a5e7765ba8c7a1cee2339f4eae8543e6e851c2d0d92c77c27a6

data/lib/logstash/agent.rb

@@ -23,7 +23,7 @@ class LogStash::Agent < Clamp::Command
   option ["-w", "--filterworkers"], "COUNT",
     I18n.t("logstash.agent.flag.filterworkers"),
     :attribute_name => :filter_workers,
-    :default => LogStash::Config::CpuCoreStrategy.fifty_percent, &:to_i
+    :default => 0, &:to_i
 
   option ["-l", "--log"], "FILE",
     I18n.t("logstash.agent.flag.log"),
@@ -50,6 +50,11 @@ class LogStash::Agent < Clamp::Command
     I18n.t("logstash.agent.flag.configtest"),
     :attribute_name => :config_test
 
+  option "--[no-]allow-unsafe-shutdown", :flag,
+    I18n.t("logstash.agent.flag.unsafe_shutdown"),
+    :attribute_name => :unsafe_shutdown,
+    :default => false
+
   # Emit a warning message.
   def warn(message)
     # For now, all warnings are fatal.
@@ -75,6 +80,9 @@ class LogStash::Agent < Clamp::Command
     require "logstash/plugin"
     @logger = Cabin::Channel.get(LogStash)
 
+    LogStash::ShutdownController.unsafe_shutdown = unsafe_shutdown?
+    LogStash::ShutdownController.logger = @logger
+
     if version?
       show_version
       return 0
@@ -143,7 +151,7 @@ class LogStash::Agent < Clamp::Command
       configure_logging(log_file)
     end
 
-    pipeline.configure("filter-workers", filter_workers)
+    pipeline.configure("filter-workers", filter_workers) if filter_workers > 0
 
     # Stop now if we are only asking for a config test.
     if config_test?
@@ -176,8 +184,7 @@ class LogStash::Agent < Clamp::Command
 
   def shutdown(pipeline)
     pipeline.shutdown do
-      InflightEventsReporter.logger = @logger
-      InflightEventsReporter.start(pipeline.input_to_filter, pipeline.filter_to_output, pipeline.outputs)
+      ::LogStash::ShutdownController.start(pipeline)
     end
   end
 
@@ -312,19 +319,27 @@ class LogStash::Agent < Clamp::Command
     Dir.glob(path).sort.each do |file|
       next unless File.file?(file)
       if file.match(/~$/)
-        @logger.debug("NOT reading config file because it is a temp file", :file => file)
+        @logger.debug("NOT reading config file because it is a temp file", :config_file => file)
         next
       end
-      @logger.debug("Reading config file", :file => file)
+      @logger.debug("Reading config file", :config_file => file)
       cfg = File.read(file)
       if !cfg.ascii_only? && !cfg.valid_encoding?
         encoding_issue_files << file
       end
       config << cfg + "\n"
+      if config_test?
+        @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
+        @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
+      end
     end
     if (encoding_issue_files.any?)
       fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
     end
+    if config_test?
+      @logger.debug? && @logger.debug("\nThe following is the merged configuration")
+      @logger.debug? && @logger.debug("\n" + config + "\n\n")
+    end
     return config
   end # def load_config
 

data/lib/logstash/config/mixin.rb

@@ -35,12 +35,6 @@ module LogStash::Config::Mixin
   attr_accessor :config
   attr_accessor :original_params
 
-  CONFIGSORT = {
-    Symbol => 0,
-    String => 0,
-    Regexp => 100,
-  }
-
   PLUGIN_VERSION_1_0_0 = LogStash::Util::PluginVersion.new(1, 0, 0)
   PLUGIN_VERSION_0_9_0 = LogStash::Util::PluginVersion.new(0, 9, 0)
 

data/lib/logstash/event.rb

@@ -106,7 +106,7 @@ class LogStash::Event
 
   public
   def to_s
-    self.sprintf("#{timestamp.to_iso8601} %{host} %{message}")
+    "#{timestamp.to_iso8601} #{self.sprintf("%{host} %{message}")}"
   end # def to_s
 
   public

data/lib/logstash/filters/base.rb

@@ -117,8 +117,6 @@ class LogStash::Filters::Base < LogStash::Plugin
   # Optional.
   config :periodic_flush, :validate => :boolean, :default => false
 
-  RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field", "include_any", "exclude_any"]
-
   public
   def initialize(params)
     super
@@ -145,6 +143,7 @@ class LogStash::Filters::Base < LogStash::Plugin
   # @return [Array<LogStash::Event] filtered events and any new events generated by the filter
   public
   def multi_filter(events)
+    LogStash::Util.set_thread_plugin(self)
     result = []
     events.each do |event|
       unless event.cancelled?

data/lib/logstash/inputs/base.rb

@@ -89,11 +89,6 @@ class LogStash::Inputs::Base < LogStash::Plugin
     @stop_called.value
   end
 
-  protected
-  def to_event(raw, source)
-    raise LogStash::ThisMethodWasRemoved("LogStash::Inputs::Base#to_event - you should use codecs now instead of to_event. Not sure what this means? Get help on https://discuss.elastic.co/c/logstash")
-  end # def to_event
-
   protected
   def decorate(event)
     # Only set 'type' if not already set. This is backwards-compatible behavior

data/lib/logstash/outputs/base.rb

@@ -1,11 +1,9 @@
 # encoding: utf-8
-require "cgi"
 require "logstash/event"
 require "logstash/logging"
 require "logstash/plugin"
 require "logstash/namespace"
 require "logstash/config/mixin"
-require "uri"
 
 class LogStash::Outputs::Base < LogStash::Plugin
   include LogStash::Config::Mixin
@@ -25,7 +23,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
   # Note that this setting may not be useful for all outputs.
   config :workers, :validate => :number, :default => 1
 
-  attr_reader :worker_plugins, :worker_queue
+  attr_reader :worker_plugins, :worker_queue, :worker_threads
 
   public
   def workers_not_supported(message=nil)
@@ -58,13 +56,15 @@ class LogStash::Outputs::Base < LogStash::Plugin
   def worker_setup
     if @workers == 1
       @worker_plugins = [self]
+      @worker_threads = []
     else
       define_singleton_method(:handle, method(:handle_worker))
       @worker_queue = SizedQueue.new(20)
       @worker_plugins = @workers.times.map { self.class.new(@original_params.merge("workers" => 1)) }
-      @worker_plugins.map.with_index do |plugin, i|
+      @worker_threads = @worker_plugins.map.with_index do |plugin, i|
         Thread.new(original_params, @worker_queue) do |params, queue|
-          LogStash::Util::set_thread_name(">#{self.class.config_name}.#{i}")
+          LogStash::Util.set_thread_name(">#{self.class.config_name}.#{i}")
+          LogStash::Util.set_thread_plugin(self)
           plugin.register
           while true
             event = queue.pop
@@ -77,10 +77,12 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   public
   def handle(event)
+    LogStash::Util.set_thread_plugin(self)
     receive(event)
   end # def handle
 
   def handle_worker(event)
+    LogStash::Util.set_thread_plugin(self)
     @worker_queue.push(event)
   end
 

data/lib/logstash/pipeline.rb

@@ -9,11 +9,11 @@ require "logstash/config/file"
 require "logstash/filters/base"
 require "logstash/inputs/base"
 require "logstash/outputs/base"
-require "logstash/util/reporter"
 require "logstash/config/cpu_core_strategy"
 require "logstash/util/defaults_printer"
+require "logstash/shutdown_controller"
 
-class LogStash::Pipeline
+module LogStash; class Pipeline
   attr_reader :inputs, :filters, :outputs, :input_to_filter, :filter_to_output
 
   def initialize(configstr)
@@ -25,6 +25,7 @@ class LogStash::Pipeline
 
     grammar = LogStashConfigParser.new
     @config = grammar.parse(configstr)
+
     if @config.nil?
       raise LogStash::ConfigurationError, grammar.failure_reason
     end
@@ -46,7 +47,7 @@ class LogStash::Pipeline
     @filter_to_output = filters? ? SizedQueue.new(20) : @input_to_filter
 
     @settings = {
-      "filter-workers" => LogStash::Config::CpuCoreStrategy.fifty_percent
+      "default-filter-workers" => LogStash::Config::CpuCoreStrategy.fifty_percent
     }
 
     # @ready requires thread safety since it is typically polled from outside the pipeline thread
@@ -59,14 +60,32 @@ class LogStash::Pipeline
   end
 
   def configure(setting, value)
-    if setting == "filter-workers" && value > 1
-      # Abort if we have any filters that aren't threadsafe
-      plugins = @filters.select { |f| !f.threadsafe? }.collect { |f| f.class.config_name }
-      if !plugins.size.zero?
-        raise LogStash::ConfigurationError, "Cannot use more than 1 filter worker because the following plugins don't work with more than one worker: #{plugins.join(", ")}"
+    @settings[setting] = value
+  end
+
+  def safe_filter_worker_count
+    default = @settings["default-filter-workers"]
+    thread_count = @settings["filter-workers"] #override from args "-w 8" or config
+    safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
+    if unsafe_filters.any?
+      plugins = unsafe_filters.collect { |f| f.class.config_name }
+      case thread_count
+      when nil
+        # user did not specify a worker thread count
+        # warn if the default is multiple
+        @logger.warn("Defaulting filter worker threads to 1 because there are some filters that might not work with multiple worker threads",
+          :count_was => default, :filters => plugins) if default > 1
+        1 # can't allow the default value to propagate if there are unsafe filters
+      when 0, 1
+        1
+      else
+        @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads",
+          :worker_threads => thread_count, :filters => plugins)
+        thread_count # allow user to force this even if there are unsafe filters
       end
+    else
+      thread_count || default
     end
-    @settings[setting] = value
   end
 
   def filters?
@@ -149,9 +168,14 @@ class LogStash::Pipeline
 
   def start_filters
     @filters.each(&:register)
-    to_start = @settings["filter-workers"]
-    @filter_threads = to_start.times.collect do
-      Thread.new { filterworker }
+    # dynamically get thread count based on filter threadsafety
+    # moved this test to here to allow for future config reloading
+    to_start = safe_filter_worker_count
+    @filter_threads = to_start.times.collect do |i|
+      Thread.new do
+        LogStash::Util.set_thread_name("|filterworker.#{i}")
+        filterworker
+      end
     end
     actually_started = @filter_threads.select(&:alive?).size
     msg = "Worker threads expected: #{to_start}, worker threads started: #{actually_started}"
@@ -175,7 +199,8 @@ class LogStash::Pipeline
   end
 
   def inputworker(plugin)
-    LogStash::Util::set_thread_name("<#{plugin.class.config_name}")
+    LogStash::Util.set_thread_name("<#{plugin.class.config_name}")
+    LogStash::Util.set_thread_plugin(plugin)
     begin
       plugin.run(@input_to_filter)
     rescue => e
@@ -208,7 +233,6 @@ class LogStash::Pipeline
   end # def inputworker
 
   def filterworker
-    LogStash::Util.set_thread_name("|worker")
     begin
       while true
         event = @input_to_filter.pop
@@ -250,6 +274,7 @@ class LogStash::Pipeline
       event = @filter_to_output.pop
       break if event == LogStash::SHUTDOWN
       output_func(event)
+      LogStash::Util.set_thread_plugin(nil)
     end
   ensure
     @outputs.each do |output|
@@ -309,4 +334,49 @@ class LogStash::Pipeline
     end
   end # flush_filters_to_output!
 
-end # class Pipeline
+  def inflight_count
+    data = {}
+    total = 0
+
+    input_to_filter = @input_to_filter.size
+    total += input_to_filter
+    filter_to_output = @filter_to_output.size
+    total += filter_to_output
+
+    data["input_to_filter"] = input_to_filter if input_to_filter > 0
+    data["filter_to_output"] = filter_to_output if filter_to_output > 0
+
+    output_worker_queues = []
+    @outputs.each do |output|
+      next unless output.worker_queue && output.worker_queue.size > 0
+      plugin_info = output.debug_info
+      size = output.worker_queue.size
+      total += size
+      plugin_info << size
+      output_worker_queues << plugin_info
+    end
+    data["output_worker_queues"] = output_worker_queues unless output_worker_queues.empty?
+    data["total"] = total
+    data
+  end
+
+  def stalling_threads
+    plugin_threads
+     .reject {|t| t["blocked_on"] } # known begnin blocking statuses
+     .each {|t| t.delete("backtrace") }
+     .each {|t| t.delete("blocked_on") }
+     .each {|t| t.delete("status") }
+  end
+
+  def plugin_threads
+    input_threads = @input_threads.select {|t| t.alive? }.map {|t| thread_info(t) }
+    filter_threads = @filter_threads.select {|t| t.alive? }.map {|t| thread_info(t) }
+    output_threads = @output_threads.select {|t| t.alive? }.map {|t| thread_info(t) }
+    output_worker_threads = @outputs.flat_map {|output| output.worker_threads }.map {|t| thread_info(t) }
+    input_threads + filter_threads + output_threads + output_worker_threads
+  end
+
+  def thread_info(thread)
+    LogStash::Util.thread_info(thread)
+  end
+end; end

data/lib/logstash/plugin.rb

@@ -59,6 +59,11 @@ class LogStash::Plugin
     end
   end
 
+  public
+  def debug_info
+    [self.class.to_s, original_params]
+  end
+
   # Look up a plugin by type and name.
   public
   def self.lookup(type, name)

data/lib/logstash/shutdown_controller.rb

@@ -0,0 +1,127 @@
+# encoding: utf-8
+
+module LogStash
+  class ShutdownController
+
+    CHECK_EVERY = 1 # second
+    REPORT_EVERY = 5 # checks
+    ABORT_AFTER = 3 # stalled reports
+
+    attr_reader :cycle_period, :report_every, :abort_threshold
+
+    def initialize(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
+      @pipeline = pipeline
+      @cycle_period = cycle_period
+      @report_every = report_every
+      @abort_threshold = abort_threshold
+      @reports = []
+    end
+
+    def self.unsafe_shutdown=(boolean)
+      @unsafe_shutdown = boolean
+    end
+
+    def self.unsafe_shutdown?
+      @unsafe_shutdown
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def self.logger
+      @logger ||= Cabin::Channel.get(LogStash)
+    end
+
+    def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
+      controller = self.new(pipeline, cycle_period, report_every, abort_threshold)
+      Thread.new(controller) { |controller| controller.start }
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    def start
+      sleep(@cycle_period)
+      cycle_number = 0
+      stalled_count = 0
+      Stud.interval(@cycle_period) do
+        @reports << Report.from_pipeline(@pipeline)
+        @reports.delete_at(0) if @reports.size > @report_every # expire old report
+        if cycle_number == (@report_every - 1) # it's report time!
+          logger.warn(@reports.last.to_hash)
+
+          if shutdown_stalled?
+            logger.error("The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.") if stalled_count == 0
+            stalled_count += 1
+
+            if self.class.unsafe_shutdown? && @abort_threshold == stalled_count
+              logger.fatal("Forcefully quitting logstash..")
+              force_exit()
+              break
+            end
+          else
+            stalled_count = 0
+          end
+        end
+        cycle_number = (cycle_number + 1) % @report_every
+      end
+    end
+
+    # A pipeline shutdown is stalled if
+    # * at least REPORT_EVERY reports have been created
+    # * the inflight event count is in monotonically increasing
+    # * there are worker threads running which aren't blocked on SizedQueue pop/push
+    # * the stalled thread list is constant in the previous REPORT_EVERY reports
+    def shutdown_stalled?
+      return false unless @reports.size == @report_every #
+      # is stalled if inflight count is either constant or increasing
+      stalled_event_count = @reports.each_cons(2).all? do |prev_report, next_report|
+        prev_report.inflight_count["total"] <= next_report.inflight_count["total"]
+      end
+      if stalled_event_count
+        @reports.each_cons(2).all? do |prev_report, next_report|
+          prev_report.stalling_threads == next_report.stalling_threads
+        end
+      else
+        false
+      end
+    end
+
+    def force_exit
+      exit(-1)
+    end
+  end
+
+  class Report
+
+    attr_reader :inflight_count, :stalling_threads
+
+    def self.from_pipeline(pipeline)
+      new(pipeline.inflight_count, pipeline.stalling_threads)
+    end
+
+    def initialize(inflight_count, stalling_threads)
+      @inflight_count = inflight_count
+      @stalling_threads = format_threads_by_plugin(stalling_threads)
+    end
+
+    def to_hash
+      {
+        "INFLIGHT_EVENT_COUNT" => @inflight_count,
+        "STALLING_THREADS" => @stalling_threads
+      }
+    end
+
+    def format_threads_by_plugin(stalling_threads)
+      stalled_plugins = {}
+      stalling_threads.each do |thr|
+        key = (thr.delete("plugin") || "other")
+        stalled_plugins[key] ||= []
+        stalled_plugins[key] << thr
+      end
+      stalled_plugins
+    end
+  end
+end