logstash-core 2.2.4.snapshot1

data/lib/logstash/pipeline_reporter.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require 'ostruct'
+
+module LogStash; class PipelineReporter
+  attr_reader :logger, :pipeline
+
+  # This is an immutable copy of the pipeline state,
+  # It is a proxy to a hash to allow us to add methods dynamically to the hash
+  class Snapshot
+    def initialize(data)
+      @data = data
+    end
+
+    def to_hash
+      @data
+    end
+
+    def to_simple_hash
+      {"inflight_count" => inflight_count, "stalling_thread_info" => format_threads_by_plugin}
+    end
+
+    def to_str
+      to_simple_hash.to_s
+    end
+    alias_method :to_s, :to_str
+
+    def method_missing(meth)
+      @data[meth]
+    end
+
+    def format_threads_by_plugin
+      stalled_plugins = {}
+      stalling_threads_info.each do |thr|
+        key = (thr.delete("plugin") || "other")
+        stalled_plugins[key] ||= []
+        stalled_plugins[key] << thr
+      end
+      stalled_plugins
+    end
+  end
+
+  def initialize(logger,pipeline)
+    @logger = logger
+    @pipeline = pipeline
+  end
+
+  # The main way of accessing data from the reporter,,
+  # this provides a (more or less) consistent snapshot of what's going on in the
+  # pipeline with some extra decoration
+  def snapshot
+    Snapshot.new(self.to_hash)
+  end
+
+  def to_hash
+    pipeline.inflight_batches_synchronize do |batch_map|
+      worker_states_snap = worker_states(batch_map) # We only want to run this once
+      inflight_count = worker_states_snap.map {|s| s[:inflight_count] }.reduce(0, :+)
+
+      {
+        :events_filtered => events_filtered,
+        :events_consumed => events_consumed,
+        :worker_count => pipeline.worker_threads.size,
+        :inflight_count => inflight_count,
+        :worker_states => worker_states_snap,
+        :output_info => output_info,
+        :thread_info => pipeline.plugin_threads_info,
+        :stalling_threads_info => pipeline.stalling_threads_info
+      }
+    end
+  end
+
+  private
+
+  def events_filtered
+    pipeline.events_filtered.value
+  end
+
+  def events_consumed
+    pipeline.events_consumed.value
+  end
+
+  def plugin_threads
+    pipeline.plugin_threads
+  end
+
+  # Not threadsafe! must be called within an `inflight_batches_synchronize` block
+  def worker_states(batch_map)
+      pipeline.worker_threads.map.with_index do |thread,idx|
+        status = thread.status || "dead"
+        inflight_count = batch_map[thread] ? batch_map[thread].size : 0
+        {
+          :status => status,
+          :alive => thread.alive?,
+          :index => idx,
+          :inflight_count => inflight_count
+        }
+    end
+  end
+
+  def output_info
+    pipeline.outputs.map do |output_delegator|
+      is_multi_worker = output_delegator.worker_count > 1
+
+      {
+        :type => output_delegator.config_name,
+        :config => output_delegator.config,
+        :is_multi_worker => is_multi_worker,
+        :events_received => output_delegator.events_received,
+        :workers => output_delegator.workers,
+        :busy_workers => output_delegator.busy_workers
+      }
+    end
+  end
+end end

data/lib/logstash/plugin.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "cabin"
+require "concurrent"
+
+class LogStash::Plugin
+  attr_accessor :params
+  attr_accessor :logger
+
+  NL = "\n"
+
+  public
+  def hash
+    params.hash ^
+    self.class.name.hash
+  end
+
+  public
+  def eql?(other)
+    self.class.name == other.class.name && @params == other.params
+  end
+
+  public
+  def initialize(params=nil)
+    @params = LogStash::Util.deep_clone(params)
+    @logger = Cabin::Channel.get(LogStash)
+  end
+
+  # close is called during shutdown, after the plugin worker
+  # main task terminates
+  public
+  def do_close
+    @logger.debug("closing", :plugin => self)
+    close
+  end
+
+  # Subclasses should implement this close method if you need to perform any
+  # special tasks during shutdown (like flushing, etc.)
+  public
+  def close
+    # ..
+  end
+
+  def to_s
+    return "#{self.class.name}: #{@params}"
+  end
+
+  public
+  def inspect
+    if !@params.nil?
+      description = @params
+        .reject { |k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?) }
+        .collect { |k, v| "#{k}=>#{v.inspect}" }
+      return "<#{self.class.name} #{description.join(", ")}>"
+    else
+      return "<#{self.class.name} --->"
+    end
+  end
+
+  public
+  def debug_info
+    [self.class.to_s, original_params]
+  end
+
+  # Look up a plugin by type and name.
+  public
+  def self.lookup(type, name)
+    path = "logstash/#{type}s/#{name}"
+
+    # first check if plugin already exists in namespace and continue to next step if not
+    begin
+      return namespace_lookup(type, name)
+    rescue NameError
+      logger.debug("Plugin not defined in namespace, checking for plugin file", :type => type, :name => name, :path => path)
+    end
+
+    # try to load the plugin file. ex.: lookup("filter", "grok") will require logstash/filters/grok
+    require(path)
+
+    # check again if plugin is now defined in namespace after the require
+    namespace_lookup(type, name)
+  rescue LoadError, NameError => e
+    raise(LogStash::PluginLoadingError, I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :path => path, :error => e.to_s))
+  end
+
+  private
+
+  # lookup a plugin by type and name in the existing LogStash module namespace
+  # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
+  # @param type [String] plugin type, "input", "ouput", "filter"
+  # @param name [String] plugin name, ex.: "grok"
+  # @return [Class] the plugin class or raises NameError
+  # @raise NameError if plugin class does not exist or is invalid
+  def self.namespace_lookup(type, name)
+    type_const = "#{type.capitalize}s"
+    namespace = LogStash.const_get(type_const)
+    # the namespace can contain constants which are not for plugins classes (do not respond to :config_name)
+    # namespace.constants is the shallow collection of all constants symbols in namespace
+    # note that below namespace.const_get(c) should never result in a NameError since c is from the constants collection
+    klass_sym = namespace.constants.find { |c| is_a_plugin?(namespace.const_get(c), name) }
+    klass = klass_sym && namespace.const_get(klass_sym)
+    raise(NameError) unless klass
+    klass
+  end
+
+  # check if klass is a valid plugin for name
+  # @param klass [Class] plugin class
+  # @param name [String] plugin name
+  # @return [Boolean] true if klass is a valid plugin for name
+  def self.is_a_plugin?(klass, name)
+    klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
+  end
+
+  # @return [Cabin::Channel] logger channel for class methods
+  def self.logger
+    @logger ||= Cabin::Channel.get(LogStash)
+  end
+end # class LogStash::Plugin

data/lib/logstash/program.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash::Program
+  public
+  def exit(value)
+    if RUBY_ENGINE == "jruby"
+      # Kernel::exit() in jruby just tosses an exception? Let's actually exit.
+      Java::java.lang.System.exit(value)
+    else
+      Kernel::exit(value)
+    end
+  end # def exit
+end # module LogStash::Program

data/lib/logstash/runner.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+Thread.abort_on_exception = true
+Encoding.default_external = Encoding::UTF_8
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+require "logstash/environment"
+
+LogStash::Environment.load_locale!
+
+require "logstash/namespace"
+require "logstash/program"
+
+class LogStash::Runner
+  include LogStash::Program
+
+  def main(args)
+    require "logstash/util"
+    require "logstash/util/java_version"
+    require "stud/trap"
+    require "stud/task"
+    @startup_interruption_trap = Stud::trap("INT") { puts "Interrupted"; exit 0 }
+
+    LogStash::Util::set_thread_name(self.class.name)
+
+    if RUBY_VERSION < "1.9.2"
+      $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      return 1
+    end
+
+    # Print a warning to STDERR for bad java versions
+    LogStash::Util::JavaVersion.warn_on_bad_java_version
+
+    Stud::untrap("INT", @startup_interruption_trap)
+
+    task = run(args)
+    exit(task.wait)
+  end # def self.main
+
+  def run(args)
+    command = args.shift
+    commands = {
+      "version" => lambda do
+        require "logstash/agent"
+        agent_args = ["--version"]
+        if args.include?("--verbose")
+          agent_args << "--verbose"
+        end
+        return LogStash::Agent.run($0, agent_args)
+      end,
+      "irb" => lambda do
+        require "irb"
+        return IRB.start(__FILE__)
+      end,
+      "pry" => lambda do
+        require "pry"
+        return binding.pry
+      end,
+      "docgen" => lambda do
+        require 'docs/asciidocgen'
+        opts = OptionParser.new
+        settings = {}
+        opts.on("-o DIR", "--output DIR",
+          "Directory to output to; optional. If not specified,"\
+          "we write to stdout.") do |val|
+          settings[:output] = val
+        end
+        args = opts.parse(ARGV)
+        docs = LogStashConfigAsciiDocGenerator.new
+        args.each do |arg|
+          docs.generate(arg, settings)
+        end
+        return 0
+      end,
+      "agent" => lambda do
+        require "logstash/agent"
+        # Hack up a runner
+        agent = LogStash::Agent.new("/bin/logstash agent", $0)
+        begin
+          agent.parse(args)
+        rescue Clamp::HelpWanted => e
+          show_help(e.command)
+          return 0
+        rescue Clamp::UsageError => e
+          # If 'too many arguments' then give the arguments to
+          # the next command. Otherwise it's a real error.
+          raise if e.message != "too many arguments"
+          remaining = agent.remaining_arguments
+        end
+
+        return agent.execute
+      end
+    } # commands
+
+    if commands.include?(command)
+      return Stud::Task.new { commands[command].call }
+    else
+      if command.nil?
+        $stderr.puts "No command given"
+      else
+        if !%w(--help -h help).include?(command)
+          # Emit 'no such command' if it's not someone asking for help.
+          $stderr.puts "No such command #{command.inspect}"
+        end
+      end
+      $stderr.puts %q[
+Usage: logstash <command> [command args]
+Run a command with the --help flag to see the arguments.
+For example: logstash agent --help
+
+Available commands:
+  agent - runs the logstash agent
+  version - emits version info about this logstash
+]
+      #$stderr.puts commands.keys.map { |s| "  #{s}" }.join("\n")
+      return Stud::Task.new { 1 }
+    end
+  end # def run
+
+  private
+
+  def show_help(command)
+    puts command.help
+  end
+end # class LogStash::Runner

data/lib/logstash/shutdown_watcher.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+
+module LogStash
+  class ShutdownWatcher
+
+    CHECK_EVERY = 1 # second
+    REPORT_EVERY = 5 # checks
+    ABORT_AFTER = 3 # stalled reports
+
+    attr_reader :cycle_period, :report_every, :abort_threshold
+
+    def initialize(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
+      @pipeline = pipeline
+      @cycle_period = cycle_period
+      @report_every = report_every
+      @abort_threshold = abort_threshold
+      @reports = []
+    end
+
+    def self.unsafe_shutdown=(boolean)
+      @unsafe_shutdown = boolean
+    end
+
+    def self.unsafe_shutdown?
+      @unsafe_shutdown
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def self.logger
+      @logger ||= Cabin::Channel.get(LogStash)
+    end
+
+    def self.start(pipeline, cycle_period=CHECK_EVERY, report_every=REPORT_EVERY, abort_threshold=ABORT_AFTER)
+      watcher = self.new(pipeline, cycle_period, report_every, abort_threshold)
+      Thread.new(watcher) { |watcher| watcher.start }
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    def start
+      sleep(@cycle_period)
+      cycle_number = 0
+      stalled_count = 0
+      Stud.interval(@cycle_period) do
+        @reports << pipeline_report_snapshot
+        @reports.delete_at(0) if @reports.size > @report_every # expire old report
+        if cycle_number == (@report_every - 1) # it's report time!
+          logger.warn(@reports.last)
+
+          if shutdown_stalled?
+            logger.error("The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.") if stalled_count == 0
+            stalled_count += 1
+
+            if self.class.unsafe_shutdown? && @abort_threshold == stalled_count
+              logger.fatal("Forcefully quitting logstash..")
+              force_exit()
+              break
+            end
+          else
+            stalled_count = 0
+          end
+        end
+        cycle_number = (cycle_number + 1) % @report_every
+      end
+    end
+
+    def pipeline_report_snapshot
+      @pipeline.reporter.snapshot
+    end
+
+    # A pipeline shutdown is stalled if
+    # * at least REPORT_EVERY reports have been created
+    # * the inflight event count is in monotonically increasing
+    # * there are worker threads running which aren't blocked on SizedQueue pop/push
+    # * the stalled thread list is constant in the previous REPORT_EVERY reports
+    def shutdown_stalled?
+      return false unless @reports.size == @report_every #
+      # is stalled if inflight count is either constant or increasing
+      stalled_event_count = @reports.each_cons(2).all? do |prev_report, next_report|
+        prev_report.inflight_count <= next_report.inflight_count
+      end
+      if stalled_event_count
+        @reports.each_cons(2).all? do |prev_report, next_report|
+          prev_report.stalling_threads == next_report.stalling_threads
+        end
+      else
+        false
+      end
+    end
+
+    def force_exit
+      exit(-1)
+    end
+  end
+end