logstash-core 2.2.4.snapshot1

data/lib/logstash/outputs/base.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/namespace"
+require "logstash/config/mixin"
+require "logstash/util/wrapped_synchronous_queue"
+require "concurrent/atomic/atomic_fixnum"
+
+class LogStash::Outputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "output"
+
+  config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
+
+  # The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
+
+  # The number of workers to use for this output.
+  # Note that this setting may not be useful for all outputs.
+  config :workers, :validate => :number, :default => 1
+
+  attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
+
+  def self.declare_threadsafe!
+    declare_workers_not_supported!
+    @threadsafe = true
+  end
+
+  def self.threadsafe?
+    @threadsafe == true
+  end
+
+  def self.declare_workers_not_supported!(message=nil)
+    @workers_not_supported_message = message
+    @workers_not_supported = true
+  end
+
+  def self.workers_not_supported_message
+    @workers_not_supported_message
+  end
+
+  def self.workers_not_supported?
+    !!@workers_not_supported
+  end
+
+  public
+  # TODO: Remove this in the next major version after Logstash 2.x
+  # Post 2.x it should raise an error and tell people to use the class level
+  # declaration
+  def workers_not_supported(message=nil)
+    self.class.declare_workers_not_supported!(message)
+  end
+
+  public
+  def initialize(params={})
+    super
+    config_init(@params)
+
+    # If we're running with a single thread we must enforce single-threaded concurrency by default
+    # Maybe in a future version we'll assume output plugins are threadsafe
+    @single_worker_mutex = Mutex.new
+  end
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def receive(event)
+    raise "#{self.class}#receive must be overidden"
+  end # def receive
+
+  public
+  # To be overriden in implementations
+  def multi_receive(events)
+    events.each {|event| receive(event) }
+  end
+
+  private
+  def output?(event)
+    # TODO: noop for now, remove this once we delete this call from all plugins
+    true
+  end # def output?
+end # class LogStash::Outputs::Base

data/lib/logstash/patches.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require "logstash/patches/bugfix_jruby_2558"
+require "logstash/patches/cabin"
+require "logstash/patches/profile_require_calls"
+require "logstash/patches/stronger_openssl_defaults"

data/lib/logstash/patches/bugfix_jruby_2558.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "logstash/environment"
+
+if LogStash::Environment.windows? && LogStash::Environment.jruby?
+  require "socket"
+  module JRubyBug2558SocketPeerAddrBugFix
+    def peeraddr(*args)
+      orig_peeraddr(*args).map do |v|
+        case v
+        when String
+          v.force_encoding(Encoding::UTF_8)
+        else
+          v
+        end
+      end
+    end
+  end
+
+  class << Socket
+    # Bugfix for jruby #2558
+    alias_method :orig_gethostname, :gethostname
+    def gethostname
+      return orig_gethostname.force_encoding(Encoding::UTF_8)
+    end
+  end
+
+  class TCPSocket
+    alias_method :orig_peeraddr, :peeraddr
+    include JRubyBug2558SocketPeerAddrBugFix
+  end
+  class UDPSocket
+    alias_method :orig_peeraddr, :peeraddr
+    include JRubyBug2558SocketPeerAddrBugFix
+  end
+end
+
+if LogStash::Environment.windows?
+  # make sure all strings pulled out of ENV are UTF8
+  class <<ENV
+    alias_method :orig_getter, :[]
+    def [](key)
+      case value = orig_getter(key)
+      when String
+        # dup is necessary since force_encoding is destructive
+        value.dup.force_encoding(Encoding::UTF_8)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/logstash/patches/cabin.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+if ENV["PROFILE_BAD_LOG_CALLS"] || ($DEBUGLIST || []).include?("log")
+  # Set PROFILE_BAD_LOG_CALLS=1 in your environment if you want
+  # to track down logger calls that cause performance problems
+  #
+  # Related research here:
+  #   https://github.com/jordansissel/experiments/tree/master/ruby/logger-string-vs-block
+  #
+  # Basically, the following is wastes tons of effort creating objects that are
+  # never used if the log level hides the log:
+  #
+  #     logger.debug("something happend", :what => Happened)
+  #
+  # This is shown to be 4x faster:
+  #
+  #     logger.debug(...) if logger.debug?
+  #
+  # I originally intended to use RubyParser and SexpProcessor to
+  # process all the logstash ruby code offline, but it was much
+  # faster to write this monkeypatch to warn as things are called.
+  require "cabin/mixins/logger"
+  module Cabin::Mixins::Logger
+    LEVELS.keys.each do |level|
+      m = "original_#{level}".to_sym
+      predicate = "#{level}?".to_sym
+      alias_method m, level
+      define_method(level) do |*args|
+        if !send(predicate)
+          warn("Unconditional log call", :location => caller[0])
+        end
+        send(m, *args)
+      end
+    end
+  end
+end # PROFILE_BAD_LOG_CALLS

data/lib/logstash/patches/profile_require_calls.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+if ($DEBUGLIST || []).include?("require")
+  ROOT = File.dirname(__FILE__)
+  module Kernel
+    alias_method :require_debug, :require
+
+    def require(path)
+      start = Time.now
+      result = require_debug(path)
+      duration = Time.now - start
+
+      origin = caller[1]
+      if origin =~ /rubygems\/custom_require/
+        origin = caller[3]
+        if origin.nil?
+          STDERR.puts "Unknown origin"
+          STDERR.puts caller.join("\n")
+        end
+      end
+      origin = origin.gsub(/:[0-9]+:in .*/, "") if origin
+
+      # Only print require() calls that did actual work.
+      # require() returns true on load, false if already loaded.
+      if result
+        source = caller[0]
+        #p source.include?("/lib/polyglot.rb:63:in `require'") => source
+        if source.include?("/lib/polyglot.rb:63:in `require'")
+          source = caller[1]
+        end
+
+        #target = $LOADED_FEATURES.grep(/#{path}/).first
+        #puts path
+        #puts caller.map { |c| "  #{c}" }.join("\n")
+        #fontsize = [10, duration * 48].max
+        puts "#{duration},#{path},#{source}"
+      end
+      #puts caller.map { |c| " => #{c}" }.join("\n")
+    end
+
+    alias_method :load_debug, :load
+
+    def load(path)
+      puts "load(\"#{path}\")"
+      return load_debug(path)
+    end
+  end
+end

data/lib/logstash/patches/rubygems.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+# monkey patch RubyGems to silence ffi warnings:
+#
+# WARN: Unresolved specs during Gem::Specification.reset:
+#       ffi (>= 0)
+# WARN: Clearing out unresolved specs.
+# Please report a bug if this causes problems.
+#
+# see https://github.com/elasticsearch/logstash/issues/2556 and https://github.com/rubygems/rubygems/issues/1070
+#
+# this code is from Rubygems v2.1.9 in JRuby 1.7.17. Per tickets this issue should be solved at JRuby >= 1.7.20.
+#
+# this method implementation works for Rubygems version 2.1.0 and up, verified up to 2.4.6
+if ::Gem::Version.new(::Gem::VERSION) >= ::Gem::Version.new("2.1.0") && ::Gem::Version.new(::Gem::VERSION) < ::Gem::Version.new("2.5.0")
+  class ::Gem::Specification
+    def self.reset
+      @@dirs = nil
+      ::Gem.pre_reset_hooks.each { |hook| hook.call }
+      @@all = nil
+      @@stubs = nil
+      _clear_load_cache
+      unresolved = unresolved_deps
+      unless unresolved.empty?
+        unless (unresolved.size == 1 && unresolved["ffi"])
+          w = "W" + "ARN"
+          warn "#{w}: Unresolved specs during Gem::Specification.reset:"
+          unresolved.values.each do |dep|
+            warn "      #{dep}"
+          end
+          warn "#{w}: Clearing out unresolved specs."
+          warn "Please report a bug if this causes problems."
+        end
+        unresolved.clear
+      end
+      ::Gem.post_reset_hooks.each { |hook| hook.call }
+    end
+  end
+end

data/lib/logstash/patches/stronger_openssl_defaults.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "openssl"
+
+# :nodoc:
+class OpenSSL::SSL::SSLContext
+  # Wrap SSLContext.new to a stronger default settings.
+  class << self
+    alias_method :orig_new, :new
+    def new(*args)
+      c = orig_new(*args)
+
+      # MRI nor JRuby seem to actually invoke `SSLContext#set_params` by
+      # default, which makes the default ciphers (and other settings) not
+      # actually defaults. Oops!
+      # To force this, and force our (hopefully more secure) defaults on
+      # all things using openssl in Ruby, we will invoke set_params
+      # on all new SSLContext objects.
+      c.set_params
+      c
+    end
+  end
+
+  # This cipher selection comes from https://wiki.mozilla.org/Security/Server_Side_TLS
+  MOZILLA_INTERMEDIATE_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
+
+  # Returns the value that should be used for the default SSLContext options
+  #
+  # This is a method instead of a constant because some constants (like
+  # OpenSSL::SSL::OP_NO_COMPRESSION) may not be available in all Ruby
+  # versions/platforms.
+  def self.__default_options
+    # ruby-core is refusing to patch ruby's default openssl settings to be more
+    # secure, so let's fix that here. The next few lines setting options and
+    # ciphers come from jmhodges' proposed patch
+    ssloptions = OpenSSL::SSL::OP_ALL
+ 
+    # TODO(sissel): JRuby doesn't have this. Maybe work on a fix?
+    if defined?(OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS)
+      ssloptions &= ~OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS
+    end
+
+    # TODO(sissel): JRuby doesn't have this. Maybe work on a fix?
+    if defined?(OpenSSL::SSL::OP_NO_COMPRESSION)
+      ssloptions |= OpenSSL::SSL::OP_NO_COMPRESSION
+    end
+
+    # Disable SSLv2 and SSLv3. They are insecure and highly discouraged.
+    ssloptions |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
+    ssloptions |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
+    ssloptions
+  end
+
+  # Overwriting the DEFAULT_PARAMS const idea from here: https://www.ruby-lang.org/en/news/2014/10/27/changing-default-settings-of-ext-openssl/
+  #
+  # This monkeypatch doesn't enforce a `VERIFY_MODE` on the SSLContext,
+  # SSLContext are both used for the client and the server implementation,
+  # If set the `verify_mode` to peer the server wont accept any connection,
+  # because it will try to verify the client certificate, this is a protocol
+  # details implemented at the plugin level.
+  #
+  # For more details see: https://github.com/elastic/logstash/issues/3657
+  remove_const(:DEFAULT_PARAMS) if const_defined?(:DEFAULT_PARAMS)
+  DEFAULT_PARAMS = {
+    :ssl_version => "TLS",
+    :ciphers => MOZILLA_INTERMEDIATE_CIPHERS,
+    :options => __default_options # Not a constant because it's computed at start-time.
+  }
+end

data/lib/logstash/pipeline.rb

@@ -0,0 +1,499 @@
+# encoding: utf-8
+require "thread"
+require "stud/interval"
+require "concurrent"
+require "logstash/namespace"
+require "logstash/errors"
+require "logstash/event"
+require "logstash/config/file"
+require "logstash/filters/base"
+require "logstash/inputs/base"
+require "logstash/outputs/base"
+require "logstash/config/cpu_core_strategy"
+require "logstash/util/defaults_printer"
+require "logstash/shutdown_watcher"
+require "logstash/util/wrapped_synchronous_queue"
+require "logstash/pipeline_reporter"
+require "logstash/output_delegator"
+
+module LogStash; class Pipeline
+  attr_reader :inputs, :filters, :outputs, :worker_threads, :events_consumed, :events_filtered, :reporter, :pipeline_id, :logger
+
+  DEFAULT_SETTINGS = {
+    :default_pipeline_workers => LogStash::Config::CpuCoreStrategy.maximum,
+    :pipeline_batch_size => 125,
+    :pipeline_batch_delay => 5, # in milliseconds
+    :flush_interval => 5, # in seconds
+    :flush_timeout_interval => 60, # in seconds
+    :debug_config => false 
+  }
+  MAX_INFLIGHT_WARN_THRESHOLD = 10_000
+
+  def initialize(config_str, settings = {})
+    @pipeline_id = settings[:pipeline_id] || self.object_id
+    @logger = Cabin::Channel.get(LogStash)
+    @settings = DEFAULT_SETTINGS.clone
+    settings.each {|setting, value| configure(setting, value) }
+    @reporter = LogStash::PipelineReporter.new(@logger, self)
+
+    @inputs = nil
+    @filters = nil
+    @outputs = nil
+
+    @worker_threads = []
+
+    grammar = LogStashConfigParser.new
+    @config = grammar.parse(config_str)
+    if @config.nil?
+      raise LogStash::ConfigurationError, grammar.failure_reason
+    end
+    # This will compile the config to ruby and evaluate the resulting code.
+    # The code will initialize all the plugins and define the
+    # filter and output methods.
+    code = @config.compile
+    # The config code is hard to represent as a log message...
+    # So just print it.
+    if @settings[:debug_config]
+      @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
+    end
+    begin
+      eval(code)
+    rescue => e
+      raise
+    end
+
+    @input_queue = LogStash::Util::WrappedSynchronousQueue.new
+    @events_filtered = Concurrent::AtomicFixnum.new(0)
+    @events_consumed = Concurrent::AtomicFixnum.new(0)
+
+    # We generally only want one thread at a time able to access pop/take/poll operations
+    # from this queue. We also depend on this to be able to block consumers while we snapshot
+    # in-flight buffers
+    @input_queue_pop_mutex = Mutex.new
+    @input_threads = []
+    # @ready requires thread safety since it is typically polled from outside the pipeline thread
+    @ready = Concurrent::AtomicBoolean.new(false)
+    @running = Concurrent::AtomicBoolean.new(false)
+    @flushing = Concurrent::AtomicReference.new(false)
+  end # def initialize
+
+  def ready?
+    @ready.value
+  end
+
+  def configure(setting, value)
+    @settings[setting] = value
+  end
+
+  def safe_pipeline_worker_count
+    default = DEFAULT_SETTINGS[:default_pipeline_workers]
+    thread_count = @settings[:pipeline_workers] #override from args "-w 8" or config
+    safe_filters, unsafe_filters = @filters.partition(&:threadsafe?)
+
+    if unsafe_filters.any?
+      plugins = unsafe_filters.collect { |f| f.class.config_name }
+      case thread_count
+      when nil
+        # user did not specify a worker thread count
+        # warn if the default is multiple
+
+        if default > 1
+          @logger.warn("Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads",
+                       :count_was => default, :filters => plugins)
+        end
+
+        1 # can't allow the default value to propagate if there are unsafe filters
+      when 0, 1
+        1
+      else
+        @logger.warn("Warning: Manual override - there are filters that might not work with multiple worker threads",
+                     :worker_threads => thread_count, :filters => plugins)
+        thread_count # allow user to force this even if there are unsafe filters
+      end
+    else
+      thread_count || default
+    end
+  end
+
+  def filters?
+    return @filters.any?
+  end
+
+  def run
+    LogStash::Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
+    @logger.terminal(LogStash::Util::DefaultsPrinter.print(@settings))
+
+    start_workers
+
+    @logger.info("Pipeline started")
+    @logger.terminal("Logstash startup completed")
+
+    # Block until all inputs have stopped
+    # Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
+
+    transition_to_running
+    start_flusher # Launches a non-blocking thread for flush events
+    wait_inputs
+    transition_to_stopped
+
+    @logger.info("Input plugins stopped! Will shutdown filter/output workers.")
+
+    shutdown_flusher
+    shutdown_workers
+
+    @logger.info("Pipeline shutdown complete.")
+    @logger.terminal("Logstash shutdown completed")
+
+    # exit code
+    return 0
+  end # def run
+
+  def transition_to_running
+    @running.make_true
+  end
+
+  def transition_to_stopped
+    @running.make_false
+  end
+
+  def running?
+    @running.true?
+  end
+
+  def stopped?
+    @running.false?
+  end
+
+  def start_workers
+    @inflight_batches = {}
+
+    @worker_threads.clear # In case we're restarting the pipeline
+    begin
+      start_inputs
+      @outputs.each {|o| o.register }
+      @filters.each {|f| f.register}
+
+      pipeline_workers = safe_pipeline_worker_count
+      batch_size = @settings[:pipeline_batch_size]
+      batch_delay = @settings[:pipeline_batch_delay]
+      max_inflight = batch_size * pipeline_workers
+      @logger.info("Starting pipeline",
+                   :id => self.pipeline_id,
+                   :pipeline_workers => pipeline_workers,
+                   :batch_size => batch_size,
+                   :batch_delay => batch_delay,
+                   :max_inflight => max_inflight)
+      if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
+        @logger.warn "CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})"
+      end
+
+      pipeline_workers.times do |t|
+        @worker_threads << Thread.new do
+          LogStash::Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
+          worker_loop(batch_size, batch_delay)
+        end
+      end
+    ensure
+      # it is important to garantee @ready to be true after the startup sequence has been completed
+      # to potentially unblock the shutdown method which may be waiting on @ready to proceed
+      @ready.make_true
+    end
+  end
+
+  # Main body of what a worker thread does
+  # Repeatedly takes batches off the queu, filters, then outputs them
+  def worker_loop(batch_size, batch_delay)
+    running = true
+
+    while running
+      # To understand the purpose behind this synchronize please read the body of take_batch
+      input_batch, signal = @input_queue_pop_mutex.synchronize { take_batch(batch_size, batch_delay) }
+      running = false if signal == LogStash::SHUTDOWN
+
+      @events_consumed.increment(input_batch.size)
+
+      filtered_batch = filter_batch(input_batch)
+
+      if signal # Flush on SHUTDOWN or FLUSH
+        flush_options = (signal == LogStash::SHUTDOWN) ? {:final => true} : {}
+        flush_filters_to_batch(filtered_batch, flush_options)
+      end
+
+      @events_filtered.increment(filtered_batch.size)
+
+      output_batch(filtered_batch)
+
+      inflight_batches_synchronize { set_current_thread_inflight_batch(nil) }
+    end
+  end
+
+  def take_batch(batch_size, batch_delay)
+    batch = []
+    # Since this is externally synchronized in `worker_look` wec can guarantee that the visibility of an insight batch
+    # guaranteed to be a full batch not a partial batch
+    set_current_thread_inflight_batch(batch)
+
+    signal = false
+    batch_size.times do |t|
+      event = (t == 0) ? @input_queue.take : @input_queue.poll(batch_delay)
+
+      if event.nil?
+        next
+      elsif event == LogStash::SHUTDOWN || event == LogStash::FLUSH
+        # We MUST break here. If a batch consumes two SHUTDOWN events
+        # then another worker may have its SHUTDOWN 'stolen', thus blocking
+        # the pipeline. We should stop doing work after flush as well.
+        signal = event
+        break
+      else
+        batch << event
+      end
+    end
+
+    [batch, signal]
+  end
+
+  def filter_batch(batch)
+    batch.reduce([]) do |acc,e|
+      if e.is_a?(LogStash::Event)
+        filtered = filter_func(e)
+        filtered.each {|fe| acc << fe unless fe.cancelled?}
+      end
+      acc
+    end
+  rescue Exception => e
+    # Plugins authors should manage their own exceptions in the plugin code
+    # but if an exception is raised up to the worker thread they are considered
+    # fatal and logstash will not recover from this situation.
+    #
+    # Users need to check their configuration or see if there is a bug in the
+    # plugin.
+    @logger.error("Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash.",
+                  "exception" => e, "backtrace" => e.backtrace)
+    raise
+  end
+
+  # Take an array of events and send them to the correct output
+  def output_batch(batch)
+    # Build a mapping of { output_plugin => [events...]}
+    outputs_events = batch.reduce(Hash.new { |h, k| h[k] = [] }) do |acc, event|
+      # We ask the AST to tell us which outputs to send each event to
+      # Then, we stick it in the correct bin
+
+      # output_func should never return anything other than an Array but we have lots of legacy specs
+      # that monkeypatch it and return nil. We can deprecate  "|| []" after fixing these specs
+      outputs_for_event = output_func(event) || []
+
+      outputs_for_event.each { |output| acc[output] << event }
+      acc
+    end
+
+    # Now that we have our output to event mapping we can just invoke each output
+    # once with its list of events
+    outputs_events.each { |output, events| output.multi_receive(events) }
+  end
+
+  def set_current_thread_inflight_batch(batch)
+    @inflight_batches[Thread.current] = batch
+  end
+
+  def inflight_batches_synchronize
+    @input_queue_pop_mutex.synchronize do
+      yield(@inflight_batches)
+    end
+  end
+
+  def wait_inputs
+    @input_threads.each(&:join)
+  end
+
+  def start_inputs
+    moreinputs = []
+    @inputs.each do |input|
+      if input.threadable && input.threads > 1
+        (input.threads - 1).times do |i|
+          moreinputs << input.clone
+        end
+      end
+    end
+    @inputs += moreinputs
+
+    @inputs.each do |input|
+      input.register
+      start_input(input)
+    end
+  end
+
+  def start_input(plugin)
+    @input_threads << Thread.new { inputworker(plugin) }
+  end
+
+  def inputworker(plugin)
+    LogStash::Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
+    begin
+      plugin.run(@input_queue)
+    rescue => e
+      if plugin.stop?
+        @logger.debug("Input plugin raised exception during shutdown, ignoring it.",
+                      :plugin => plugin.class.config_name, :exception => e,
+                      :backtrace => e.backtrace)
+        return
+      end
+
+      # otherwise, report error and restart
+      if @logger.debug?
+        @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
+                             :plugin => plugin.inspect, :error => e.to_s,
+                             :exception => e.class,
+                             :stacktrace => e.backtrace.join("\n")))
+      else
+        @logger.error(I18n.t("logstash.pipeline.worker-error",
+                             :plugin => plugin.inspect, :error => e))
+      end
+
+      # Assuming the failure that caused this exception is transient,
+      # let's sleep for a bit and execute #run again
+      sleep(1)
+      retry
+    ensure
+      plugin.do_close
+    end
+  end # def inputworker
+
+  # initiate the pipeline shutdown sequence
+  # this method is intended to be called from outside the pipeline thread
+  # @param before_stop [Proc] code block called before performing stop operation on input plugins
+  def shutdown(&before_stop)
+    # shutdown can only start once the pipeline has completed its startup.
+    # avoid potential race conditoon between the startup sequence and this
+    # shutdown method which can be called from another thread at any time
+    sleep(0.1) while !ready?
+
+    # TODO: should we also check against calling shutdown multiple times concurently?
+
+    before_stop.call if block_given?
+
+    @logger.info "Closing inputs"
+    @inputs.each(&:do_stop)
+    @logger.info "Closed inputs"
+  end # def shutdown
+
+  # After `shutdown` is called from an external thread this is called from the main thread to
+  # tell the worker threads to stop and then block until they've fully stopped
+  # This also stops all filter and output plugins
+  def shutdown_workers
+    # Each worker thread will receive this exactly once!
+    @worker_threads.each do |t|
+      @logger.debug("Pushing shutdown", :thread => t)
+      @input_queue.push(LogStash::SHUTDOWN)
+    end
+
+    @worker_threads.each do |t|
+      @logger.debug("Shutdown waiting for worker thread #{t}")
+      t.join
+    end
+
+    @filters.each(&:do_close)
+    @outputs.each(&:do_close)
+  end
+
+  def plugin(plugin_type, name, *args)
+    args << {} if args.empty?
+
+    klass = LogStash::Plugin.lookup(plugin_type, name)
+
+    if plugin_type == "output"
+      LogStash::OutputDelegator.new(@logger, klass, default_output_workers, *args)
+    else
+      klass.new(*args)
+    end
+  end
+
+  def default_output_workers
+    @settings[:pipeline_workers] || @settings[:default_pipeline_workers]
+  end
+
+  # for backward compatibility in devutils for the rspec helpers, this method is not used
+  # in the pipeline anymore.
+  def filter(event, &block)
+    # filter_func returns all filtered events, including cancelled ones
+    filter_func(event).each { |e| block.call(e) }
+  end
+
+
+  # perform filters flush and yeild flushed event to the passed block
+  # @param options [Hash]
+  # @option options [Boolean] :final => true to signal a final shutdown flush
+  def flush_filters(options = {}, &block)
+    flushers = options[:final] ? @shutdown_flushers : @periodic_flushers
+
+    flushers.each do |flusher|
+      flusher.call(options, &block)
+    end
+  end
+
+  def start_flusher
+    # Invariant to help detect improper initialization
+    raise "Attempted to start flusher on a stopped pipeline!" if stopped?
+
+    @flusher_thread = Thread.new do
+      while Stud.stoppable_sleep(5, 0.1) { stopped? }
+        flush
+        break if stopped?
+      end
+    end
+  end
+
+  def shutdown_flusher
+    @flusher_thread.join
+  end
+
+  def flush
+    if @flushing.compare_and_set(false, true)
+      @logger.debug? && @logger.debug("Pushing flush onto pipeline")
+      @input_queue.push(LogStash::FLUSH)
+    end
+  end
+
+  # perform filters flush into the output queue
+  # @param options [Hash]
+  # @option options [Boolean] :final => true to signal a final shutdown flush
+  def flush_filters_to_batch(batch, options = {})
+    flush_filters(options) do |event|
+      unless event.cancelled?
+        @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
+        batch << event
+      end
+    end
+
+    @flushing.set(false)
+  end # flush_filters_to_output!
+
+  def plugin_threads_info
+    input_threads = @input_threads.select {|t| t.alive? }
+    worker_threads = @worker_threads.select {|t| t.alive? }
+    (input_threads + worker_threads).map {|t| LogStash::Util.thread_info(t) }
+  end
+
+  def stalling_threads_info
+    plugin_threads_info
+      .reject {|t| t["blocked_on"] } # known benign blocking statuses
+      .each {|t| t.delete("backtrace") }
+      .each {|t| t.delete("blocked_on") }
+      .each {|t| t.delete("status") }
+  end
+
+  # Sometimes we log stuff that will dump the pipeline which may contain
+  # sensitive information (like the raw syntax tree which can contain passwords)
+  # We want to hide most of what's in here
+  def inspect
+    {
+      :pipeline_id => @pipeline_id,
+      :settings => @settings.inspect,
+      :ready => @ready,
+      :running => @running,
+      :flushing => @flushing
+    }
+  end
+
+end end