logstash-core 1.5.0.beta2-java

data/lib/logstash/patches/cabin.rb

@@ -0,0 +1,34 @@
+if ENV["PROFILE_BAD_LOG_CALLS"] || ($DEBUGLIST || []).include?("log")
+  # Set PROFILE_BAD_LOG_CALLS=1 in your environment if you want
+  # to track down logger calls that cause performance problems
+  #
+  # Related research here:
+  #   https://github.com/jordansissel/experiments/tree/master/ruby/logger-string-vs-block
+  #
+  # Basically, the following is wastes tons of effort creating objects that are
+  # never used if the log level hides the log:
+  #
+  #     logger.debug("something happend", :what => Happened)
+  #
+  # This is shown to be 4x faster:
+  #
+  #     logger.debug(...) if logger.debug?
+  #
+  # I originally intended to use RubyParser and SexpProcessor to
+  # process all the logstash ruby code offline, but it was much
+  # faster to write this monkeypatch to warn as things are called.
+  require "cabin/mixins/logger"
+  module Cabin::Mixins::Logger
+    LEVELS.keys.each do |level|
+      m = "original_#{level}".to_sym
+      predicate = "#{level}?".to_sym
+      alias_method m, level
+      define_method(level) do |*args|
+        if !send(predicate)
+          warn("Unconditional log call", :location => caller[0])
+        end
+        send(m, *args)
+      end
+    end
+  end
+end # PROFILE_BAD_LOG_CALLS

data/lib/logstash/patches/profile_require_calls.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+if ($DEBUGLIST || []).include?("require")
+  ROOT = File.dirname(__FILE__)
+  module Kernel
+    alias_method :require_debug, :require
+
+    def require(path)
+      start = Time.now
+      result = require_debug(path)
+      duration = Time.now - start
+
+      origin = caller[1]
+      if origin =~ /rubygems\/custom_require/
+        origin = caller[3]
+        if origin.nil?
+          STDERR.puts "Unknown origin"
+          STDERR.puts caller.join("\n")
+        end
+      end
+      origin = origin.gsub(/:[0-9]+:in .*/, "") if origin
+
+      # Only print require() calls that did actual work.
+      # require() returns true on load, false if already loaded.
+      if result
+        source = caller[0]
+        #p source.include?("/lib/polyglot.rb:63:in `require'") => source
+        if source.include?("/lib/polyglot.rb:63:in `require'")
+          source = caller[1]
+        end
+
+        #target = $LOADED_FEATURES.grep(/#{path}/).first
+        #puts path
+        #puts caller.map { |c| "  #{c}" }.join("\n")
+        #fontsize = [10, duration * 48].max
+        puts "#{duration},#{path},#{source}"
+      end
+      #puts caller.map { |c| " => #{c}" }.join("\n")
+    end
+
+    alias_method :load_debug, :load
+
+    def load(path)
+      puts "load(\"#{path}\")"
+      return load_debug(path)
+    end
+  end
+end

data/lib/logstash/pipeline.rb

@@ -0,0 +1,305 @@
+# encoding: utf-8
+require "thread" #
+require "stud/interval"
+require "logstash/namespace"
+require "logstash/errors"
+require "logstash/event"
+require "logstash/config/file"
+require "logstash/filters/base"
+require "logstash/inputs/base"
+require "logstash/outputs/base"
+
+class LogStash::Pipeline
+
+  FLUSH_EVENT = LogStash::FlushEvent.new
+
+  def initialize(configstr)
+    @logger = Cabin::Channel.get(LogStash)
+    grammar = LogStashConfigParser.new
+    @config = grammar.parse(configstr)
+    if @config.nil?
+      raise LogStash::ConfigurationError, grammar.failure_reason
+    end
+
+    # This will compile the config to ruby and evaluate the resulting code.
+    # The code will initialize all the plugins and define the
+    # filter and output methods.
+    code = @config.compile
+    # The config code is hard to represent as a log message...
+    # So just print it.
+    @logger.debug? && @logger.debug("Compiled pipeline code:\n#{code}")
+    begin
+      eval(code)
+    rescue => e
+      raise
+    end
+
+    @input_to_filter = SizedQueue.new(20)
+
+    # If no filters, pipe inputs directly to outputs
+    if !filters?
+      @filter_to_output = @input_to_filter
+    else
+      @filter_to_output = SizedQueue.new(20)
+    end
+    @settings = {
+      "filter-workers" => 1,
+    }
+  end # def initialize
+
+  def ready?
+    return @ready
+  end
+
+  def started?
+    return @started
+  end
+
+  def configure(setting, value)
+    if setting == "filter-workers"
+      # Abort if we have any filters that aren't threadsafe
+      if value > 1 && @filters.any? { |f| !f.threadsafe? }
+        plugins = @filters.select { |f| !f.threadsafe? }.collect { |f| f.class.config_name }
+        raise LogStash::ConfigurationError, "Cannot use more than 1 filter worker because the following plugins don't work with more than one worker: #{plugins.join(", ")}"
+      end
+    end
+    @settings[setting] = value
+  end
+
+  def filters?
+    return @filters.any?
+  end
+
+  def run
+    @started = true
+    @input_threads = []
+
+    start_inputs
+    start_filters if filters?
+    start_outputs
+
+    @ready = true
+
+    @logger.info("Pipeline started")
+    @logger.terminal("Logstash startup completed")
+
+    wait_inputs
+
+    if filters?
+      shutdown_filters
+      wait_filters
+      flush_filters_to_output!(:final => true)
+    end
+
+    shutdown_outputs
+    wait_outputs
+
+    @logger.info("Pipeline shutdown complete.")
+    @logger.terminal("Logstash shutdown completed")
+
+    # exit code
+    return 0
+  end # def run
+
+  def wait_inputs
+    @input_threads.each(&:join)
+  rescue Interrupt
+    # rbx does weird things during do SIGINT that I haven't debugged
+    # so we catch Interrupt here and signal a shutdown. For some reason the
+    # signal handler isn't invoked it seems? I dunno, haven't looked much into
+    # it.
+    shutdown
+  end
+
+  def shutdown_filters
+    @flusher_lock.synchronize { @flusher_thread.kill }
+    @input_to_filter.push(LogStash::ShutdownEvent.new)
+  end
+
+  def wait_filters
+    @filter_threads.each(&:join) if @filter_threads
+  end
+
+  def shutdown_outputs
+    # nothing, filters will do this
+    @filter_to_output.push(LogStash::ShutdownEvent.new)
+  end
+
+  def wait_outputs
+    # Wait for the outputs to stop
+    @output_threads.each(&:join)
+  end
+
+  def start_inputs
+    moreinputs = []
+    @inputs.each do |input|
+      if input.threadable && input.threads > 1
+        (input.threads-1).times do |i|
+          moreinputs << input.clone
+        end
+      end
+    end
+    @inputs += moreinputs
+
+    @inputs.each do |input|
+      input.register
+      start_input(input)
+    end
+  end
+
+  def start_filters
+    @filters.each(&:register)
+    @filter_threads = @settings["filter-workers"].times.collect do
+      Thread.new { filterworker }
+    end
+
+    @flusher_lock = Mutex.new
+    @flusher_thread = Thread.new { Stud.interval(5) { @flusher_lock.synchronize { @input_to_filter.push(FLUSH_EVENT) } } }
+  end
+
+  def start_outputs
+    @outputs.each(&:register)
+    @output_threads = [
+      Thread.new { outputworker }
+    ]
+  end
+
+  def start_input(plugin)
+    @input_threads << Thread.new { inputworker(plugin) }
+  end
+
+  def inputworker(plugin)
+    LogStash::Util::set_thread_name("<#{plugin.class.config_name}")
+    begin
+      plugin.run(@input_to_filter)
+    rescue LogStash::ShutdownSignal
+      return
+    rescue => e
+      if @logger.debug?
+        @logger.error(I18n.t("logstash.pipeline.worker-error-debug",
+                             :plugin => plugin.inspect, :error => e.to_s,
+                             :exception => e.class,
+                             :stacktrace => e.backtrace.join("\n")))
+      else
+        @logger.error(I18n.t("logstash.pipeline.worker-error",
+                             :plugin => plugin.inspect, :error => e))
+      end
+      puts e.backtrace if @logger.debug?
+      plugin.teardown
+      sleep 1
+      retry
+    end
+  rescue LogStash::ShutdownSignal
+    # nothing
+  ensure
+    plugin.teardown
+  end # def inputworker
+
+  def filterworker
+    LogStash::Util::set_thread_name("|worker")
+    begin
+      while true
+        event = @input_to_filter.pop
+
+        case event
+        when LogStash::Event
+          # use events array to guarantee ordering of origin vs created events
+          # where created events are emitted by filters like split or metrics
+          events = []
+          filter(event) { |newevent| events << newevent }
+          events.each { |event| @filter_to_output.push(event) }
+        when LogStash::FlushEvent
+          # handle filter flushing here so that non threadsafe filters (thus only running one filterworker)
+          # don't have to deal with thread safety implementing the flush method
+          @flusher_lock.synchronize { flush_filters_to_output! }
+        when LogStash::ShutdownEvent
+          # pass it down to any other filterworker and stop this worker
+          @input_to_filter.push(event)
+          break
+        end
+      end
+    rescue => e
+      @logger.error("Exception in filterworker", "exception" => e, "backtrace" => e.backtrace)
+    end
+
+    @filters.each(&:teardown)
+  end # def filterworker
+
+  def outputworker
+    LogStash::Util::set_thread_name(">output")
+    @outputs.each(&:worker_setup)
+
+    while true
+      event = @filter_to_output.pop
+      break if event.is_a?(LogStash::ShutdownEvent)
+      output(event)
+    end # while true
+
+    @outputs.each do |output|
+      output.worker_plugins.each(&:teardown)
+    end
+  end # def outputworker
+
+  # Shutdown this pipeline.
+  #
+  # This method is intended to be called from another thread
+  def shutdown
+    @input_threads.each do |thread|
+      # Interrupt all inputs
+      @logger.info("Sending shutdown signal to input thread",
+                   :thread => thread)
+      thread.raise(LogStash::ShutdownSignal)
+      begin
+        thread.wakeup # in case it's in blocked IO or sleeping
+      rescue ThreadError
+      end
+
+      # Sometimes an input is stuck in a blocking I/O
+      # so we need to tell it to teardown directly
+      @inputs.each do |input|
+        input.teardown
+      end
+    end
+
+    # No need to send the ShutdownEvent to the filters/outputs nor to wait for
+    # the inputs to finish, because in the #run method we wait for that anyway.
+  end # def shutdown
+
+  def plugin(plugin_type, name, *args)
+    args << {} if args.empty?
+    klass = LogStash::Plugin.lookup(plugin_type, name)
+    return klass.new(*args)
+  end
+
+  def filter(event, &block)
+    @filter_func.call(event, &block)
+  end
+
+  def output(event)
+    @output_func.call(event)
+  end
+
+  # perform filters flush and yeild flushed event to the passed block
+  # @param options [Hash]
+  # @option options [Boolean] :final => true to signal a final shutdown flush
+  def flush_filters(options = {}, &block)
+    flushers = options[:final] ? @shutdown_flushers : @periodic_flushers
+
+    flushers.each do |flusher|
+      flusher.call(options, &block)
+    end
+  end
+
+  # perform filters flush into the output queue
+  # @param options [Hash]
+  # @option options [Boolean] :final => true to signal a final shutdown flush
+  def flush_filters_to_output!(options = {})
+    flush_filters(options) do |event|
+      unless event.cancelled?
+        @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
+        @filter_to_output.push(event)
+      end
+    end
+  end # flush_filters_to_output!
+
+end # class Pipeline

data/lib/logstash/plugin.rb

@@ -0,0 +1,177 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "cabin"
+
+class LogStash::Plugin
+  attr_accessor :params
+  attr_accessor :logger
+
+  NL = "\n"
+
+  public
+  def hash
+    params.hash ^
+    self.class.name.hash
+  end
+
+  public
+  def eql?(other)
+    self.class.name == other.class.name && @params == other.params
+  end
+
+  public
+  def initialize(params=nil)
+    @params = params
+    @logger = Cabin::Channel.get(LogStash)
+  end
+
+  # This method is called when someone or something wants this plugin to shut
+  # down. When you successfully shutdown, you must call 'finished'
+  # You must also call 'super' in any subclasses.
+  public
+  def shutdown(queue)
+    # By default, shutdown is assumed a no-op for all plugins.
+    # If you need to take special efforts to shutdown (like waiting for
+    # an operation to complete, etc)
+    teardown
+    @logger.info("Received shutdown signal", :plugin => self)
+
+    @shutdown_queue = queue
+    if @plugin_state == :finished
+      finished
+    else
+      @plugin_state = :terminating
+    end
+  end # def shutdown
+
+  # You should call this method when you (the plugin) are done with work
+  # forever.
+  public
+  def finished
+    # TODO(sissel): I'm not sure what I had planned for this shutdown_queue
+    # thing
+    if @shutdown_queue
+      @logger.info("Sending shutdown event to agent queue", :plugin => self)
+      @shutdown_queue << self
+    end
+
+    if @plugin_state != :finished
+      @logger.info("Plugin is finished", :plugin => self)
+      @plugin_state = :finished
+    end
+  end # def finished
+
+  # Subclasses should implement this teardown method if you need to perform any
+  # special tasks during shutdown (like flushing, etc.)
+  public
+  def teardown
+    # nothing by default
+    finished
+  end
+
+  # This method is called when a SIGHUP triggers a reload operation
+  public
+  def reload
+    # Do nothing by default
+  end
+
+  public
+  def finished?
+    return @plugin_state == :finished
+  end # def finished?
+
+  public
+  def running?
+    return @plugin_state != :finished
+  end # def finished?
+
+  public
+  def terminating?
+    return @plugin_state == :terminating
+  end # def terminating?
+
+  public
+  def to_s
+    return "#{self.class.name}: #{@params}"
+  end
+
+  protected
+  def update_watchdog(state)
+    Thread.current[:watchdog] = Time.now
+    Thread.current[:watchdog_state] = state
+  end
+
+  protected
+  def clear_watchdog
+    Thread.current[:watchdog] = nil
+    Thread.current[:watchdog_state] = nil
+  end
+
+  public
+  def inspect
+    if !@config.nil?
+      description = @config \
+        .select { |k,v| !v.nil? && (v.respond_to?(:empty?) && !v.empty?) } \
+        .collect { |k,v| "#{k}=>#{v.inspect}" }
+      return "<#{self.class.name} #{description.join(", ")}>"
+    else
+      return "<#{self.class.name} --->"
+    end
+  end
+
+  # Look up a plugin by type and name.
+  public
+  def self.lookup(type, name)
+    path = "logstash/#{type}s/#{name}"
+
+    # first check if plugin already exists in namespace and continue to next step if not
+    begin
+      return namespace_lookup(type, name)
+    rescue NameError
+      logger.debug("Plugin not defined in namespace, checking for plugin file", :type => type, :name => name, :path => path)
+    end
+
+    # try to load the plugin file. ex.: lookup("filter", "grok") will require logstash/filters/grok
+    require(path)
+
+    # check again if plugin is now defined in namespace after the require
+    namespace_lookup(type, name)
+  rescue LoadError, NameError => e
+    raise(LogStash::PluginLoadingError, I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :path => path, :error => e.to_s))
+  end
+
+  private
+
+  # lookup a plugin by type and name in the existing LogStash module namespace
+  # ex.: namespace_lookup("filter", "grok") looks for LogStash::Filters::Grok
+  # @param type [String] plugin type, "input", "ouput", "filter"
+  # @param name [String] plugin name, ex.: "grok"
+  # @return [Class] the plugin class or raises NameError
+  # @raise NameError if plugin class does not exist or is invalid
+  def self.namespace_lookup(type, name)
+    type_const = "#{type.capitalize}s"
+    namespace = LogStash.const_get(type_const)
+    # the namespace can contain constants which are not for plugins classes (do not respond to :config_name)
+    # namespace.constants is the shallow collection of all constants symbols in namespace
+    # note that below namespace.const_get(c) should never result in a NameError since c is from the constants collection
+    klass_sym = namespace.constants.find { |c| is_a_plugin?(namespace.const_get(c), name) }
+    klass = klass_sym && namespace.const_get(klass_sym)
+    raise(NameError) unless klass
+    klass
+  end
+
+  # check if klass is a valid plugin for name
+  # @param klass [Class] plugin class
+  # @param name [String] plugin name
+  # @return [Boolean] true if klass is a valid plugin for name
+  def self.is_a_plugin?(klass, name)
+    klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
+  end
+
+  # @return [Cabin::Channel] logger channel for class methods
+  def self.logger
+    @logger ||= Cabin::Channel.get(LogStash)
+  end
+end # class LogStash::Plugin