logstash-core 2.2.4.snapshot1

data/lib/logstash/java_integration.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require "java"
+
+# this is mainly for usage with JrJackson json parsing in :raw mode which genenerates
+# Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.
+# these object already quacks like their Ruby equivalents Array and Hash but they will
+# not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
+# both classes everywhere. see LogStash::JSon.
+
+class Array
+  # enable class equivalence between Array and ArrayList
+  # so that ArrayList will work with case o when Array ...
+  def self.===(other)
+    return true if other.is_a?(Java::JavaUtil::Collection)
+    super
+  end
+end
+
+class Hash
+  # enable class equivalence between Hash and LinkedHashMap
+  # so that LinkedHashMap will work with case o when Hash ...
+  def self.===(other)
+    return true if other.is_a?(Java::JavaUtil::Map)
+    super
+  end
+end
+
+# map_mixin to patch LinkedHashMap and HashMap. it must be done directly on the classes,
+# using a module mixin does not work, and injecting in the Map interface does not work either
+# but injecting in the class works.
+
+map_mixin = lambda do
+  # this is a temporary fix to solve a bug in JRuby where classes implementing the Map interface, like LinkedHashMap
+  # have a bug in the has_key? method that is implemented in the Enumerable module that is somehow mixed in the Map interface.
+  # this bug makes has_key? (and all its aliases) return false for a key that has a nil value.
+  # Only LinkedHashMap is patched here because patching the Map interface is not working.
+  # TODO find proper fix, and submit upstream
+  # releavant JRuby files:
+  # https://github.com/jruby/jruby/blob/master/core/src/main/ruby/jruby/java/java_ext/java.util.rb
+  # https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/java/proxies/MapJavaProxy.java
+  def has_key?(key)
+    self.containsKey(key)
+  end
+  alias_method :include?, :has_key?
+  alias_method :member?, :has_key?
+  alias_method :key?, :has_key?
+
+  # Java 8 Map implements a merge method with a different signature from
+  # the Ruby Hash#merge. see https://github.com/jruby/jruby/issues/1249
+  # this can be removed when fixed upstream
+  if ENV_JAVA['java.specification.version'] >= '1.8'
+    def merge(other)
+      dup.merge!(other)
+    end
+  end
+end
+
+Java::JavaUtil::LinkedHashMap.module_exec(&map_mixin)
+Java::JavaUtil::HashMap.module_exec(&map_mixin)
+
+module java::util::Map
+  # have Map objects like LinkedHashMap objects report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Hash
+    super
+  end
+end
+
+module java::util::Collection
+  # have Collections objects like ArrayList report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Array
+    super
+  end
+
+  # support the Ruby Array delete method on a Java Collection
+  def delete(o)
+    self.removeAll([o]) ? o : block_given? ? yield : nil
+  end
+
+  def compact
+    duped = Java::JavaUtil::ArrayList.new(self)
+    duped.compact!
+    duped
+  end
+
+  def compact!
+    size_before = self.size
+    self.removeAll(java::util::Collections.singleton(nil))
+    if size_before == self.size
+      nil
+    else
+      self
+    end
+  end
+
+  # support the Ruby intersection method on Java Collection
+  def &(other)
+    # transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array intersection contract
+    duped = Java::JavaUtil::LinkedHashSet.new(self)
+    duped.retainAll(other)
+    duped
+  end
+
+  # support the Ruby union method on Java Collection
+  def |(other)
+    # transform self into a LinkedHashSet to remove duplicates and preserve order as defined by the Ruby Array union contract
+    duped = Java::JavaUtil::LinkedHashSet.new(self)
+    duped.addAll(other)
+    duped
+  end
+
+  def inspect
+    "<#{self.class.name}:#{self.hashCode} #{self.to_a(&:inspect)}>"
+  end
+end

data/lib/logstash/json.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "logstash/environment"
+require "logstash/errors"
+if LogStash::Environment.jruby?
+  require "jrjackson"
+  require "logstash/java_integration"
+else
+  require  "oj"
+end
+
+module LogStash
+  module Json
+    class ParserError < LogStash::Error; end
+    class GeneratorError < LogStash::Error; end
+
+    extend self
+
+    ### MRI
+
+    def mri_load(data, options = {})
+      Oj.load(data)
+    rescue Oj::ParseError => e
+      raise LogStash::Json::ParserError.new(e.message)
+    end
+
+    def mri_dump(o)
+      Oj.dump(o, :mode => :compat, :use_to_json => true)
+    rescue => e
+      raise LogStash::Json::GeneratorError.new(e.message)
+    end
+
+    ### JRuby
+
+    def jruby_load(data, options = {})
+      # TODO [guyboertje] remove these comments in 5.0
+      # options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
+
+      JrJackson::Ruby.parse(data, options)
+
+    rescue JrJackson::ParseError => e
+      raise LogStash::Json::ParserError.new(e.message)
+    end
+
+    def jruby_dump(o)
+      # TODO [guyboertje] remove these comments in 5.0
+      # test for enumerable here to work around an omission in JrJackson::Json.dump to
+      # also look for Java::JavaUtil::ArrayList, see TODO submit issue
+      # o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
+
+      JrJackson::Base.generate(o, {})
+
+    rescue => e
+      raise LogStash::Json::GeneratorError.new(e.message)
+    end
+
+    prefix = LogStash::Environment.jruby? ? "jruby" : "mri"
+    alias_method :load, "#{prefix}_load".to_sym
+    alias_method :dump, "#{prefix}_dump".to_sym
+
+  end
+end

data/lib/logstash/logging.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+require "logger"
+
+class LogStash::Logger
+  attr_accessor :target
+
+  public
+  def initialize(*args)
+    super()
+
+    #self[:program] = File.basename($0)
+    #subscribe(::Logger.new(*args))
+    @target = args[0]
+    @channel = Cabin::Channel.get(LogStash)
+
+    # lame hack until cabin's smart enough not to doubley-subscribe something.
+    # without this subscription count check, running the test suite
+    # causes Cabin to subscribe to STDOUT maaaaaany times.
+    subscriptions = @channel.instance_eval { @subscribers.count }
+    @channel.subscribe(@target) unless subscriptions > 0
+
+    # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
+    @level = $DEBUG ? :debug : :warn
+    if ENV["LOGSTASH_DEBUG"]
+      @level = :debug
+    end
+
+    # Direct metrics elsewhere.
+    @channel.metrics.channel = Cabin::Channel.new
+  end # def initialize
+
+  # Delegation
+  def level=(value) @channel.level = value; end
+  def debug(*args); @channel.debug(*args); end
+  def debug?(*args); @channel.debug?(*args); end
+  def info(*args); @channel.info(*args); end
+  def info?(*args); @channel.info?(*args); end
+  def warn(*args); @channel.warn(*args); end
+  def warn?(*args); @channel.warn?(*args); end
+  def error(*args); @channel.error(*args); end
+  def error?(*args); @channel.error?(*args); end
+  def fatal(*args); @channel.fatal(*args); end
+  def fatal?(*args); @channel.fatal?(*args); end
+
+  def self.setup_log4j(logger)
+    require "java"
+
+    properties = java.util.Properties.new
+    log4j_level = "WARN"
+    case logger.level
+      when :debug
+        log4j_level = "DEBUG"
+      when :info
+        log4j_level = "INFO"
+      when :warn
+        log4j_level = "WARN"
+    end # case level
+    properties.setProperty("log4j.rootLogger", "#{log4j_level},logstash")
+
+    # TODO(sissel): This is a shitty hack to work around the fact that
+    # LogStash::Logger isn't used anymore. We should fix that.
+    target = logger.instance_eval { @subscribers }.values.first.instance_eval { @io }
+    case target
+      when STDOUT
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.out")
+      when STDERR
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.err")
+      when target.is_a?(File)
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.FileAppender")
+        properties.setProperty("log4j.appender.logstash.File", target.path)
+      else
+        properties.setProperty("log4j.appender.logstash", "org.apache.log4j.varia.NullAppender")
+    end # case target
+
+    properties.setProperty("log4j.appender.logstash.layout",
+                  "org.apache.log4j.PatternLayout")
+    properties.setProperty("log4j.appender.logstash.layout.conversionPattern",
+                  "log4j, [%d{yyyy-MM-dd}T%d{HH:mm:ss.SSS}] %5p: %c: %m%n")
+
+    org.apache.log4j.LogManager.resetConfiguration
+    org.apache.log4j.PropertyConfigurator.configure(properties)
+    logger.debug("log4j java properties setup", :log4j_level => log4j_level)
+  end
+end # class LogStash::Logger

data/lib/logstash/namespace.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module LogStash
+  module Inputs; end
+  module Outputs; end
+  module Filters; end
+  module Search; end
+  module Config; end
+  module File; end
+  module Web; end
+  module Util; end
+  module PluginMixins; end
+  module PluginManager; end
+end # module LogStash

data/lib/logstash/output_delegator.rb

@@ -0,0 +1,172 @@
+# encoding: utf-8
+require "concurrent/atomic/atomic_fixnum"
+java_import "java.util.concurrent.CopyOnWriteArrayList"
+
+# This class goes hand in hand with the pipeline to provide a pool of
+# free workers to be used by pipeline worker threads. The pool is
+# internally represented with a SizedQueue set the the size of the number
+# of 'workers' the output plugin is configured with.
+#
+# This plugin also records some basic statistics
+module LogStash class OutputDelegator
+  attr_reader :workers, :config, :threadsafe
+
+  # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
+  # Internally these just get merged together into a single hash
+  def initialize(logger, klass, default_worker_count, *plugin_args)
+    @logger = logger
+    @threadsafe = klass.threadsafe?
+    @config = plugin_args.reduce({}, :merge)
+    @klass = klass
+    @workers = java.util.concurrent.CopyOnWriteArrayList.new
+    @default_worker_count = default_worker_count
+    @registered = false
+    @events_received = Concurrent::AtomicFixnum.new(0)
+  end
+
+  def threadsafe?
+    !!@threadsafe
+  end
+
+  def warn_on_worker_override!
+    # The user has configured extra workers, but this plugin doesn't support it :(
+    if worker_limits_overriden?
+      message = @klass.workers_not_supported_message
+      warning_meta = {:plugin => @klass.config_name, :worker_count => @config["workers"]}
+      if message
+        warning_meta[:message] = message
+        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", warning_meta))
+      else
+        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", warning_meta))
+      end
+    end
+  end
+
+  def worker_limits_overriden?
+    @config["workers"] && @config["workers"] > 1 && @klass.workers_not_supported?
+  end
+
+  def target_worker_count
+    # Remove in 5.0 after all plugins upgraded to use class level declarations
+    raise ArgumentError, "Attempted to detect target worker count before instantiating a worker to test for legacy workers_not_supported!" if @workers.size == 0
+
+    if @threadsafe || @klass.workers_not_supported?
+      1
+    else
+      @config["workers"] || @default_worker_count
+    end
+  end
+
+  def config_name
+    @klass.config_name
+  end
+
+  def register
+    raise ArgumentError, "Attempted to register #{self} twice!" if @registered
+    @registered = true
+    # We define this as an array regardless of threadsafety
+    # to make reporting simpler, even though a threadsafe plugin will just have
+    # a single instance
+    #
+    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
+    # To detect these we need an instance to be created first :()
+    # TODO: In the next major version after 2.x remove support for this
+    @workers << @klass.new(@config)
+    @workers.first.register # Needed in case register calls `workers_not_supported`
+
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+
+    # Threadsafe versions don't need additional workers
+    setup_additional_workers!(target_worker_count) unless @threadsafe
+    # We skip the first worker because that's pre-registered to deal with legacy workers_not_supported
+    @workers.subList(1,@workers.size).each(&:register)
+    setup_multi_receive!
+  end
+
+  def setup_additional_workers!(target_worker_count)
+    warn_on_worker_override!
+
+    (target_worker_count - 1).times do
+      inst = @klass.new(@config)
+      @workers << inst
+    end
+
+    # This queue is used to manage sharing across threads
+    @worker_queue = SizedQueue.new(target_worker_count)
+    @workers.each {|w| @worker_queue << w }
+  end
+
+  def setup_multi_receive!
+    # One might wonder why we don't use something like
+    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
+    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
+    # The other 2% you get weird errors about rebinding to the same object
+    # Until we switch to Jruby 9.x keep the define_singleton_method parts
+    # the way they are, with a block
+    # See https://github.com/jruby/jruby/issues/3582
+    if threadsafe?
+      @threadsafe_worker = @workers.first
+      define_singleton_method(:multi_receive) do |events|
+        threadsafe_multi_receive(events)
+      end
+    else
+      define_singleton_method(:multi_receive) do |events|
+        worker_multi_receive(events)
+      end
+    end
+  end
+
+  def threadsafe_multi_receive(events)
+    @events_received.increment(events.length)
+
+    @threadsafe_worker.multi_receive(events)
+  end
+
+  def worker_multi_receive(events)
+    @events_received.increment(events.length)
+
+    worker = @worker_queue.pop
+    begin
+      worker.multi_receive(events)
+    ensure
+      @worker_queue.push(worker)
+    end
+  end
+
+  def do_close
+    @logger.debug("closing output delegator", :klass => @klass)
+
+    if @threadsafe
+      @workers.each(&:do_close)
+    else
+      worker_count.times do
+        worker = @worker_queue.pop
+        worker.do_close
+      end
+    end
+  end
+
+  def events_received
+    @events_received.value
+  end
+
+  # There's no concept of 'busy' workers for a threadsafe plugin!
+  def busy_workers
+    if @threadsafe
+      0
+    else
+      # The pipeline reporter can run before the outputs are registered trying to pull a value here
+      # In that case @worker_queue is empty, we just return 0
+      return 0 unless @worker_queue
+      @workers.size - @worker_queue.size
+    end
+  end
+
+  def worker_count
+    @workers.size
+  end
+
+  private
+  # Needed for testing, so private
+  attr_reader :threadsafe_worker, :worker_queue
+end end