logstash-core 1.5.0.beta2-java

data/lib/logstash/inputs/threadable.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/inputs/base"
+
+# This is the threadable class for logstash inputs. 
+# Use this class in your inputs if it can support multiple threads
+class LogStash::Inputs::Threadable < LogStash::Inputs::Base
+
+  # Set this to the number of threads you want this input to spawn.
+  # This is the same as declaring the input multiple times
+  config :threads, :validate => :number, :default => 1
+ 
+  def initialize(params)
+    super
+    @threadable = true
+  end
+
+end # class LogStash::Inputs::Threadable

data/lib/logstash/java_integration.rb

@@ -0,0 +1,41 @@
+require "java"
+
+# this is mainly for usage with JrJackson json parsing in :raw mode which genenerates
+# Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.
+# these object already quacks like their Ruby equivalents Array and Hash but they will
+# not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
+# both classes everywhere. see LogStash::JSon.
+
+class Java::JavaUtil::ArrayList
+  # have ArrayList objects report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Array
+    super
+  end
+end
+
+class Java::JavaUtil::LinkedHashMap
+  # have LinkedHashMap objects report is_a?(Array) == true
+  def is_a?(clazz)
+    return true if clazz == Hash
+    super
+  end
+end
+
+class Array
+  # enable class equivalence between Array and ArrayList
+  # so that ArrayList will work with case o when Array ...
+  def self.===(other)
+    return true if other.is_a?(Java::JavaUtil::ArrayList)
+    super
+  end
+end
+
+class Hash
+  # enable class equivalence between Hash and LinkedHashMap
+  # so that LinkedHashMap will work with case o when Hash ...
+  def self.===(other)
+    return true if other.is_a?(Java::JavaUtil::LinkedHashMap)
+    super
+  end
+end

data/lib/logstash/json.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "logstash/environment"
+require "logstash/errors"
+if LogStash::Environment.jruby?
+  require "jrjackson"
+  require "logstash/java_integration"
+else
+  require  "oj"
+end
+
+module LogStash
+  module Json
+    class ParserError < LogStash::Error; end
+    class GeneratorError < LogStash::Error; end
+
+    extend self
+
+    ### MRI
+
+    def mri_load(data, options = {})
+      Oj.load(data)
+    rescue Oj::ParseError => e
+      raise LogStash::Json::ParserError.new(e.message)
+    end
+
+    def mri_dump(o)
+      Oj.dump(o, :mode => :compat, :use_to_json => true)
+    rescue => e
+      raise LogStash::Json::GeneratorError.new(e.message)
+    end
+
+    ### JRuby
+
+    def jruby_load(data, options = {})
+      options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
+    rescue JrJackson::ParseError => e
+      raise LogStash::Json::ParserError.new(e.message)
+    end
+
+    def jruby_dump(o)
+      # test for enumerable here to work around an omission in JrJackson::Json.dump to
+      # also look for Java::JavaUtil::ArrayList, see TODO submit issue
+      o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
+    rescue => e
+      raise LogStash::Json::GeneratorError.new(e.message)
+    end
+
+    prefix = LogStash::Environment.jruby? ? "jruby" : "mri"
+    alias_method :load, "#{prefix}_load".to_sym
+    alias_method :dump, "#{prefix}_dump".to_sym
+
+  end
+end

data/lib/logstash/logging.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+require "logger"
+
+class LogStash::Logger
+  attr_accessor :target
+
+  public
+  def initialize(*args)
+    super()
+
+    #self[:program] = File.basename($0)
+    #subscribe(::Logger.new(*args))
+    @target = args[0]
+    @channel = Cabin::Channel.get(LogStash)
+
+    # lame hack until cabin's smart enough not to doubley-subscribe something.
+    # without this subscription count check, running the test suite
+    # causes Cabin to subscribe to STDOUT maaaaaany times.
+    subscriptions = @channel.instance_eval { @subscribers.count }
+    @channel.subscribe(@target) unless subscriptions > 0
+
+    # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
+    @level = $DEBUG ? :debug : :warn
+    if ENV["LOGSTASH_DEBUG"]
+      @level = :debug
+    end
+
+    # Direct metrics elsewhere.
+    @channel.metrics.channel = Cabin::Channel.new
+  end # def initialize
+
+  # Delegation
+  def level=(value) @channel.level = value; end
+  def debug(*args); @channel.debug(*args); end
+  def debug?(*args); @channel.debug?(*args); end
+  def info(*args); @channel.info(*args); end
+  def info?(*args); @channel.info?(*args); end
+  def warn(*args); @channel.warn(*args); end
+  def warn?(*args); @channel.warn?(*args); end
+  def error(*args); @channel.error(*args); end
+  def error?(*args); @channel.error?(*args); end
+  def fatal(*args); @channel.fatal(*args); end
+  def fatal?(*args); @channel.fatal?(*args); end
+
+  def self.setup_log4j(logger)
+    require "java"
+
+    properties = java.util.Properties.new
+    log4j_level = "WARN"
+    case logger.level
+      when :debug
+        log4j_level = "DEBUG"
+      when :info
+        log4j_level = "INFO"
+      when :warn
+        log4j_level = "WARN"
+    end # case level
+    properties.setProperty("log4j.rootLogger", "#{log4j_level},logstash")
+
+    # TODO(sissel): This is a shitty hack to work around the fact that
+    # LogStash::Logger isn't used anymore. We should fix that.
+    target = logger.instance_eval { @subscribers }.values.first.instance_eval { @io }
+    case target
+      when STDOUT
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.out")
+      when STDERR
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.err")
+      when target.is_a?(File)
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.FileAppender")
+        properties.setProperty("log4j.appender.logstash.File", target.path)
+      else
+        properties.setProperty("log4j.appender.logstash", "org.apache.log4j.varia.NullAppender")
+    end # case target
+
+    properties.setProperty("log4j.appender.logstash.layout",
+                  "org.apache.log4j.PatternLayout")
+    properties.setProperty("log4j.appender.logstash.layout.conversionPattern",
+                  "log4j, [%d{yyyy-MM-dd}T%d{HH:mm:ss.SSS}] %5p: %c: %m%n")
+
+    org.apache.log4j.LogManager.resetConfiguration
+    org.apache.log4j.PropertyConfigurator.configure(properties)
+    logger.debug("log4j java properties setup", :log4j_level => log4j_level)
+  end
+end # class LogStash::Logger

data/lib/logstash/multiqueue.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+
+class LogStash::MultiQueue
+  attr_accessor :logger
+
+  public
+  def initialize(*queues)
+    @logger = Cabin::Channel.get(LogStash)
+    @mutex = Mutex.new
+    @queues = queues
+  end # def initialize
+
+  public
+  def logger=(_logger)
+    @logger = _logger
+
+    # Set the logger for all known queues, too.
+    @queues.each do |q|
+      q.logger = _logger
+    end
+  end # def logger=
+
+  # Push an object to all queues.
+  public
+  def push(object)
+    @queues.each { |q| q.push(object) }
+  end # def push
+  alias :<< :push
+
+  alias_method :<<, :push
+
+  # Add a new Queue to this queue.
+  public
+  def add_queue(queue)
+    @mutex.synchronize do
+      @queues << queue
+    end
+  end # def add_queue
+
+  public
+  def remove_queue(queue)
+    @mutex.synchronize do
+      @queues.delete(queue)
+    end
+  end # def remove_queue
+
+  public
+  def size
+    return @queues.collect { |q| q.size }
+  end # def size
+end # class LogStash::MultiQueue

data/lib/logstash/namespace.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+#$: << File.join(File.dirname(__FILE__), "..", "..", "vendor", "bundle")
+
+module LogStash
+  module Inputs; end
+  module Outputs; end
+  module Filters; end
+  module Search; end
+  module Config; end
+  module File; end
+  module Web; end
+  module Util; end
+  module PluginMixins; end
+  module PluginManager; end
+
+  SHUTDOWN = :shutdown
+end # module LogStash

data/lib/logstash/outputs/base.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "cgi"
+require "logstash/event"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/namespace"
+require "logstash/config/mixin"
+require "uri"
+
+class LogStash::Outputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "output"
+
+  # The type to act on. If a type is given, then this output will only
+  # act on messages with the same type. See any input plugin's `type`
+  # attribute for more.
+  # Optional.
+  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  # Only handle events with all of these tags.
+  # Optional.
+  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  # Only handle events without any of these tags.
+  # Optional.
+  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+
+  # The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
+
+  # The number of workers to use for this output.
+  # Note that this setting may not be useful for all outputs.
+  config :workers, :validate => :number, :default => 1
+
+  attr_reader :worker_plugins
+
+  public
+  def workers_not_supported(message=nil)
+    return if @workers == 1
+    if message
+      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", :plugin => self.class.config_name, :worker_count => @workers, :message => message))
+    else
+      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", :plugin => self.class.config_name, :worker_count => @workers))
+    end
+    @workers = 1
+  end
+
+  public
+  def initialize(params={})
+    super
+    config_init(params)
+  end
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def receive(event)
+    raise "#{self.class}#receive must be overidden"
+  end # def receive
+
+  public
+  def worker_setup
+    if @workers == 1
+      @worker_plugins = [self]
+    else
+      define_singleton_method(:handle, method(:handle_worker))
+      @worker_queue = SizedQueue.new(20)
+      @worker_plugins = @workers.times.map { self.class.new(params.merge("workers" => 1, "codec" => @codec.clone)) }
+      @worker_plugins.map.with_index do |plugin, i|
+        Thread.new(original_params, @worker_queue) do |params, queue|
+          LogStash::Util::set_thread_name(">#{self.class.config_name}.#{i}")
+          plugin.register
+          while true
+            event = queue.pop
+            plugin.handle(event)
+          end
+        end
+      end
+    end
+  end
+
+  public
+  def handle(event)
+    receive(event)
+  end # def handle
+
+  def handle_worker(event)
+    @worker_queue.push(event)
+  end
+
+  private
+  def output?(event)
+    if !@type.empty?
+      if event["type"] != @type
+        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because type doesn't match",
+                                         :type => @type, :event => event)
+        return false
+      end
+    end
+
+    if !@tags.empty?
+      return false if !event["tags"]
+      if (event["tags"] & @tags).size != @tags.size
+        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags don't match",
+                                         :tags => @tags, :event => event)
+        return false
+      end
+    end
+
+    if !@exclude_tags.empty? && event["tags"]
+      if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
+        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags contains excluded tags",
+                                         :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
+        return false
+      end
+    end
+
+    return true
+  end
+end # class LogStash::Outputs::Base

data/lib/logstash/patches.rb

@@ -0,0 +1,3 @@
+require "logstash/patches/bugfix_jruby_2558"
+require "logstash/patches/cabin"
+require "logstash/patches/profile_require_calls"

data/lib/logstash/patches/bugfix_jruby_2558.rb

@@ -0,0 +1,50 @@
+require "logstash/environment"
+
+if LogStash::Environment.windows? && LogStash::Environment.jruby?
+  require "socket"
+  module JRubyBug2558SocketPeerAddrBugFix
+    def peeraddr
+      orig_peeraddr.map do |v|
+        case v
+        when String
+          v.force_encoding(Encoding::UTF_8)
+        else
+          v
+        end
+      end
+    end
+  end
+
+  class << Socket
+    # Bugfix for jruby #2558
+    alias_method :orig_gethostname, :gethostname
+    def gethostname
+      return orig_gethostname.force_encoding(Encoding::UTF_8)
+    end
+  end
+
+  class TCPSocket
+    alias_method :orig_peeraddr, :peeraddr
+    include JRubyBug2558SocketPeerAddrBugFix
+  end
+  class UDPSocket
+    alias_method :orig_peeraddr, :peeraddr
+    include JRubyBug2558SocketPeerAddrBugFix
+  end
+end
+
+if LogStash::Environment.windows?
+  # make sure all strings pulled out of ENV are UTF8
+  class <<ENV
+    alias_method :orig_getter, :[]
+    def [](key)
+      case value = orig_getter(key)
+      when String
+        # dup is necessary since force_encoding is destructive
+        value.dup.force_encoding(Encoding::UTF_8)
+      else
+        value
+      end
+    end
+  end
+end