logstash-lib 1.3.2

data/lib/logstash/kibana.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require "rack/handler/ftw" # gem ftw
+require "ftw" # gem ftw
+require "sinatra/base" # gem sinatra
+require "optparse"
+require "mime/types"
+
+class Rack::Handler::FTW
+  alias_method :handle_connection_, :handle_connection
+  def handle_connection(connection)
+    #require "pry"; binding.pry
+    return handle_connection_(connection)
+  end
+end
+
+module LogStash::Kibana
+  class App < Sinatra::Base
+    set :logging, true
+
+    use Rack::CommonLogger
+    use Rack::ShowExceptions
+
+    get "/" do
+      redirect "index.html"
+    end
+    
+    # Sinatra has problems serving static files from 
+    # jar files, so let's hack this by hand.
+    #set :public, "#{File.dirname(__FILE__)}/public"
+    get "/config.js" do static_file end
+    get "/index.html" do static_file end
+    get "/app/*" do static_file end
+    get "/css/*" do static_file end
+    get "/font/*" do static_file end
+    get "/img/*" do static_file end
+    get "/vendor/*" do static_file end
+
+    def static_file
+      # request.path_info is the full path of the request.
+      docroot =  File.expand_path(File.join(File.dirname(__FILE__), "../../vendor/kibana"))
+      path = File.join(docroot, *request.path_info.split("/"))
+      if File.exists?(path)
+        ext = path.split(".").last
+        content_type MIME::Types.type_for(ext).first.to_s
+        body File.new(path, "r").read
+      else
+        status 404
+        content_type "text/plain"
+        body "File not found: #{path}"
+      end
+    end # def static_file
+  end # class App
+
+  class Runner
+    Settings = Struct.new(:logfile, :address, :port, :backend)
+
+    public
+    def run(args)
+      settings = Settings.new
+      settings.address = "0.0.0.0"
+      settings.port = 9292
+      settings.backend = "localhost"
+
+      progname = File.basename($0)
+
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: #{progname} [options]"
+        opts.on("-a", "--address ADDRESS", "Address on which to start webserver. Default is 0.0.0.0.") do |address|
+          settings.address = address
+        end
+
+        opts.on("-p", "--port PORT", "Port on which to start webserver. Default is 9292.") do |port|
+          settings.port = port.to_i
+        end
+
+        #opts.on("-b", "--backend host",
+                #"The backend host to use. Default is 'localhost'") do |host|
+          #settings.backend = host
+        #end
+      end
+
+      begin
+        args = opts.parse(args)
+      rescue SystemExit
+        # if you ask for --help, optparse will exit.
+        # capture it and return normally
+        return []
+      end
+
+      @thread = Thread.new do
+        Cabin::Channel.get.info("Starting web server", :settings => settings)
+        ftw = Rack::Handler::FTW.new(LogStash::Kibana::App.new,
+                               :Host => settings.address,
+                               :Port => settings.port)
+        trap_id = Stud::trap("INT") do
+          puts "Stopping web..."
+          ftw.stop rescue nil
+          raise SystemExit
+        end
+
+        ftw.run
+      end
+
+      return args
+    end # def run
+
+    public
+    def wait
+      @thread.join if @thread
+      return 0
+    end # def wait
+  end
+end

data/lib/logstash/loadlibs.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+jarpath = File.join(File.dirname(__FILE__), "../../vendor/**/*.jar")
+Dir[jarpath].each do |jar|
+  if $DEBUG
+    puts "Loading #{jar}"
+  end
+  require jar
+end
+

data/lib/logstash/logging.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+require "logger"
+
+class LogStash::Logger
+  attr_accessor :target
+
+  public
+  def initialize(*args)
+    super()
+
+    #self[:program] = File.basename($0)
+    #subscribe(::Logger.new(*args))
+    @target = args[0]
+    @channel = Cabin::Channel.get(LogStash)
+
+    # lame hack until cabin's smart enough not to doubley-subscribe something.
+    # without this subscription count check, running the test suite
+    # causes Cabin to subscribe to STDOUT maaaaaany times.
+    subscriptions = @channel.instance_eval { @subscribers.count }
+    @channel.subscribe(@target) unless subscriptions > 0
+
+    # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
+    @level = $DEBUG ? :debug : :warn
+    if ENV["LOGSTASH_DEBUG"]
+      @level = :debug
+    end
+
+    # Direct metrics elsewhere.
+    @channel.metrics.channel = Cabin::Channel.new
+  end # def initialize
+
+  # Delegation
+  def level=(value) @channel.level = value; end
+  def debug(*args); @channel.debug(*args); end
+  def debug?(*args); @channel.debug?(*args); end
+  def info(*args); @channel.info(*args); end
+  def info?(*args); @channel.info?(*args); end
+  def warn(*args); @channel.warn(*args); end
+  def warn?(*args); @channel.warn?(*args); end
+  def error(*args); @channel.error(*args); end
+  def error?(*args); @channel.error?(*args); end
+  def fatal(*args); @channel.fatal(*args); end
+  def fatal?(*args); @channel.fatal?(*args); end
+
+  def self.setup_log4j(logger)
+    require "java"
+
+    properties = java.util.Properties.new
+    log4j_level = "WARN"
+    case logger.level
+      when :debug
+        log4j_level = "DEBUG"
+      when :info
+        log4j_level = "INFO"
+      when :warn
+        log4j_level = "WARN"
+    end # case level
+    properties.setProperty("log4j.rootLogger", "#{log4j_level},logstash")
+
+    # TODO(sissel): This is a shitty hack to work around the fact that
+    # LogStash::Logger isn't used anymore. We should fix that.
+    target = logger.instance_eval { @subscribers }.values.first.instance_eval { @io }
+    case target
+      when STDOUT
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.out")
+      when STDERR
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.ConsoleAppender")
+        properties.setProperty("log4j.appender.logstash.Target", "System.err")
+      else
+        properties.setProperty("log4j.appender.logstash",
+                      "org.apache.log4j.FileAppender")
+        properties.setProperty("log4j.appender.logstash.File", target.path)
+    end # case target
+
+    properties.setProperty("log4j.appender.logstash.layout",
+                  "org.apache.log4j.PatternLayout")
+    properties.setProperty("log4j.appender.logstash.layout.conversionPattern",
+                  "log4j, [%d{yyyy-MM-dd}T%d{HH:mm:ss.SSS}] %5p: %c: %m%n")
+
+    org.apache.log4j.LogManager.resetConfiguration
+    org.apache.log4j.PropertyConfigurator.configure(properties)
+    logger.debug("log4j java properties setup", :log4j_level => log4j_level)
+  end
+end # class LogStash::Logger

data/lib/logstash/monkeypatches-for-bugs.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/JRUBY-6970" if RUBY_PLATFORM == "java"

data/lib/logstash/monkeypatches-for-debugging.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+if $DEBUGLIST.include?("require")
+  ROOT = File.dirname(__FILE__)
+  module Kernel
+    alias_method :require_debug, :require
+
+    def require(path)
+      start = Time.now
+      result = require_debug(path)
+      duration = Time.now - start
+
+      origin = caller[1]
+      if origin =~ /rubygems\/custom_require/
+        origin = caller[3]
+        if origin.nil?
+          STDERR.puts "Unknown origin"
+          STDERR.puts caller.join("\n")
+        end
+      end
+      origin = origin.gsub(/:[0-9]+:in .*/, "") if origin
+
+      # Only print require() calls that did actual work.
+      # require() returns true on load, false if already loaded.
+      if result
+        source = caller[0]
+        #p source.include?("/lib/polyglot.rb:63:in `require'") => source
+        if source.include?("/lib/polyglot.rb:63:in `require'")
+          source = caller[1]
+        end
+
+        #target = $LOADED_FEATURES.grep(/#{path}/).first
+        #puts path
+        #puts caller.map { |c| "  #{c}" }.join("\n")
+        #fontsize = [10, duration * 48].max
+        puts "#{duration},#{path},#{source}"
+      end
+      #puts caller.map { |c| " => #{c}" }.join("\n")
+    end
+
+    alias_method :load_debug, :load
+
+    def load(path)
+      puts "load(\"#{path}\")"
+      return load_debug(path)
+    end
+  end
+end

data/lib/logstash/monkeypatches-for-performance.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+if RUBY_PLATFORM == "java"
+  class << File
+    # mpp == monkey patch for performance
+    alias_method :mpp_file?, :file?
+    alias_method :mpp_exist?, :exist?
+    alias_method :mpp_exists?, :exists?
+
+    JAR_RE = /^(?:jar:)?file:(\/.*\.jar)!(\/.*$)/
+    def file?(path)
+      #return mpp_file?(path)
+      # If path is in a jar (file://blah/foo.jar!/some/path)
+      #   - create a cache for this jar of all files
+      #   - return cached results only
+      if RUBY_PLATFORM == "java" 
+        m = JAR_RE.match(path)
+        return mpp_file?(path) if !m # not a jar file
+        c = __zipcache(m[1], m[2]) # m[1] == the jar path
+        # ZipEntry has only 'isDirectory()' so I assume any
+        # non-directories are files.
+        rc = (!c.nil? && !c.isDirectory)
+        #p path => rc
+        return rc
+      end
+      return mpp_file?(path)
+    end
+
+    def exist?(path)
+      #return mpp_exist?(path)
+      # If path is in a jar (file://blah/foo.jar!/some/path)
+      #   - create a cache for this jar of all files
+      #   - return cached results only
+      if RUBY_PLATFORM == "java" 
+        m = JAR_RE.match(path)
+        return mpp_exists?(path) if !m # not a jar file
+        c = __zipcache(m[1], m[2]) # m[1] == the jar path
+        return !c.nil?
+      end
+      return mpp_exists?(path)
+    end
+
+    def exists?(path)
+      return exist?(path)
+    end
+
+    def __zipcache(jarpath, path)
+      @jarcache ||= Hash.new { |h,k| h[k] = {} }
+
+      if @jarcache[jarpath].empty?
+        #puts "Caching file entries for #{jarpath}"
+        s = Time.now
+        zip = java.util.zip.ZipFile.new(jarpath)
+        zip.entries.each do |entry|
+          #puts "Caching file entries for #{jarpath}: /#{entry.name}"
+          # Prefix entry name with "/" because that's what the jar path looks
+          # like in jruby: file://some.jar!/some/path
+          @jarcache[jarpath]["/" + entry.name] = entry
+        end
+      end
+
+      entry = @jarcache[jarpath][path]
+      #puts "Serving cached file info #{path}: #{entry}"
+      return entry
+    end
+  end
+end

data/lib/logstash/multiqueue.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+
+class LogStash::MultiQueue
+  attr_accessor :logger
+
+  public
+  def initialize(*queues)
+    @logger = Cabin::Channel.get(LogStash)
+    @mutex = Mutex.new
+    @queues = queues
+  end # def initialize
+
+  public
+  def logger=(_logger)
+    @logger = _logger
+
+    # Set the logger for all known queues, too.
+    @queues.each do |q|
+      q.logger = _logger
+    end
+  end # def logger=
+
+  # Push an object to all queues.
+  public
+  def push(object)
+    @queues.each { |q| q.push(object) }
+  end # def push
+  alias :<< :push
+
+  alias_method :<<, :push
+
+  # Add a new Queue to this queue.
+  public
+  def add_queue(queue)
+    @mutex.synchronize do
+      @queues << queue
+    end
+  end # def add_queue
+
+  public
+  def remove_queue(queue)
+    @mutex.synchronize do
+      @queues.delete(queue)
+    end
+  end # def remove_queue
+
+  public
+  def size
+    return @queues.collect { |q| q.size }
+  end # def size
+end # class LogStash::MultiQueue

data/lib/logstash/namespace.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+#$: << File.join(File.dirname(__FILE__), "..", "..", "vendor", "bundle")
+
+module LogStash
+  module Inputs; end
+  module Outputs; end
+  module Filters; end
+  module Search; end
+  module Config; end
+  module File; end
+  module Web; end
+  module Util; end
+  module PluginMixins; end
+
+  SHUTDOWN = :shutdown
+end # module LogStash

data/lib/logstash/outputs/base.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "cgi"
+require "logstash/event"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/namespace"
+require "logstash/config/mixin"
+require "uri"
+
+class LogStash::Outputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "output"
+
+  # The type to act on. If a type is given, then this output will only
+  # act on messages with the same type. See any input plugin's "type"
+  # attribute for more.
+  # Optional.
+  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  # Only handle events with all of these tags.  Note that if you specify
+  # a type, the event must also match that type.
+  # Optional.
+  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  # Only handle events without any of these tags. Note this check is additional to type and tags.
+  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+
+  # The codec used for output data
+  config :codec, :validate => :codec, :default => "plain"
+
+  # The number of workers to use for this output.
+  # Note that this setting may not be useful for all outputs.
+  config :workers, :validate => :number, :default => 1
+
+  public
+  def workers_not_supported(message=nil)
+    return if @workers == 1
+    if message
+      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", :plugin => self.class.config_name, :worker_count => @workers, :message => message))
+    else
+      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", :plugin => self.class.config_name, :worker_count => @workers))
+    end
+    @workers = 1
+  end
+
+  public
+  def initialize(params={})
+    super
+    config_init(params)
+  end
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def receive(event)
+    raise "#{self.class}#receive must be overidden"
+  end # def receive
+
+  public
+  def worker_setup
+    return unless @workers > 1
+
+    define_singleton_method(:handle, method(:handle_worker))
+    @worker_queue = SizedQueue.new(20)
+
+    @worker_threads = @workers.times do |i|
+      Thread.new(original_params, @worker_queue) do |params, queue|
+        LogStash::Util::set_thread_name(">#{self.class.config_name}.#{i}")
+        worker_params = params.merge("workers" => 1, "codec" => @codec.clone)
+        worker_plugin = self.class.new(worker_params)
+        worker_plugin.register
+        while true
+          event = queue.pop
+          worker_plugin.handle(event)
+        end
+      end
+    end
+  end
+
+  public
+  def handle(event)
+    receive(event)
+  end # def handle
+  
+  def handle_worker(event)
+    @worker_queue.push(event)
+  end
+
+  private
+  def output?(event)
+    if !@type.empty?
+      if event["type"] != @type
+        @logger.debug? and @logger.debug(["outputs/#{self.class.name}: Dropping event because type doesn't match #{@type}", event])
+        return false
+      end
+    end
+
+    if !@tags.empty?
+      return false if !event["tags"]
+      @include_method = :any?
+      if !@tags.send(@include_method) {|tag| event["tags"].include?(tag)}
+        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags don't match #{@tags.inspect}", event)
+        return false
+      end
+    end
+
+    if !@exclude_tags.empty? && event["tags"]
+      if @exclude_tags.send(@exclude_method) {|tag| event["tags"].include?(tag)}
+        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags contains excluded tags: #{exclude_tags.inspect}", event)
+        return false
+      end
+    end
+
+    return true
+  end
+end # class LogStash::Outputs::Base