logstash-lib 1.3.2

data/lib/logstash/sized_queue.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require "thread" # for SizedQueue
+class LogStash::SizedQueue < SizedQueue
+  # TODO(sissel): Soon will implement push/pop stats, etc
+end

data/lib/logstash/test.rb

@@ -0,0 +1,183 @@
+# encoding: utf-8
+require "rubygems"
+require "optparse"
+
+# TODO(sissel): Are these necessary anymore?
+#$:.unshift "#{File.dirname(__FILE__)}/../lib"
+#$:.unshift "#{File.dirname(__FILE__)}/../test"
+
+require "logstash/namespace"
+require "logstash/loadlibs"
+require "logstash/logging"
+
+class LogStash::Test
+  public
+  def initialize
+    log_to(STDERR)
+
+    # This is lib/logstash/test.rb, so go up 2 directories for the plugin path
+    if jarred?(__FILE__)
+      @plugin_paths = [ File.dirname(File.dirname(__FILE__)) ]
+    else
+      @plugin_paths = [ File.dirname(File.dirname(File.dirname(__FILE__))) ]
+    end 
+    @verbose = 0
+  end # def initialize
+
+  private
+  def jarred?(path)
+    return path =~ /^file:/
+  end # def jarred?
+
+  public
+  def log_to(target)
+    @logger = LogStash::Logger.new(target)
+  end # def log_to
+
+  def check_lib(lib, provider, is=:optional, message=nil)
+    optional = (is == :optional)
+    begin
+      require lib
+      puts "+ Found #{optional ? "optional" : "required"} library '#{lib}'"
+      return { :optional => optional, :found => true }
+    rescue LoadError => e
+      puts "- Missing #{optional ? "optional" : "required"} library '#{lib}'" \
+           "- try 'gem install #{provider}'" \
+           "#{optional ? " if you want this library" : ""}. #{message}"
+      return { :optional => optional, :found => false }
+    end
+  end # def check_lib
+
+  def report_ruby_version
+    puts "Running #{RUBY_VERSION}p#{RUBY_PATCHLEVEL} on #{RUBY_PLATFORM}"
+  end # def report_ruby_version
+
+  def check_libraries
+    results = [
+      # main agent
+      check_lib("grok-pure", "jls-grok", :optional, "needed for the grok filter."),
+      check_lib("bunny", "bunny", :optional, "needed for AMQP input and output"),
+      check_lib("uuidtools", "uuidtools", :required,
+                "needed for AMQP input and output"),
+      check_lib("ap", "awesome_print", :optional, "improve debug logging output"),
+      check_lib("json", "json", :required, "required for logstash to function"),
+      check_lib("filewatch/tail", "filewatch", :optional,
+                "required for file input"),
+      check_lib("jruby-elasticsearch", "jruby-elasticsearch", :optional,
+                "required for elasticsearch output and for logstash web"),
+      check_lib("stomp", "stomp", :optional,
+                "required for stomp input and output"),
+      check_lib("mongo", "mongo", :optional, "required for mongo output"),
+      check_lib("redis", "redis", :optional,
+                "required for stomp input and output"),
+      check_lib("gelf", "gelf", :optional, "required for gelf (graylog2) output"),
+      check_lib("statsd", "statsd-ruby", :optional, "required for statsd output"),
+
+      # logstash web
+      check_lib("ftw", "ftw", :required, "needed for logstash web"),
+      check_lib("rack", "rack", :required, "needed for logstash web"),
+      check_lib("sinatra", "sinatra", :required, "needed for logstash web"),
+      check_lib("sass", "sass", :required, "needed for logstash web"),
+      check_lib("haml", "haml", :required, "needed for logstash web"),
+    ]
+
+    missing_required = results.count { |r| !r[:optional] and !r[:found] }
+    if missing_required == 0
+      puts "All required libraries found :)"
+    else
+      suffix = (missing_required > 1) ? "ies" : "y"
+      puts "FATAL: Missing #{missing_required} required librar#{suffix}"
+      return false
+    end
+
+    return true
+  end # def check_libraries
+
+  # Parse options.
+  private
+  def options(args)
+    # strip out the pluginpath argument if it exists and 
+    # extend the LOAD_PATH for the ruby runtime
+    opts = OptionParser.new
+
+    opts.on("-v", "Increase verbosity") do
+      @verbose += 1
+    end
+
+    # Step one is to add test flags.
+    opts.on("--pluginpath PLUGINPATH", 
+            "Load plugins and test from a pluginpath") do |path|
+      @plugin_paths << path
+    end # --pluginpath PLUGINPATH
+
+    begin
+      remainder = opts.parse(args)
+    rescue OptionParser::InvalidOption => e
+      @logger.info("Invalid option", :exception => e)
+      raise e
+    end
+    return remainder
+  end # def options
+
+  public
+  def run(args)
+    remainder = options(args)
+
+    if @verbose >= 3  # Uber debugging.
+      @logger.level = :debug
+      $DEBUG = true
+    elsif @verbose == 2 # logstash debug logs
+      @logger.level = :debug
+    elsif @verbose == 1 # logstash info logs
+      @logger.level = :info
+    else # Default log level
+      @logger.level = :warn
+    end
+
+    @success = true
+    @thread = Thread.new do
+      report_ruby_version
+
+      # TODO(sissel): Rewrite this into a proper test?
+      #if !check_libraries
+        #puts "Library check failed."
+        #@success = false
+      #end
+
+      @plugin_paths.each do |path|
+        load_tests(path)
+      end
+
+      require "minitest/spec"
+      @status = MiniTest::Unit.new.run(ARGV)
+    end # the runner thread
+    return remainder
+  end # def run
+
+  def wait
+    @thread.join
+    return @status
+  end # def wait
+
+  # Find tests in a given path. Tests must be in the plugin path +
+  # "/test/.../test_*.rb"
+  def each_test(basepath, &block)
+    if jarred?(basepath)
+      # No test/logstash/... hierarchy in the jar, not right now anyway.
+      glob_path = File.join(basepath, "logstash", "**", "test_*.rb")
+    else
+      glob_path = File.join(basepath, "test", "**", "test_*.rb")
+    end
+    @logger.info("Searching for tests", :path => glob_path)
+    Dir.glob(glob_path).each do |path|
+      block.call(path)
+    end
+  end # def each_test
+
+  def load_tests(path)
+    each_test(path) do |test|
+      @logger.info("Loading test", :test => test)
+      require test
+    end
+  end # def load_tests
+end # class LogStash::Test

data/lib/logstash/threadwatchdog.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::ThreadWatchdog
+  attr_accessor :logger
+  attr_accessor :threads
+
+  class TimeoutError < StandardError; end
+
+  public
+  def initialize(threads, watchdog_timeout=10)
+    @threads = threads
+    @watchdog_timeout = watchdog_timeout
+  end # def initialize
+
+  public
+  def watch
+    while sleep(1)
+      cutoff = Time.now - @watchdog_timeout
+      @threads.each do |t|
+        watchdog = t[:watchdog]
+        if watchdog and watchdog <= cutoff
+          age = Time.now - watchdog
+          @logger.fatal("thread watchdog timeout",
+                        :thread => t,
+                        :backtrace => t.backtrace,
+                        :thread_watchdog => watchdog,
+                        :age => age,
+                        :cutoff => @watchdog_timeout,
+                        :state => t[:watchdog_state])
+          raise TimeoutError, "watchdog timeout"
+        end
+      end
+    end
+  end # def watch
+end # class LogStash::ThreadWatchdog

data/lib/logstash/time_addon.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+# Provide our own Time wrapper for ISO8601 support
+# Example:
+#   >> LogStash::Time.now.to_iso8601
+#   => "2010-10-17 00:25:24.619014-0700"
+#
+#   >> LogStash::Time.now.utc.to_iso8601
+#   => "2010-10-17 07:25:26.788704Z"
+module LogStash::Time
+  ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
+  def self.now
+    return Time.new.utc
+  end
+
+  if RUBY_PLATFORM == "java"
+    #JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser
+    #JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser
+    JODA_ISO8601_PARSER = org.joda.time.format.DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+    UTC = org.joda.time.DateTimeZone.forID("UTC")
+    def self.parse_iso8601(t)
+      millis = JODA_ISO8601_PARSER.parseMillis(t)
+      return Time.at(millis / 1000, (millis % 1000) * 1000)
+    end
+  else
+    def self.parse_iso8601(t)
+      # Warning, ruby's Time.parse is *really* terrible and slow.
+      return unless t.is_a?(String)
+      return Time.parse(t).gmtime
+    end
+  end
+end # module LogStash::Time

data/lib/logstash/util.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+module LogStash::Util
+  UNAME = case RbConfig::CONFIG["host_os"]
+    when /^linux/; "linux"
+    else; RbConfig::CONFIG["host_os"]
+  end
+
+  PR_SET_NAME = 15
+  def self.set_thread_name(name)
+    if RUBY_ENGINE == "jruby"
+      # Keep java and ruby thread names in sync.
+      Java::java.lang.Thread.currentThread.setName(name)
+    end
+    Thread.current[:name] = name
+    
+    if UNAME == "linux"
+      require "logstash/util/prctl"
+      # prctl PR_SET_NAME allows up to 16 bytes for a process name
+      # since MRI 1.9, JRuby, and Rubinius use system threads for this.
+      LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
+    end
+  end # def set_thread_name
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array) 
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue |= svalue
+          else
+            dvalue = [dvalue] | svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+ 
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  # Arrays merged will simply be appended.
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge_with_dups(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array) 
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue += svalue
+          else
+            dvalue = [dvalue] + svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  def self.hash_merge_many(*hashes)
+    dst = {}
+    hashes.each do |hash|
+      hash_merge_with_dups(dst, hash)
+    end
+    return dst
+  end # def hash_merge_many
+end # module LogStash::Util

data/lib/logstash/util/buftok.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+# BufferedTokenizer - Statefully split input data by a specifiable token
+#
+# Authors:: Tony Arcieri, Martin Emde
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Tony Arcieri and Martin Emde
+# 
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+#
+#---------------------------------------------------------------------------
+#
+
+# (C)2006 Tony Arcieri, Martin Emde
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+
+# BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
+# by default.  It allows input to be spoon-fed from some outside source which
+# receives arbitrary length datagrams which may-or-may-not contain the token
+# by which entities are delimited.
+#
+# Commonly used to parse lines out of incoming data:
+#
+#  module LineBufferedConnection
+#    def receive_data(data)
+#      (@buffer ||= BufferedTokenizer.new).extract(data).each do |line|
+#        receive_line(line)
+#      end
+#    end
+#  end
+
+module FileWatch; class BufferedTokenizer
+  # New BufferedTokenizers will operate on lines delimited by "\n" by default
+  # or allow you to specify any delimiter token you so choose, which will then
+  # be used by String#split to tokenize the input data
+  def initialize(delimiter = "\n", size_limit = nil)
+    # Store the specified delimiter
+    @delimiter = delimiter
+
+    # Store the specified size limitation
+    @size_limit = size_limit
+
+    # The input buffer is stored as an array.  This is by far the most efficient
+    # approach given language constraints (in C a linked list would be a more
+    # appropriate data structure).  Segments of input data are stored in a list
+    # which is only joined when a token is reached, substantially reducing the
+    # number of objects required for the operation.
+    @input = []
+
+    # Size of the input buffer
+    @input_size = 0
+  end
+
+  # Extract takes an arbitrary string of input data and returns an array of
+  # tokenized entities, provided there were any available to extract.  This
+  # makes for easy processing of datagrams using a pattern like:
+  #
+  #   tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
+  def extract(data)
+    # Extract token-delimited entities from the input string with the split command.
+    # There's a bit of craftiness here with the -1 parameter.  Normally split would
+    # behave no differently regardless of if the token lies at the very end of the 
+    # input buffer or not (i.e. a literal edge case)  Specifying -1 forces split to
+    # return "" in this case, meaning that the last entry in the list represents a
+    # new segment of data where the token has not been encountered
+    entities = data.split @delimiter, -1
+
+    # Check to see if the buffer has exceeded capacity, if we're imposing a limit
+    if @size_limit
+      raise 'input buffer full' if @input_size + entities.first.size > @size_limit
+      @input_size += entities.first.size
+    end
+    
+    # Move the first entry in the resulting array into the input buffer.  It represents
+    # the last segment of a token-delimited entity unless it's the only entry in the list.
+    @input << entities.shift
+
+    # If the resulting array from the split is empty, the token was not encountered
+    # (not even at the end of the buffer).  Since we've encountered no token-delimited
+    # entities this go-around, return an empty array.
+    return [] if entities.empty?
+
+    # At this point, we've hit a token, or potentially multiple tokens.  Now we can bring
+    # together all the data we've buffered from earlier calls without hitting a token,
+    # and add it to our list of discovered entities.
+    entities.unshift @input.join
+
+=begin
+    # Note added by FC, 10Jul07. This paragraph contains a regression. It breaks
+    # empty tokens. Think of the empty line that delimits an HTTP header. It will have
+    # two "\n" delimiters in a row, and this code mishandles the resulting empty token.
+    # It someone figures out how to fix the problem, we can re-enable this code branch.
+    # Multi-character token support.
+    # Split any tokens that were incomplete on the last iteration buf complete now.
+    entities.map! do |e|
+      e.split @delimiter, -1
+    end
+    # Flatten the resulting array.  This has the side effect of removing the empty
+    # entry at the end that was produced by passing -1 to split.  Add it again if
+    # necessary.
+    if (entities[-1] == [])
+      entities.flatten! << []
+    else
+      entities.flatten!
+    end
+=end
+
+    # Now that we've hit a token, joined the input buffer and added it to the entities
+    # list, we can go ahead and clear the input buffer.  All of the segments that were
+    # stored before the join can now be garbage collected.
+    @input.clear
+    
+    # The last entity in the list is not token delimited, however, thanks to the -1
+    # passed to split.  It represents the beginning of a new list of as-yet-untokenized  
+    # data, so we add it to the start of the list.
+    @input << entities.pop
+    
+    # Set the new input buffer size, provided we're keeping track
+    @input_size = @input.first.size if @size_limit
+
+    # Now we're left with the list of extracted token-delimited entities we wanted
+    # in the first place.  Hooray!
+    entities
+  end
+  
+  # Flush the contents of the input buffer, i.e. return the input buffer even though
+  # a token has not yet been encountered
+  def flush
+    buffer = @input.join
+    @input.clear
+    buffer
+  end
+
+  # Is the buffer empty?
+  def empty?
+    @input.empty?
+  end
+end; end