logstash-core 1.5.0.beta2-java

data/lib/logstash/sized_queue.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+require "thread" # for SizedQueue
+class LogStash::SizedQueue < SizedQueue
+  # TODO(sissel): Soon will implement push/pop stats, etc
+end

data/lib/logstash/threadwatchdog.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::ThreadWatchdog
+  attr_accessor :logger
+  attr_accessor :threads
+
+  class TimeoutError < StandardError; end
+
+  public
+  def initialize(threads, watchdog_timeout=10)
+    @threads = threads
+    @watchdog_timeout = watchdog_timeout
+  end # def initialize
+
+  public
+  def watch
+    while sleep(1)
+      cutoff = Time.now - @watchdog_timeout
+      @threads.each do |t|
+        watchdog = t[:watchdog]
+        if watchdog and watchdog <= cutoff
+          age = Time.now - watchdog
+          @logger.fatal("thread watchdog timeout",
+                        :thread => t,
+                        :backtrace => t.backtrace,
+                        :thread_watchdog => watchdog,
+                        :age => age,
+                        :cutoff => @watchdog_timeout,
+                        :state => t[:watchdog_state])
+          raise TimeoutError, "watchdog timeout"
+        end
+      end
+    end
+  end # def watch
+end # class LogStash::ThreadWatchdog

data/lib/logstash/timestamp.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+require "logstash/environment"
+require "logstash/json"
+require "forwardable"
+require "date"
+require "time"
+
+module LogStash
+  class TimestampParserError < StandardError; end
+
+  class Timestamp
+    extend Forwardable
+    include Comparable
+
+    def_delegators :@time, :tv_usec, :usec, :year, :iso8601, :to_i, :tv_sec, :to_f, :to_edn, :<=>, :+
+
+    attr_reader :time
+
+    ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
+    ISO8601_PRECISION = 3
+
+    def initialize(time = Time.new)
+      @time = time.utc
+    end
+
+    def self.at(*args)
+      Timestamp.new(::Time.at(*args))
+    end
+
+    def self.parse(*args)
+      Timestamp.new(::Time.parse(*args))
+    end
+
+    def self.now
+      Timestamp.new(::Time.now)
+    end
+
+    # coerce tries different strategies based on the time object class to convert into a Timestamp.
+    # @param [String, Time, Timestamp] time the time object to try coerce
+    # @return [Timestamp, nil] Timestamp will be returned if successful otherwise nil
+    # @raise [TimestampParserError] on String with invalid format
+    def self.coerce(time)
+      case time
+      when String
+        LogStash::Timestamp.parse_iso8601(time)
+      when LogStash::Timestamp
+        time
+      when Time
+        LogStash::Timestamp.new(time)
+      else
+        nil
+      end
+    end
+
+    if LogStash::Environment.jruby?
+      JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser
+      UTC = org.joda.time.DateTimeZone.forID("UTC")
+
+      def self.parse_iso8601(t)
+        millis = JODA_ISO8601_PARSER.parseMillis(t)
+        LogStash::Timestamp.at(millis / 1000, (millis % 1000) * 1000)
+      rescue => e
+        raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}")
+      end
+
+    else
+
+      def self.parse_iso8601(t)
+        # warning, ruby's Time.parse is *really* terrible and slow.
+        LogStash::Timestamp.new(::Time.parse(t))
+      rescue => e
+        raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}")
+      end
+    end
+
+    def utc
+      @time.utc # modifies the receiver
+      self
+    end
+    alias_method :gmtime, :utc
+
+    def to_json(*args)
+      # ignore arguments to respect accepted to_json method signature
+      "\"" + to_iso8601 + "\""
+    end
+    alias_method :inspect, :to_json
+
+    def to_iso8601
+      @iso8601 ||= @time.iso8601(ISO8601_PRECISION)
+    end
+    alias_method :to_s, :to_iso8601
+
+    def -(value)
+      @time - (value.is_a?(Timestamp) ? value.time : value)
+    end
+  end
+end

data/lib/logstash/util.rb

@@ -0,0 +1,152 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/environment"
+
+module LogStash::Util
+  UNAME = case RbConfig::CONFIG["host_os"]
+    when /^linux/; "linux"
+    else; RbConfig::CONFIG["host_os"]
+  end
+
+  PR_SET_NAME = 15
+  def self.set_thread_name(name)
+    if RUBY_ENGINE == "jruby"
+      # Keep java and ruby thread names in sync.
+      Java::java.lang.Thread.currentThread.setName(name)
+    end
+    Thread.current[:name] = name
+
+    if UNAME == "linux"
+      require "logstash/util/prctl"
+      # prctl PR_SET_NAME allows up to 16 bytes for a process name
+      # since MRI 1.9, JRuby, and Rubinius use system threads for this.
+      LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
+    end
+  end # def set_thread_name
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue |= svalue
+          else
+            dvalue = [dvalue] | svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  # Arrays merged will simply be appended.
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge_with_dups(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue += svalue
+          else
+            dvalue = [dvalue] + svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  def self.hash_merge_many(*hashes)
+    dst = {}
+    hashes.each do |hash|
+      hash_merge_with_dups(dst, hash)
+    end
+    return dst
+  end # def hash_merge_many
+
+
+  # nomalize method definition based on platform.
+  # normalize is used to convert an object create through
+  # json deserialization from JrJackson in :raw mode to pure Ruby
+  # to support these pure Ruby object monkey patches.
+  # see logstash/json.rb and logstash/java_integration.rb
+
+  if LogStash::Environment.jruby?
+    require "java"
+
+    # recursively convert any Java LinkedHashMap and ArrayList to pure Ruby.
+    # will not recurse into pure Ruby objects. Pure Ruby object should never
+    # contain LinkedHashMap and ArrayList since these are only created at
+    # initial deserialization, anything after (deeper) will be pure Ruby.
+    def self.normalize(o)
+      case o
+      when Java::JavaUtil::LinkedHashMap
+        o.inject({}){|r, (k, v)| r[k] = normalize(v); r}
+      when Java::JavaUtil::ArrayList
+        o.map{|i| normalize(i)}
+      else
+        o
+      end
+    end
+
+  else
+
+    # identity function, pure Ruby object don't need normalization.
+    def self.normalize(o); o; end
+  end
+
+  def self.stringify_symbols(o)
+    case o
+    when Hash
+      o.inject({}){|r, (k, v)| r[k.is_a?(Symbol) ? k.to_s : k] = stringify_symbols(v); r}
+    when Array
+      o.map{|i| stringify_symbols(i)}
+    when Symbol
+      o.to_s
+    else
+      o
+    end
+  end
+
+end # module LogStash::Util

data/lib/logstash/util/accessors.rb

@@ -0,0 +1,88 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+require "logstash/util"
+
+module LogStash::Util
+
+  # PathCache is a singleton which globally caches a parsed fields path for the path to the
+  # container hash and key in that hash.
+  module PathCache
+    extend self
+
+    def get(accessor)
+      @cache ||= {}
+      @cache[accessor] ||= parse(accessor)
+    end
+
+    def parse(accessor)
+      path = accessor.split(/[\[\]]/).select{|s| !s.empty?}
+      [path.pop, path]
+    end
+  end
+
+  # Accessors uses a lookup table to speedup access of an accessor field of the type
+  # "[hello][world]" to the underlying store hash into {"hello" => {"world" => "foo"}}
+  class Accessors
+
+    def initialize(store)
+      @store = store
+      @lut = {}
+    end
+
+    def get(accessor)
+      target, key = lookup(accessor)
+      unless target.nil?
+        target.is_a?(Array) ? target[key.to_i] : target[key]
+      end
+    end
+
+    def set(accessor, value)
+      target, key = store_and_lookup(accessor)
+      target[target.is_a?(Array) ? key.to_i : key] = value
+    end
+
+    def strict_set(accessor, value)
+      set(accessor, LogStash::Event.validate_value(value))
+    end
+
+    def del(accessor)
+      target, key = lookup(accessor)
+      unless target.nil?
+        target.is_a?(Array) ? target.delete_at(key.to_i) : target.delete(key)
+      end
+    end
+
+    private
+
+    def lookup(accessor)
+      target, key = lookup_path(accessor)
+      if target.nil?
+        [target, key]
+      else
+        @lut[accessor] = [target, key]
+      end
+    end
+
+    def store_and_lookup(accessor)
+      @lut[accessor] ||= store_path(accessor)
+    end
+
+    def lookup_path(accessor)
+      key, path = PathCache.get(accessor)
+      target = path.inject(@store) do |r, k|
+        if r.nil?
+          return nil
+        end
+        r[r.is_a?(Array) ? k.to_i : k]
+      end
+      [target, key]
+    end
+
+    def store_path(accessor)
+      key, path = PathCache.get(accessor)
+      target = path.inject(@store) {|r, k| r[r.is_a?(Array) ? k.to_i : k] ||= {}}
+      [target, key]
+    end
+  end # class Accessors
+end # module LogStash::Util

data/lib/logstash/util/buftok.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+# BufferedTokenizer - Statefully split input data by a specifiable token
+#
+# Authors:: Tony Arcieri, Martin Emde
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Tony Arcieri and Martin Emde
+# 
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+#
+#---------------------------------------------------------------------------
+#
+
+# (C)2006 Tony Arcieri, Martin Emde
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+
+# BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
+# by default.  It allows input to be spoon-fed from some outside source which
+# receives arbitrary length datagrams which may-or-may-not contain the token
+# by which entities are delimited.
+#
+# Commonly used to parse lines out of incoming data:
+#
+#  module LineBufferedConnection
+#    def receive_data(data)
+#      (@buffer ||= BufferedTokenizer.new).extract(data).each do |line|
+#        receive_line(line)
+#      end
+#    end
+#  end
+
+module FileWatch; class BufferedTokenizer
+  # New BufferedTokenizers will operate on lines delimited by "\n" by default
+  # or allow you to specify any delimiter token you so choose, which will then
+  # be used by String#split to tokenize the input data
+  def initialize(delimiter = "\n", size_limit = nil)
+    # Store the specified delimiter
+    @delimiter = delimiter
+
+    # Store the specified size limitation
+    @size_limit = size_limit
+
+    # The input buffer is stored as an array.  This is by far the most efficient
+    # approach given language constraints (in C a linked list would be a more
+    # appropriate data structure).  Segments of input data are stored in a list
+    # which is only joined when a token is reached, substantially reducing the
+    # number of objects required for the operation.
+    @input = []
+
+    # Size of the input buffer
+    @input_size = 0
+  end
+
+  # Extract takes an arbitrary string of input data and returns an array of
+  # tokenized entities, provided there were any available to extract.  This
+  # makes for easy processing of datagrams using a pattern like:
+  #
+  #   tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
+  def extract(data)
+    # Extract token-delimited entities from the input string with the split command.
+    # There's a bit of craftiness here with the -1 parameter.  Normally split would
+    # behave no differently regardless of if the token lies at the very end of the 
+    # input buffer or not (i.e. a literal edge case)  Specifying -1 forces split to
+    # return "" in this case, meaning that the last entry in the list represents a
+    # new segment of data where the token has not been encountered
+    entities = data.split @delimiter, -1
+
+    # Check to see if the buffer has exceeded capacity, if we're imposing a limit
+    if @size_limit
+      raise 'input buffer full' if @input_size + entities.first.size > @size_limit
+      @input_size += entities.first.size
+    end
+    
+    # Move the first entry in the resulting array into the input buffer.  It represents
+    # the last segment of a token-delimited entity unless it's the only entry in the list.
+    @input << entities.shift
+
+    # If the resulting array from the split is empty, the token was not encountered
+    # (not even at the end of the buffer).  Since we've encountered no token-delimited
+    # entities this go-around, return an empty array.
+    return [] if entities.empty?
+
+    # At this point, we've hit a token, or potentially multiple tokens.  Now we can bring
+    # together all the data we've buffered from earlier calls without hitting a token,
+    # and add it to our list of discovered entities.
+    entities.unshift @input.join
+
+=begin
+    # Note added by FC, 10Jul07. This paragraph contains a regression. It breaks
+    # empty tokens. Think of the empty line that delimits an HTTP header. It will have
+    # two "\n" delimiters in a row, and this code mishandles the resulting empty token.
+    # It someone figures out how to fix the problem, we can re-enable this code branch.
+    # Multi-character token support.
+    # Split any tokens that were incomplete on the last iteration buf complete now.
+    entities.map! do |e|
+      e.split @delimiter, -1
+    end
+    # Flatten the resulting array.  This has the side effect of removing the empty
+    # entry at the end that was produced by passing -1 to split.  Add it again if
+    # necessary.
+    if (entities[-1] == [])
+      entities.flatten! << []
+    else
+      entities.flatten!
+    end
+=end
+
+    # Now that we've hit a token, joined the input buffer and added it to the entities
+    # list, we can go ahead and clear the input buffer.  All of the segments that were
+    # stored before the join can now be garbage collected.
+    @input.clear
+    
+    # The last entity in the list is not token delimited, however, thanks to the -1
+    # passed to split.  It represents the beginning of a new list of as-yet-untokenized  
+    # data, so we add it to the start of the list.
+    @input << entities.pop
+    
+    # Set the new input buffer size, provided we're keeping track
+    @input_size = @input.first.size if @size_limit
+
+    # Now we're left with the list of extracted token-delimited entities we wanted
+    # in the first place.  Hooray!
+    entities
+  end
+  
+  # Flush the contents of the input buffer, i.e. return the input buffer even though
+  # a token has not yet been encountered
+  def flush
+    buffer = @input.join
+    @input.clear
+    buffer
+  end
+
+  # Is the buffer empty?
+  def empty?
+    @input.empty?
+  end
+end; end