logstash-core 2.2.4.snapshot1

data/lib/logstash/util.rb

@@ -0,0 +1,203 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/environment"
+
+module LogStash::Util
+  UNAME = case RbConfig::CONFIG["host_os"]
+    when /^linux/; "linux"
+    else; RbConfig::CONFIG["host_os"]
+  end
+
+  PR_SET_NAME = 15
+  def self.set_thread_name(name)
+    if RUBY_ENGINE == "jruby"
+      # Keep java and ruby thread names in sync.
+      Java::java.lang.Thread.currentThread.setName(name)
+    end
+    Thread.current[:name] = name
+
+    if UNAME == "linux"
+      require "logstash/util/prctl"
+      # prctl PR_SET_NAME allows up to 16 bytes for a process name
+      # since MRI 1.9, JRuby, and Rubinius use system threads for this.
+      LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
+    end
+  end # def set_thread_name
+
+  def self.set_thread_plugin(plugin)
+    Thread.current[:plugin] = plugin
+  end
+
+  def self.get_thread_id(thread)
+    if RUBY_ENGINE == "jruby"
+      JRuby.reference(thread).native_thread.id
+    else
+      raise Exception.new("Native thread IDs aren't supported outside of JRuby")
+    end
+  end
+
+  def self.thread_info(thread)
+    backtrace = thread.backtrace.map do |line|
+      line.gsub(LogStash::Environment::LOGSTASH_HOME, "[...]")
+    end
+
+    blocked_on = case backtrace.first
+                 when /in `push'/ then "blocked_on_push"
+                 when /(?:pipeline|base).*pop/ then "waiting_for_events"
+                 else nil
+                 end
+
+    {
+      "thread_id" => get_thread_id(thread),
+      "name" => thread[:name],
+      "plugin" => (thread[:plugin] ? thread[:plugin].debug_info : nil),
+      "backtrace" => backtrace,
+      "blocked_on" => blocked_on,
+      "status" => thread.status,
+      "current_call" => backtrace.first
+    }
+  end
+
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue |= svalue
+          else
+            dvalue = [dvalue] | svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  # Arrays merged will simply be appended.
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge_with_dups(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue += svalue
+          else
+            dvalue = [dvalue] + svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  def self.hash_merge_many(*hashes)
+    dst = {}
+    hashes.each do |hash|
+      hash_merge_with_dups(dst, hash)
+    end
+    return dst
+  end # def hash_merge_many
+
+
+  # nomalize method definition based on platform.
+  # normalize is used to convert an object create through
+  # json deserialization from JrJackson in :raw mode to pure Ruby
+  # to support these pure Ruby object monkey patches.
+  # see logstash/json.rb and logstash/java_integration.rb
+
+  if LogStash::Environment.jruby?
+    require "java"
+
+    # recursively convert any Java LinkedHashMap and ArrayList to pure Ruby.
+    # will not recurse into pure Ruby objects. Pure Ruby object should never
+    # contain LinkedHashMap and ArrayList since these are only created at
+    # initial deserialization, anything after (deeper) will be pure Ruby.
+    def self.normalize(o)
+      case o
+      when Java::JavaUtil::LinkedHashMap
+        o.inject({}){|r, (k, v)| r[k] = normalize(v); r}
+      when Java::JavaUtil::ArrayList
+        o.map{|i| normalize(i)}
+      else
+        o
+      end
+    end
+
+  else
+
+    # identity function, pure Ruby object don't need normalization.
+    def self.normalize(o); o; end
+  end
+
+  def self.stringify_symbols(o)
+    case o
+    when Hash
+      o.inject({}){|r, (k, v)| r[k.is_a?(Symbol) ? k.to_s : k] = stringify_symbols(v); r}
+    when Array
+      o.map{|i| stringify_symbols(i)}
+    when Symbol
+      o.to_s
+    else
+      o
+    end
+  end
+
+  def self.deep_clone(o)
+    case o
+    when Hash
+      o.inject({}) {|h, (k,v)| h[k] = deep_clone(v); h }
+    when Array
+      o.map {|v| deep_clone(v) }
+    when Fixnum, Symbol, IO, TrueClass, FalseClass, NilClass
+      o
+    when LogStash::Codecs::Base
+      o.clone
+    when String
+      o.clone #need to keep internal state e.g. frozen
+    else
+      Marshal.load(Marshal.dump(o))
+    end
+  end
+end # module LogStash::Util

data/lib/logstash/util/buftok.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+# BufferedTokenizer - Statefully split input data by a specifiable token
+#
+# Authors:: Tony Arcieri, Martin Emde
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Tony Arcieri and Martin Emde
+# 
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+#
+#---------------------------------------------------------------------------
+#
+
+# (C)2006 Tony Arcieri, Martin Emde
+# Distributed under the Ruby license (http://www.ruby-lang.org/en/LICENSE.txt)
+
+# BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
+# by default.  It allows input to be spoon-fed from some outside source which
+# receives arbitrary length datagrams which may-or-may-not contain the token
+# by which entities are delimited.
+#
+# Commonly used to parse lines out of incoming data:
+#
+#  module LineBufferedConnection
+#    def receive_data(data)
+#      (@buffer ||= BufferedTokenizer.new).extract(data).each do |line|
+#        receive_line(line)
+#      end
+#    end
+#  end
+
+module FileWatch; class BufferedTokenizer
+  # New BufferedTokenizers will operate on lines delimited by "\n" by default
+  # or allow you to specify any delimiter token you so choose, which will then
+  # be used by String#split to tokenize the input data
+  def initialize(delimiter = "\n", size_limit = nil)
+    # Store the specified delimiter
+    @delimiter = delimiter
+
+    # Store the specified size limitation
+    @size_limit = size_limit
+
+    # The input buffer is stored as an array.  This is by far the most efficient
+    # approach given language constraints (in C a linked list would be a more
+    # appropriate data structure).  Segments of input data are stored in a list
+    # which is only joined when a token is reached, substantially reducing the
+    # number of objects required for the operation.
+    @input = []
+
+    # Size of the input buffer
+    @input_size = 0
+  end
+
+  # Extract takes an arbitrary string of input data and returns an array of
+  # tokenized entities, provided there were any available to extract.  This
+  # makes for easy processing of datagrams using a pattern like:
+  #
+  #   tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
+  def extract(data)
+    # Extract token-delimited entities from the input string with the split command.
+    # There's a bit of craftiness here with the -1 parameter.  Normally split would
+    # behave no differently regardless of if the token lies at the very end of the 
+    # input buffer or not (i.e. a literal edge case)  Specifying -1 forces split to
+    # return "" in this case, meaning that the last entry in the list represents a
+    # new segment of data where the token has not been encountered
+    entities = data.split @delimiter, -1
+
+    # Check to see if the buffer has exceeded capacity, if we're imposing a limit
+    if @size_limit
+      raise 'input buffer full' if @input_size + entities.first.size > @size_limit
+      @input_size += entities.first.size
+    end
+    
+    # Move the first entry in the resulting array into the input buffer.  It represents
+    # the last segment of a token-delimited entity unless it's the only entry in the list.
+    @input << entities.shift
+
+    # If the resulting array from the split is empty, the token was not encountered
+    # (not even at the end of the buffer).  Since we've encountered no token-delimited
+    # entities this go-around, return an empty array.
+    return [] if entities.empty?
+
+    # At this point, we've hit a token, or potentially multiple tokens.  Now we can bring
+    # together all the data we've buffered from earlier calls without hitting a token,
+    # and add it to our list of discovered entities.
+    entities.unshift @input.join
+
+=begin
+    # Note added by FC, 10Jul07. This paragraph contains a regression. It breaks
+    # empty tokens. Think of the empty line that delimits an HTTP header. It will have
+    # two "\n" delimiters in a row, and this code mishandles the resulting empty token.
+    # It someone figures out how to fix the problem, we can re-enable this code branch.
+    # Multi-character token support.
+    # Split any tokens that were incomplete on the last iteration buf complete now.
+    entities.map! do |e|
+      e.split @delimiter, -1
+    end
+    # Flatten the resulting array.  This has the side effect of removing the empty
+    # entry at the end that was produced by passing -1 to split.  Add it again if
+    # necessary.
+    if (entities[-1] == [])
+      entities.flatten! << []
+    else
+      entities.flatten!
+    end
+=end
+
+    # Now that we've hit a token, joined the input buffer and added it to the entities
+    # list, we can go ahead and clear the input buffer.  All of the segments that were
+    # stored before the join can now be garbage collected.
+    @input.clear
+    
+    # The last entity in the list is not token delimited, however, thanks to the -1
+    # passed to split.  It represents the beginning of a new list of as-yet-untokenized  
+    # data, so we add it to the start of the list.
+    @input << entities.pop
+    
+    # Set the new input buffer size, provided we're keeping track
+    @input_size = @input.first.size if @size_limit
+
+    # Now we're left with the list of extracted token-delimited entities we wanted
+    # in the first place.  Hooray!
+    entities
+  end
+  
+  # Flush the contents of the input buffer, i.e. return the input buffer even though
+  # a token has not yet been encountered
+  def flush
+    buffer = @input.join
+    @input.clear
+    buffer
+  end
+
+  # Is the buffer empty?
+  def empty?
+    @input.empty?
+  end
+end; end

data/lib/logstash/util/charset.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+class LogStash::Util::Charset
+  attr_accessor :logger
+
+  def initialize(charset)
+    @charset = charset
+    @charset_encoding = Encoding.find(charset)
+  end
+
+  def convert(data)
+    data.force_encoding(@charset_encoding)
+
+    # NON UTF-8 charset declared.
+    # Let's convert it (as cleanly as possible) into UTF-8 so we can use it with JSON, etc.
+    return data.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace) unless @charset_encoding == Encoding::UTF_8
+
+    # UTF-8 charset declared.
+    # Some users don't know the charset of their logs or just don't know they
+    # can set the charset setting.
+    unless data.valid_encoding?
+      # A silly hack to help convert some of the unknown bytes to
+      # somewhat-readable escape codes. The [1..-2] is to trim the quotes
+      # ruby puts on the value.
+      return data.inspect[1..-2].tap do |escaped|
+        @logger.warn("Received an event that has a different character encoding than you configured.", :text => escaped, :expected_charset => @charset)
+      end
+    end
+
+    return data
+  end # def convert
+
+end # class LogStash::Util::Charset

data/lib/logstash/util/decorators.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+module LogStash::Util
+
+  # Decorators provides common manipulation on the event data.
+  module Decorators
+    extend self
+    
+    @logger = Cabin::Channel.get(LogStash)
+
+    # fields is a hash of field => value
+    # where both `field` and `value` can use sprintf syntax.
+    def add_fields(fields,event, pluginname)
+      fields.each do |field, value|
+        field = event.sprintf(field)
+        value = Array(value)
+        value.each do |v|
+          v = event.sprintf(v)
+          if event.include?(field)
+            # note below that the array field needs to be updated then reassigned to the event.
+            # this is important because a construct like event[field] << v will not work
+            # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+            a = Array(event[field])
+            a << v
+            event[field] = a
+          else
+            event[field] = v
+          end
+          @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value)
+        end
+      end
+    end
+
+    # tags is an array of string. sprintf syntax can be used.
+    def add_tags(tags, event, pluginname)
+      tags.each do |tag|
+        tag = event.sprintf(tag)
+        @logger.debug? and @logger.debug("#{pluginname}: adding tag", :tag => tag)
+        # note below that the tags array field needs to be updated then reassigned to the event.
+        # this is important because a construct like event["tags"] << tag will not work
+        # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+        tags = event["tags"] || []
+        tags << tag
+        event["tags"] = tags
+      end
+    end
+
+  end # module LogStash::Util::Decorators
+
+end # module LogStash::Util

data/lib/logstash/util/defaults_printer.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+require "logstash/util/worker_threads_default_printer"
+
+
+# This class exists to format the settings for defaults used
+module LogStash module Util class DefaultsPrinter
+  def self.print(settings)
+    new(settings).print
+  end
+
+  def initialize(settings)
+    @settings = settings
+    @printers = [workers]
+  end
+
+  def print
+    collector = []
+    @printers.each do |printer|
+      printer.visit(collector)
+    end
+    "Settings: " + collector.join(', ')
+  end
+
+  private
+
+  def workers
+    WorkerThreadsDefaultPrinter.new(@settings)
+  end
+end end end