logstash-core 1.5.0.beta2-java

data/lib/logstash/util/charset.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+class LogStash::Util::Charset
+  attr_accessor :logger
+
+  def initialize(charset)
+    @charset = charset
+    @charset_encoding = Encoding.find(charset)
+  end
+
+  def convert(data)
+    data.force_encoding(@charset_encoding)
+
+    # NON UTF-8 charset declared.
+    # Let's convert it (as cleanly as possible) into UTF-8 so we can use it with JSON, etc.
+    return data.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace) unless @charset_encoding == Encoding::UTF_8
+
+    # UTF-8 charset declared.
+    # Some users don't know the charset of their logs or just don't know they
+    # can set the charset setting.
+    unless data.valid_encoding?
+      # A silly hack to help convert some of the unknown bytes to
+      # somewhat-readable escape codes. The [1..-2] is to trim the quotes
+      # ruby puts on the value.
+      return data.inspect[1..-2].tap do |escaped|
+        @logger.warn("Received an event that has a different character encoding than you configured.", :text => escaped, :expected_charset => @charset)
+      end
+    end
+
+    return data
+  end # def convert
+
+end # class LogStash::Util::Charset

data/lib/logstash/util/fieldreference.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+module LogStash::Util::FieldReference
+
+  def compile(accessor)
+    if accessor[0,1] != '['
+      return <<-"CODE"
+        lambda do |store, &block|
+          return block.nil? ? store[#{accessor.inspect}] : block.call(store, #{accessor.inspect})
+        end
+      CODE
+    end
+
+    code = "lambda do |store, &block|\n"
+    selectors = accessor.scan(/(?<=\[).+?(?=\])/)
+    selectors.each_with_index do |tok, i|
+      last = (i == selectors.count() - 1)
+      code << "   # [#{tok}]#{ last ? " (last selector)" : "" }\n"
+
+      if last
+        code << <<-"CODE"
+          return block.call(store, #{tok.inspect}) unless block.nil?
+        CODE
+      end
+
+      code << <<-"CODE"
+        store = store.is_a?(Array) ? store[#{tok.to_i}] : store[#{tok.inspect}]
+        return store if store.nil?
+      CODE
+
+    end
+    code << "return store\nend"
+    #puts code
+    return code
+  end # def compile
+
+  def exec(accessor, store, &block)
+    @__fieldeval_cache ||= {}
+    @__fieldeval_cache[accessor] ||= eval(compile(accessor))
+    return @__fieldeval_cache[accessor].call(store, &block)
+  end
+
+  def set(accessor, value, store)
+    # The assignment can fail if the given field reference (accessor) does not exist
+    # In this case, we'll want to set the value manually.
+    if exec(accessor, store) { |hash, key| hash[key] = value }.nil?
+      return (store[accessor] = value) if accessor[0,1] != "["
+
+      # No existing element was found, so let's set one.
+      *parents, key = accessor.scan(/(?<=\[)[^\]]+(?=\])/)
+      parents.each do |p|
+        if store.include?(p)
+          store = store[p]
+        else
+          store[p] = {}
+          store = store[p]
+        end
+      end
+      store[key] = value
+    end
+
+    return value
+  end
+
+  extend self
+end # module LogStash::Util::FieldReference

data/lib/logstash/util/filetools.rb

@@ -0,0 +1,185 @@
+require "net/http"
+require "uri"
+require "digest/sha1"
+require 'fileutils'
+
+module LogStash::Util::FileTools
+  extend self
+
+  def fetch(url, sha1, output)
+
+    puts "Downloading #{url}"
+    actual_sha1 = download(url, output)
+
+    if actual_sha1 != sha1
+      fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
+    end
+  end # def fetch
+
+  def file_fetch(url, sha1, target)
+    filename = File.basename( URI(url).path )
+    output = "#{target}/#{filename}"
+    begin
+      actual_sha1 = file_sha1(output)
+      if actual_sha1 != sha1
+        fetch(url, sha1, output)
+      end
+    rescue Errno::ENOENT
+      fetch(url, sha1, output)
+    end
+    return output
+  end
+
+  def file_sha1(path)
+    digest = Digest::SHA1.new
+    fd = File.new(path, "r")
+    while true
+      begin
+        digest << fd.sysread(16384)
+      rescue EOFError
+        break
+      end
+    end
+    return digest.hexdigest
+  ensure
+    fd.close if fd
+  end
+
+  def download(url, output)
+    uri = URI(url)
+    digest = Digest::SHA1.new
+    tmp = "#{output}.tmp"
+    Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
+      request = Net::HTTP::Get.new(uri.path)
+      http.request(request) do |response|
+        fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
+        size = (response["content-length"].to_i || -1).to_f
+        count = 0
+        File.open(tmp, "w") do |fd|
+          response.read_body do |chunk|
+            fd.write(chunk)
+            digest << chunk
+            if size > 0 && $stdout.tty?
+              count += chunk.bytesize
+              $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
+            end
+          end
+        end
+        $stdout.write("\r      \r") if $stdout.tty?
+      end
+    end
+
+    File.rename(tmp, output)
+
+    return digest.hexdigest
+  rescue SocketError => e
+    puts "Failure while downloading #{url}: #{e}"
+    raise
+  ensure
+    File.unlink(tmp) if File.exist?(tmp)
+  end # def download
+
+  def untar(tarball, &block)
+    require "archive/tar/minitar"
+    tgz = Zlib::GzipReader.new(File.open(tarball))
+    # Pull out typesdb
+    tar = Archive::Tar::Minitar::Input.open(tgz)
+    tar.each do |entry|
+      path = block.call(entry)
+      next if path.nil?
+      parent = File.dirname(path)
+
+      FileUtils.mkdir_p(parent) unless File.directory?(parent)
+
+      # Skip this file if the output file is the same size
+      if entry.directory?
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      else
+        entry_mode = entry.instance_eval { @mode } & 0777
+        if File.exists?(path)
+          stat = File.stat(path)
+          # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
+          # expose headers in the entry.
+          entry_size = entry.instance_eval { @size }
+          # If file sizes are same, skip writing.
+          next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
+        end
+        puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
+        File.open(path, "w") do |fd|
+          # eof? check lets us skip empty files. Necessary because the API provided by
+          # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
+          # IO object. Something about empty files in this EntryStream causes
+          # IO.copy_stream to throw "can't convert nil into String" on JRuby
+          # TODO(sissel): File a bug about this.
+          while !entry.eof?
+            chunk = entry.read(16384)
+            fd.write(chunk)
+          end
+            #IO.copy_stream(entry, fd)
+        end
+        File.chmod(entry_mode, path)
+      end
+    end
+    tar.close
+    File.unlink(tarball) if File.file?(tarball)
+  end # def untar
+
+  def do_ungz(file)
+
+    outpath = file.gsub('.gz', '')
+    tgz = Zlib::GzipReader.new(File.open(file))
+    begin
+      File.open(outpath, "w") do |out|
+        IO::copy_stream(tgz, out)
+      end
+      File.unlink(file)
+    rescue
+      File.unlink(outpath) if File.file?(outpath)
+     raise
+    end
+    tgz.close
+  end
+
+  def eval_file(entry, files, prefix)
+    return false if entry.full_name =~ /PaxHeaders/
+    if !files.nil?
+      if files.is_a?(Array)
+        return false unless files.include?(entry.full_name.gsub(prefix, ''))
+        entry.full_name.split("/").last
+      elsif files.is_a?(String)
+        return false unless entry.full_name =~ Regexp.new(files)
+        entry.full_name.split("/").last
+      end
+    else
+      entry.full_name.gsub(prefix, '')
+    end
+  end
+
+  def process_downloads(files,target='')
+
+    FileUtils.mkdir_p(target) unless File.directory?(target)
+
+    files.each do |file|
+      download = file_fetch(file['url'], file['sha1'],target)
+
+      if download =~ /.tar.gz/
+        prefix = download.gsub('.tar.gz', '').gsub("#{target}/", '')
+        untar(download) do |entry|
+          next unless out = eval_file(entry, file['files'], prefix)
+          File.join(target, out)
+        end
+
+      elsif download =~ /.tgz/
+        prefix = download.gsub('.tgz', '').gsub("#{target}/", '')
+        untar(download) do |entry|
+          next unless out = eval_file(entry, file['files'], prefix)
+          File.join(target, out)
+        end
+
+      elsif download =~ /.gz/
+        do_ungz(download)
+      end
+    end
+  end
+
+end

data/lib/logstash/util/password.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+# This class exists to quietly wrap a password string so that, when printed or
+# logged, you don't accidentally print the password itself.
+class LogStash::Util::Password
+  attr_reader :value
+
+  public
+  def initialize(password)
+    @value = password
+  end # def initialize
+
+  public
+  def to_s
+    return "<password>"
+  end # def to_s
+
+  public
+  def inspect
+    return to_s
+  end # def inspect
+end # class LogStash::Util::Password
+

data/lib/logstash/util/plugin_version.rb

@@ -0,0 +1,43 @@
+require 'logstash/errors'
+require 'rubygems/version'
+require 'forwardable'
+
+module LogStash::Util
+  class PluginVersion
+    extend Forwardable
+    include Comparable
+
+    GEM_NAME_PREFIX = 'logstash'
+
+    def_delegators :@version, :to_s
+    attr_reader :version
+
+    def initialize(*options)
+      if options.size == 1 && options.first.is_a?(Gem::Version)
+        @version = options.first
+      else
+        @version = Gem::Version.new(options.join('.'))
+      end
+    end
+
+    def self.find_version!(name)
+      begin
+        specification = Gem::Specification.find_by_name(name)
+        new(specification.version)
+      rescue Gem::LoadError
+        # Rescuing the LoadError and raise a Logstash specific error.
+        # Likely we can't find the gem in the current GEM_PATH
+        raise LogStash::PluginNoVersionError
+      end
+    end
+
+    def self.find_plugin_version!(type, name)
+      plugin_name = [GEM_NAME_PREFIX, type, name].join('-')
+      find_version!(plugin_name)
+    end
+
+    def <=>(other)
+      version <=> other.version
+    end
+  end
+end

data/lib/logstash/util/prctl.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module LibC
+  require "ffi"
+  extend FFI::Library
+  ffi_lib 'c'
+
+  # Ok so the 2nd arg isn't really a string... but whaatever
+  attach_function :prctl, [:int, :string, :long, :long, :long], :int
+end
+

data/lib/logstash/util/require-helper.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+
+module LogStash::Util::Require
+  class << self
+    attr_accessor :logger
+
+    def require(lib, gemdep, message=nil)
+      @logger ||= LogStash::Logger.new(STDERR)
+      begin
+        require lib
+      rescue LoadError => e
+        @logger.error("Failed loading '#{lib}'")
+      end
+    end # def require
+  end # class << self
+end # def LogStash::Util::Require

data/lib/logstash/util/retryable.rb

@@ -0,0 +1,39 @@
+module LogStash
+  module Retryable
+    # execute retryable code block
+    # @param [Hash] options retryable options
+    # @option options [Fixnum] :tries retries to perform, default 1, set to 0 for infite retries. 1 means that upon exception the block will be retried once
+    # @option options [Fixnum] :base_sleep seconds to sleep on first retry, default 1
+    # @option options [Fixnum] :max_sleep max seconds to sleep upon exponential backoff, default 1
+    # @option options [Exception] :rescue exception class list to retry on, defaults is Exception, which retries on any Exception.
+    # @option options [Proc] :on_retry call the given Proc/lambda before each retry with the raised exception as parameter
+    def retryable(options = {}, &block)
+      options = {
+        :tries => 1,
+        :rescue => Exception,
+        :on_retry => nil,
+        :base_sleep => 1,
+        :max_sleep => 1,
+      }.merge(options)
+
+      rescue_classes = Array(options[:rescue])
+      max_sleep_retry = Math.log2(options[:max_sleep] / options[:base_sleep])
+      retry_count = 0
+
+      begin
+        return yield(retry_count)
+      rescue *rescue_classes => e
+        raise e if options[:tries] > 0 && retry_count >= options[:tries]
+
+        options[:on_retry].call(retry_count + 1, e) if options[:on_retry]
+
+        # dont compute and maybe overflow exponent on too big a retry count
+        seconds = retry_count < max_sleep_retry ? options[:base_sleep] * (2 ** retry_count) : options[:max_sleep]
+        sleep(seconds)
+
+        retry_count += 1
+        retry
+      end
+    end
+  end
+end

data/lib/logstash/util/socket_peer.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+module ::LogStash::Util::SocketPeer
+  public
+  def peer
+    "#{peeraddr[3]}:#{peeraddr[1]}"
+  end # def peer
+end # module SocketPeer

data/lib/logstash/version.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+# The version of logstash.
+LOGSTASH_VERSION = "1.5.0.beta2"
+
+# Note to authors: this should not include dashes because 'gem' barfs if
+# you include a dash in the version string.