logstash-core 2.2.4.snapshot1

data/lib/logstash/util/filetools.rb

@@ -0,0 +1,186 @@
+# encoding: utf-8
+require "net/http"
+require "uri"
+require "digest/sha1"
+require 'fileutils'
+
+module LogStash::Util::FileTools
+  extend self
+
+  def fetch(url, sha1, output)
+
+    puts "Downloading #{url}"
+    actual_sha1 = download(url, output)
+
+    if actual_sha1 != sha1
+      fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
+    end
+  end # def fetch
+
+  def file_fetch(url, sha1, target)
+    filename = File.basename( URI(url).path )
+    output = "#{target}/#{filename}"
+    begin
+      actual_sha1 = file_sha1(output)
+      if actual_sha1 != sha1
+        fetch(url, sha1, output)
+      end
+    rescue Errno::ENOENT
+      fetch(url, sha1, output)
+    end
+    return output
+  end
+
+  def file_sha1(path)
+    digest = Digest::SHA1.new
+    fd = File.new(path, "r")
+    while true
+      begin
+        digest << fd.sysread(16384)
+      rescue EOFError
+        break
+      end
+    end
+    return digest.hexdigest
+  ensure
+    fd.close if fd
+  end
+
+  def download(url, output)
+    uri = URI(url)
+    digest = Digest::SHA1.new
+    tmp = "#{output}.tmp"
+    Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
+      request = Net::HTTP::Get.new(uri.path)
+      http.request(request) do |response|
+        fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
+        size = (response["content-length"].to_i || -1).to_f
+        count = 0
+        File.open(tmp, "w") do |fd|
+          response.read_body do |chunk|
+            fd.write(chunk)
+            digest << chunk
+            if size > 0 && $stdout.tty?
+              count += chunk.bytesize
+              $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
+            end
+          end
+        end
+        $stdout.write("\r      \r") if $stdout.tty?
+      end
+    end
+
+    File.rename(tmp, output)
+
+    return digest.hexdigest
+  rescue SocketError => e
+    puts "Failure while downloading #{url}: #{e}"
+    raise
+  ensure
+    File.unlink(tmp) if File.exist?(tmp)
+  end # def download
+
+  def untar(tarball, &block)
+    require "archive/tar/minitar"
+    tgz = Zlib::GzipReader.new(File.open(tarball))
+    # Pull out typesdb
+    tar = Archive::Tar::Minitar::Input.open(tgz)
+    tar.each do |entry|
+      path = block.call(entry)
+      next if path.nil?
+      parent = File.dirname(path)
+
+      FileUtils.mkdir_p(parent) unless File.directory?(parent)
+
+      # Skip this file if the output file is the same size
+      if entry.directory?
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      else
+        entry_mode = entry.instance_eval { @mode } & 0777
+        if File.exists?(path)
+          stat = File.stat(path)
+          # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
+          # expose headers in the entry.
+          entry_size = entry.instance_eval { @size }
+          # If file sizes are same, skip writing.
+          next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
+        end
+        puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
+        File.open(path, "w") do |fd|
+          # eof? check lets us skip empty files. Necessary because the API provided by
+          # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
+          # IO object. Something about empty files in this EntryStream causes
+          # IO.copy_stream to throw "can't convert nil into String" on JRuby
+          # TODO(sissel): File a bug about this.
+          while !entry.eof?
+            chunk = entry.read(16384)
+            fd.write(chunk)
+          end
+            #IO.copy_stream(entry, fd)
+        end
+        File.chmod(entry_mode, path)
+      end
+    end
+    tar.close
+    File.unlink(tarball) if File.file?(tarball)
+  end # def untar
+
+  def do_ungz(file)
+
+    outpath = file.gsub('.gz', '')
+    tgz = Zlib::GzipReader.new(File.open(file))
+    begin
+      File.open(outpath, "w") do |out|
+        IO::copy_stream(tgz, out)
+      end
+      File.unlink(file)
+    rescue
+      File.unlink(outpath) if File.file?(outpath)
+     raise
+    end
+    tgz.close
+  end
+
+  def eval_file(entry, files, prefix)
+    return false if entry.full_name =~ /PaxHeaders/
+    if !files.nil?
+      if files.is_a?(Array)
+        return false unless files.include?(entry.full_name.gsub(prefix, ''))
+        entry.full_name.split("/").last
+      elsif files.is_a?(String)
+        return false unless entry.full_name =~ Regexp.new(files)
+        entry.full_name.split("/").last
+      end
+    else
+      entry.full_name.gsub(prefix, '')
+    end
+  end
+
+  def process_downloads(files,target='')
+
+    FileUtils.mkdir_p(target) unless File.directory?(target)
+
+    files.each do |file|
+      download = file_fetch(file['url'], file['sha1'],target)
+
+      if download =~ /.tar.gz/
+        prefix = download.gsub('.tar.gz', '').gsub("#{target}/", '')
+        untar(download) do |entry|
+          next unless out = eval_file(entry, file['files'], prefix)
+          File.join(target, out)
+        end
+
+      elsif download =~ /.tgz/
+        prefix = download.gsub('.tgz', '').gsub("#{target}/", '')
+        untar(download) do |entry|
+          next unless out = eval_file(entry, file['files'], prefix)
+          File.join(target, out)
+        end
+
+      elsif download =~ /.gz/
+        do_ungz(download)
+      end
+    end
+  end
+
+end

data/lib/logstash/util/java_version.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require 'cabin'
+
+module LogStash::Util::JavaVersion
+  def self.logger
+    @logger ||= Cabin::Channel.get(LogStash)
+  end
+
+  # Print a warning if we're on a bad version of java
+  def self.warn_on_bad_java_version
+    if self.bad_java_version?(self.version)
+      msg = "!!! Please upgrade your java version, the current version '#{self.version}' may cause problems. We recommend a minimum version of 1.7.0_51"
+      STDERR.puts(msg)
+      logger.warn(msg)
+    end
+  end
+
+  # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
+  def self.version
+    return nil unless LogStash::Environment.jruby?
+    java.lang.System.getProperty("java.runtime.version")
+  end
+
+  # Takes a string of a java version ex: "1.8.0_24-beta"
+  # and returns a parsed map of the components.
+  # nil inputs will be returned as nil.
+  def self.parse_java_version(version_string)
+    return nil if version_string.nil?
+
+    # Crazy java versioning rules @ http://www.oracle.com/technetwork/java/javase/versioning-naming-139433.html
+    # The regex below parses this all correctly http://rubular.com/r/sInQc3Nc7f
+
+    match = version_string.match(/\A(\d+)\.(\d+)\.(\d+)(_(\d+))?(-(.+))?\Z/)
+
+    return nil unless match
+
+    major, minor, patch, ufull, update, bfull, build = match.captures
+
+    {
+      :full => version_string,
+      :major => major.to_i,
+      :minor => minor.to_i,
+      :patch => patch.to_i,
+      :update => update.to_i, # this is always coerced to an int (a nil will be zero) to make comparisons easier
+      :build => build # not an integer, could be b06 for instance!,
+    }
+  end
+
+  # Determine if the given java version string is a bad version of java
+  # If it is, return true, if it isn't return false.
+  # Accepts nil, returning nil.
+  def self.bad_java_version?(version_string)
+    return nil if version_string.nil?
+
+    parsed = parse_java_version(version_string)
+    return false unless parsed
+
+    if parsed[:major] == 1 && parsed[:minor] == 7 && parsed[:patch] == 0 && parsed[:update] < 51
+      true
+    elsif parsed[:major] == 1 && parsed[:minor] < 7
+      true
+    else
+      false
+    end
+  end
+end

data/lib/logstash/util/password.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+# This class exists to quietly wrap a password string so that, when printed or
+# logged, you don't accidentally print the password itself.
+class LogStash::Util::Password
+  attr_reader :value
+
+  public
+  def initialize(password)
+    @value = password
+  end # def initialize
+
+  public
+  def to_s
+    return "<password>"
+  end # def to_s
+
+  public
+  def inspect
+    return to_s
+  end # def inspect
+end # class LogStash::Util::Password
+

data/lib/logstash/util/plugin_version.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require 'logstash/errors'
+require 'rubygems/version'
+require 'forwardable'
+
+module LogStash::Util
+  class PluginVersion
+    extend Forwardable
+    include Comparable
+
+    GEM_NAME_PREFIX = 'logstash'
+
+    def_delegators :@version, :to_s
+    attr_reader :version
+
+    def initialize(*options)
+      if options.size == 1 && options.first.is_a?(Gem::Version)
+        @version = options.first
+      else
+        @version = Gem::Version.new(options.join('.'))
+      end
+    end
+
+    def self.find_version!(name)
+      begin
+        spec = Gem::Specification.find_by_name(name)
+        if spec.nil?
+          # Checking for nil? is a workaround for situations where find_by_name
+          # is not able to find the real spec, as for example with pre releases
+          # of plugins
+          spec = Gem::Specification.find_all_by_name(name).first
+        end
+        new(spec.version)
+      rescue Gem::LoadError
+        # Rescuing the LoadError and raise a Logstash specific error.
+        # Likely we can't find the gem in the current GEM_PATH
+        raise LogStash::PluginNoVersionError
+      end
+    end
+
+    def self.find_plugin_version!(type, name)
+      plugin_name = [GEM_NAME_PREFIX, type, name].join('-')
+      find_version!(plugin_name)
+    end
+
+    def <=>(other)
+      version <=> other.version
+    end
+
+    private
+
+    def self.build_from_spec(spec)
+      new(spec.version)
+    end
+  end
+end

data/lib/logstash/util/prctl.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+module LibC
+  require "ffi"
+  extend FFI::Library
+  ffi_lib 'c'
+
+  # Ok so the 2nd arg isn't really a string... but whaatever
+  attach_function :prctl, [:int, :string, :long, :long, :long], :int
+end
+

data/lib/logstash/util/retryable.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+module LogStash
+  module Retryable
+    # execute retryable code block
+    # @param [Hash] options retryable options
+    # @option options [Fixnum] :tries retries to perform, default 1, set to 0 for infite retries. 1 means that upon exception the block will be retried once
+    # @option options [Fixnum] :base_sleep seconds to sleep on first retry, default 1
+    # @option options [Fixnum] :max_sleep max seconds to sleep upon exponential backoff, default 1
+    # @option options [Exception] :rescue exception class list to retry on, defaults is Exception, which retries on any Exception.
+    # @option options [Proc] :on_retry call the given Proc/lambda before each retry with the raised exception as parameter
+    def retryable(options = {}, &block)
+      options = {
+        :tries => 1,
+        :rescue => Exception,
+        :on_retry => nil,
+        :base_sleep => 1,
+        :max_sleep => 1,
+      }.merge(options)
+
+      rescue_classes = Array(options[:rescue])
+      max_sleep_retry = Math.log2(options[:max_sleep] / options[:base_sleep])
+      retry_count = 0
+
+      begin
+        return yield(retry_count)
+      rescue *rescue_classes => e
+        raise e if options[:tries] > 0 && retry_count >= options[:tries]
+
+        options[:on_retry].call(retry_count + 1, e) if options[:on_retry]
+
+        # dont compute and maybe overflow exponent on too big a retry count
+        seconds = retry_count < max_sleep_retry ? options[:base_sleep] * (2 ** retry_count) : options[:max_sleep]
+        sleep(seconds)
+
+        retry_count += 1
+        retry
+      end
+    end
+  end
+end

data/lib/logstash/util/socket_peer.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+module ::LogStash::Util::SocketPeer
+  public
+  def peer
+    "#{peeraddr[3]}:#{peeraddr[1]}"
+  end # def peer
+end # module SocketPeer

data/lib/logstash/util/unicode_trimmer.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+module LogStash::Util::UnicodeTrimmer
+  # The largest possible unicode chars are 4 bytes
+  # http://stackoverflow.com/questions/9533258/what-is-the-maximum-number-of-bytes-for-a-utf-8-encoded-character
+  # http://tools.ietf.org/html/rfc3629
+  MAX_CHAR_BYTES = 4
+
+  # Takes a unicode string and makes sure it fits in a max of `desired_bytes`
+  # This aims to be somewhat efficient about this for the average case and get as close to
+  # O(1) as possible. Given certain distributions of multi-byte characters it'll be slower
+  # It tries to find the point the truncation *should* happen based on the average byte size.
+  # If that snips it in the wrong place it'll try to add or remove chars to get it to the right
+  # spot and preserve as much data as possible.
+  public
+  def self.trim_bytes(orig_str, desired_bytes)
+    return orig_str if orig_str.bytesize <= desired_bytes
+
+    pre_shortened = pre_shorten(orig_str, desired_bytes)
+
+    case pre_shortened.bytesize <=> desired_bytes
+    when 0
+      pre_shortened
+    when 1
+      shrink_bytes(pre_shortened, orig_str, desired_bytes)
+    when -1
+      grow_bytes(pre_shortened, orig_str, desired_bytes)
+    end
+  end
+
+  private
+  # Try to cut the string at the right place based on the avg. byte size
+  def self.pre_shorten(orig_str, desired_bytes)
+    # Compute the average size to get an idea of where should chop
+    orig_len = orig_str.length
+    orig_bs = orig_str.bytesize
+    avg_size = (orig_bs.to_f / orig_len.to_f)
+
+    # Try to do an initial shortening based on the average char size
+    # The goal here is to get us somewhere above or below the boundary quickly
+    orig_extra_bytes = orig_bs - desired_bytes
+    pre_shorten_by = (orig_extra_bytes  / avg_size).to_i
+    orig_str.slice(0, orig_len - pre_shorten_by)
+  end
+
+  private
+  def self.grow_bytes(pre_shortened, orig_str, desired_bytes)
+    res_str = pre_shortened.clone()
+
+    loop do
+      bs = res_str.bytesize
+      deficit = desired_bytes - bs
+      lengthen_by = deficit / MAX_CHAR_BYTES
+      lengthen_by = 1 if lengthen_by < 1
+      append = orig_str.slice(res_str.length, lengthen_by)
+
+      break if (bs + append.bytesize) > desired_bytes
+
+      res_str << append
+    end
+
+    res_str
+  end
+
+  private
+  def self.shrink_bytes(pre_shortened, orig_str, desired_bytes)
+    res_str = pre_shortened.clone()
+
+    loop do
+      bs = res_str.bytesize
+      break if bs <= desired_bytes
+
+      extra = bs - desired_bytes
+      shorten_by = extra / MAX_CHAR_BYTES
+      shorten_by = 1 if shorten_by < 1
+
+      res_str.slice!(res_str.length - shorten_by)
+    end
+
+    res_str
+  end
+end