logstash-core 1.5.1.snapshot1-java → 1.5.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfdb72c84f82dbcc3a0fe2186266df55d9fa5ad1
-  data.tar.gz: fd6f4f326ebcabd35cbfefca906d3e0ecc1e62c2
+  metadata.gz: c1d4a0fc2ffd05d5ca9bb3d6d9c8c1ffeeed3e8e
+  data.tar.gz: 62546e37ec4e051369782ea1f3ad37d444e774aa
 SHA512:
-  metadata.gz: bbaa4a864f7c1f7ce118787f86a6786a7183d7de29788d73263b4b05a53514054ce7b8a733ee39361e652ac15899912b6e33c449a72cdbd68972634cb8c54a2b
-  data.tar.gz: ceefa0a85beab7a89b10d99b0598276beabccc4023c095ece38e1f4d589cb3415aa40975199c5457dfd2804a29cf034c82f94a3b3915387fd2107467465b76a7
+  metadata.gz: 949ed2af6b2b3100f7bf57a708c8b3081d60cb310374f19262beef13fa9c8426154f9eb85c3ba26ed06a641a1a8dd4d0198949805c53838d485047fd542d347f
+  data.tar.gz: ab49a4f44669506b310d2f80195177f563bb513320a3e0d7e44d86ddfb28deb713756e69c9fac7d73a993dd7a3134406cfdf9278f09af9863e13ba38f1329c5e

data/lib/logstash/agent.rb

@@ -172,14 +172,8 @@ class LogStash::Agent < Clamp::Command
 
     if [:info, :debug].include?(verbosity?) || debug? || verbose?
       show_version_ruby
-
-      if RUBY_PLATFORM == "java"
-        show_version_java
-      end
-
-      if [:debug].include?(verbosity?) || debug?
-        show_gems
-      end
+      show_version_java if LogStash::Environment.jruby?
+      show_gems if [:debug].include?(verbosity?) || debug?
     end
   end # def show_version
 

data/lib/logstash/environment.rb

@@ -77,7 +77,7 @@ module LogStash
     end
 
     def windows?
-      Gem.win_platform?
+      ::Gem.win_platform?
     end
 
     def vendor_path(path)

data/lib/logstash/event.rb

@@ -3,10 +3,10 @@ require "time"
 require "date"
 require "cabin"
 require "logstash/namespace"
-require "logstash/util/fieldreference"
 require "logstash/util/accessors"
 require "logstash/timestamp"
 require "logstash/json"
+require "logstash/string_interpolation"
 
 # transcient pipeline events for normal in-flow signaling as opposed to
 # flow altering exceptions. for now having base classes is adequate and
@@ -217,57 +217,10 @@ class LogStash::Event
   #
   # If a %{name} value is an array, then we will join by ','
   # If a %{name} value does not exist, then no substitution occurs.
-  #
-  # TODO(sissel): It is not clear what the value of a field that
-  # is an array (or hash?) should be. Join by comma? Something else?
   public
   def sprintf(format)
-    if format.is_a?(Float) and
-        (format < MIN_FLOAT_BEFORE_SCI_NOT or format >= MAX_FLOAT_BEFORE_SCI_NOT) then
-      format = ("%.15f" % format).sub(/0*$/,"")
-    else
-      format = format.to_s
-    end
-    if format.index("%").nil?
-      return format
-    end
-
-    return format.gsub(/%\{[^}]+\}/) do |tok|
-      # Take the inside of the %{ ... }
-      key = tok[2 ... -1]
-
-      if key[0] == "+" && !@data.has_key?(TIMESTAMP)
-        raise LogStash::Error, "Unable to format \"#{key}\" in string \"#{format}\", #{TIMESTAMP} field not found"
-      end
-
-      if key == "+%s"
-        # Got %{+%s}, support for unix epoch time
-        next @data[TIMESTAMP].to_i
-      elsif key[0,1] == "+"
-        t = @data[TIMESTAMP]
-        formatter = org.joda.time.format.DateTimeFormat.forPattern(key[1 .. -1])\
-          .withZone(org.joda.time.DateTimeZone::UTC)
-        #next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).toDateTime.toString(formatter)
-        # Invoke a specific Instant constructor to avoid this warning in JRuby
-        #  > ambiguous Java methods found, using org.joda.time.Instant(long)
-        org.joda.time.Instant.java_class.constructor(Java::long).new_instance(
-          t.tv_sec * 1000 + t.tv_usec / 1000
-        ).to_java.toDateTime.toString(formatter)
-      else
-        value = self[key]
-        case value
-          when nil
-            tok # leave the %{foo} if this field does not exist in this event.
-          when Array
-            value.join(",") # Join by ',' if value is an array
-          when Hash
-            LogStash::Json.dump(value) # Convert hashes to json
-          else
-            value # otherwise return the value
-        end # case value
-      end # 'key' checking
-    end # format.gsub...
-  end # def sprintf
+    LogStash::StringInterpolation.evaluate(self, format)
+  end
 
   def tag(value)
     # Generalize this method for more usability

data/lib/logstash/filters/base.rb

@@ -4,6 +4,7 @@ require "logstash/event"
 require "logstash/logging"
 require "logstash/plugin"
 require "logstash/config/mixin"
+require "logstash/util/decorators"
 
 class LogStash::Filters::Base < LogStash::Plugin
   include LogStash::Config::Mixin
@@ -179,21 +180,7 @@ class LogStash::Filters::Base < LogStash::Plugin
   # matches the filter's conditions (right type, etc)
   protected
   def filter_matched(event)
-    @add_field.each do |field, value|
-      field = event.sprintf(field)
-      value = [value] if !value.is_a?(Array)
-      value.each do |v|
-        v = event.sprintf(v)
-        if event.include?(field)
-          event[field] = [event[field]] if !event[field].is_a?(Array)
-          event[field] << v
-        else
-          event[field] = v
-        end
-        @logger.debug? and @logger.debug("filters/#{self.class.name}: adding value to field",
-                                       :field => field, :value => value)
-      end
-    end
+    LogStash::Util::Decorators.add_fields(@add_field,event,"filters/#{self.class.name}")
 
     @remove_field.each do |field|
       field = event.sprintf(field)
@@ -202,12 +189,7 @@ class LogStash::Filters::Base < LogStash::Plugin
       event.remove(field)
     end
 
-    @add_tag.each do |tag|
-      tag = event.sprintf(tag)
-      @logger.debug? and @logger.debug("filters/#{self.class.name}: adding tag",
-                                       :tag => tag)
-      (event["tags"] ||= []) << tag
-    end
+    LogStash::Util::Decorators.add_tags(@add_tag,event,"filters/#{self.class.name}")
 
     @remove_tag.each do |tag|
       break if event["tags"].nil?

data/lib/logstash/inputs/base.rb

@@ -5,6 +5,7 @@ require "logstash/plugin"
 require "logstash/logging"
 require "logstash/config/mixin"
 require "logstash/codecs/base"
+require "logstash/util/decorators"
 
 # This is the base class for Logstash inputs.
 class LogStash::Inputs::Base < LogStash::Plugin
@@ -109,14 +110,8 @@ class LogStash::Inputs::Base < LogStash::Plugin
     # Only set 'type' if not already set. This is backwards-compatible behavior
     event["type"] = @type if @type && !event.include?("type")
 
-    if @tags.any?
-      event["tags"] ||= []
-      event["tags"] += @tags
-    end
-
-    @add_field.each do |field, value|
-      event[field] = value
-    end
+    LogStash::Util::Decorators.add_fields(@add_field,event,"inputs/#{self.class.name}")
+    LogStash::Util::Decorators.add_tags(@tags,event,"inputs/#{self.class.name}")
   end
 
   protected

data/lib/logstash/java_integration.rb

@@ -92,4 +92,8 @@ module java::util::Collection
     duped.addAll(other)
     duped
   end
-end
+
+  def inspect
+    "<#{self.class.name}:#{self.hashCode} #{self.to_a(&:inspect)}>"
+  end
+end

data/lib/logstash/runner.rb

@@ -16,6 +16,7 @@ class LogStash::Runner
 
   def main(args)
     require "logstash/util"
+    require "logstash/util/java_version"
     require "stud/trap"
     require "stud/task"
     @startup_interruption_trap = Stud::trap("INT") { puts "Interrupted"; exit 0 }
@@ -27,6 +28,9 @@ class LogStash::Runner
       return 1
     end
 
+    # Print a warning to STDERR for bad java versions
+    LogStash::Util::JavaVersion.warn_on_bad_java_version
+
     Stud::untrap("INT", @startup_interruption_trap)
 
     task = run(args)

data/lib/logstash/string_interpolation.rb

@@ -0,0 +1,139 @@
+require "thread_safe"
+require "forwardable"
+
+module LogStash
+  module StringInterpolation
+    extend self 
+
+    # Floats outside of these upper and lower bounds are forcibly converted
+    # to scientific notation by Float#to_s
+    MIN_FLOAT_BEFORE_SCI_NOT = 0.0001
+    MAX_FLOAT_BEFORE_SCI_NOT = 1000000000000000.0
+
+    CACHE = ThreadSafe::Cache.new
+    TEMPLATE_TAG_REGEXP = /%\{[^}]+\}/
+
+    def evaluate(event, template)
+      if template.is_a?(Float) && (template < MIN_FLOAT_BEFORE_SCI_NOT || template >= MAX_FLOAT_BEFORE_SCI_NOT)
+        return ("%.15f" % template).sub(/0*$/,"")
+      end
+      
+      template = template.to_s
+      
+      return template if not_cachable?(template)
+
+      compiled = CACHE.get_or_default(template, nil) || CACHE.put(template, compile_template(template))
+      compiled.evaluate(event)
+    end
+
+    private
+    def not_cachable?(template)
+      template.index("%").nil?
+    end
+
+    def compile_template(template)
+      nodes = Template.new
+
+      position = 0
+      matches = template.to_enum(:scan, TEMPLATE_TAG_REGEXP).map { |m| $~ }
+
+      matches.each do |match|
+        tag = match[0][2..-2]
+        start = match.offset(0).first
+        nodes << StaticNode.new(template[position..(start-1)]) if start > 0
+        nodes << identify(tag)
+        position = match.offset(0).last
+      end
+
+      if position < template.size - 1
+        nodes << StaticNode.new(template[position..-1])
+      end
+
+      optimize(nodes)
+    end
+
+    def optimize(nodes)
+      nodes.size == 1 ?  nodes.first : nodes
+    end
+
+    def identify(tag)
+      if tag == "+%s"
+        EpocNode.new
+      elsif tag[0, 1] == "+"
+        DateNode.new(tag[1..-1])
+      else
+        KeyNode.new(tag)
+      end
+    end
+  end
+
+  class Template
+    extend Forwardable
+    def_delegators :@nodes, :<<, :push, :size, :first
+
+    def initialize
+      @nodes = []
+    end
+
+    def evaluate(event)
+      @nodes.collect { |node| node.evaluate(event) }.join
+    end
+  end
+
+  class EpocNode
+    def evaluate(event)
+      t = event.timestamp
+      raise LogStash::Error, "Unable to format in string \"#{@format}\", #{LogStash::Event::TIMESTAMP} field not found" unless t
+      t.to_i.to_s
+    end
+  end
+
+  class StaticNode
+    def initialize(content)
+      @content = content
+    end
+
+    def evaluate(event)
+      @content
+    end
+  end
+
+  class KeyNode
+    def initialize(key)
+      @key = key
+    end
+
+    def evaluate(event)
+      value = event[@key]
+
+      case value
+      when nil
+        "%{#{@key}}"
+      when Array
+        value.join(",")
+      when Hash
+        Logstash::Json.dump(value)
+      else
+        value
+      end
+    end
+  end
+
+  class DateNode
+    def initialize(format)
+      @format = format
+      @formatter = org.joda.time.format.DateTimeFormat.forPattern(@format)
+          .withZone(org.joda.time.DateTimeZone::UTC)
+    end
+
+    def evaluate(event)
+      t = event.timestamp
+
+      raise LogStash::Error, "Unable to format in string \"#{@format}\", #{LogStash::Event::TIMESTAMP} field not found" unless t
+
+      org.joda.time.Instant.java_class.constructor(Java::long).new_instance(
+        t.tv_sec * 1000 + t.tv_usec / 1000
+      ).to_java.toDateTime.toString(@formatter)
+    end
+  end
+end

data/lib/logstash/util/decorators.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+require "logstash/util"
+
+module LogStash::Util
+
+  # Decorators provides common manipulation on the event data.
+  module Decorators
+    extend self
+    
+    @logger = Cabin::Channel.get(LogStash)
+
+    # fields is a hash of field => value
+    # where both `field` and `value` can use sprintf syntax.
+    def add_fields(fields,event, pluginname)
+      fields.each do |field, value|
+        field = event.sprintf(field)
+        value = Array(value)
+        value.each do |v|
+          v = event.sprintf(v)
+          if event.include?(field)
+            event[field] = Array(event[field])
+            event[field] << v
+          else
+            event[field] = v
+          end
+          @logger.debug? and @logger.debug("#{pluginname}: adding value to field",
+                                         :field => field, :value => value)
+        end
+      end
+    end
+
+    # tags is an array of string. sprintf syntax can be used.
+    def add_tags(tags, event, pluginname)
+      tags.each do |tag|
+        tag = event.sprintf(tag)
+        @logger.debug? and @logger.debug("#{pluginname}: adding tag",
+                                       :tag => tag)
+        (event["tags"] ||= []) << tag
+      end
+    end
+
+  end # module LogStash::Util::Decorators
+
+end # module LogStash::Util

data/lib/logstash/util/java_version.rb

@@ -0,0 +1,61 @@
+require 'cabin'
+
+module LogStash::Util::JavaVersion
+  def self.logger
+    @logger ||= Cabin::Channel.get(LogStash)
+  end
+
+  # Print a warning if we're on a bad version of java
+  def self.warn_on_bad_java_version
+    if self.bad_java_version?(self.version)
+      msg = "!!! Please upgrade your java version, the current version '#{self.version}' may cause problems. We recommend a minimum version of 1.7.0_51"
+      STDERR.puts(msg)
+      logger.warn(msg)
+    end
+  end
+
+  # Return the current java version string. Returns nil if this is a non-java platform (e.g. MRI).
+  def self.version
+    return nil unless LogStash::Environment.jruby?
+    java.lang.System.getProperty("java.runtime.version")
+  end
+
+  # Takes a string of a java version ex: "1.8.0_24-beta"
+  # and returns a parsed map of the components.
+  # nil inputs will be returned as nil.
+  def self.parse_java_version(version_string)
+    return nil if version_string.nil?
+
+    # Crazy java versioning rules @ http://www.oracle.com/technetwork/java/javase/versioning-naming-139433.html
+    # The regex below parses this all correctly http://rubular.com/r/sInQc3Nc7f
+
+    match = version_string.match(/\A(\d+)\.(\d+)\.(\d+)(_(\d+))?(-(.+))?\Z/)
+    major, minor, patch, ufull, update, bfull, build = match.captures
+
+    {
+      :full => version_string,
+      :major => major.to_i,
+      :minor => minor.to_i,
+      :patch => patch.to_i,
+      :update => update.to_i, # this is always coerced to an int (a nil will be zero) to make comparisons easier
+      :build => build # not an integer, could be b06 for instance!,
+    }
+  end
+
+  # Determine if the given java version string is a bad version of java
+  # If it is, return true, if it isn't return false.
+  # Accepts nil, returning nil.
+  def self.bad_java_version?(version_string)
+    return nil if version_string.nil?
+
+    parsed = parse_java_version(version_string)
+
+    if parsed[:major] == 1 && parsed[:minor] == 7 && parsed[:patch] == 0 && parsed[:update] < 51
+      true
+    elsif parsed[:major] == 1 && parsed[:minor] < 7
+      true
+    else
+      false
+    end
+  end
+end

data/lib/logstash/util/unicode_trimmer.rb

@@ -0,0 +1,80 @@
+module UnicodeTrimmer
+  # The largest possible unicode chars are 4 bytes
+  # http://stackoverflow.com/questions/9533258/what-is-the-maximum-number-of-bytes-for-a-utf-8-encoded-character
+  # http://tools.ietf.org/html/rfc3629
+  MAX_CHAR_BYTES = 4
+
+  # Takes a unicode string and makes sure it fits in a max of `desired_bytes`
+  # This aims to be somewhat efficient about this for the average case and get as close to
+  # O(1) as possible. Given certain distributions of multi-byte characters it'll be slower
+  # It tries to find the point the truncation *should* happen based on the average byte size.
+  # If that snips it in the wrong place it'll try to add or remove chars to get it to the right
+  # spot and preserve as much data as possible.
+  public
+  def self.trim_bytes(orig_str, desired_bytes)
+    return orig_str if orig_str.bytesize <= desired_bytes
+
+    pre_shortened = pre_shorten(orig_str, desired_bytes)
+
+    case pre_shortened.bytesize <=> desired_bytes
+    when 0
+      pre_shortened
+    when 1
+      shrink_bytes(pre_shortened, orig_str, desired_bytes)
+    when -1
+      grow_bytes(pre_shortened, orig_str, desired_bytes)
+    end
+  end
+
+  private
+  # Try to cut the string at the right place based on the avg. byte size
+  def self.pre_shorten(orig_str, desired_bytes)
+    # Compute the average size to get an idea of where should chop
+    orig_len = orig_str.length
+    orig_bs = orig_str.bytesize
+    avg_size = (orig_bs.to_f / orig_len.to_f)
+
+    # Try to do an initial shortening based on the average char size
+    # The goal here is to get us somewhere above or below the boundary quickly
+    orig_extra_bytes = orig_bs - desired_bytes
+    pre_shorten_by = (orig_extra_bytes  / avg_size).to_i
+    orig_str.slice(0, orig_len - pre_shorten_by)
+  end
+
+  private
+  def self.grow_bytes(pre_shortened, orig_str, desired_bytes)
+    res_str = pre_shortened.clone()
+
+    loop do
+      bs = res_str.bytesize
+      deficit = desired_bytes - bs
+      lengthen_by = deficit / MAX_CHAR_BYTES
+      lengthen_by = 1 if lengthen_by < 1
+      append = orig_str.slice(res_str.length, lengthen_by)
+
+      break if (bs + append.bytesize) > desired_bytes
+
+      res_str << append
+    end
+
+    res_str
+  end
+
+  private
+  def self.shrink_bytes(pre_shortened, orig_str, desired_bytes)
+    res_str = pre_shortened.clone()
+
+    loop do
+      bs = res_str.bytesize
+      break if bs <= desired_bytes
+
+      extra = bs - desired_bytes
+      shorten_by = extra / MAX_CHAR_BYTES
+      shorten_by = 1 if shorten_by < 1
+
+      res_str.slice!(res_str.length - shorten_by)
+    end
+
+    res_str
+  end
+end

data/lib/logstash/version.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # The version of logstash.
-LOGSTASH_VERSION = "1.5.1.snapshot1"
+LOGSTASH_VERSION = "1.5.2"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

data/logstash-core.gemspec

@@ -18,10 +18,11 @@ Gem::Specification.new do |gem|
   gem.version       = LOGSTASH_VERSION.gsub(/-/, '.')
 
   gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
-  gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
+  gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
+  gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan
@@ -32,7 +33,6 @@ Gem::Specification.new do |gem|
 
   # filetools and rakelib
   gem.add_runtime_dependency "minitar", "~> 0.5.4"
-
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
   if RUBY_PLATFORM == 'java'

data/spec/core/event_spec.rb

@@ -63,6 +63,10 @@ describe LogStash::Event do
         expect(subject.sprintf("%{+HH}")).to eq("00")
       end
 
+      it "should support mixed string" do
+        expect(subject.sprintf("foo %{+YYYY-MM-dd} %{type}")).to eq("foo 2013-01-01 sprintf")
+      end
+
       it "should raise error with %{+format} syntax when @timestamp field is missing", :if => RUBY_ENGINE == "jruby" do
         str = "logstash-%{+YYYY}"
         subj = subject.clone

data/spec/coverage_helper.rb

@@ -0,0 +1,14 @@
+# Useful module to help loading all logstash content when
+# running coverage analysis
+module CoverageHelper
+
+  SKIP_LIST = ["lib/bootstrap/rspec.rb", "lib/logstash/util/prctl.rb"]
+
+  def self.eager_load
+    Dir.glob("lib/**/*.rb") do |file|
+      next if SKIP_LIST.include?(file)
+      require file
+    end
+  end
+
+end

data/spec/inputs/base_spec.rb

@@ -1,6 +1,67 @@
 # encoding: utf-8
 require "spec_helper"
 
+# use a dummy NOOP input to test Inputs::Base
+class LogStash::Inputs::NOOP < LogStash::Inputs::Base
+  config_name "noop"
+  milestone 2
+
+  def register; end
+
+end
+
+describe "LogStash::Inputs::Base#decorate" do
+  it "should add tag" do
+    input = LogStash::Inputs::NOOP.new("tags" => "value")
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["tags"]).to eq(["value"])
+  end
+
+  it "should add multiple tag" do
+    input = LogStash::Inputs::NOOP.new("tags" => ["value1","value2"])
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["tags"]).to eq(["value1","value2"])
+  end
+
+  it "should allow duplicates  tag" do
+    input = LogStash::Inputs::NOOP.new("tags" => ["value","value"])
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["tags"]).to eq(["value","value"])
+  end
+
+  it "should add tag with sprintf" do
+    input = LogStash::Inputs::NOOP.new("tags" => "%{type}")
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["tags"]).to eq(["noop"])
+  end
+
+  it "should add single field" do
+    input = LogStash::Inputs::NOOP.new("add_field" => {"field" => "value"})
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["field"]).to eq("value")
+  end
+
+  it "should add single field with sprintf" do
+    input = LogStash::Inputs::NOOP.new("add_field" => {"%{type}" => "%{type}"})
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["noop"]).to eq("noop")
+  end
+
+  it "should add multiple field" do
+    input = LogStash::Inputs::NOOP.new("add_field" => {"field" => ["value1", "value2"], "field2" => "value"})
+    evt = LogStash::Event.new({"type" => "noop"})
+    input.instance_eval {decorate(evt)}
+    expect(evt["field"]).to eq(["value1","value2"])
+    expect(evt["field2"]).to eq("value")
+  end
+end
+
 describe "LogStash::Inputs::Base#fix_streaming_codecs" do
   it "should carry the charset setting along when switching" do
     require "logstash/inputs/tcp"

data/spec/lib/logstash/java_integration_spec.rb

@@ -82,6 +82,23 @@ describe "Java integration" do
   context "Java::JavaUtil::Collection" do
     subject{Java::JavaUtil::ArrayList.new(initial_array)}
 
+    context "when inspecting" do
+      let(:items) { [:a, {:b => :c}] }
+      subject { java.util.ArrayList.new(items) }
+
+      it "should include the contents of the Collection" do
+        expect(subject.inspect).to include(items.inspect)
+      end
+
+      it "should include the class name" do
+        expect(subject.inspect).to include("ArrayList")
+      end
+
+      it "should include the hash code of the collection" do
+        expect(subject.inspect).to include(subject.hashCode.to_s)
+      end
+    end
+
     context "when deleting a unique instance" do
       let(:initial_array) {["foo", "bar"]}
 

data/spec/license_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'rakelib/default_plugins'
+
+describe "Project licenses" do
+
+  let(:expected_licenses) {
+    ##
+    # Expected licenses are Apache License 2.0, BSD license, MIT license and the ruby one,
+    # this not exclude that this list change in the feature.
+    ##
+    Regexp.union([ /mit/,
+                   /apache*/,
+                   /bsd/,
+                   /ruby/,
+                   /lgpl/])
+  }
+
+  shared_examples "runtime license test" do
+
+    subject(:gem_name) do |example|
+      example.metadata[:example_group][:parent_example_group][:description]
+    end
+
+    let(:spec) { Gem::Specification.find_all_by_name(gem_name)[0] }
+
+    it "have an expected license" do
+      spec.licenses.each do |license|
+        expect(license.downcase).to match(expected_licenses)
+      end
+    end
+
+    it "has runtime dependencies with expected licenses" do
+      spec.runtime_dependencies.map { |dep| dep.to_spec }.each do |runtime_spec|
+        next unless runtime_spec
+        runtime_spec.licenses.each do |license|
+          expect(license.downcase).to match(expected_licenses)
+        end
+      end
+    end
+  end
+
+  describe "logstash-core" do
+    it_behaves_like "runtime license test"
+  end
+
+  installed_plugins.each do |plugin|
+    describe plugin do
+      it_behaves_like "runtime license test"
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -1 +1,10 @@
+require_relative 'coverage_helper'
+# In order to archive an expected coverage analysis we need to eager load
+# all logstash code base, otherwise it will not get a good analysis.
+CoverageHelper.eager_load if ENV['COVERAGE']
+
 require "logstash/devutils/rspec/spec_helper"
+
+def installed_plugins
+  Gem::Specification.find_all.select { |spec| spec.metadata["logstash_plugin"] }.map { |plugin| plugin.name }
+end

data/spec/util/java_version_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+require 'logstash/util/java_version'
+
+describe "LogStash::Util::JavaVersion" do
+  subject(:mod) { LogStash::Util::JavaVersion }
+
+  it "should get the current java version if we're on Java" do
+    if LogStash::Environment.jruby?
+      expect(LogStash::Util::JavaVersion.version).to be_a(String)
+    end
+  end
+
+  it "should mark a bad beta version as bad" do
+    expect(mod.bad_java_version?("1.7.0_45-beta")).to be_truthy
+  end
+
+  it "should mark a bad standard version as bad" do
+    expect(mod.bad_java_version?("1.6.0")).to be_truthy
+  end
+
+  it "should mark a good standard java version as good" do
+    expect(mod.bad_java_version?("1.7.0_51")).to be_falsey
+  end
+
+  it "should mark a good beta version as good" do
+    expect(mod.bad_java_version?("1.8.0-beta")).to be_falsey
+  end
+
+  describe "parsing java versions" do
+    it "should return nil on a nil version" do
+      expect(mod.parse_java_version(nil)).to be_nil
+    end
+
+    shared_examples("version parsing") do |desc, string, major, minor, patch, update, build|
+      context("#{desc} with version #{string}") do
+        subject(:parsed) { LogStash::Util::JavaVersion.parse_java_version(string) }
+
+        it "should have the correct major version" do
+          expect(parsed[:major]).to eql(major)
+        end
+
+        it "should have the correct minor version" do
+          expect(parsed[:minor]).to eql(minor)
+        end
+
+        it "should have the correct patch version" do
+          expect(parsed[:patch]).to eql(patch)
+        end
+
+        it "should have the correct update version" do
+          expect(parsed[:update]).to eql(update)
+        end
+
+        it "should have the correct build string" do
+          expect(parsed[:build]).to eql(build)
+        end
+      end
+    end
+
+    include_examples("version parsing", "a plain version", "1.3.0", 1, 3, 0, 0, nil)
+    include_examples("version parsing", "an update", "1.4.0_03", 1, 4, 0, 3, nil)
+    include_examples("version parsing", "a build", "1.4.0-beta", 1, 4, 0, 0,"beta")
+    include_examples("version parsing", "an update+build", "1.4.0_03-beta", 1, 4, 0, 3, "beta")
+  end
+
+end

data/spec/util/unicode_trimmer_spec.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/util/unicode_trimmer"
+require "flores/rspec"
+require "flores/random"
+
+RSpec.configure do |config|
+  Flores::RSpec.configure(config)
+end
+
+describe "truncating unicode strings correctly" do
+  context "with extra bytes before the snip" do
+    let(:ustr) { "Testing «ταБЬℓσ»: 1<2 & 4+1>3, now 20% off!" }
+
+    it "should truncate to exact byte boundaries when possible" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, 21).bytesize).to eql(21)
+    end
+
+    it "should truncate below the bytesize when splitting a byte" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, 20).bytesize).to eql(18)
+    end
+
+    it "should not truncate the string when the bytesize is already OK" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
+    end
+  end
+
+  context "with extra bytes after the snip" do
+    let(:ustr) { ": 1<2 & 4+1>3, now 20% off! testing «ταБЬℓσ»" }
+
+    it "should truncate to exact byte boundaries when possible" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, 21).bytesize).to eql(21)
+    end
+
+    it "should truncate below the bytesize when splitting a byte" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, 52).bytesize).to eql(51)
+    end
+
+    it "should not truncate the string when the bytesize is already OK" do
+      expect(UnicodeTrimmer.trim_bytes(ustr, ustr.bytesize)).to eql(ustr)
+    end
+  end
+
+  context "randomized testing" do
+    let(:text) { Flores::Random.text(1..1000) }
+    let(:size) { Flores::Random.integer(1..text.bytesize) }
+    let(:expected_range) { (size - 4)..size }
+
+    stress_it "should be near the boundary of requested size" do
+      expect(expected_range).to include(UnicodeTrimmer.trim_bytes(text, size).bytesize)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 1.5.1.snapshot1
+  version: 1.5.2
 platform: java
 authors:
 - Jordan Sissel
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-11 00:00:00.000000000 Z
+date: 2015-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cabin
@@ -82,6 +82,20 @@ dependencies:
         version: 0.0.4
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: gems
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: treetop
   version_requirements: !ruby/object:Gem::Requirement
@@ -192,20 +206,23 @@ files:
 - lib/logstash/program.rb
 - lib/logstash/runner.rb
 - lib/logstash/sized_queue.rb
+- lib/logstash/string_interpolation.rb
 - lib/logstash/threadwatchdog.rb
 - lib/logstash/timestamp.rb
 - lib/logstash/util.rb
 - lib/logstash/util/accessors.rb
 - lib/logstash/util/buftok.rb
 - lib/logstash/util/charset.rb
-- lib/logstash/util/fieldreference.rb
+- lib/logstash/util/decorators.rb
 - lib/logstash/util/filetools.rb
+- lib/logstash/util/java_version.rb
 - lib/logstash/util/password.rb
 - lib/logstash/util/plugin_version.rb
 - lib/logstash/util/prctl.rb
 - lib/logstash/util/require-helper.rb
 - lib/logstash/util/retryable.rb
 - lib/logstash/util/socket_peer.rb
+- lib/logstash/util/unicode_trimmer.rb
 - lib/logstash/version.rb
 - locales/en.yml
 - logstash-core.gemspec
@@ -218,19 +235,22 @@ files:
 - spec/core/plugin_spec.rb
 - spec/core/runner_spec.rb
 - spec/core/timestamp_spec.rb
+- spec/coverage_helper.rb
 - spec/filters/base_spec.rb
 - spec/inputs/base_spec.rb
 - spec/lib/logstash/bundler_spec.rb
 - spec/lib/logstash/java_integration_spec.rb
+- spec/license_spec.rb
 - spec/logstash/agent_spec.rb
 - spec/outputs/base_spec.rb
 - spec/spec_helper.rb
 - spec/util/accessors_spec.rb
 - spec/util/charset_spec.rb
-- spec/util/fieldeval_spec.rb
 - spec/util/gemfile_spec.rb
+- spec/util/java_version_spec.rb
 - spec/util/json_spec.rb
 - spec/util/plugin_version_spec.rb
+- spec/util/unicode_trimmer_spec.rb
 - spec/util_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -247,9 +267,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
 rubygems_version: 2.2.2
@@ -266,17 +286,20 @@ test_files:
 - spec/core/plugin_spec.rb
 - spec/core/runner_spec.rb
 - spec/core/timestamp_spec.rb
+- spec/coverage_helper.rb
 - spec/filters/base_spec.rb
 - spec/inputs/base_spec.rb
 - spec/lib/logstash/bundler_spec.rb
 - spec/lib/logstash/java_integration_spec.rb
+- spec/license_spec.rb
 - spec/logstash/agent_spec.rb
 - spec/outputs/base_spec.rb
 - spec/spec_helper.rb
 - spec/util/accessors_spec.rb
 - spec/util/charset_spec.rb
-- spec/util/fieldeval_spec.rb
 - spec/util/gemfile_spec.rb
+- spec/util/java_version_spec.rb
 - spec/util/json_spec.rb
 - spec/util/plugin_version_spec.rb
+- spec/util/unicode_trimmer_spec.rb
 - spec/util_spec.rb

data/lib/logstash/util/fieldreference.rb

@@ -1,68 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/util"
-
-module LogStash::Util::FieldReference
-
-  def compile(accessor)
-    if accessor[0,1] != '['
-      return <<-"CODE"
-        lambda do |store, &block|
-          return block.nil? ? store[#{accessor.inspect}] : block.call(store, #{accessor.inspect})
-        end
-      CODE
-    end
-
-    code = "lambda do |store, &block|\n"
-    selectors = accessor.scan(/(?<=\[).+?(?=\])/)
-    selectors.each_with_index do |tok, i|
-      last = (i == selectors.count() - 1)
-      code << "   # [#{tok}]#{ last ? " (last selector)" : "" }\n"
-
-      if last
-        code << <<-"CODE"
-          return block.call(store, #{tok.inspect}) unless block.nil?
-        CODE
-      end
-
-      code << <<-"CODE"
-        store = store.is_a?(Array) ? store[#{tok.to_i}] : store[#{tok.inspect}]
-        return store if store.nil?
-      CODE
-
-    end
-    code << "return store\nend"
-    #puts code
-    return code
-  end # def compile
-
-  def exec(accessor, store, &block)
-    @__fieldeval_cache ||= {}
-    @__fieldeval_cache[accessor] ||= eval(compile(accessor))
-    return @__fieldeval_cache[accessor].call(store, &block)
-  end
-
-  def set(accessor, value, store)
-    # The assignment can fail if the given field reference (accessor) does not exist
-    # In this case, we'll want to set the value manually.
-    if exec(accessor, store) { |hash, key| hash[key] = value }.nil?
-      return (store[accessor] = value) if accessor[0,1] != "["
-
-      # No existing element was found, so let's set one.
-      *parents, key = accessor.scan(/(?<=\[)[^\]]+(?=\])/)
-      parents.each do |p|
-        if store.include?(p)
-          store = store[p]
-        else
-          store[p] = {}
-          store = store[p]
-        end
-      end
-      store[key] = value
-    end
-
-    return value
-  end
-
-  extend self
-end # module LogStash::Util::FieldReference

data/spec/util/fieldeval_spec.rb

@@ -1,96 +0,0 @@
-require "spec_helper"
-require "logstash/util/fieldreference"
-
-describe LogStash::Util::FieldReference, :if => true do
-
-  context "using simple accessor" do
-
-    it "should retrieve value" do
-      str = "hello"
-      m = eval(subject.compile(str))
-      data = { "hello" => "world" }
-      expect(m.call(data)).to eq(data[str])
-    end
-
-    it "should handle delete in block" do
-      str = "simple"
-      m = eval(subject.compile(str))
-      data = { "simple" => "things" }
-      m.call(data) { |obj, key| obj.delete(key) }
-      expect(data).to be_empty
-    end
-
-    it "should handle assignment in block" do
-      str = "simple"
-      m = eval(subject.compile(str))
-      data = {}
-      expect(m.call(data) { |obj, key| obj[key] = "things" }).to eq("things")
-      expect(data).to eq({ "simple" => "things" })
-    end
-
-    it "should handle assignment using set" do
-      str = "simple"
-      data = {}
-      expect(subject.set(str, "things", data)).to eq("things")
-      expect(data).to eq({ "simple" => "things" })
-    end
-  end
-
-  context "using accessor path" do
-
-    it "should retrieve shallow value" do
-      str = "[hello]"
-      m = eval(subject.compile(str))
-      data = { "hello" =>  "world" }
-      expect(m.call(data)).to eq("world")
-    end
-
-    it "should retrieve deep value" do
-      str = "[hello][world]"
-      m = eval(subject.compile(str))
-      data = { "hello" => { "world" => "foo", "bar" => "baz" } }
-      expect(m.call(data)).to eq(data["hello"]["world"])
-    end
-
-    it "should handle delete in block" do
-      str = "[hello][world]"
-      m = eval(subject.compile(str))
-      data = { "hello" => { "world" => "foo", "bar" => "baz" } }
-      m.call(data) { |obj, key| obj.delete(key) }
-
-      # Make sure the "world" key is removed.
-      expect(data["hello"]).to eq({ "bar" => "baz" })
-    end
-
-    it "should not handle assignment in block" do
-      str = "[hello][world]"
-      m = eval(subject.compile(str))
-      data = {}
-      expect(m.call(data) { |obj, key| obj[key] = "things" }).to be_nil
-      expect(data).to be_empty
-    end
-
-    it "should set shallow value" do
-      str = "[hello]"
-      data = {}
-      expect(subject.set(str, "foo", data)).to eq("foo")
-      expect(data).to eq({ "hello" => "foo" })
-    end
-
-    it "should set deep value" do
-      str = "[hello][world]"
-      data = {}
-      expect(subject.set(str, "foo", data)).to eq("foo")
-      expect(data).to eq({ "hello" => { "world" => "foo" } })
-    end
-
-    it "should retrieve array item" do
-      data = { "hello" => { "world" => ["a", "b"], "bar" => "baz" } }
-      m = eval(subject.compile("[hello][world][0]"))
-      expect(m.call(data)).to eq(data["hello"]["world"][0])
-
-      m = eval(subject.compile("[hello][world][1]"))
-      expect(m.call(data)).to eq(data["hello"]["world"][1])
-    end
-  end
-end