logstash-core 1.5.0.rc2.snapshot-java → 1.5.0.rc3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d71faa6ec440c5cb9057a50d51147d590b25984
-  data.tar.gz: 52f21ca86f005baa4e5eb3c3886040e1657fc9a2
+  metadata.gz: 33f2eca5fac0bd51fb51dcaa864f11e5dad12614
+  data.tar.gz: e16ce45358620cd81a1044b42f31744ac4106274
 SHA512:
-  metadata.gz: 2202931d2edce9a77cb4a8b9f84a36c1b9ff1a9c38a1cf319ab8f3b48b65eb59654e4841c17ef7f062ba92f4b0b9ab082d227702118b51f94b513b1d45f3a20d
-  data.tar.gz: e0fa89dfd73c248c93999986ca023716a24ea62670fa2781e58b6d7d2edde5f2f280319aac23ea48ac83cffab289b4031b7443d0e795bb1c9567495d53d3b078
+  metadata.gz: 6fed2572b9d8d055330b405efe6524c9a9cbc228b946bca904ec9967206d6c7753e8f6a08ac8a3b3cfce6ef1cb36c62a9d6d876ab551ba5b711b018343e5a96c
+  data.tar.gz: 88c502df536ec73769152280fac1172737db4c7b49bc1c081b2ed821b3f2b185472adf8185095d876e95e26f6108c13e6a8ad5df0f42bd3d8f3b3a970c34d729

data/lib/logstash/agent.rb

@@ -172,7 +172,6 @@ class LogStash::Agent < Clamp::Command
 
       if RUBY_PLATFORM == "java"
         show_version_java
-        show_version_elasticsearch
       end
 
       if [:debug].include?(verbosity?) || debug?
@@ -190,13 +189,6 @@ class LogStash::Agent < Clamp::Command
     puts RUBY_DESCRIPTION
   end # def show_version_ruby
 
-  def show_version_elasticsearch
-    LogStash::Environment.load_elasticsearch_jars!
-
-    $stdout.write("Elasticsearch: ");
-    org.elasticsearch.Version::main([])
-  end # def show_version_elasticsearch
-
   def show_version_java
     properties = java.lang.System.getProperties
     puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"

data/lib/logstash/bundler.rb

@@ -1,28 +1,55 @@
-require "bundler"
-require "bundler/cli"
-
-module Bundler
-  # Patch bundler to write a .lock file specific to the version of ruby.
-  # This keeps MRI/JRuby/RBX from conflicting over the Gemfile.lock updates
-  module SharedHelpers
-    def default_lockfile
-      ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}"
-      Pathname.new("#{default_gemfile}.#{ruby}.lock")
-    end
-  end
-
-  # Add the Bundler.reset! method which has been added in master but is not in 1.7.9.
-  class << self
-    unless self.method_defined?("reset!")
-      def reset!
-        @definition = nil
-      end
-    end
-  end
-end
+require "logstash/environment"
 
 module LogStash
   module Bundler
+    # Take a gem package and extract it to a specific target
+    # @param [String] Gem file, this must be a path
+    # @param [String, String] Return a Gem::Package and the installed path
+    def self.unpack(file, path)
+      require "rubygems/package"
+      require "securerandom"
+
+      # We are creating a random directory per extract,
+      # if we dont do this bundler will not trigger download of the dependencies.
+      # Use case is:
+      # - User build his own gem with a fix
+      # - User doesnt increment the version
+      # - User install the same version but different code or dependencies multiple times..
+      basename  = ::File.basename(file, '.gem') 
+      unique = SecureRandom.hex(4)
+      target_path = ::File.expand_path(::File.join(path, unique, basename))
+      
+      package = ::Gem::Package.new(file)
+      package.extract_files(target_path)
+
+      return [package, target_path]
+    end
+
+    def self.setup!(options = {})
+      options = {:without => [:development]}.merge(options)
+      options[:without] = Array(options[:without])
+
+      # make sure we use our own installed bundler
+      require "logstash/patches/rubygems" # patch rubygems before clear_paths
+      ::Gem.clear_paths
+      ::Gem.paths = ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home
+
+      # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s)
+      ENV["BUNDLE_GEMFILE"] = LogStash::Environment::GEMFILE_PATH
+
+      require "bundler"
+      require "logstash/bundler"
+      require "logstash/patches/bundler"
+
+      ::Bundler.settings[:path]    = LogStash::Environment::BUNDLE_DIR
+      ::Bundler.settings[:without] = options[:without].join(":")
+      # in the context of Bundler.setup it looks like this is useless here because Gemfile path can only be specified using
+      # the ENV, see https://github.com/bundler/bundler/blob/v1.8.3/lib/bundler/shared_helpers.rb#L103
+      ::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
+
+      ::Bundler.reset!
+      ::Bundler.setup
+    end
 
     # capture any $stdout from the passed block. also trap any exception in that block, in which case the trapped exception will be returned
     # @param [Proc] the code block to execute
@@ -51,7 +78,22 @@ module LogStash
       options[:without] = Array(options[:without])
       options[:update] = Array(options[:update]) if options[:update]
 
-      ENV["GEM_PATH"] = LogStash::Environment.logstash_gem_home
+      # make sure we use our own installed bundler
+      require "logstash/patches/rubygems" # patch rubygems before clear_paths
+      ::Gem.clear_paths
+      ::Gem.paths = ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home
+
+      # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s).
+      # in the context of calling Bundler::CLI this is not really required since Bundler::CLI will look at
+      # Bundler.settings[:gemfile] unlike Bundler.setup. For the sake of consistency and defensive/future proofing, let's keep it here.
+      ENV["BUNDLE_GEMFILE"] = LogStash::Environment::GEMFILE_PATH
+
+      require "bundler"
+      require "bundler/cli"
+      require "logstash/patches/bundler"
+
+      # force Rubygems sources to our Gemfile sources
+      ::Gem.sources = options[:rubygems_source] if options[:rubygems_source]
 
       ::Bundler.settings[:path] = LogStash::Environment::BUNDLE_DIR
       ::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
@@ -60,7 +102,7 @@ module LogStash
       try = 0
 
       # capture_stdout also traps any raised exception and pass them back as the function return [output, exception]
-      capture_stdout do
+      output, exception = capture_stdout do
         loop do
           begin
             ::Bundler.reset!
@@ -81,11 +123,15 @@ module LogStash
 
             try += 1
             $stderr.puts("Error #{e.class}, retrying #{try}/#{options[:max_tries]}")
-            $stderr.puts(e.message) if ENV["DEBUG"]
+            $stderr.puts(e.message)
             sleep(0.5)
           end
         end
       end
+
+      raise exception if exception
+
+      return output
     end
 
     # build Bundler::CLI.start arguments array from the given options hash
@@ -107,4 +153,4 @@ module LogStash
       arguments.flatten
     end
   end
-end
+end

data/lib/logstash/config/config_ast.rb

@@ -1,7 +1,9 @@
 # encoding: utf-8
 require 'logstash/errors'
 require "treetop"
+
 class Treetop::Runtime::SyntaxNode
+
   def compile
     return "" if elements.nil?
     return elements.collect(&:compile).reject(&:empty?).join("")
@@ -55,11 +57,31 @@ class Treetop::Runtime::SyntaxNode
   end
 end
 
+
 module LogStash; module Config; module AST
+
+  def self.defered_conditionals=(val)
+    @defered_conditionals = val
+  end
+
+  def self.defered_conditionals
+    @defered_conditionals
+  end
+
+  def self.defered_conditionals_index
+    @defered_conditionals_index
+  end
+
+  def self.defered_conditionals_index=(val)
+    @defered_conditionals_index = val
+  end
+
   class Node < Treetop::Runtime::SyntaxNode; end
 
   class Config < Node
     def compile
+      LogStash::Config::AST.defered_conditionals = []
+      LogStash::Config::AST.defered_conditionals_index = 0
       code = []
 
       code << <<-CODE
@@ -81,20 +103,22 @@ module LogStash; module Config; module AST
       ["filter", "output"].each do |type|
         # defines @filter_func and @output_func
 
-        definitions << "@#{type}_func = lambda do |event, &block|"
-        definitions << "  events = [event]"
+        definitions << "def #{type}_func(event)"
+        definitions << "  events = [event]" if type == "filter"
         definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", :event => event.to_hash)"
+
         sections.select { |s| s.plugin_type.text_value == type }.each do |s|
           definitions << s.compile.split("\n", -1).map { |e| "  #{e}" }
         end
 
-        if type == "filter"
-          definitions << "  events.flatten.each{|e| block.call(e) }"
-        end
+        definitions << "  events" if type == "filter"
         definitions << "end"
       end
 
       code += definitions.join("\n").split("\n", -1).collect { |l| "  #{l}" }
+
+      code += LogStash::Config::AST.defered_conditionals
+
       return code.join("\n")
     end
   end
@@ -125,20 +149,15 @@ module LogStash; module Config; module AST
             #{name}_flush = lambda do |options, &block|
               @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
 
-              flushed_events = #{name}.flush(options)
+              events = #{name}.flush(options)
 
-              return if flushed_events.nil? || flushed_events.empty?
+              return if events.nil? || events.empty?
 
-              flushed_events.each do |event|
-                @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :event => event)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
 
-                events = [event]
-                #{plugin.compile_starting_here.gsub(/^/, "  ")}
-
-                block.call(event)
-                events.flatten.each{|e| block.call(e) if e != event}
-              end
+              #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
+              events.each{|e| block.call(e)}
             end
 
             if #{name}.respond_to?(:flush)
@@ -211,13 +230,7 @@ module LogStash; module Config; module AST
         return "start_input(#{variable_name})"
       when "filter"
         return <<-CODE
-          events = events.flat_map do |event|
-            next [] if event.cancelled?
-
-            new_events = []
-            #{variable_name}.filter(event){|new_event| new_events << new_event}
-            event.cancelled? ? new_events : new_events.unshift(event)
-          end
+          #{variable_name}.filter(event) {|new_event| events << new_event }
         CODE
       when "output"
         return "#{variable_name}.handle(event)\n"
@@ -287,7 +300,7 @@ module LogStash; module Config; module AST
 
   module Unicode
     def self.wrap(text)
-      return "(" + text.inspect + ".force_encoding(Encoding::UTF_8)" + ")"
+      return "(" + text.force_encoding(Encoding::UTF_8).inspect + ")"
     end
   end
 
@@ -364,18 +377,38 @@ module LogStash; module Config; module AST
       # at the end, events is returned to handle the case where no branch match and no branch code is executed
       # so we must make sure to return the current event.
 
-      return <<-CODE
-        events = events.flat_map do |event|
-          events = [event]
+      type = recursive_select_parent(PluginSection).first.plugin_type.text_value
+
+      if type == "filter"
+        i = LogStash::Config::AST.defered_conditionals_index += 1
+        source = <<-CODE
+          def cond_func_#{i}(input_events)
+            result = []
+            input_events.each do |event|
+              events = [event]
+              #{super}
+              end
+              result += events
+            end
+            result
+          end
+        CODE
+        LogStash::Config::AST.defered_conditionals << source
+
+        <<-CODE
+          events = cond_func_#{i}(events)
+        CODE
+      else
+        <<-CODE
           #{super}
           end
-          events
-        end
-      CODE
+        CODE
+      end
     end
   end
 
   class BranchEntry < Node; end
+
   class If < BranchEntry
     def compile
       children = recursive_inject { |e| e.is_a?(Branch) || e.is_a?(Plugin) }

data/lib/logstash/config/mixin.rb

@@ -472,7 +472,7 @@ module LogStash::Config::Mixin
               bytes = Integer(value.first) rescue nil
               result = bytes || Filesize.from(value.first).to_i
             rescue ArgumentError
-              return false, "Unparseable filesize: #{value.first}. possible units (KiB, MiB, ...) e.g. '10 KiB'. doc reference: http://www.elasticsearch.org/guide/en/logstash/current/_logstash_config_language.html#bytes"
+              return false, "Unparseable filesize: #{value.first}. possible units (KiB, MiB, ...) e.g. '10 KiB'. doc reference: http://www.elastic.co/guide/en/logstash/current/configuration.html#bytes"
             end
           else
             return false, "Unknown validator symbol #{validator}"

data/lib/logstash/environment.rb

@@ -1,75 +1,19 @@
 require "logstash/errors"
-require 'logstash/version'
-
-# monkey patch RubyGems to silence ffi warnings:
-#
-# WARN: Unresolved specs during Gem::Specification.reset:
-#       ffi (>= 0)
-# WARN: Clearing out unresolved specs.
-# Please report a bug if this causes problems.
-#
-# see https://github.com/elasticsearch/logstash/issues/2556 and https://github.com/rubygems/rubygems/issues/1070
-#
-# this code is from Rubygems v2.1.9 in JRuby 1.7.17. Per tickets this issue should be solved at JRuby >= 1.7.20.
-
-# this method implementation works for Rubygems version 2.1.0 and up, verified up to 2.4.6
-if Gem::Version.new(Gem::VERSION) >= Gem::Version.new("2.1.0") && Gem::Version.new(Gem::VERSION) < Gem::Version.new("2.5.0")
-  class Gem::Specification
-    def self.reset
-      @@dirs = nil
-      Gem.pre_reset_hooks.each { |hook| hook.call }
-      @@all = nil
-      @@stubs = nil
-      _clear_load_cache
-      unresolved = unresolved_deps
-      unless unresolved.empty?
-        unless (unresolved.size == 1 && unresolved["ffi"])
-          w = "W" + "ARN"
-          warn "#{w}: Unresolved specs during Gem::Specification.reset:"
-          unresolved.values.each do |dep|
-            warn "      #{dep}"
-          end
-          warn "#{w}: Clearing out unresolved specs."
-          warn "Please report a bug if this causes problems."
-        end
-        unresolved.clear
-      end
-      Gem.post_reset_hooks.each { |hook| hook.call }
-    end
-  end
-end
+require "logstash/version"
 
 module LogStash
   module Environment
     extend self
 
     LOGSTASH_HOME = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
-    JAR_DIR = ::File.join(LOGSTASH_HOME, "vendor", "jar")
-    ELASTICSEARCH_DIR = ::File.join(LOGSTASH_HOME, "vendor", "elasticsearch")
     BUNDLE_DIR = ::File.join(LOGSTASH_HOME, "vendor", "bundle")
     GEMFILE_PATH = ::File.join(LOGSTASH_HOME, "Gemfile")
     BUNDLE_CONFIG_PATH = ::File.join(LOGSTASH_HOME, ".bundle", "config")
     BOOTSTRAP_GEM_PATH = ::File.join(LOGSTASH_HOME, 'build', 'bootstrap')
+    LOCAL_GEM_PATH = ::File.join(LOGSTASH_HOME, 'vendor', 'local_gems')
 
     LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
-    # loads currently embedded elasticsearch jars
-    # @raise LogStash::EnvironmentError if not running under JRuby or if no jar files are found
-    def load_elasticsearch_jars!
-      raise(LogStash::EnvironmentError, "JRuby is required") unless jruby?
-
-      require "java"
-      jars_path = ::File.join(ELASTICSEARCH_DIR, "**", "*.jar")
-      jar_files = Dir.glob(jars_path)
-
-      raise(LogStash::EnvironmentError, "Could not find Elasticsearch jar files under #{ELASTICSEARCH_DIR}") if jar_files.empty?
-
-      jar_files.each do |jar|
-        loaded = require jar
-        puts("Loaded #{jar}") if $DEBUG && loaded
-      end
-    end
-
     def logstash_gem_home
       ::File.join(BUNDLE_DIR, ruby_engine, gem_ruby_version)
     end
@@ -90,27 +34,41 @@ module LogStash
       env.downcase == "test"
     end
 
-    def bundler_setup!(options = {})
-      options = {:without => [:development]}.merge(options)
-      options[:without] = Array(options[:without])
-      # make sure we use our own nicely installed bundler and not a rogue, bad, mean, ugly, stupid other bundler. bad bundler, bad bad bundler go away.
-      ::Gem.clear_paths
-      ::Gem.paths = ENV['GEM_HOME'] = ENV['GEM_PATH'] = logstash_gem_home
+    def runtime_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "runtime-jars")
+    end
+
+    def test_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "test-jars")
+    end
+
+    def load_runtime_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(runtime_jars_root(dir_name, package), "*.jar"))
+    end
 
-      # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s)
-      ENV["BUNDLE_GEMFILE"] = LogStash::Environment::GEMFILE_PATH
+    def load_test_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(test_jars_root(dir_name, package), "*.jar"))
+    end
 
-      require "bundler"
-      require "logstash/bundler"
+    def load_jars!(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.jruby-required")) unless LogStash::Environment.jruby?
 
-      ::Bundler.settings[:path]    = LogStash::Environment::BUNDLE_DIR
-      ::Bundler.settings[:without] = options[:without].join(":")
-      # in the context of Bundler.setup it looks like this is useless here because Gemfile path can only be specified using
-      # the ENV, see https://github.com/bundler/bundler/blob/v1.8.3/lib/bundler/shared_helpers.rb#L103
-      ::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
+      jar_files = find_jars(pattern)
+      require_jars! jar_files
+    end
 
-      ::Bundler.reset!
-      ::Bundler.setup
+    def find_jars(pattern)
+      require 'java'
+      jar_files = Dir.glob(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.missing-jars", :pattern => pattern)) if jar_files.empty?
+      jar_files
+    end
+
+    def require_jars!(files)
+      files.each do |jar_file|
+        loaded = require jar_file
+        puts("Loaded #{jar_file}") if $DEBUG && loaded
+      end
     end
 
     def ruby_bin