logstash-core 1.5.0.beta2-java → 1.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c512265bcb6873ce0def467329ba6b4a058aea5
-  data.tar.gz: a376414ac39751f9c4e7b67fbb109410fe4d04b8
+  metadata.gz: 69a97d7d7b23b0e8c46eec7149ab4370f943d222
+  data.tar.gz: bab042c02f71ac9a9e7319cce8321e631553ab20
 SHA512:
-  metadata.gz: 5520cb37d6482b810ab0501e370bb08e859365d7b5e4217683fc065ecde3c859dde0f3da0c8447ff5b28ab19f9264db9cc4c8e5e5523a6a53bb8c57f3054608a
-  data.tar.gz: b722a0fe0d550a82adafc116bd579489f7871c633d22dce7d9724ca6617c107e09b138a92632babf75eebb704a9f2fb34022e05ed544d0b0381cfc05b4befd64
+  metadata.gz: 18d78f1d0762674a720dc763c8843b74ca2c0af16c4321b9496e1636511ec2f79271e934c9f068e6bc92c0897d93d8306d9ce305d1fa63b8cd3386ce1fc0ee55
+  data.tar.gz: ff72f2f8ad1af596dbe4cf6c89b40b0ae52966d90e950e046dd1fdee4e1bdbc6b3fed1fe441252d524d631daaada7cf71e21d23093a6ca7b7f06c48d5f5d812e

data/lib/logstash-core.rb

@@ -0,0 +1,2 @@
+module LogStash
+end

data/lib/logstash/agent.rb

@@ -39,11 +39,6 @@ class LogStash::Agent < Clamp::Command
   option ["-V", "--version"], :flag,
     I18n.t("logstash.agent.flag.version")
 
-  option ["-p", "--pluginpath"] , "PATH",
-    I18n.t("logstash.agent.flag.pluginpath"),
-    :multivalued => true,
-    :attribute_name => :plugin_paths
-
   option ["-t", "--configtest"], :flag,
     I18n.t("logstash.agent.flag.configtest"),
     :attribute_name => :config_test
@@ -172,7 +167,6 @@ class LogStash::Agent < Clamp::Command
 
       if RUBY_PLATFORM == "java"
         show_version_java
-        show_version_elasticsearch
       end
 
       if [:debug].include?(verbosity?) || debug?
@@ -190,13 +184,6 @@ class LogStash::Agent < Clamp::Command
     puts RUBY_DESCRIPTION
   end # def show_version_ruby
 
-  def show_version_elasticsearch
-    LogStash::Environment.load_elasticsearch_jars!
-
-    $stdout.write("Elasticsearch: ");
-    org.elasticsearch.Version::main([])
-  end # def show_version_elasticsearch
-
   def show_version_java
     properties = java.lang.System.getProperties
     puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
@@ -215,7 +202,6 @@ class LogStash::Agent < Clamp::Command
   # Log file stuff, plugin path checking, etc.
   def configure
     configure_logging(log_file)
-    configure_plugin_path(plugin_paths) if !plugin_paths.nil?
   end # def configure
 
   # Point logging at a specific path.
@@ -266,33 +252,6 @@ class LogStash::Agent < Clamp::Command
     # http://jira.codehaus.org/browse/JRUBY-7003
   end # def configure_logging
 
-  # Validate and add any paths to the list of locations
-  # logstash will look to find plugins.
-  def configure_plugin_path(paths)
-    # Append any plugin paths to the ruby search path
-    paths.each do |path|
-      # Verify the path exists
-      if !Dir.exists?(path)
-        warn(I18n.t("logstash.agent.configuration.plugin_path_missing",
-                    :path => path))
-
-      end
-
-      # TODO(sissel): Verify the path looks like the correct form.
-      # aka, there must be file in path/logstash/{inputs,codecs,filters,outputs}/*.rb
-      plugin_glob = File.join(path, "logstash", "{inputs,codecs,filters,outputs}", "*.rb")
-      if Dir.glob(plugin_glob).empty?
-        @logger.warn(I18n.t("logstash.agent.configuration.no_plugins_found",
-                    :path => path, :plugin_glob => plugin_glob))
-      end
-
-      # We push plugin paths to the front of the LOAD_PATH so that folks
-      # can override any core logstash plugins if they need to.
-      @logger.debug("Adding plugin path", :path => path)
-      $LOAD_PATH.unshift(path)
-    end
-  end # def configure_plugin_path
-
   def load_config(path)
     begin
       uri = URI.parse(path)

data/lib/logstash/config/config_ast.rb

@@ -1,7 +1,9 @@
 # encoding: utf-8
 require 'logstash/errors'
 require "treetop"
+
 class Treetop::Runtime::SyntaxNode
+
   def compile
     return "" if elements.nil?
     return elements.collect(&:compile).reject(&:empty?).join("")
@@ -55,11 +57,31 @@ class Treetop::Runtime::SyntaxNode
   end
 end
 
+
 module LogStash; module Config; module AST
+
+  def self.defered_conditionals=(val)
+    @defered_conditionals = val
+  end
+
+  def self.defered_conditionals
+    @defered_conditionals
+  end
+
+  def self.defered_conditionals_index
+    @defered_conditionals_index
+  end
+
+  def self.defered_conditionals_index=(val)
+    @defered_conditionals_index = val
+  end
+
   class Node < Treetop::Runtime::SyntaxNode; end
 
   class Config < Node
     def compile
+      LogStash::Config::AST.defered_conditionals = []
+      LogStash::Config::AST.defered_conditionals_index = 0
       code = []
 
       code << <<-CODE
@@ -81,20 +103,22 @@ module LogStash; module Config; module AST
       ["filter", "output"].each do |type|
         # defines @filter_func and @output_func
 
-        definitions << "@#{type}_func = lambda do |event, &block|"
-        definitions << "  events = [event]"
+        definitions << "def #{type}_func(event)"
+        definitions << "  events = [event]" if type == "filter"
         definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", :event => event.to_hash)"
+
         sections.select { |s| s.plugin_type.text_value == type }.each do |s|
           definitions << s.compile.split("\n", -1).map { |e| "  #{e}" }
         end
 
-        if type == "filter"
-          definitions << "  events.flatten.each{|e| block.call(e) }"
-        end
+        definitions << "  events" if type == "filter"
         definitions << "end"
       end
 
       code += definitions.join("\n").split("\n", -1).collect { |l| "  #{l}" }
+
+      code += LogStash::Config::AST.defered_conditionals
+
       return code.join("\n")
     end
   end
@@ -125,20 +149,15 @@ module LogStash; module Config; module AST
             #{name}_flush = lambda do |options, &block|
               @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name})
 
-              flushed_events = #{name}.flush(options)
+              events = #{name}.flush(options)
 
-              return if flushed_events.nil? || flushed_events.empty?
+              return if events.nil? || events.empty?
 
-              flushed_events.each do |event|
-                @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :event => event)
+              @logger.debug? && @logger.debug(\"Flushing\", :plugin => #{name}, :events => events)
 
-                events = [event]
-                #{plugin.compile_starting_here.gsub(/^/, "  ")}
-
-                block.call(event)
-                events.flatten.each{|e| block.call(e) if e != event}
-              end
+              #{plugin.compile_starting_here.gsub(/^/, "  ")}
 
+              events.each{|e| block.call(e)}
             end
 
             if #{name}.respond_to?(:flush)
@@ -211,13 +230,7 @@ module LogStash; module Config; module AST
         return "start_input(#{variable_name})"
       when "filter"
         return <<-CODE
-          events = events.flat_map do |event|
-            next [] if event.cancelled?
-
-            new_events = []
-            #{variable_name}.filter(event){|new_event| new_events << new_event}
-            event.cancelled? ? new_events : new_events.unshift(event)
-          end
+          events = #{variable_name}.multi_filter(events)
         CODE
       when "output"
         return "#{variable_name}.handle(event)\n"
@@ -287,7 +300,7 @@ module LogStash; module Config; module AST
 
   module Unicode
     def self.wrap(text)
-      return "(" + text.inspect + ".force_encoding(Encoding::UTF_8)" + ")"
+      return "(" + text.force_encoding(Encoding::UTF_8).inspect + ")"
     end
   end
 
@@ -364,18 +377,38 @@ module LogStash; module Config; module AST
       # at the end, events is returned to handle the case where no branch match and no branch code is executed
       # so we must make sure to return the current event.
 
-      return <<-CODE
-        events = events.flat_map do |event|
-          events = [event]
+      type = recursive_select_parent(PluginSection).first.plugin_type.text_value
+
+      if type == "filter"
+        i = LogStash::Config::AST.defered_conditionals_index += 1
+        source = <<-CODE
+          def cond_func_#{i}(input_events)
+            result = []
+            input_events.each do |event|
+              events = [event]
+              #{super}
+              end
+              result += events
+            end
+            result
+          end
+        CODE
+        LogStash::Config::AST.defered_conditionals << source
+
+        <<-CODE
+          events = cond_func_#{i}(events)
+        CODE
+      else
+        <<-CODE
           #{super}
           end
-          events
-        end
-      CODE
+        CODE
+      end
     end
   end
 
   class BranchEntry < Node; end
+
   class If < BranchEntry
     def compile
       children = recursive_inject { |e| e.is_a?(Branch) || e.is_a?(Plugin) }

data/lib/logstash/config/mixin.rb

@@ -137,7 +137,7 @@ module LogStash::Config::Mixin
     # inside the gemspec.
     def milestone(m = nil)
       @logger = Cabin::Channel.get(LogStash)
-      @logger.error(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
+      @logger.warn(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
     end
 
     # Define a new configuration setting
@@ -451,7 +451,7 @@ module LogStash::Config::Mixin
               return false, "Expected password (one value), got #{value.size} values?"
             end
 
-            result = ::LogStash::Util::Password.new(value.first)
+            result = value.first.is_a?(::LogStash::Util::Password) ? value.first : ::LogStash::Util::Password.new(value.first)
           when :path
             if value.size > 1 # Only 1 value wanted
               return false, "Expected path (one value), got #{value.size} values?"
@@ -472,7 +472,7 @@ module LogStash::Config::Mixin
               bytes = Integer(value.first) rescue nil
               result = bytes || Filesize.from(value.first).to_i
             rescue ArgumentError
-              return false, "Unparseable filesize: #{value.first}. possible units (KiB, MiB, ...) e.g. '10 KiB'. doc reference: http://www.elasticsearch.org/guide/en/logstash/current/_logstash_config_language.html#bytes"
+              return false, "Unparseable filesize: #{value.first}. possible units (KiB, MiB, ...) e.g. '10 KiB'. doc reference: http://www.elastic.co/guide/en/logstash/current/configuration.html#bytes"
             end
           else
             return false, "Unknown validator symbol #{validator}"

data/lib/logstash/environment.rb

@@ -1,78 +1,19 @@
 require "logstash/errors"
-require 'logstash/version'
-
-# monkey patch RubyGems to silence ffi warnings:
-#
-# WARN: Unresolved specs during Gem::Specification.reset:
-#       ffi (>= 0)
-# WARN: Clearing out unresolved specs.
-# Please report a bug if this causes problems.
-#
-# see https://github.com/elasticsearch/logstash/issues/2556 and https://github.com/rubygems/rubygems/issues/1070
-#
-# this code is from Rubygems v2.1.9 in JRuby 1.7.17. Per tickets this issue should be solved at JRuby >= 1.7.20.
-
-# this method implementation works for Rubygems version 2.1.0 and up, verified up to 2.4.6
-if Gem::Version.new(Gem::VERSION) >= Gem::Version.new("2.1.0") && Gem::Version.new(Gem::VERSION) < Gem::Version.new("2.5.0")
-  class Gem::Specification
-    def self.reset
-      @@dirs = nil
-      Gem.pre_reset_hooks.each { |hook| hook.call }
-      @@all = nil
-      @@stubs = nil
-      _clear_load_cache
-      unresolved = unresolved_deps
-      unless unresolved.empty?
-        unless (unresolved.size == 1 && unresolved["ffi"])
-          w = "W" + "ARN"
-          warn "#{w}: Unresolved specs during Gem::Specification.reset:"
-          unresolved.values.each do |dep|
-            warn "      #{dep}"
-          end
-          warn "#{w}: Clearing out unresolved specs."
-          warn "Please report a bug if this causes problems."
-        end
-        unresolved.clear
-      end
-      Gem.post_reset_hooks.each { |hook| hook.call }
-    end
-  end
-end
+require "logstash/version"
 
 module LogStash
   module Environment
     extend self
 
-    LOGSTASH_HOME = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
-    JAR_DIR = ::File.join(LOGSTASH_HOME, "vendor", "jar")
-    ELASTICSEARCH_DIR = ::File.join(LOGSTASH_HOME, "vendor", "elasticsearch")
-    BUNDLE_DIR = ::File.join(LOGSTASH_HOME, "vendor", "bundle")
-    GEMFILE_PATH = ::File.join(LOGSTASH_HOME, "Gemfile")
-    BUNDLE_CONFIG_PATH = ::File.join(LOGSTASH_HOME, ".bundle", "config")
-    BOOTSTRAP_GEM_PATH = ::File.join(LOGSTASH_HOME, 'build', 'bootstrap')
-
-    LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
-
-    # loads currently embedded elasticsearch jars
-    # @raise LogStash::EnvironmentError if not running under JRuby or if no jar files are found
-    def load_elasticsearch_jars!
-      raise(LogStash::EnvironmentError, "JRuby is required") unless jruby?
-
-      require "java"
-      jars_path = ::File.join(ELASTICSEARCH_DIR, "**", "*.jar")
-      jar_files = Dir.glob(jars_path)
-
-      raise(LogStash::EnvironmentError, "Could not find Elasticsearch jar files under #{ELASTICSEARCH_DIR}") if jar_files.empty?
-
-      jar_files.each do |jar|
-        loaded = require jar
-        puts("Loaded #{jar}") if $DEBUG && loaded
-      end
+    # rehydrate the bootstrap environment if the startup was not done by executing bootstrap.rb
+    # and we are in the context of the logstash package
+    if !LogStash::Environment.const_defined?("LOGSTASH_HOME") &&  !ENV["LOGSTASH_HOME"].to_s.empty?
+      $LOAD_PATH << ::File.join(ENV["LOGSTASH_HOME"], "lib")
+      require "bootstrap/environment"
     end
 
-    def logstash_gem_home
-      ::File.join(BUNDLE_DIR, ruby_engine, gem_ruby_version)
-    end
+    LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
+    LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
     def env
       LOGSTASH_ENV
@@ -90,45 +31,45 @@ module LogStash
       env.downcase == "test"
     end
 
-    def bundler_setup!
-      # if there's no .bundle/config setup the runtime environment.
-      unless ::File.exists?(BUNDLE_CONFIG_PATH)
-        ENV["BUNDLE_PATH"] = LogStash::Environment::BUNDLE_DIR
-        ENV["BUNDLE_WITHOUT"] = "development"
-      end
-      # force BUNDLE_GEMFILE since Bundler does not store it in its ./bundle/config. this is required otherwise Bundler will look for the Gemfile in the CWD
-      # and will crash when invoking logstash outside its home dir.
-      ENV["BUNDLE_GEMFILE"] = LogStash::Environment::GEMFILE_PATH
+    def runtime_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "runtime-jars")
+    end
 
-      # make sure we use our own nicely installed bundler and not a rogue, bad, mean, ugly, stupid other bundler. bad bundler, bad bad bundler go away.
-      Gem.clear_paths
-      Gem.paths = ENV['GEM_HOME'] = ENV['GEM_PATH'] = logstash_gem_home
-      require "bundler"
+    def test_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "test-jars")
+    end
 
-      require "logstash/bundler"
+    def load_runtime_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(runtime_jars_root(dir_name, package), "*.jar"))
+    end
 
-      ::Bundler.reset_settings # our monkey patched method
-      ::Bundler.reset!
-      ::Bundler.setup
+    def load_test_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(test_jars_root(dir_name, package), "*.jar"))
     end
 
-    def ruby_bin
-      ENV["USE_RUBY"] == "1" ? "ruby" : File.join("vendor", "jruby", "bin", "jruby")
+    def load_jars!(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.jruby-required")) unless LogStash::Environment.jruby?
+
+      jar_files = find_jars(pattern)
+      require_jars! jar_files
     end
 
-    # @return [String] major.minor ruby version, ex 1.9
-    def ruby_abi_version
-      RUBY_VERSION[/(\d+\.\d+)(\.\d+)*/, 1]
+    def find_jars(pattern)
+      require 'java'
+      jar_files = Dir.glob(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.missing-jars", :pattern => pattern)) if jar_files.empty?
+      jar_files
     end
 
-    # @return [String] the ruby version string bundler uses to craft its gem path
-    def gem_ruby_version
-      RbConfig::CONFIG["ruby_version"]
+    def require_jars!(files)
+      files.each do |jar_file|
+        loaded = require jar_file
+        puts("Loaded #{jar_file}") if $DEBUG && loaded
+      end
     end
 
-    # @return [String] jruby, ruby, rbx, ...
-    def ruby_engine
-      RUBY_ENGINE
+    def ruby_bin
+      ENV["USE_RUBY"] == "1" ? "ruby" : File.join("vendor", "jruby", "bin", "jruby")
     end
 
     def jruby?
@@ -143,16 +84,12 @@ module LogStash
       return ::File.join(LOGSTASH_HOME, "vendor", path)
     end
 
-    def plugin_path(path)
-      return ::File.join(LOGSTASH_HOME, "lib", "logstash", path)
-    end
-
     def pattern_path(path)
       return ::File.join(LOGSTASH_HOME, "patterns", path)
     end
 
     def locales_path(path)
-      return ::File.join(LOGSTASH_HOME, "locales", path)
+      return ::File.join(LOGSTASH_CORE, "locales", path)
     end
 
     def load_locale!