logstash-core 1.5.0.beta2-java → 1.5.0-java

data/lib/logstash/runner.rb

@@ -1,32 +1,16 @@
 # encoding: utf-8
 
+Thread.abort_on_exception = true
 Encoding.default_external = Encoding::UTF_8
-$START = Time.now
 $DEBUGLIST = (ENV["DEBUG"] || "").split(",")
 
 require "logstash/environment"
-LogStash::Environment.bundler_setup!
-LogStash::Environment.load_locale!
 
-Thread.abort_on_exception = true
+LogStash::Environment.load_locale!
 
 require "logstash/namespace"
 require "logstash/program"
 
-class LogStash::RSpecsRunner
-  def initialize(args)
-    @args = args
-  end
-
-  def run
-    @result = RSpec::Core::Runner.run(@args)
-  end
-
-  def wait
-    return @result
-  end
-end
-
 class LogStash::Runner
   include LogStash::Program
 
@@ -37,7 +21,6 @@ class LogStash::Runner
     @startup_interruption_trap = Stud::trap("INT") { puts "Interrupted"; exit 0 }
 
     LogStash::Util::set_thread_name(self.class.name)
-    #$LOAD_PATH << File.join(File.dirname(__FILE__), "..")
 
     if RUBY_VERSION < "1.9.2"
       $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
@@ -61,15 +44,6 @@ class LogStash::Runner
         end
         return LogStash::Agent.run($0, agent_args)
       end,
-      "rspec" => lambda do
-        require "rspec/core/runner"
-        require "rspec"
-        spec_path = File.expand_path(File.join(File.dirname(__FILE__), "/../../spec"))
-        $LOAD_PATH << spec_path
-        all_specs = Dir.glob(File.join(spec_path, "/**/*_spec.rb"))
-        rspec = LogStash::RSpecsRunner.new(args.empty? ? all_specs : args)
-        return rspec.run
-      end,
       "irb" => lambda do
         require "irb"
         return IRB.start(__FILE__)
@@ -94,21 +68,10 @@ class LogStash::Runner
         end
         return 0
       end,
-      "plugin" => lambda do
-        require 'logstash/pluginmanager'
-        plugin_manager = LogStash::PluginManager::Main.new($0)
-        begin
-          plugin_manager.parse(args)
-          return plugin_manager.execute
-        rescue Clamp::HelpWanted => e
-          show_help(e.command)
-          return 0
-        end
-      end,
       "agent" => lambda do
         require "logstash/agent"
         # Hack up a runner
-        agent = LogStash::Agent.new($0)
+        agent = LogStash::Agent.new("/bin/logstash agent", $0)
         begin
           agent.parse(args)
         rescue Clamp::HelpWanted => e
@@ -144,24 +107,15 @@ For example: logstash agent --help
 Available commands:
   agent - runs the logstash agent
   version - emits version info about this logstash
-  rspec - runs tests
-      ]
+]
       #$stderr.puts commands.keys.map { |s| "  #{s}" }.join("\n")
       return Stud::Task.new { 1 }
     end
   end # def run
 
-  # @return true if this file is the main file being run and not via rspec
-  def self.autorun?
-    # caller is the current execution stack
-    $0 == __FILE__ && caller.none?{|entry| entry =~ /rspec/}
-  end
-
   private
 
   def show_help(command)
     puts command.help
   end
 end # class LogStash::Runner
-
-LogStash::Runner.new.main(ARGV) if LogStash::Runner.autorun?

data/lib/logstash/util.rb

@@ -148,5 +148,4 @@ module LogStash::Util
       o
     end
   end
-
 end # module LogStash::Util

data/lib/logstash/util/accessors.rb

@@ -53,6 +53,12 @@ module LogStash::Util
       end
     end
 
+    def include?(accessor)
+      target, key = lookup_path(accessor)
+      return false unless target
+      target.is_a?(Array) ? !target[key.to_i].nil? : target.include?(key)
+    end
+
     private
 
     def lookup(accessor)

data/lib/logstash/version.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 # The version of logstash.
-LOGSTASH_VERSION = "1.5.0.beta2"
+LOGSTASH_VERSION = "1.5.0"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

data/locales/en.yml

@@ -7,6 +7,11 @@ en:
     The error reported is: 
       %{error}
   logstash:
+    environment:
+      jruby-required:  >-
+        JRuby is required
+      missing-jars: >-
+        Could not find jar files under %{pattern}
     pipeline:
       worker-error: |-
         A plugin had an unrecoverable error. Will restart this plugin.

data/logstash-core.gemspec

@@ -0,0 +1,51 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logstash/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"]
+  gem.email         = ["jls@semicomplete.com", "petef@databits.net", "info@elasticsearch.com"]
+  gem.description   = %q{The core components of logstash, the scalable log and event management tool}
+  gem.summary       = %q{logstash-core - The core components of logstash}
+  gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  gem.license       = "Apache License (2.0)"
+
+  gem.files         = Dir.glob(["logstash-core.gemspec", "lib/logstash-core.rb", "lib/logstash/**/*.rb", "spec/**/*.rb", "locales/*"])
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "logstash-core"
+  gem.require_paths = ["lib"]
+  gem.version       = LOGSTASH_VERSION.gsub(/-/, '.')
+
+  gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
+  gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
+  gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
+  gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
+  gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
+
+  # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
+  # investigated what the cause might be. -Jordan
+  gem.add_runtime_dependency "treetop", "< 1.5.0" #(MIT license)
+
+  # upgrade i18n only post 0.6.11, see https://github.com/svenfuchs/i18n/issues/270
+  gem.add_runtime_dependency "i18n", "= 0.6.9" #(MIT license)
+
+  # filetools and rakelib
+  gem.add_runtime_dependency "minitar", "~> 0.5.4"
+
+  if RUBY_PLATFORM == 'java'
+    gem.platform = RUBY_PLATFORM
+    gem.add_runtime_dependency "jrjackson", "~> 0.2.8" #(Apache 2.0 license)
+  else
+    gem.add_runtime_dependency "oj" #(MIT-style license)
+  end
+
+  if RUBY_ENGINE == "rbx"
+    # rubinius puts the ruby stdlib into gems.
+    gem.add_runtime_dependency "rubysl"
+
+    # Include racc to make the xml tests pass.
+    # https://github.com/rubinius/rubinius/issues/2632#issuecomment-26954565
+    gem.add_runtime_dependency "racc"
+  end
+end

data/spec/core/conditionals_spec.rb

@@ -0,0 +1,428 @@
+require 'spec_helper'
+
+module ConditionalFanciness
+  def description
+    return example.metadata[:example_group][:description_args][0]
+  end
+
+  def conditional(expression, &block)
+    describe(expression) do
+      config <<-CONFIG
+        filter {
+          if #{expression} {
+            mutate { add_tag => "success" }
+          } else {
+            mutate { add_tag => "failure" }
+          }
+        }
+      CONFIG
+      instance_eval(&block)
+    end
+  end
+end
+
+describe "conditionals in output" do
+  extend ConditionalFanciness
+
+  describe "simple" do
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":{"bar"},"baz": "quux"}'
+          count => 1
+        }
+      }
+      output {
+        if [foo] == "bar" {
+          stdout { }
+        }
+      }
+    CONFIG
+
+    agent do
+      #LOGSTASH-2288, should not fail raising an exception
+    end
+  end
+end
+
+describe "conditionals in filter" do
+  extend ConditionalFanciness
+
+  describe "simple" do
+    config <<-CONFIG
+      filter {
+        mutate { add_field => { "always" => "awesome" } }
+        if [foo] == "bar" {
+          mutate { add_field => { "hello" => "world" } }
+        } else if [bar] == "baz" {
+          mutate { add_field => { "fancy" => "pants" } }
+        } else {
+          mutate { add_field => { "free" => "hugs" } }
+        }
+      }
+    CONFIG
+
+    sample({"foo" => "bar"}) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to eq("world")
+      expect(subject["fancy"]).to be_nil
+      expect(subject["free"]).to be_nil
+    end
+
+    sample({"notfoo" => "bar"}) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to be_nil
+      expect(subject["fancy"]).to be_nil
+      expect(subject["free"]).to eq("hugs")
+    end
+
+    sample({"bar" => "baz"}) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to be_nil
+      expect(subject["fancy"]).to eq("pants")
+      expect(subject["free"]).to be_nil
+    end
+  end
+
+  describe "nested" do
+    config <<-CONFIG
+      filter {
+        if [nest] == 123 {
+          mutate { add_field => { "always" => "awesome" } }
+          if [foo] == "bar" {
+            mutate { add_field => { "hello" => "world" } }
+          } else if [bar] == "baz" {
+            mutate { add_field => { "fancy" => "pants" } }
+          } else {
+            mutate { add_field => { "free" => "hugs" } }
+          }
+        }
+      }
+    CONFIG
+
+    sample("foo" => "bar", "nest" => 124) do
+      expect(subject["always"]).to be_nil
+      expect(subject["hello"]).to be_nil
+      expect(subject["fancy"]).to be_nil
+      expect(subject["free"]).to be_nil
+    end
+
+    sample("foo" => "bar", "nest" => 123) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to eq("world")
+      expect(subject["fancy"]).to be_nil
+      expect(subject["free"]).to be_nil
+    end
+
+    sample("notfoo" => "bar", "nest" => 123) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to be_nil
+      expect(subject["fancy"]).to be_nil
+      expect(subject["free"]).to eq("hugs")
+    end
+
+    sample("bar" => "baz", "nest" => 123) do
+      expect(subject["always"]).to eq("awesome")
+      expect(subject["hello"]).to be_nil
+      expect(subject["fancy"]).to eq("pants")
+      expect(subject["free"]).to be_nil
+    end
+  end
+
+  describe "comparing two fields" do
+    config <<-CONFIG
+      filter {
+        if [foo] == [bar] {
+          mutate { add_tag => woot }
+        }
+      }
+    CONFIG
+
+    sample("foo" => 123, "bar" => 123) do
+      expect(subject["tags"] ).to include("woot")
+    end
+  end
+
+  describe "the 'in' operator" do
+    config <<-CONFIG
+      filter {
+        if [foo] in [foobar] {
+          mutate { add_tag => "field in field" }
+        }
+        if [foo] in "foo" {
+          mutate { add_tag => "field in string" }
+        }
+        if "hello" in [greeting] {
+          mutate { add_tag => "string in field" }
+        }
+        if [foo] in ["hello", "world", "foo"] {
+          mutate { add_tag => "field in list" }
+        }
+        if [missing] in [alsomissing] {
+          mutate { add_tag => "shouldnotexist" }
+        }
+        if !("foo" in ["hello", "world"]) {
+          mutate { add_tag => "shouldexist" }
+        }
+      }
+    CONFIG
+
+    sample("foo" => "foo", "foobar" => "foobar", "greeting" => "hello world") do
+      expect(subject["tags"]).to include("field in field")
+      expect(subject["tags"]).to include("field in string")
+      expect(subject["tags"]).to include("string in field")
+      expect(subject["tags"]).to include("field in list")
+      expect(subject["tags"]).not_to include("shouldnotexist")
+      expect(subject["tags"]).to include("shouldexist")
+    end
+  end
+
+  describe "the 'not in' operator" do
+    config <<-CONFIG
+      filter {
+        if "foo" not in "baz" { mutate { add_tag => "baz" } }
+        if "foo" not in "foo" { mutate { add_tag => "foo" } }
+        if !("foo" not in "foo") { mutate { add_tag => "notfoo" } }
+        if "foo" not in [somelist] { mutate { add_tag => "notsomelist" } }
+        if "one" not in [somelist] { mutate { add_tag => "somelist" } }
+        if "foo" not in [alsomissing] { mutate { add_tag => "no string in missing field" } }
+      }
+    CONFIG
+
+    sample("foo" => "foo", "somelist" => [ "one", "two" ], "foobar" => "foobar", "greeting" => "hello world", "tags" => [ "fancypantsy" ]) do
+      # verify the original exists
+      expect(subject["tags"]).to include("fancypantsy")
+
+      expect(subject["tags"]).to include("baz")
+      expect(subject["tags"]).not_to include("foo")
+      expect(subject["tags"]).to include("notfoo")
+      expect(subject["tags"]).to include("notsomelist")
+      expect(subject["tags"]).not_to include("somelist")
+      expect(subject["tags"]).to include("no string in missing field")
+    end
+  end
+
+  describe "operators" do
+    conditional "[message] == 'sample'" do
+      sample("sample") { expect(subject["tags"] ).to include("success") }
+      sample("different") { expect(subject["tags"] ).to include("failure") }
+    end
+
+    conditional "[message] != 'sample'" do
+      sample("sample") { expect(subject["tags"] ).to include("failure") }
+      sample("different") { expect(subject["tags"] ).to include("success") }
+    end
+
+    conditional "[message] < 'sample'" do
+      sample("apple") { expect(subject["tags"] ).to include("success") }
+      sample("zebra") { expect(subject["tags"] ).to include("failure") }
+    end
+
+    conditional "[message] > 'sample'" do
+      sample("zebra") { expect(subject["tags"] ).to include("success") }
+      sample("apple") { expect(subject["tags"] ).to include("failure") }
+    end
+
+    conditional "[message] <= 'sample'" do
+      sample("apple") { expect(subject["tags"] ).to include("success") }
+      sample("zebra") { expect(subject["tags"] ).to include("failure") }
+      sample("sample") { expect(subject["tags"] ).to include("success") }
+    end
+
+    conditional "[message] >= 'sample'" do
+      sample("zebra") { expect(subject["tags"] ).to include("success") }
+      sample("sample") { expect(subject["tags"] ).to include("success") }
+      sample("apple") { expect(subject["tags"] ).to include("failure") }
+    end
+
+    conditional "[message] =~ /sample/" do
+      sample("apple") { expect(subject["tags"] ).to include("failure") }
+      sample("sample") { expect(subject["tags"] ).to include("success") }
+      sample("some sample") { expect(subject["tags"] ).to include("success") }
+    end
+
+    conditional "[message] !~ /sample/" do
+      sample("apple") { expect(subject["tags"]).to include("success") }
+      sample("sample") { expect(subject["tags"]).to include("failure") }
+      sample("some sample") { expect(subject["tags"]).to include("failure") }
+    end
+
+  end
+
+  describe "negated expressions" do
+    conditional "!([message] == 'sample')" do
+      sample("sample") { expect(subject["tags"]).not_to include("success") }
+      sample("different") { expect(subject["tags"]).not_to include("failure") }
+    end
+
+    conditional "!([message] != 'sample')" do
+      sample("sample") { expect(subject["tags"]).not_to include("failure") }
+      sample("different") { expect(subject["tags"]).not_to include("success") }
+    end
+
+    conditional "!([message] < 'sample')" do
+      sample("apple") { expect(subject["tags"]).not_to include("success") }
+      sample("zebra") { expect(subject["tags"]).not_to include("failure") }
+    end
+
+    conditional "!([message] > 'sample')" do
+      sample("zebra") { expect(subject["tags"]).not_to include("success") }
+      sample("apple") { expect(subject["tags"]).not_to include("failure") }
+    end
+
+    conditional "!([message] <= 'sample')" do
+      sample("apple") { expect(subject["tags"]).not_to include("success") }
+      sample("zebra") { expect(subject["tags"]).not_to include("failure") }
+      sample("sample") { expect(subject["tags"]).not_to include("success") }
+    end
+
+    conditional "!([message] >= 'sample')" do
+      sample("zebra") { expect(subject["tags"]).not_to include("success") }
+      sample("sample") { expect(subject["tags"]).not_to include("success") }
+      sample("apple") { expect(subject["tags"]).not_to include("failure") }
+    end
+
+    conditional "!([message] =~ /sample/)" do
+      sample("apple") { expect(subject["tags"]).not_to include("failure") }
+      sample("sample") { expect(subject["tags"]).not_to include("success") }
+      sample("some sample") { expect(subject["tags"]).not_to include("success") }
+    end
+
+    conditional "!([message] !~ /sample/)" do
+      sample("apple") { expect(subject["tags"]).not_to include("success") }
+      sample("sample") { expect(subject["tags"]).not_to include("failure") }
+      sample("some sample") { expect(subject["tags"]).not_to include("failure") }
+    end
+
+  end
+
+  describe "value as an expression" do
+    # testing that a field has a value should be true.
+    conditional "[message]" do
+      sample("apple") { expect(subject["tags"]).to include("success") }
+      sample("sample") { expect(subject["tags"]).to include("success") }
+      sample("some sample") { expect(subject["tags"]).to include("success") }
+    end
+
+    # testing that a missing field has a value should be false.
+    conditional "[missing]" do
+      sample("apple") { expect(subject["tags"]).to include("failure") }
+      sample("sample") { expect(subject["tags"]).to include("failure") }
+      sample("some sample") { expect(subject["tags"]).to include("failure") }
+    end
+  end
+
+  describe "logic operators" do
+    describe "and" do
+      conditional "[message] and [message]" do
+        sample("whatever") { expect(subject["tags"]).to include("success") }
+      end
+      conditional "[message] and ![message]" do
+        sample("whatever") { expect(subject["tags"]).to include("failure") }
+      end
+      conditional "![message] and [message]" do
+        sample("whatever") { expect(subject["tags"]).to include("failure") }
+      end
+      conditional "![message] and ![message]" do
+        sample("whatever") { expect(subject["tags"]).to include("failure") }
+      end
+    end
+
+    describe "or" do
+      conditional "[message] or [message]" do
+        sample("whatever") { expect(subject["tags"]).to include("success") }
+      end
+      conditional "[message] or ![message]" do
+        sample("whatever") { expect(subject["tags"]).to include("success") }
+      end
+      conditional "![message] or [message]" do
+        sample("whatever") { expect(subject["tags"]).to include("success") }
+      end
+      conditional "![message] or ![message]" do
+        sample("whatever") { expect(subject["tags"]).to include("failure") }
+      end
+    end
+  end
+
+  describe "field references" do
+    conditional "[field with space]" do
+      sample("field with space" => "hurray") do
+        expect(subject["tags"]).to include("success")
+      end
+    end
+
+    conditional "[field with space] == 'hurray'" do
+      sample("field with space" => "hurray") do
+        expect(subject["tags"]).to include("success")
+      end
+    end
+
+    conditional "[nested field][reference with][some spaces] == 'hurray'" do
+      sample({"nested field" => { "reference with" => { "some spaces" => "hurray" } } }) do
+        expect(subject["tags"]).to include("success")
+      end
+    end
+  end
+
+  describe "new events from root" do
+    config <<-CONFIG
+      filter {
+        if [type] == "original" {
+          clone {
+            clones => ["clone"]
+          }
+        }
+        if [type] == "original" {
+          mutate { add_field => { "cond1" => "true" } }
+        } else {
+          mutate { add_field => { "cond2" => "true" } }
+        }
+      }
+    CONFIG
+
+    sample({"type" => "original"}) do
+      expect(subject).to be_an(Array)
+      expect(subject.length).to eq(2)
+
+      expect(subject[0]["type"]).to eq("original")
+      expect(subject[0]["cond1"]).to eq("true")
+      expect(subject[0]["cond2"]).to eq(nil)
+
+      expect(subject[1]["type"]).to eq("clone")
+      # expect(subject[1]["cond1"]).to eq(nil)
+      # expect(subject[1]["cond2"]).to eq("true")
+    end
+  end
+
+  describe "multiple new events from root" do
+    config <<-CONFIG
+      filter {
+        if [type] == "original" {
+          clone {
+            clones => ["clone1", "clone2"]
+          }
+        }
+        if [type] == "clone1" {
+          mutate { add_field => { "cond1" => "true" } }
+        } else if [type] == "clone2" {
+          mutate { add_field => { "cond2" => "true" } }
+        }
+      }
+    CONFIG
+
+    sample({"type" => "original"}) do
+      # puts subject.inspect
+      expect(subject[0]["cond1"]).to eq(nil)
+      expect(subject[0]["cond2"]).to eq(nil)
+
+      expect(subject[1]["type"]).to eq("clone1")
+      expect(subject[1]["cond1"]).to eq("true")
+      expect(subject[1]["cond2"]).to eq(nil)
+
+      expect(subject[2]["type"]).to eq("clone2")
+      expect(subject[2]["cond1"]).to eq(nil)
+      expect(subject[2]["cond2"]).to eq("true")
+    end
+  end
+
+end