logstash-core 1.5.0.rc3.snapshot6-java → 1.5.0.rc4-java

data/spec/core/pipeline_spec.rb

@@ -0,0 +1,198 @@
+require "spec_helper"
+
+class DummyInput < LogStash::Inputs::Base
+  config_name "dummyinput"
+  milestone 2
+
+  def register
+  end
+
+  def run(queue)
+  end
+
+  def teardown
+  end
+end
+
+class DummyCodec < LogStash::Codecs::Base
+  config_name "dummycodec"
+  milestone 2
+
+  def decode(data)
+    data
+  end
+
+  def encode(event)
+    event
+  end
+
+  def teardown
+  end
+end
+
+class DummyOutput < LogStash::Outputs::Base
+  config_name "dummyoutput"
+  milestone 2
+
+  attr_reader :num_teardowns
+
+  def initialize(params={})
+    super
+    @num_teardowns = 0
+  end
+
+  def register
+  end
+
+  def receive(event)
+  end
+
+  def teardown
+    @num_teardowns += 1
+  end
+end
+
+class TestPipeline < LogStash::Pipeline
+  attr_reader :outputs
+end
+
+describe LogStash::Pipeline do
+
+  context "teardown" do
+
+    before(:each) do
+      LogStash::Plugin.stub(:lookup)
+        .with("input", "dummyinput").and_return(DummyInput)
+      LogStash::Plugin.stub(:lookup)
+        .with("codec", "plain").and_return(DummyCodec)
+      LogStash::Plugin.stub(:lookup)
+        .with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    let(:test_config_without_output_workers) {
+      <<-eos
+      input {
+        dummyinput {}
+      }
+
+      output {
+        dummyoutput {}
+      }
+      eos
+    }
+
+    let(:test_config_with_output_workers) {
+      <<-eos
+      input {
+        dummyinput {}
+      }
+
+      output {
+        dummyoutput {
+          workers => 2
+        }
+      }
+      eos
+    }
+
+    context "output teardown" do
+      it "should call teardown of output without output-workers" do
+        pipeline = TestPipeline.new(test_config_without_output_workers)
+        pipeline.run
+
+        expect(pipeline.outputs.size ).to eq(1)
+        expect(pipeline.outputs.first.worker_plugins.size ).to eq(1)
+        expect(pipeline.outputs.first.worker_plugins.first.num_teardowns ).to eq(1)
+      end
+
+      it "should call output teardown correctly with output workers" do
+        pipeline = TestPipeline.new(test_config_with_output_workers)
+        pipeline.run
+
+        expect(pipeline.outputs.size ).to eq(1)
+        expect(pipeline.outputs.first.num_teardowns).to eq(0)
+        pipeline.outputs.first.worker_plugins.each do |plugin|
+          expect(plugin.num_teardowns ).to eq(1)
+        end
+      end
+    end
+  end
+
+  context "compiled flush function" do
+
+    context "cancelled events should not propagate down the filters" do
+      config <<-CONFIG
+        filter {
+          multiline {
+           pattern => "hello"
+           what => next
+          }
+          multiline {
+           pattern => "hello"
+           what => next
+          }
+        }
+      CONFIG
+
+      sample("hello") do
+        expect(subject["message"]).to eq("hello")
+      end
+    end
+
+    context "new events should propagate down the filters" do
+      config <<-CONFIG
+        filter {
+          clone {
+            clones => ["clone1"]
+          }
+          multiline {
+            pattern => "bar"
+            what => previous
+          }
+        }
+      CONFIG
+
+      sample(["foo", "bar"]) do
+        expect(subject.size).to eq(2)
+
+        expect(subject[0]["message"]).to eq("foo\nbar")
+        expect(subject[0]["type"]).to be_nil
+        expect(subject[1]["message"]).to eq("foo\nbar")
+        expect(subject[1]["type"]).to eq("clone1")
+      end
+    end
+  end
+
+  context "compiled filter funtions" do
+
+    context "new events should propagate down the filters" do
+      config <<-CONFIG
+        filter {
+          clone {
+            clones => ["clone1", "clone2"]
+          }
+          mutate {
+            add_field => {"foo" => "bar"}
+          }
+        }
+      CONFIG
+
+      sample("hello") do
+        expect(subject.size).to eq(3)
+
+        expect(subject[0]["message"]).to eq("hello")
+        expect(subject[0]["type"]).to be_nil
+        expect(subject[0]["foo"]).to eq("bar")
+
+        expect(subject[1]["message"]).to eq("hello")
+        expect(subject[1]["type"]).to eq("clone1")
+        expect(subject[1]["foo"]).to eq("bar")
+
+        expect(subject[2]["message"]).to eq("hello")
+        expect(subject[2]["type"]).to eq("clone2")
+        expect(subject[2]["foo"]).to eq("bar")
+      end
+    end
+
+  end
+end

data/spec/core/plugin_spec.rb

@@ -0,0 +1,106 @@
+require "spec_helper"
+require "logstash/plugin"
+
+describe LogStash::Plugin do
+  it "should fail lookup on inexisting type" do
+    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    expect { LogStash::Plugin.lookup("badbadtype", "badname") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail lookup on inexisting name" do
+    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    expect { LogStash::Plugin.lookup("filter", "badname") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail on bad plugin class" do
+    LogStash::Filters::BadSuperClass = Class.new
+    expect { LogStash::Plugin.lookup("filter", "bad_super_class") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail on missing config_name method" do
+    LogStash::Filters::MissingConfigName = Class.new(LogStash::Filters::Base)
+    expect { LogStash::Plugin.lookup("filter", "missing_config_name") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should lookup an already defined plugin class" do
+    class LogStash::Filters::LadyGaga < LogStash::Filters::Base
+      config_name "lady_gaga"
+    end
+    expect(LogStash::Plugin.lookup("filter", "lady_gaga")).to eq(LogStash::Filters::LadyGaga)
+  end
+
+  context "when validating the plugin version" do
+    let(:plugin_name) { 'logstash-filter-stromae' }
+    subject do
+      Class.new(LogStash::Filters::Base) do
+        config_name 'stromae'
+      end
+    end
+
+    it "doesn't warn the user if the version is superior or equal to 1.0.0" do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('1.0.0')))
+
+      expect_any_instance_of(Cabin::Channel).not_to receive(:info)
+      subject.validate({})
+    end
+
+    it 'warns the user if the plugin version is between 0.9.x and 1.0.0' do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.9.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .with(/Using version 0.9.x/)
+
+      subject.validate({})
+    end
+
+    it 'warns the user if the plugin version is inferior to 0.9.x' do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.1.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .with(/Using version 0.1.x/)
+      subject.validate({})
+    end
+
+    it "doesnt show the version notice more than once" do
+      one_notice = Class.new(LogStash::Filters::Base) do
+        config_name "stromae"
+      end
+
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.1.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .once
+        .with(/Using version 0.1.x/)
+
+      one_notice.validate({})
+      one_notice.validate({})
+    end
+
+    it "warns the user if we can't find a defined version" do
+      expect_any_instance_of(Cabin::Channel).to receive(:warn)
+        .once
+        .with(/plugin doesn't have a version/)
+
+      subject.validate({})
+    end
+    
+
+    it 'logs a warning if the plugin use the milestone option' do
+      expect_any_instance_of(Cabin::Channel).to receive(:warn)
+        .with(/stromae plugin is using the 'milestone' method/)
+
+      class LogStash::Filters::Stromae < LogStash::Filters::Base
+        config_name "stromae"
+        milestone 2
+      end
+    end
+  end
+end

data/spec/core/runner_spec.rb

@@ -0,0 +1,39 @@
+require "spec_helper"
+require "logstash/runner"
+require "stud/task"
+
+class NullRunner
+  def run(args); end
+end
+
+describe LogStash::Runner do
+
+  context "argument parsing" do
+    it "should run agent" do
+      expect(Stud::Task).to receive(:new).once.and_return(nil)
+      args = ["agent", "-e", ""]
+      expect(subject.run(args)).to eq(nil)
+    end
+
+    it "should run agent help" do
+      expect(subject).to receive(:show_help).once.and_return(nil)
+      args = ["agent", "-h"]
+      expect(subject.run(args).wait).to eq(0)
+    end
+
+    it "should show help with no arguments" do
+      expect($stderr).to receive(:puts).once.and_return("No command given")
+      expect($stderr).to receive(:puts).once
+      args = []
+      expect(subject.run(args).wait).to eq(1)
+    end
+
+    it "should show help for unknown commands" do
+      expect($stderr).to receive(:puts).once.and_return("No such command welp")
+      expect($stderr).to receive(:puts).once
+      args = ["welp"]
+      expect(subject.run(args).wait).to eq(1)
+    end
+
+  end
+end

data/spec/core/timestamp_spec.rb

@@ -0,0 +1,83 @@
+require "spec_helper"
+require "logstash/timestamp"
+
+describe LogStash::Timestamp do
+
+  it "should parse its own iso8601 output" do
+    t = Time.now
+    ts = LogStash::Timestamp.new(t)
+    expect(LogStash::Timestamp.parse_iso8601(ts.to_iso8601).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce iso8601 string" do
+    t = Time.now
+    ts = LogStash::Timestamp.new(t)
+    expect(LogStash::Timestamp.coerce(ts.to_iso8601).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce Time" do
+    t = Time.now
+    expect(LogStash::Timestamp.coerce(t).to_i).to eq(t.to_i)
+  end
+
+  it "should coerce Timestamp" do
+    t = LogStash::Timestamp.now
+    expect(LogStash::Timestamp.coerce(t).to_i).to eq(t.to_i)
+  end
+
+  it "should raise on invalid string coerce" do
+    expect{LogStash::Timestamp.coerce("foobar")}.to raise_error LogStash::TimestampParserError
+  end
+
+  it "should return nil on invalid object coerce" do
+    expect(LogStash::Timestamp.coerce(:foobar)).to be_nil
+  end
+
+  it "should support to_json" do
+    expect(LogStash::Timestamp.parse_iso8601("2014-09-23T00:00:00-0800").to_json).to eq("\"2014-09-23T08:00:00.000Z\"")
+  end
+
+  it "should support to_json and ignore arguments" do
+    expect(LogStash::Timestamp.parse_iso8601("2014-09-23T00:00:00-0800").to_json(:some => 1, :argumnents => "test")).to eq("\"2014-09-23T08:00:00.000Z\"")
+  end
+
+  it "should support timestamp comparaison" do
+   current = LogStash::Timestamp.new(Time.now) 
+   future = LogStash::Timestamp.new(Time.now + 100)
+
+   expect(future > current).to eq(true)
+   expect(future < current).to eq(false)
+   expect(current == current).to eq(true)
+
+   expect(current <=> current).to eq(0)
+   expect(current <=> future).to eq(-1)
+   expect(future <=> current).to eq(1)
+  end
+
+  it "should allow unary operation +" do
+    current = Time.now
+    t = LogStash::Timestamp.new(current) + 10
+    expect(t).to eq(current + 10)
+  end
+
+  describe "subtraction" do
+    it "should work on a timestamp object" do
+      t = Time.now
+      current = LogStash::Timestamp.new(t)
+      future = LogStash::Timestamp.new(t + 10)
+      expect(future - current).to eq(10)
+    end
+
+    it "should work on with time object" do
+      current = Time.now
+      t = LogStash::Timestamp.new(current + 10)
+      expect(t - current).to eq(10)
+    end
+
+    it "should work with numeric value" do
+      current = Time.now
+      t = LogStash::Timestamp.new(current + 10)
+      expect(t - 10).to eq(current)
+    end
+  end
+end

data/spec/filters/base_spec.rb

@@ -0,0 +1,318 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/json"
+
+# use a dummy NOOP filter to test Filters::Base
+class LogStash::Filters::NOOP < LogStash::Filters::Base
+  config_name "noop"
+  milestone 2
+
+  def register; end
+
+  def filter(event)
+    return unless filter?(event)
+    filter_matched(event)
+  end
+end
+
+describe LogStash::Filters::Base do
+  subject {LogStash::Filters::Base.new({})}
+
+  it "should provide method interfaces to override" do
+    expect{subject.register}.to raise_error(RuntimeError)
+    expect{subject.filter(:foo)}.to raise_error(RuntimeError)
+  end
+
+  it "should provide class public API" do
+    [:register, :filter, :multi_filter, :execute, :threadsafe?, :filter_matched, :filter?, :teardown].each do |method|
+      expect(subject).to respond_to(method)
+    end
+  end
+
+  context "multi_filter" do
+    let(:event1){LogStash::Event.new}
+    let(:event2){LogStash::Event.new}
+
+    it "should multi_filter without new events" do
+      allow(subject).to receive(:filter) do |event, &block|
+        nil
+      end
+      expect(subject.multi_filter([event1])).to eq([event1])
+    end
+
+    it "should multi_filter with new events" do
+      allow(subject).to receive(:filter) do |event, &block|
+        block.call(event2)
+      end
+      expect(subject.multi_filter([event1])).to eq([event1, event2])
+    end
+  end
+end
+
+describe LogStash::Filters::NOOP do
+
+  describe "adding multiple values to one field" do
+    config <<-CONFIG
+    filter {
+      noop {
+        add_field => ["new_field", "new_value"]
+        add_field => ["new_field", "new_value_2"]
+      }
+    }
+    CONFIG
+
+    sample "example" do
+      insist { subject["new_field"] } == ["new_value", "new_value_2"]
+    end
+  end
+
+  describe "type parsing" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        add_tag => ["test"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop") do
+      insist { subject["tags"] } == ["test"]
+    end
+
+    sample("type" => "not_noop") do
+      insist { subject["tags"] }.nil?
+    end
+  end
+
+  describe "tags parsing with one tag" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1"]
+        add_tag => ["test"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop") do
+      insist { subject["tags"] }.nil?
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2"]) do
+      insist { subject["tags"] } == ["t1", "t2", "test"]
+    end
+  end
+
+  describe "tags parsing with multiple tags" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1", "t2"]
+        add_tag => ["test"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop") do
+      insist { subject["tags"] }.nil?
+    end
+
+    sample("type" => "noop", "tags" => ["t1"]) do
+      insist { subject["tags"] } == ["t1"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2"]) do
+      insist { subject["tags"] } == ["t1", "t2", "test"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
+      insist { subject["tags"] } == ["t1", "t2", "t3", "test"]
+    end
+  end
+
+  describe "exclude_tags with 1 tag" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1"]
+        add_tag => ["test"]
+        exclude_tags => ["t2"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop") do
+      insist { subject["tags"] }.nil?
+    end
+
+    sample("type" => "noop", "tags" => ["t1"]) do
+      insist { subject["tags"] } == ["t1", "test"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2"]) do
+      insist { subject["tags"] } == ["t1", "t2"]
+    end
+  end
+
+  describe "exclude_tags with >1 tags" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1"]
+        add_tag => ["test"]
+        exclude_tags => ["t2", "t3"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "tags" => ["t1", "t2", "t4"]) do
+      insist { subject["tags"] } == ["t1", "t2", "t4"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t3", "t4"]) do
+      insist { subject["tags"] } == ["t1", "t3", "t4"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t4", "t5"]) do
+      insist { subject["tags"] } == ["t1", "t4", "t5", "test"]
+    end
+  end
+
+  describe "remove_tag" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1"]
+        remove_tag => ["t2", "t3"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "tags" => ["t4"]) do
+      insist { subject["tags"] } == ["t4"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
+      insist { subject["tags"] } == ["t1"]
+    end
+
+    # also test from Json deserialized data to test the handling of native Java collections by JrJackson
+    # see https://github.com/elastic/logstash/issues/2261
+    sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\", \"t3\"]}")) do
+      insist { subject["tags"] } == ["t1"]
+    end
+
+    sample("type" => "noop", "tags" => ["t1", "t2"]) do
+      insist { subject["tags"] } == ["t1"]
+    end
+
+    # also test from Json deserialized data to test the handling of native Java collections by JrJackson
+    # see https://github.com/elastic/logstash/issues/2261
+    sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\"]}")) do
+      insist { subject["tags"] } == ["t1"]
+    end
+  end
+
+  describe "remove_tag with dynamic value" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        tags => ["t1"]
+        remove_tag => ["%{blackhole}"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "tags" => ["t1", "goaway", "t3"], "blackhole" => "goaway") do
+      insist { subject["tags"] } == ["t1", "t3"]
+    end
+
+    # also test from Json deserialized data to test the handling of native Java collections by JrJackson
+    # see https://github.com/elastic/logstash/issues/2261
+    sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"goaway\", \"t3\"], \"blackhole\":\"goaway\"}")) do
+      insist { subject["tags"] } == ["t1", "t3"]
+    end
+  end
+
+  describe "remove_field" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        remove_field => ["t2", "t3"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "t4" => "four") do
+      insist { subject }.include?("t4")
+    end
+
+    sample("type" => "noop", "t1" => "one", "t2" => "two", "t3" => "three") do
+      insist { subject }.include?("t1")
+      reject { subject }.include?("t2")
+      reject { subject }.include?("t3")
+    end
+
+    sample("type" => "noop", "t1" => "one", "t2" => "two") do
+      insist { subject }.include?("t1")
+      reject { subject }.include?("t2")
+    end
+  end
+
+ describe "remove_field on deep objects" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        remove_field => ["[t1][t2]"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "t1" => {"t2" => "two", "t3" => "three"}) do
+      insist { subject }.include?("t1")
+      reject { subject }.include?("[t1][t2]")
+      insist { subject }.include?("[t1][t3]")
+    end
+  end
+
+ describe "remove_field on array" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        remove_field => ["[t1][0]"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "t1" => ["t2", "t3"]) do
+      insist { subject }.include?("t1")
+      insist { subject["[t1][0]"] } == "t3"
+    end
+  end
+
+  describe "remove_field with dynamic value in field name" do
+    config <<-CONFIG
+    filter {
+      noop {
+        type => "noop"
+        remove_field => ["%{blackhole}"]
+      }
+    }
+    CONFIG
+
+    sample("type" => "noop", "blackhole" => "go", "go" => "away") do
+      insist { subject }.include?("blackhole")
+      reject { subject }.include?("go")
+    end
+  end
+end