logstash-core 5.0.0.alpha3.snapshot2-java → 5.0.0.alpha3.snapshot4-java

data/spec/logstash/config/config_ast_spec.rb

@@ -169,10 +169,12 @@ describe LogStashConfigParser do
     end
 
     describe "generated conditional functionals" do
-      it "should be defined at instance level" do
+      it "should be created per instance" do
         instance_1 = pipeline_klass.new(config_string)
         instance_2 = pipeline_klass.new(config_string)
-        expect(instance_1.method(:cond_func_1).owner).to_not be(instance_2.method(:cond_func_1).owner)
+        generated_method_1 = instance_1.instance_variable_get("@generated_objects")[:cond_func_1]
+        generated_method_2 = instance_2.instance_variable_get("@generated_objects")[:cond_func_1]
+        expect(generated_method_1).to_not be(generated_method_2)
       end
     end
   end

data/spec/logstash/config/mixin_spec.rb

@@ -160,14 +160,14 @@ describe LogStash::Config::Mixin do
     let(:plugin_class) do
       Class.new(LogStash::Filters::Base)  do
         config_name "one_plugin"
-        config :oneString, :validate => :string
-        config :oneBoolean, :validate => :boolean
-        config :oneNumber, :validate => :number
-        config :oneArray, :validate => :array
-        config :oneHash, :validate => :hash
+        config :oneString, :validate => :string, :required => false
+        config :oneBoolean, :validate => :boolean, :required => false
+        config :oneNumber, :validate => :number, :required => false
+        config :oneArray, :validate => :array, :required => false
+        config :oneHash, :validate => :hash, :required => false
 
         def initialize(params)
-          super(params.merge(LogStash::Config::Mixin::ALLOW_ENV_FLAG => true))
+          super(params)
         end
       end
     end
@@ -231,8 +231,34 @@ describe LogStash::Config::Mixin do
         expect(subject.oneArray).to(be == [ "first array value", "fancy" ])
         expect(subject.oneHash).to(be == { "key1" => "fancy", "key2" => "fancy is true", "key3" => "true or false" })
       end
+    end
+
+    context "should support $ in values" do
+      before do
+        ENV["bar"] = "foo"
+        ENV["f$$"] = "bar"
+      end
+
+      after do
+        ENV.delete("bar")
+        ENV.delete("f$$")
+      end
+
+      subject do
+        plugin_class.new(
+          "oneString" => "${f$$:val}",
+          "oneArray" => ["foo$bar", "${bar:my$val}"]
+          # "dollar_in_env" => "${f$$:final}"
+        )
+      end
 
+      it "should support $ in values" do
+        expect(subject.oneArray).to(be == ["foo$bar", "foo"])
+      end
+
+      it "should not support $ in environment variable name" do
+        expect(subject.oneString).to(be == "${f$$:val}")
+      end
     end
   end
-
 end

data/spec/logstash/filters/base_spec.rb

@@ -62,7 +62,7 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample "example" do
-      insist { subject["new_field"] } == ["new_value", "new_value_2"]
+      insist { subject.get("new_field") } == ["new_value", "new_value_2"]
     end
   end
 
@@ -76,7 +76,7 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
   end
 
@@ -90,11 +90,11 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1", "t2", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "test"]
     end
   end
 
@@ -108,19 +108,19 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1"]) do
-      insist { subject["tags"] } == ["t1", "test"]
+      insist { subject.get("tags") } == ["t1", "test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1", "t2", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
-      insist { subject["tags"] } == ["t1", "t2", "t3", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "t3", "test"]
     end
   end
 
@@ -134,27 +134,27 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop", "tags" => ["t4"]) do
-      insist { subject["tags"] } == ["t4"]
+      insist { subject.get("tags") } == ["t4"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\", \"t3\"]}")) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\"]}")) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
   end
 
@@ -168,13 +168,13 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop", "tags" => ["t1", "goaway", "t3"], "blackhole" => "goaway") do
-      insist { subject["tags"] } == ["t1", "t3"]
+      insist { subject.get("tags") } == ["t1", "t3"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"goaway\", \"t3\"], \"blackhole\":\"goaway\"}")) do
-      insist { subject["tags"] } == ["t1", "t3"]
+      insist { subject.get("tags") } == ["t1", "t3"]
     end
   end
 
@@ -230,7 +230,7 @@ describe LogStash::Filters::NOOP do
 
     sample("type" => "noop", "t1" => ["t2", "t3"]) do
       insist { subject }.include?("t1")
-      insist { subject["[t1][0]"] } == "t3"
+      insist { subject.get("[t1][0]") } == "t3"
     end
   end
 

data/spec/logstash/inputs/base_spec.rb

@@ -15,50 +15,50 @@ describe "LogStash::Inputs::Base#decorate" do
     input = LogStash::Inputs::NOOP.new("tags" => "value")
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value"])
+    expect(evt.get("tags")).to eq(["value"])
   end
 
   it "should add multiple tag" do
     input = LogStash::Inputs::NOOP.new("tags" => ["value1","value2"])
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value1","value2"])
+    expect(evt.get("tags")).to eq(["value1","value2"])
   end
 
   it "should allow duplicates  tag" do
     input = LogStash::Inputs::NOOP.new("tags" => ["value","value"])
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value","value"])
+    expect(evt.get("tags")).to eq(["value","value"])
   end
 
   it "should add tag with sprintf" do
     input = LogStash::Inputs::NOOP.new("tags" => "%{type}")
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["noop"])
+    expect(evt.get("tags")).to eq(["noop"])
   end
 
   it "should add single field" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"field" => "value"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["field"]).to eq("value")
+    expect(evt.get("field")).to eq("value")
   end
 
   it "should add single field with sprintf" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"%{type}" => "%{type}"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["noop"]).to eq("noop")
+    expect(evt.get("noop")).to eq("noop")
   end
 
   it "should add multiple field" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"field" => ["value1", "value2"], "field2" => "value"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["field"]).to eq(["value1","value2"])
-    expect(evt["field2"]).to eq("value")
+    expect(evt.get("field")).to eq(["value1","value2"])
+    expect(evt.get("field2")).to eq("value")
   end
 end
 

data/spec/logstash/output_delegator_spec.rb

@@ -13,10 +13,12 @@ describe LogStash::OutputDelegator do
     let(:out_klass) { double("output klass") }
     let(:out_inst) { double("output instance") }
 
+
     before(:each) do
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:threadsafe?).and_return(false)
       allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      allow(out_klass).to receive(:name).and_return("example")
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)

data/spec/logstash/pipeline_spec.rb

@@ -84,9 +84,21 @@ class TestPipeline < LogStash::Pipeline
 end
 
 describe LogStash::Pipeline do
-  let(:worker_thread_count)     { LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers] }
+  let(:worker_thread_count)     { 5 }
   let(:safe_thread_count)       { 1 }
   let(:override_thread_count)   { 42 }
+  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings) { {} }
+
+  before :each do
+    pipeline_workers_setting = LogStash::SETTINGS.get_setting("pipeline.workers")
+    allow(pipeline_workers_setting).to receive(:default).and_return(worker_thread_count)
+    pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
+  end
+
+  after :each do
+    pipeline_settings_obj.reset
+  end
 
   describe "defaulting the pipeline workers based on thread safety" do
     before(:each) do
@@ -123,13 +135,15 @@ describe LogStash::Pipeline do
         end
 
         it "should not receive a debug message with the compiled code" do
+          pipeline_settings_obj.set("config.debug", false)
           expect(logger).not_to receive(:debug).with(/Compiled pipeline/, anything)
           pipeline = TestPipeline.new(test_config_with_filters)
         end
 
-        it "should print the compiled code if debug_config is set to true" do
+        it "should print the compiled code if config.debug is set to true" do
+          pipeline_settings_obj.set("config.debug", true)
           expect(logger).to receive(:debug).with(/Compiled pipeline/, anything)
-          pipeline = TestPipeline.new(test_config_with_filters, :debug_config => true)
+          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
         end
       end
 
@@ -145,12 +159,13 @@ describe LogStash::Pipeline do
       end
 
       context "when there is command line -w N set" do
+        let(:pipeline_settings) { {"pipeline.workers" => override_thread_count } }
         it "starts multiple filter thread" do
-          msg = "Warning: Manual override - there are filters that might not work with multiple worker threads"
-          pipeline = TestPipeline.new(test_config_with_filters)
+          msg = "Warning: Manual override - there are filters that might" +
+                " not work with multiple worker threads"
+          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:worker_threads=> override_thread_count, :filters=>["dummyfilter"]})
-          pipeline.configure(:pipeline_workers, override_thread_count)
           pipeline.run
           expect(pipeline.worker_threads.size).to eq(override_thread_count)
         end
@@ -222,7 +237,7 @@ describe LogStash::Pipeline do
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
-        expect(pipeline.outputs.first.workers.size ).to eq(::LogStash::Pipeline::DEFAULT_OUTPUT_WORKERS)
+        expect(pipeline.outputs.first.workers.size ).to eq(::LogStash::SETTINGS.get("pipeline.output.workers"))
         expect(pipeline.outputs.first.workers.first.num_closes ).to eq(1)
       end
 
@@ -279,7 +294,7 @@ describe LogStash::Pipeline do
       CONFIG
 
       sample("hello") do
-        expect(subject["message"]).to eq("hello")
+        expect(subject.get("message")).to eq("hello")
       end
     end
 
@@ -299,10 +314,10 @@ describe LogStash::Pipeline do
       sample(["foo", "bar"]) do
         expect(subject.size).to eq(2)
 
-        expect(subject[0]["message"]).to eq("foo\nbar")
-        expect(subject[0]["type"]).to be_nil
-        expect(subject[1]["message"]).to eq("foo\nbar")
-        expect(subject[1]["type"]).to eq("clone1")
+        expect(subject[0].get("message")).to eq("foo\nbar")
+        expect(subject[0].get("type")).to be_nil
+        expect(subject[1].get("message")).to eq("foo\nbar")
+        expect(subject[1].get("type")).to eq("clone1")
       end
     end
   end
@@ -310,7 +325,8 @@ describe LogStash::Pipeline do
   describe "max inflight warning" do
     let(:config) { "input { dummyinput {} } output { dummyoutput {} }" }
     let(:batch_size) { 1 }
-    let(:pipeline) { LogStash::Pipeline.new(config, :pipeline_batch_size => batch_size, :pipeline_workers => 1) }
+    let(:pipeline_settings) { { "pipeline.batch.size" => batch_size, "pipeline.workers" => 1 } }
+    let(:pipeline) { LogStash::Pipeline.new(config, pipeline_settings_obj) }
     let(:logger) { pipeline.logger }
     let(:warning_prefix) { /CAUTION: Recommended inflight events max exceeded!/ }
 
@@ -354,17 +370,17 @@ describe LogStash::Pipeline do
       sample("hello") do
         expect(subject.size).to eq(3)
 
-        expect(subject[0]["message"]).to eq("hello")
-        expect(subject[0]["type"]).to be_nil
-        expect(subject[0]["foo"]).to eq("bar")
+        expect(subject[0].get("message")).to eq("hello")
+        expect(subject[0].get("type")).to be_nil
+        expect(subject[0].get("foo")).to eq("bar")
 
-        expect(subject[1]["message"]).to eq("hello")
-        expect(subject[1]["type"]).to eq("clone1")
-        expect(subject[1]["foo"]).to eq("bar")
+        expect(subject[1].get("message")).to eq("hello")
+        expect(subject[1].get("type")).to eq("clone1")
+        expect(subject[1].get("foo")).to eq("bar")
 
-        expect(subject[2]["message"]).to eq("hello")
-        expect(subject[2]["type"]).to eq("clone2")
-        expect(subject[2]["foo"]).to eq("bar")
+        expect(subject[2].get("message")).to eq("hello")
+        expect(subject[2].get("type")).to eq("clone2")
+        expect(subject[2].get("foo")).to eq("bar")
       end
     end
   end
@@ -435,13 +451,13 @@ describe LogStash::Pipeline do
 
     it "flushes the buffered contents of the filter" do
       Thread.abort_on_exception = true
-      pipeline = LogStash::Pipeline.new(config, { :flush_interval => 1 })
+      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
       # give us a bit of time to flush the events
       wait(5).for do
         next unless output && output.events && output.events.first
-        output.events.first["message"].split("\n").count
+        output.events.first.get("message").split("\n").count
       end.to eq(number_of_events)
       pipeline.shutdown
     end
@@ -526,8 +542,9 @@ describe LogStash::Pipeline do
   end
 
   context "when collecting metrics in the pipeline" do
-    subject { described_class.new(config, { :metric => metric, :pipeline_id => pipeline_id }) }
-    let(:pipeline_id) { :main }
+    let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
+    subject { described_class.new(config, pipeline_settings_obj) }
+    let(:pipeline_id) { "main" }
     let(:metric) { LogStash::Instrument::Metric.new }
     let(:number_of_events) { 1000 }
     let(:multiline_id) { "my-multiline" }
@@ -573,6 +590,7 @@ describe LogStash::Pipeline do
       # Reset the metric store
       LogStash::Instrument::Collector.instance.clear
 
+      subject.metric = metric
       Thread.new { subject.run }
       # make sure we have received all the generated events
       sleep 1 while dummyoutput.events.size < number_of_events
@@ -616,4 +634,20 @@ describe LogStash::Pipeline do
       end
     end
   end
+
+  context "Pipeline object" do
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+
+    it "should not add ivars" do
+       expect(pipeline1.instance_variables).to eq(pipeline2.instance_variables)
+    end
+  end
 end

data/spec/logstash/plugin_spec.rb

@@ -8,12 +8,12 @@ require "logstash/filters/base"
 
 describe LogStash::Plugin do
   it "should fail lookup on inexisting type" do
-    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    #expect_any_instance_of(Cabin::Channel).to receive(:debug).once
     expect { LogStash::Plugin.lookup("badbadtype", "badname") }.to raise_error(LogStash::PluginLoadingError)
   end
 
   it "should fail lookup on inexisting name" do
-    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    #expect_any_instance_of(Cabin::Channel).to receive(:debug).once
     expect { LogStash::Plugin.lookup("filter", "badname") }.to raise_error(LogStash::PluginLoadingError)
   end