logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

data/spec/logstash/config/config_ast_spec.rb

@@ -169,10 +169,12 @@ describe LogStashConfigParser do
     end
 
     describe "generated conditional functionals" do
-      it "should be defined at instance level" do
+      it "should be created per instance" do
         instance_1 = pipeline_klass.new(config_string)
         instance_2 = pipeline_klass.new(config_string)
-        expect(instance_1.method(:cond_func_1).owner).to_not be(instance_2.method(:cond_func_1).owner)
+        generated_method_1 = instance_1.instance_variable_get("@generated_objects")[:cond_func_1]
+        generated_method_2 = instance_2.instance_variable_get("@generated_objects")[:cond_func_1]
+        expect(generated_method_1).to_not be(generated_method_2)
       end
     end
   end

data/spec/logstash/config/mixin_spec.rb

@@ -160,14 +160,14 @@ describe LogStash::Config::Mixin do
     let(:plugin_class) do
       Class.new(LogStash::Filters::Base)  do
         config_name "one_plugin"
-        config :oneString, :validate => :string
-        config :oneBoolean, :validate => :boolean
-        config :oneNumber, :validate => :number
-        config :oneArray, :validate => :array
-        config :oneHash, :validate => :hash
+        config :oneString, :validate => :string, :required => false
+        config :oneBoolean, :validate => :boolean, :required => false
+        config :oneNumber, :validate => :number, :required => false
+        config :oneArray, :validate => :array, :required => false
+        config :oneHash, :validate => :hash, :required => false
 
         def initialize(params)
-          super(params.merge(LogStash::Config::Mixin::ALLOW_ENV_FLAG => true))
+          super(params)
         end
       end
     end
@@ -231,8 +231,34 @@ describe LogStash::Config::Mixin do
         expect(subject.oneArray).to(be == [ "first array value", "fancy" ])
         expect(subject.oneHash).to(be == { "key1" => "fancy", "key2" => "fancy is true", "key3" => "true or false" })
       end
+    end
+
+    context "should support $ in values" do
+      before do
+        ENV["bar"] = "foo"
+        ENV["f$$"] = "bar"
+      end
+
+      after do
+        ENV.delete("bar")
+        ENV.delete("f$$")
+      end
+
+      subject do
+        plugin_class.new(
+          "oneString" => "${f$$:val}",
+          "oneArray" => ["foo$bar", "${bar:my$val}"]
+          # "dollar_in_env" => "${f$$:final}"
+        )
+      end
 
+      it "should support $ in values" do
+        expect(subject.oneArray).to(be == ["foo$bar", "foo"])
+      end
+
+      it "should not support $ in environment variable name" do
+        expect(subject.oneString).to(be == "${f$$:val}")
+      end
     end
   end
-
 end

data/spec/logstash/filters/base_spec.rb

@@ -62,7 +62,7 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample "example" do
-      insist { subject["new_field"] } == ["new_value", "new_value_2"]
+      insist { subject.get("new_field") } == ["new_value", "new_value_2"]
     end
   end
 
@@ -76,7 +76,7 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
   end
 
@@ -90,11 +90,11 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1", "t2", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "test"]
     end
   end
 
@@ -108,19 +108,19 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] } == ["test"]
+      insist { subject.get("tags") } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1"]) do
-      insist { subject["tags"] } == ["t1", "test"]
+      insist { subject.get("tags") } == ["t1", "test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1", "t2", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
-      insist { subject["tags"] } == ["t1", "t2", "t3", "test"]
+      insist { subject.get("tags") } == ["t1", "t2", "t3", "test"]
     end
   end
 
@@ -134,27 +134,27 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop", "tags" => ["t4"]) do
-      insist { subject["tags"] } == ["t4"]
+      insist { subject.get("tags") } == ["t4"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2", "t3"]) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\", \"t3\"]}")) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"t2\"]}")) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject.get("tags") } == ["t1"]
     end
   end
 
@@ -168,13 +168,13 @@ describe LogStash::Filters::NOOP do
     CONFIG
 
     sample("type" => "noop", "tags" => ["t1", "goaway", "t3"], "blackhole" => "goaway") do
-      insist { subject["tags"] } == ["t1", "t3"]
+      insist { subject.get("tags") } == ["t1", "t3"]
     end
 
     # also test from Json deserialized data to test the handling of native Java collections by JrJackson
     # see https://github.com/elastic/logstash/issues/2261
     sample(LogStash::Json.load("{\"type\":\"noop\", \"tags\":[\"t1\", \"goaway\", \"t3\"], \"blackhole\":\"goaway\"}")) do
-      insist { subject["tags"] } == ["t1", "t3"]
+      insist { subject.get("tags") } == ["t1", "t3"]
     end
   end
 
@@ -230,7 +230,7 @@ describe LogStash::Filters::NOOP do
 
     sample("type" => "noop", "t1" => ["t2", "t3"]) do
       insist { subject }.include?("t1")
-      insist { subject["[t1][0]"] } == "t3"
+      insist { subject.get("[t1][0]") } == "t3"
     end
   end
 

data/spec/logstash/inputs/base_spec.rb

@@ -15,50 +15,50 @@ describe "LogStash::Inputs::Base#decorate" do
     input = LogStash::Inputs::NOOP.new("tags" => "value")
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value"])
+    expect(evt.get("tags")).to eq(["value"])
   end
 
   it "should add multiple tag" do
     input = LogStash::Inputs::NOOP.new("tags" => ["value1","value2"])
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value1","value2"])
+    expect(evt.get("tags")).to eq(["value1","value2"])
   end
 
   it "should allow duplicates  tag" do
     input = LogStash::Inputs::NOOP.new("tags" => ["value","value"])
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["value","value"])
+    expect(evt.get("tags")).to eq(["value","value"])
   end
 
   it "should add tag with sprintf" do
     input = LogStash::Inputs::NOOP.new("tags" => "%{type}")
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["tags"]).to eq(["noop"])
+    expect(evt.get("tags")).to eq(["noop"])
   end
 
   it "should add single field" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"field" => "value"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["field"]).to eq("value")
+    expect(evt.get("field")).to eq("value")
   end
 
   it "should add single field with sprintf" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"%{type}" => "%{type}"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["noop"]).to eq("noop")
+    expect(evt.get("noop")).to eq("noop")
   end
 
   it "should add multiple field" do
     input = LogStash::Inputs::NOOP.new("add_field" => {"field" => ["value1", "value2"], "field2" => "value"})
     evt = LogStash::Event.new({"type" => "noop"})
     input.instance_eval {decorate(evt)}
-    expect(evt["field"]).to eq(["value1","value2"])
-    expect(evt["field2"]).to eq("value")
+    expect(evt.get("field")).to eq(["value1","value2"])
+    expect(evt.get("field2")).to eq("value")
   end
 end
 

data/spec/logstash/instrument/periodic_poller/jvm_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require 'logstash/instrument/periodic_poller/jvm'
+
+describe LogStash::Instrument::PeriodicPoller::JVM do
+  let(:metric) { LogStash::Instrument::Metric.new }
+  let(:options) { {} }
+  subject(:jvm) { described_class.new(metric, options) }
+  
+  it "should initialize cleanly" do
+    expect { jvm }.not_to raise_error
+  end
+
+  describe "collections" do
+    subject(:collection) { jvm.collect }
+    
+    it "should run cleanly" do
+      expect { collection }.not_to raise_error
+    end
+
+    describe "metrics" do
+      let(:snapshot_store) { metric.collector.snapshot_metric.metric_store }
+      subject(:jvm_metrics) { snapshot_store.get_shallow(:jvm, :process) }
+
+      # Make looking up metric paths easy when given varargs of keys
+      # e.g. mval(:parent, :child)
+      def mval(*metric_path)
+        metric_path.reduce(jvm_metrics) {|acc,k| acc[k]}.value
+      end          
+
+      [
+        :max_file_descriptors,
+        :open_file_descriptors,
+        :peak_open_file_descriptors,
+        [:mem, :total_virtual_in_bytes],
+        [:cpu, :total_in_millis],
+        [:cpu, :percent]
+      ].each do |path|
+        path = Array(path)
+        it "should have a value for #{path} that is Numeric" do
+          expect(mval(*path)).to be_a(Numeric)
+        end
+      end
+    end
+  end
+end

data/spec/logstash/output_delegator_spec.rb

@@ -13,10 +13,12 @@ describe LogStash::OutputDelegator do
     let(:out_klass) { double("output klass") }
     let(:out_inst) { double("output instance") }
 
+
     before(:each) do
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:threadsafe?).and_return(false)
       allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      allow(out_klass).to receive(:name).and_return("example")
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)

data/spec/logstash/pipeline_reporter_spec.rb

@@ -47,6 +47,10 @@ describe LogStash::PipelineReporter do
     @post_snapshot = reporter.snapshot
   end
 
+  after do
+    pipeline.shutdown
+  end
+
   describe "events filtered" do
     it "should start at zero" do
       expect(@pre_snapshot.events_filtered).to eql(0)
@@ -82,4 +86,4 @@ describe LogStash::PipelineReporter do
       expect(@post_snapshot.output_info.first[:events_received]).to eql(generator_count)
     end
   end
-end
+end

data/spec/logstash/pipeline_spec.rb

@@ -84,9 +84,21 @@ class TestPipeline < LogStash::Pipeline
 end
 
 describe LogStash::Pipeline do
-  let(:worker_thread_count)     { LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers] }
+  let(:worker_thread_count)     { 5 }
   let(:safe_thread_count)       { 1 }
   let(:override_thread_count)   { 42 }
+  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings) { {} }
+
+  before :each do
+    pipeline_workers_setting = LogStash::SETTINGS.get_setting("pipeline.workers")
+    allow(pipeline_workers_setting).to receive(:default).and_return(worker_thread_count)
+    pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
+  end
+
+  after :each do
+    pipeline_settings_obj.reset
+  end
 
   describe "defaulting the pipeline workers based on thread safety" do
     before(:each) do
@@ -123,13 +135,15 @@ describe LogStash::Pipeline do
         end
 
         it "should not receive a debug message with the compiled code" do
+          pipeline_settings_obj.set("config.debug", false)
           expect(logger).not_to receive(:debug).with(/Compiled pipeline/, anything)
           pipeline = TestPipeline.new(test_config_with_filters)
         end
 
-        it "should print the compiled code if debug_config is set to true" do
+        it "should print the compiled code if config.debug is set to true" do
+          pipeline_settings_obj.set("config.debug", true)
           expect(logger).to receive(:debug).with(/Compiled pipeline/, anything)
-          pipeline = TestPipeline.new(test_config_with_filters, :debug_config => true)
+          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
         end
       end
 
@@ -141,18 +155,21 @@ describe LogStash::Pipeline do
             {:count_was=>worker_thread_count, :filters=>["dummyfilter"]})
           pipeline.run
           expect(pipeline.worker_threads.size).to eq(safe_thread_count)
+          pipeline.shutdown
         end
       end
 
       context "when there is command line -w N set" do
+        let(:pipeline_settings) { {"pipeline.workers" => override_thread_count } }
         it "starts multiple filter thread" do
-          msg = "Warning: Manual override - there are filters that might not work with multiple worker threads"
-          pipeline = TestPipeline.new(test_config_with_filters)
+          msg = "Warning: Manual override - there are filters that might" +
+                " not work with multiple worker threads"
+          pipeline = TestPipeline.new(test_config_with_filters, pipeline_settings_obj)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:worker_threads=> override_thread_count, :filters=>["dummyfilter"]})
-          pipeline.configure(:pipeline_workers, override_thread_count)
           pipeline.run
           expect(pipeline.worker_threads.size).to eq(override_thread_count)
+          pipeline.shutdown
         end
       end
     end
@@ -178,6 +195,7 @@ describe LogStash::Pipeline do
         pipeline = TestPipeline.new(test_config_with_filters)
         pipeline.run
         expect(pipeline.worker_threads.size).to eq(worker_thread_count)
+        pipeline.shutdown
       end
     end
   end
@@ -222,8 +240,9 @@ describe LogStash::Pipeline do
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
-        expect(pipeline.outputs.first.workers.size ).to eq(::LogStash::Pipeline::DEFAULT_OUTPUT_WORKERS)
+        expect(pipeline.outputs.first.workers.size ).to eq(::LogStash::SETTINGS.get("pipeline.output.workers"))
         expect(pipeline.outputs.first.workers.first.num_closes ).to eq(1)
+        pipeline.shutdown
       end
 
       it "should call output close correctly with output workers" do
@@ -240,6 +259,7 @@ describe LogStash::Pipeline do
         output_delegator.workers.each do |plugin|
           expect(plugin.num_closes ).to eq(1)
         end
+        pipeline.shutdown
       end
     end
   end
@@ -261,6 +281,7 @@ describe LogStash::Pipeline do
         expect(pipeline).to receive(:start_flusher).ordered.and_call_original
 
         pipeline.run
+        pipeline.shutdown
       end
     end
 
@@ -279,7 +300,7 @@ describe LogStash::Pipeline do
       CONFIG
 
       sample("hello") do
-        expect(subject["message"]).to eq("hello")
+        expect(subject.get("message")).to eq("hello")
       end
     end
 
@@ -299,10 +320,10 @@ describe LogStash::Pipeline do
       sample(["foo", "bar"]) do
         expect(subject.size).to eq(2)
 
-        expect(subject[0]["message"]).to eq("foo\nbar")
-        expect(subject[0]["type"]).to be_nil
-        expect(subject[1]["message"]).to eq("foo\nbar")
-        expect(subject[1]["type"]).to eq("clone1")
+        expect(subject[0].get("message")).to eq("foo\nbar")
+        expect(subject[0].get("type")).to be_nil
+        expect(subject[1].get("message")).to eq("foo\nbar")
+        expect(subject[1].get("type")).to eq("clone1")
       end
     end
   end
@@ -310,7 +331,8 @@ describe LogStash::Pipeline do
   describe "max inflight warning" do
     let(:config) { "input { dummyinput {} } output { dummyoutput {} }" }
     let(:batch_size) { 1 }
-    let(:pipeline) { LogStash::Pipeline.new(config, :pipeline_batch_size => batch_size, :pipeline_workers => 1) }
+    let(:pipeline_settings) { { "pipeline.batch.size" => batch_size, "pipeline.workers" => 1 } }
+    let(:pipeline) { LogStash::Pipeline.new(config, pipeline_settings_obj) }
     let(:logger) { pipeline.logger }
     let(:warning_prefix) { /CAUTION: Recommended inflight events max exceeded!/ }
 
@@ -354,17 +376,17 @@ describe LogStash::Pipeline do
       sample("hello") do
         expect(subject.size).to eq(3)
 
-        expect(subject[0]["message"]).to eq("hello")
-        expect(subject[0]["type"]).to be_nil
-        expect(subject[0]["foo"]).to eq("bar")
+        expect(subject[0].get("message")).to eq("hello")
+        expect(subject[0].get("type")).to be_nil
+        expect(subject[0].get("foo")).to eq("bar")
 
-        expect(subject[1]["message"]).to eq("hello")
-        expect(subject[1]["type"]).to eq("clone1")
-        expect(subject[1]["foo"]).to eq("bar")
+        expect(subject[1].get("message")).to eq("hello")
+        expect(subject[1].get("type")).to eq("clone1")
+        expect(subject[1].get("foo")).to eq("bar")
 
-        expect(subject[2]["message"]).to eq("hello")
-        expect(subject[2]["type"]).to eq("clone2")
-        expect(subject[2]["foo"]).to eq("bar")
+        expect(subject[2].get("message")).to eq("hello")
+        expect(subject[2].get("type")).to eq("clone2")
+        expect(subject[2].get("foo")).to eq("bar")
       end
     end
   end
@@ -376,8 +398,14 @@ describe LogStash::Pipeline do
     output { }
     CONFIG
 
-    it "uses a `NullMetric` object if no metric is given" do
-      pipeline = LogStash::Pipeline.new(config)
+    it "uses a `NullMetric` object if `metric.collect` is set to false" do
+      settings = double("LogStash::SETTINGS")
+
+      allow(settings).to receive(:get_value).with("pipeline.id").and_return("main")
+      allow(settings).to receive(:get_value).with("metric.collect").and_return(false)
+      allow(settings).to receive(:get_value).with("config.debug").and_return(false)
+
+      pipeline = LogStash::Pipeline.new(config, settings)
       expect(pipeline.metric).to be_kind_of(LogStash::Instrument::NullMetric)
     end
   end
@@ -435,13 +463,13 @@ describe LogStash::Pipeline do
 
     it "flushes the buffered contents of the filter" do
       Thread.abort_on_exception = true
-      pipeline = LogStash::Pipeline.new(config, { :flush_interval => 1 })
+      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
       # give us a bit of time to flush the events
       wait(5).for do
         next unless output && output.events && output.events.first
-        output.events.first["message"].split("\n").count
+        output.events.first.get("message").split("\n").count
       end.to eq(number_of_events)
       pipeline.shutdown
     end
@@ -495,7 +523,7 @@ describe LogStash::Pipeline do
       t = Thread.new { subject.run }
       sleep(0.1)
       expect(subject.started_at).to be < Time.now
-      t.kill rescue nil
+      subject.shutdown
     end
   end
 
@@ -520,14 +548,15 @@ describe LogStash::Pipeline do
         t = Thread.new { subject.run }
         sleep(0.1)
         expect(subject.uptime).to be > 0
-        t.kill rescue nil
+        subject.shutdown
       end
     end
   end
 
   context "when collecting metrics in the pipeline" do
-    subject { described_class.new(config, { :metric => metric, :pipeline_id => pipeline_id }) }
-    let(:pipeline_id) { :main }
+    let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
+    subject { described_class.new(config, pipeline_settings_obj) }
+    let(:pipeline_id) { "main" }
     let(:metric) { LogStash::Instrument::Metric.new }
     let(:number_of_events) { 1000 }
     let(:multiline_id) { "my-multiline" }
@@ -616,4 +645,20 @@ describe LogStash::Pipeline do
       end
     end
   end
+
+  context "Pipeline object" do
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+
+    it "should not add ivars" do
+       expect(pipeline1.instance_variables).to eq(pipeline2.instance_variables)
+    end
+  end
 end