logstash-core 5.0.0.alpha4.snapshot1-java → 5.0.0.alpha4.snapshot2-java

data/spec/logstash/agent_spec.rb

@@ -328,11 +328,11 @@ describe LogStash::Agent do
     end
   end
 
+
   context "metrics after config reloading" do
-    let(:dummy_output) { DummyOutput.new }
     let(:config) { "input { generator { } } output { dummyoutput { } }" }
-    let(:new_config_generator_counter) { 50 }
-    let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput {} }" }
+    let(:new_config_generator_counter) { 500 }
+    let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
     let(:config_path) do
       f = Stud::Temporary.file
       f.write(config)
@@ -353,11 +353,24 @@ describe LogStash::Agent do
                     "metric.collect" => true })
     end 
 
+    # We need to create theses dummy classes to know how many
+    # events where actually generated by the pipeline and successfully send to the output.
+    # Theses values are compared with what we store in the metric store.
+    let!(:dummy_output) { DummyOutput.new }
+    let!(:dummy_output2) { DummyOutput.new }
+    class DummyOutput2 < LogStash::Outputs::Base; end
+
     before :each do
       allow(DummyOutput).to receive(:new).at_least(:once).with(anything).and_return(dummy_output)
+      allow(DummyOutput2).to receive(:new).at_least(:once).with(anything).and_return(dummy_output2)
+
       allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput2").and_return(DummyOutput2)
+
+      @abort_on_exception = Thread.abort_on_exception
+      Thread.abort_on_exception = true
 
       @t = Thread.new do
         subject.register_pipeline("main",  pipeline_settings)
@@ -368,28 +381,38 @@ describe LogStash::Agent do
     end
 
     after :each do
-      subject.shutdown
-      Stud.stop!(@t)
-      @t.join
+      begin
+        subject.shutdown
+        Stud.stop!(@t)
+        @t.join
+      ensure
+        Thread.abort_on_exception = @abort_on_exception
+      end
     end
 
     it "resets the metric collector" do
       # We know that the store has more events coming in.
-      sleep(0.01) while dummy_output.events.size < new_config_generator_counter
-      snapshot = LogStash::Instrument::Collector.instance.snapshot_metric
+      while dummy_output.events.size <= new_config_generator_counter
+        sleep(0.1)
+      end
+
+      snapshot = subject.metric.collector.snapshot_metric
       expect(snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value).to be > new_config_generator_counter
 
       # update the configuration and give some time to logstash to pick it up and do the work
-      IO.write(config_path, new_config)
+      # Also force a flush to disk to make sure ruby reload it.
+      File.open(config_path, "w") do |f|
+        f.write(new_config)
+        f.fsync
+      end
 
       sleep(interval * 3) # Give time to reload the config
       
-      # Since there is multiple threads involved with the configuration reload, 
-      # It can take some time to the stats be visible in the store but it will
       # be eventually consistent.
-      sleep(0.01) while dummy_output.events.size < new_config_generator_counter
+      sleep(0.01) while dummy_output2.events.size < new_config_generator_counter
 
-      value = LogStash::Instrument::Collector.instance.snapshot_metric.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
+      snapshot = subject.metric.collector.snapshot_metric
+      value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
       expect(value).to eq(new_config_generator_counter)
     end
   end

data/spec/logstash/config/config_ast_spec.rb

@@ -144,6 +144,49 @@ describe LogStashConfigParser do
     end
   end
 
+  context "when using two plugin sections of the same type" do
+    let(:pipeline_klass) do
+      Class.new do
+        def initialize(config)
+          grammar = LogStashConfigParser.new
+          @config = grammar.parse(config)
+          @code = @config.compile
+          eval(@code)
+        end
+        def plugin(*args);end
+      end
+    end
+    context "(filters)" do
+      let(:config_string) {
+        "input { generator { } }
+         filter { filter1 { } }
+         filter { filter1 { } }
+         output { output1 { } }"
+      }
+
+
+      it "should create a pipeline with both sections" do
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        filters = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^filter.+?_\d+$/) }
+        expect(filters.size).to eq(2)
+      end
+    end
+
+    context "(filters)" do
+      let(:config_string) {
+        "input { generator { } }
+         output { output1 { } }
+         output { output1 { } }"
+      }
+
+
+      it "should create a pipeline with both sections" do
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
+        outputs = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^output.+?_\d+$/) }
+        expect(outputs.size).to eq(2)
+      end
+    end
+  end
   context "when creating two instances of the same configuration" do
 
     let(:config_string) {

data/spec/logstash/config/mixin_spec.rb

@@ -68,6 +68,74 @@ describe LogStash::Config::Mixin do
     end
   end
 
+  context "when validating lists of items" do
+    let(:klass) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "multiuri"
+        config :uris, :validate => :uri, :list => true
+        config :strings, :validate => :string, :list => true
+        config :required_strings, :validate => :string, :list => true, :required => true
+      end
+    end
+
+    let(:uris) { ["http://example.net/1", "http://example.net/2"] }
+    let(:safe_uris) { uris.map {|str| ::LogStash::Util::SafeURI.new(str) } }
+    let(:strings) { ["I am a", "modern major general"] }
+    let(:required_strings) { ["required", "strings"] }
+
+    subject { klass.new("uris" => uris, "strings" => strings, "required_strings" => required_strings) }
+
+    it "a URI list should return an array of URIs" do
+      expect(subject.uris).to match_array(safe_uris)
+    end
+
+    it "a string list should return an array of strings" do
+      expect(subject.strings).to match_array(strings)
+    end
+
+    context "with a scalar value" do
+      let(:strings) { "foo" }
+
+      it "should return the scalar value as a single element array" do
+        expect(subject.strings).to match_array([strings])
+      end
+    end
+
+    context "with an empty list" do
+      let(:strings) { [] }
+
+      it "should return nil" do
+        expect(subject.strings).to be_nil
+      end
+    end
+
+    describe "with required => true" do
+      context "and a single element" do
+        let(:required_strings) { ["foo"] }
+
+        it "should return the single value" do
+          expect(subject.required_strings).to eql(required_strings)
+        end
+      end
+
+      context "with an empty list" do
+        let (:required_strings) { [] }
+
+        it "should raise a configuration error" do
+          expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
+        end        
+      end
+
+      context "with no value specified" do
+        let (:required_strings) { nil }
+
+        it "should raise a configuration error" do
+          expect { subject.required_strings }.to raise_error(LogStash::ConfigurationError)
+        end
+      end          
+    end
+  end
+
   context "when validating :password" do
     let(:klass) do
       Class.new(LogStash::Filters::Base)  do
@@ -102,6 +170,76 @@ describe LogStash::Config::Mixin do
     end
   end
 
+  context "when validating :uri" do
+    let(:klass) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "fakeuri"
+        config :uri, :validate => :uri
+      end
+    end
+
+    shared_examples("safe URI") do            
+      subject { klass.new("uri" => uri_str) }
+
+      it "should be a SafeURI object" do
+        expect(subject.uri).to(be_a(LogStash::Util::SafeURI))
+      end
+
+      it "should make password values hidden with #to_s" do
+        expect(subject.uri.to_s).to eql(uri_hidden)
+      end
+
+      it "should make password values hidden with #inspect" do
+        expect(subject.uri.inspect).to eql(uri_hidden)
+      end
+
+      it "should correctly copy URI types" do
+        clone = subject.class.new(subject.params)
+        expect(clone.uri.to_s).to eql(uri_hidden)
+      end
+
+      it "should make the real URI object availale under #uri" do
+        expect(subject.uri.uri).to be_a(::URI)
+      end
+
+      it "should obfuscate original_params" do
+        expect(subject.original_params['uri']).to(be_a(LogStash::Util::SafeURI))
+      end
+
+      context "attributes" do
+        [:scheme, :user, :password, :hostname, :path].each do |attr|
+          it "should make #{attr} available" do
+            expect(subject.uri.send(attr)).to eql(self.send(attr))
+          end
+        end
+      end
+    end
+
+    context "with a username / password" do
+      let(:scheme) { "myscheme" }
+      let(:user) { "myuser" }
+      let(:password) { "fancypants" }
+      let(:hostname) { "myhostname" }
+      let(:path) { "/my/path" }
+      let(:uri_str) { "#{scheme}://#{user}:#{password}@#{hostname}#{path}" }
+      let(:uri_hidden) { "#{scheme}://#{user}:#{LogStash::Util::SafeURI::PASS_PLACEHOLDER}@#{hostname}#{path}" }
+
+      include_examples("safe URI")
+    end
+
+    context "without a username / password" do
+      let(:scheme) { "myscheme" }
+      let(:user) { nil }
+      let(:password) { nil }
+      let(:hostname) { "myhostname" }
+      let(:path) { "/my/path" }
+      let(:uri_str) { "#{scheme}://#{hostname}#{path}" }
+      let(:uri_hidden) { "#{scheme}://#{hostname}#{path}" }
+
+      include_examples("safe URI")
+    end
+  end
+
   describe "obsolete settings" do
     let(:plugin_class) do
       Class.new(LogStash::Inputs::Base) do

data/spec/logstash/inputs/metrics_spec.rb

@@ -3,15 +3,17 @@ require "logstash/inputs/metrics"
 require "spec_helper"
 
 describe LogStash::Inputs::Metrics do
+  let(:collector) { LogStash::Instrument::Collector.new }
+  let(:metric) { LogStash::Instrument::Metric.new(collector) }
+  let(:queue) { [] }
+
   before :each do
-    LogStash::Instrument::Collector.instance.clear
+    allow(subject).to receive(:metric).and_return(metric)
   end
 
-  let(:queue) { [] }
-
   describe "#run" do
     it "should register itself to the collector observer" do
-      expect(LogStash::Instrument::Collector.instance).to receive(:add_observer).with(subject)
+      expect(collector).to receive(:add_observer).with(subject)
       t = Thread.new { subject.run(queue) }
       sleep(0.1) # give a bit of time to the thread to start
       subject.stop
@@ -19,24 +21,21 @@ describe LogStash::Inputs::Metrics do
   end
 
   describe "#update" do
-    let(:namespaces)  { [:root, :base] }
-    let(:key)        { :foo }
-    let(:metric_store) { LogStash::Instrument::MetricStore.new }
-
     it "should fill up the queue with received events" do
       Thread.new { subject.run(queue) }
       sleep(0.1)
       subject.stop
 
-      metric_store.fetch_or_store(namespaces, key, LogStash::Instrument::MetricType::Counter.new(namespaces, key))
-      subject.update(LogStash::Instrument::Snapshot.new(metric_store))
+      metric.increment([:root, :test], :plugin)
+
+      subject.update(collector.snapshot_metric)
       expect(queue.count).to eq(1)
     end
   end
 
   describe "#stop" do
     it "should remove itself from the the collector observer" do
-      expect(LogStash::Instrument::Collector.instance).to receive(:delete_observer).with(subject)
+      expect(collector).to receive(:delete_observer).with(subject)
       t = Thread.new { subject.run(queue) }
       sleep(0.1) # give a bit of time to the thread to start
       subject.stop

data/spec/logstash/instrument/collector_spec.rb

@@ -3,7 +3,7 @@ require "logstash/instrument/collector"
 require "spec_helper"
 
 describe LogStash::Instrument::Collector do
-  subject { LogStash::Instrument::Collector.instance }
+  subject { LogStash::Instrument::Collector.new }
   describe "#push" do
     let(:namespaces_path) { [:root, :pipelines, :pipelines01] }
     let(:key) { :my_key }

data/spec/logstash/instrument/metric_store_spec.rb

@@ -142,6 +142,67 @@ describe LogStash::Instrument::MetricStore do
       end
     end
 
+    describe "get_shallow" do
+      it "should retrieve a path as a single value" do
+        r = subject.get_shallow(:node, :sashimi, :pipelines, :pipeline01, :processed_events_in)
+        expect(r.value).to eql(1)
+      end
+    end
+
+    describe "extract_metrics" do
+      it "should retrieve non-nested values correctly" do
+        r = subject.extract_metrics(
+          [:node, :sashimi, :pipelines, :pipeline01],
+          :processed_events_in,
+          :processed_events_out,
+        )
+        expect(r[:processed_events_in]).to eql(1)
+        expect(r[:processed_events_out]).to eql(1)
+      end
+
+      it "should retrieve nested values correctly alongside non-nested ones" do
+        r = subject.extract_metrics(
+          [:node, :sashimi, :pipelines, :pipeline01],
+          :processed_events_in,
+          [:plugins, :"logstash-output-elasticsearch", :event_in]
+        )
+       expect(r[:processed_events_in]).to eql(1)
+        expect(r[:plugins][:"logstash-output-elasticsearch"][:event_in]).to eql(1)
+      end
+
+      it "should retrieve multiple nested keys at a given location" do
+        r = subject.extract_metrics(
+          [:node, :sashimi, :pipelines],
+          [:pipeline01, [:processed_events_in, :processed_events_out]]
+        )
+
+        expect(r[:pipeline01][:processed_events_in]).to eql(1)
+        expect(r[:pipeline01][:processed_events_out]).to eql(1)
+      end
+
+      it "should retrieve a single key nested in multiple places" do
+        r = subject.extract_metrics(
+          [:node, :sashimi, :pipelines],
+          [[:pipeline01, :pipeline02], :processed_events_out]
+        )
+
+        expect(r[:pipeline01][:processed_events_out]).to eql(1)
+        expect(r[:pipeline02][:processed_events_out]).to eql(1)
+      end
+
+      it "handle overlaps of paths" do
+        r = subject.extract_metrics(
+          [:node, :sashimi, :pipelines],
+          [:pipeline01, :processed_events_in],
+          [[:pipeline01, :pipeline02], :processed_events_out]
+        )
+
+        expect(r[:pipeline01][:processed_events_in]).to eql(1)
+        expect(r[:pipeline01][:processed_events_out]).to eql(1)
+        expect(r[:pipeline02][:processed_events_out]).to eql(1)
+      end
+    end
+
     describe "#each" do
       it "retrieves all the metric" do
         expect(subject.each.size).to eq(metric_events.size)

data/spec/logstash/instrument/periodic_poller/jvm_spec.rb

@@ -1,8 +1,10 @@
-require 'spec_helper'
-require 'logstash/instrument/periodic_poller/jvm'
+# encoding: utf-8
+require "spec_helper"
+require "logstash/instrument/periodic_poller/jvm"
+require "logstash/instrument/collector"
 
 describe LogStash::Instrument::PeriodicPoller::JVM do
-  let(:metric) { LogStash::Instrument::Metric.new }
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
   let(:options) { {} }
   subject(:jvm) { described_class.new(metric, options) }
   
@@ -18,6 +20,7 @@ describe LogStash::Instrument::PeriodicPoller::JVM do
     end
 
     describe "metrics" do
+      before(:each) { jvm.collect }
       let(:snapshot_store) { metric.collector.snapshot_metric.metric_store }
       subject(:jvm_metrics) { snapshot_store.get_shallow(:jvm, :process) }
 

data/spec/logstash/pipeline_spec.rb

@@ -468,8 +468,8 @@ describe LogStash::Pipeline do
       sleep 0.1 while !pipeline.ready?
       # give us a bit of time to flush the events
       wait(5).for do
-        next unless output && output.events && output.events.first
-        output.events.first.get("message").split("\n").count
+        next unless output && output.events && !(event = output.events.pop).nil?
+        event.get("message").split("\n").count
       end.to eq(number_of_events)
       pipeline.shutdown
     end
@@ -554,10 +554,12 @@ describe LogStash::Pipeline do
   end
 
   context "when collecting metrics in the pipeline" do
+    let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+
+    subject { described_class.new(config, pipeline_settings_obj, metric) }
+
     let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
-    subject { described_class.new(config, pipeline_settings_obj) }
     let(:pipeline_id) { "main" }
-    let(:metric) { LogStash::Instrument::Metric.new }
     let(:number_of_events) { 1000 }
     let(:multiline_id) { "my-multiline" }
     let(:multiline_id_other) { "my-multiline_other" }
@@ -591,6 +593,7 @@ describe LogStash::Pipeline do
       EOS
     end
     let(:dummyoutput) { DummyOutput.new({ "id" => dummy_output_id }) }
+    let(:metric_store) { subject.metric.collector.snapshot_metric.metric_store }
 
     before :each do
       allow(DummyOutput).to receive(:new).with(any_args).and_return(dummyoutput)
@@ -599,9 +602,6 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
 
-      # Reset the metric store
-      LogStash::Instrument::Collector.instance.clear
-
       Thread.new { subject.run }
       # make sure we have received all the generated events
       sleep 1 while dummyoutput.events.size < number_of_events
@@ -612,7 +612,7 @@ describe LogStash::Pipeline do
     end
 
     context "global metric" do
-      let(:collected_metric) { LogStash::Instrument::Collector.instance.snapshot_metric.metric_store.get_with_path("stats/events") }
+      let(:collected_metric) { metric_store.get_with_path("stats/events") }
 
       it "populates the differents" do
         expect(collected_metric[:stats][:events][:in].value).to eq(number_of_events)
@@ -622,7 +622,7 @@ describe LogStash::Pipeline do
     end
 
     context "pipelines" do
-      let(:collected_metric) { LogStash::Instrument::Collector.instance.snapshot_metric.metric_store.get_with_path("stats/pipelines/") }
+      let(:collected_metric) { metric_store.get_with_path("stats/pipelines/") }
 
       it "populates the pipelines core metrics" do
         expect(collected_metric[:stats][:pipelines][:main][:events][:in].value).to eq(number_of_events)