logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/spec/logstash/json_spec.rb

@@ -59,6 +59,20 @@ describe "LogStash::Json" do
         expect(LogStash::Json.dump(array)).to eql(json_array)
       end
 
+      context "pretty print" do
+
+        let(:hash) { { "foo" => "bar", :zoo => 2 } }
+
+        it "should serialize with pretty print" do
+          pprint_json = LogStash::Json.dump(hash, :pretty => true)
+          expect(pprint_json).to include("\n")
+        end
+
+        it "should by default do no pretty print" do
+          pprint_json = LogStash::Json.dump(hash)
+          expect(pprint_json).not_to include("\n")
+        end
+      end
     end
 
   else

data/spec/logstash/output_delegator_spec.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash/output_delegator"
 require 'spec_helper'
 
 describe LogStash::OutputDelegator do
@@ -6,19 +7,21 @@ describe LogStash::OutputDelegator do
   let(:events) { 7.times.map { LogStash::Event.new }}
   let(:default_worker_count) { 1 }
 
-  subject { described_class.new(logger, out_klass, default_worker_count) }
+  subject { described_class.new(logger, out_klass, default_worker_count, LogStash::Instrument::NullMetric.new) }
 
   context "with a plain output plugin" do
     let(:out_klass) { double("output klass") }
     let(:out_inst) { double("output instance") }
 
-    before do
+    before(:each) do
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:threadsafe?).and_return(false)
       allow(out_klass).to receive(:workers_not_supported?).and_return(false)
-      allow(out_klass).to receive(:name).and_return("example")
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
+      allow(out_inst).to receive(:metric=).with(any_args)
+      allow(out_inst).to receive(:id).and_return("a-simple-plugin")
+      allow(out_inst).to receive(:plugin_unique_name).and_return("hello-123")
       allow(logger).to receive(:debug).with(any_args)
     end
 

data/spec/logstash/outputs/base_spec.rb

@@ -15,31 +15,6 @@ class LogStash::Outputs::NOOP < LogStash::Outputs::Base
   end
 end
 
-
-# use a dummy NOOP output to test Outputs::Base
-class LogStash::Outputs::NOOPSingle < LogStash::Outputs::Base
-  config_name "noop single"
-  concurrency :single
-
-  config :dummy_option, :validate => :string
-
-  def register; end
-
-  def receive(event)
-    return output?(event)
-  end
-end
-
-class LogStash::Outputs::NOOPShared < ::LogStash::Outputs::Base
-  concurrency :shared
-  
-  def register; end
-end
-
-class LogStash::Outputs::NOOPLegacy < ::LogStash::Outputs::Base
-  def register; end
-end
-
 class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
   LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
 
@@ -48,52 +23,7 @@ class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
   end
 end
 
-class LogStash::Outputs::NOOPMultiReceiveEncoded < ::LogStash::Outputs::Base
-  concurrency :single
-  
-  def register; end
-
-  def multi_receive_encoded(events_and_encoded)
-  end
-end
-
 describe "LogStash::Outputs::Base#new" do
-  describe "concurrency" do
-    subject { klass.new({}) }
-    
-    context "single" do   
-      let(:klass) { LogStash::Outputs::NOOPSingle }
-
-      it "should set concurrency correctly" do
-        expect(subject.concurrency).to eq(:single)
-      end
-    end
-
-    context "shared" do
-      let(:klass) { LogStash::Outputs::NOOPShared }
-      
-      it "should set concurrency correctly" do
-        expect(subject.concurrency).to eq(:shared)
-      end
-    end
-
-    context "legacy" do
-      let(:klass) { LogStash::Outputs::NOOPLegacy }
-      
-      it "should set concurrency correctly" do
-        expect(subject.concurrency).to eq(:legacy)
-      end
-
-      it "should default the # of workers to 1" do
-        expect(subject.workers).to eq(1)
-      end
-
-      it "should default concurrency to :legacy" do
-        expect(subject.concurrency).to eq(:legacy)
-      end
-    end
-  end
-  
   it "should instantiate cleanly" do
     params = { "dummy_option" => "potatoes", "codec" => "json", "workers" => 2 }
     worker_params = params.dup; worker_params["workers"] = 1
@@ -107,41 +37,4 @@ describe "LogStash::Outputs::Base#new" do
     LogStash::Outputs::NOOPLegacyNoWorkers.new.register
     expect(LogStash::Outputs::NOOPLegacyNoWorkers.workers_not_supported?).to eql(true)
   end
-
-  describe "dispatching multi_receive" do
-    let(:event) { double("event") }
-    let(:events) { [event] }
-    subject { klass.new({}) }
-    
-    context "with multi_receive_encoded" do
-      let(:klass) { LogStash::Outputs::NOOPMultiReceiveEncoded }
-      let(:codec) { double("codec") }
-      let(:encoded) { double("encoded") }
-      
-      before do
-        allow(codec).to receive(:multi_encode).with(events).and_return(encoded)
-        allow(subject).to receive(:codec).and_return(codec)
-        allow(subject).to receive(:multi_receive_encoded)
-        subject.multi_receive(events)
-      end
-
-      it "should invoke multi_receive_encoded if it exists" do
-        expect(subject).to have_received(:multi_receive_encoded).with(encoded)
-      end
-    end
-
-    context "with plain #receive" do
-      let(:klass) { LogStash::Outputs::NOOPSingle }
-
-      before do
-        allow(subject).to receive(:multi_receive).and_call_original
-        allow(subject).to receive(:receive).with(event)
-        subject.multi_receive(events)
-      end
-
-      it "should receive the event by itself" do
-        expect(subject).to have_received(:receive).with(event)
-      end
-    end
-  end
 end

data/spec/logstash/pipeline_spec.rb

@@ -17,6 +17,21 @@ class DummyInput < LogStash::Inputs::Base
   end
 end
 
+class DummyInputGenerator < LogStash::Inputs::Base
+  config_name "dummyinputgenerator"
+  milestone 2
+
+  def register
+  end
+
+  def run(queue)
+    queue << Logstash::Event.new while !stop?
+  end
+
+  def close
+  end
+end
+
 class DummyCodec < LogStash::Codecs::Base
   config_name "dummycodec"
   milestone 2
@@ -47,16 +62,20 @@ class DummyOutput < LogStash::Outputs::Base
 
   def register
   end
-  
+
   def receive(event)
     @events << event
   end
 
   def close
-    @num_closes += 1
+    @num_closes = 1
   end
 end
 
+class DummyOutputMore < DummyOutput
+  config_name "dummyoutputmore"
+end
+
 class DummyFilter < LogStash::Filters::Base
   config_name "dummyfilter"
   milestone 2
@@ -118,10 +137,28 @@ describe LogStash::Pipeline do
         eos
       }
 
+      describe "debug compiled" do
+        let(:logger) { double("pipeline logger").as_null_object }
+
+        before do
+          expect(Cabin::Channel).to receive(:get).with(LogStash).and_return(logger).at_least(:once)
+          allow(logger).to receive(:debug?).and_return(true)
+        end
+
+        it "should not receive a debug message with the compiled code" do
+          expect(logger).not_to receive(:debug).with(/Compiled pipeline/, anything)
+          pipeline = TestPipeline.new(test_config_with_filters)
+        end
+
+        it "should print the compiled code if debug_config is set to true" do
+          expect(logger).to receive(:debug).with(/Compiled pipeline/, anything)
+          pipeline = TestPipeline.new(test_config_with_filters, :debug_config => true)
+        end
+      end
+
       context "when there is no command line -w N set" do
         it "starts one filter thread" do
-          msg = "Defaulting pipeline worker threads to 1 because there are some" +
-                " filters that might not work with multiple worker threads"
+          msg = "Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads"
           pipeline = TestPipeline.new(test_config_with_filters)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:count_was=>worker_thread_count, :filters=>["dummyfilter"]})
@@ -132,8 +169,7 @@ describe LogStash::Pipeline do
 
       context "when there is command line -w N set" do
         it "starts multiple filter thread" do
-          msg = "Warning: Manual override - there are filters that might" +
-                " not work with multiple worker threads"
+          msg = "Warning: Manual override - there are filters that might not work with multiple worker threads"
           pipeline = TestPipeline.new(test_config_with_filters)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:worker_threads=> override_thread_count, :filters=>["dummyfilter"]})
@@ -356,26 +392,36 @@ describe LogStash::Pipeline do
     end
   end
 
-  describe "stalling_threads" do
-    before(:each) do
-      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
+  context "metrics" do
+    config <<-CONFIG
+    input { }
+    filter { }
+    output { }
+    CONFIG
+
+    it "uses a `NullMetric` object if no metric is given" do
+      pipeline = LogStash::Pipeline.new(config)
+      expect(pipeline.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+    end
+  end
+
+  context "Multiples pipelines" do
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinputgenerator").and_return(DummyInputGenerator)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputmore").and_return(DummyOutputMore)
     end
 
-    context "when the pipeline doesn't have filters" do
-      let(:pipeline_with_no_filters) do
-        <<-eos
-        input { dummyinput {} }
-        output { dummyoutput {} }
-        eos
-      end
+    let(:pipeline1) { LogStash::Pipeline.new("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { LogStash::Pipeline.new("input { dummyinputgenerator {} } filter { dummyfilter {} } output { dummyoutputmore {}}") }
 
-      it "doesn't raise an error" do
-        pipeline = TestPipeline.new(pipeline_with_no_filters)
-        pipeline.run
-        expect { pipeline.stalling_threads_info }.to_not raise_error
-      end
+    it "should handle evaluating different config" do
+      expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
     end
   end
 
@@ -389,7 +435,7 @@ describe LogStash::Pipeline do
         }
       }
       filter {
-        multiline { 
+        multiline {
           pattern => "^NeverMatch"
           negate => true
           what => "previous"
@@ -401,7 +447,7 @@ describe LogStash::Pipeline do
       EOS
     end
     let(:output) { DummyOutput.new }
-    
+
     before do
       allow(DummyOutput).to receive(:new).with(any_args).and_return(output)
       allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
@@ -437,8 +483,8 @@ describe LogStash::Pipeline do
 
     it "should handle evaluating different config" do
       # When the functions are compiled from the AST it will generate instance
-      # variables that are unique to the actual config, the intance are pointing
-      # to conditionals/plugins.
+      # variables that are unique to the actual config, the intances are pointing
+      # to conditionals and/or plugins.
       #
       # Before the `defined_singleton_method`, the definition of the method was
       # not unique per class, but the `instance variables` were unique per class.
@@ -453,19 +499,144 @@ describe LogStash::Pipeline do
     end
   end
 
-  context "Pipeline object" do
-    before do
+  context "#started_at" do
+    let(:config) do
+      <<-EOS
+      input {
+        generator {}
+      }
+      EOS
+    end
+
+    subject { described_class.new(config) }
+
+    it "returns nil when the pipeline isnt started" do
+      expect(subject.started_at).to be_nil
+    end
+
+    it "return when the pipeline started working" do
+      t = Thread.new { subject.run }
+      sleep(0.1)
+      expect(subject.started_at).to be < Time.now
+      t.kill rescue nil
+    end
+  end
+
+  context "#uptime" do
+    let(:config) do
+      <<-EOS
+      input {
+        generator {}
+      }
+      EOS
+    end
+    subject { described_class.new(config) }
+
+    context "when the pipeline is not started" do
+      it "returns 0" do
+        expect(subject.uptime).to eq(0)
+      end
+    end
+
+    context "when the pipeline is started" do
+      it "return the duration in milliseconds" do
+        t = Thread.new { subject.run }
+        sleep(0.1)
+        expect(subject.uptime).to be > 0
+        t.kill rescue nil
+      end
+    end
+  end
+
+  context "when collecting metrics in the pipeline" do
+    subject { described_class.new(config, { :metric => metric, :pipeline_id => pipeline_id }) }
+    let(:pipeline_id) { :main }
+    let(:metric) { LogStash::Instrument::Metric.new }
+    let(:number_of_events) { 1000 }
+    let(:multiline_id) { "my-multiline" }
+    let(:multiline_id_other) { "my-multiline_other" }
+    let(:dummy_output_id) { "my-dummyoutput" }
+    let(:generator_id) { "my-generator" }
+    let(:config) do
+      <<-EOS
+      input {
+        generator {
+           count => #{number_of_events}
+           id => "#{generator_id}"
+        }
+      }
+      filter {
+         multiline {
+              id => "#{multiline_id}"
+              pattern => "hello"
+              what => next
+          }
+          multiline {
+               id => "#{multiline_id_other}"
+               pattern => "hello"
+               what => next
+           }
+      }
+      output {
+        dummyoutput {
+          id => "#{dummy_output_id}"
+        }
+      }
+      EOS
+    end
+    let(:dummyoutput) { DummyOutput.new({ "id" => dummy_output_id }) }
+
+    before :each do
+      allow(DummyOutput).to receive(:new).with(any_args).and_return(dummyoutput)
       allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
-      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
-      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+
+      # Reset the metric store
+      LogStash::Instrument::Collector.instance.clear
+
+      Thread.new { subject.run }
+      # make sure we have received all the generated events
+      sleep 1 while dummyoutput.events.size < number_of_events
     end
 
-    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
-    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    after :each do
+      subject.shutdown
+    end
+
+    context "global metric" do
+      let(:collected_metric) { LogStash::Instrument::Collector.instance.snapshot_metric.metric_store.get_with_path("stats/events") }
+
+      it "populates the differents" do
+        expect(collected_metric[:stats][:events][:in].value).to eq(number_of_events)
+        expect(collected_metric[:stats][:events][:filtered].value).to eq(number_of_events)
+        expect(collected_metric[:stats][:events][:out].value).to eq(number_of_events)
+      end
+    end
+
+    context "pipelines" do
+      let(:collected_metric) { LogStash::Instrument::Collector.instance.snapshot_metric.metric_store.get_with_path("stats/pipelines/") }
 
-    it "should not add ivars" do
-       expect(pipeline1.instance_variables).to eq(pipeline2.instance_variables)
+      it "populates the pipelines core metrics" do
+        expect(collected_metric[:stats][:pipelines][:main][:events][:in].value).to eq(number_of_events)
+        expect(collected_metric[:stats][:pipelines][:main][:events][:filtered].value).to eq(number_of_events)
+        expect(collected_metric[:stats][:pipelines][:main][:events][:out].value).to eq(number_of_events)
+      end
+
+      it "populates the filter metrics" do
+        [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
+          [:in, :out].each do |metric_key|
+            plugin_name = "multiline_#{id}".to_sym
+            expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:events][metric_key].value).to eq(number_of_events)
+          end
+        end
+      end
+
+      it "populates the output metrics" do
+        plugin_name = "dummyoutput_#{dummy_output_id}".to_sym
+        expect(collected_metric[:stats][:pipelines][:main][:plugins][:outputs][plugin_name][:events][:out].value).to eq(number_of_events)
+      end
     end
   end
 end