logstash-core 2.1.3-java → 2.2.0-java

data/spec/logstash/outputs/base_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "spec_helper"
+
+# use a dummy NOOP output to test Outputs::Base
+class LogStash::Outputs::NOOP < LogStash::Outputs::Base
+  config_name "noop"
+  milestone 2
+
+  config :dummy_option, :validate => :string
+
+  def register; end
+
+  def receive(event)
+    return output?(event)
+  end
+end
+
+class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
+  LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
+
+  def register
+    workers_not_supported(LEGACY_WORKERS_NOT_SUPPORTED_REASON)
+  end
+end
+
+describe "LogStash::Outputs::Base#new" do
+  it "should instantiate cleanly" do
+    params = { "dummy_option" => "potatoes", "codec" => "json", "workers" => 2 }
+    worker_params = params.dup; worker_params["workers"] = 1
+
+    expect do
+      LogStash::Outputs::NOOP.new(params.dup)
+    end.not_to raise_error
+  end
+
+  it "should move workers_not_supported declarations up to the class level" do
+    LogStash::Outputs::NOOPLegacyNoWorkers.new.register
+    expect(LogStash::Outputs::NOOPLegacyNoWorkers.workers_not_supported?).to eql(true)
+  end
+end

data/spec/logstash/pipeline_reporter_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/pipeline"
+require "logstash/pipeline_reporter"
+
+class DummyOutput < LogStash::Outputs::Base
+  config_name "dummyoutput"
+  milestone 2
+
+  attr_reader :num_closes, :events
+
+  def initialize(params={})
+    super
+    @num_closes = 0
+    @events = []
+  end
+
+  def register
+  end
+
+  def receive(event)
+    @events << event
+  end
+
+  def close
+    @num_closes += 1
+  end
+end
+
+#TODO: Figure out how to add more tests that actually cover inflight events
+#This will require some janky multithreading stuff
+describe LogStash::PipelineReporter do
+  let(:generator_count) { 5 }
+  let(:config) do
+    "input { generator { count => #{generator_count} } } output { dummyoutput {} } "
+  end
+  let(:pipeline) { LogStash::Pipeline.new(config)}
+  let(:reporter) { pipeline.reporter }
+
+  before do
+    allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_call_original
+    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
+
+    @pre_snapshot = reporter.snapshot
+    pipeline.run
+    @post_snapshot = reporter.snapshot
+  end
+
+  describe "events filtered" do
+    it "should start at zero" do
+      expect(@pre_snapshot.events_filtered).to eql(0)
+    end
+
+    it "should end at the number of generated events" do
+      expect(@post_snapshot.events_filtered).to eql(generator_count)
+    end
+  end
+
+  describe "events consumed" do
+    it "should start at zero" do
+      expect(@pre_snapshot.events_consumed).to eql(0)
+    end
+
+    it "should end at the number of generated events" do
+      expect(@post_snapshot.events_consumed).to eql(generator_count)
+    end
+  end
+
+  describe "inflight count" do
+    it "should be zero before running" do
+      expect(@pre_snapshot.inflight_count).to eql(0)
+    end
+
+    it "should be zero after running" do
+      expect(@post_snapshot.inflight_count).to eql(0)
+    end
+  end
+
+  describe "output states" do
+    it "should include the count of received events" do
+      expect(@post_snapshot.output_info.first[:events_received]).to eql(generator_count)
+    end
+  end
+end

data/spec/{core → logstash}/pipeline_spec.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/inputs/generator"
+require "logstash/filters/multiline"
 
 class DummyInput < LogStash::Inputs::Base
   config_name "dummyinput"
@@ -35,17 +37,19 @@ class DummyOutput < LogStash::Outputs::Base
   config_name "dummyoutput"
   milestone 2
 
-  attr_reader :num_closes
+  attr_reader :num_closes, :events
 
   def initialize(params={})
     super
     @num_closes = 0
+    @events = []
   end
 
   def register
   end
-
+  
   def receive(event)
+    @events << event
   end
 
   def close
@@ -80,22 +84,21 @@ class DummySafeFilter < LogStash::Filters::Base
 end
 
 class TestPipeline < LogStash::Pipeline
-  attr_reader :outputs, :filter_threads, :settings, :logger
+  attr_reader :outputs, :settings, :logger
 end
 
 describe LogStash::Pipeline do
-  let(:worker_thread_count)     { 8 }
+  let(:worker_thread_count)     { LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers] }
   let(:safe_thread_count)       { 1 }
   let(:override_thread_count)   { 42 }
 
-  describe "defaulting the filter workers based on thread safety" do
+  describe "defaulting the pipeline workers based on thread safety" do
     before(:each) do
       allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
       allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
       allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummysafefilter").and_return(DummySafeFilter)
-      allow(LogStash::Config::CpuCoreStrategy).to receive(:fifty_percent).and_return(worker_thread_count)
     end
 
     context "when there are some not threadsafe filters" do
@@ -117,13 +120,13 @@ describe LogStash::Pipeline do
 
       context "when there is no command line -w N set" do
         it "starts one filter thread" do
-          msg = "Defaulting filter worker threads to 1 because there are some" +
+          msg = "Defaulting pipeline worker threads to 1 because there are some" +
                 " filters that might not work with multiple worker threads"
           pipeline = TestPipeline.new(test_config_with_filters)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:count_was=>worker_thread_count, :filters=>["dummyfilter"]})
           pipeline.run
-          expect(pipeline.filter_threads.size).to eq(safe_thread_count)
+          expect(pipeline.worker_threads.size).to eq(safe_thread_count)
         end
       end
 
@@ -134,9 +137,9 @@ describe LogStash::Pipeline do
           pipeline = TestPipeline.new(test_config_with_filters)
           expect(pipeline.logger).to receive(:warn).with(msg,
             {:worker_threads=> override_thread_count, :filters=>["dummyfilter"]})
-          pipeline.configure("filter-workers", override_thread_count)
+          pipeline.configure(:pipeline_workers, override_thread_count)
           pipeline.run
-          expect(pipeline.filter_threads.size).to eq(override_thread_count)
+          expect(pipeline.worker_threads.size).to eq(override_thread_count)
         end
       end
     end
@@ -161,7 +164,7 @@ describe LogStash::Pipeline do
       it "starts multiple filter threads" do
         pipeline = TestPipeline.new(test_config_with_filters)
         pipeline.run
-        expect(pipeline.filter_threads.size).to eq(worker_thread_count)
+        expect(pipeline.worker_threads.size).to eq(worker_thread_count)
       end
     end
   end
@@ -206,8 +209,8 @@ describe LogStash::Pipeline do
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
-        expect(pipeline.outputs.first.worker_plugins.size ).to eq(1)
-        expect(pipeline.outputs.first.worker_plugins.first.num_closes ).to eq(1)
+        expect(pipeline.outputs.first.workers.size ).to eq(pipeline.default_output_workers)
+        expect(pipeline.outputs.first.workers.first.num_closes ).to eq(1)
       end
 
       it "should call output close correctly with output workers" do
@@ -215,8 +218,13 @@ describe LogStash::Pipeline do
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
-        expect(pipeline.outputs.first.num_closes).to eq(0)
-        pipeline.outputs.first.worker_plugins.each do |plugin|
+        # We even close the parent output worker, even though it doesn't receive messages
+
+        output_delegator = pipeline.outputs.first
+        output = output_delegator.workers.first
+
+        expect(output.num_closes).to eq(1)
+        output_delegator.workers.each do |plugin|
           expect(plugin.num_closes ).to eq(1)
         end
       end
@@ -268,6 +276,36 @@ describe LogStash::Pipeline do
     end
   end
 
+  describe "max inflight warning" do
+    let(:config) { "input { dummyinput {} } output { dummyoutput {} }" }
+    let(:batch_size) { 1 }
+    let(:pipeline) { LogStash::Pipeline.new(config, :pipeline_batch_size => batch_size, :pipeline_workers => 1) }
+    let(:logger) { pipeline.logger }
+    let(:warning_prefix) { /CAUTION: Recommended inflight events max exceeded!/ }
+
+    before(:each) do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+      allow(logger).to receive(:warn)
+      thread = Thread.new { pipeline.run }
+      pipeline.shutdown
+      thread.join
+    end
+
+    it "should not raise a max inflight warning if the max_inflight count isn't exceeded" do
+      expect(logger).not_to have_received(:warn).with(warning_prefix)
+    end
+
+    context "with a too large inflight count" do
+      let(:batch_size) { LogStash::Pipeline::MAX_INFLIGHT_WARN_THRESHOLD + 1 }
+
+      it "should raise a max inflight warning if the max_inflight count is exceeded" do
+        expect(logger).to have_received(:warn).with(warning_prefix)
+      end
+    end
+  end
+
   context "compiled filter funtions" do
 
     context "new events should propagate down the filters" do
@@ -318,8 +356,82 @@ describe LogStash::Pipeline do
       it "doesn't raise an error" do
         pipeline = TestPipeline.new(pipeline_with_no_filters)
         pipeline.run
-        expect { pipeline.stalling_threads }.to_not raise_error
+        expect { pipeline.stalling_threads_info }.to_not raise_error
       end
     end
   end
+
+  context "Periodic Flush" do
+    let(:number_of_events) { 100 }
+    let(:config) do
+      <<-EOS
+      input {
+        generator {
+          count => #{number_of_events}
+        }
+      }
+      filter {
+        multiline { 
+          pattern => "^NeverMatch"
+          negate => true
+          what => "previous"
+        }
+      }
+      output {
+        dummyoutput {}
+      }
+      EOS
+    end
+    let(:output) { DummyOutput.new }
+    
+    before do
+      allow(DummyOutput).to receive(:new).with(any_args).and_return(output)
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    it "flushes the buffered contents of the filter" do
+      Thread.abort_on_exception = true
+      pipeline = LogStash::Pipeline.new(config, { :flush_interval => 1 })
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+      # give us a bit of time to flush the events
+      wait(5).for do
+        next unless output && output.events && output.events.first
+        output.events.first["message"].split("\n").count
+      end.to eq(number_of_events)
+      pipeline.shutdown
+    end
+  end
+
+  context "Multiple pipelines" do
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(DummyCodec)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(DummyFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    end
+
+    let(:pipeline1) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+    let(:pipeline2) { LogStash::Pipeline.new("input { generator {} } filter { dummyfilter {} } output { dummyoutput {}}") }
+
+    it "should handle evaluating different config" do
+      # When the functions are compiled from the AST it will generate instance
+      # variables that are unique to the actual config, the intance are pointing
+      # to conditionals/plugins.
+      #
+      # Before the `defined_singleton_method`, the definition of the method was
+      # not unique per class, but the `instance variables` were unique per class.
+      #
+      # So the methods were trying to access instance variables that did not exist
+      # in the current instance and was returning an array containing nil values for
+      # the match.
+      expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+    end
+  end
 end

data/spec/{core → logstash}/plugin_spec.rb

@@ -108,7 +108,7 @@ describe LogStash::Plugin do
 
       subject.validate({})
     end
-    
+
 
     it 'logs a warning if the plugin use the milestone option' do
       expect_any_instance_of(Cabin::Channel).to receive(:warn)
@@ -120,4 +120,50 @@ describe LogStash::Plugin do
       end
     end
   end
+
+  describe "subclass initialize" do
+    let(:args) { Hash.new }
+
+    [
+      StromaeCodec = Class.new(LogStash::Codecs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeFilter = Class.new(LogStash::Filters::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeInput = Class.new(LogStash::Inputs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeOutput = Class.new(LogStash::Outputs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end
+    ].each do |klass|
+
+      it "subclass #{klass.name} does not modify params" do
+        klass.new(args)
+        expect(args).to be_empty
+      end
+    end
+
+    context "codec initialization" do
+
+      class LogStash::Codecs::Noop < LogStash::Codecs::Base
+        config_name "noop"
+
+        config :format, :validate => :string
+        def register; end
+      end
+
+      it "should only register once" do
+        args   = { "codec" => LogStash::Codecs::Noop.new("format" => ".") }
+        expect_any_instance_of(LogStash::Codecs::Noop).to receive(:register).once
+        LogStash::Plugin.new(args)
+      end
+
+    end
+  end
 end

data/spec/logstash/runner_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/runner"
+require "stud/task"
+require "stud/trap"
+
+class NullRunner
+  def run(args); end
+end
+
+describe LogStash::Runner do
+
+  context "argument parsing" do
+    it "should run agent" do
+      expect(Stud::Task).to receive(:new).once.and_return(nil)
+      args = ["agent", "-e", ""]
+      expect(subject.run(args)).to eq(nil)
+    end
+
+    it "should run agent help" do
+      expect(subject).to receive(:show_help).once.and_return(nil)
+      args = ["agent", "-h"]
+      expect(subject.run(args).wait).to eq(0)
+    end
+
+    it "should show help with no arguments" do
+      expect($stderr).to receive(:puts).once.and_return("No command given")
+      expect($stderr).to receive(:puts).once
+      args = []
+      expect(subject.run(args).wait).to eq(1)
+    end
+
+    it "should show help for unknown commands" do
+      expect($stderr).to receive(:puts).once.and_return("No such command welp")
+      expect($stderr).to receive(:puts).once
+      args = ["welp"]
+      expect(subject.run(args).wait).to eq(1)
+    end
+  end
+
+  describe "pipeline settings" do
+    let(:pipeline_string) { "input { stdin {} } output { stdout {} }" }
+    let(:base_pipeline_settings) { { :pipeline_id => "base" } }
+    let(:pipeline) { double("pipeline") }
+
+    before(:each) do
+      task = Stud::Task.new { 1 }
+      allow(pipeline).to receive(:run).and_return(task)
+    end
+
+    context "when pipeline workers is not defined by the user" do
+      it "should not pass the value to the pipeline" do
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, base_pipeline_settings).and_return(pipeline)
+        args = ["agent", "-e", pipeline_string]
+        subject.run(args).wait
+      end
+    end
+
+    context "when pipeline workers is defined by the user" do
+      it "should pass the value to the pipeline" do
+        base_pipeline_settings[:pipeline_workers] = 2
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, base_pipeline_settings).and_return(pipeline)
+        args = ["agent", "-w", "2", "-e", pipeline_string]
+        subject.run(args).wait
+      end
+    end
+  end
+end