logstash-core 5.0.2-java → 5.1.1.1-java

data/spec/logstash/pipeline_spec.rb

@@ -104,15 +104,15 @@ describe LogStash::Pipeline do
   describe "event cancellation" do
     # test harness for https://github.com/elastic/logstash/issues/6055
 
-    let(:output) { DummyOutputWithEventsArray.new }
+    let(:output) { LogStash::Outputs::DummyOutputWithEventsArray.new }
 
     before do
       allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
-      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputwitheventsarray").and_return(DummyOutputWithEventsArray)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputwitheventsarray").and_return(LogStash::Outputs::DummyOutputWithEventsArray)
       allow(LogStash::Plugin).to receive(:lookup).with("filter", "drop").and_call_original
       allow(LogStash::Plugin).to receive(:lookup).with("filter", "mutate").and_call_original
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
-      allow(DummyOutputWithEventsArray).to receive(:new).with(any_args).and_return(output)
+      allow(LogStash::Outputs::DummyOutputWithEventsArray).to receive(:new).with(any_args).and_return(output)
     end
 
     let(:config) do
@@ -447,6 +447,13 @@ describe LogStash::Pipeline do
       allow(settings).to receive(:get_value).with("pipeline.id").and_return("main")
       allow(settings).to receive(:get_value).with("metric.collect").and_return(false)
       allow(settings).to receive(:get_value).with("config.debug").and_return(false)
+      allow(settings).to receive(:get).with("queue.type").and_return("memory")
+      allow(settings).to receive(:get).with("queue.page_capacity").and_return(1024 * 1024)
+      allow(settings).to receive(:get).with("queue.max_events").and_return(250)
+      allow(settings).to receive(:get).with("queue.max_bytes").and_return(1024 * 1024 * 1024)
+      allow(settings).to receive(:get).with("queue.checkpoint.acks").and_return(1024)
+      allow(settings).to receive(:get).with("queue.checkpoint.writes").and_return(1024)
+      allow(settings).to receive(:get).with("queue.checkpoint.interval").and_return(1000)
 
       pipeline = LogStash::Pipeline.new(config, settings)
       expect(pipeline.metric).to be_kind_of(LogStash::Instrument::NullMetric)
@@ -509,7 +516,7 @@ describe LogStash::Pipeline do
       pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
-      wait(5).for do
+      wait(3).for do
         # give us a bit of time to flush the events
         output.events.empty?
       end.to be_falsey
@@ -549,10 +556,11 @@ describe LogStash::Pipeline do
   end
 
   context "#started_at" do
+    # use a run limiting count to shutdown the pipeline automatically
     let(:config) do
       <<-EOS
       input {
-        generator {}
+        generator { count => 10 }
       }
       EOS
     end
@@ -564,8 +572,7 @@ describe LogStash::Pipeline do
     end
 
     it "return when the pipeline started working" do
-      t = Thread.new { subject.run }
-      sleep(0.1)
+      subject.run
       expect(subject.started_at).to be < Time.now
       subject.shutdown
     end
@@ -604,7 +611,7 @@ describe LogStash::Pipeline do
 
     let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
     let(:pipeline_id) { "main" }
-    let(:number_of_events) { 1000 }
+    let(:number_of_events) { 420 }
     let(:multiline_id) { "my-multiline" }
     let(:multiline_id_other) { "my-multiline_other" }
     let(:dummy_output_id) { "my-dummyoutput" }
@@ -648,13 +655,10 @@ describe LogStash::Pipeline do
 
       Thread.new { subject.run }
       # make sure we have received all the generated events
-
-      times = 0
-      while dummyoutput.events.size < number_of_events
-        times += 1
-        sleep 0.25
-        raise "Waited too long" if times > 4
-      end
+      wait(3).for do
+        # give us a bit of time to flush the events
+        dummyoutput.events.size < number_of_events
+      end.to be_falsey
     end
 
     after :each do
@@ -685,7 +689,7 @@ describe LogStash::Pipeline do
       it "populates the filter metrics" do
         [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
           [:in, :out].each do |metric_key|
-            plugin_name = "multiline_#{id}".to_sym
+            plugin_name = id.to_sym
             expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:events][metric_key].value).to eq(number_of_events)
           end
         end
@@ -703,7 +707,7 @@ describe LogStash::Pipeline do
 
       it "populates the name of the filter plugin" do
         [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
-          plugin_name = "multiline_#{id}".to_sym
+          plugin_name = id.to_sym
           expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:name].value).to eq(LogStash::Filters::Multiline.config_name)
         end
       end

data/spec/logstash/plugin_spec.rb

@@ -31,20 +31,8 @@ describe LogStash::Plugin do
     class LogStash::Filters::LadyGaga < LogStash::Filters::Base
       config_name "lady_gaga"
     end
-    expect(LogStash::Plugin.lookup("filter", "lady_gaga")).to eq(LogStash::Filters::LadyGaga)
-  end
-
-  describe "plugin signup in the registry" do
-
-    let(:registry) { LogStash::Registry.instance }
 
-    it "should be present in the registry" do
-      class LogStash::Filters::MyPlugin < LogStash::Filters::Base
-        config_name "my_plugin"
-      end
-      path     = "logstash/filters/my_plugin"
-      expect(registry.registered?(path)).to eq(true)
-    end
+    expect(LogStash::Plugin.lookup("filter", "lady_gaga")).to eq(LogStash::Filters::LadyGaga)
   end
 
   describe "#inspect" do
@@ -296,7 +284,7 @@ describe LogStash::Plugin do
           end
 
           it "should use a `NullMetric`" do
-            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NamespacedNullMetric)
           end
         end
 
@@ -308,7 +296,7 @@ describe LogStash::Plugin do
           end
 
           it "should use a `NullMetric`" do
-            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NamespacedNullMetric)
           end
         end
       end
@@ -338,7 +326,7 @@ describe LogStash::Plugin do
           end
 
           it "should use a `NullMetric`" do
-            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NamespacedNullMetric)
           end
         end
       end

data/spec/logstash/plugins/hooks_registry_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/event_dispatcher"
+require "logstash/plugins/hooks_registry"
+
+describe LogStash::Plugins::HooksRegistry do
+  class DummyEmitter
+    attr_reader :dispatcher
+
+    def initialize
+      @dispatcher = LogStash::EventDispatcher.new(self)
+    end
+
+    def do_work
+      dispatcher.fire(:do_work)
+    end
+  end
+
+  class DummyListener
+    def initialize
+      @work = false
+    end
+
+    def do_work(emitter = nil)
+      @work = true
+    end
+
+    def work?
+      @work
+    end
+  end
+
+  subject { described_class.new }
+
+  let(:emitter) { DummyEmitter.new }
+  let(:listener) { DummyListener.new }
+
+  it "allow to register an emitter" do
+    expect { subject.register_emitter(emitter.class, emitter.dispatcher) }.to change { subject.emmitters_count }.by(1)
+  end
+
+  it "allow to remove an emitter" do
+    subject.register_emitter(emitter.class, emitter.dispatcher)
+    expect { subject.remove_emitter(emitter.class)}.to change { subject.emmitters_count }.by(-1)
+  end
+
+  it "allow to register hooks to emitters" do
+    expect { subject.register_hooks(emitter.class, listener) }.to change { subject.hooks_count }.by(1)
+    expect { subject.register_hooks(emitter.class, listener) }.to change { subject.hooks_count(emitter.class) }.by(1)
+  end
+
+  it "link the emitter class to the listener" do
+    subject.register_emitter(emitter.class, emitter.dispatcher)
+    subject.register_hooks(emitter.class, listener)
+
+    expect(listener.work?).to be_falsey
+    emitter.do_work
+
+    expect(listener.work?).to be_truthy
+  end
+end

data/spec/logstash/plugins/registry_spec.rb

@@ -8,15 +8,19 @@ class LogStash::Inputs::Dummy < LogStash::Inputs::Base
   config_name "dummy"
 
   def register; end
-
 end
 
-describe LogStash::Registry do
 
-  let(:registry) { described_class.instance }
+class LogStash::Inputs::NewPlugin < LogStash::Inputs::Base
+  config_name "new_plugin"
 
-  context "when loading installed plugins" do
+  def register; end
+end
+
+describe LogStash::Plugins::Registry do
+  let(:registry) { described_class.new }
 
+  context "when loading installed plugins" do
     let(:plugin) { double("plugin") }
 
     it "should return the expected class" do
@@ -25,18 +29,13 @@ describe LogStash::Registry do
     end
 
     it "should raise an error if can not find the plugin class" do
-      expect(LogStash::Registry::Plugin).to receive(:new).with("input", "elastic").and_return(plugin)
-      expect(plugin).to receive(:path).and_return("logstash/input/elastic").twice
-      expect(plugin).to receive(:installed?).and_return(true)
-      expect { registry.lookup("input", "elastic") }.to raise_error(LoadError)
+      expect { registry.lookup("input", "do-not-exist-elastic") }.to raise_error(LoadError)
     end
 
     it "should load from registry is already load" do
-      registry.lookup("input", "stdin")
-      expect(registry).to receive(:registered?).and_return(true).once
-      registry.lookup("input", "stdin")
-      internal_registry = registry.instance_variable_get("@registry")
-      expect(internal_registry).to include("logstash/inputs/stdin" => LogStash::Inputs::Stdin)
+      expect(registry.exists?(:input, "stdin")).to be_falsey
+      expect { registry.lookup("input", "new_plugin") }.to change { registry.size }.by(1)
+      expect { registry.lookup("input", "new_plugin") }.not_to change { registry.size }
     end
   end
 
@@ -53,5 +52,14 @@ describe LogStash::Registry do
     end
   end
 
-end
+  context "when loading plugin manually configured" do
+    it "should return the plugin" do
+      class SimplePlugin
+      end
 
+      expect { registry.lookup("filter", "simple_plugin") }.to raise_error(LoadError)
+      registry.add(:filter, "simple_plugin", SimplePlugin)
+      expect(registry.lookup("filter", "simple_plugin")).to eq(SimplePlugin)
+    end
+  end
+end

data/spec/logstash/settings/bytes_spec.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/settings"
+
+describe LogStash::Setting::Bytes do
+  let(:multipliers) do
+    {
+      "b" => 1,
+      "kb" => 1 << 10,
+      "mb" => 1 << 20,
+      "gb" => 1 << 30,
+      "tb" => 1 << 40,
+      "pb" => 1 << 50,
+    }
+  end
+
+  let(:number) { Flores::Random.number(0..1000) }
+  let(:unit) { Flores::Random.item(multipliers.keys) }
+  let(:default) { "0b" }
+
+  subject { described_class.new("a byte value", default, false) }
+
+  describe "#set" do
+
+    # Hard-coded test just to make sure at least one known case is working
+    context "when given '10mb'" do
+      it "returns 10485760" do
+        expect(subject.set("10mb")).to be == 10485760
+      end
+    end
+
+    context "when given a string" do
+      context "which is a valid byte unit" do
+        let(:text) { "#{number}#{unit}" }
+
+        before { subject.set(text) }
+
+        it "should coerce it to a Fixnum" do
+          expect(subject.value).to be_a(Fixnum)
+        end
+      end
+
+      context "which is not a valid byte unit" do
+        values = [ "hello world", "1234", "", "-__-" ]
+        values.each do |value|
+          it "should fail" do
+            expect { subject.set(value) }.to raise_error
+          end
+        end
+      end
+    end
+  end
+end

data/spec/logstash/settings/time_value_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/settings"
+
+describe LogStash::Setting::TimeValue do
+  subject { described_class.new("option", "-1") }
+  describe "#set" do
+    it "should coerce the default correctly" do
+      expect(subject.value).to eq(LogStash::Util::TimeValue.new(-1, :nanosecond).to_nanos)
+    end
+
+    context "when a value is given outside of possible_values" do
+      it "should raise an ArgumentError" do
+        expect { subject.set("invalid") }.to raise_error(ArgumentError)
+      end
+    end
+    context "when a value is given as a time value" do
+      it "should set the value" do
+        subject.set("18m")
+        expect(subject.value).to eq(LogStash::Util::TimeValue.new(18, :minute).to_nanos)
+      end
+    end
+
+    context "when a value is given as a nanosecond" do
+      it "should set the value" do
+        subject.set(5)
+        expect(subject.value).to eq(LogStash::Util::TimeValue.new(5, :nanosecond).to_nanos)
+      end
+    end
+  end
+end

data/spec/logstash/settings/writable_directory_spec.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/settings"
+require "tmpdir"
+require "socket" # for UNIXSocket
+
+describe LogStash::Setting::WritableDirectory do
+  let(:mode_rx) { 0555 }
+  # linux is 108, Macos is 104, so use a safe value
+  # Stud::Temporary.pathname, will exceed that size without adding anything
+  let(:parent) { File.join(Dir.tmpdir, Time.now.to_f.to_s) }
+  let(:path) { File.join(parent, "fancy") }
+
+  before { Dir.mkdir(parent) }
+  after { Dir.exist?(path) && Dir.unlink(path) rescue nil }
+  after { Dir.unlink(parent) }
+
+  shared_examples "failure" do
+    before { subject.set(path) }
+    it "should fail" do
+      expect { subject.validate_value }.to raise_error
+    end
+  end
+
+  subject do
+    # Create a new WritableDirectory setting with no default value strict
+    # disabled.
+    described_class.new("fancy.path", "", false)
+  end
+
+  describe "#value" do
+    before { subject.set(path) }
+
+    context "when the directory is missing" do
+
+      context "and the parent is writable" do
+        after { 
+          Dir.unlink(path) 
+        }
+        it "creates the directory" do
+          subject.value # need to invoke `#value` to make it do the work.
+          expect(::File.directory?(path)).to be_truthy
+        end
+      end
+
+      context "and the directory cannot be created" do
+        before { File.chmod(mode_rx, parent) }
+        it "should fail" do
+          expect { subject.value }.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "#set and #validate_value" do
+    context "when the directory exists" do
+      before { Dir.mkdir(path) }
+      after { Dir.unlink(path) }
+
+      context "and is writable" do
+        before { subject.set(path) }
+        # assume this spec already created a directory that's writable... fair? :)
+        it "should return true" do
+          expect(subject.validate_value).to be_truthy
+        end
+      end
+
+      context "but is not writable" do
+        before { File.chmod(0, path) }
+        it_behaves_like "failure"
+      end
+    end
+
+    context "when the path exists" do
+      after { File.unlink(path) }
+
+      context "but is a file" do
+        before { File.new(path, "w").close }
+        it_behaves_like "failure"
+      end
+
+      context "but is a socket" do
+        let(:socket) { UNIXServer.new(path) }
+        before { socket } # realize `socket` value
+        after { socket.close }
+        it_behaves_like "failure"
+      end
+
+      context "but is a symlink" do
+        before { File::symlink("whatever", path) }
+        it_behaves_like "failure"
+      end
+    end
+
+    context "when the directory is missing" do
+      # Create a path with at least one subdirectory we can try to fiddle with permissions
+
+      context "but can be created" do
+        before do
+          # If the path doesn't exist, we want to try creating it, so let's be
+          # extra careful and make sure the path doesn't exist yet.
+          expect(File.directory?(path)).to be_falsey
+          subject.set(path)
+        end
+
+        after do
+          Dir.unlink(path)
+        end
+
+        it "should return true" do
+          expect(subject.validate_value).to be_truthy
+        end
+      end
+
+      context "and cannot be created" do
+        before do
+          # Remove write permission on the parent
+          File.chmod(mode_rx, parent)
+        end
+
+        it_behaves_like "failure"
+      end
+    end
+  end
+end