logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -7,8 +7,8 @@ require_relative "../../support/mocks_classes"
 require "spec_helper"
 
 describe LogStash::Instrument::WrappedWriteClient do
-  let!(:write_client) { queue.write_client }
-  let!(:read_client) { queue.read_client }
+  let(:write_client) { queue.write_client }
+  let(:read_client) { queue.read_client }
   let(:pipeline) { double("pipeline", :pipeline_id => :main) }
   let(:collector)   { LogStash::Instrument::Collector.new }
   let(:metric) { LogStash::Instrument::Metric.new(collector) }
@@ -18,54 +18,39 @@ describe LogStash::Instrument::WrappedWriteClient do
 
   subject { described_class.new(write_client, pipeline, metric, plugin) }
 
-  def threaded_read_client
-    Thread.new do
-      started_at = Time.now
-
-      batch_size = 0
-      loop {
-        if Time.now - started_at > 60
-          raise "Took too much time to read from the queue"
-        end
-        batch_size = read_client.read_batch.size
-
-        break if batch_size > 0
-      }
-      expect(batch_size).to eq(1)
-    end
-  end
 
   shared_examples "queue tests" do
     it "pushes single event to the `WriteClient`" do
-      pusher_thread = Thread.new(subject, event) do |_subject, _event|
-        _subject.push(_event)
+      t = Thread.new do
+        subject.push(event)
       end
-
-      reader_thread = threaded_read_client
-
-      [pusher_thread, reader_thread].collect(&:join)
+      sleep(0.01) while !t.status
+      expect(read_client.read_batch.size).to eq(1)
+      t.kill rescue nil
     end
 
     it "pushes batch to the `WriteClient`" do
       batch = write_client.get_new_batch
       batch << event
 
-      pusher_thread = Thread.new(subject, batch) do |_subject, _batch|
-        _subject.push_batch(_batch)
+      t = Thread.new do
+        subject.push_batch(batch)
       end
 
-      reader_thread = threaded_read_client
-      [pusher_thread, reader_thread].collect(&:join)
+      sleep(0.01) while !t.status
+      expect(read_client.read_batch.size).to eq(1)
+      t.kill rescue nil
     end
 
     context "recorded metrics" do
       before do
-        pusher_thread = Thread.new(subject, event) do |_subject, _event|
-          _subject.push(_event)
+        t = Thread.new do
+          subject.push(event)
         end
-
-        reader_thread = threaded_read_client
-        [pusher_thread, reader_thread].collect(&:join)
+        sleep(0.01) while !t.status
+        sleep(0.250) # make it block for some time, so duration isn't 0
+        read_client.read_batch.size
+        t.kill rescue nil
       end
 
       let(:snapshot_store) { collector.snapshot_metric.metric_store }
@@ -110,4 +95,19 @@ describe LogStash::Instrument::WrappedWriteClient do
 
     include_examples "queue tests"
   end
+
+  context "AckedMemoryQueue" do
+    let(:queue) { LogStash::Util::WrappedAckedQueue.create_memory_based("", 1024, 10, 1024) }
+
+    before do
+      read_client.set_events_metric(metric.namespace([:stats, :events]))
+      read_client.set_pipeline_metric(metric.namespace([:stats, :pipelines, :main, :events]))
+    end
+
+    after do
+      queue.close
+    end
+
+    include_examples "queue tests"
+  end
 end

data/spec/logstash/legacy_ruby_event_spec.rb

@@ -399,17 +399,17 @@ describe LogStash::Event do
       end
 
       it "should assign current time when no timestamp" do
-        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(2).of (Time.now.to_i)
+        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of (Time.now.to_i)
       end
 
       it "should tag for invalid value" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
 
         event = LogStash::Event.new("@timestamp" => 666)
-        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq(666)
       end
@@ -421,7 +421,7 @@ describe LogStash::Event do
 
       it "should tag for invalid string format" do
         event = LogStash::Event.new("@timestamp" => "foo")
-        expect(event.timestamp.to_i).to be_within(2).of Time.now.to_i
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
         expect(event.get("tags")).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
         expect(event.get(LogStash::Event::TIMESTAMP_FAILURE_FIELD)).to eq("foo")
       end
@@ -590,6 +590,15 @@ describe LogStash::Event do
   describe "#to_s" do
     let(:timestamp) { LogStash::Timestamp.new }
     let(:event1) { LogStash::Event.new({ "@timestamp" => timestamp, "host" => "foo", "message" => "bar"}) }
+    let(:event2) { LogStash::Event.new({ "host" => "bar", "message" => "foo"}) }
+
+    it "should cache only one template" do
+      LogStash::StringInterpolation.clear_cache
+      expect {
+        event1.to_s
+        event2.to_s
+      }.to change { LogStash::StringInterpolation.cache_size }.by(1)
+    end
 
     it "return the string containing the timestamp, the host and the message" do
       expect(event1.to_s).to eq("#{timestamp.to_iso8601} #{event1.get("host")} #{event1.get("message")}")

data/spec/logstash/output_delegator_spec.rb

@@ -2,26 +2,18 @@
 require "logstash/output_delegator"
 require "logstash/execution_context"
 require "spec_helper"
+require "support/shared_contexts"
 
 describe LogStash::OutputDelegator do
-
-  class MockGauge
-    def increment(_)
-    end
-  end
-
   let(:logger) { double("logger") }
   let(:events) { 7.times.map { LogStash::Event.new }}
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
   let(:collector) { [] }
-  let(:counter_in) { MockGauge.new }
-  let(:counter_out) { MockGauge.new }
-  let(:counter_time) { MockGauge.new }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
-  let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "foo", "output",
-                                                                   LogStash::Util::DummyDeadLetterQueueWriter.new) }
+  
+  include_context "execution_context"
 
-  subject { described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
+  subject { described_class.new(logger, out_klass, metric, execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
 
   context "with a plain output plugin" do
     let(:out_klass) { double("output klass") }
@@ -31,9 +23,6 @@ describe LogStash::OutputDelegator do
     before(:each) do
       # use the same metric instance
       allow(metric).to receive(:namespace).with(any_args).and_return(metric)
-      allow(metric).to receive(:counter).with(:in).and_return(counter_in)
-      allow(metric).to receive(:counter).with(:out).and_return(counter_out)
-      allow(metric).to receive(:counter).with(:duration_in_millis).and_return(counter_time)
 
       allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
       allow(out_klass).to receive(:name).and_return("example")
@@ -42,7 +31,7 @@ describe LogStash::OutputDelegator do
       allow(out_inst).to receive(:register)
       allow(out_inst).to receive(:multi_receive)
       allow(out_inst).to receive(:metric=).with(any_args)
-      allow(out_inst).to receive(:execution_context=).with(default_execution_context)
+      allow(out_inst).to receive(:execution_context=).with(execution_context)
       allow(out_inst).to receive(:id).and_return("a-simple-plugin")
 
       allow(subject.metric_events).to receive(:increment).with(any_args)
@@ -55,7 +44,7 @@ describe LogStash::OutputDelegator do
 
     it "should push the name of the plugin to the metric" do
       expect(metric).to receive(:gauge).with(:name, out_klass.config_name)
-      described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
+      described_class.new(logger, out_klass, metric, execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args)
     end
 
     context "after having received a batch of events" do
@@ -69,13 +58,15 @@ describe LogStash::OutputDelegator do
       end
 
       it "should increment the number of events received" do
-        expect(counter_in).to receive(:increment).with(events.length)
-        expect(counter_out).to receive(:increment).with(events.length)
+        expect(subject.metric_events).to receive(:increment).with(:in, events.length)
+        expect(subject.metric_events).to receive(:increment).with(:out, events.length)
         subject.multi_receive(events)
       end
 
       it "should record the `duration_in_millis`" do
-        expect(counter_time).to receive(:increment).with(Integer)
+        clock = spy("clock")
+        expect(subject.metric_events).to receive(:time).with(:duration_in_millis).and_return(clock)
+        expect(clock).to receive(:stop)
         subject.multi_receive(events)
       end
     end

data/spec/logstash/outputs/base_spec.rb

@@ -2,6 +2,7 @@
 require "spec_helper"
 require "logstash/outputs/base"
 require "logstash/execution_context"
+require "support/shared_contexts"
 
 # use a dummy NOOP output to test Outputs::Base
 class LogStash::Outputs::NOOPSingle < LogStash::Outputs::Base
@@ -80,23 +81,24 @@ describe "LogStash::Outputs::Base#new" do
   end
 
   context "execution context" do
-    let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "id", "output", LogStash::Util::DummyDeadLetterQueueWriter.new) }
+    include_context "execution_context"
+    
     let(:klass) { LogStash::Outputs::NOOPSingle }
 
     subject(:instance) { klass.new(params.dup) }
 
     it "allow to set the context" do
       expect(instance.execution_context).to be_nil
-      instance.execution_context = default_execution_context
+      instance.execution_context = execution_context
 
-      expect(instance.execution_context).to eq(default_execution_context)
+      expect(instance.execution_context).to eq(execution_context)
     end
 
     it "propagate the context to the codec" do
       expect(instance.codec.execution_context).to be_nil
-      instance.execution_context = default_execution_context
+      instance.execution_context = execution_context
 
-      expect(instance.codec.execution_context).to eq(default_execution_context)
+      expect(instance.codec.execution_context).to eq(execution_context)
     end
   end
 

data/spec/logstash/pipeline_action/create_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require "spec_helper"
+require_relative "../../support/helpers"
+require_relative "../../support/matchers"
+require "logstash/pipeline_action/create"
+require "logstash/instrument/null_metric"
+require "logstash/inputs/generator"
+
+describe LogStash::PipelineAction::Create do
+  let(:metric) { LogStash::Instrument::NullMetric.new(LogStash::Instrument::Collector.new) }
+  let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { null {} }") }
+  let(:pipelines) {  Hash.new }
+  let(:agent) { double("agent") }
+
+  before do
+    clear_data_dir
+  end
+
+  subject { described_class.new(pipeline_config, metric) }
+
+  after do
+    pipelines.each { |_, pipeline| pipeline.shutdown }
+  end
+
+  it "returns the pipeline_id" do
+    expect(subject.pipeline_id).to eq(:main)
+  end
+
+
+  context "when we have really short lived pipeline" do
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 1 } } output { null {} }") }
+
+    it "returns a successful execution status" do
+      expect(subject.execute(agent, pipelines)).to be_truthy
+    end
+  end
+
+  context "when the pipeline successfully start" do
+    it "adds the pipeline to the current pipelines" do
+      expect { subject.execute(agent, pipelines) }.to change(pipelines, :size).by(1)
+    end
+
+    it "starts the pipeline" do
+      subject.execute(agent, pipelines)
+      expect(pipelines[:main].running?).to be_truthy
+    end
+
+    it "returns a successful execution status" do
+      expect(subject.execute(agent, pipelines)).to be_truthy
+    end
+  end
+
+  context  "when the pipeline doesn't start" do
+    context "with a syntax error" do
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { stdout ") } # bad syntax
+
+      it "raises the exception upstream" do
+        expect { subject.execute(agent, pipelines) }.to raise_error
+      end
+    end
+
+    context "with an error raised during `#register`" do
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } filter { ruby { init => '1/0' code => '1+2' } } output { null {} }") }
+
+      it "returns false" do
+        expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
+      end
+    end
+  end
+
+  context "when sorting create action" do
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { id => '123' } } output { null {} }") }
+    let(:system_pipeline_config) { mock_pipeline_config(:main_2, "input { generator { id => '123' } } output { null {} }", { "pipeline.system" => true }) }
+
+    it "should give higher priority to system pipeline" do
+      action_user_pipeline = described_class.new(pipeline_config, metric)
+      action_system_pipeline = described_class.new(system_pipeline_config, metric)
+
+      sorted = [action_user_pipeline, action_system_pipeline].sort
+      expect(sorted).to eq([action_system_pipeline, action_user_pipeline])
+    end
+  end
+end

data/spec/logstash/pipeline_action/reload_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require "spec_helper"
+require_relative "../../support/helpers"
+require_relative "../../support/matchers"
+require "logstash/pipeline_action/reload"
+require "logstash/instrument/null_metric"
+
+describe LogStash::PipelineAction::Reload do
+  let(:metric) { LogStash::Instrument::NullMetric.new(LogStash::Instrument::Collector.new) }
+  let(:pipeline_id) { :main }
+  let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => true}) }
+  let(:pipeline_config) { "input { generator {} } output { null {} }" }
+  let(:pipeline) { LogStash::Pipeline.new(pipeline_config, mock_settings("pipeline.reloadable" => true)) }
+  let(:pipelines) { { pipeline_id => pipeline } }
+  let(:agent) { double("agent") }
+
+  subject { described_class.new(new_pipeline_config, metric) }
+
+  before do
+    clear_data_dir
+    pipeline.start
+  end
+
+  after do
+    pipelines.each { |_, pipeline| pipeline.shutdown }
+  end
+
+  it "returns the pipeline_id" do
+    expect(subject.pipeline_id).to eq(pipeline_id)
+  end
+
+  context "when existing pipeline and new pipeline are both reloadable" do
+    it "stop the previous pipeline" do
+      expect { subject.execute(agent, pipelines) }.to change(pipeline, :running?).from(true).to(false)
+    end
+
+    it "start the new pipeline" do
+      subject.execute(agent, pipelines)
+      expect(pipelines[pipeline_id].running?).to be_truthy
+    end
+
+    it "run the new pipeline code" do
+      subject.execute(agent, pipelines)
+      expect(pipelines[pipeline_id].config_hash).to eq(new_pipeline_config.config_hash)
+    end
+  end
+
+  context "when the existing pipeline is not reloadable" do
+    before do
+      allow(pipeline).to receive(:reloadable?).and_return(false)
+    end
+
+    it "cannot successfully execute the action" do
+      expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
+    end
+  end
+
+  context "when the new pipeline is not reloadable" do
+    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input { generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
+
+    it "cannot successfully execute the action" do
+      expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
+    end
+  end
+
+  context "when the new pipeline has syntax errors" do
+    let(:new_pipeline_config) { mock_pipeline_config(pipeline_id, "input generator { id => 'new' } } output { null {} }", { "pipeline.reloadable" => false}) }
+
+    it "cannot successfully execute the action" do
+      expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
+    end
+  end
+
+  context "when there is an error in the register" do
+    before do
+      allow_any_instance_of(LogStash::Inputs::Generator).to receive(:register).and_raise("Bad value")
+    end
+
+    it "cannot successfully execute the action" do
+      expect(subject.execute(agent, pipelines)).not_to be_a_successful_action
+    end
+  end
+end

data/spec/logstash/pipeline_action/stop_spec.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require "spec_helper"
+require_relative "../../support/helpers"
+require "logstash/pipeline_action/stop"
+require "logstash/pipeline"
+require "logstash/instrument/null_metric"
+
+describe LogStash::PipelineAction::Stop do
+  let(:pipeline_config) { "input { generator {} } output { null {} }" }
+  let(:pipeline_id) { :main }
+  let(:pipeline) { LogStash::Pipeline.new(pipeline_config) }
+  let(:pipelines) { { :main => pipeline } }
+  let(:agent) { double("agent") }
+
+  subject { described_class.new(pipeline_id) }
+
+  before do
+    clear_data_dir
+    pipeline.start
+  end
+
+  after do
+    pipeline.shutdown
+  end
+
+  it "returns the pipeline_id" do
+    expect(subject.pipeline_id).to eq(:main)
+  end
+
+  it "shutdown the running pipeline" do
+    expect { subject.execute(agent, pipelines) }.to change(pipeline, :running?).from(true).to(false)
+  end
+
+  it "removes the pipeline from the running pipelines" do
+    expect { subject.execute(agent, pipelines) }.to change { pipelines.include?(pipeline_id) }.from(true).to(false)
+  end
+end