logstash-core 5.4.0-java → 5.4.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b75f61585cb54f56f39ba3df892c501842fe618b
-  data.tar.gz: 1a2c44621cdc55841d2d6a9f03e869cc4b58af81
+  metadata.gz: 1a5a0bad713dd49bae31f5ce35302a518688e034
+  data.tar.gz: fce1138b7e1a930dae5e232bff1bbe74afc505e0
 SHA512:
-  metadata.gz: 62549f54412120fafee19c383bd22885ea7e53fc091b65291aeda0d2b42b2ee885e36776385ff88a5f8996c326d8607b44f6a8430315e4940e4c92f1c446a7f8
-  data.tar.gz: be4c9ab4a85b0e86c33d74a0013d9cd49f44cb45aeba386b92d3489e330ce4bcb41568fdc0d8efa0dca19907d505a15d5c6e34acbba87592a32e75c2acc6a717
+  metadata.gz: 27455700bac45bcf838b065c0fa716299f6167d1f70b658f31e2a5cedf6c678e4be7dcacdb7aba04a2cfc78c2c92850c46a93f14ac120d48a369940f054c3e18
+  data.tar.gz: 30278142ebf380fc9e1254652e028ed31d24902a65ab2b363614978a7b6193104b4f40467f964af189cc5396779bd7da30de11c13cac897081b012cc6ba0f3d1

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.4.0"
+LOGSTASH_CORE_VERSION = "5.4.1"

data/lib/logstash/api/commands/hot_threads_reporter.rb

@@ -42,7 +42,7 @@ class HotThreadsReport
       _hash["thread.stacktrace"].each do |trace|
         traces << trace
       end
-      thread[:traces] = traces unless traces.empty?
+      thread[:traces] = traces
       hash[:threads] << thread
     end
     { :hot_threads => hash }

data/lib/logstash/runner.rb

@@ -202,7 +202,7 @@ class LogStash::Runner < Clamp::StrictCommand
     java.lang.System.setProperty("ls.log.level", setting("log.level"))
     unless java.lang.System.getProperty("log4j.configurationFile")
       log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
-      LogStash::Logging::Logger::initialize("file://" + log4j_config_location)
+      LogStash::Logging::Logger::initialize("file:///" + log4j_config_location)
     end
     # override log level that may have been introduced from a custom log4j config file
     LogStash::Logging::Logger::configure_logging(setting("log.level"))

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.4.0"
+LOGSTASH_VERSION = "5.4.1"

data/spec/logstash/agent_spec.rb

@@ -15,6 +15,7 @@ describe LogStash::Agent do
   let(:pipeline_args) { {} }
   let(:config_file) { Stud::Temporary.pathname }
   let(:config_file_txt) { "input { generator { count => 100000 } } output { }" }
+  let(:logger) { double("logger") }
 
   subject { LogStash::Agent.new(agent_settings) }
 
@@ -30,6 +31,9 @@ describe LogStash::Agent do
     pipeline_args.each do |key, value|
       pipeline_settings.set(key, value)
     end
+    allow(described_class).to receive(:logger).and_return(logger)
+    [:debug, :info, :error, :warn, :fatal, :trace].each {|level| allow(logger).to receive(level) }
+    [:debug?, :info?, :error?, :warn?, :fatal?, :trace?].each {|level| allow(logger).to receive(level) }
   end
 
   after :each do
@@ -134,55 +138,12 @@ describe LogStash::Agent do
           expect(subject).to_not receive(:reload_state!)
           t = Thread.new { subject.execute }
 
-          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-          # a bad test design or missing class functionality.
-          sleep(0.1)
           Stud.stop!(t)
           t.join
           subject.shutdown
         end
       end
 
-      context "when calling reload_pipeline!" do
-        context "with a config that contains reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
-
-          it "does not upgrade the new config" do
-            t = Thread.new { subject.execute }
-            sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.ready?
-            expect(subject).to_not receive(:upgrade_pipeline)
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            subject.send(:"reload_pipeline!", "main")
-
-            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-            # a bad test design or missing class functionality.
-            sleep(0.1)
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
-          end
-        end
-
-        context "with a config that does not contain reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-
-          it "does upgrade the new config" do
-            t = Thread.new { subject.execute }
-            sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.ready?
-            expect(subject).to receive(:upgrade_pipeline).once.and_call_original
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-            subject.send(:"reload_pipeline!", "main")
-
-            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-            # a bad test design or missing class functionality.
-            sleep(0.1)
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
-          end
-        end
-
-      end
       context "when calling reload_state!" do
         context "with a pipeline with auto reloading turned off" do
           let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
@@ -197,7 +158,6 @@ describe LogStash::Agent do
 
             # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
             # a bad test design or missing class functionality.
-            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -215,9 +175,6 @@ describe LogStash::Agent do
             File.open(config_file, "w") { |f| f.puts second_pipeline_config }
             subject.reload_state!
 
-            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-            # a bad test design or missing class functionality.
-            sleep(0.1)
             Stud.stop!(t)
             t.join
             subject.shutdown
@@ -244,60 +201,25 @@ describe LogStash::Agent do
           allow(subject).to receive(:clean_state?).and_return(false)
           t = Thread.new { subject.execute }
           sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
+
           expect(subject).to receive(:reload_state!).at_least(2).times
 
-          # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-          # a bad test design or missing class functionality.
-          sleep(0.1)
+          sleep 1
+
           Stud.stop!(t)
           t.join
           subject.shutdown
         end
       end
+    end
 
-      context "when calling reload_state!" do
-        context "with a config that contains reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
-
-          it "does not upgrade the new config" do
-            t = Thread.new { subject.execute }
-            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-            expect(subject).to_not receive(:upgrade_pipeline)
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-
-            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-            # a bad test design or missing class functionality.
-            sleep(0.1)
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
-          end
-        end
-
-        context "with a config that does not contain reload incompatible plugins" do
-          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
-
-          it "does upgrade the new config" do
-            t = Thread.new { subject.execute }
-            sleep(0.01) until subject.running_pipelines? && subject.pipelines.values.first.running?
-            expect(subject).to receive(:upgrade_pipeline).once.and_call_original
-            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
-
-            # TODO: refactor this. forcing an arbitrary fixed delay for thread concurrency issues is an indication of
-            # a bad test design or missing class functionality.
-            sleep(0.1)
-            Stud.stop!(t)
-            t.join
-            subject.shutdown
-          end
-        end
-      end
+    context "when calling reload_state!" do
     end
   end
 
   describe "#reload_state!" do
     let(:first_pipeline_config) { "input { } filter { } output { }" }
-    let(:second_pipeline_config) { "input { generator {} } filter { } output { }" }
+    let(:second_pipeline_config) { "input { generator { count => 10000 } } filter { } output { }" }
     let(:pipeline_args) { {
       "config.string" => first_pipeline_config,
       "pipeline.workers" => 4,
@@ -326,6 +248,16 @@ describe LogStash::Agent do
         subject.reload_state!
       end
     end
+
+    context "with a config that contains reload incompatible plugins" do
+      let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+
+      it "does not upgrade the new config" do
+        expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
+        expect(subject).to_not receive(:upgrade_pipeline)
+        subject.reload_state!
+      end
+    end
   end
 
   describe "Environment Variables In Configs" do

data/spec/logstash/instrument/wrapped_write_client_spec.rb

@@ -7,8 +7,8 @@ require_relative "../../support/mocks_classes"
 require "spec_helper"
 
 describe LogStash::Instrument::WrappedWriteClient do
-  let(:write_client) { queue.write_client }
-  let(:read_client) { queue.read_client }
+  let!(:write_client) { queue.write_client }
+  let!(:read_client) { queue.read_client }
   let(:pipeline) { double("pipeline", :pipeline_id => :main) }
   let(:collector)   { LogStash::Instrument::Collector.new }
   let(:metric) { LogStash::Instrument::Metric.new(collector) }
@@ -18,39 +18,54 @@ describe LogStash::Instrument::WrappedWriteClient do
 
   subject { described_class.new(write_client, pipeline, metric, plugin) }
 
+  def threaded_read_client
+    Thread.new do
+      started_at = Time.now
+
+      batch_size = 0
+      loop {
+        if Time.now - started_at > 60
+          raise "Took too much time to read from the queue"
+        end
+        batch_size = read_client.read_batch.size
+
+        break if batch_size > 0
+      }
+      expect(batch_size).to eq(1)
+    end
+  end
 
   shared_examples "queue tests" do
     it "pushes single event to the `WriteClient`" do
-      t = Thread.new do
-        subject.push(event)
+      pusher_thread = Thread.new(subject, event) do |_subject, _event|
+        _subject.push(_event)
       end
-      sleep(0.01) while !t.status
-      expect(read_client.read_batch.size).to eq(1)
-      t.kill rescue nil
+
+      reader_thread = threaded_read_client
+
+      [pusher_thread, reader_thread].collect(&:join)
     end
 
     it "pushes batch to the `WriteClient`" do
       batch = write_client.get_new_batch
       batch << event
 
-      t = Thread.new do
-        subject.push_batch(batch)
+      pusher_thread = Thread.new(subject, batch) do |_subject, _batch|
+        _subject.push_batch(_batch)
       end
 
-      sleep(0.01) while !t.status
-      expect(read_client.read_batch.size).to eq(1)
-      t.kill rescue nil
+      reader_thread = threaded_read_client
+      [pusher_thread, reader_thread].collect(&:join)
     end
 
     context "recorded metrics" do
       before do
-        t = Thread.new do
-          subject.push(event)
+        pusher_thread = Thread.new(subject, event) do |_subject, _event|
+          _subject.push(_event)
         end
-        sleep(0.01) while !t.status
-        sleep(0.250) # make it block for some time, so duration isn't 0
-        read_client.read_batch.size
-        t.kill rescue nil
+
+        reader_thread = threaded_read_client
+        [pusher_thread, reader_thread].collect(&:join)
       end
 
       let(:snapshot_store) { collector.snapshot_metric.metric_store }
@@ -97,7 +112,7 @@ describe LogStash::Instrument::WrappedWriteClient do
   end
 
   context "AckedMemoryQueue" do
-    let(:queue) { LogStash::Util::WrappedAckedQueue.create_memory_based("", 1024, 10, 1024) }
+    let(:queue) { LogStash::Util::WrappedAckedQueue.create_memory_based("", 1024, 10, 4096) }
 
     before do
       read_client.set_events_metric(metric.namespace([:stats, :events]))

data/spec/logstash/pipeline_spec.rb

@@ -80,6 +80,21 @@ class DummySafeFilter < LogStash::Filters::Base
   def close() end
 end
 
+class DummyFlushingFilter < LogStash::Filters::Base
+  config_name "dummyflushingfilter"
+  milestone 2
+
+  def register() end
+  def filter(event) end
+  def periodic_flush
+    true
+  end
+  def flush(options)
+    return [::LogStash::Event.new("message" => "dummy_flush")]
+  end
+  def close() end
+end
+
 class TestPipeline < LogStash::Pipeline
   attr_reader :outputs, :settings
 end
@@ -250,6 +265,7 @@ describe LogStash::Pipeline do
       }
 
       it "starts multiple filter threads" do
+        skip("This test has been failing periodically since November 2016. Tracked as https://github.com/elastic/logstash/issues/6245")
         pipeline = TestPipeline.new(test_config_with_filters)
         pipeline.run
         expect(pipeline.worker_threads.size).to eq(worker_thread_count)
@@ -564,23 +580,16 @@ describe LogStash::Pipeline do
   end
 
   context "Periodic Flush" do
-    let(:number_of_events) { 100 }
     let(:config) do
       <<-EOS
       input {
-        generator {
-          count => #{number_of_events}
-        }
+        dummy_input {}
       }
       filter {
-        multiline {
-          pattern => "^NeverMatch"
-          negate => true
-          what => "previous"
-        }
+        dummy_flushing_filter {}
       }
       output {
-        dummyoutput {}
+        dummy_output {}
       }
       EOS
     end
@@ -588,24 +597,27 @@ describe LogStash::Pipeline do
 
     before do
       allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output)
-      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "dummy_input").and_return(DummyInput)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummy_flushing_filter").and_return(DummyFlushingFilter)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
-      allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
-      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
     end
 
-    it "flushes the buffered contents of the filter" do
+    it "flush periodically" do
       Thread.abort_on_exception = true
+
       pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
       sleep(0.1) until pipeline.ready?
-      wait(3).for do
+      wait(10).for do
         # give us a bit of time to flush the events
         output.events.empty?
       end.to be_falsey
-      event = output.events.pop
-      expect(event.get("message").count("\n")).to eq(99)
+
+      expect(output.events.any? {|e| e.get("message") == "dummy_flush"}).to eq(true)
+
       pipeline.shutdown
+
       t.join
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 5.4.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-28 00:00:00.000000000 Z
+date: 2017-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement