logstash-core 6.8.14-java → 7.0.0.alpha1-java

data/logstash-core.gemspec

@@ -1,4 +1,10 @@
 # -*- encoding: utf-8 -*-
+
+# NOTE: please use `rake artifact:gems` or `rake artifact:build-logstash-core` to build LS gems
+# You can add a version qualifier (e.g. alpha1) via the VERSION_QUALIFIER env var, e.g.
+# VERSION_QUALIFIER=beta2 RELEASE=1 rake artifact:build-logstash-core
+# `require 'logstash-core/version'` is aware of this env var
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
@@ -28,7 +34,7 @@ Gem::Specification.new do |gem|
   gem.description   = %q{The core components of logstash, the scalable log and event management tool}
   gem.summary       = %q{logstash-core - The core components of logstash}
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
-  gem.license       = "Apache License (2.0)"
+  gem.license       = "Apache-2.0"
 
   gem.files         = Dir.glob(
     %w(versions-gem-copy.yml logstash-core.gemspec gemspec_jars.rb lib/**/*.rb spec/**/*.rb locales/*
@@ -47,7 +53,10 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
   gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
-  gem.add_runtime_dependency "rack", '~> 1.6', '>= 1.6.11'
+
+  # Later versions are ruby 2.0 only. We should remove the rack dep once we support 9k
+  gem.add_runtime_dependency "rack", '1.6.6'
+
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2.16'
   gem.add_runtime_dependency "jruby-openssl", ">= 0.9.20" # >= 0.9.13 Required to support TLSv1.2

data/spec/logstash/acked_queue_concurrent_stress_spec.rb

@@ -14,7 +14,7 @@ describe LogStash::WrappedAckedQueue, :stress_test => true do
     let(:reject_memo_keys) { [:reject_memo_keys, :path, :queue, :writer_threads, :collector, :metric, :reader_threads, :output_strings] }
 
     let(:queue) do
-      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, false, queue_capacity)
+      described_class.new(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, queue_capacity)
     end
 
     let(:writer_threads) do
@@ -70,7 +70,7 @@ describe LogStash::WrappedAckedQueue, :stress_test => true do
       it "writes, reads, closes and reopens" do
         Thread.abort_on_exception = true
 
-        # force lazy initialization to avoid concurency issues within threads
+        # force lazy initialization to avoid concurrency issues within threads
         counts
         queue
 

data/spec/logstash/agent/converge_spec.rb

@@ -49,7 +49,7 @@ describe LogStash::Agent do
 
     context "system pipeline" do
       
-      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { dummyblockinginput { } } output { null {} }", { "pipeline.system" => true }) }
+      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { generator { } } output { null {} }", { "pipeline.system" => true }) }
 
       context "when we have a finite pipeline and a system pipeline running" do
 
@@ -65,40 +65,40 @@ describe LogStash::Agent do
       end
 
       context "when we have an infinite pipeline and a system pipeline running" do
-        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { } } output { null {} }") }
+        let(:infinite_pipeline_config) { mock_pipeline_config(:main, "input { generator { } } output { null {} }") }
 
         let(:source_loader) do
           TestSourceLoader.new(infinite_pipeline_config, system_pipeline_config)
         end
 
         before(:each) do
-          @agent_task = start_agent(subject)
+            @agent_task = start_agent(subject)
         end
 
         after(:each) do
-          @agent_task.stop!
-          @agent_task.wait
-          subject.shutdown
+            @agent_task.stop!
         end
 
         describe "#running_user_defined_pipelines" do
           it "returns the user defined pipelines" do
-            # wait is necessary to accommodate for pipelines startup time
-            wait(60).for {subject.running_user_defined_pipelines.keys}.to eq([:main])
-           end
+            wait_for do
+              subject.with_running_user_defined_pipelines {|pipelines| pipelines.keys }
+            end.to eq([:main])
+          end
         end
 
         describe "#running_user_defined_pipelines?" do
           it "returns true" do
-            # wait is necessary to accommodate for pipelines startup time
-            wait(60).for {subject.running_user_defined_pipelines?}.to be_truthy
+            wait_for do
+              subject.running_user_defined_pipelines?
+            end.to be_truthy
           end
         end
       end
     end
 
     context "when `config.reload.automatic`" do
-      let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
 
       let(:source_loader) do
         TestSourceLoader.new(pipeline_config)
@@ -114,14 +114,14 @@ describe LogStash::Agent do
 
           after(:each) do
             @agent_task.stop!
-            @agent_task.wait
-            subject.shutdown
           end
 
           it "converge only once" do
             wait(60).for { source_loader.fetch_count }.to eq(1)
-            # no need to wait here because have_running_pipeline? does the wait
+
             expect(subject).to have_running_pipeline?(pipeline_config)
+
+            subject.shutdown
           end
         end
 
@@ -135,6 +135,8 @@ describe LogStash::Agent do
 
             expect(source_loader.fetch_count).to eq(1)
             expect(subject.pipelines_count).to eq(0)
+
+            subject.shutdown
           end
         end
       end
@@ -147,25 +149,26 @@ describe LogStash::Agent do
             "config.reload.interval" =>  interval
           )
         end
-
         before(:each) do
           @agent_task = start_agent(subject)
         end
 
         after(:each) do
           @agent_task.stop!
-          @agent_task.wait
-          subject.shutdown
         end
 
         context "and successfully load the config" do
           it "converges periodically the pipelines from the configs source" do
-            # no need to wait here because have_running_pipeline? does the wait
+            sleep(2) # let the interval reload a few times
             expect(subject).to have_running_pipeline?(pipeline_config)
 
             # we rely on a periodic thread to call fetch count, we have seen unreliable run on
             # travis, so lets add a few retries
-            try { expect(source_loader.fetch_count).to be > 1 }
+            try do
+              expect(source_loader.fetch_count).to be > 1
+            end
+
+            subject.shutdown
           end
         end
 
@@ -175,9 +178,12 @@ describe LogStash::Agent do
           end
 
           it "it will keep trying to converge" do
+
             sleep(agent_settings.get("config.reload.interval") / 1_000_000_000.0 * 20) # let the interval reload a few times
             expect(subject.pipelines_count).to eq(0)
             expect(source_loader.fetch_count).to be > 1
+
+            subject.shutdown
           end
         end
       end
@@ -185,8 +191,8 @@ describe LogStash::Agent do
   end
 
   context "when shutting down the agent" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }") }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput { id => 'new' } } output { null {} }") }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator { id => 'new' } } output { null {} }") }
 
     let(:source_loader) do
       TestSourceLoader.new([pipeline_config, new_pipeline_config])
@@ -199,8 +205,8 @@ describe LogStash::Agent do
   end
 
   context "Configuration converge scenario" do
-    let(:pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
-    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { dummyblockinginput {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
 
     before do
       # Set the Agent to an initial state of pipelines
@@ -257,7 +263,7 @@ describe LogStash::Agent do
     end
 
     context "when the source return a modified pipeline" do
-      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { dummyblockinginput { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
+      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { generator { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
 
       let(:source_loader) do
         TestSequenceSourceLoader.new(

data/spec/logstash/agent/metrics_spec.rb

@@ -229,7 +229,7 @@ describe LogStash::Agent do
         # since the pipeline is async, it can actually take some time to have metrics recordings
         # so we try a few times
         try(20) do
-          expect { mhash(:stats, :pipelines, :main, :events) }.not_to raise_error , "Events pipelien stats should exist"
+          expect { mhash(:stats, :pipelines, :main, :events) }.not_to raise_error , "Events pipeline stats should exist"
           expect { mhash(:stats, :pipelines, :main, :plugins) }.not_to raise_error, "Plugins pipeline stats should exist"
         end
 

data/spec/logstash/agent_spec.rb

@@ -8,6 +8,7 @@ require_relative "../support/mocks_classes"
 require "fileutils"
 require_relative "../support/helpers"
 require_relative "../support/matchers"
+require 'timeout'
 
 java_import org.logstash.Timestamp
 
@@ -67,7 +68,7 @@ describe LogStash::Agent do
       let(:agent_args) { { "config.string" => config_string } }
 
       it "should delegate settings to new pipeline" do
-        expect(LogStash::Pipeline).to receive(:new) do |arg1, arg2|
+        expect(LogStash::JavaPipeline).to receive(:new) do |arg1, arg2|
           expect(arg1).to eq(config_string)
           expect(arg2.to_hash).to include(agent_args)
         end
@@ -118,7 +119,7 @@ describe LogStash::Agent do
         context "if state is clean" do
           before :each do
             allow(subject).to receive(:running_user_defined_pipelines?).and_return(true)
-            allow(subject).to receive(:no_pipeline?).and_return(false)
+            allow(subject).to receive(:clean_state?).and_return(false)
           end
 
           it "should not converge state more than once" do
@@ -141,7 +142,7 @@ describe LogStash::Agent do
             it "does not upgrade the new config" do
               t = Thread.new { subject.execute }
               wait(timeout)
-                  .for { subject.running_pipelines? && subject.running_pipelines.values.first.ready? }
+                  .for { subject.running_pipelines? && subject.pipelines.values.first.ready? }
                   .to eq(true)
               expect(subject.converge_state_and_update).not_to be_a_successful_converge
               expect(subject).to have_running_pipeline?(mock_config_pipeline)
@@ -161,7 +162,7 @@ describe LogStash::Agent do
             it "does upgrade the new config" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.running_pipelines_count > 0 && subject.running_pipelines.values.first.ready?
+                sleep(0.1) until subject.pipelines_count > 0 && subject.pipelines.values.first.ready?
               end
 
               expect(subject.converge_state_and_update).to be_a_successful_converge
@@ -185,7 +186,7 @@ describe LogStash::Agent do
             it "does not try to reload the pipeline" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.running_pipelines? && subject.running_pipelines.values.first.running?
+                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
               end
               expect(subject.converge_state_and_update).not_to be_a_successful_converge
               expect(subject).to have_running_pipeline?(mock_config_pipeline)
@@ -205,7 +206,7 @@ describe LogStash::Agent do
             it "tries to reload the pipeline" do
               t = Thread.new { subject.execute }
               Timeout.timeout(timeout) do
-                sleep(0.1) until subject.running_pipelines? && subject.running_pipelines.values.first.running?
+                sleep(0.1) until subject.running_pipelines? && subject.pipelines.values.first.running?
               end
 
               expect(subject.converge_state_and_update).to be_a_successful_converge

data/spec/logstash/compiler/compiler_spec.rb

@@ -252,34 +252,6 @@ describe LogStash::Compiler do
           expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
         end
 
-        describe "a filter plugin with a repeated hash directive with duplicated keys" do
-          let(:source) { "input { } filter { #{plugin_source} } output { } " }
-          let(:plugin_source) do
-            %q[
-              grok {
-                match => { "message" => "foo" }
-                match => { "message" => "bar" }
-                break_on_match => false
-              }
-          ]
-          end
-          subject(:c_plugin) { compiled[:filter] }
-
-          let(:expected_plugin_args) do
-            {
-                "match" => {
-                    "message" => ["foo", "bar"]
-                },
-                "break_on_match" => "false"
-            }
-          end
-
-          it "should merge the values of the duplicate keys into an array" do
-            expect(c_plugin).to ir_eql(j.iPlugin(rand_meta, FILTER, "grok", expected_plugin_args))
-          end
-
-        end
-
         describe "a filter plugin that has nested Hash directives" do
           let(:source) { "input { } filter { #{plugin_source} } output { } " }
           let(:plugin_source) do

data/spec/logstash/config/config_ast_spec.rb

@@ -143,21 +143,6 @@ describe LogStashConfigParser do
 
         expect(config).to be_nil
       end
-
-      it "supports octal literals" do
-        parser = LogStashConfigParser.new
-        config = parser.parse(%q(
-          input {
-            example {
-              foo => 010
-            }
-          }
-        ))
-
-        compiled_number = eval(config.recursive_select(LogStash::Config::AST::Number).first.compile)
-
-        expect(compiled_number).to be == 8
-      end
     end
 
     context "when config.support_escapes" do

data/spec/logstash/config/mixin_spec.rb

@@ -132,8 +132,8 @@ describe LogStash::Config::Mixin do
     context "with an empty list" do
       let(:strings) { [] }
 
-      it "should return an empty list" do
-        expect(subject.strings).to be_empty
+      it "should return nil" do
+        expect(subject.strings).to be_nil
       end
     end
 
@@ -419,7 +419,6 @@ describe LogStash::Config::Mixin do
       end
 
       it "should use the value in the variable" do
-        skip("This test fails on Windows, tracked in https://github.com/elastic/logstash/issues/10454")
         expect(subject.oneString).to(be == "fancy")
         expect(subject.oneBoolean).to(be_truthy)
         expect(subject.oneArray).to(be == [ "first array value", "fancy" ])

data/spec/logstash/converge_result_spec.rb

@@ -90,7 +90,7 @@ describe LogStash::ConvergeResult do
   end
 
   context "when all the actions are executed" do
-    context "all succesfull" do
+    context "all successful" do
       let(:success_action) { LogStash::PipelineAction::Stop.new(:success) }
       let(:success_action_2) { LogStash::PipelineAction::Stop.new(:success_2) }
 

data/spec/logstash/environment_spec.rb

@@ -57,14 +57,14 @@ describe LogStash::Environment do
     context "windows" do
       windows_host_os.each do |host|
         it "#{host} returns true" do
-          allow(LogStash::Environment).to receive(:host_os).and_return(host)
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
           expect(LogStash::Environment.windows?).to be_truthy
         end
       end
 
       linux_host_os.each do |host|
         it "#{host} returns false" do
-          allow(LogStash::Environment).to receive(:host_os).and_return(host)
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
           expect(LogStash::Environment.windows?).to be_falsey
         end
       end
@@ -73,14 +73,14 @@ describe LogStash::Environment do
     context "Linux" do
       windows_host_os.each do |host|
         it "#{host} returns true" do
-          allow(LogStash::Environment).to receive(:host_os).and_return(host)
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
           expect(LogStash::Environment.linux?).to be_falsey
         end
       end
 
       linux_host_os.each do |host|
         it "#{host} returns false" do
-          allow(LogStash::Environment).to receive(:host_os).and_return(host)
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return(host)
           expect(LogStash::Environment.linux?).to be_truthy
         end
       end

data/spec/logstash/event_spec.rb

@@ -141,10 +141,10 @@ describe LogStash::Event do
       expect(e.get("foo")).to eq(BigDecimal.new(1))
     end
 
-    it "should set RubyInteger" do
+    it "should set RubyBignum" do
       e = LogStash::Event.new()
       e.set("[foo]", -9223372036854776000)
-      expect(e.get("foo")).to be_kind_of(Integer)
+      expect(e.get("foo")).to be_kind_of(Bignum)
       expect(e.get("foo")).to eq(-9223372036854776000)
     end
 
@@ -341,14 +341,6 @@ describe LogStash::Event do
   context "method missing exception messages" do
     subject { LogStash::Event.new({"foo" => "bar"}) }
 
-    it "#[] method raises a better exception message" do
-      expect { subject["foo"] }.to raise_error(NoMethodError, /Direct event field references \(i\.e\. event\['field'\]\)/)
-    end
-
-    it "#[]= method raises a better exception message" do
-      expect { subject["foo"] = "baz" }.to raise_error(NoMethodError, /Direct event field references \(i\.e\. event\['field'\] = 'value'\)/)
-    end
-
     it "other missing method raises normal exception message" do
       expect { subject.baz() }.to raise_error(NoMethodError, /undefined method `baz' for/)
     end

data/spec/logstash/filter_delegator_spec.rb

@@ -51,7 +51,7 @@ describe LogStash::FilterDelegator do
     end
 
     it "defines a flush method" do
-      expect(subject.has_flush).to be_truthy
+      expect(subject.respond_to?(:flush)).to be_truthy
     end
 
     context "when the flush return events" do
@@ -128,7 +128,7 @@ describe LogStash::FilterDelegator do
     end
 
     it "doesnt define a flush method" do
-      expect(subject.has_flush).to be_falsey
+      expect(subject.respond_to?(:flush)).to be_falsey
     end
 
     it "increments the in/out of the metric" do
@@ -145,4 +145,14 @@ describe LogStash::FilterDelegator do
     end
   end
 
+  context "delegate methods to the original plugin" do
+    # I am not testing the behavior of these methods
+    # this is done in the plugin tests. I just want to make sure
+    # the proxy delegates the methods.
+    LogStash::FilterDelegator::DELEGATED_METHODS.each do |method|
+      it "delegate method: `#{method}` to the filter" do
+        expect(subject.respond_to?(method))
+      end
+    end
+  end
 end