logstash-core 6.0.1-java → 6.1.0-java

data/spec/logstash/json_spec.rb

@@ -33,60 +33,42 @@ describe "LogStash::Json" do
     ]
   }
 
-  if LogStash::Environment.jruby?
-
-    ### JRuby specific
-    # Former expectation in this code were removed because of https://github.com/rspec/rspec-mocks/issues/964
-    # as soon as is fix we can re introduce them if desired, however for now the completeness of the test
-    # is also not affected as the conversion would not work if the expectation where not meet.
-    ###
-    context "jruby deserialize" do
-      it "should respond to load and deserialize object" do
-        expect(LogStash::Json.load(json_hash)).to eql(hash)
-      end
+  # Former expectation in this code were removed because of https://github.com/rspec/rspec-mocks/issues/964
+  # as soon as is fix we can re introduce them if desired, however for now the completeness of the test
+  # is also not affected as the conversion would not work if the expectation where not meet.
+  ###
+  context "jruby deserialize" do
+    it "should respond to load and deserialize object" do
+      expect(LogStash::Json.load(json_hash)).to eql(hash)
     end
+  end
 
-    context "jruby serialize" do
-      it "should respond to dump and serialize object" do
-        expect(LogStash::Json.dump(string)).to eql(json_string)
-      end
-
-      it "should call JrJackson::Raw.generate for Hash" do
-        expect(LogStash::Json.dump(hash)).to eql(json_hash)
-      end
-
-      it "should call JrJackson::Raw.generate for Array" do
-        expect(LogStash::Json.dump(array)).to eql(json_array)
-      end
-
-      context "pretty print" do
-
-        let(:hash) { { "foo" => "bar", :zoo => 2 } }
+  context "jruby serialize" do
+    it "should respond to dump and serialize object" do
+      expect(LogStash::Json.dump(string)).to eql(json_string)
+    end
 
-        it "should serialize with pretty print" do
-          pprint_json = LogStash::Json.dump(hash, :pretty => true)
-          expect(pprint_json).to include("\n")
-        end
+    it "should call JrJackson::Raw.generate for Hash" do
+      expect(LogStash::Json.dump(hash)).to eql(json_hash)
+    end
 
-        it "should by default do no pretty print" do
-          pprint_json = LogStash::Json.dump(hash)
-          expect(pprint_json).not_to include("\n")
-        end
-      end
+    it "should call JrJackson::Raw.generate for Array" do
+      expect(LogStash::Json.dump(array)).to eql(json_array)
     end
 
-  else
+    context "pretty print" do
 
-    ### MRI specific
+      let(:hash) { { "foo" => "bar", :zoo => 2 } }
 
-    it "should respond to load and deserialize object on mri" do
-      expect(Oj).to receive(:load).with(json).and_call_original
-      expect(LogStash::Json.load(json)).to eql(hash)
-    end
+      it "should serialize with pretty print" do
+        pprint_json = LogStash::Json.dump(hash, :pretty => true)
+        expect(pprint_json).to include("\n")
+      end
 
-    it "should respond to dump and serialize object on mri" do
-      expect(Oj).to receive(:dump).with(hash, anything).and_call_original
-      expect(LogStash::Json.dump(hash)).to eql(json)
+      it "should by default do no pretty print" do
+        pprint_json = LogStash::Json.dump(hash)
+        expect(pprint_json).not_to include("\n")
+      end
     end
   end
 

data/spec/logstash/plugins/registry_spec.rb

@@ -61,6 +61,13 @@ describe LogStash::Plugins::Registry do
       expect(registry.lookup("filter", "simple_plugin")).to eq(simple_plugin)
     end
 
+    it "should be possible to remove the plugin" do
+      registry.add(:filter, "simple_plugin", simple_plugin)
+      expect(registry.lookup("filter", "simple_plugin")).to eq(simple_plugin)
+      registry.remove(:filter, "simple_plugin")
+      expect { registry.lookup("filter", "simple_plugin") }.to raise_error(LoadError)
+    end
+
     it "doesn't add multiple time the same plugin" do
       plugin1 = Class.new
       plugin2 = Class.new

data/spec/logstash/queue_factory_spec.rb

@@ -15,7 +15,9 @@ describe LogStash::QueueFactory do
       LogStash::Setting::Numeric.new("queue.checkpoint.acks", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.writes", 1024),
       LogStash::Setting::Numeric.new("queue.checkpoint.interval", 1000),
-      LogStash::Setting::String.new("pipeline.id", pipeline_id)
+      LogStash::Setting::String.new("pipeline.id", pipeline_id),
+      LogStash::Setting::PositiveInteger.new("pipeline.batch.size", 125),
+      LogStash::Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum)
     ]
   end
 
@@ -72,9 +74,10 @@ describe LogStash::QueueFactory do
   context "when `queue.type` is `memory`" do
     before do
       settings.set("queue.type", "memory")
+      settings.set("pipeline.batch.size", 1024)
     end
 
-    it "returns a `WrappedAckedQueue`" do
+    it "returns a `WrappedSynchronousQueue`" do
       queue =  subject.create(settings)
       expect(queue).to be_kind_of(LogStash::Util::WrappedSynchronousQueue)
       queue.close

data/spec/logstash/settings_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "spec_helper"
-require "logstash/util/environment_variables"
+require "logstash/util/substitution_variables"
 require "logstash/settings"
 require "fileutils"
 

data/spec/logstash/util/java_version_spec.rb

@@ -6,9 +6,7 @@ describe "LogStash::Util::JavaVersion" do
   subject(:mod) { LogStash::Util::JavaVersion }
 
   it "should get the current java version if we're on Java" do
-    if LogStash::Environment.jruby?
-      expect(LogStash::Util::JavaVersion.version).to be_a(String)
-    end
+    expect(LogStash::Util::JavaVersion.version).to be_a(String)
   end
 
   it "should mark a bad beta version as bad" do

data/spec/logstash/util/wrapped_synchronous_queue_spec.rb

@@ -5,6 +5,8 @@ require "logstash/instrument/collector"
 
 describe LogStash::Util::WrappedSynchronousQueue do
 
+  subject {LogStash::Util::WrappedSynchronousQueue.new(5)}
+
   describe "queue clients" do
     context "when requesting a write client" do
       it "returns a client" do
@@ -18,15 +20,9 @@ describe LogStash::Util::WrappedSynchronousQueue do
       end
     end
 
-    class DummyQueue < Array
-      def take() shift(); end
-      def poll(*) shift(); end
-    end
-
     describe "WriteClient | ReadClient" do
-      let(:queue) { DummyQueue.new }
-      let(:write_client) { LogStash::Util::WrappedSynchronousQueue::WriteClient.new(queue)}
-      let(:read_client)  { LogStash::Util::WrappedSynchronousQueue::ReadClient.new(queue)}
+      let(:write_client) { subject.write_client }
+      let(:read_client)  { subject.read_client }
 
       context "when reading from the queue" do
         let(:collector) { LogStash::Instrument::Collector.new }
@@ -64,16 +60,16 @@ describe LogStash::Util::WrappedSynchronousQueue do
 
         context "when we have item in the queue" do
           it "records the `duration_in_millis`" do
-            batch = write_client.get_new_batch
-            5.times {|i| batch.push("value-#{i}")}
+            batch = []
+            5.times {|i| batch.push(LogStash::Event.new({"message" => "value-#{i}"}))}
             write_client.push_batch(batch)
 
             read_batch = read_client.read_batch
             sleep(0.1) # simulate some work for the `duration_in_millis`
             # TODO: this interaction should be cleaned in an upcoming PR,
             # This is what the current pipeline does.
-            read_client.add_filtered_metrics(read_batch)
-            read_client.add_output_metrics(read_batch)
+            read_client.add_filtered_metrics(read_batch.filtered_size)
+            read_client.add_output_metrics(read_batch.filtered_size)
             read_client.close_batch(read_batch)
             store = collector.snapshot_metric.metric_store
 
@@ -94,26 +90,33 @@ describe LogStash::Util::WrappedSynchronousQueue do
         end
 
         it "appends batches to the queue" do
-          batch = write_client.get_new_batch
-          5.times {|i| batch.push(LogStash::Event.new({"message" => "value-#{i}"}))}
+          batch = []
+          messages = []
+          5.times do |i|
+            message = "value-#{i}"
+            batch.push(LogStash::Event.new({"message" => message}))
+            messages << message
+          end
           write_client.push_batch(batch)
           read_batch = read_client.read_batch
           expect(read_batch.size).to eq(5)
-          i = 0
-          read_batch.each do |data|
-            expect(data.get("message")).to eq("value-#{i}")
+          read_batch.to_a.each do |data|
+            message = data.get("message")
+            expect(messages).to include(message)
+            messages.delete(message)
             # read_batch.cancel("value-#{i}") if i > 2     # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
-            data.cancel if i > 2
-            read_batch.merge(LogStash::Event.new({"message" => "generated-#{i}"})) if i > 2
-            i += 1
+            if message.match /value-[3-4]/
+              data.cancel
+              read_batch.merge(LogStash::Event.new({ "message" => message.gsub(/value/, 'generated') }))
+            end
           end
           # expect(read_batch.cancelled_size).to eq(2) # disabled for https://github.com/elastic/logstash/issues/6055
-          i = 0
+          received = []
           read_batch.each do |data|
-            expect(data.get("message")).to eq("value-#{i}") if i < 3
-            expect(data.get("message")).to eq("generated-#{i}") if i > 2
-            i += 1
+            received << data.get("message")
           end
+          (0..2).each {|i| expect(received).to include("value-#{i}")}
+          (3..4).each {|i| expect(received).to include("generated-#{i}")}
         end
       end
     end

data/spec/logstash/webserver_spec.rb

@@ -13,16 +13,12 @@ def block_ports(range)
   range.each do |port|
     begin
       server = TCPServer.new("localhost", port)
-      Thread.new do
-        client = server.accept rescue nil
-      end
       servers << server
     rescue => e
       # The port can already be taken
     end
   end
 
-  sleep(1)
   servers
 end
 
@@ -78,7 +74,8 @@ describe LogStash::WebServer do
       sleep(1)
 
       # We cannot use stop here, since the code is stuck in an infinite loop
-      t.kill rescue nil
+      t.kill
+      t.join
 
       silence_warnings do
         STDERR = backup_stderr
@@ -107,7 +104,7 @@ describe LogStash::WebServer do
         expect(subject.address).to eq("localhost:10006")
 
         subject.stop
-        t.kill rescue nil
+        t.join
       end
     end
 

data/spec/support/helpers.rb

@@ -54,6 +54,11 @@ def mock_pipeline_from_string(config_string, settings = LogStash::SETTINGS, metr
   LogStash::Pipeline.new(pipeline_config, metric)
 end
 
+def mock_java_pipeline_from_string(config_string, settings = LogStash::SETTINGS, metric = nil)
+  pipeline_config = mock_pipeline_config(settings.get("pipeline.id"), config_string, settings)
+  LogStash::JavaPipeline.new(pipeline_config, metric)
+end
+
 def mock_pipeline_config(pipeline_id, config_string = nil, settings = {})
   config_string = "input { stdin { id => '#{pipeline_id}' }}" if config_string.nil?
 

data/spec/support/pipeline/pipeline_helpers.rb

@@ -0,0 +1,97 @@
+require "logstash/agent"
+require "logstash/java_pipeline"
+require "logstash/event"
+require "stud/try"
+require "rspec/expectations"
+require "thread"
+
+java_import org.logstash.common.SourceWithMetadata
+
+module PipelineHelpers
+
+  class SpecSamplerInput < LogStash::Inputs::Base
+    config_name "spec_sampler_input"
+
+    def register
+    end
+
+    def run(queue)
+      unless @@event.nil?
+        queue.push_batch(@@event)
+        @@event = nil
+      end
+    end
+
+    def close
+      @@event = nil
+    end
+
+    def self.set_event(event)
+      @@event = event
+    end
+  end
+
+  class SpecSamplerOutput < LogStash::Outputs::Base
+    config_name "spec_sampler_output"
+
+    def register
+      @@seen = []
+    end
+
+    def multi_receive(events)
+      @@seen += events
+    end
+
+    def self.seen
+      @@seen
+    end
+  end
+
+  def sample_one(sample_event, &block)
+    name = sample_event.is_a?(String) ? sample_event : LogStash::Json.dump(sample_event)
+    name = name[0..50] + "..." if name.length > 50
+
+    before do
+      LogStash::PLUGIN_REGISTRY.add(:input, "spec_sampler_input", SpecSamplerInput)
+      LogStash::PLUGIN_REGISTRY.add(:output, "spec_sampler_output", SpecSamplerOutput)
+    end
+
+    describe "\"#{name}\"" do
+      let(:pipeline) do
+        settings = ::LogStash::SETTINGS.clone
+        settings.set_value("queue.drain", true)
+        settings.set_value("pipeline.workers", 1)
+        LogStash::JavaPipeline.new(
+          LogStash::Config::PipelineConfig.new(
+            LogStash::Config::Source::Local, :main,
+            SourceWithMetadata.new(
+              "config_string", "config_string",
+              "input { spec_sampler_input {} }\n" + config + "\noutput { spec_sampler_output {} }"
+            ), settings
+          )
+        )
+      end
+      let(:event) do
+        sample_event = [sample_event] unless sample_event.is_a?(Array)
+        next sample_event.collect do |e|
+          e = { "message" => e } if e.is_a?(String)
+          next LogStash::Event.new(e)
+        end
+      end
+
+      let(:results) do
+        SpecSamplerInput.set_event event
+        pipeline.run
+        SpecSamplerOutput.seen
+      end
+
+      after do
+        pipeline.close
+      end
+
+      subject {results.length > 1 ? results : results.first}
+
+      it("when processed", &block)
+    end
+  end
+end

data/versions-gem-copy.yml

@@ -1,7 +1,10 @@
 ---
-logstash: 6.0.1
-logstash-core: 6.0.1
+logstash: 6.1.0
+logstash-core: 6.1.0
 logstash-core-plugin-api: 2.1.16
+
+# jruby must reference a *released* version of jruby which can be downloaded from the official download url
+# *and* for which jars artifacts are published for compile-time
 jruby:
   version: 9.1.13.0
   sha1: 815bac27d5daa1459a4477d6d80584f007ce6a68

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 6.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-04 00:00:00.000000000 Z
+date: 2017-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -417,7 +417,9 @@ files:
 - lib/logstash/instrument/periodic_pollers.rb
 - lib/logstash/instrument/snapshot.rb
 - lib/logstash/instrument/wrapped_write_client.rb
+- lib/logstash/java_filter_delegator.rb
 - lib/logstash/java_integration.rb
+- lib/logstash/java_pipeline.rb
 - lib/logstash/json.rb
 - lib/logstash/logging.rb
 - lib/logstash/logging/json.rb
@@ -453,7 +455,6 @@ files:
 - lib/logstash/patches/exception_to_json.rb
 - lib/logstash/patches/profile_require_calls.rb
 - lib/logstash/patches/puma.rb
-- lib/logstash/patches/rubygems.rb
 - lib/logstash/patches/stronger_openssl_defaults.rb
 - lib/logstash/pipeline.rb
 - lib/logstash/pipeline_action.rb
@@ -465,6 +466,7 @@ files:
 - lib/logstash/pipeline_settings.rb
 - lib/logstash/plugin.rb
 - lib/logstash/plugins/hooks_registry.rb
+- lib/logstash/plugins/plugin_factory.rb
 - lib/logstash/plugins/registry.rb
 - lib/logstash/queue_factory.rb
 - lib/logstash/runner.rb
@@ -482,7 +484,6 @@ files:
 - lib/logstash/util/dead_letter_queue_manager.rb
 - lib/logstash/util/decorators.rb
 - lib/logstash/util/duration_formatter.rb
-- lib/logstash/util/environment_variables.rb
 - lib/logstash/util/filetools.rb
 - lib/logstash/util/java_version.rb
 - lib/logstash/util/loggable.rb
@@ -492,7 +493,9 @@ files:
 - lib/logstash/util/prctl.rb
 - lib/logstash/util/retryable.rb
 - lib/logstash/util/safe_uri.rb
+- lib/logstash/util/settings_helper.rb
 - lib/logstash/util/socket_peer.rb
+- lib/logstash/util/substitution_variables.rb
 - lib/logstash/util/thread_dump.rb
 - lib/logstash/util/time_value.rb
 - lib/logstash/util/unicode_trimmer.rb
@@ -551,7 +554,9 @@ files:
 - spec/logstash/instrument/periodic_poller/load_average_spec.rb
 - spec/logstash/instrument/periodic_poller/os_spec.rb
 - spec/logstash/instrument/wrapped_write_client_spec.rb
+- spec/logstash/java_filter_delegator_spec.rb
 - spec/logstash/java_integration_spec.rb
+- spec/logstash/java_pipeline_spec.rb
 - spec/logstash/json_spec.rb
 - spec/logstash/legacy_ruby_event_spec.rb
 - spec/logstash/legacy_ruby_timestamp_spec.rb
@@ -609,6 +614,7 @@ files:
 - spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
+- spec/support/pipeline/pipeline_helpers.rb
 - spec/support/shared_contexts.rb
 - spec/support/shared_examples.rb
 - versions-gem-copy.yml
@@ -637,7 +643,7 @@ requirements:
 - jar com.fasterxml.jackson.core:jackson-core, 2.9.1
 - jar com.fasterxml.jackson.core:jackson-databind, 2.9.1
 - jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1
-- jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.9.1
+- jar org.codehaus.janino:janino, 3.0.7
 - jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1
 rubyforge_project:
 rubygems_version: 2.6.13
@@ -693,7 +699,9 @@ test_files:
 - spec/logstash/instrument/periodic_poller/load_average_spec.rb
 - spec/logstash/instrument/periodic_poller/os_spec.rb
 - spec/logstash/instrument/wrapped_write_client_spec.rb
+- spec/logstash/java_filter_delegator_spec.rb
 - spec/logstash/java_integration_spec.rb
+- spec/logstash/java_pipeline_spec.rb
 - spec/logstash/json_spec.rb
 - spec/logstash/legacy_ruby_event_spec.rb
 - spec/logstash/legacy_ruby_timestamp_spec.rb
@@ -751,5 +759,6 @@ test_files:
 - spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
+- spec/support/pipeline/pipeline_helpers.rb
 - spec/support/shared_contexts.rb
 - spec/support/shared_examples.rb