logstash-core 5.0.0.alpha6.snapshot5-java → 5.0.0-java

data/spec/logstash/pipeline_spec.rb

@@ -100,6 +100,60 @@ describe LogStash::Pipeline do
     pipeline_settings_obj.reset
   end
 
+
+  describe "event cancellation" do
+    # test harness for https://github.com/elastic/logstash/issues/6055
+
+    let(:output) { DummyOutputWithEventsArray.new }
+
+    before do
+      allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
+      allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputwitheventsarray").and_return(DummyOutputWithEventsArray)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "drop").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "mutate").and_call_original
+      allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
+      allow(DummyOutputWithEventsArray).to receive(:new).with(any_args).and_return(output)
+    end
+
+    let(:config) do
+      <<-CONFIG
+        input {
+          generator {
+            lines => ["1", "2", "END"]
+            count => 1
+          }
+        }
+        filter {
+          if [message] == "1" {
+            drop {}
+          }
+          mutate { add_tag => ["notdropped"] }
+        }
+        output { dummyoutputwitheventsarray {} }
+      CONFIG
+    end
+
+    it "should not propage cancelled events from filter to output" do
+      abort_on_exception_state = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+
+      pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+      wait(3).for do
+        # give us a bit of time to flush the events
+        # puts("*****" + output.events.map{|e| e.message}.to_s)
+        output.events.map{|e| e.get("message")}.include?("END")
+      end.to be_truthy
+      expect(output.events.size).to eq(2)
+      expect(output.events[0].get("tags")).to eq(["notdropped"])
+      expect(output.events[1].get("tags")).to eq(["notdropped"])
+      pipeline.shutdown
+
+      Thread.abort_on_exception = abort_on_exception_state
+    end
+  end
+
   describe "defaulting the pipeline workers based on thread safety" do
     before(:each) do
       allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)

data/spec/logstash/runner_spec.rb

@@ -21,6 +21,7 @@ describe LogStash::Runner do
     allow(LogStash::Runner).to receive(:logger).and_return(logger)
     allow(logger).to receive(:debug?).and_return(true)
     allow(logger).to receive(:subscribe).with(any_args)
+    allow(logger).to receive(:debug) {}
     allow(logger).to receive(:log) {}
     allow(logger).to receive(:info) {}
     allow(logger).to receive(:fatal) {}
@@ -279,7 +280,7 @@ describe LogStash::Runner do
       it "should set log level to warn" do
         args = ["--version"]
         subject.run("bin/logstash", args)
-        expect(logger.level).to eq(:warn)
+        expect(logger.level).to eq(:info)
       end
     end
     context "when setting to debug" do

data/spec/logstash/setting_spec.rb

@@ -63,6 +63,7 @@ describe LogStash::Setting do
       end
     end
   end
+
   describe "#set" do
     subject { described_class.new("number", Numeric, 1) }
     it "should change the value of a setting" do
@@ -77,12 +78,33 @@ describe LogStash::Setting do
         expect(subject.set?).to eq(true)
       end
     end
-
     context "when the argument's class does not match @klass" do
       it "should throw an exception" do
         expect { subject.set("not a number") }.to raise_error
       end
     end
+    context "when strict=false" do
+      let(:strict) { false }
+      subject { described_class.new("number", Numeric, 1, strict) }
+      before do
+        expect(subject).not_to receive(:validate)
+      end
+
+      it "should not call #validate" do
+        subject.set(123)
+      end
+    end
+    context "when strict=true" do
+      let(:strict) { true }
+      subject { described_class.new("number", Numeric, 1, strict) }
+      before do
+        expect(subject).to receive(:validate)
+      end
+
+      it "should call #validate" do
+        subject.set(123)
+      end
+    end
   end
 
   describe "#reset" do

data/spec/logstash/settings/string_spec.rb

@@ -4,7 +4,7 @@ require "logstash/settings"
 
 describe LogStash::Setting::String do
   let(:possible_values) { ["a", "b", "c"] }
-  subject { described_class.new("mytext", nil, false, possible_values) }
+  subject { described_class.new("mytext", possible_values.first, true, possible_values) }
   describe "#set" do
     context "when a value is given outside of possible_values" do
       it "should raise an ArgumentError" do

data/spec/logstash/settings_spec.rb

@@ -59,4 +59,31 @@ describe LogStash::Settings do
       expect(subset.get("num.2")).to eq(1000)
     end
   end
+
+  describe "#validate_all" do
+    subject { described_class.new }
+    let(:numeric_setting_name) { "example" }
+    let(:numeric_setting) { LogStash::Setting.new(numeric_setting_name, Numeric, 1, false) }
+
+    before do
+      subject.register(numeric_setting)
+      subject.set_value(numeric_setting_name, value)
+    end
+
+    context "when any setting is invalid" do
+      let(:value) { "some string" }
+
+      it "should fail" do
+        expect { subject.validate_all }.to raise_error
+      end
+    end
+
+    context "when all settings are valid" do
+      let(:value) { 123 }
+
+      it "should succeed" do
+        expect { subject.validate_all }.not_to raise_error
+      end
+    end
+  end
 end

data/spec/logstash/util/wrapped_synchronous_queue_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "spec_helper"
 require "logstash/util/wrapped_synchronous_queue"
+require "logstash/instrument/collector"
 
 describe LogStash::Util::WrappedSynchronousQueue do
   context "#offer" do
@@ -45,11 +46,47 @@ describe LogStash::Util::WrappedSynchronousQueue do
     end
 
     describe "WriteClient | ReadClient" do
-      context "when writing to the queue" do
-        let(:queue) { DummyQueue.new }
-        let(:write_client) { LogStash::Util::WrappedSynchronousQueue::WriteClient.new(queue)}
-        let(:read_client)  { LogStash::Util::WrappedSynchronousQueue::ReadClient.new(queue)}
+      let(:queue) { DummyQueue.new }
+      let(:write_client) { LogStash::Util::WrappedSynchronousQueue::WriteClient.new(queue)}
+      let(:read_client)  { LogStash::Util::WrappedSynchronousQueue::ReadClient.new(queue)}
+
+      context "when reading from the queue" do
+        let(:collector) { LogStash::Instrument::Collector.new }
+
+        before do
+          read_client.set_events_metric(LogStash::Instrument::Metric.new(collector).namespace(:events))
+          read_client.set_pipeline_metric(LogStash::Instrument::Metric.new(collector).namespace(:pipeline))
+        end
+
+        context "when the queue is empty" do
+          it "doesnt record the `duration_in_millis`" do
+            batch = read_client.take_batch
+            read_client.close_batch(batch)
+            store = collector.snapshot_metric.metric_store
+            expect(store.size).to eq(0)
+          end
+        end
 
+        context "when we have item in the queue" do
+          it "records the `duration_in_millis`" do
+            batch = write_client.get_new_batch
+            5.times {|i| batch.push("value-#{i}")}
+            write_client.push_batch(batch)
+            read_batch = read_client.take_batch
+            sleep(0.1) # simulate some work?
+            read_client.close_batch(batch)
+            store = collector.snapshot_metric.metric_store
+
+            expect(store.size).to eq(4)
+            expect(store.get_shallow(:events, :in).value).to eq(5)
+            expect(store.get_shallow(:events, :duration_in_millis).value).to be > 0
+            expect(store.get_shallow(:pipeline, :in).value).to eq(5)
+            expect(store.get_shallow(:pipeline, :duration_in_millis).value).to be > 0
+          end
+        end
+      end
+
+      context "when writing to the queue" do
         before :each do
           read_client.set_events_metric(LogStash::Instrument::NullMetric.new)
           read_client.set_pipeline_metric(LogStash::Instrument::NullMetric.new)
@@ -57,22 +94,23 @@ describe LogStash::Util::WrappedSynchronousQueue do
 
         it "appends batches to the queue" do
           batch = write_client.get_new_batch
-          5.times {|i| batch.push("value-#{i}")}
+          5.times {|i| batch.push(LogStash::Event.new({"message" => "value-#{i}"}))}
           write_client.push_batch(batch)
           read_batch = read_client.take_batch
           expect(read_batch.size).to eq(5)
           i = 0
           read_batch.each do |data|
-            expect(data).to eq("value-#{i}")
-            read_batch.cancel("value-#{i}") if i > 2
-            read_batch.merge("generated-#{i}") if i > 2
+            expect(data.get("message")).to eq("value-#{i}")
+            # read_batch.cancel("value-#{i}") if i > 2     # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+            data.cancel if i > 2
+            read_batch.merge(LogStash::Event.new({"message" => "generated-#{i}"})) if i > 2
             i += 1
           end
-          expect(read_batch.cancelled_size).to eq(2)
+          # expect(read_batch.cancelled_size).to eq(2) # disabled for https://github.com/elastic/logstash/issues/6055
           i = 0
           read_batch.each do |data|
-            expect(data).to eq("value-#{i}") if i < 3
-            expect(data).to eq("generated-#{i}") if i > 2
+            expect(data.get("message")).to eq("value-#{i}") if i < 3
+            expect(data.get("message")).to eq("generated-#{i}") if i > 2
             i += 1
           end
         end

data/spec/logstash/webserver_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 # require "logstash/json"
 require "logstash/webserver"
+require_relative "../support/helpers"
 require "socket"
 require "spec_helper"
 require "open-uri"
@@ -9,11 +10,15 @@ def block_ports(range)
   servers = []
 
   range.each do |port|
-    server = TCPServer.new("localhost", port)
-    Thread.new do
-      client = server.accept rescue nil
+    begin
+      server = TCPServer.new("localhost", port)
+      Thread.new do
+        client = server.accept rescue nil
+      end
+      servers << server
+    rescue => e
+      # The port can already be taken
     end
-    servers << server
   end
 
   sleep(1)
@@ -36,29 +41,56 @@ describe LogStash::WebServer do
     Thread.abort_on_exception = @abort
   end
 
-  let(:logger) { double("logger") }
+  let(:logger) { LogStash::Logging::Logger.new("testing") }
   let(:agent) { double("agent") }
   let(:webserver) { double("webserver") }
 
   before :each do
-    [:info, :warn, :error, :fatal, :debug].each do |level|
-      allow(logger).to receive(level)
-    end
-    [:info?, :warn?, :error?, :fatal?, :debug?].each do |level|
-      allow(logger).to receive(level)
-    end
-
     allow(webserver).to receive(:address).and_return("127.0.0.1")
     allow(agent).to receive(:webserver).and_return(webserver)
   end
 
-  context "when the port is already in use and a range is provided" do
-    subject { LogStash::WebServer.new(logger,
-                                      agent,
-                                      { :http_host => "localhost", :http_ports => port_range
-                                      })}
+  subject { LogStash::WebServer.new(logger,
+                                    agent,
+                                    { :http_host => "localhost", :http_ports => port_range })}
+
+  let(:port_range) { 10000..10010 }
+
+  context "when an exception occur in the server thread" do
+    let(:spy_output) { spy("stderr").as_null_object }
+
+    it "should not log to STDERR" do
+      backup_stderr = STDERR
+      backup_stdout = STDOUT
+
+      # We are redefining constants, so lets silence the warning
+      silence_warnings do
+        STDOUT = STDERR = spy_output
+      end
+
+      expect(spy_output).not_to receive(:puts).with(any_args)
+      expect(spy_output).not_to receive(:write).with(any_args)
+
+      # This trigger an infinite loop in the reactor
+      expect(IO).to receive(:select).and_raise(IOError).at_least(:once)
+
+      t = Thread.new do
+        subject.run
+      end
+
+      sleep(1)
+
+      # We cannot use stop here, since the code is stuck in an infinite loop
+      t.kill rescue nil
 
-    let(:port_range) { 10000..10010 }
+      silence_warnings do
+        STDERR = backup_stderr
+        STDOUT = backup_stdout
+      end
+    end
+  end
+
+  context "when the port is already in use and a range is provided" do
     after(:each) { free_ports(@servers) }
 
     context "when we have available ports" do
@@ -93,3 +125,29 @@ describe LogStash::WebServer do
     end
   end
 end
+
+describe LogStash::IOWrappedLogger do
+  let(:logger)  { spy("logger") }
+  let(:message) { "foobar" }
+
+  subject { described_class.new(logger) }
+
+  it "responds to puts" do
+    subject.puts(message)
+    expect(logger).to have_received(:debug).with(message)
+  end
+
+  it "responds to write" do
+    subject.write(message)
+    expect(logger).to have_received(:debug).with(message)
+  end
+
+  it "reponds to <<" do
+    subject << message
+    expect(logger).to have_received(:debug).with(message)
+  end
+
+  it "responds to sync=(v)" do
+    expect{ subject.sync = true }.not_to raise_error
+  end
+end

data/spec/support/helpers.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+def silence_warnings
+  warn_level = $VERBOSE
+  $VERBOSE = nil
+  yield
+ensure
+  $VERBOSE = warn_level
+end

data/spec/support/mocks_classes.rb

@@ -25,3 +25,25 @@ class DummyOutput < LogStash::Outputs::Base
     @num_closes = 1
   end
 end
+
+class DummyOutputWithEventsArray < LogStash::Outputs::Base
+  config_name "dummyoutput2"
+  milestone 2
+
+  attr_reader :events
+
+  def initialize(params={})
+    super
+    @events = []
+  end
+
+  def register
+  end
+
+  def receive(event)
+    @events << event
+  end
+
+  def close
+  end
+end

data/spec/support/shared_examples.rb

@@ -95,4 +95,14 @@ shared_examples "metrics commons operations" do
   end
 end
 
+shared_examples "not found" do
+  it "should return a 404 to unknown request" do
+    do_request { get "/i_want_to_believe-#{Time.now.to_i}" }
+    expect(last_response.content_type).to eq("application/json")
+    expect(last_response).not_to be_ok
+    expect(last_response.status).to eq(404)
+    expect(LogStash::Json.load(last_response.body)).to include("status" => 404)
+    expect(LogStash::Json.load(last_response.body)["path"]).not_to be_nil
+  end
+end
 

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.0.0.alpha6.snapshot5
+  version: 5.0.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-26 00:00:00.000000000 Z
+date: 2016-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha6.snapshot5
+        version: 5.0.0
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha6.snapshot5
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -303,7 +303,9 @@ files:
 - lib/logstash/api/commands/stats.rb
 - lib/logstash/api/commands/system/basicinfo_command.rb
 - lib/logstash/api/commands/system/plugins_command.rb
+- lib/logstash/api/errors.rb
 - lib/logstash/api/modules/base.rb
+- lib/logstash/api/modules/logging.rb
 - lib/logstash/api/modules/node.rb
 - lib/logstash/api/modules/node_stats.rb
 - lib/logstash/api/modules/plugins.rb
@@ -361,6 +363,7 @@ files:
 - lib/logstash/patches/cabin.rb
 - lib/logstash/patches/clamp.rb
 - lib/logstash/patches/profile_require_calls.rb
+- lib/logstash/patches/puma.rb
 - lib/logstash/patches/rubygems.rb
 - lib/logstash/patches/stronger_openssl_defaults.rb
 - lib/logstash/pipeline.rb
@@ -393,6 +396,7 @@ files:
 - lib/logstash/webserver.rb
 - locales/en.yml
 - logstash-core.gemspec
+- spec/api/lib/api/logging_spec.rb
 - spec/api/lib/api/node_plugins_spec.rb
 - spec/api/lib/api/node_spec.rb
 - spec/api/lib/api/node_stats_spec.rb
@@ -400,6 +404,7 @@ files:
 - spec/api/lib/api/root_spec.rb
 - spec/api/lib/api/support/resource_dsl_methods.rb
 - spec/api/lib/commands/stats.rb
+- spec/api/lib/errors_spec.rb
 - spec/api/lib/rack_app_spec.rb
 - spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
@@ -450,6 +455,7 @@ files:
 - spec/logstash/util_spec.rb
 - spec/logstash/webserver_spec.rb
 - spec/static/i18n_spec.rb
+- spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
 - spec/support/shared_examples.rb
@@ -458,7 +464,7 @@ files:
 - vendor/jars/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
 - vendor/jars/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
 - vendor/jars/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
-- vendor/jars/org/logstash/logstash-core/5.0.0-alpha6/logstash-core-5.0.0-alpha6.jar
+- vendor/jars/org/logstash/logstash-core/5.0.0/logstash-core-5.0.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -475,9 +481,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
 rubygems_version: 2.4.8
@@ -485,6 +491,7 @@ signing_key:
 specification_version: 4
 summary: logstash-core - The core components of logstash
 test_files:
+- spec/api/lib/api/logging_spec.rb
 - spec/api/lib/api/node_plugins_spec.rb
 - spec/api/lib/api/node_spec.rb
 - spec/api/lib/api/node_stats_spec.rb
@@ -492,6 +499,7 @@ test_files:
 - spec/api/lib/api/root_spec.rb
 - spec/api/lib/api/support/resource_dsl_methods.rb
 - spec/api/lib/commands/stats.rb
+- spec/api/lib/errors_spec.rb
 - spec/api/lib/rack_app_spec.rb
 - spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
@@ -542,6 +550,7 @@ test_files:
 - spec/logstash/util_spec.rb
 - spec/logstash/webserver_spec.rb
 - spec/static/i18n_spec.rb
+- spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
 - spec/support/shared_examples.rb