logstash-core 6.0.0.alpha2-java → 6.0.0.beta1-java

data/spec/logstash/pipeline_dlq_commit_spec.rb

@@ -36,7 +36,7 @@ class DLQCommittingFilter < LogStash::Filters::Base
 end
 
 describe LogStash::Pipeline do
-  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings_obj) { LogStash::SETTINGS.clone }
   let(:pipeline_settings) do
     {
       "pipeline.workers" => 2,

data/spec/logstash/pipeline_spec.rb

@@ -5,6 +5,8 @@ require "logstash/filters/multiline"
 require_relative "../support/mocks_classes"
 require_relative "../support/helpers"
 require_relative "../logstash/pipeline_reporter_spec" # for DummyOutput class
+require "stud/try"
+require 'timeout'
 
 class DummyInput < LogStash::Inputs::Base
   config_name "dummyinput"
@@ -104,12 +106,21 @@ describe LogStash::Pipeline do
   let(:worker_thread_count)     { 5 }
   let(:safe_thread_count)       { 1 }
   let(:override_thread_count)   { 42 }
+  let(:dead_letter_queue_enabled) { false }
+  let(:dead_letter_queue_path) { }
   let(:pipeline_settings_obj) { LogStash::SETTINGS }
   let(:pipeline_settings) { {} }
+  let(:max_retry) {10} #times
+  let(:timeout) {120} #seconds
 
   before :each do
     pipeline_workers_setting = LogStash::SETTINGS.get_setting("pipeline.workers")
     allow(pipeline_workers_setting).to receive(:default).and_return(worker_thread_count)
+    dlq_enabled_setting = LogStash::SETTINGS.get_setting("dead_letter_queue.enable")
+    allow(dlq_enabled_setting).to receive(:value).and_return(dead_letter_queue_enabled)
+    dlq_path_setting = LogStash::SETTINGS.get_setting("path.dead_letter_queue")
+    allow(dlq_path_setting).to receive(:value).and_return(dead_letter_queue_path)
+
     pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
   end
 
@@ -117,6 +128,18 @@ describe LogStash::Pipeline do
     pipeline_settings_obj.reset
   end
 
+  describe "#ephemeral_id" do
+    it "creates an ephemeral_id at creation time" do
+      pipeline = mock_pipeline_from_string("input { generator { count =>  1 } } output { null {} }")
+      expect(pipeline.ephemeral_id).to_not be_nil
+      pipeline.close
+
+      second_pipeline = mock_pipeline_from_string("input { generator { count => 1 } } output { null {} }")
+      expect(second_pipeline.ephemeral_id).not_to eq(pipeline.ephemeral_id)
+      second_pipeline.close
+    end
+  end
+
 
   describe "event cancellation" do
     # test harness for https://github.com/elastic/logstash/issues/6055
@@ -156,12 +179,16 @@ describe LogStash::Pipeline do
 
       pipeline = mock_pipeline_from_string(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
-      sleep(0.1) until pipeline.ready?
-      wait(3).for do
-        # give us a bit of time to flush the events
-        # puts("*****" + output.events.map{|e| e.message}.to_s)
-        output.events.map{|e| e.get("message")}.include?("END")
-      end.to be_truthy
+      Timeout.timeout(timeout) do
+        sleep(0.1) until pipeline.ready?
+      end
+      Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(3).for do
+          # give us a bit of time to flush the events
+          # puts("*****" + output.events.map{|e| e.message}.to_s)
+          output.events.map{|e| e.get("message")}.include?("END")
+        end.to be_truthy
+      end
       expect(output.events.size).to eq(2)
       expect(output.events[0].get("tags")).to eq(["notdropped"])
       expect(output.events[1].get("tags")).to eq(["notdropped"])
@@ -409,7 +436,9 @@ describe LogStash::Pipeline do
       # race condition if called in the thread
       p = pipeline
       t = Thread.new { p.run }
-      sleep(0.1) until pipeline.ready?
+      Timeout.timeout(timeout) do
+        sleep(0.1) until pipeline.ready?
+      end
       pipeline.shutdown
       t.join
     end
@@ -607,11 +636,15 @@ describe LogStash::Pipeline do
       Thread.abort_on_exception = true
       pipeline = mock_pipeline_from_string(config, pipeline_settings_obj)
       t = Thread.new { pipeline.run }
-      sleep(0.1) until pipeline.ready?
-      wait(10).for do
-        # give us a bit of time to flush the events
-        output.events.empty?
-      end.to be_falsey
+      Timeout.timeout(timeout) do
+        sleep(0.1) until pipeline.ready?
+      end
+      Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(10).for do
+          # give us a bit of time to flush the events
+          output.events.empty?
+        end.to be_falsey
+      end
 
       expect(output.events.any? {|e| e.get("message") == "dummy_flush"}).to eq(true)
 
@@ -711,9 +744,12 @@ describe LogStash::Pipeline do
         # subject must be first call outside the thread context because of lazy initialization
         s = subject
         t = Thread.new { s.run }
-        sleep(0.1) until subject.ready?
-
-        sleep(0.1)
+        Timeout.timeout(timeout) do
+          sleep(0.1) until subject.ready?
+        end
+        Timeout.timeout(timeout) do
+          sleep(0.1)
+        end
         expect(subject.uptime).to be > 0
         subject.shutdown
         t.join
@@ -777,13 +813,17 @@ describe LogStash::Pipeline do
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
 
       pipeline_thread
-      sleep(0.1) until subject.ready?
+      Timeout.timeout(timeout) do
+        sleep(0.1) until subject.ready?
+      end
 
       # make sure we have received all the generated events
-      wait(3).for do
-        # give us a bit of time to flush the events
-        dummyoutput.events.size >= number_of_events
-      end.to be_truthy
+      Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(3).for do
+          # give us a bit of time to flush the events
+          dummyoutput.events.size >= number_of_events
+        end.to be_truthy
+      end
     end
 
     after :each do
@@ -840,6 +880,33 @@ describe LogStash::Pipeline do
           expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:name].value).to eq(LogStash::Filters::Multiline.config_name)
         end
       end
+
+      context 'when dlq is disabled' do
+        let (:collect_stats) { subject.collect_dlq_stats}
+        let (:collected_stats) { collected_metric[:stats][:pipelines][:main][:dlq]}
+        let (:available_stats) {[:path, :queue_size_in_bytes]}
+
+        it 'should show not show any dlq stats' do
+          collect_stats
+          expect(collected_stats).to be_nil
+        end
+
+      end
+
+      context 'when dlq is enabled' do
+        let (:dead_letter_queue_enabled) { true }
+        let (:dead_letter_queue_path) { Stud::Temporary.directory }
+        let (:pipeline_dlq_path) { "#{dead_letter_queue_path}/#{pipeline_id}"}
+
+        let (:collect_stats) { subject.collect_dlq_stats }
+        let (:collected_stats) { collected_metric[:stats][:pipelines][:main][:dlq]}
+
+        it 'should show dlq stats' do
+          collect_stats
+          # A newly created dead letter queue with no entries will have a size of 1 (the version 'header')
+          expect(collected_stats[:queue_size_in_bytes].value).to eq(1)
+        end
+      end
     end
   end
 

data/spec/logstash/runner_spec.rb

@@ -7,12 +7,12 @@ require "stud/temporary"
 require "logstash/util/java_version"
 require "logstash/logging/json"
 require "logstash/config/source_loader"
+require "logstash/config/modules_common"
+require "logstash/modules/util"
+require "logstash/elasticsearch_client"
 require "json"
 require_relative "../support/helpers"
-
-class NullRunner
-  def run(args); end
-end
+require_relative "../support/matchers"
 
 describe LogStash::Runner do
 
@@ -43,6 +43,10 @@ describe LogStash::Runner do
     allow(agent).to receive(:shutdown)
   end
 
+  after :each do
+    LogStash::Logging::Logger::configure_logging("info")
+  end
+
   describe "argument precedence" do
     let(:config) { "input {} output {}" }
     let(:cli_args) { ["-e", config, "-w", "20"] }
@@ -126,6 +130,7 @@ describe LogStash::Runner do
     context "with a good configuration" do
       let(:pipeline_string) { "input { } filter { } output { }" }
       it "should exit successfully" do
+        expect(logger).not_to receive(:fatal)
         expect(subject.run(args)).to eq(0)
       end
     end
@@ -170,12 +175,14 @@ describe LogStash::Runner do
         let(:queue_override_path) { "/tmp/queue-override_path" }
 
         it "should set data paths" do
+          LogStash::SETTINGS.set("path.queue", queue_override_path)
+
           expect(LogStash::Agent).to receive(:new) do |settings|
             expect(settings.get("path.data")).to eq(test_data_path)
             expect(settings.get("path.queue")).to eq(queue_override_path)
           end
 
-          LogStash::SETTINGS.set("path.queue", queue_override_path)
+
 
           args = ["--path.data", test_data_path, "-e", pipeline_string]
           subject.run("bin/logstash", args)
@@ -305,6 +312,116 @@ describe LogStash::Runner do
     end
   end
 
+  describe "logstash modules" do
+    before(:each) do
+      test_modules_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", "modules_test_files"))
+      LogStash::Modules::Util.register_local_modules(test_modules_dir)
+    end
+
+    describe "--config.test_and_exit" do
+      subject { LogStash::Runner.new("") }
+      let(:args) { ["-t", "--modules", module_string] }
+
+      context "with a good configuration" do
+        let(:module_string) { "tester" }
+        it "should exit successfully" do
+          expect(logger).not_to receive(:fatal)
+          expect(subject.run(args)).to eq(0)
+        end
+      end
+
+      context "with a bad configuration" do
+        let(:module_string) { "rlwekjhrewlqrkjh" }
+        it "should fail by returning a bad exit code" do
+          expect(logger).to receive(:fatal)
+          expect(subject.run(args)).to eq(1)
+        end
+      end
+    end
+
+    describe "--modules" do
+      let(:args) { ["--modules", module_string] }
+
+      context "with an available module specified but no connection to elasticsearch" do
+        let(:module_string) { "tester" }
+        before do
+          expect(logger).to receive(:fatal) do |msg, hash|
+            expect(msg).to eq("An unexpected error occurred!")
+            expect(hash).to be_a_config_loading_error_hash(
+              /Failed to import module configurations to Elasticsearch and\/or Kibana. Module: tester has/)
+          end
+          expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
+            pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
+            expect(pipelines).to eq([])
+            agent
+          end
+        end
+        it "should log fatally and return a bad exit code" do
+          expect(subject.run("bin/logstash", args)).to eq(1)
+        end
+      end
+
+      context "with an available module specified and a mocked connection to elasticsearch" do
+        let(:module_string) { "tester" }
+        let(:kbn_version) { "6.0.0" }
+        let(:esclient) { double(:esclient) }
+        let(:kbnclient) { double(:kbnclient) }
+        let(:response) { double(:response) }
+        before do
+          allow(response).to receive(:status).and_return(404)
+          allow(esclient).to receive(:head).and_return(response)
+          allow(esclient).to receive(:can_connect?).and_return(true)
+          allow(kbnclient).to receive(:version).and_return(kbn_version)
+          allow(kbnclient).to receive(:version_parts).and_return(kbn_version.split('.'))
+          allow(kbnclient).to receive(:can_connect?).and_return(true)
+          allow(LogStash::ElasticsearchClient).to receive(:build).and_return(esclient)
+          allow(LogStash::Modules::KibanaClient).to receive(:new).and_return(kbnclient)
+
+          expect(esclient).to receive(:put).once do |path, content|
+            LogStash::ElasticsearchClient::Response.new(201, "", {})
+          end
+          expect(kbnclient).to receive(:post).twice do |path, content|
+            LogStash::Modules::KibanaClient::Response.new(201, "", {})
+          end
+
+          expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
+            pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
+            expect(pipelines).not_to be_empty
+            module_pipeline = pipelines.first
+            expect(module_pipeline).to include("pipeline_id", "config_string")
+            expect(module_pipeline["pipeline_id"]).to include('tester')
+            expect(module_pipeline["config_string"]).to include('index => "tester-')
+            agent
+          end
+          expect(logger).not_to receive(:fatal)
+          expect(logger).not_to receive(:error)
+        end
+        it "should not terminate logstash" do
+          expect(subject.run("bin/logstash", args)).to be_nil
+        end
+      end
+
+      context "with an unavailable module specified" do
+        let(:module_string) { "fancypants" }
+        before do
+          expect(logger).to receive(:fatal) do |msg, hash|
+            expect(msg).to eq("An unexpected error occurred!")
+            expect(hash).to be_a_config_loading_error_hash(
+              /The modules specified are not available yet. Specified modules: \["fancypants"\] Available modules:/)
+          end
+          expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
+            pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
+            expect(pipelines).to eq([])
+            agent
+          end
+        end
+        it "should log fatally and return a bad exit code" do
+          expect(subject.run("bin/logstash", args)).to eq(1)
+        end
+      end
+    end
+  end
+
   describe "--log.level" do
     before :each do
       allow_any_instance_of(subject).to receive(:show_version)

data/spec/logstash/setting_spec.rb

@@ -80,7 +80,7 @@ describe LogStash::Setting do
     end
     context "when the argument's class does not match @klass" do
       it "should throw an exception" do
-        expect { subject.set("not a number") }.to raise_error
+        expect { subject.set("not a number") }.to raise_error ArgumentError
       end
     end
     context "when strict=false" do
@@ -131,7 +131,7 @@ describe LogStash::Setting do
     context "when validation fails" do
       let(:new_value) { "very very very very very big text" }
       it "should raise an exception" do
-        expect { subject.set(new_value) }.to raise_error
+        expect { subject.set(new_value) }.to raise_error ArgumentError
       end
       it "should not change the value" do
         subject.set(new_value) rescue nil

data/spec/logstash/settings/splittable_string_array_spec.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/settings"
+
+describe LogStash::Setting::SplittableStringArray do
+  let(:element_class) { String }
+  let(:default_value) { [] }
+
+  subject { described_class.new("testing", element_class, default_value) }
+
+  before do
+    subject.set(candidate)
+  end
+
+  context "when giving an array" do
+    let(:candidate) { ["hello,", "ninja"] }
+
+    it "returns the same elements" do
+      expect(subject.value).to match(candidate)
+    end
+  end
+
+  context "when given a string" do
+    context "with 1 element" do
+      let(:candidate) { "hello" }
+
+      it "returns 1 element" do
+        expect(subject.value).to match(["hello"])
+      end
+    end
+
+    context "with multiple element" do
+      let(:candidate) { "hello,ninja" }
+
+      it "returns an array of string" do
+        expect(subject.value).to match(["hello", "ninja"])
+      end
+    end
+  end
+
+  context "when defining a custom tokenizer" do
+    subject { described_class.new("testing", element_class, default_value, strict=true, ";") }
+
+    let(:candidate) { "hello;ninja" }
+
+    it "returns an array of string" do
+      expect(subject.value).to match(["hello", "ninja"])
+    end
+  end
+end
+

data/spec/logstash/timestamp_spec.rb

@@ -5,6 +5,12 @@ require "logstash/timestamp"
 
 describe LogStash::Timestamp do
   context "constructors" do
+    # Via JRuby 9k time see logstash/issues/7463
+    # JRuby 9k now uses Java 8 Time with nanosecond precision but
+    # our Timestamp use Joda with millisecond precision
+    # expected: 2017-06-15 10:34:08.389999999 +0000
+    #      got: 2017-06-15 10:34:08.389000000 +0000
+    # we may need to use `be_within(0.000999999).of()` in other places too
     it "should work" do
       t = LogStash::Timestamp.new
       expect(t.time.to_i).to be_within(1).of Time.now.to_i
@@ -12,9 +18,9 @@ describe LogStash::Timestamp do
       t = LogStash::Timestamp.now
       expect(t.time.to_i).to be_within(1).of Time.now.to_i
 
-      now = Time.now.utc
+      now = DateTime.now.to_time.utc
       t = LogStash::Timestamp.new(now)
-      expect(t.time).to eq(now)
+      expect(t.time.to_f).to be_within(0.000999999).of(now.to_f)
 
       t = LogStash::Timestamp.at(now.to_i)
       expect(t.time.to_i).to eq(now.to_i)

data/spec/logstash/util/safe_uri_spec.rb

@@ -16,5 +16,21 @@ module LogStash module Util
         expect(cloned_safe_uri.query).to eq("a=b")
       end
     end
+
+    describe "handling escapable fields" do
+      let(:user) { "u%20" }
+      let(:password) { "p%20ss" }
+      let(:path) { "/a%20/path" }
+      let(:query) { "a%20query&another=es%3dq" }
+      let(:fragment) { "spacey%20fragment" }
+      subject { LogStash::Util::SafeURI.new("http://#{user}:#{password}@example.net#{path}?#{query}\##{fragment}") }
+
+      [:user, :password, :path, :query, :fragment].each do |field|
+        it "should not escape the #{field} field" do
+          expected = self.send(field)
+          expect(subject.send(field)).to eq(expected)
+        end
+      end
+    end
   end
 end end