logstash-core 6.0.0.beta2-java → 6.0.0-java

data/spec/logstash/modules/kibana_client_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/modules/kibana_client"
+module LogStash module Modules
+  KibanaTestResponse = Struct.new(:code, :body, :headers)
+  class KibanaTestClient
+    def http(method, endpoint, options)
+      self
+    end
+    def call
+      KibanaTestResponse.new(200, '{"version":{"number":"1.2.3","build_snapshot":false}}', {})
+    end
+  end
+  describe KibanaClient do
+    let(:settings) { Hash.new }
+    let(:test_client) { KibanaTestClient.new }
+    let(:kibana_host) { "https://foo.bar:4321" }
+    subject(:kibana_client) { described_class.new(settings, test_client) }
+
+    context "when supplied with conflicting scheme data" do
+      let(:settings) { {"var.kibana.scheme" => "http", "var.kibana.host" => kibana_host} }
+      it "a new instance will throw an error" do
+        expect{described_class.new(settings, test_client)}.to raise_error(ArgumentError, /Detected differing Kibana host schemes as sourced from var\.kibana\.host: 'https' and var\.kibana\.scheme: 'http'/)
+      end
+    end
+
+    context "when supplied with invalid schemes" do
+      ["httpd", "ftp", "telnet"].each do |uri_scheme|
+        it "a new instance will throw an error" do
+          re = /Kibana host scheme given is invalid, given value: '#{uri_scheme}' - acceptable values: 'http', 'https'/
+          expect{described_class.new({"var.kibana.scheme" => uri_scheme}, test_client)}.to raise_error(ArgumentError, re)
+        end
+      end
+    end
+
+    context "when supplied with the scheme in the host only" do
+      let(:settings) { {"var.kibana.host" => kibana_host} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq("https://foo.bar:4321")
+      end
+    end
+
+    context "when supplied with the scheme in the scheme setting" do
+      let(:settings) { {"var.kibana.scheme" => "https", "var.kibana.host" => "foo.bar:4321"} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq(kibana_host)
+      end
+    end
+
+    context "when supplied with a no scheme host setting and ssl is enabled" do
+      let(:settings) { {"var.kibana.ssl.enabled" => "true", "var.kibana.host" => "foo.bar:4321"} }
+      it "has a version and an endpoint" do
+        expect(kibana_client.version).to eq("1.2.3")
+        expect(kibana_client.endpoint).to eq(kibana_host)
+      end
+    end
+  end
+end end

data/spec/logstash/modules/logstash_config_spec.rb

@@ -3,7 +3,7 @@ require "logstash/modules/logstash_config"
 
 describe LogStash::Modules::LogStashConfig do
   let(:mod) { instance_double("module", :directory => Stud::Temporary.directory, :module_name => "testing") }
-  let(:settings) { {"var.logstash.testing.pants" => "fancy" }}
+  let(:settings) { {"var.logstash.testing.pants" => "fancy", "var.elasticsearch.password" => LogStash::Util::Password.new('correct_horse_battery_staple') }}
   subject { described_class.new(mod, settings) }
 
   describe "configured inputs" do
@@ -36,6 +36,12 @@ describe LogStash::Modules::LogStashConfig do
     end
   end
 
+  describe 'elastic_search_config' do
+    it 'should put the password in correctly' do
+      expect(subject.elasticsearch_output_config()).to include("password => \"correct_horse_battery_staple\"")
+    end
+  end
+
   describe "alias modules options" do
     let(:alias_table) do
       { "var.logstash.testing" => "var.logstash.better" }

data/spec/logstash/modules/scaffold_spec.rb

@@ -81,7 +81,7 @@ ERB
 
     it "provides a logstash config" do
       expect(test_module.logstash_configuration).to be_nil
-      test_module.with_settings(module_settings)
+      test_module.with_settings(LogStash::Util::ModulesSettingArray.new([module_settings]).first)
       expect(test_module.logstash_configuration).not_to be_nil
       config_string = test_module.config_string
       expect(config_string).to include("port => 5606")

data/spec/logstash/modules/settings_merger_spec.rb

@@ -28,6 +28,36 @@ describe LogStash::Modules::SettingsMerger do
     end
   end
 
+  describe "#merge_kibana_auth" do
+
+    before do
+      described_class.merge_kibana_auth!(mod_settings)
+    end
+
+    context 'only elasticsearch username and password is set' do
+      let(:mod_settings) { {"name"=>"mod1", "var.input.tcp.port"=>2222, "var.elasticsearch.username"=>"rupert", "var.elasticsearch.password"=>"fotherington" } }
+      it "sets kibana username and password" do
+        expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
+        expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
+        expect(mod_settings["var.kibana.username"]).to eq("rupert")
+        expect(mod_settings["var.kibana.password"]).to eq("fotherington")
+      end
+    end
+
+    context 'elasticsearch and kibana usernames and passwords are set' do
+      let(:mod_settings) { {"name"=>"mod1", "var.input.tcp.port"=>2222, "var.elasticsearch.username"=>"rupert", "var.elasticsearch.password"=>"fotherington",
+                                                               "var.kibana.username"=>"davey", "var.kibana.password"=>"stott"} }
+
+      it "keeps existing kibana username and password" do
+        expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
+        expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
+        expect(mod_settings["var.kibana.username"]).to eq("davey")
+        expect(mod_settings["var.kibana.password"]).to eq("stott")
+      end
+    end
+
+  end
+
   describe "#merge_cloud_settings" do
     let(:cloud_id) { LogStash::Util::CloudSettingId.new("label:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }
     let(:cloud_auth) { LogStash::Util::CloudSettingAuth.new("elastix:bigwhoppingfairytail") }
@@ -38,7 +68,7 @@ describe LogStash::Modules::SettingsMerger do
         {
           "var.kibana.scheme" => "https",
           "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
-          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
           "var.elasticsearch.username" => "elastix",
           "var.kibana.username" => "elastix"
         }
@@ -63,7 +93,7 @@ describe LogStash::Modules::SettingsMerger do
         {
           "var.kibana.scheme" => "https",
           "var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
-          "var.elasticsearch.hosts" => "notareal.us-east-1.aws.found.io:443",
+          "var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
         }
       end
       let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id}) }

data/spec/logstash/pipeline_action/create_spec.rb

@@ -19,7 +19,10 @@ describe LogStash::PipelineAction::Create do
   subject { described_class.new(pipeline_config, metric) }
 
   after do
-    pipelines.each { |_, pipeline| pipeline.shutdown }
+    pipelines.each do |_, pipeline| 
+      pipeline.shutdown 
+      pipeline.thread.join
+    end
   end
 
   it "returns the pipeline_id" do

data/spec/logstash/pipeline_action/reload_spec.rb

@@ -22,7 +22,10 @@ describe LogStash::PipelineAction::Reload do
   end
 
   after do
-    pipelines.each { |_, pipeline| pipeline.shutdown }
+    pipelines.each do |_, pipeline| 
+      pipeline.shutdown
+      pipeline.thread.join
+    end
   end
 
   it "returns the pipeline_id" do

data/spec/logstash/pipeline_dlq_commit_spec.rb

@@ -67,7 +67,7 @@ describe LogStash::Pipeline do
   end
 
   after(:each) do
-    FileUtils.remove_entry pipeline_settings["path.dead_letter_queue"]
+    FileUtils.rm_rf(pipeline_settings["path.dead_letter_queue"])
   end
 
   context "dlq is enabled" do
@@ -85,6 +85,7 @@ describe LogStash::Pipeline do
       entry = dlq_reader.pollEntry(40)
       expect(entry).to_not be_nil
       expect(entry.reason).to eq("my reason")
+      subject.shutdown
     end
   end
 
@@ -101,6 +102,7 @@ describe LogStash::Pipeline do
       subject.run
       dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
       expect(java.nio.file.Files.exists(dlq_path)).to eq(false)
+      subject.shutdown
     end
   end
 

data/spec/logstash/pipeline_pq_file_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 require "spec_helper"
 require "logstash/inputs/generator"
-require "logstash/filters/multiline"
 require_relative "../support/helpers"
+require_relative "../support/mocks_classes"
 
 class PipelinePqFileOutput < LogStash::Outputs::Base
   config_name "pipelinepqfileoutput"
@@ -40,7 +40,7 @@ describe LogStash::Pipeline do
   let(:pipeline_settings_obj) { LogStash::SETTINGS.clone }
   let(:pipeline_id) { "main" }
 
-  let(:multiline_id) { "my-multiline" }
+  let(:dummy_id) { "my-dummyid" }
   let(:output_id) { "my-pipelinepqfileoutput" }
   let(:generator_id) { "my-generator" }
   let(:config) do
@@ -52,10 +52,8 @@ describe LogStash::Pipeline do
       }
     }
     filter {
-      multiline {
-        id => "#{multiline_id}"
-        pattern => "hello"
-        what => next
+      dummyfilter {
+        id => "#{dummy_id}"
       }
     }
     output {
@@ -98,7 +96,7 @@ describe LogStash::Pipeline do
     allow(PipelinePqFileOutput).to receive(:new).with(any_args).and_return(counting_output)
     allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
     allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
-    allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
+    allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(LogStash::Filters::DummyFilter)
     allow(LogStash::Plugin).to receive(:lookup).with("output", "pipelinepqfileoutput").and_return(PipelinePqFileOutput)
 
     pipeline_workers_setting = LogStash::SETTINGS.get_setting("pipeline.workers")

data/spec/logstash/pipeline_spec.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require "spec_helper"
 require "logstash/inputs/generator"
-require "logstash/filters/multiline"
+require "logstash/filters/drop"
 require_relative "../support/mocks_classes"
 require_relative "../support/helpers"
 require_relative "../logstash/pipeline_reporter_spec" # for DummyOutput class
@@ -254,6 +254,14 @@ describe LogStash::Pipeline do
           pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
           pipeline.close
         end
+
+        it "should log each filtered event if config.debug is set to true" do
+          pipeline_settings_obj.set("config.debug", true)
+          pipeline = mock_pipeline_from_string(test_config_with_filters, pipeline_settings_obj)
+          expect(logger).to receive(:debug).with(/filter received/, anything)
+          pipeline.filter_func([LogStash::Event.new])
+          pipeline.close
+        end
       end
 
       context "when there is no command line -w N set" do
@@ -411,19 +419,12 @@ describe LogStash::Pipeline do
     context "cancelled events should not propagate down the filters" do
       config <<-CONFIG
         filter {
-          multiline {
-           pattern => "hello"
-           what => next
-          }
-          multiline {
-           pattern => "hello"
-           what => next
-          }
+          drop {}
         }
       CONFIG
 
       sample("hello") do
-        expect(subject.get("message")).to eq("hello")
+        expect(subject).to eq(nil)
       end
     end
 
@@ -433,19 +434,10 @@ describe LogStash::Pipeline do
           clone {
             clones => ["clone1"]
           }
-          multiline {
-            pattern => "bar"
-            what => previous
-          }
         }
       CONFIG
-
       sample(["foo", "bar"]) do
-        expect(subject.size).to eq(2)
-        expect(subject[0].get("message")).to eq("foo\nbar")
-        expect(subject[0].get("type")).to be_nil
-        expect(subject[1].get("message")).to eq("foo\nbar")
-        expect(subject[1].get("type")).to eq("clone1")
+        expect(subject.size).to eq(4)
       end
     end
   end
@@ -634,9 +626,9 @@ describe LogStash::Pipeline do
 
     it "should handle evaluating different config" do
       expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
       expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
     end
   end
 
@@ -716,9 +708,9 @@ describe LogStash::Pipeline do
       # in the current instance and was returning an array containing nil values for
       # the match.
       expect(pipeline1.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
       expect(pipeline2.output_func(LogStash::Event.new)).not_to include(nil)
-      expect(pipeline1.filter_func(LogStash::Event.new)).not_to include(nil)
+      expect(pipeline1.filter_func([LogStash::Event.new])).not_to include(nil)
     end
   end
 
@@ -797,8 +789,8 @@ describe LogStash::Pipeline do
     let(:pipeline_settings) { { "pipeline.id" => pipeline_id } }
     let(:pipeline_id) { "main" }
     let(:number_of_events) { 420 }
-    let(:multiline_id) { "my-multiline" }
-    let(:multiline_id_other) { "my-multiline_other" }
+    let(:dummy_id) { "my-multiline" }
+    let(:dummy_id_other) { "my-multiline_other" }
     let(:dummy_output_id) { "my-dummyoutput" }
     let(:generator_id) { "my-generator" }
     let(:config) do
@@ -810,15 +802,11 @@ describe LogStash::Pipeline do
         }
       }
       filter {
-         multiline {
-              id => "#{multiline_id}"
-              pattern => "hello"
-              what => next
+          dummyfilter {
+              id => "#{dummy_id}"
           }
-          multiline {
-               id => "#{multiline_id_other}"
-               pattern => "hello"
-               what => next
+          dummyfilter {
+               id => "#{dummy_id_other}"
            }
       }
       output {
@@ -841,7 +829,7 @@ describe LogStash::Pipeline do
       allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(dummyoutput)
       allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
       allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
-      allow(LogStash::Plugin).to receive(:lookup).with("filter", "multiline").and_return(LogStash::Filters::Multiline)
+      allow(LogStash::Plugin).to receive(:lookup).with("filter", "dummyfilter").and_return(LogStash::Filters::DummyFilter)
       allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
 
       pipeline_thread
@@ -885,7 +873,7 @@ describe LogStash::Pipeline do
       end
 
       it "populates the filter metrics" do
-        [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
+        [dummy_id, dummy_id_other].map(&:to_sym).each do |id|
           [:in, :out].each do |metric_key|
             plugin_name = id.to_sym
             expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:events][metric_key].value).to eq(number_of_events)
@@ -907,9 +895,9 @@ describe LogStash::Pipeline do
       end
 
       it "populates the name of the filter plugin" do
-        [multiline_id, multiline_id_other].map(&:to_sym).each do |id|
+        [dummy_id, dummy_id_other].map(&:to_sym).each do |id|
           plugin_name = id.to_sym
-          expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:name].value).to eq(LogStash::Filters::Multiline.config_name)
+          expect(collected_metric[:stats][:pipelines][:main][:plugins][:filters][plugin_name][:name].value).to eq(LogStash::Filters::DummyFilter.config_name)
         end
       end
 

data/spec/logstash/runner_spec.rb

@@ -35,7 +35,7 @@ describe LogStash::Runner do
     allow(LogStash::Logging::Logger).to receive(:configure_logging) do |level, path|
       allow(logger).to receive(:level).and_return(level.to_sym)
     end
-
+    allow(LogStash::Logging::Logger).to receive(:reconfigure).with(any_args)
     # Make sure we don't start a real pipeline here.
     # because we cannot easily close the pipeline
     allow(LogStash::Agent).to receive(:new).with(any_args).and_return(agent)
@@ -43,10 +43,6 @@ describe LogStash::Runner do
     allow(agent).to receive(:shutdown)
   end
 
-  after :each do
-    LogStash::Logging::Logger::configure_logging("info")
-  end
-
   describe "argument precedence" do
     let(:config) { "input {} output {}" }
     let(:cli_args) { ["-e", config, "-w", "20"] }

data/spec/logstash/settings/modules_spec.rb

@@ -9,12 +9,23 @@ describe LogStash::Setting::Modules do
   describe "Modules.Cli" do
     subject { described_class.new("mycloudid", LogStash::Util::ModulesSettingArray, []) }
     context "when given an array of hashes that contains a password key" do
+      let(:secret) { 'some_secret'}
       it "should convert password Strings to Password" do
-        source = [{"var.kibana.password" => "some_secret"}]
+        source = [{"var.kibana.password" => secret}]
         setting = subject.set(source)
         expect(setting).to be_a(Array)
         expect(setting.__class__).to eq(LogStash::Util::ModulesSettingArray)
         expect(setting.first.fetch("var.kibana.password")).to be_a(LogStash::Util::Password)
+        expect(setting.first.fetch("var.kibana.password").value).to eq(secret)
+      end
+
+      it 'should not wrap values that are already passwords' do
+        source = [{"var.kibana.password" => LogStash::Util::Password.new(secret)}]
+        setting = subject.set(source)
+        expect(setting).to be_a(Array)
+        expect(setting.__class__).to eq(LogStash::Util::ModulesSettingArray)
+        expect(setting.first.fetch("var.kibana.password")).to be_a(LogStash::Util::Password)
+        expect(setting.first.fetch("var.kibana.password").value).to eq(secret)
       end
     end
   end
@@ -42,7 +53,7 @@ describe LogStash::Setting::Modules do
     context "when given a badly formatted encoded id" do
       it "should not raise an error" do
         encoded = Base64.urlsafe_encode64("foo$$bal")
-        expect { subject.set(encoded) }.to raise_error(ArgumentError, /Cloud Id, after decoding, is invalid. Format: '<part1>\$<part2>\$<part3>'/)
+        expect { subject.set(encoded) }.to raise_error(ArgumentError, "Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"foo$$bal\".")
       end
     end
 

data/spec/logstash/settings/writable_directory_spec.rb

@@ -3,17 +3,17 @@ require "spec_helper"
 require "logstash/settings"
 require "tmpdir"
 require "socket" # for UNIXSocket
+require "fileutils"
 
 describe LogStash::Setting::WritableDirectory do
-  let(:mode_rx) { 0555 }
   # linux is 108, Macos is 104, so use a safe value
   # Stud::Temporary.pathname, will exceed that size without adding anything
   let(:parent) { File.join(Dir.tmpdir, Time.now.to_f.to_s) }
   let(:path) { File.join(parent, "fancy") }
 
   before { Dir.mkdir(parent) }
-  after { Dir.exist?(path) && Dir.unlink(path) rescue nil }
-  after { Dir.unlink(parent) }
+  after { Dir.exist?(path) && FileUtils.rm_rf(path)}
+  after { FileUtils.rm_rf(parent) }
 
   shared_examples "failure" do
     before { subject.set(path) }
@@ -44,8 +44,9 @@ describe LogStash::Setting::WritableDirectory do
       end
 
       context "and the directory cannot be created" do
-        before { File.chmod(mode_rx, parent) }
         it "should fail" do
+          # using chmod does not work on Windows better mock and_raise("message")
+          expect(FileUtils).to receive(:mkdir_p).and_raise("foobar")
           expect { subject.value }.to raise_error
         end
       end
@@ -66,7 +67,8 @@ describe LogStash::Setting::WritableDirectory do
       end
 
       context "but is not writable" do
-        before { File.chmod(0, path) }
+        # chmod does not work on Windows, mock writable? instead
+        before { expect(File).to receive(:writable?).and_return(false) }
         it_behaves_like "failure"
       end
     end
@@ -84,12 +86,13 @@ describe LogStash::Setting::WritableDirectory do
         before { socket } # realize `socket` value
         after { socket.close }
         it_behaves_like "failure"
-      end
+      end unless LogStash::Environment.windows?
 
+      
       context "but is a symlink" do
-        before { File::symlink("whatever", path) }
+        before { FileUtils.symlink("whatever", path) }
         it_behaves_like "failure"
-      end
+      end unless LogStash::Environment.windows?
     end
 
     context "when the directory is missing" do
@@ -114,8 +117,8 @@ describe LogStash::Setting::WritableDirectory do
 
       context "and cannot be created" do
         before do
-          # Remove write permission on the parent
-          File.chmod(mode_rx, parent)
+          # chmod does not work on Windows, mock writable? instead
+          expect(File).to receive(:writable?).and_return(false)
         end
 
         it_behaves_like "failure"