logstash-core 6.0.0.alpha1-java → 6.0.0.alpha2-java

data/spec/logstash/runner_spec.rb

@@ -43,10 +43,6 @@ describe LogStash::Runner do
     allow(agent).to receive(:shutdown)
   end
 
-  after :each do
-    LogStash::SETTINGS.reset
-  end
-
   describe "argument precedence" do
     let(:config) { "input {} output {}" }
     let(:cli_args) { ["-e", config, "-w", "20"] }
@@ -56,10 +52,6 @@ describe LogStash::Runner do
       allow(LogStash::SETTINGS).to receive(:read_yaml).and_return(settings_yml_hash)
     end
 
-    after :each do
-      LogStash::SETTINGS.reset
-    end
-
     it "favors the last occurence of an option" do
       expect(LogStash::Agent).to receive(:new) do |settings|
         expect(settings.get("config.string")).to eq(config)
@@ -87,21 +79,6 @@ describe LogStash::Runner do
         subject.run(args)
       end
     end
-
-    context "with no arguments" do
-      let(:args) { [] }
-
-      before(:each) do
-        allow(LogStash::Util::JavaVersion).to receive(:warn_on_bad_java_version)
-      end
-
-      it "should show help" do
-        expect($stderr).to receive(:puts).once
-        expect(subject).to receive(:signal_usage_error).once.and_call_original
-        expect(subject).to receive(:show_short_help).once
-        subject.run(args)
-      end
-    end
   end
 
   context "--pluginpath" do
@@ -130,7 +107,7 @@ describe LogStash::Runner do
 
   context "--auto-reload" do
     subject { LogStash::Runner.new("") }
-    context "when -f is not given" do
+    context "when -e is given" do
 
       let(:args) { ["-r", "-e", "input {} output {}"] }
 
@@ -143,10 +120,6 @@ describe LogStash::Runner do
   end
 
   describe "--config.test_and_exit" do
-    before do
-      # Reset the source in a clean state before any asserts
-      LogStash::Config::SOURCE_LOADER.configure_sources([])
-    end
     subject { LogStash::Runner.new("") }
     let(:args) { ["-t", "-e", pipeline_string] }
 
@@ -176,38 +149,56 @@ describe LogStash::Runner do
       allow(pipeline).to receive(:run).and_return(task)
       allow(pipeline).to receive(:shutdown)
     end
-    
+
     context "when :path.data is defined by the user" do
       let(:test_data_path) { "/tmp/ls-test-data" }
       let(:test_queue_path) { test_data_path + "/" + "queue" }
-      
+      let(:test_dlq_path) { test_data_path + "/" + "dead_letter_queue" }
+
       it "should set data paths" do
         expect(LogStash::Agent).to receive(:new) do |settings|
           expect(settings.get("path.data")).to eq(test_data_path)
           expect(settings.get("path.queue")).to eq(test_queue_path)
+          expect(settings.get("path.dead_letter_queue")).to eq(test_dlq_path)
         end
-        
+
         args = ["--path.data", test_data_path, "-e", pipeline_string]
         subject.run("bin/logstash", args)
       end
-      
+
       context "and path.queue is manually set" do
         let(:queue_override_path) { "/tmp/queue-override_path" }
-        
+
         it "should set data paths" do
           expect(LogStash::Agent).to receive(:new) do |settings|
             expect(settings.get("path.data")).to eq(test_data_path)
             expect(settings.get("path.queue")).to eq(queue_override_path)
           end
-          
+
           LogStash::SETTINGS.set("path.queue", queue_override_path)
-          
+
+          args = ["--path.data", test_data_path, "-e", pipeline_string]
+          subject.run("bin/logstash", args)
+        end
+      end
+
+      context "and path.dead_letter_queue is manually set" do
+        let(:queue_override_path) { "/tmp/queue-override_path" }
+
+        it "should set data paths" do
+          expect(LogStash::Agent).to receive(:new) do |settings|
+            expect(settings.get("path.data")).to eq(test_data_path)
+            expect(settings.get("path.dead_letter_queue")).to eq(queue_override_path)
+          end
+
+          LogStash::SETTINGS.set("path.dead_letter_queue", queue_override_path)
+
           args = ["--path.data", test_data_path, "-e", pipeline_string]
           subject.run("bin/logstash", args)
         end
       end
     end
-    
+
     context "when :http.host is defined by the user" do
       it "should pass the value to the webserver" do
         expect(LogStash::Agent).to receive(:new) do |settings|

data/spec/logstash/settings/array_coercible_spec.rb

@@ -43,4 +43,69 @@ describe LogStash::Setting::ArrayCoercible do
       end
     end
   end
+
+  describe "#==" do
+    context "when comparing two settings" do
+      let(:setting_1) { described_class.new("option_1", element_class_1, value_1) }
+      let(:element_class_1) { String }
+      let(:setting_2) { described_class.new("option_1", element_class_2, value_2) }
+      let(:element_class_2) { String }
+
+      context "where one was given a non array value" do
+        let(:value_1) { "a string" }
+        context "and the other also the same non array value" do
+          let(:value_2) { "a string" }
+          it "should be equal" do
+            expect(setting_1).to be == setting_2
+          end
+        end
+        context "and the other also the same value in an array" do
+          let(:value_2) { [ "a string" ] }
+          it "should be equal" do
+            expect(setting_1).to be == setting_2
+          end
+        end
+        context "and the other a different non array value" do
+          let(:value_2) { "a different string" }
+          it "should be equal" do
+            expect(setting_1).to_not be == setting_2
+          end
+        end
+        context "and the other a different value in an array" do
+          let(:value_2) { [ "a different string" ] }
+          it "should be equal" do
+            expect(setting_1).to_not be == setting_2
+          end
+        end
+      end
+
+      context "where one was given a value in an array" do
+        let(:value_1) { [ "a string"] }
+        context "and the other the same value in an array" do
+          let(:value_2) { [ "a string" ] }
+          it "should be equal" do
+            expect(setting_1).to be == setting_2
+          end
+        end
+        context "and the other the same value not in an array" do
+          let(:value_2) { "a string" }
+          it "should be equal" do
+            expect(setting_1).to be == setting_2
+          end
+        end
+        context "and the other a different value in an array" do
+          let(:value_2) { [ "a different string" ] }
+          it "should be equal" do
+            expect(setting_1).to_not be == setting_2
+          end
+        end
+        context "and the other a different value in an array" do
+          let(:value_2) { "a different string" }
+          it "should be equal" do
+            expect(setting_1).to_not be == setting_2
+          end
+        end
+      end
+    end
+  end
 end

data/spec/logstash/settings_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/util/environment_variables"
 require "logstash/settings"
 require "fileutils"
 
@@ -146,4 +147,94 @@ describe LogStash::Settings do
       end
     end
   end
+
+  describe "#from_yaml" do
+
+    context "env placeholders in flat logstash.yml" do
+
+      after do
+        ENV.delete('SOME_LOGSTASH_SPEC_ENV_VAR')
+        ENV.delete('some.logstash.spec.env.var')
+      end
+      
+      subject do
+        settings = described_class.new
+        settings.register(LogStash::Setting::String.new("interpolated", "missing"))
+        settings.register(LogStash::Setting::String.new("with_dot", "missing"))
+        settings
+      end
+
+      let(:values) {{
+        "interpolated" => "${SOME_LOGSTASH_SPEC_ENV_VAR}",
+        "with_dot" => "${some.logstash.spec.env.var}"
+      }}
+      let(:yaml_path) do
+        p = Stud::Temporary.pathname
+        FileUtils.mkdir_p(p)
+
+        ::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
+          f.write(YAML.dump(values))
+        end
+        p
+      end
+
+      it "can interpolate environment into settings" do
+        expect(subject.get('interpolated')).to eq("missing")
+        expect(subject.get('with_dot')).to eq("missing")
+        ENV['SOME_LOGSTASH_SPEC_ENV_VAR'] = "correct_setting"
+        ENV['some.logstash.spec.env.var'] = "correct_setting_for_dotted"
+        subject.from_yaml(yaml_path)
+        expect(subject.get('interpolated')).to eq("correct_setting")
+        expect(subject.get('with_dot')).to eq("correct_setting_for_dotted")
+      end
+    end
+  end
+
+  context "env placeholders in nested logstash.yml" do
+
+    before do
+      ENV['lsspecdomain'] = "domain1"
+      ENV['lsspecdomain2'] = "domain2"
+    end
+
+    after do
+      ENV.delete('lsspecdomain')
+      ENV.delete('lsspecdomain2')
+    end
+
+    subject do
+      settings = described_class.new
+      settings.register(LogStash::Setting::ArrayCoercible.new("host", String, []))
+      settings.register(LogStash::Setting::ArrayCoercible.new("modules", Hash, []))
+      settings
+    end
+
+    let(:values) {{
+      "host" => ["dev1.${lsspecdomain}", "dev2.${lsspecdomain}"],
+      "modules" => [
+        {"name" => "${lsspecdomain}", "testing" => "${lsspecdomain}"}, 
+        {"name" => "${lsspecdomain2}", "testing" => "${lsspecdomain2}"}
+      ]
+    }}
+    let(:yaml_path) do
+      p = Stud::Temporary.pathname
+      FileUtils.mkdir_p(p)
+
+      ::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
+        f.write(YAML.dump(values))
+      end
+      p
+    end
+
+    it "can interpolate environment into settings" do
+      expect(subject.get('host')).to match_array([])
+      expect(subject.get('modules')).to match_array([])
+      subject.from_yaml(yaml_path)
+      expect(subject.get('host')).to match_array(["dev1.domain1", "dev2.domain1"])
+      expect(subject.get('modules')).to match_array([
+                                                      {"name" => "domain1", "testing" => "domain1"},
+                                                      {"name" => "domain2", "testing" => "domain2"}
+                                                    ])
+    end
+  end
 end

data/spec/logstash/shutdown_watcher_spec.rb

@@ -9,22 +9,12 @@ describe LogStash::ShutdownWatcher do
   let(:pipeline) { double("pipeline") }
   let(:reporter) { double("reporter") }
   let(:reporter_snapshot) { double("reporter snapshot") }
-  report_count = 0
 
   before :each do
     allow(pipeline).to receive(:reporter).and_return(reporter)
     allow(pipeline).to receive(:thread).and_return(Thread.current)
     allow(reporter).to receive(:snapshot).and_return(reporter_snapshot)
     allow(reporter_snapshot).to receive(:o_simple_hash).and_return({})
-
-    allow(subject).to receive(:pipeline_report_snapshot).and_wrap_original do |m, *args|
-      report_count += 1
-      m.call(*args)
-    end
-  end
-
-  after :each do
-    report_count = 0
   end
 
   context "when pipeline is stalled" do
@@ -69,8 +59,9 @@ describe LogStash::ShutdownWatcher do
       it "shouldn't force the shutdown" do
         expect(subject).to_not receive(:force_exit)
         thread = Thread.new(subject) {|subject| subject.start }
-        sleep 0.1 until report_count > check_threshold
-        thread.kill
+        sleep 0.1 until subject.attempts_count > check_threshold
+        subject.stop!
+        expect(thread.join(60)).to_not be_nil
       end
     end
   end
@@ -91,8 +82,9 @@ describe LogStash::ShutdownWatcher do
       it "should force the shutdown" do
         expect(subject).to_not receive(:force_exit)
         thread = Thread.new(subject) {|subject| subject.start }
-        sleep 0.1 until report_count > check_threshold
-        thread.kill
+        sleep 0.1 until subject.attempts_count > check_threshold
+        subject.stop!
+        expect(thread.join(60)).to_not be_nil
       end
     end
 
@@ -105,8 +97,10 @@ describe LogStash::ShutdownWatcher do
       it "shouldn't force the shutdown" do
         expect(subject).to_not receive(:force_exit)
         thread = Thread.new(subject) {|subject| subject.start }
-        sleep 0.1 until report_count > check_threshold
-        thread.kill
+        sleep 0.1 until subject.attempts_count > check_threshold
+        subject.stop!
+        thread.join
+        expect(thread.join(60)).to_not be_nil
       end
     end
   end

data/spec/logstash/state_resolver_spec.rb

@@ -46,10 +46,12 @@ describe LogStash::StateResolver do
 
   context "when some pipeline are running" do
     context "when a pipeline is running" do
-      let(:running_pipelines) { { :main => mock_pipeline(:main) } }
+      let(:main_pipeline) { mock_pipeline(:main) }
+      let(:main_pipeline_config) { main_pipeline.pipeline_config }
+      let(:running_pipelines) { { :main => main_pipeline } }
 
       context "when the pipeline config contains a new one and the existing" do
-        let(:pipeline_configs) { [mock_pipeline_config(:hello_world), mock_pipeline_config(:main)] }
+        let(:pipeline_configs) { [mock_pipeline_config(:hello_world), main_pipeline_config ] }
 
         it "creates the new one and keep the other one" do
           expect(subject.resolve(running_pipelines, pipeline_configs)).to have_actions(
@@ -105,7 +107,7 @@ describe LogStash::StateResolver do
       context "without system pipeline" do
         let(:pipeline_configs) do
           [
-            mock_pipeline_config(:main1),
+            running_pipelines[:main1].pipeline_config,
             mock_pipeline_config(:main9),
             mock_pipeline_config(:main5, "input { generator {}}"),
             mock_pipeline_config(:main3, "input { generator {}}"),
@@ -129,7 +131,7 @@ describe LogStash::StateResolver do
       context "with system pipeline" do
         let(:pipeline_configs) do
           [
-            mock_pipeline_config(:main1),
+            running_pipelines[:main1].pipeline_config,
             mock_pipeline_config(:main9),
             mock_pipeline_config(:main5, "input { generator {}}"),
             mock_pipeline_config(:main3, "input { generator {}}"),

data/spec/support/helpers.rb

@@ -22,7 +22,7 @@ def clear_data_dir
   end
 end
 
-def mock_settings(settings_values)
+def mock_settings(settings_values={})
   settings = LogStash::SETTINGS.clone
 
   settings_values.each do |key, value|
@@ -32,13 +32,26 @@ def mock_settings(settings_values)
   settings
 end
 
+def make_test_agent(settings=mock_settings)
+    sl = LogStash::Config::SourceLoader.new
+    sl.add_source(LogStash::Config::Source::Local.new(settings))
+    sl
+
+    ::LogStash::Agent.new(settings, sl)
+end
+
 def mock_pipeline(pipeline_id, reloadable = true, config_hash = nil)
   config_string = "input { stdin { id => '#{pipeline_id}' }}"
   settings = mock_settings("pipeline.id" => pipeline_id.to_s,
                            "config.string" => config_string,
                            "config.reload.automatic" => reloadable)
-  pipeline = LogStash::Pipeline.new(config_string, settings)
-  pipeline
+  pipeline_config = mock_pipeline_config(pipeline_id, config_string, settings)
+  LogStash::Pipeline.new(pipeline_config)
+end
+
+def mock_pipeline_from_string(config_string, settings = LogStash::SETTINGS, metric = nil)
+  pipeline_config = mock_pipeline_config(settings.get("pipeline.id"), config_string, settings)
+  LogStash::Pipeline.new(pipeline_config, metric)
 end
 
 def mock_pipeline_config(pipeline_id, config_string = nil, settings = {})

data/spec/support/shared_contexts.rb

@@ -2,12 +2,36 @@ shared_context "execution_context" do
   let(:pipeline) { double("pipeline") }
   let(:pipeline_id) { :main }
   let(:agent) { double("agent") }
+  let(:plugin_id) { :plugin_id }
+  let(:plugin_type) { :plugin_type }
+  let(:dlq_writer) { double("dlq_writer") }
   let(:execution_context) do
-    ::LogStash::ExecutionContext.new(pipeline, agent)
+    ::LogStash::ExecutionContext.new(pipeline, agent, plugin_id, plugin_type, dlq_writer)
   end
-  
+
   before do
     allow(pipeline).to receive(:pipeline_id).and_return(pipeline_id)
     allow(pipeline).to receive(:agent).and_return(agent)
   end
 end
+
+shared_context "api setup" do
+  before :all do
+    clear_data_dir
+    settings = mock_settings
+    config_string = "input { generator {id => 'api-generator-pipeline' count => 100 } } output { dummyoutput {} }"
+    settings.set("config.string", config_string)
+    @agent = make_test_agent(settings)
+    @agent.execute
+  end
+
+  after :all do
+    @agent.shutdown
+  end
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class.new(nil, @agent)
+  end
+end