logstash-core 5.4.3-java → 5.5.0-java

data/spec/logstash/filter_delegator_spec.rb

@@ -14,7 +14,8 @@ describe LogStash::FilterDelegator do
   let(:collector) { [] }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
   let(:events) { [LogStash::Event.new, LogStash::Event.new] }
-  let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
+  let(:default_execution_context) { LogStash::ExecutionContext.new(:main, filter_id, "filter",
+                                                                   LogStash::Util::DummyDeadLetterQueueWriter.new) }
 
   before :each do
     allow(metric).to receive(:namespace).with(anything).and_return(metric)

data/spec/logstash/inputs/base_spec.rb

@@ -64,7 +64,7 @@ describe "LogStash::Inputs::Base#decorate" do
   end
 
   context "execution context" do
-    let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
+    let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "id", "input", LogStash::Util::DummyDeadLetterQueueWriter.new) }
     let(:klass) { LogStash::Inputs::NOOP }
 
     subject(:instance) { klass.new({}) }

data/spec/logstash/output_delegator_spec.rb

@@ -9,7 +9,8 @@ describe LogStash::OutputDelegator do
   let(:plugin_args) { {"id" => "foo", "arg1" => "val1"} }
   let(:collector) { [] }
   let(:metric) { LogStash::Instrument::NamespacedNullMetric.new(collector, :null) }
-  let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
+  let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "foo", "output",
+                                                                   LogStash::Util::DummyDeadLetterQueueWriter.new) }
 
   subject { described_class.new(logger, out_klass, metric, default_execution_context, ::LogStash::OutputDelegatorStrategyRegistry.instance, plugin_args) }
 

data/spec/logstash/outputs/base_spec.rb

@@ -80,7 +80,7 @@ describe "LogStash::Outputs::Base#new" do
   end
 
   context "execution context" do
-    let(:default_execution_context) { LogStash::ExecutionContext.new(:main) }
+    let(:default_execution_context) { LogStash::ExecutionContext.new(:main, "id", "output", LogStash::Util::DummyDeadLetterQueueWriter.new) }
     let(:klass) { LogStash::Outputs::NOOPSingle }
 
     subject(:instance) { klass.new(params.dup) }

data/spec/logstash/pipeline_dlq_commit_spec.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "tmpdir"
+require "spec_helper"
+require "logstash/codecs/plain"
+require_relative "../support/mocks_classes"
+
+class SingleGeneratorInput < LogStash::Inputs::Base
+  config_name "singlegenerator"
+  milestone 2
+
+  def register
+  end
+
+  def run(queue)
+    queue << LogStash::Event.new
+  end
+
+  def close
+  end
+end
+
+class DLQCommittingFilter < LogStash::Filters::Base
+  config_name "dlq_commit"
+  milestone 2
+
+  def register()
+  end
+
+  def filter(event)
+    execution_context.dlq_writer.write(event, "my reason")
+  end
+
+  def threadsafe?() true; end
+
+  def close() end
+end
+
+describe LogStash::Pipeline do
+  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings) do
+    {
+      "pipeline.workers" => 2,
+      "pipeline.id" => pipeline_id,
+      "dead_letter_queue.enable" => enable_dlq,
+      "path.dead_letter_queue" => Dir.mktmpdir
+    }
+  end
+  let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
+  let(:test_config) {
+    <<-eos
+        input { singlegenerator { id => input_id } }
+
+        filter { dlq_commit { id => filter_id } }
+
+        output { dummyoutput { id => output_id } }
+    eos
+  }
+
+  subject { LogStash::Pipeline.new(test_config, pipeline_settings_obj, metric) }
+
+  before(:each) do
+    pipeline_settings.each {|k, v| pipeline_settings_obj.set(k, v) }
+    allow(LogStash::Plugin).to receive(:lookup).with("input", "singlegenerator").and_return(SingleGeneratorInput)
+    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain)
+    allow(LogStash::Plugin).to receive(:lookup).with("filter", "dlq_commit").and_return(DLQCommittingFilter)
+    allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(::LogStash::Outputs::DummyOutput)
+  end
+
+  after(:each) do
+    FileUtils.remove_entry pipeline_settings["path.dead_letter_queue"]
+  end
+
+  context "dlq is enabled" do
+    let(:enable_dlq) { true }
+    let(:pipeline_id) { "test-dlq" }
+
+    it "retrieves proper pipeline-level DLQ writer" do
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
+      expect_any_instance_of(org.logstash.common.io.DeadLetterQueueWriter).to receive(:close).and_call_original
+      subject.run
+      dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
+      dlq_reader = org.logstash.common.io.DeadLetterQueueReader.new(dlq_path)
+      entry = dlq_reader.pollEntry(40)
+      expect(entry).to_not be_nil
+      expect(entry.reason).to eq("my reason")
+    end
+  end
+
+  context "dlq is disabled" do
+    let(:enable_dlq) { false }
+    let(:pipeline_id) { "test-without-dlq" }
+
+    it "does not write to the DLQ" do
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "input_id", "singlegenerator").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "filter_id", "dlq_commit").and_call_original
+      expect(LogStash::Util::PluginDeadLetterQueueWriter).to receive(:new).with(anything, "output_id", "dummyoutput").and_call_original
+      expect(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:new).and_call_original
+      expect_any_instance_of(LogStash::Util::DummyDeadLetterQueueWriter).to receive(:close).and_call_original
+      subject.run
+      dlq_path = java.nio.file.Paths.get(pipeline_settings_obj.get("path.dead_letter_queue"), pipeline_id)
+      expect(java.nio.file.Files.exists(dlq_path)).to eq(false)
+    end
+  end
+
+end

data/spec/logstash/pipeline_pq_file_spec.rb

@@ -36,7 +36,7 @@ class PipelinePqFileOutput < LogStash::Outputs::Base
 end
 
 describe LogStash::Pipeline do
-  let(:pipeline_settings_obj) { LogStash::SETTINGS }
+  let(:pipeline_settings_obj) { LogStash::SETTINGS.clone }
   let(:pipeline_id) { "main" }
 
   let(:multiline_id) { "my-multiline" }

data/spec/logstash/plugin_spec.rb

@@ -48,7 +48,7 @@ describe LogStash::Plugin do
 
     it "can be set and get" do
       expect(subject.execution_context).to be_nil
-      execution_context = LogStash::ExecutionContext.new(:main)
+      execution_context = LogStash::ExecutionContext.new(:main, "id", "type", nil)
       subject.execution_context = execution_context
       expect(subject.execution_context).to eq(execution_context)
     end

data/spec/logstash/plugins/registry_spec.rb

@@ -53,13 +53,30 @@ describe LogStash::Plugins::Registry do
   end
 
   context "when loading plugin manually configured" do
-    it "should return the plugin" do
-      class SimplePlugin
-      end
+    let(:simple_plugin) { Class.new }
 
+    it "should return the plugin" do
       expect { registry.lookup("filter", "simple_plugin") }.to raise_error(LoadError)
-      registry.add(:filter, "simple_plugin", SimplePlugin)
-      expect(registry.lookup("filter", "simple_plugin")).to eq(SimplePlugin)
+      registry.add(:filter, "simple_plugin", simple_plugin)
+      expect(registry.lookup("filter", "simple_plugin")).to eq(simple_plugin)
+    end
+
+    it "doesn't add multiple time the same plugin" do
+      plugin1 = Class.new
+      plugin2 = Class.new
+
+      registry.add(:filter, "simple_plugin", plugin1)
+      registry.add(:filter, "simple_plugin", plugin2)
+
+      expect(registry.plugins_with_type(:filter)).to include(plugin1)
+      expect(registry.plugins_with_type(:filter).size).to eq(1)
+    end
+
+    it "allow you find plugin by type" do
+      registry.add(:filter, "simple_plugin", simple_plugin)
+
+      expect(registry.plugins_with_type(:filter)).to include(simple_plugin)
+      expect(registry.plugins_with_type(:modules)).to match([])
     end
   end
 end

data/spec/logstash/runner_spec.rb

@@ -6,7 +6,11 @@ require "stud/trap"
 require "stud/temporary"
 require "logstash/util/java_version"
 require "logstash/logging/json"
+require "logstash/config/modules_common"
+require "logstash/elasticsearch_client"
 require "json"
+require_relative "../support/helpers"
+require_relative "../support/matchers"
 
 class NullRunner
   def run(args); end
@@ -32,13 +36,6 @@ describe LogStash::Runner do
     end
   end
 
-  after :each do
-    LogStash::SETTINGS.reset
-  end
-
-  after :all do
-  end
-
   describe "argument precedence" do
     let(:config) { "input {} output {}" }
     let(:cli_args) { ["-e", config, "-w", "20"] }
@@ -48,10 +45,6 @@ describe LogStash::Runner do
       allow(LogStash::SETTINGS).to receive(:read_yaml).and_return(settings_yml_hash)
     end
 
-    after :each do
-      LogStash::SETTINGS.reset
-    end
-
     it "favors the last occurence of an option" do
       expect(LogStash::Agent).to receive(:new) do |settings|
         expect(settings.get("config.string")).to eq(config)
@@ -142,6 +135,7 @@ describe LogStash::Runner do
     context "with a good configuration" do
       let(:pipeline_string) { "input { } filter { } output { }" }
       it "should exit successfully" do
+        expect(logger).not_to receive(:fatal)
         expect(subject.run(args)).to eq(0)
       end
     end
@@ -165,38 +159,56 @@ describe LogStash::Runner do
       allow(pipeline).to receive(:run).and_return(task)
       allow(pipeline).to receive(:shutdown)
     end
-    
+
     context "when :path.data is defined by the user" do
       let(:test_data_path) { "/tmp/ls-test-data" }
       let(:test_queue_path) { test_data_path + "/" + "queue" }
-      
+      let(:test_dlq_path) { test_data_path + "/" + "dead_letter_queue" }
+
       it "should set data paths" do
         expect(LogStash::Agent).to receive(:new) do |settings|
           expect(settings.get("path.data")).to eq(test_data_path)
           expect(settings.get("path.queue")).to eq(test_queue_path)
+          expect(settings.get("path.dead_letter_queue")).to eq(test_dlq_path)
         end
-        
+
         args = ["--path.data", test_data_path, "-e", pipeline_string]
         subject.run("bin/logstash", args)
       end
-      
+
       context "and path.queue is manually set" do
         let(:queue_override_path) { "/tmp/queue-override_path" }
-        
+
         it "should set data paths" do
           expect(LogStash::Agent).to receive(:new) do |settings|
             expect(settings.get("path.data")).to eq(test_data_path)
             expect(settings.get("path.queue")).to eq(queue_override_path)
           end
-          
+
           LogStash::SETTINGS.set("path.queue", queue_override_path)
-          
+
+          args = ["--path.data", test_data_path, "-e", pipeline_string]
+          subject.run("bin/logstash", args)
+        end
+      end
+
+      context "and path.dead_letter_queue is manually set" do
+        let(:queue_override_path) { "/tmp/queue-override_path" }
+
+        it "should set data paths" do
+          expect(LogStash::Agent).to receive(:new) do |settings|
+            expect(settings.get("path.data")).to eq(test_data_path)
+            expect(settings.get("path.dead_letter_queue")).to eq(queue_override_path)
+          end
+
+          LogStash::SETTINGS.set("path.dead_letter_queue", queue_override_path)
+
           args = ["--path.data", test_data_path, "-e", pipeline_string]
           subject.run("bin/logstash", args)
         end
       end
     end
-    
+
     context "when :http.host is defined by the user" do
       it "should pass the value to the webserver" do
         expect(LogStash::Agent).to receive(:new) do |settings|
@@ -303,6 +315,97 @@ describe LogStash::Runner do
     end
   end
 
+  describe "logstash modules" do
+    describe "--config.test_and_exit" do
+      subject { LogStash::Runner.new("") }
+      let(:args) { ["-t", "--modules", module_string] }
+
+      context "with a good configuration" do
+        let(:module_string) { "cef" }
+        it "should exit successfully" do
+          skip("Skipped until cef module is added back to the codebase as explained in #7455")
+          expect(logger).not_to receive(:fatal)
+          expect(subject.run(args)).to eq(0)
+        end
+      end
+
+      context "with a bad configuration" do
+        let(:module_string) { "rlwekjhrewlqrkjh" }
+        it "should fail by returning a bad exit code" do
+          expect(logger).to receive(:fatal)
+          expect(subject.run(args)).to eq(1)
+        end
+      end
+    end
+
+    describe "--modules" do
+      let(:args) { ["--modules", module_string] }
+      let(:agent_instance) { double("agent") }
+      context "with an available module specified but no connection to elasticsearch" do
+        let(:module_string) { "cef" }
+        before do
+          expect(logger).to receive(:fatal) do |msg, hash|
+            expect(msg).to eq("An unexpected error occurred!")
+            expect(hash).to be_a_config_loading_error_hash(
+              /Failed to import module configurations to Elasticsearch. Module: cef/)
+          end
+        end
+        it "should log fatally and return a bad exit code" do
+          skip("Skipped until cef module is added back to the codebase as explained in #7455")
+          expect(subject.run("bin/logstash", args)).to eq(1)
+        end
+      end
+
+      context "with an available module specified and a mocked connection to elasticsearch" do
+        let(:module_string) { "cef" }
+        let(:client) { double(:client) }
+        let(:response) { double(:response) }
+        before do
+          allow(response).to receive(:status).and_return(404)
+          allow(client).to receive(:head).and_return(response)
+          allow(client).to receive(:can_connect?).and_return(true)
+          allow(agent_instance).to receive(:register_pipeline)
+          allow(agent_instance).to receive(:execute)
+          allow(agent_instance).to receive(:shutdown)
+          allow(LogStash::ElasticsearchClient).to receive(:build).and_return(client)
+
+          expect(client).to receive(:put).at_least(15).times do |path, content|
+            LogStash::ElasticsearchClient::Response.new(201, "", {})
+          end
+          expect(LogStash::Agent).to receive(:new) do |settings|
+            pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
+            expect(pipelines).not_to be_empty
+            cef_pipeline = pipelines.first
+            expect(cef_pipeline).to include("pipeline_id", "config_string")
+            expect(cef_pipeline["pipeline_id"]).to include('cef')
+            expect(cef_pipeline["config_string"]).to include('index => "cef-')
+            agent_instance
+          end
+          expect(logger).not_to receive(:fatal)
+          expect(logger).not_to receive(:error)
+        end
+        it "should not terminate logstash" do
+          skip("Skipped until cef module is added back to the codebase as explained in #7455")
+          expect(subject.run("bin/logstash", args)).to be_nil
+        end
+      end
+
+      context "with an unavailable module specified" do
+        let(:module_string) { "fancypants" }
+        before do
+          expect(logger).to receive(:fatal) do |msg, hash|
+            expect(msg).to eq("An unexpected error occurred!")
+            expect(hash).to be_a_config_loading_error_hash(
+              /The modules specified are not available yet. Specified modules: \["fancypants"\] Available modules:/)
+          end
+        end
+        it "should log fatally and return a bad exit code" do
+          expect(subject.run("bin/logstash", args)).to eq(1)
+        end
+      end
+    end
+  end
+
   describe "--log.level" do
     before :each do
       allow_any_instance_of(subject).to receive(:show_version)

data/spec/logstash/settings_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "spec_helper"
+require "logstash/util/environment_variables"
 require "logstash/settings"
 require "fileutils"
 
@@ -146,4 +147,94 @@ describe LogStash::Settings do
       end
     end
   end
+
+  describe "#from_yaml" do
+
+    context "env placeholders in flat logstash.yml" do
+
+      after do
+        ENV.delete('SOME_LOGSTASH_SPEC_ENV_VAR')
+        ENV.delete('some.logstash.spec.env.var')
+      end
+      
+      subject do
+        settings = described_class.new
+        settings.register(LogStash::Setting::String.new("interpolated", "missing"))
+        settings.register(LogStash::Setting::String.new("with_dot", "missing"))
+        settings
+      end
+
+      let(:values) {{
+        "interpolated" => "${SOME_LOGSTASH_SPEC_ENV_VAR}",
+        "with_dot" => "${some.logstash.spec.env.var}"
+      }}
+      let(:yaml_path) do
+        p = Stud::Temporary.pathname
+        FileUtils.mkdir_p(p)
+
+        ::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
+          f.write(YAML.dump(values))
+        end
+        p
+      end
+
+      it "can interpolate environment into settings" do
+        expect(subject.get('interpolated')).to eq("missing")
+        expect(subject.get('with_dot')).to eq("missing")
+        ENV['SOME_LOGSTASH_SPEC_ENV_VAR'] = "correct_setting"
+        ENV['some.logstash.spec.env.var'] = "correct_setting_for_dotted"
+        subject.from_yaml(yaml_path)
+        expect(subject.get('interpolated')).to eq("correct_setting")
+        expect(subject.get('with_dot')).to eq("correct_setting_for_dotted")
+      end
+    end
+  end
+
+  context "env placeholders in nested logstash.yml" do
+
+    before do
+      ENV['lsspecdomain'] = "domain1"
+      ENV['lsspecdomain2'] = "domain2"
+    end
+
+    after do
+      ENV.delete('lsspecdomain')
+      ENV.delete('lsspecdomain2')
+    end
+
+    subject do
+      settings = described_class.new
+      settings.register(LogStash::Setting::ArrayCoercible.new("host", String, []))
+      settings.register(LogStash::Setting::ArrayCoercible.new("modules", Hash, []))
+      settings
+    end
+
+    let(:values) {{
+      "host" => ["dev1.${lsspecdomain}", "dev2.${lsspecdomain}"],
+      "modules" => [
+        {"name" => "${lsspecdomain}", "testing" => "${lsspecdomain}"}, 
+        {"name" => "${lsspecdomain2}", "testing" => "${lsspecdomain2}"}
+      ]
+    }}
+    let(:yaml_path) do
+      p = Stud::Temporary.pathname
+      FileUtils.mkdir_p(p)
+
+      ::File.open(::File.join(p, "logstash.yml"), "w+") do |f|
+        f.write(YAML.dump(values))
+      end
+      p
+    end
+
+    it "can interpolate environment into settings" do
+      expect(subject.get('host')).to match_array([])
+      expect(subject.get('modules')).to match_array([])
+      subject.from_yaml(yaml_path)
+      expect(subject.get('host')).to match_array(["dev1.domain1", "dev2.domain1"])
+      expect(subject.get('modules')).to match_array([
+                                                      {"name" => "domain1", "testing" => "domain1"},
+                                                      {"name" => "domain2", "testing" => "domain2"}
+                                                    ])
+    end
+  end
 end