logstash-output-elasticsearch 11.12.4-java → 11.15.9-java

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -50,6 +50,143 @@ describe LogStash::Outputs::ElasticSearch do
     subject.close
   end
 
+  context "check aborting of a batch" do
+    context "on an unreachable ES instance" do
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:shutdown_value) { true }
+
+      let(:logger) { double("logger") }
+
+      let(:never_ending) { Thread.new { sleep 1 while true } }
+
+      let(:do_register) { false }
+
+      before(:each) do
+        spy_http_client_builder!
+        stub_http_client_pool!
+
+        allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
+
+        # emulate 'failed' ES connection, which sleeps forever
+        allow(subject).to receive(:after_successful_connection) { |&block| never_ending }
+        allow(subject).to receive(:stop_after_successful_connection_thread)
+
+        subject.register
+
+        allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
+        allow(subject.client).to receive(:get_xpack_info)
+
+        subject.client.pool.adapter.manticore.respond_with(:body => "{}")
+
+        allow(subject).to receive(:logger).and_return(logger)
+        allow(logger).to receive(:info)
+
+        allow(subject).to receive(:pipeline_shutdown_requested?) do
+          shutdown_value
+        end
+      end
+
+      it "the #multi_receive abort while waiting on unreachable and a shutdown is requested" do
+        expect { subject.multi_receive(events) }.to raise_error(org.logstash.execution.AbortedBatchException)
+        expect(logger).to have_received(:info).with(/Aborting the batch due to shutdown request while waiting for connections to become live/i)
+      end
+    end
+
+    context "on a reachable ES instance" do
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:logger) { double("logger") }
+
+      before(:each) do
+        allow(subject).to receive(:logger).and_return(logger)
+        allow(logger).to receive(:info)
+
+        allow(subject).to receive(:pipeline_shutdown_requested?).and_return(true)
+        allow(subject).to receive(:retrying_submit)
+      end
+
+      it "the #multi_receive doesn't abort when waiting for a connection on alive ES and a shutdown is requested" do
+        subject.multi_receive(events)
+        expect(logger).to_not have_received(:info).with(/Aborting the batch due to shutdown request while waiting for connections to become live/i)
+      end
+    end
+
+    context "when a connected ES becomes unreachable" do
+#       let(:error) do
+#                 ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+#                   429, double("url").as_null_object, request_body, double("response body")
+#                 )
+#               end
+
+      shared_examples 'raise an abort error' do
+        let(:options) {
+          {
+            "index" => "my-index",
+            "hosts" => ["localhost","localhost:9202"],
+            "path" => "some-path",
+            "manage_template" => false
+          }
+        }
+
+        let(:manticore_urls) { subject.client.pool.urls }
+        let(:manticore_url) { manticore_urls.first }
+
+        let(:stub_http_client_pool!) do
+          [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+            allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+          end
+        end
+
+        let(:event) { ::LogStash::Event.new("foo" => "bar") }
+
+        let(:logger) { double("logger").as_null_object }
+        let(:response) { { :errors => [], :items => [] } }
+
+        let(:request_body) { double(:request_body, :bytesize => 1023) }
+
+        before(:each) do
+          bulk_param =  [["index", anything, event.to_hash]]
+
+          allow(subject).to receive(:logger).and_return(logger)
+
+          # fail consistently for ever
+          allow(subject.client).to receive(:bulk).with(bulk_param).and_raise(error)
+        end
+
+        it "should exit the retry with an abort exception if shutdown is requested" do
+          # trigger the shutdown signal
+          allow(subject).to receive(:pipeline_shutdown_requested?) { true }
+
+          # execute in another thread because it blocks in a retry loop until the shutdown is triggered
+          th = Thread.new do
+            subject.multi_receive([event])
+          rescue org.logstash.execution.AbortedBatchException => e
+            # return exception's class so that it can be verified when retrieving the thread's value
+            e.class
+          end
+
+          expect(th.value).to eq(org.logstash.execution.AbortedBatchException)
+        end
+      end
+
+      context "with 429 error" do
+        let(:error) do
+          ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+            429, double("url").as_null_object, request_body, double("response body")
+          )
+        end
+
+        it_behaves_like 'raise an abort error'
+      end
+
+      context "with 'no connections' error" do
+        let(:error) { ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError.new }
+
+        it_behaves_like 'raise an abort error'
+      end
+    end
+  end if LOGSTASH_VERSION >= '8.8'
 
   context "with an active instance" do
     let(:options) {
@@ -590,7 +727,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
-      it "should interpolate the pipeline value and set it" do
+      it "interpolate the pipeline value and set it" do
         expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
@@ -600,7 +737,66 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
-      it "should interpolate the pipeline value but not set it because it is empty" do
+      it "interpolates the pipeline value but not set it because it is empty" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with both pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "my-ingest-pipeline", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the plugin's pipeline value" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
+      end
+
+      context "when the plugin's `pipeline` is constant" do
+        let(:options) { super().merge("pipeline" => "my-constant-pipeline") }
+         it "uses plugin's pipeline value" do
+          expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-constant-pipeline")
+        end
+      end
+
+      context "when the plugin's `pipeline` includes an unresolvable sprintf placeholder" do
+        let(:options) { super().merge("pipeline" => "reference-%{unset}-field") }
+        it "does not use the target_ingest_pipeline" do
+          # when sprintf doesn't resolve a placeholder, the behaviour of our `pipeline` is UNSPECIFIED.
+          # here we only validate that the presence of the magic field does not
+          # override an explicitly-configured pipeline.
+          expect(subject.send(:event_action_tuple, event)[1]).to_not include(:pipeline => "my-ingest-pipeline")
+        end
+      end
+    end
+
+    context "with empty pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the pipeline value but not set it because pipeline is empty" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"target_ingest_pipeline" => "meta-ingest-pipeline"}}) }
+
+      it "interpolates the target_ingest_pipeline value and set it" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "meta-ingest-pipeline")
+      end
+    end
+
+    context "with empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"host" => "elastic"}}) }
+
+      it "does not set pipeline" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with empty pipeline and empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new }
+
+      it "does not set pipeline" do
         expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end
@@ -640,9 +836,8 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-
-    context "With the 'ssl' option" do
-      let(:options) { {"ssl" => true}}
+    context "With the 'ssl_enabled' option" do
+      let(:options) { {"ssl_enabled" => true}}
 
       include_examples("an encrypted client connection")
     end
@@ -653,6 +848,81 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "SSL deprecated settings" do
+    let(:base_options) { {"ssl" => "true"} }
+
+    context "with client certificate" do
+      let(:do_register) { true }
+      let(:cacert) { Stud::Temporary.file.path }
+      let(:options) { base_options.merge(
+        "cacert" => cacert,
+        "ssl_certificate_verification" => false
+      ) }
+
+      after :each do
+        File.delete(cacert)
+      end
+
+      it "should map new configs into params" do
+        expect(subject.params).to match hash_including(
+                                          "ssl_enabled" => true,
+                                          "ssl_verification_mode" => "none",
+                                          "ssl_certificate_authorities" => [cacert]
+                                        )
+      end
+
+      it "should set new configs variables" do
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
+        expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
+      end
+    end
+
+    context "with java stores" do
+      let(:do_register) { true }
+      let(:keystore) { Stud::Temporary.file.path }
+      let(:truststore) { Stud::Temporary.file.path }
+      let(:options) { base_options.merge(
+        "keystore" => keystore,
+        "keystore_password" => "keystore",
+        "truststore" => truststore,
+        "truststore_password" => "truststore",
+        "ssl_certificate_verification" => true
+      ) }
+
+      let(:spy_http_client_builder!) do
+        allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
+        allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
+      end
+
+      after :each do
+        File.delete(keystore)
+        File.delete(truststore)
+      end
+
+      it "should map new configs into params" do
+        expect(subject.params).to match hash_including(
+                                          "ssl_enabled" => true,
+                                          "ssl_keystore_path" => keystore,
+                                          "ssl_truststore_path" => truststore,
+                                          "ssl_verification_mode" => "full"
+                                        )
+
+        expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
+        expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
+      end
+
+      it "should set new configs variables" do
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
+        expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
+        expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
+        expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
+        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
+      end
+    end
+  end
+
   describe "retry_on_conflict" do
     let(:num_retries) { 123 }
     let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
@@ -890,41 +1160,63 @@ describe LogStash::Outputs::ElasticSearch do
   end if LOGSTASH_VERSION > '6.0'
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
+    let(:es_api_action) { "CUSTOM_ACTION" }
+    let(:es_api_params) { Hash['_index' => 'MY_INDEX'] }
+
     let(:options) { { "manage_template" => false, "data_stream" => 'false' } }
-    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
+    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(es_api_action, es_api_params, LogStash::Event.new("foo" => "bar")) }
+
+    let(:logger) { double('logger').as_null_object }
+    before(:each) { subject.instance_variable_set(:@logger, logger) }
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
-      let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
       context 'resorting to previous behaviour of logging the error' do
         context 'getting an invalid_index_name_exception' do
           it 'should log at ERROR level' do
-            subject.instance_variable_set(:@logger, double("logger").as_null_object)
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:error).with(a_string_including("Could not index event to Elasticsearch"),
+                                                         a_hash_including(:status => :some_status,
+                                                                          :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                          :response => mock_response))
           end
         end
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
             end.to_not raise_error
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
       end
@@ -944,6 +1236,8 @@ describe LogStash::Outputs::ElasticSearch do
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
         action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
         subject.handle_dlq_response("Could not index event to Elasticsearch.", action, 404, mock_response)
+
+        expect(logger).to_not have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"))
       end
     end
 
@@ -1034,12 +1328,12 @@ describe LogStash::Outputs::ElasticSearch do
       it 'adds the appropriate Authorization header to the manticore client' do
         expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
       end
-      it 'is provides ssl=>true to the http client builder' do; aggregate_failures do
-        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl'=>true))
+      it 'is provides ssl_enabled=>true to the http client builder' do; aggregate_failures do
+        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl_enabled'=>true))
       end; end
     end
 
-    context "when set without ssl => true" do
+    context "when set without ssl_enabled => true" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "api_key" => api_key } }
 
@@ -1055,14 +1349,14 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context "when set without ssl specified but with an https host" do
+    context "when set without ssl_enabled specified but with an https host" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["https://some.host.com"], "api_key" => api_key } }
 
       it_behaves_like 'secure api-key authenticated client'
     end
 
-    context "when set without ssl specified but with an http host`" do
+    context "when set without ssl_enabled specified but with an http host`" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
 
@@ -1071,9 +1365,9 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context "when set with `ssl => false`" do
+    context "when set with `ssl_enabled => false`" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => "false", "api_key" => api_key } }
+      let(:options) { { "ssl_enabled" => "false", "api_key" => api_key } }
 
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
@@ -1083,13 +1377,13 @@ describe LogStash::Outputs::ElasticSearch do
     context "when set" do
       let(:options) { { "api_key" => ::LogStash::Util::Password.new(api_key) } }
 
-      context "with ssl => true" do
-        let(:options) { super().merge("ssl" => true) }
+      context "with ssl_enabled => true" do
+        let(:options) { super().merge("ssl_enabled" => true) }
         it_behaves_like 'secure api-key authenticated client'
       end
 
-      context "with ssl => false" do
-        let(:options) { super().merge("ssl" => "false") }
+      context "with ssl_enabled => false" do
+        let(:options) { super().merge("ssl_enabled" => "false") }
 
         let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
         it "should raise a configuration error" do
@@ -1097,7 +1391,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
       end
 
-      context "without ssl specified" do
+      context "without ssl_enabled specified" do
         context "with an https host" do
           let(:options) { super().merge("hosts" => ["https://some.host.com"]) }
           it_behaves_like 'secure api-key authenticated client'
@@ -1121,7 +1415,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context 'user also set' do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => true, "api_key" => api_key, 'user' => 'another' } }
+      let(:options) { { "ssl_enabled" => true, "api_key" => api_key, 'user' => 'another' } }
 
       it "should fail" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -1130,7 +1424,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context 'cloud_auth also set' do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => true, "api_key" => api_key, 'cloud_auth' => 'foobar' } }
+      let(:options) { { "ssl_enabled" => true, "api_key" => api_key, 'cloud_auth' => 'foobar' } }
 
       it "should fail" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -1225,6 +1519,42 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
     end
+
+    context 'during register/finish_register' do
+
+      let(:options) { { 'hosts' => '127.0.0.1:9999', 'data_stream' => 'true' } }
+      let(:es_version) { '8.7.0' } # DS default on LS 8.x
+
+      before do
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:maybe_create_rollover_alias)
+        allow(subject).to receive(:maybe_create_ilm_policy)
+        allow(subject).to receive(:build_client)
+      end
+
+      # this test could not have been done using latches as the fix to the race
+      # condition alters the order of the instructions in elasticsearch.rb
+      #
+      # the race condition happened when the @index was set to the datastream
+      # after `ilm_in_use?` is called but before `setup_ilm`
+      it 'doesn\'t have a race condition leading to resetting back to ILM' do
+        ilm_in_use = subject.method(:ilm_in_use?)
+        expect(subject).to receive(:ilm_in_use?) do |params|
+          ret = ilm_in_use.call
+          sleep 3
+          ret
+        end
+        setup_mapper_and_target = subject.method(:setup_mapper_and_target)
+        expect(subject).to receive(:setup_mapper_and_target).once do |params|
+          sleep 1
+          setup_mapper_and_target.call(params)
+        end
+        subject.register
+        sleep 6
+        expect(subject.index).to eq("logs-generic-default")
+      end
+
+    end
   end
 
   @private