logstash-output-elasticsearch 10.8.2-java → 11.0.1-java

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -1,5 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
   let(:logger) { Cabin::Channel.get }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -1,6 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "json"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
@@ -9,7 +9,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
   let(:oss) { true }
-  let(:valid_license) { true }
+  let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
@@ -26,8 +26,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
-    allow(subject.license_checker).to receive(:oss?).and_return(oss)
-    allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
+    allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
   after do
@@ -68,7 +67,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
       context "and setting healthcheck_path" do
         let(:healthcheck_path) { "/my/health" }
-        let(:options) { super.merge(:healthcheck_path => healthcheck_path) }
+        let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
         it "performs the healthcheck to the healthcheck_path" do
           expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
             expect(method).to eq(:head)
@@ -130,7 +129,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "when enabled" do
-      let(:options) { super.merge(:sniffing => true)}
+      let(:options) { super().merge(:sniffing => true)}
 
       it "should start the sniffer" do
         expect(subject.sniffer_alive?).to eql(true)
@@ -247,7 +246,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     let(:options) do
-      super.merge(:license_checker => license_checker)
+      super().merge(:license_checker => license_checker)
     end
 
     context 'when LicenseChecker#acceptable_license? returns false' do
@@ -283,21 +282,21 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       let(:oss) { false }
 
       context "if ES doesn't return a valid license" do
-        let(:valid_license) { false }
+        let(:license_status) { nil }
 
-        it "marks the url as active" do
+        it "marks the url as dead" do
           subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
+          expect(subject.alive_urls_count).to eq(0)
         end
 
         it "logs a warning" do
-          expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
+          expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
           subject.update_initial_urls
         end
       end
 
       context "if ES returns a valid license" do
-        let(:valid_license) { true }
+        let(:license_status) { 'active' }
 
         it "marks the url as active" do
           subject.update_initial_urls
@@ -305,7 +304,22 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         end
 
         it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
+          expect(subject.license_checker).to_not receive(:warn_no_license)
+          expect(subject.license_checker).to_not receive(:warn_invalid_license)
+          subject.update_initial_urls
+        end
+      end
+
+      context "if ES returns an invalid license" do
+        let(:license_status) { 'invalid' }
+
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+
+        it "logs a warning" do
+          expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
           subject.update_initial_urls
         end
       end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,7 +1,6 @@
-require_relative "../../../../spec/es_spec_helper"
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "java"
+require "cabin"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
   let(:ssl) { nil }
@@ -48,7 +47,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       describe "ssl" do
         context "when SSL is true" do
           let(:ssl) { true }
-          let(:base_options) { super.merge(:hosts => [http_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [http_hostname_port]) }
 
           it "should refuse to handle an http url" do
             expect {
@@ -59,7 +58,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
         context "when SSL is false" do
           let(:ssl) { false }
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           
           it "should refuse to handle an https url" do
             expect {
@@ -69,7 +68,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         end
 
         describe "ssl is nil" do
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           it "should handle an ssl url correctly when SSL is nil" do
             subject
             expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
@@ -79,14 +78,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
       describe "path" do
         let(:url) { http_hostname_port_path }
-        let(:base_options) { super.merge(:hosts => [url]) }
+        let(:base_options) { super().merge(:hosts => [url]) }
         
         it "should allow paths in a url" do
           expect(subject.host_to_url(url)).to eq(url)
         end
 
         context "with the path option set" do
-          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "/otherpath"}) }
 
           it "should not allow paths in two places" do
             expect {
@@ -97,7 +96,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         
         context "with a path missing a leading /" do
           let(:url) { http_hostname_port }
-          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "otherpath"}) }
           
           
           it "should automatically insert a / in front of path overlays" do
@@ -204,7 +203,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
   end
 
   describe "#bulk" do
-    subject { described_class.new(base_options) }
+    subject(:http_client) { described_class.new(base_options) }
 
     require "json"
     let(:message) { "hey" }
@@ -212,42 +211,61 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
     ]}
 
-    context "if a message is over TARGET_BULK_BYTES" do
-      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-      let(:message) { "a" * (target_bulk_bytes + 1) }
+    [true,false].each do |http_compression_enabled|
+      context "with `http_compression => #{http_compression_enabled}`" do
 
-      it "should be handled properly" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once do |data|
-          expect(data.size).to be > target_bulk_bytes
+        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+
+        before(:each) do
+          if http_compression_enabled
+            expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
+          else
+            expect(http_client).to_not receive(:gzip_writer)
+          end
         end
-        s = subject.send(:bulk, actions)
-      end
-    end
 
-    context "with two messages" do
-      let(:message1) { "hey" }
-      let(:message2) { "you" }
-      let(:actions) { [
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
-      ]}
-      it "executes one bulk_send operation" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once
-        s = subject.send(:bulk, actions)
-      end
+        context "if a message is over TARGET_BULK_BYTES" do
+          let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+          let(:message) { "a" * (target_bulk_bytes + 1) }
+
+          it "should be handled properly" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once do |data|
+              if !http_compression_enabled
+                expect(data.size).to be > target_bulk_bytes
+              else
+                expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes
+              end
+            end
+            s = subject.send(:bulk, actions)
+          end
+        end
+
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
+          let(:actions) { [
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+          ]}
+          it "executes one bulk_send operation" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once
+            s = subject.send(:bulk, actions)
+          end
 
-      context "if one exceeds TARGET_BULK_BYTES" do
-        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-        let(:message1) { "a" * (target_bulk_bytes + 1) }
-        it "executes two bulk_send operations" do
-          allow(subject).to receive(:join_bulk_responses)
-          expect(subject).to receive(:bulk_send).twice
-          s = subject.send(:bulk, actions)
+          context "if one exceeds TARGET_BULK_BYTES" do
+            let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            it "executes two bulk_send operations" do
+              allow(subject).to receive(:join_bulk_responses)
+              expect(subject).to receive(:bulk_send).twice
+              s = subject.send(:bulk, actions)
+            end
+          end
         end
-      end
-    end
+       end
+     end
   end
 
   describe "sniffing" do

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,7 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/elasticsearch/http_client"
-require "java"
-require "json"
+require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -1,6 +1,5 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require 'stud/temporary'
-require "logstash/outputs/elasticsearch"
 require 'manticore/client'
 
 describe "Proxy option" do
@@ -24,7 +23,7 @@ describe "Proxy option" do
 
     context "when specified as a URI" do
       shared_examples("hash conversion") do |hash|
-        let(:settings) { super.merge("proxy" => proxy)}
+        let(:settings) { super().merge("proxy" => proxy)}
         
         it "should set the proxy to the correct hash value" do
           expect(::Manticore::Client).to have_received(:new) do |options|
@@ -71,7 +70,7 @@ describe "Proxy option" do
   end
 
   context "when specified as ''" do
-    let(:settings) { super.merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
+    let(:settings) { super().merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
 
     it "should not send the proxy option to manticore" do
       expect { subject.register }.not_to raise_error
@@ -85,7 +84,7 @@ describe "Proxy option" do
   end
 
   context "when specified as invalid uri" do
-    let(:settings) { super.merge("proxy" => ":")}
+    let(:settings) { super().merge("proxy" => ":")}
 
     it "should fail" do
       # SafeURI isn't doing the proper exception wrapping for us, we can not simply :

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,27 +1,40 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require "base64"
 require "flores/random"
+require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
 describe LogStash::Outputs::ElasticSearch do
-  subject { described_class.new(options) }
+  subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
   let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
 
   let(:do_register) { true }
 
+  let(:stub_http_client_pool!) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
+  end
+
+  let(:after_successful_connection_thread_mock) do
+    double('after_successful_connection_thread', value: true)
+  end
+
   before(:each) do
     if do_register
-      # Build the client and set mocks before calling register to avoid races.
-      subject.build_client
+      stub_http_client_pool!
+
+      allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
+
+      # emulate 'successful' ES connection on the same thread
+      allow(subject).to receive(:after_successful_connection) { |&block| block.call }.
+          and_return after_successful_connection_thread_mock
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+
+      subject.register
 
-      # Rspec mocks can't handle background threads, so... we can't use any
-      allow(subject.client.pool).to receive(:start_resurrectionist)
-      allow(subject.client.pool).to receive(:start_sniffer)
-      allow(subject.client.pool).to receive(:healthcheck!)
       allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
       allow(subject.client).to receive(:get_xpack_info)
-      subject.register
+
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end
@@ -44,9 +57,15 @@ describe LogStash::Outputs::ElasticSearch do
     let(:manticore_urls) { subject.client.pool.urls }
     let(:manticore_url) { manticore_urls.first }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     describe "getting a document type" do
       context "if document_type isn't set" do
-        let(:options) { super.merge("document_type" => nil)}
+        let(:options) { super().merge("document_type" => nil)}
         context "for 7.x elasticsearch clusters" do
           let(:maximum_seen_major_version) { 7 }
           it "should return '_doc'" do
@@ -70,7 +89,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
       context "with 'document type set'" do
-        let(:options) { super.merge("document_type" => "bar")}
+        let(:options) { super().merge("document_type" => "bar")}
         it "should get the event type from the 'document_type' setting" do
           expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
@@ -80,14 +99,14 @@ describe LogStash::Outputs::ElasticSearch do
     describe "building an event action tuple" do
       context "for 7.x elasticsearch clusters" do
         let(:maximum_seen_major_version) { 7 }
-        it "should include '_type'" do
+        it "should not include '_type' when 'document_type' is not explicitly defined" do
           action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
           action_params = action_tuple[1]
-          expect(action_params).to include(:_type => "_doc")
+          expect(action_params).not_to include(:_type => "_doc")
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should get the event type from the 'document_type' setting" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -105,7 +124,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should not include '_type'" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -127,7 +146,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as part of a URL" do
         let(:options) {
-          super.merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
+          super().merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
         }
 
         include_examples("an authenticated config")
@@ -135,7 +154,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as a hash option" do
           let(:options) {
-            super.merge!(
+            super().merge!(
               "user" => user,
               "password" => password
             )
@@ -175,7 +194,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "with extra slashes" do
         let(:path) { "/slashed-path/ "}
-        let(:options) { super.merge("path" => "/some-path/") }
+        let(:options) { super().merge("path" => "/some-path/") }
 
         it "should properly set the path on the HTTP client without adding slashes" do
           expect(manticore_url.path).to eql(options["path"])
@@ -234,13 +253,13 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     describe "without a port specified" do
-      let(:options) { super.merge('hosts' => 'localhost') }
+      let(:options) { super().merge('hosts' => 'localhost') }
       it "should properly set the default port (9200) on the HTTP client" do
         expect(manticore_url.port).to eql(9200)
       end
     end
     describe "with a port other than 9200 specified" do
-      let(:options) { super.merge('hosts' => 'localhost:9202') }
+      let(:options) { super().merge('hosts' => 'localhost:9202') }
       it "should properly set the specified port on the HTTP client" do
         expect(manticore_url.port).to eql(9202)
       end
@@ -253,8 +272,7 @@ describe LogStash::Outputs::ElasticSearch do
       before do
         allow(subject).to receive(:retrying_submit).with(anything)
         events.each_with_index do |e,i|
-          et = events_tuples[i]
-          allow(subject).to receive(:event_action_tuple).with(e).and_return(et)
+          allow(subject).to receive(:event_action_tuple).with(e).and_return(events_tuples[i])
         end
         subject.multi_receive(events)
       end
@@ -265,12 +283,14 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { ::LogStash::Event.new("foo" => "bar") }
       let(:error) do
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-          429, double("url").as_null_object, double("request body"), double("response body")
+          429, double("url").as_null_object, request_body, double("response body")
         )
       end
       let(:logger) { double("logger").as_null_object }
       let(:response) { { :errors => [], :items => [] } }
 
+      let(:request_body) { double(:request_body, :bytesize => 1023) }
+
       before(:each) do
 
         i = 0
@@ -296,6 +316,95 @@ describe LogStash::Outputs::ElasticSearch do
         expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
       end
     end
+
+    context "unexpected bulk response" do
+      let(:options) do
+        { "hosts" => "127.0.0.1:9999", "index" => "%{foo}", "manage_template" => false }
+      end
+
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:bulk_response) do
+        # shouldn't really happen but we've seen this happen - here ES returns more items than were sent
+        { "took"=>1, "ingest_took"=>9, "errors"=>true,
+          "items"=>[{"index"=>{"_index"=>"bar1", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                              "error"=>{"type" => "illegal_state_exception",
+                                      "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]"
+                                      }
+                              }
+                    },
+                    # NOTE: this is an artificial success (usually everything fails with a 500) but even if some doc where
+                    # to succeed due the unexpected reponse items we can not clearly identify which actions to retry ...
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>201}},
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                               "error"=>{"type" => "illegal_state_exception",
+                                        "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]];"
+                                        }
+                              }
+                    }]
+        }
+      end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
+          expect( stream.string ).to include '"foo":"bar1"'
+          expect( stream.string ).to include '"foo":"bar2"'
+        end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
+      end
+
+      it "should retry submit" do
+        allow(subject.logger).to receive(:error).with(/Encountered an unexpected error/i, anything)
+        allow(subject.client).to receive(:bulk).and_call_original # track count
+
+        subject.multi_receive(events)
+
+        expect(subject.client).to have_received(:bulk).twice
+      end
+
+      it "should log specific error message" do
+        expect(subject.logger).to receive(:error).with(/Encountered an unexpected error/i,
+                                                       hash_including(:message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
+
+        subject.multi_receive(events)
+      end
+    end
+  end
+
+  context '413 errors' do
+    let(:payload_size) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES + 1024 }
+    let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
+
+    let(:logger_stub) { double("logger").as_null_object }
+
+    before(:each) do
+      allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
+
+      allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
+
+      max_bytes = payload_size * 3 / 4 # ensure a failure first attempt
+      allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
+        if body.length > max_bytes
+          max_bytes *= 2 # ensure a successful retry
+          double("Response", :code => 413, :body => "")
+        else
+          double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
+        end
+      end
+    end
+
+    it 'retries the 413 until it goes away' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(elasticsearch_output_instance.client).to have_received(:bulk).twice
+    end
+
+    it 'logs about payload quantity and size' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(logger_stub).to have_received(:warn)
+                                 .with(a_string_matching(/413 Payload Too Large/),
+                                       hash_including(:action_count => 1, :content_length => a_value > 20_000_000))
+    end
   end
 
   context "with timeout set" do
@@ -311,7 +420,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     before do
       # Expect a timeout to be logged.
-      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
+      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request/i, anything).at_least(:once)
       expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original
     end
 
@@ -325,13 +434,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the action option" do
+
     context "with a sprintf action" do
       let(:options) { {"action" => "%{myactionfield}" } }
 
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        expect(subject.event_action_tuple(event).first).to eql("update")
+        expect(subject.send(:event_action_tuple, event).first).to eql("update")
       end
     end
 
@@ -341,7 +451,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should obtain specific action's params from event_action_tuple" do
-        expect(subject.event_action_tuple(event)[1]).to include(:_upsert)
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:_upsert)
       end
     end
 
@@ -349,6 +459,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { {"action" => "SOME Garbaaage"} }
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
 
+      before { allow(subject).to receive(:finish_register) }
+
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error(LogStash::ConfigurationError)
       end
@@ -356,13 +468,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the pipeline option" do
+
     context "with a sprintf and set pipeline" do
       let(:options) { {"pipeline" => "%{pipeline}" } }
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
       it "should interpolate the pipeline value and set it" do
-        expect(subject.event_action_tuple(event)[1]).to include(:pipeline => "my-ingest-pipeline")
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
 
@@ -372,7 +485,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
       it "should interpolate the pipeline value but not set it because it is empty" do
-        expect(subject.event_action_tuple(event)[1]).not_to include(:pipeline)
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end
   end
@@ -410,30 +523,30 @@ describe LogStash::Outputs::ElasticSearch do
     let(:options) { { 'retry_on_conflict' => num_retries } }
 
     context "with a regular index" do
-      let(:options) { super.merge("action" => "index") }
+      let(:options) { super().merge("action" => "index") }
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
         allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).not_to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).not_to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
     context "using a plain update" do
-      let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
     context "with a sprintf action that resolves to update" do
-      let(:options) { super.merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
         expect(action).to eq("update")
       end
     end
@@ -463,6 +576,8 @@ describe LogStash::Outputs::ElasticSearch do
     let(:do_register) { false }
 
     before :each do
+      allow(subject).to receive(:finish_register)
+
       allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
     end
 
@@ -486,6 +601,12 @@ describe LogStash::Outputs::ElasticSearch do
     let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
     let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     context "using non-url hosts" do
 
       let(:options) {
@@ -650,9 +771,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_status("Could not index event to Elasticsearch.",
               [:action, :params, :event], :some_status, mock_response)
@@ -661,9 +782,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_status("Could not index event to Elasticsearch.",
@@ -683,16 +804,59 @@ describe LogStash::Outputs::ElasticSearch do
       # We should still log when sending to the DLQ.
       # This shall be solved by another issue, however: logstash-output-elasticsearch#772
       it 'should send the event to the DLQ instead, and not log' do
-        expect(dlq_writer).to receive(:write).once.with(:event, /Could not index/)
+        event = LogStash::Event.new("foo" => "bar")
+        expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-        subject.handle_dlq_status("Could not index event to Elasticsearch.",
-          [:action, :params, :event], :some_status, mock_response)
+        action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
+        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
+
+    context 'with response status 400' do
+
+      let(:options) { super().merge 'document_id' => '%{foo}' }
+
+      let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+      let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+      let(:bulk_response) do
+        {
+            "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+            [{
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                           "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
+                  }
+            }]
+        }
+      end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+      end
+
+      it "should write event to DLQ" do
+        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+          expect( args.size ).to eql 2
+
+          event, reason = *args
+          expect( event ).to be_a LogStash::Event
+          expect( event ).to be events.first
+          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
+          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+
+          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+        end.once
+
+        event_action_tuples = subject.map_events(events)
+        subject.send(:submit, event_action_tuples)
+      end
+
+    end if LOGSTASH_VERSION > '7.0'
   end
 
   describe "custom headers" do
-    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) } 
+    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
 
     context "when set" do
       let(:headers) { { "X-Thing" => "Test" } }
@@ -765,6 +929,75 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "post-register ES setup" do
+    let(:do_register) { false }
+    let(:es_version) { '7.10.0' } # DS default on LS 8.x
+    let(:options) { { 'hosts' => '127.0.0.1:9999' } }
+    let(:logger) { subject.logger }
+
+    before do
+      allow(logger).to receive(:error) # expect tracking
+
+      allow(subject).to receive(:last_es_version).and_return es_version
+      # make successful_connection? return true:
+      allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+    end
+
+    it "logs inability to retrieve uuid" do
+      allow(subject).to receive(:install_template)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Unable to retrieve Elasticsearch cluster uuid/i, anything)
+    end if LOGSTASH_VERSION >= '7.0.0'
+
+    it "logs template install failure" do
+      allow(subject).to receive(:discover_cluster_uuid)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Failed to install template/i, anything)
+    end
+
+    context 'error raised' do
+
+      let(:es_version) { '7.8.0' }
+      let(:options) { super().merge('data_stream' => 'true') }
+      let(:latch) { Concurrent::CountDownLatch.new }
+
+      before do
+        allow(subject).to receive(:install_template)
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:ilm_in_use?).and_return nil
+        # executes from the after_successful_connection thread :
+        allow(subject).to receive(:finish_register) { latch.wait }.and_call_original
+        subject.register
+      end
+
+      it 'keeps logging on multi_receive' do
+        allow(subject).to receive(:retrying_submit)
+        latch.count_down; sleep(1.0)
+
+        expect_logged_error = lambda do |count|
+          expect(logger).to have_received(:error).with(
+              /Elasticsearch setup did not complete normally, please review previously logged errors/i,
+              hash_including(message:  'A data_stream configuration is only supported since Elasticsearch 7.9.0 (detected version 7.8.0), please upgrade your cluster')
+          ).exactly(count).times
+        end
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 1) ]
+        expect_logged_error.call(1)
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 2) ]
+        expect_logged_error.call(2)
+      end
+
+    end
+  end
+
   @private
 
   def stub_manticore_client!(manticore_double = nil)