logstash-output-elasticsearch 10.8.4-java → 10.8.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bbaed62e36b6543bf68420603a64e84f39a61e3ffaaf3456f28fad73295430c
-  data.tar.gz: '0682c7ac69384d7db244e33186237461c148683cc205b72660251e5d3041765c'
+  metadata.gz: 69557d21ffe4079cabafcf86949f41d85cb6781f8898cebdc54b354117333b6b
+  data.tar.gz: a65b40a961335837f9ccff55472c0aeef033c5248cdcc579ffa98c6560fa377c
 SHA512:
-  metadata.gz: 8a546bff1b7623d8e37fd11b9551691ff87072831f81b00af1c4bfd8086e8f7ee45fad2b78682f68927e97d883dfc923a90f76bc53f648885fcd3d5d3c437028
-  data.tar.gz: 4ea4dc460aa38935cd4e72d2c35c069fb372d02c99fa5cf47e48dbce13994d3615296a3601c8a6c6641f6db587ec0776bc04fcd7b7902c32f31c8c6e95eb4340
+  metadata.gz: e8be38c81c89f8dca5dad83c79106180967cb5ed6806ed4a0ce97db1296a15bd8a462da80ef4a663807648164ac410d3d57fc46b2412ef497b1f9d0a4d7b57c6
+  data.tar.gz: 1843e98054e65374fe4b72c5938b0d808fecca799294783893d75c955977ec0d34020cd688ec7a16e4e22bf8c6c2b9e53343e9bfd4dd0cbccee10e601d0b2e0f

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 10.8.6
+ - Fixed an issue where a single over-size event being rejected by Elasticsearch would cause the entire entire batch to be retried indefinitely. The oversize event will still be retried on its own and logging has been improved to include payload sizes in this situation [#972](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/972)
+ - Fixed an issue with `http_compression => true` where a well-compressed payload could fit under our outbound 20MB limit but expand beyond Elasticsearch's 100MB limit, causing bulk failures. Bulk grouping is now determined entirely by the decompressed payload size [#823](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/823)
+ - Improved debug-level logging about bulk requests.
+
+## 10.8.5
+ - Feat: assert returned item count from _bulk [#997](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/997)
+
 ## 10.8.4
  - Fixed an issue where a retried request would drop "update" parameters [#800](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/800)
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -109,27 +109,50 @@ module LogStash; module Outputs; class ElasticSearch;
       body_stream = StringIO.new
       if http_compression
         body_stream.set_encoding "BINARY"
-        stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-      else 
+        stream_writer = gzip_writer(body_stream)
+      else
         stream_writer = body_stream
       end
       bulk_responses = []
-      bulk_actions.each do |action|
+      batch_actions = []
+      bulk_actions.each_with_index do |action, index|
         as_json = action.is_a?(Array) ?
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
-          bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
+        if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
+          stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
+          logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
+                       :action_count => batch_actions.size,
+                       :payload_size => stream_writer.pos,
+                       :content_length => body_stream.size,
+                       :batch_offset => (index + 1 - batch_actions.size))
+          bulk_responses << bulk_send(body_stream, batch_actions)
+          body_stream.truncate(0) && body_stream.seek(0)
+          stream_writer = gzip_writer(body_stream) if http_compression
+          batch_actions.clear
         end
         stream_writer.write(as_json)
+        batch_actions << action
       end
       stream_writer.close if http_compression
-      bulk_responses << bulk_send(body_stream) if body_stream.size > 0
+      logger.debug("Sending final bulk request for batch.",
+                   :action_count => batch_actions.size,
+                   :payload_size => stream_writer.pos,
+                   :content_length => body_stream.size,
+                   :batch_offset => (actions.size - batch_actions.size))
+      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end
 
+    def gzip_writer(io)
+      fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
+      fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
+
+      Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+    end
+
     def join_bulk_responses(bulk_responses)
       {
         "errors" => bulk_responses.any? {|r| r["errors"] == true},
@@ -137,25 +160,37 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream)
+    def bulk_send(body_stream, batch_actions)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
-      # Discard the URL
       response = @pool.post(@bulk_path, params, body_stream.string)
-      if !body_stream.closed?
-        body_stream.truncate(0)
-        body_stream.seek(0)
-      end
 
       @bulk_response_metrics.increment(response.code.to_s)
 
-      if response.code != 200
+      case response.code
+      when 200 # OK
+        LogStash::Json.load(response.body)
+      when 413 # Payload Too Large
+        logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
+        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
+      else
         url = ::LogStash::Util::SafeURI.new(response.final_url)
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           response.code, url, body_stream.to_s, response.body
         )
       end
+    end
 
-      LogStash::Json.load(response.body)
+    def emulate_batch_error_response(actions, http_code, reason)
+      {
+          "errors" => true,
+          "items" => actions.map do |action|
+            action = action.first if action.is_a?(Array)
+            request_action, request_parameters = action.first
+            {
+                request_action => {"status" => http_code, "error" => { "type" => reason }}
+            }
+          end
+      }
     end
 
     def get(path)

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -204,8 +204,16 @@ module LogStash; module PluginMixins; module ElasticSearch
         return
       end
 
+      responses = bulk_response["items"]
+      if responses.size != actions.size # can not map action -> response reliably
+        # an ES bug (on 7.10.2, 7.11.1) where a _bulk request to index X documents would return Y (> X) items
+        msg = "Sent #{actions.size} documents but Elasticsearch returned #{responses.size} responses"
+        @logger.warn(msg, actions: actions, responses: responses)
+        fail("#{msg} (likely a bug with _bulk endpoint)")
+      end
+
       actions_to_retry = []
-      bulk_response["items"].each_with_index do |response,idx|
+      responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
         status = action_props["status"]
@@ -291,7 +299,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
-        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
         message = "Encountered a retryable error. Will Retry with exponential backoff "
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.8.4'
+  s.version         = '10.8.6'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/integration/outputs/ilm_spec.rb

@@ -5,7 +5,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy with a maximum number of documents' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
       put_policy(@es, ilm_policy_name, policy)
@@ -54,7 +54,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy where the maximum number of documents is not reached' do
     let (:policy) { large_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom-policy"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should ingest into a single index when max docs is not reached' do
       put_policy(@es,ilm_policy_name, policy)
@@ -119,7 +119,7 @@ shared_examples_for 'an ILM disabled Logstash' do
   context 'with an existing policy that will roll over' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-3_docs"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should not roll over indices' do
       subject.register
@@ -155,7 +155,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
   context 'with a custom template name' do
     let (:template_name) { "logstash_custom_template_name" }
-    let (:settings) { super.merge('template_name' => template_name)}
+    let (:settings) { super().merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
@@ -195,7 +195,7 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
   context 'when ilm is enabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => true }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
 
     it 'should raise a configuration error' do
       expect do
@@ -210,13 +210,13 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   end
 
   context 'when ilm is disabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => false }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => false }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
 
   context 'when ilm is set to auto in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => 'auto' }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => 'auto' }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
@@ -286,7 +286,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using the default policy' do
         context 'with a custom pattern' do
-          let (:settings) { super.merge("ilm_pattern" => "000001")}
+          let (:settings) { super().merge("ilm_pattern" => "000001")}
           it 'should create a rollover alias' do
             expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
             subject.register
@@ -346,7 +346,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"logstash-policy-small"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { small_max_doc_policy }
 
         before do
@@ -363,7 +363,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using a time based policy' do
         let (:ilm_policy_name) {"logstash-policy-time"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { max_age_policy("1d") }
 
         before do
@@ -409,7 +409,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
         end
 
-        let (:settings) { super.merge("template" => template,
+        let (:settings) { super().merge("template" => template,
                                       "index" => "overwrite-4")}
 
         it 'should not overwrite the index patterns' do
@@ -426,7 +426,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
                                       "template" => template,
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
@@ -480,7 +480,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
         context 'with a different template_name' do
           let (:template_name) { "logstash_custom_template_name" }
-          let (:settings) { super.merge('template_name' => template_name)}
+          let (:settings) { super().merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
 
@@ -514,7 +514,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'when ilm_enabled is the default' do
-      let (:settings) { super.tap{|x|x.delete('ilm_enabled')}}
+      let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
 
       if ESHelper.es_version_satisfies?(">=7.0")
         context 'when Elasticsearch is version 7 or above' do
@@ -530,13 +530,13 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'with ilm disabled' do
-      let (:settings) { super.merge('ilm_enabled' => false )}
+      let (:settings) { super().merge('ilm_enabled' => false )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end
 
     context 'with ilm disabled using a string' do
-      let (:settings) { super.merge('ilm_enabled' => 'false' )}
+      let (:settings) { super().merge('ilm_enabled' => 'false' )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end

data/spec/unit/http_client_builder_spec.rb

@@ -40,10 +40,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
-        let(:options) { super.merge("bulk_path" => bulk_path) }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq(bulk_path)
@@ -66,7 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets bulk_path to path+_bulk" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq("#{path}/_bulk")
@@ -88,10 +88,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }
-        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(healthcheck_path)
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets healthcheck_path to path" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(path)
@@ -136,10 +136,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "sniffing_path" do
       context "when setting sniffing_path" do
         let(:sniffing_path) { "/meh" }
-        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq(sniffing_path)
@@ -162,7 +162,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets sniffing_path to path+_nodes/http" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -68,7 +68,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
       context "and setting healthcheck_path" do
         let(:healthcheck_path) { "/my/health" }
-        let(:options) { super.merge(:healthcheck_path => healthcheck_path) }
+        let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
         it "performs the healthcheck to the healthcheck_path" do
           expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
             expect(method).to eq(:head)
@@ -130,7 +130,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "when enabled" do
-      let(:options) { super.merge(:sniffing => true)}
+      let(:options) { super().merge(:sniffing => true)}
 
       it "should start the sniffer" do
         expect(subject.sniffer_alive?).to eql(true)
@@ -247,7 +247,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     let(:options) do
-      super.merge(:license_checker => license_checker)
+      super().merge(:license_checker => license_checker)
     end
 
     context 'when LicenseChecker#acceptable_license? returns false' do

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -48,7 +48,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       describe "ssl" do
         context "when SSL is true" do
           let(:ssl) { true }
-          let(:base_options) { super.merge(:hosts => [http_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [http_hostname_port]) }
 
           it "should refuse to handle an http url" do
             expect {
@@ -59,7 +59,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
         context "when SSL is false" do
           let(:ssl) { false }
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           
           it "should refuse to handle an https url" do
             expect {
@@ -69,7 +69,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         end
 
         describe "ssl is nil" do
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           it "should handle an ssl url correctly when SSL is nil" do
             subject
             expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
@@ -79,14 +79,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
       describe "path" do
         let(:url) { http_hostname_port_path }
-        let(:base_options) { super.merge(:hosts => [url]) }
+        let(:base_options) { super().merge(:hosts => [url]) }
         
         it "should allow paths in a url" do
           expect(subject.host_to_url(url)).to eq(url)
         end
 
         context "with the path option set" do
-          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "/otherpath"}) }
 
           it "should not allow paths in two places" do
             expect {
@@ -97,7 +97,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         
         context "with a path missing a leading /" do
           let(:url) { http_hostname_port }
-          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "otherpath"}) }
           
           
           it "should automatically insert a / in front of path overlays" do
@@ -204,7 +204,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
   end
 
   describe "#bulk" do
-    subject { described_class.new(base_options) }
+    subject(:http_client) { described_class.new(base_options) }
 
     require "json"
     let(:message) { "hey" }
@@ -212,42 +212,61 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
     ]}
 
-    context "if a message is over TARGET_BULK_BYTES" do
-      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-      let(:message) { "a" * (target_bulk_bytes + 1) }
+    [true,false].each do |http_compression_enabled|
+      context "with `http_compression => #{http_compression_enabled}`" do
 
-      it "should be handled properly" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once do |data|
-          expect(data.size).to be > target_bulk_bytes
+        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+
+        before(:each) do
+          if http_compression_enabled
+            expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
+          else
+            expect(http_client).to_not receive(:gzip_writer)
+          end
         end
-        s = subject.send(:bulk, actions)
-      end
-    end
 
-    context "with two messages" do
-      let(:message1) { "hey" }
-      let(:message2) { "you" }
-      let(:actions) { [
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
-      ]}
-      it "executes one bulk_send operation" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once
-        s = subject.send(:bulk, actions)
-      end
+        context "if a message is over TARGET_BULK_BYTES" do
+          let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+          let(:message) { "a" * (target_bulk_bytes + 1) }
+
+          it "should be handled properly" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once do |data|
+              if !http_compression_enabled
+                expect(data.size).to be > target_bulk_bytes
+              else
+                expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes
+              end
+            end
+            s = subject.send(:bulk, actions)
+          end
+        end
+
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
+          let(:actions) { [
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+          ]}
+          it "executes one bulk_send operation" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once
+            s = subject.send(:bulk, actions)
+          end
 
-      context "if one exceeds TARGET_BULK_BYTES" do
-        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-        let(:message1) { "a" * (target_bulk_bytes + 1) }
-        it "executes two bulk_send operations" do
-          allow(subject).to receive(:join_bulk_responses)
-          expect(subject).to receive(:bulk_send).twice
-          s = subject.send(:bulk, actions)
+          context "if one exceeds TARGET_BULK_BYTES" do
+            let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            it "executes two bulk_send operations" do
+              allow(subject).to receive(:join_bulk_responses)
+              expect(subject).to receive(:bulk_send).twice
+              s = subject.send(:bulk, actions)
+            end
+          end
         end
-      end
-    end
+       end
+     end
   end
 
   describe "sniffing" do

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -24,7 +24,7 @@ describe "Proxy option" do
 
     context "when specified as a URI" do
       shared_examples("hash conversion") do |hash|
-        let(:settings) { super.merge("proxy" => proxy)}
+        let(:settings) { super().merge("proxy" => proxy)}
         
         it "should set the proxy to the correct hash value" do
           expect(::Manticore::Client).to have_received(:new) do |options|
@@ -71,7 +71,7 @@ describe "Proxy option" do
   end
 
   context "when specified as ''" do
-    let(:settings) { super.merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
+    let(:settings) { super().merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
 
     it "should not send the proxy option to manticore" do
       expect { subject.register }.not_to raise_error
@@ -85,7 +85,7 @@ describe "Proxy option" do
   end
 
   context "when specified as invalid uri" do
-    let(:settings) { super.merge("proxy" => ":")}
+    let(:settings) { super().merge("proxy" => ":")}
 
     it "should fail" do
       # SafeURI isn't doing the proper exception wrapping for us, we can not simply :

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "flores/random"
 require "logstash/outputs/elasticsearch"
 
 describe LogStash::Outputs::ElasticSearch do
-  subject { described_class.new(options) }
+  subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
   let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
 
@@ -46,7 +46,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     describe "getting a document type" do
       context "if document_type isn't set" do
-        let(:options) { super.merge("document_type" => nil)}
+        let(:options) { super().merge("document_type" => nil)}
         context "for 7.x elasticsearch clusters" do
           let(:maximum_seen_major_version) { 7 }
           it "should return '_doc'" do
@@ -70,7 +70,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
       context "with 'document type set'" do
-        let(:options) { super.merge("document_type" => "bar")}
+        let(:options) { super().merge("document_type" => "bar")}
         it "should get the event type from the 'document_type' setting" do
           expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
@@ -87,7 +87,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should get the event type from the 'document_type' setting" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -105,7 +105,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should not include '_type'" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -127,7 +127,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as part of a URL" do
         let(:options) {
-          super.merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
+          super().merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
         }
 
         include_examples("an authenticated config")
@@ -135,7 +135,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as a hash option" do
           let(:options) {
-            super.merge!(
+            super().merge!(
               "user" => user,
               "password" => password
             )
@@ -175,7 +175,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "with extra slashes" do
         let(:path) { "/slashed-path/ "}
-        let(:options) { super.merge("path" => "/some-path/") }
+        let(:options) { super().merge("path" => "/some-path/") }
 
         it "should properly set the path on the HTTP client without adding slashes" do
           expect(manticore_url.path).to eql(options["path"])
@@ -234,13 +234,13 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     describe "without a port specified" do
-      let(:options) { super.merge('hosts' => 'localhost') }
+      let(:options) { super().merge('hosts' => 'localhost') }
       it "should properly set the default port (9200) on the HTTP client" do
         expect(manticore_url.port).to eql(9200)
       end
     end
     describe "with a port other than 9200 specified" do
-      let(:options) { super.merge('hosts' => 'localhost:9202') }
+      let(:options) { super().merge('hosts' => 'localhost:9202') }
       it "should properly set the specified port on the HTTP client" do
         expect(manticore_url.port).to eql(9202)
       end
@@ -265,12 +265,14 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { ::LogStash::Event.new("foo" => "bar") }
       let(:error) do
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-          429, double("url").as_null_object, double("request body"), double("response body")
+          429, double("url").as_null_object, request_body, double("response body")
         )
       end
       let(:logger) { double("logger").as_null_object }
       let(:response) { { :errors => [], :items => [] } }
 
+      let(:request_body) { double(:request_body, :bytesize => 1023) }
+
       before(:each) do
 
         i = 0
@@ -296,6 +298,95 @@ describe LogStash::Outputs::ElasticSearch do
         expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
       end
     end
+
+    context "unexpected bulk response" do
+      let(:options) do
+        { "hosts" => "127.0.0.1:9999", "index" => "%{foo}", "manage_template" => false }
+      end
+
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:bulk_response) do
+        # shouldn't really happen but we've seen this happen - here ES returns more items than were sent
+        { "took"=>1, "ingest_took"=>9, "errors"=>true,
+          "items"=>[{"index"=>{"_index"=>"bar1", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                              "error"=>{"type" => "illegal_state_exception",
+                                      "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]"
+                                      }
+                              }
+                    },
+                    # NOTE: this is an artificial success (usually everything fails with a 500) but even if some doc where
+                    # to succeed due the unexpected reponse items we can not clearly identify which actions to retry ...
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>201}},
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                               "error"=>{"type" => "illegal_state_exception",
+                                        "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]];"
+                                        }
+                              }
+                    }]
+        }
+    end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
+          expect( stream.string ).to include '"foo":"bar1"'
+          expect( stream.string ).to include '"foo":"bar2"'
+        end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
+      end
+
+      it "should retry submit" do
+        allow(subject.logger).to receive(:error).with(/Encountered an unexpected error/i, anything)
+        allow(subject.client).to receive(:bulk).and_call_original # track count
+
+        subject.multi_receive(events)
+
+        expect(subject.client).to have_received(:bulk).twice
+      end
+
+      it "should log specific error message" do
+        expect(subject.logger).to receive(:error).with(/Encountered an unexpected error/i,
+                                                       hash_including(:error_message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
+
+        subject.multi_receive(events)
+      end
+    end
+  end
+
+  context '413 errors' do
+    let(:payload_size) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES + 1024 }
+    let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
+
+    let(:logger_stub) { double("logger").as_null_object }
+
+    before(:each) do
+      allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
+
+      allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
+
+      max_bytes = payload_size * 3 / 4 # ensure a failure first attempt
+      allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
+        if body.length > max_bytes
+          max_bytes *= 2 # ensure a successful retry
+          double("Response", :code => 413, :body => "")
+        else
+          double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
+        end
+      end
+    end
+
+    it 'retries the 413 until it goes away' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(elasticsearch_output_instance.client).to have_received(:bulk).twice
+    end
+
+    it 'logs about payload quantity and size' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(logger_stub).to have_received(:warn)
+                                 .with(a_string_matching(/413 Payload Too Large/),
+                                       hash_including(:action_count => 1, :content_length => a_value > 20_000_000))
+    end
   end
 
   context "with timeout set" do
@@ -410,7 +501,7 @@ describe LogStash::Outputs::ElasticSearch do
     let(:options) { { 'retry_on_conflict' => num_retries } }
 
     context "with a regular index" do
-      let(:options) { super.merge("action" => "index") }
+      let(:options) { super().merge("action" => "index") }
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
         allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
@@ -420,7 +511,7 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     context "using a plain update" do
-      let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
         action, params, event_data = subject.event_action_tuple(event)
@@ -429,7 +520,7 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     context "with a sprintf action that resolves to update" do
-      let(:options) { super.merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
         action, params, event_data = subject.event_action_tuple(event)

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -44,7 +44,7 @@ describe "whitelisting error types in expected behavior" do
   end
 
   describe "when failure logging is disabled for docuemnt exists error" do
-    let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+    let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
       expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.8.4
+  version: 10.8.6
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2021-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement