logstash-output-elasticsearch 10.8.0-java → 10.8.6-java

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -24,7 +24,7 @@ describe "Proxy option" do
 
     context "when specified as a URI" do
       shared_examples("hash conversion") do |hash|
-        let(:settings) { super.merge("proxy" => proxy)}
+        let(:settings) { super().merge("proxy" => proxy)}
         
         it "should set the proxy to the correct hash value" do
           expect(::Manticore::Client).to have_received(:new) do |options|
@@ -71,7 +71,7 @@ describe "Proxy option" do
   end
 
   context "when specified as ''" do
-    let(:settings) { super.merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
+    let(:settings) { super().merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
 
     it "should not send the proxy option to manticore" do
       expect { subject.register }.not_to raise_error
@@ -85,7 +85,7 @@ describe "Proxy option" do
   end
 
   context "when specified as invalid uri" do
-    let(:settings) { super.merge("proxy" => ":")}
+    let(:settings) { super().merge("proxy" => ":")}
 
     it "should fail" do
       # SafeURI isn't doing the proper exception wrapping for us, we can not simply :

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "flores/random"
 require "logstash/outputs/elasticsearch"
 
 describe LogStash::Outputs::ElasticSearch do
-  subject { described_class.new(options) }
+  subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
   let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
 
@@ -46,7 +46,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     describe "getting a document type" do
       context "if document_type isn't set" do
-        let(:options) { super.merge("document_type" => nil)}
+        let(:options) { super().merge("document_type" => nil)}
         context "for 7.x elasticsearch clusters" do
           let(:maximum_seen_major_version) { 7 }
           it "should return '_doc'" do
@@ -70,7 +70,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
       context "with 'document type set'" do
-        let(:options) { super.merge("document_type" => "bar")}
+        let(:options) { super().merge("document_type" => "bar")}
         it "should get the event type from the 'document_type' setting" do
           expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
@@ -80,14 +80,14 @@ describe LogStash::Outputs::ElasticSearch do
     describe "building an event action tuple" do
       context "for 7.x elasticsearch clusters" do
         let(:maximum_seen_major_version) { 7 }
-        it "should include '_type'" do
+        it "should not include '_type' when 'document_type' is not explicitly defined" do
           action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
           action_params = action_tuple[1]
-          expect(action_params).to include(:_type => "_doc")
+          expect(action_params).not_to include(:_type => "_doc")
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should get the event type from the 'document_type' setting" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -105,7 +105,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
 
         context "with 'document type set'" do
-          let(:options) { super.merge("document_type" => "bar")}
+          let(:options) { super().merge("document_type" => "bar")}
           it "should not include '_type'" do
             action_tuple = subject.send(:event_action_tuple, LogStash::Event.new("type" => "foo"))
             action_params = action_tuple[1]
@@ -127,7 +127,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as part of a URL" do
         let(:options) {
-          super.merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
+          super().merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"])
         }
 
         include_examples("an authenticated config")
@@ -135,7 +135,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "as a hash option" do
           let(:options) {
-            super.merge!(
+            super().merge!(
               "user" => user,
               "password" => password
             )
@@ -175,7 +175,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       context "with extra slashes" do
         let(:path) { "/slashed-path/ "}
-        let(:options) { super.merge("path" => "/some-path/") }
+        let(:options) { super().merge("path" => "/some-path/") }
 
         it "should properly set the path on the HTTP client without adding slashes" do
           expect(manticore_url.path).to eql(options["path"])
@@ -234,13 +234,13 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     describe "without a port specified" do
-      let(:options) { super.merge('hosts' => 'localhost') }
+      let(:options) { super().merge('hosts' => 'localhost') }
       it "should properly set the default port (9200) on the HTTP client" do
         expect(manticore_url.port).to eql(9200)
       end
     end
     describe "with a port other than 9200 specified" do
-      let(:options) { super.merge('hosts' => 'localhost:9202') }
+      let(:options) { super().merge('hosts' => 'localhost:9202') }
       it "should properly set the specified port on the HTTP client" do
         expect(manticore_url.port).to eql(9202)
       end
@@ -265,12 +265,14 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { ::LogStash::Event.new("foo" => "bar") }
       let(:error) do
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-          429, double("url").as_null_object, double("request body"), double("response body")
+          429, double("url").as_null_object, request_body, double("response body")
         )
       end
       let(:logger) { double("logger").as_null_object }
       let(:response) { { :errors => [], :items => [] } }
 
+      let(:request_body) { double(:request_body, :bytesize => 1023) }
+
       before(:each) do
 
         i = 0
@@ -296,6 +298,95 @@ describe LogStash::Outputs::ElasticSearch do
         expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
       end
     end
+
+    context "unexpected bulk response" do
+      let(:options) do
+        { "hosts" => "127.0.0.1:9999", "index" => "%{foo}", "manage_template" => false }
+      end
+
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:bulk_response) do
+        # shouldn't really happen but we've seen this happen - here ES returns more items than were sent
+        { "took"=>1, "ingest_took"=>9, "errors"=>true,
+          "items"=>[{"index"=>{"_index"=>"bar1", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                              "error"=>{"type" => "illegal_state_exception",
+                                      "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]; nested: IllegalArgumentException[no enrich index exists for policy with name [test-metadata1]];; java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]]"
+                                      }
+                              }
+                    },
+                    # NOTE: this is an artificial success (usually everything fails with a 500) but even if some doc where
+                    # to succeed due the unexpected reponse items we can not clearly identify which actions to retry ...
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>201}},
+                    {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>500,
+                               "error"=>{"type" => "illegal_state_exception",
+                                        "reason" => "pipeline with id [test-ingest] could not be loaded, caused by [ElasticsearchParseException[Error updating pipeline with id [test-ingest]]; nested: ElasticsearchException[java.lang.IllegalArgumentException: no enrich index exists for policy with name [test-metadata1]];"
+                                        }
+                              }
+                    }]
+        }
+    end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
+          expect( stream.string ).to include '"foo":"bar1"'
+          expect( stream.string ).to include '"foo":"bar2"'
+        end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
+      end
+
+      it "should retry submit" do
+        allow(subject.logger).to receive(:error).with(/Encountered an unexpected error/i, anything)
+        allow(subject.client).to receive(:bulk).and_call_original # track count
+
+        subject.multi_receive(events)
+
+        expect(subject.client).to have_received(:bulk).twice
+      end
+
+      it "should log specific error message" do
+        expect(subject.logger).to receive(:error).with(/Encountered an unexpected error/i,
+                                                       hash_including(:error_message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
+
+        subject.multi_receive(events)
+      end
+    end
+  end
+
+  context '413 errors' do
+    let(:payload_size) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES + 1024 }
+    let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
+
+    let(:logger_stub) { double("logger").as_null_object }
+
+    before(:each) do
+      allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
+
+      allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
+
+      max_bytes = payload_size * 3 / 4 # ensure a failure first attempt
+      allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
+        if body.length > max_bytes
+          max_bytes *= 2 # ensure a successful retry
+          double("Response", :code => 413, :body => "")
+        else
+          double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
+        end
+      end
+    end
+
+    it 'retries the 413 until it goes away' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(elasticsearch_output_instance.client).to have_received(:bulk).twice
+    end
+
+    it 'logs about payload quantity and size' do
+      elasticsearch_output_instance.multi_receive([event])
+
+      expect(logger_stub).to have_received(:warn)
+                                 .with(a_string_matching(/413 Payload Too Large/),
+                                       hash_including(:action_count => 1, :content_length => a_value > 20_000_000))
+    end
   end
 
   context "with timeout set" do
@@ -410,7 +501,7 @@ describe LogStash::Outputs::ElasticSearch do
     let(:options) { { 'retry_on_conflict' => num_retries } }
 
     context "with a regular index" do
-      let(:options) { super.merge("action" => "index") }
+      let(:options) { super().merge("action" => "index") }
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
         allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
@@ -420,7 +511,7 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     context "using a plain update" do
-      let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
         action, params, event_data = subject.event_action_tuple(event)
@@ -429,7 +520,7 @@ describe LogStash::Outputs::ElasticSearch do
     end
 
     context "with a sprintf action that resolves to update" do
-      let(:options) { super.merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
+      let(:options) { super().merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
         action, params, event_data = subject.event_action_tuple(event)

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -44,7 +44,7 @@ describe "whitelisting error types in expected behavior" do
   end
 
   describe "when failure logging is disabled for docuemnt exists error" do
-    let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+    let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
       expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.8.0
+  version: 10.8.6
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-02 00:00:00.000000000 Z
+date: 2021-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement