logstash-output-elasticsearch 9.0.0-java → 9.0.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5ca4ff178e96df26064af65eb8fb6a4d394c6ff7
-  data.tar.gz: 0f5d0ede03e013b1d1c332b889ff1d6166c61f81
+SHA256:
+  metadata.gz: 910353088f57a02bea121f9f2c92e27d27664372dbf6b934e256103178c093cd
+  data.tar.gz: a5ba64f0511129dbc9fa8890457f25317e7541aecaf4e694f4f1b835143ce999
 SHA512:
-  metadata.gz: df4adbd9a19b54b7997ddad28e3e976d62c0124e645bb45855ef4899d07d322e5fce715cde1fb9981cff4c21ad2b7daec4f807f768582e0ef4b44581049ef7f2
-  data.tar.gz: 4a0cf366db45cd2b438d568b0dab4b8fda540bf09f6c43c43f0c80bdbf24758d26966812b857ae04cc75e128920a9c18d5c707cc6cb2d3c3e8371e145b14f57b
+  metadata.gz: 03a87ed4822764503c6d0d5269ac5c19df9b0d5566659f0ca0df6ced8abd4f6aebd1e384cd6f2fa829787cd864730b32a90235378b88954eba25ebceff240228
+  data.tar.gz: 357011f25a5e41559e995bf35cbb25de6d0ef4c5db0524994dcb24bd2d8300e36068793a403b40e3fc5face5495a9e9963546146c89870c0d219c269864f8afb

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 9.0.2
+  - Ignore event's type field for the purpose of setting document `_type` if cluster is es 6.x or above
+
+## 9.0.1
+  - Update gemspec summary
+
 ### 9.0.0
   - Change default document type to 'doc' from 'logs' to align with beats and reflect the generic nature of logstash.
   - Deprecate 'document_type' option

data/docs/index.asciidoc

@@ -67,8 +67,8 @@ HTTP requests to the bulk API are expected to return a 200 response code. All ot
 
 The following document errors are handled as follows:
 
-* 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<dlq-policy>> for more info.
-* 409 errors (conflict) are logged as a warning and dropped. 
+  * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<dlq-policy>> for more info.
+  * 409 errors (conflict) are logged as a warning and dropped. 
 
 Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
@@ -242,8 +242,10 @@ Note: This option is deprecated due to the https://www.elastic.co/guide/en/elast
 It will be removed in the next major version of Logstash.
 This sets the document type to write events to. Generally you should try to write only
 similar events to the same 'type'. String expansion `%{foo}` works here.
-Unless you set 'document_type', the event 'type' will be used if it exists
-otherwise the document type will be assigned the value of 'doc'.
+If you don't set a value for this option:
+
+- for elasticsearch clusters 6.x and above: the value of 'doc' will be used;
+- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
 
 [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
 ===== `failure_type_logging_whitelist` 
@@ -681,4 +683,4 @@ See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-in
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -22,6 +22,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       setup_hosts # properly sets @hosts
       build_client
+
       install_template
       check_action_validity
 
@@ -48,6 +49,10 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
+    def maximum_seen_major_version
+      client.maximum_seen_major_version
+    end
+
     def install_template
       TemplateManager.install_template(self)
     end
@@ -199,12 +204,17 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     # Determine the correct value for the 'type' field for the given event
+    DEFAULT_EVENT_TYPE="doc".freeze
     def get_event_type(event)
       # Set the 'type' value for the index.
       type = if @document_type
                event.sprintf(@document_type)
              else
-               event.get("type") || "doc"
+               if client.maximum_seen_major_version < 6
+                 event.get("type") || DEFAULT_EVENT_TYPE
+               else
+                 DEFAULT_EVENT_TYPE
+               end
              end
 
       if !(type.is_a?(String) || type.is_a?(Numeric))

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -82,8 +82,8 @@ module LogStash; module Outputs; class ElasticSearch;
       template_put(name, template)
     end
 
-    def connected_es_versions
-      @pool.connected_es_versions
+    def maximum_seen_major_version
+      @pool.maximum_seen_major_version
     end
 
     def bulk(actions)

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -107,9 +107,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info }
     end
 
-    def connected_es_versions
+    def maximum_seen_major_version
       @state_mutex.synchronize do
-        @url_info.values.select {|v| v[:state] == :alive }.map {|v| v[:version] }
+        @maximum_seen_major_version
       end
     end
 
@@ -245,6 +245,14 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           es_version = get_es_version(url)
           @state_mutex.synchronize do
             meta[:version] = es_version
+            major = major_version(es_version)
+            if !@maximum_seen_major_version
+              @logger.info("ES Output version determined", :es_version => @maximum_seen_major_version)
+              set_new_major_version(major)
+            elsif major > @maximum_seen_major_version
+              @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an elasticsearch cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url)
+              set_new_major_version(major)
+            end
             meta[:state] = :alive
           end
         rescue HostUnreachableError, BadResponseCodeError => e
@@ -434,5 +442,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       request = perform_request_to_url(url, :get, ROOT_URI_PATH)
       LogStash::Json.load(request.body)["version"]["number"]
     end
+
+    def set_new_major_version(version)
+      @maximum_seen_major_version = version
+      if @maximum_seen_major_version >= 6
+        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", :es_version => @maximum_seen_major_version)
+      end
+    end
   end
 end; end; end; end;

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -4,7 +4,7 @@ module LogStash; module Outputs; class ElasticSearch
     def self.install_template(plugin)
       return unless plugin.manage_template
       plugin.logger.info("Using mapping template from", :path => plugin.template)
-      template = get_template(plugin.template, get_es_major_version(plugin.client))
+      template = get_template(plugin.template, plugin.maximum_seen_major_version)
       plugin.logger.info("Attempting to install template", :manage_template => template)
       install(plugin.client, plugin.template_name, template, plugin.template_overwrite)
     rescue => e
@@ -12,12 +12,6 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     private
-    def self.get_es_major_version(client)
-      # get the elasticsearch version of each node in the pool and
-      # pick the biggest major version
-      client.connected_es_versions.uniq.map {|version| version.split(".").first.to_i}.max
-    end
-
     def self.get_template(path, es_major_version)
       template_path = path || default_template_path(es_major_version)
       read_template_file(template_path)

data/logstash-output-elasticsearch.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.0.0'
+  s.version         = '9.0.2'
   s.licenses        = ['apache-2.0']
-  s.summary         = "Logstash Output to Elasticsearch"
+  s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -50,7 +50,11 @@ if ESHelper.es_version_satisfies?(">= 5")
         response = http_client.get("#{index_url}/_search?q=*&size=1000")
         result = LogStash::Json.load(response.body)
         result["hits"]["hits"].each do |doc|
-          expect(doc["_type"]).to eq(type)
+          if ESHelper.es_version_satisfies?(">= 6")
+            expect(doc["_type"]).to eq("doc")
+          else
+            expect(doc["_type"]).to eq(type)
+          end
           expect(doc["_index"]).to eq(index)
         end
       end

data/spec/integration/outputs/index_spec.rb

@@ -78,7 +78,11 @@ describe "indexing" do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        if ESHelper.es_version_satisfies?(">= 6")
+          expect(doc["_type"]).to eq("doc")
+        else
+          expect(doc["_type"]).to eq(type)
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -7,11 +7,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:es_node_versions) { [ "0.0.0" ] }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
   let(:manticore_double) { double("manticore a") }
-  before do
+  before(:each) do
 
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
@@ -21,10 +22,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
-    allow(subject).to receive(:connected_es_versions).with(any_args).and_return(["0.0.0"])
-    allow(subject).to receive(:get_es_version).with(any_args).and_return("0.0.0")
-
-    subject.start
+    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
   end
 
   after do
@@ -34,10 +32,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   describe "initialization" do
     it "should be successful" do
       expect { subject }.not_to raise_error
+      subject.start
     end
   end
 
   describe "the resurrectionist" do
+    before(:each) { subject.start }
     it "should start the resurrectionist when created" do
       expect(subject.resurrectionist_alive?).to eql(true)
     end
@@ -77,6 +77,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   end
 
   describe "the sniffer" do
+    before(:each) { subject.start }
     it "should not start the sniffer by default" do
       expect(subject.sniffer_alive?).to eql(nil)
     end
@@ -92,6 +93,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
   describe "closing" do
     before do
+      subject.start
       # Simulate a single in use connection on the first check of this
       allow(adapter).to receive(:close).and_call_original
       allow(subject).to receive(:wait_for_in_use_connections).and_call_original
@@ -120,6 +122,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   end
 
   describe "connection management" do
+    before(:each) { subject.start }
     context "with only one URL in the list" do
       it "should use the only URL in 'with_connection'" do
         subject.with_connection do |c|
@@ -167,4 +170,27 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       end
     end
   end
+
+  describe "version tracking" do
+    let(:initial_urls) { [
+      ::LogStash::Util::SafeURI.new("http://somehost:9200"),
+      ::LogStash::Util::SafeURI.new("http://otherhost:9201")
+    ] }
+
+    before(:each) do
+      allow(subject).to receive(:perform_request_to_url).and_return(nil)
+      subject.start
+    end
+
+    it "picks the largest major version" do
+      expect(subject.maximum_seen_major_version).to eq(0)
+    end
+
+    context "if there are nodes with multiple major versions" do
+      let(:es_node_versions) { [ "0.0.0", "6.0.0" ] }
+      it "picks the largest major version" do
+        expect(subject.maximum_seen_major_version).to eq(6)
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -5,26 +5,6 @@ require "json"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
-  describe ".get_es_major_version" do
-    let(:client) { double("client") }
-
-    before(:each) do
-      allow(client).to receive(:connected_es_versions).and_return(["5.3.0"])
-    end
-
-    it "picks the largest major version" do
-      expect(described_class.get_es_major_version(client)).to eq(5)
-    end
-    context "if there are nodes with multiple major versions" do
-      before(:each) do
-        allow(client).to receive(:connected_es_versions).and_return(["5.3.0", "6.0.0"])
-      end
-      it "picks the largest major version" do
-        expect(described_class.get_es_major_version(client)).to eq(6)
-      end
-    end
-  end
-
   describe ".default_template_path" do
     context "elasticsearch 1.x" do
       it "chooses the 2x template" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -2,7 +2,31 @@ require_relative "../../../spec/es_spec_helper"
 require "flores/random"
 require "logstash/outputs/elasticsearch"
 
-describe "outputs/elasticsearch" do
+describe LogStash::Outputs::ElasticSearch do
+  subject { described_class.new(options) }
+  let(:options) { {} }
+  let(:maximum_seen_major_version) { rand(100) }
+
+  let(:do_register) { true }
+
+  before(:each) do
+    if do_register
+      subject.register
+
+      # Rspec mocks can't handle background threads, so... we can't use any
+      allow(subject.client.pool).to receive(:start_resurrectionist)
+      allow(subject.client.pool).to receive(:start_sniffer)
+      allow(subject.client.pool).to receive(:healthcheck!)
+      allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
+      subject.client.pool.adapter.manticore.respond_with(:body => "{}")
+    end
+  end
+
+  after(:each) do
+    subject.close
+  end
+
+
   context "with an active instance" do
     let(:options) {
       {
@@ -12,57 +36,51 @@ describe "outputs/elasticsearch" do
         "manage_template" => false
       }
     }
-    
-    let(:eso) { LogStash::Outputs::ElasticSearch.new(options) }
 
-    let(:manticore_urls) { eso.client.pool.urls }
+    let(:manticore_urls) { subject.client.pool.urls }
     let(:manticore_url) { manticore_urls.first }
-
-    let(:do_register) { true }
-
-    before(:each) do
-      if do_register
-        eso.register
-        
-        # Rspec mocks can't handle background threads, so... we can't use any 
-        allow(eso.client.pool).to receive(:start_resurrectionist)
-        allow(eso.client.pool).to receive(:start_sniffer)
-        allow(eso.client.pool).to receive(:healthcheck!)
-        eso.client.pool.adapter.manticore.respond_with(:body => "{}")
-      end
-    end
-    
-    after(:each) do
-      eso.close 
-    end
     
     describe "getting a document type" do
       it "should default to 'doc'" do
-        expect(eso.send(:get_event_type, LogStash::Event.new)).to eql("doc")
+        expect(subject.send(:get_event_type, LogStash::Event.new)).to eql("doc")
       end
 
-      it "should get the type from the event if nothing else specified in the config" do
-        expect(eso.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
+      context "if document_type isn't set" do
+        let(:options) { super.merge("document_type" => nil)}
+        context "for 6.x elasticsearch clusters" do
+          let(:maximum_seen_major_version) { 6 }
+          it "should return 'doc'" do
+            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
+          end
+        end
+
+        context "for < 6.0 elasticsearch clusters" do
+          let(:maximum_seen_major_version) { 5 }
+          it "should get the type from the event" do
+            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
+          end
+        end
       end
 
       context "with 'document type set'" do
         let(:options) { super.merge("document_type" => "bar")}
         it "should get the event type from the 'document_type' setting" do
-          expect(eso.send(:get_event_type, LogStash::Event.new())).to eql("bar")
+          expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
       end
 
-      context "with a bad type" do
+      context "with a bad type event field in a < 6.0 es cluster" do
+        let(:maximum_seen_major_version) { 5 }
         let(:type_arg) { ["foo"] }
-        let(:result) { eso.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
+        let(:result) { subject.send(:get_event_type, LogStash::Event.new("type" => type_arg)) }
 
         before do
-          allow(eso.instance_variable_get(:@logger)).to receive(:warn)
+          allow(subject.instance_variable_get(:@logger)).to receive(:warn)
           result
         end
 
         it "should call @logger.warn and return nil" do
-          expect(eso.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
+          expect(subject.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once
         end
 
         it "should set the type to the stringified value" do
@@ -103,7 +121,7 @@ describe "outputs/elasticsearch" do
 
     describe "with path" do
       it "should properly create a URI with the path" do
-        expect(eso.path).to eql(options["path"])
+        expect(subject.path).to eql(options["path"])
       end
 
         it "should properly set the path on the HTTP client adding slashes" do
@@ -129,7 +147,7 @@ describe "outputs/elasticsearch" do
         let(:client_host_path) { manticore_url.path }
 
         it "should initialize without error" do
-          expect { eso }.not_to raise_error
+          expect { subject }.not_to raise_error
         end
 
         it "should use the URI path" do
@@ -145,7 +163,7 @@ describe "outputs/elasticsearch" do
           end
 
           it "should initialize without error" do
-            expect { eso }.not_to raise_error
+            expect { subject }.not_to raise_error
           end
 
           it "should use the option path" do
@@ -164,7 +182,7 @@ describe "outputs/elasticsearch" do
           end
 
           it "should initialize with an error" do
-            expect { eso.register }.to raise_error(LogStash::ConfigurationError)
+            expect { subject.register }.to raise_error(LogStash::ConfigurationError)
           end
         end
       end
@@ -188,12 +206,12 @@ describe "outputs/elasticsearch" do
       let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
 
       before do
-        allow(eso).to receive(:retrying_submit).with(anything)
+        allow(subject).to receive(:retrying_submit).with(anything)
         events.each_with_index do |e,i|
           et = events_tuples[i]
-          allow(eso).to receive(:event_action_tuple).with(e).and_return(et)
+          allow(subject).to receive(:event_action_tuple).with(e).and_return(et)
         end
-        eso.multi_receive(events)
+        subject.multi_receive(events)
       end
 
     end
@@ -213,24 +231,24 @@ describe "outputs/elasticsearch" do
         i = 0
         bulk_param =  [["index", anything, event.to_hash]]
 
-        allow(eso).to receive(:logger).and_return(logger)
+        allow(subject).to receive(:logger).and_return(logger)
 
         # Fail the first time bulk is called, succeed the next time
-        allow(eso.client).to receive(:bulk).with(bulk_param) do
+        allow(subject.client).to receive(:bulk).with(bulk_param) do
           i += 1
           if i == 1
             raise error
           end
         end.and_return(response)
-        eso.multi_receive([event])
+        subject.multi_receive([event])
       end
 
       it "should retry the 429 till it goes away" do
-        expect(eso.client).to have_received(:bulk).twice
+        expect(subject.client).to have_received(:bulk).twice
       end
 
       it "should log a debug message" do
-        expect(eso.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
+        expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
       end
     end
   end
@@ -245,19 +263,16 @@ describe "outputs/elasticsearch" do
         "timeout" => 0.1, # fast timeout
       }
     end
-    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
 
     before do
-      eso.register
-
       # Expect a timeout to be logged.
-      expect(eso.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
-      expect(eso.client).to receive(:bulk).at_least(:twice).and_call_original
+      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
+      expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original
     end
 
     it "should fail after the timeout" do
       #pending("This is tricky now that we do healthchecks on instantiation")
-      Thread.new { eso.multi_receive([LogStash::Event.new]) }
+      Thread.new { subject.multi_receive([LogStash::Event.new]) }
 
       # Allow the timeout to occur
       sleep 6
@@ -265,19 +280,19 @@ describe "outputs/elasticsearch" do
   end
 
   describe "the action option" do
-    subject(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
     context "with a sprintf action" do
-      let(:options) { {"action" => "%{myactionfield}"} }
+      let(:options) { {"action" => "%{myactionfield}" } }
 
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        expect(eso.event_action_tuple(event).first).to eql("update")
+        expect(subject.event_action_tuple(event).first).to eql("update")
       end
     end
 
     context "with an invalid action" do
       let(:options) { {"action" => "SOME Garbaaage"} }
+      let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
 
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error(LogStash::ConfigurationError)
@@ -289,8 +304,6 @@ describe "outputs/elasticsearch" do
     let(:manticore_double) do 
       double("manticoreX#{self.inspect}") 
     end
-    
-    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
 
     before(:each) do
       response_double = double("manticore response").as_null_object
@@ -300,17 +313,12 @@ describe "outputs/elasticsearch" do
       allow(manticore_double).to receive(:close)
         
       allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
-      eso.register
-    end
-    
-    after(:each) do
-      eso.close
+      subject.register
     end
     
-    
     shared_examples("an encrypted client connection") do
       it "should enable SSL in manticore" do
-        expect(eso.client.pool.urls.map(&:scheme).uniq).to eql(['https'])
+        expect(subject.client.pool.urls.map(&:scheme).uniq).to eql(['https'])
       end
     end
 
@@ -330,22 +338,22 @@ describe "outputs/elasticsearch" do
   describe "retry_on_conflict" do
     let(:num_retries) { 123 }
     let(:event) { LogStash::Event.new("message" => "blah") }
-    subject(:eso) {LogStash::Outputs::ElasticSearch.new(options.merge('retry_on_conflict' => num_retries))}
+    let(:options) { { 'retry_on_conflict' => num_retries } }
 
     context "with a regular index" do
-      let(:options) { {"action" => "index"} }
+      let(:options) { super.merge("action" => "index") }
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        action, params, event_data = eso.event_action_tuple(event)
+        action, params, event_data = subject.event_action_tuple(event)
         expect(params).not_to include({:_retry_on_conflict => num_retries})
       end
     end
 
     context "using a plain update" do
-      let(:options) { {"action" => "update", "retry_on_conflict" => num_retries} }
+      let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) } 
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        action, params, event_data = eso.event_action_tuple(event)
+        action, params, event_data = subject.event_action_tuple(event)
         expect(params).to include({:_retry_on_conflict => num_retries})
       end
     end
@@ -353,17 +361,17 @@ describe "outputs/elasticsearch" do
 
   describe "sleep interval calculation" do
     let(:retry_max_interval) { 64 }
-    subject(:eso) { LogStash::Outputs::ElasticSearch.new("retry_max_interval" => retry_max_interval) }
+    let(:options) { { "retry_max_interval" => retry_max_interval } }
 
     it "should double the given value" do
-      expect(eso.next_sleep_interval(2)).to eql(4)
-      expect(eso.next_sleep_interval(32)).to eql(64)
+      expect(subject.next_sleep_interval(2)).to eql(4)
+      expect(subject.next_sleep_interval(32)).to eql(64)
     end
 
     it "should not increase the value past the max retry interval" do
       sleep_interval = 2
       100.times do
-        sleep_interval = eso.next_sleep_interval(sleep_interval)
+        sleep_interval = subject.next_sleep_interval(sleep_interval)
         expect(sleep_interval).to be <= retry_max_interval
       end
     end
@@ -371,14 +379,15 @@ describe "outputs/elasticsearch" do
 
   describe "stale connection check" do
     let(:validate_after_inactivity) { 123 }
-    subject(:eso) { LogStash::Outputs::ElasticSearch.new("validate_after_inactivity" => validate_after_inactivity) }
+    let(:options) { { "validate_after_inactivity" => validate_after_inactivity } }
+    let(:do_register) { false }
 
-    before do
+    before :each do
       allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
-      subject.register
     end
 
-    it "should set the correct http client option for 'validate_after_inactivity" do
+    it "should set the correct http client option for 'validate_after_inactivity'" do
+      subject.register
       expect(::Manticore::Client).to have_received(:new) do |options|
         expect(options[:check_connection_timeout]).to eq(validate_after_inactivity)
       end
@@ -387,22 +396,12 @@ describe "outputs/elasticsearch" do
 
   describe "custom parameters" do
 
-    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
-
-    let(:manticore_urls) { eso.client.pool.urls }
+    let(:manticore_urls) { subject.client.pool.urls }
     let(:manticore_url) { manticore_urls.first }
 
     let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
     let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
 
-    before(:each) do
-      eso.register
-    end
-
-    after(:each) do
-      eso.close rescue nil
-    end
-
     context "using non-url hosts" do
 
       let(:options) {
@@ -415,7 +414,7 @@ describe "outputs/elasticsearch" do
       }
 
       it "creates a URI with the added parameters" do
-        expect(eso.parameters).to eql(custom_parameters_hash)
+        expect(subject.parameters).to eql(custom_parameters_hash)
       end
 
       it "sets the query string on the HTTP client" do

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -14,6 +14,7 @@ describe "whitelisting error types in expected behavior" do
 
     subject.register
 
+    allow(subject.client).to receive(:maximum_seen_major_version).and_return(0)
     allow(subject.client).to receive(:bulk).and_return(
       {
         "errors" => true,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.0.0
+  version: 9.0.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-29 00:00:00.000000000 Z
+date: 2017-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -196,7 +196,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -280,10 +282,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
-summary: Logstash Output to Elasticsearch
+summary: Stores logs in Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
 - spec/fixtures/htpasswd