logstash-output-elasticsearch 11.0.3-java → 11.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 143b78ff484990a35c4d9a1b54ccb040bdba7e6aa2fd69928b0553544883fabd
-  data.tar.gz: a14e65b2b499d65d9a8f3389d0c0168f0447e1358338d867dd5b05a78ca28c13
+  metadata.gz: 0bd2f7b95f4a121df8712ce601a2c56c228de6e113f90c96058c58f966af31db
+  data.tar.gz: bac417c7d999e1676ed451cf5216b0b6085203b533fd7d5039abf3efbc070578
 SHA512:
-  metadata.gz: fd6495287f9d5dff77d6006c5495013335eb869b7a9678973567358b9ce6ed772822cda5b38bfbeff1831e72406a4e550125cfafd9cbaeab20b588ccf584a7b6
-  data.tar.gz: d6eda43b3338ad401b889ff20f7bd246b3a44f921b0f006989572cb95145a518cb7103a1c23fd930018c2552316af33d356d4fecc8210ca95f0a6259a2b1037f
+  metadata.gz: '09274b381e8026eb8527b5926318c603ffc776e645538671263f61fec3d00731e7695a13e2f72cdccd4037f1d7298975a6424f03115a38060f4e2e34f7498049'
+  data.tar.gz: 47fddb97c3634cd68588bebaed1d0fac38d4eb33c3cc69083e59a9c13c1f74a87f7835e6fc7982762a2db19009150900ca7ac2bee213d973e56dc0a8fbcc1158

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 11.2.0
+ - Added preflight checks on Elasticsearch [#1026](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1026)
+
+## 11.1.0
+ - Feat: add `user-agent` header passed to the Elasticsearch HTTP connection [#1038](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1038)
+
+## 11.0.5
+ - Fixed running post-register action when Elasticsearch status change from unhealthy to healthy [#1035](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1035)
+
+## 11.0.4
+ - [DOC] Clarify that `http_compression` applies to _requests_, and remove noise about _response_ decompression [#1000](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1000)
+
 ## 11.0.3
  - Fixed SSL handshake hang indefinitely with proxy setup [#1032](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1032)
 

data/docs/index.asciidoc

@@ -278,16 +278,11 @@ not reevaluate its DNS value while the keepalive is in effect.
 
 ==== HTTP Compression
 
-This plugin supports request and response compression. Response compression is
-enabled by default for HTTP and for Elasticsearch versions 5.0 and later.
+This plugin supports request compression, and handles compressed responses
+from Elasticsearch.
 
-You don't have to set any configs in Elasticsearch for it to send back a
-compressed response. For versions before 5.0, or if HTTPS is enabled,
-`http.compression` must be set to `true` {ref}/modules-http.html#modules-http[in
-Elasticsearch] to take advantage of response compression when using this plugin.
-
-For requests compression, regardless of the Elasticsearch version, enable the
-`http_compression` setting in the Logstash config file.
+To enable request compression, use the <<plugins-{type}s-{plugin}-http_compression>>
+setting on this plugin.
 
 ==== Authentication
 
@@ -640,8 +635,13 @@ Any special characters present in the URLs here MUST be URL escaped! This means
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Enable gzip compression on requests. Note that response compression is on by
-default for Elasticsearch v5.0 and beyond
+Enable gzip compression on requests.
+
+This setting allows you to reduce this plugin's outbound network traffic by
+compressing each bulk _request_ to {es}.
+
+NOTE: This output plugin reads compressed _responses_ from {es} regardless
+      of the value of this setting.
 
 [id="plugins-{type}s-{plugin}-ilm_enabled"]
 ===== `ilm_enabled`

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -37,6 +37,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     ROOT_URI_PATH = '/'.freeze
     LICENSE_PATH = '/_license'.freeze
 
+    VERSION_6_TO_7 = Gem::Requirement.new([">= 6.0.0", "< 7.0.0"])
+    VERSION_7_TO_7_14 = Gem::Requirement.new([">= 7.0.0", "< 7.14.0"])
+
     DEFAULT_OPTIONS = {
       :healthcheck_path => ROOT_URI_PATH,
       :sniffing_path => "/_nodes/http",
@@ -211,7 +214,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def start_resurrectionist
       @resurrectionist = Thread.new do
         until_stopped("resurrection", @resurrect_delay) do
-          healthcheck!
+          healthcheck!(false)
         end
       end
     end
@@ -232,11 +235,18 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       perform_request_to_url(url, :head, @healthcheck_path)
     end
 
-    def healthcheck!
+    def healthcheck!(register_phase = true)
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
           health_check_request(url)
+
+          # when called from resurrectionist skip the product check done during register phase
+          if register_phase
+            if !elasticsearch?(url)
+              raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+            end
+          end
           # If no exception was raised it must have succeeded!
           logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
           # We reconnected to this node, check its ES version
@@ -254,6 +264,42 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
+    def elasticsearch?(url)
+      begin
+        response = perform_request_to_url(url, :get, ROOT_URI_PATH)
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        return false if response.code == 401 || response.code == 403
+        raise e
+      end
+
+      version_info = LogStash::Json.load(response.body)
+      return false if version_info['version'].nil?
+
+      version = Gem::Version.new(version_info["version"]['number'])
+      return false if version < Gem::Version.new('6.0.0')
+
+      if VERSION_6_TO_7.satisfied_by?(version)
+        return valid_tagline?(version_info)
+      elsif VERSION_7_TO_7_14.satisfied_by?(version)
+        build_flavor = version_info["version"]['build_flavor']
+        return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
+      else
+        # case >= 7.14
+        lower_headers = response.headers.transform_keys {|key| key.to_s.downcase }
+        product_header = lower_headers['x-elastic-product']
+        return false if product_header != 'Elasticsearch'
+      end
+      return true
+    rescue => e
+      logger.error("Unable to retrieve Elasticsearch version", url: url.sanitized.to_s, exception: e.class, message: e.message)
+      false
+    end
+
+    def valid_tagline?(version_info)
+      tagline = version_info['tagline']
+      tagline == "You Know, for Search"
+    end
+
     def stop_resurrectionist
       @resurrectionist.join if @resurrectionist
     end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -4,6 +4,7 @@ require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 require 'cgi'
 require 'zlib'
 require 'stringio'
+require 'java'
 
 module LogStash; module Outputs; class ElasticSearch;
   # This is a constant instead of a config option because
@@ -92,6 +93,10 @@ module LogStash; module Outputs; class ElasticSearch;
       @pool.maximum_seen_major_version
     end
 
+    def alive_urls_count
+      @pool.alive_urls_count
+    end
+
     def bulk(actions)
       @action_count ||= 0
       @action_count += actions.size
@@ -297,6 +302,8 @@ module LogStash; module Outputs; class ElasticSearch;
         :request_timeout => timeout,
       }
 
+      adapter_options[:user_agent] = prepare_user_agent
+
       adapter_options[:proxy] = client_settings[:proxy] if client_settings[:proxy]
 
       adapter_options[:check_connection_timeout] = client_settings[:check_connection_timeout] if client_settings[:check_connection_timeout]
@@ -318,6 +325,18 @@ module LogStash; module Outputs; class ElasticSearch;
       adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
       adapter = adapter_class.new(@logger, adapter_options)
     end
+
+    def prepare_user_agent
+      os_name = java.lang.System.getProperty('os.name')
+      os_version = java.lang.System.getProperty('os.version')
+      os_arch = java.lang.System.getProperty('os.arch')
+      jvm_vendor = java.lang.System.getProperty('java.vendor')
+      jvm_version = java.lang.System.getProperty('java.version')
+
+      plugin_version = Gem.loaded_specs['logstash-output-elasticsearch'].version
+      # example: Logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-output-elasticsearch/11.0.1
+      "Logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-output-elasticsearch/#{plugin_version}"
+    end
     
     def build_pool(options)
       adapter = build_adapter(options)

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -128,8 +128,12 @@ module LogStash; module PluginMixins; module ElasticSearch
       client.maximum_seen_major_version
     end
 
+    def alive_urls_count
+      client.alive_urls_count
+    end
+
     def successful_connection?
-      !!maximum_seen_major_version
+      !!maximum_seen_major_version && alive_urls_count > 0
     end
 
     # launch a thread that waits for an initial successful connection to the ES cluster to call the given block

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.0.3'
+  s.version         = '11.2.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
+  s.add_development_dependency 'webmock'
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/es_spec_helper.rb

@@ -80,7 +80,7 @@ module ESHelper
   end
 
   def self.es_version_satisfies?(*requirement)
-    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    es_version = nilify(RSpec.configuration.filter[:es_version]) || nilify(ENV['ES_VERSION']) || nilify(ENV['ELASTIC_STACK_VERSION'])
     if es_version.nil?
       puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
       return false
@@ -89,6 +89,15 @@ module ESHelper
     Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
   end
 
+  private
+  def self.nilify(str)
+    if str.nil?
+      return str
+    end
+    str.empty? ? nil : str
+  end
+
+  public
   def clean(client)
     client.indices.delete_template(:name => "*")
     client.indices.delete_index_template(:name => "logstash*") rescue nil

data/spec/integration/outputs/no_es_on_startup_spec.rb

@@ -55,4 +55,18 @@ describe "elasticsearch is down on startup", :integration => true do
     expect(r).to have_hits(2)
   end
 
+  it 'should get cluster_uuid when Elasticsearch recovers from license check failure' do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_raise(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    subject.register
+    Thread.new do
+      sleep 4
+      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_call_original
+    end
+    subject.multi_receive([event1, event2])
+    @es.indices.refresh
+    r = @es.search(index: 'logstash-*')
+    expect(r).to have_hits(2)
+    expect(subject.plugin_metadata.get(:cluster_uuid)).not_to be_empty
+    expect(subject.plugin_metadata.get(:cluster_uuid)).not_to eq("_na_")
+  end if ESHelper.es_version_satisfies?(">=7")
 end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -50,15 +50,18 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     describe "healthcheck url handling" do
       let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+      before(:example) do
+        expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
+          expect(url.path).to be_empty
+        end
+      end
 
       context "and not setting healthcheck_path" do
         it "performs the healthcheck to the root" do
-          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
-            expect(method).to eq(:head)
+          expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
-            expect(req_path).to eq("/")
           end
-          subject.healthcheck!
+          expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
       end
 
@@ -66,12 +69,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         let(:healthcheck_path) { "/my/health" }
         let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
         it "performs the healthcheck to the healthcheck_path" do
-          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
-            expect(method).to eq(:head)
+          expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
-            expect(req_path).to eq(healthcheck_path)
           end
-          subject.healthcheck!
+          expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
       end
     end
@@ -164,6 +165,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
 
+  class MockResponse
+    attr_reader :code, :headers
+
+    def initialize(code = 200, body = nil, headers = {})
+        @code = code
+        @body = body
+        @headers = headers
+    end
+
+    def body
+      @body.to_json
+    end
+  end
+
   describe "connection management" do
     before(:each) { subject.start }
     context "with only one URL in the list" do
@@ -175,8 +190,17 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "with multiple URLs in the list" do
+      let(:version_ok) do
+        MockResponse.new(200, {"tagline" => "You Know, for Search",
+                               "version" => {
+                                 "number" => '7.13.0',
+                                 "build_flavor" => 'default'}
+                               })
+      end
+
       before :each do
         allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+        allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
       end
       let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
 
@@ -220,8 +244,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       ::LogStash::Util::SafeURI.new("http://otherhost:9201")
     ] }
 
+    let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
+                                                          "version" => {
+                                                            "number" => '7.13.0',
+                                                            "build_flavor" => 'default'}
+                                                          }) }
+
     before(:each) do
-      allow(subject).to receive(:perform_request_to_url).and_return(nil)
+      allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
       subject.start
     end
 
@@ -240,6 +270,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   describe "license checking" do
     before(:each) do
       allow(subject).to receive(:health_check_request)
+      allow(subject).to receive(:elasticsearch?).and_return(true)
     end
 
     let(:options) do
@@ -273,6 +304,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     before(:each) do
       allow(subject).to receive(:health_check_request)
+      allow(subject).to receive(:elasticsearch?).and_return(true)
     end
 
     context "if ES doesn't return a valid license" do
@@ -319,3 +351,120 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
 end
+
+describe "#elasticsearch?" do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { double("Manticore Adapter") }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:es_node_versions) { [ "0.0.0" ] }
+  let(:license_status) { 'active' }
+
+  subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
+
+  let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+  context "in case HTTP error code" do
+    it "should fail for 401" do
+      allow(adapter).to receive(:perform_request)
+        .with(anything, :get, "/", anything, anything)
+        .and_return(MockResponse.new(401))
+
+      expect(subject.elasticsearch?(url)).to be false
+    end
+
+    it "should fail for 403" do
+      allow(adapter).to receive(:perform_request)
+              .with(anything, :get, "/", anything, anything)
+              .and_return(status: 403)
+      expect(subject.elasticsearch?(url)).to be false
+    end
+  end
+
+  context "when connecting to a cluster which reply without 'version' field" do
+    it "should fail" do
+      allow(adapter).to receive(:perform_request)
+                    .with(anything, :get, "/", anything, anything)
+                    .and_return(body: {"field" => "funky.com"}.to_json)
+      expect(subject.elasticsearch?(url)).to be false
+    end
+  end
+
+  context "when connecting to a cluster with version < 6.0.0" do
+    it "should fail" do
+      allow(adapter).to receive(:perform_request)
+                          .with(anything, :get, "/", anything, anything)
+                          .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
+      expect(subject.elasticsearch?(url)).to be false
+    end
+  end
+
+  context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
+    it "must be successful with valid 'tagline'" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
+      expect(subject.elasticsearch?(url)).to be true
+    end
+
+    it "should fail if invalid 'tagline'" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+
+    it "should fail if 'tagline' is not present" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+  end
+
+  context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
+    it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
+      expect(subject.elasticsearch?(url)).to be true
+    end
+
+    it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+
+    it "should fail if 'build_flavor' is not present and tagline is correct" do
+      allow(adapter).to receive(:perform_request)
+                                .with(anything, :get, "/", anything, anything)
+                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+  end
+
+  context "when connecting to a cluster with version >= 7.14.0" do
+    it "should fail if 'X-elastic-product' header is not present" do
+      allow(adapter).to receive(:perform_request)
+                    .with(anything, :get, "/", anything, anything)
+                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+
+    it "should fail if 'X-elastic-product' header is present but with bad value" do
+      allow(adapter).to receive(:perform_request)
+                    .with(anything, :get, "/", anything, anything)
+                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
+      expect(subject.elasticsearch?(url)).to be false
+    end
+
+    it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
+      allow(adapter).to receive(:perform_request)
+                    .with(anything, :get, "/", anything, anything)
+                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
+      expect(subject.elasticsearch?(url)).to be true
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,6 +1,8 @@
 require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "cabin"
+require "webrick"
+require "java"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
   let(:ssl) { nil }
@@ -287,4 +289,71 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       end
     end
   end
+
+  class StoppableServer
+
+    attr_reader :port
+
+    def initialize()
+      queue = Queue.new
+      @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+      @first_request = nil
+
+      @t = java.lang.Thread.new(
+        proc do
+          begin
+            @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                     :Logger => Cabin::Channel.get, # silence WEBrick logging
+                     :StartCallback => Proc.new {
+                           queue.push("started")
+                         }
+            @port = @server.config[:Port]
+            @server.mount_proc '/headers_check' do |req, res|
+              res.body = 'Hello, world from WEBrick mocking server!'
+              @first_request = req
+              @first_req_waiter.countDown()
+            end
+
+            @server.start
+          rescue => e
+            puts "Error in webserver thread #{e}"
+            # ignore
+          end
+        end
+      )
+      @t.daemon = true
+      @t.start
+      queue.pop # blocks until the server is up
+    end
+
+    def stop
+      @server.shutdown
+    end
+
+    def wait_receive_request
+      @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+      @first_request
+    end
+  end
+
+  describe "#build_adapter" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options) }
+    let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+    after :each do
+      webserver.stop
+    end
+
+    context "the 'user-agent' header" do
+      it "contains the Logstash environment details" do
+        adapter = client.build_adapter(client.options)
+        adapter.perform_request(::LogStash::Util::SafeURI.new("http://localhost:#{webserver.port}"), :get, "/headers_check")
+
+        request = webserver.wait_receive_request
+
+        transmitted_user_agent = request.header['user-agent'][0]
+        expect(transmitted_user_agent).to match(/Logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-output-elasticsearch\/\d*\.\d*\.\d*/)
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -156,7 +156,7 @@ describe LogStash::Outputs::ElasticSearch do
         include_examples("an authenticated config")
       end
 
-      context 'claud_auth also set' do
+      context 'cloud_auth also set' do
         let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
         let(:options) { { "user" => user, "password" => password, "cloud_auth" => "elastic:my-passwd-00" } }
 
@@ -934,6 +934,7 @@ describe LogStash::Outputs::ElasticSearch do
       allow(subject).to receive(:last_es_version).and_return es_version
       # make successful_connection? return true:
       allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
+      allow(subject).to receive(:alive_urls_count).and_return Integer(1)
       allow(subject).to receive(:stop_after_successful_connection_thread)
     end
 

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -12,6 +12,7 @@ describe "whitelisting error types in expected behavior" do
   before :each do
     allow(subject.logger).to receive(:warn)
     allow(subject).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject).to receive(:alive_urls_count).and_return(1)
     allow(subject).to receive(:finish_register)
 
     subject.register

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.0.3
+  version: 11.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,6 +140,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webmock
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -184,7 +212,6 @@ files:
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
-- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
 - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
 - lib/logstash/plugin_mixins/elasticsearch/common.rb
 - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb

data/lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json

@@ -1 +0,0 @@
-lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta