logstash-output-elasticsearch 11.17.0-java → 11.19.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37124c3a166313a2fb9f3831273def114770178158439a44ebfa1bc1d32f9d0f
-  data.tar.gz: a09fd3ce2c54908fedc14dc2d780bb3ec48d7091d05e2e8fcb5789e4ec1e30b9
+  metadata.gz: '0490f311461b39c49e3edaf045af9a7172dbd97da8bc456bf044a031c18e5afb'
+  data.tar.gz: fbf05190ed352c12f0948aecfa6ed08e760b03985b83aa2e6830167a00c431bc
 SHA512:
-  metadata.gz: caac996badd1bbdeb231fad3f40f96a50386baf78ee356587d5fc5d2b4a095f1073bee417a99aab081c13cb1a18802785abed618dc85db504f56145c620c46b6
-  data.tar.gz: 697a89b810998154a44338e8e73f02351b72afa00fc35e1fc115a22ce5ecfeddd27d29bd8fdf4fb779a3b17fbaa2f4f361a2c30b68c0b5ce0cd49a6edeba1a1d
+  metadata.gz: b50084b6fb4fd31e7592be7e35e23963dce4cf284763baa4c1c042d1f159f20cebe09c1013ace38a986dbf4c8ee3cc9ae4eda7f38042956bd10dbdfc0ad20114
+  data.tar.gz: 91643fa15c48a29a02659967c75e0b879f98af2a6e7f1582c83b141f67826337ccaa3d0513b1372266dd068cbccfcdbbe59a5fcf36ceff81318126dd1bdf64cc

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.19.0
+  - Added `filter_path` to bulk requests to reduce the size of responses from elasticsearch
+  
+## 11.18.0
+  - Added request header `Elastic-Api-Version` for serverless [#1147](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1147)
+
 ## 11.17.0
   - Added support to http compression level. Deprecated `http_compression` in favour of `compression_level` and enabled compression level 1 by default. [#1148](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1148)
 

data/docs/index.asciidoc

@@ -423,10 +423,12 @@ Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
 ===== `bulk_path`
 
   * Value type is <<string,string>>
-  * There is no default value for this setting.
+  * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status`
 
 HTTP Path to perform the _bulk requests to
-this defaults to a concatenation of the path parameter and "_bulk"
+* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status`
+* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used.
+
 
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -16,6 +16,18 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @response_body = response_body
       end
 
+      def invalid_eav_header?
+        @response_code == 400 && @response_body&.include?(ELASTIC_API_VERSION)
+      end
+
+      def invalid_credentials?
+        @response_code == 401
+      end
+
+      def forbidden?
+        @response_code == 403
+      end
+
     end
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
@@ -48,7 +60,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       :sniffer_delay => 10,
     }.freeze
 
-    BUILD_FLAVOUR_SERVERLESS = 'serverless'.freeze
+    BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
+    ELASTIC_API_VERSION = "Elastic-Api-Version".freeze
+    DEFAULT_EAV_HEADER = { ELASTIC_API_VERSION => "2023-10-31" }.freeze
 
     def initialize(logger, adapter, initial_urls=[], options={})
       @logger = logger
@@ -77,7 +91,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @license_checker = options[:license_checker] || LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE
 
       @last_es_version = Concurrent::AtomicReference.new
-      @build_flavour = Concurrent::AtomicReference.new
+      @build_flavor = Concurrent::AtomicReference.new
     end
 
     def start
@@ -232,39 +246,56 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def health_check_request(url)
-      response = perform_request_to_url(url, :head, @healthcheck_path)
-      raise BadResponseCodeError.new(response.code, url, nil, response.body) unless (200..299).cover?(response.code)
+      logger.debug("Running health check to see if an Elasticsearch connection is working",
+                   :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
+      begin
+        response = perform_request_to_url(url, :head, @healthcheck_path)
+        return response, nil
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
+        return nil, e
+      end
     end
 
     def healthcheck!(register_phase = true)
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
-          logger.debug("Running health check to see if an Elasticsearch connection is working",
-                        :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
-          health_check_request(url)
+          _, health_bad_code_err = health_check_request(url)
+          root_response, root_bad_code_err = get_root_path(url) if health_bad_code_err.nil? || register_phase
 
           # when called from resurrectionist skip the product check done during register phase
           if register_phase
-            if !elasticsearch?(url)
-              raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
-            end
+            raise LogStash::ConfigurationError,
+                  "Could not read Elasticsearch. Please check the credentials" if root_bad_code_err&.invalid_credentials?
+            raise LogStash::ConfigurationError,
+                  "Could not read Elasticsearch. Please check the privileges" if root_bad_code_err&.forbidden?
+            # when customer_headers is invalid
+            raise LogStash::ConfigurationError,
+                  "The Elastic-Api-Version header is not valid" if root_bad_code_err&.invalid_eav_header?
+            # when it is not Elasticserach
+            raise LogStash::ConfigurationError,
+                  "Could not connect to a compatible version of Elasticsearch" if root_bad_code_err.nil? && !elasticsearch?(root_response)
+
+            test_serverless_connection(url, root_response)
           end
+
+          raise health_bad_code_err if health_bad_code_err
+          raise root_bad_code_err if root_bad_code_err
+
           # If no exception was raised it must have succeeded!
           logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
-          # We reconnected to this node, check its ES version
-          version_info = get_es_version(url)
-          es_version = version_info.fetch('number', nil)
-          build_flavour = version_info.fetch('build_flavor', nil)
-
-          if es_version.nil?
-            logger.warn("Failed to retrieve Elasticsearch version data from connected endpoint, connection aborted", :url => url.sanitized.to_s)
-            next
-          end
+
+          # We check its ES version
+          es_version, build_flavor = parse_es_version(root_response)
+          logger.warn("Failed to retrieve Elasticsearch build flavor") if build_flavor.nil?
+          logger.warn("Failed to retrieve Elasticsearch version data from connected endpoint, connection aborted", :url => url.sanitized.to_s) if es_version.nil?
+          next if es_version.nil?
+
           @state_mutex.synchronize do
             meta[:version] = es_version
             set_last_es_version(es_version, url)
-            set_build_flavour(build_flavour)
+            set_build_flavor(build_flavor)
 
             alive = @license_checker.appropriate_license?(self, url)
             meta[:state] = alive ? :alive : :dead
@@ -275,40 +306,21 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    def elasticsearch?(url)
+    def get_root_path(url, params={})
       begin
-        response = perform_request_to_url(url, :get, ROOT_URI_PATH)
+        resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
+        return resp, nil
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        return false if response.code == 401 || response.code == 403
-        raise e
+        logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
+        return nil, e
       end
-
-      version_info = LogStash::Json.load(response.body)
-      return false if version_info['version'].nil?
-
-      version = ::Gem::Version.new(version_info["version"]['number'])
-      return false if version < ::Gem::Version.new('6.0.0')
-
-      if VERSION_6_TO_7.satisfied_by?(version)
-        return valid_tagline?(version_info)
-      elsif VERSION_7_TO_7_14.satisfied_by?(version)
-        build_flavor = version_info["version"]['build_flavor']
-        return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
-      else
-        # case >= 7.14
-        lower_headers = response.headers.transform_keys {|key| key.to_s.downcase }
-        product_header = lower_headers['x-elastic-product']
-        return false if product_header != 'Elasticsearch'
-      end
-      return true
-    rescue => e
-      logger.error("Unable to retrieve Elasticsearch version", url: url.sanitized.to_s, exception: e.class, message: e.message)
-      false
     end
 
-    def valid_tagline?(version_info)
-      tagline = version_info['tagline']
-      tagline == "You Know, for Search"
+    def test_serverless_connection(url, root_response)
+      _, build_flavor = parse_es_version(root_response)
+      params = { :headers => DEFAULT_EAV_HEADER }
+      _, bad_code_err = get_root_path(url, params) if build_flavor == BUILD_FLAVOR_SERVERLESS
+      raise LogStash::ConfigurationError, "The Elastic-Api-Version header is not valid" if bad_code_err&.invalid_eav_header?
     end
 
     def stop_resurrectionist
@@ -334,6 +346,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
+      params[:headers] = DEFAULT_EAV_HEADER.merge(params[:headers] || {}) if serverless?
       @adapter.perform_request(url, method, path, params, body)
     end
 
@@ -476,15 +489,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    def get_es_version(url)
-      response = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      return nil unless (200..299).cover?(response.code)
-
-      response = LogStash::Json.load(response.body)
-
-      response.fetch('version', {})
-    end
-
     def last_es_version
       @last_es_version.get
     end
@@ -494,7 +498,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def serverless?
-      @build_flavour.get == BUILD_FLAVOUR_SERVERLESS
+      @build_flavor.get == BUILD_FLAVOR_SERVERLESS
     end
 
     private
@@ -526,9 +530,50 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
                    previous_major: @maximum_seen_major_version, new_major: major, node_url: url.sanitized.to_s)
     end
 
-    def set_build_flavour(flavour)
-      @build_flavour.set(flavour)
+    def set_build_flavor(flavor)
+      @build_flavor.set(flavor)
+    end
+
+    def parse_es_version(response)
+      return nil, nil unless (200..299).cover?(response&.code)
+
+      response = LogStash::Json.load(response&.body)
+      version_info = response.fetch('version', {})
+      es_version = version_info.fetch('number', nil)
+      build_flavor = version_info.fetch('build_flavor', nil)
+
+      return es_version, build_flavor
+    end
+
+    def elasticsearch?(response)
+      return false if response.nil?
+
+      version_info = LogStash::Json.load(response.body)
+      return false if version_info['version'].nil?
+
+      version = ::Gem::Version.new(version_info["version"]['number'])
+      return false if version < ::Gem::Version.new('6.0.0')
+
+      if VERSION_6_TO_7.satisfied_by?(version)
+        return valid_tagline?(version_info)
+      elsif VERSION_7_TO_7_14.satisfied_by?(version)
+        build_flavor = version_info["version"]['build_flavor']
+        return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
+      else
+        # case >= 7.14
+        lower_headers = response.headers.transform_keys {|key| key.to_s.downcase }
+        product_header = lower_headers['x-elastic-product']
+        return false if product_header != 'Elasticsearch'
+      end
+      return true
+    rescue => e
+      logger.error("Unable to retrieve Elasticsearch version", exception: e.class, message: e.message)
+      false
     end
 
+    def valid_tagline?(version_info)
+      tagline = version_info['tagline']
+      tagline == "You Know, for Search"
+    end
   end
 end; end; end; end;

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -177,6 +177,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def bulk_send(body_stream, batch_actions)
       params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
+
       response = @pool.post(@bulk_path, params, body_stream.string)
 
       @bulk_response_metrics.increment(response.code.to_s)
@@ -209,7 +210,7 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def get(path)
-      response = @pool.get(path, nil)
+      response = @pool.get(path)
       LogStash::Json.load(response.body)
     end
 

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -33,9 +33,9 @@ module LogStash; module Outputs; class ElasticSearch;
       end
 
       common_options[:bulk_path] = if params["bulk_path"]
-         dedup_slashes("/#{params["bulk_path"]}")
+        resolve_filter_path(dedup_slashes("/#{params["bulk_path"]}"))
       else
-         dedup_slashes("/#{params["path"]}/_bulk")
+        resolve_filter_path(dedup_slashes("/#{params["path"]}/_bulk"))
       end
 
       common_options[:sniffing_path] = if params["sniffing_path"]
@@ -197,5 +197,16 @@ module LogStash; module Outputs; class ElasticSearch;
     def self.dedup_slashes(url)
       url.gsub(/\/+/, "/")
     end
+
+    # Set a `filter_path` query parameter if it is not already set to be
+    # `filter_path=errors,items.*.error,items.*.status` to reduce the payload between Logstash and Elasticsearch
+    def self.resolve_filter_path(url)
+      return url if url.match?(/(?:[&|?])filter_path=/)
+      ("#{url}#{query_param_separator(url)}filter_path=errors,items.*.error,items.*.status")
+    end
+
+    def self.query_param_separator(url)
+      url.match?(/\?[^\s#]+/) ? '&' : '?'
+    end
   end
 end; end; end

data/lib/logstash/outputs/elasticsearch.rb

@@ -596,7 +596,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   def install_template
     TemplateManager.install_template(self)
   rescue => e
-    @logger.error("Failed to install template", message: e.message, exception: e.class, backtrace: e.backtrace)
+    details = { message: e.message, exception: e.class, backtrace: e.backtrace }
+    details[:body] = e.response_body if e.respond_to?(:response_body)
+    @logger.error("Failed to install template", details)
   end
 
   def setup_ecs_compatibility_related_defaults

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -179,7 +179,9 @@ module LogStash; module PluginMixins; module ElasticSearch
       cluster_info = client.get('/')
       plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
     rescue => e
-      @logger.error("Unable to retrieve Elasticsearch cluster uuid", message: e.message, exception: e.class, backtrace: e.backtrace)
+      details = { message: e.message, exception: e.class, backtrace: e.backtrace }
+      details[:body] = e.response_body if e.respond_to?(:response_body)
+      @logger.error("Unable to retrieve Elasticsearch cluster uuid", details)
     end
 
     def retrying_submit(actions)

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.17.0'
+  s.version         = '11.19.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/index_spec.rb

@@ -156,7 +156,7 @@ describe "indexing" do
   let(:config) { "not implemented" }
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::ElasticSearch.new(config) }
-  
+  let(:filter_path) { "filter_path=errors,items.*.error,items.*.status"}
   let(:es_url) { "http://#{get_host_port}" }
   let(:index_url) { "#{es_url}/#{index}" }
 
@@ -178,7 +178,7 @@ describe "indexing" do
     subject.do_close
   end
 
-  shared_examples "an indexer" do |secure|
+  shared_examples "an indexer" do |secure, expected_path|
     before(:each) do
       host_unreachable_error_class = LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError
       allow(host_unreachable_error_class).to receive(:new).with(any_args).and_wrap_original do |m, original, url|
@@ -212,13 +212,13 @@ describe "indexing" do
         expect(doc["_index"]).to eq(index)
       end
     end
-    
+
     it "sets the correct content-type header" do
       expected_manticore_opts = {:headers => {"Content-Type" => "application/json"}, :body => anything}
       if secure
         expected_manticore_opts = {
-          :headers => {"Content-Type" => "application/json"}, 
-          :body => anything, 
+          :headers => {"Content-Type" => "application/json"},
+          :body => anything,
           :auth => {
             :user => user,
             :password => password,
@@ -230,6 +230,20 @@ describe "indexing" do
         and_call_original
       subject.multi_receive(events)
     end
+
+    it "sets the bulk path URL and filter path parameter correctly" do
+      expect(subject.client.pool.adapter.client).to receive(:send).
+          with(anything, expected_path != nil ? expected_path : anything, anything).at_least(:once).and_call_original
+      subject.multi_receive(events)
+    end
+
+    it "receives a filtered response" do
+      expect(subject.client).to receive(:join_bulk_responses).
+          with([{"errors"=>false, "items"=>[{"index"=>{"status"=>201}}]}]).
+          and_call_original
+      subject.multi_receive([event])
+    end
+
   end
 
   shared_examples "PKIX path failure" do
@@ -269,6 +283,65 @@ describe "indexing" do
     it_behaves_like("an indexer")
   end
 
+  describe "an indexer with custom bulk path", :integration => true do
+    let(:bulk_path) { "/_bulk?routing=true"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}#{bulk_path}&#{filter_path}" }
+    end
+  end
+
+  describe "an indexer with filter path as second parameter", :integration => true do
+    let(:bulk_path) { "/_bulk?routing=true&#{filter_path}"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/#{bulk_path}" }
+    end
+  end
+
+  describe "an indexer with filter path as first  parameter", :integration => true do
+    let(:bulk_path) { "/_bulk?#{filter_path}&routing=true"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/#{bulk_path}" }
+    end
+  end
+
+  describe "an indexer with the standard bulk path", :integration => true do
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/_bulk?#{filter_path}" }
+    end
+
+  end
+
   describe "an indexer with no type value set (default to doc)", :integration => true do
     let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
     let(:config) {
@@ -296,7 +369,7 @@ describe "indexing" do
         "index" => index,
         "http_compression" => false
       }
-    end 
+    end
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 

data/spec/unit/http_client_builder_spec.rb

@@ -36,7 +36,17 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
       end
     end
 
-    describe "healthcheck_path" do
+    describe "bulk_path" do
+      let (:filter_path) {"filter_path=errors,items.*.error,items.*.status"}
+
+      shared_examples("filter_path added to bulk path appropriately") do
+        it "sets the bulk_path option to the expected bulk path" do
+          expect(described_class).to receive(:create_http_client) do |options|
+            expect(options[:bulk_path]).to eq(expected_bulk_path)
+          end
+          described_class.build(logger, hosts, options)
+          end
+      end
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
@@ -44,21 +54,31 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:options) { super().merge("path" => "/path") }
-          it "ignores the path setting" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq(bulk_path)
-            end
-            described_class.build(logger, hosts, options)
-          end
+          let(:expected_bulk_path) { "#{bulk_path}?#{filter_path}" }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
+        end
+
+        context "when setting a filter path as first parameter"  do
+          let (:filter_path) {"filter_path=error"}
+          let(:bulk_path) { "/meh?#{filter_path}&routing=true" }
+          let(:expected_bulk_path) { bulk_path }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
+        end
+
+        context "when setting a filter path as second parameter" do
+          let (:filter_path) {"filter_path=error"}
+          let(:bulk_path) { "/meh?routing=true&#{filter_path}" }
+          let(:expected_bulk_path) { bulk_path }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
+
         context "when not using path" do
+          let(:expected_bulk_path) { "#{bulk_path}?#{filter_path}"}
 
-          it "uses the bulk_path setting" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq(bulk_path)
-            end
-            described_class.build(logger, hosts, options)
-          end
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
       end
 
@@ -66,25 +86,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
+          let(:expected_bulk_path) { "#{path}/_bulk?#{filter_path}"}
           let(:options) { super().merge("path" => path) }
-          it "sets bulk_path to path+_bulk" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq("#{path}/_bulk")
-            end
-            described_class.build(logger, hosts, options)
-          end
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
 
         context "when not using path" do
-          it "sets the bulk_path to _bulk" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq("/_bulk")
-            end
-            described_class.build(logger, hosts, options)
-          end
+          let(:expected_bulk_path) { "/_bulk?#{filter_path}"}
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
       end
     end
+
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -7,8 +7,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 3, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
-  let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'} ] }
   let(:license_status) { 'active' }
+  let(:root_response) { MockResponse.new(200,
+                                          {"tagline" => "You Know, for Search",
+                                           "version" => {
+                                             "number" => '8.9.0',
+                                             "build_flavor" => 'default'} },
+                                          { "X-Elastic-Product" => "Elasticsearch" }
+  ) }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
@@ -22,7 +28,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
-    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_version_info)
     allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
@@ -37,35 +42,42 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
 
-  describe "the resurrectionist" do
-    before(:each) { subject.start }
-    it "should start the resurrectionist when created" do
-      expect(subject.resurrectionist_alive?).to eql(true)
-    end
+  describe "healthcheck" do
 
-    it "should attempt to resurrect connections after the ressurrect delay" do
-      expect(subject).to receive(:healthcheck!).once
-      sleep(subject.resurrect_delay + 1)
+    describe "the resurrectionist" do
+      before(:each) { subject.start }
+      it "should start the resurrectionist when created" do
+        expect(subject.resurrectionist_alive?).to eql(true)
+      end
+
+      it "should attempt to resurrect connections after the ressurrect delay" do
+        expect(subject).to receive(:healthcheck!).once
+        sleep(subject.resurrect_delay + 1)
+      end
     end
 
-    describe "healthcheck url handling" do
+    describe "healthcheck path handling" do
       let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
-      let(:success_response) { double("Response", :code => 200) }
+      let(:healthcheck_response) { double("Response", :code => 200) }
 
       before(:example) do
+        subject.start
+
+        expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
+          expect(url.path).to be_empty
+          healthcheck_response
+        end
+
         expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
           expect(url.path).to be_empty
+          root_response
         end
       end
 
       context "and not setting healthcheck_path" do
+        let(:healthcheck_path) { "/" }
         it "performs the healthcheck to the root" do
-          expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
-            expect(url.path).to be_empty
-
-            success_response
-          end
-          expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
+          subject.healthcheck!
         end
       end
 
@@ -73,14 +85,116 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         let(:healthcheck_path) { "/my/health" }
         let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
         it "performs the healthcheck to the healthcheck_path" do
-          expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
-            expect(url.path).to be_empty
+          subject.healthcheck!
+        end
+      end
+    end
+
+    describe "register phase" do
+      shared_examples_for "root path returns bad code error" do |err_msg|
+        before :each do
+          subject.update_initial_urls
+          expect(subject).to receive(:elasticsearch?).never
+        end
+
+        it "raises ConfigurationError" do
+          expect(subject).to receive(:health_check_request).with(anything).and_return(["", nil])
+          expect(subject).to receive(:get_root_path).with(anything).and_return([nil,
+                                                                                ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(mock_resp.code, nil, nil, mock_resp.body)])
+          expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, err_msg)
+        end
+      end
+
+      context "with 200 without version" do
+        let(:mock_resp) { MockResponse.new(200, {"tagline" => "You Know, for Search"}) }
 
-            success_response
-          end
+        it "raises ConfigurationError" do
+          subject.update_initial_urls
+
+          expect(subject).to receive(:health_check_request).with(anything).and_return(["", nil])
+          expect(subject).to receive(:get_root_path).with(anything).and_return([mock_resp, nil])
           expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
       end
+
+      context "with 200 serverless" do
+        let(:good_resp) { MockResponse.new(200,
+                                           { "tagline" => "You Know, for Search",
+                                             "version" => { "number" => '8.10.0', "build_flavor" => 'serverless'}
+                                           },
+                                           { "X-Elastic-Product" => "Elasticsearch" }
+        ) }
+        let(:bad_400_err) do
+          ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(400,
+           nil, nil,
+           "The requested [Elastic-Api-Version] header value of [2024-10-31] is not valid. Only [2023-10-31] is supported")
+        end
+
+        it "raises ConfigurationError when the serverless connection test fails" do
+          subject.update_initial_urls
+
+          expect(subject).to receive(:health_check_request).with(anything).and_return(["", nil])
+          expect(subject).to receive(:get_root_path).with(anything).and_return([good_resp, nil])
+          expect(subject).to receive(:get_root_path).with(anything, hash_including(:headers => LogStash::Outputs::ElasticSearch::HttpClient::Pool::DEFAULT_EAV_HEADER)).and_return([nil, bad_400_err])
+          expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "The Elastic-Api-Version header is not valid")
+        end
+
+        it "passes when the serverless connection test succeeds" do
+          subject.update_initial_urls
+
+          expect(subject).to receive(:health_check_request).with(anything).and_return(["", nil])
+          expect(subject).to receive(:get_root_path).with(anything).and_return([good_resp, nil])
+          expect(subject).to receive(:get_root_path).with(anything, hash_including(:headers => LogStash::Outputs::ElasticSearch::HttpClient::Pool::DEFAULT_EAV_HEADER)).and_return([good_resp, nil])
+          expect { subject.healthcheck! }.not_to raise_error
+        end
+      end
+
+      context "with 200 default" do
+        let(:good_resp) { MockResponse.new(200,
+                                           { "tagline" => "You Know, for Search",
+                                             "version" => { "number" => '8.10.0', "build_flavor" => 'default'}
+                                           },
+                                           { "X-Elastic-Product" => "Elasticsearch" }
+        ) }
+
+        it "passes without checking serverless connection" do
+          subject.update_initial_urls
+
+          expect(subject).to receive(:health_check_request).with(anything).and_return(["", nil])
+          expect(subject).to receive(:get_root_path).with(anything).and_return([good_resp, nil])
+          expect(subject).not_to receive(:get_root_path).with(anything, hash_including(:headers => LogStash::Outputs::ElasticSearch::HttpClient::Pool::DEFAULT_EAV_HEADER))
+          expect { subject.healthcheck! }.not_to raise_error
+        end
+      end
+
+      context "with 400" do
+        let(:mock_resp) { MockResponse.new(400, "The requested [Elastic-Api-Version] header value of [2024-10-31] is not valid. Only [2023-10-31] is supported") }
+        it_behaves_like "root path returns bad code error", "The Elastic-Api-Version header is not valid"
+      end
+
+      context "with 401" do
+        let(:mock_resp) { MockResponse.new(401, "missing authentication") }
+        it_behaves_like "root path returns bad code error", "Could not read Elasticsearch. Please check the credentials"
+      end
+
+      context "with 403" do
+        let(:mock_resp) { MockResponse.new(403, "Forbidden") }
+        it_behaves_like "root path returns bad code error", "Could not read Elasticsearch. Please check the privileges"
+      end
+    end
+
+    describe "non register phase" do
+      let(:health_bad_code_err) { ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(400, nil, nil, nil) }
+
+      before :each do
+        subject.update_initial_urls
+      end
+
+      it "does not call root path when health check request fails" do
+        expect(subject).to receive(:health_check_request).with(anything).and_return(["", health_bad_code_err])
+        expect(subject).to receive(:get_root_path).never
+        subject.healthcheck!(false)
+      end
     end
   end
 
@@ -251,23 +365,23 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       ::LogStash::Util::SafeURI.new("http://otherhost:9201")
     ] }
 
-    let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
-                                                          "version" => {
-                                                            "number" => '7.13.0',
-                                                            "build_flavor" => 'default'}
-                                                          }) }
-
-    before(:each) do
-      allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
-      subject.start
-    end
-
-    it "picks the largest major version" do
-      expect(subject.maximum_seen_major_version).to eq(0)
-    end
+    let(:root_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
+                                                  "version" => {
+                                                    "number" => '0.0.0',
+                                                    "build_flavor" => 'default'}
+    }) }
+    let(:root_response2) { MockResponse.new(200, {"tagline" => "You Know, for Search",
+                                                  "version" => {
+                                                    "number" => '6.0.0',
+                                                    "build_flavor" => 'default'}
+    }) }
 
     context "if there are nodes with multiple major versions" do
-      let(:es_version_info) { [ { "number" => '0.0.0', "build_flavor" => 'default'}, { "number" => '6.0.0', "build_flavor" => 'default'} ] }
+      before(:each) do
+        allow(subject).to receive(:perform_request_to_url).and_return(root_response, root_response2)
+        subject.start
+      end
+
       it "picks the largest major version" do
         expect(subject.maximum_seen_major_version).to eq(6)
       end
@@ -275,32 +389,31 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   end
 
 
-  describe "build flavour tracking" do
+  describe "build flavor tracking" do
     let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://somehost:9200")] }
 
-    let(:es_version_info) { [ { "number" => '8.9.0', "build_flavor" => "serverless" } ] }
-
-    let(:valid_response) { MockResponse.new(200,
+    let(:root_response) { MockResponse.new(200,
                                             {"tagline" => "You Know, for Search",
                                                   "version" => {
                                                     "number" => '8.9.0',
-                                                    "build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOUR_SERVERLESS} },
+                                                    "build_flavor" => LogStash::Outputs::ElasticSearch::HttpClient::Pool::BUILD_FLAVOR_SERVERLESS} },
                                             { "X-Elastic-Product" => "Elasticsearch" }
     ) }
 
     before(:each) do
-      allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
+      allow(subject).to receive(:perform_request_to_url).and_return(root_response)
       subject.start
     end
 
-    it "picks the build flavour" do
+    it "picks the build flavor" do
       expect(subject.serverless?).to be_truthy
     end
   end
 
   describe "license checking" do
     before(:each) do
-      allow(subject).to receive(:health_check_request)
+      allow(subject).to receive(:health_check_request).and_return(["", nil])
+      allow(subject).to receive(:perform_request_to_url).and_return(root_response)
       allow(subject).to receive(:elasticsearch?).and_return(true)
     end
 
@@ -327,6 +440,36 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
 
+  describe "elastic api version header" do
+    let(:eav) { "Elastic-Api-Version" }
+
+    context "when it is serverless" do
+      before(:each) do
+        expect(subject).to receive(:serverless?).and_return(true)
+      end
+
+      it "add the default header" do
+        expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |_, _, _, params, _|
+          expect(params[:headers]).to eq({ "User-Agent" => "chromium",  "Elastic-Api-Version" => "2023-10-31"})
+        end
+        subject.perform_request_to_url(initial_urls, :get, "/", { :headers => { "User-Agent" => "chromium" }} )
+      end
+    end
+
+    context "when it is stateful" do
+      before(:each) do
+        expect(subject).to receive(:serverless?).and_return(false)
+      end
+
+      it "add the default header" do
+        expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |_, _, _, params, _|
+          expect(params[:headers]).to be_nil
+        end
+        subject.perform_request_to_url(initial_urls, :get, "/" )
+      end
+    end
+  end
+
   # TODO: extract to ElasticSearchOutputLicenseChecker unit spec
   describe "license checking with ElasticSearchOutputLicenseChecker" do
     let(:options) do
@@ -334,7 +477,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     before(:each) do
-      allow(subject).to receive(:health_check_request)
+      allow(subject).to receive(:health_check_request).and_return(["", nil])
+      allow(subject).to receive(:perform_request_to_url).and_return(root_response)
       allow(subject).to receive(:elasticsearch?).and_return(true)
     end
 
@@ -388,114 +532,71 @@ describe "#elasticsearch?" do
   let(:adapter) { double("Manticore Adapter") }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
-  let(:es_version_info) { [{ "number" => '0.0.0', "build_flavor" => 'default'}] }
-  let(:license_status) { 'active' }
 
   subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
 
-  let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
-
-  context "in case HTTP error code" do
-    it "should fail for 401" do
-      allow(adapter).to receive(:perform_request)
-        .with(anything, :get, "/", anything, anything)
-        .and_return(MockResponse.new(401))
-
-      expect(subject.elasticsearch?(url)).to be false
-    end
-
-    it "should fail for 403" do
-      allow(adapter).to receive(:perform_request)
-              .with(anything, :get, "/", anything, anything)
-              .and_return(status: 403)
-      expect(subject.elasticsearch?(url)).to be false
-    end
-  end
-
   context "when connecting to a cluster which reply without 'version' field" do
     it "should fail" do
-      allow(adapter).to receive(:perform_request)
-                    .with(anything, :get, "/", anything, anything)
-                    .and_return(body: {"field" => "funky.com"}.to_json)
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"field" => "funky.com"} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
   end
 
   context "when connecting to a cluster with version < 6.0.0" do
     it "should fail" do
-      allow(adapter).to receive(:perform_request)
-                          .with(anything, :get, "/", anything, anything)
-                          .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version" => { "number" => "5.0.0" }})
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
   end
 
   context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
     it "must be successful with valid 'tagline'" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
-      expect(subject.elasticsearch?(url)).to be true
+      resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"} )
+      expect(subject.send(:elasticsearch?, resp)).to be true
     end
 
     it "should fail if invalid 'tagline'" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
 
     it "should fail if 'tagline' is not present" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version" => {"number" => "6.5.0"}} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
   end
 
   context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
     it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
-      expect(subject.elasticsearch?(url)).to be true
+      resp = MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"} )
+      expect(subject.send(:elasticsearch?, resp)).to be true
     end
 
     it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
 
     it "should fail if 'build_flavor' is not present and tagline is correct" do
-      allow(adapter).to receive(:perform_request)
-                                .with(anything, :get, "/", anything, anything)
-                                .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
   end
 
   context "when connecting to a cluster with version >= 7.14.0" do
     it "should fail if 'X-elastic-product' header is not present" do
-      allow(adapter).to receive(:perform_request)
-                    .with(anything, :get, "/", anything, anything)
-                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version": {"number": "7.14.0"}} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
 
     it "should fail if 'X-elastic-product' header is present but with bad value" do
-      allow(adapter).to receive(:perform_request)
-                    .with(anything, :get, "/", anything, anything)
-                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
-      expect(subject.elasticsearch?(url)).to be false
+      resp = MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'} )
+      expect(subject.send(:elasticsearch?, resp)).to be false
     end
 
     it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
-      allow(adapter).to receive(:perform_request)
-                    .with(anything, :get, "/", anything, anything)
-                    .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
-      expect(subject.elasticsearch?(url)).to be true
+      resp = MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'} )
+      expect(subject.send(:elasticsearch?, resp)).to be true
     end
   end
 end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -135,7 +135,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     }
 
     it "returns the hash response" do
-      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.pool).to receive(:get).with(path).and_return(get_response)
       expect(subject.get(path)["body"]).to eq(body)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.17.0
+  version: 11.19.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-14 00:00:00.000000000 Z
+date: 2023-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement