logstash-output-elasticsearch 11.18.0-java → 11.20.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b42642e174e8a6f0bf30c67cbdf25e27cc8197f2aeb56cb2268fb11f65426a03
-  data.tar.gz: 26ca32e908ef3d42ec4281259ea9a6bf31746031b12c14ad08fbfd189c069c3b
+  metadata.gz: d6658ce0c72fc994e737d823dc03e53abf84d00c2692450173684fa9fc8a5865
+  data.tar.gz: 466cc891690d71b25df8992e93c7df1b64a09ab68b8e109fe4c4e8cf0faf25dc
 SHA512:
-  metadata.gz: dd0b5731beb34c4e331a8ea52252c4b53af5d8c62ffb97c106eab961709d0d884a02a228d48a236eee635629d23a6c2243528a20e3f1f7368f609e232f26d7f7
-  data.tar.gz: 79e980b61c3bdd1b3339f1f03eccff4c52fe596fc34e6069fac1239a5c67e17357d118a95d9536bc1937f930f1c60f935858c236447f5b87f8517ec9b79ef53e
+  metadata.gz: 8116f03d6cd876a5e6997c07e3a12119c667e30061ca44bd8cdb70987d75659f4cd02bdb6bce15413c4e5c2d7fdac639aa10f95142006e85dcf2e9d248eedc85
+  data.tar.gz: 67a56fc6ea82742bbc2605fb09708f017d199fb39a1b59015ccd4b5ffdd8fd9a917dde68d7f2ff23448ef78eb4006af40e1521a8c754d0d38ace29a0a267109b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.20.0
+  - Changed the register to initiate pipeline shutdown upon bootstrap failure instead of simply logging the error [#1151](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1151)
+
+## 11.19.0
+  - Added `filter_path` to bulk requests to reduce the size of responses from elasticsearch [#1154](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1154)
+  
 ## 11.18.0
   - Added request header `Elastic-Api-Version` for serverless [#1147](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1147)
 

data/docs/index.asciidoc

@@ -423,10 +423,12 @@ Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
 ===== `bulk_path`
 
   * Value type is <<string,string>>
-  * There is no default value for this setting.
+  * The default value for this settings is `/_bulk?filter_path=errors,items.*.error,items.*.status`
 
 HTTP Path to perform the _bulk requests to
-this defaults to a concatenation of the path parameter and "_bulk"
+* This default bulk path is the concatenation of the value of `path` parameter and `/_bulk?filter_path=errors,items.*.error,items.*.status`
+* The `filter_path` query parameter is appended to the bulk path to reduce the payload between logstash and elasticsearch. However, if a custom `filter_path` query parameter is included in the `bulk_path` setting, then that value will be used.
+
 
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -145,12 +145,12 @@ module LogStash module Outputs class ElasticSearch
     # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
-      fail 'no last_es_version' unless last_es_version # assert - should not happen
+      raise LogStash::ConfigurationError 'no last_es_version' unless last_es_version # assert - should not happen
       es_version = ::Gem::Version.create(last_es_version)
       if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
         @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
         # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
-        raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
+        raise LogStash::ConfigurationError, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
                                "(detected version #{es_version.version}), please upgrade your cluster"
       end
       es_version # return truthy

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -28,7 +28,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @response_code == 403
       end
 
+      def too_many_requests?
+        @response_code == 429
+      end
+
     end
+
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
 
@@ -69,7 +74,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @adapter = adapter
       @metric = options[:metric]
       @initial_urls = initial_urls
-      
+
       raise ArgumentError, "No URL Normalizer specified!" unless options[:url_normalizer]
       @url_normalizer = options[:url_normalizer]
       DEFAULT_OPTIONS.merge(options).tap do |merged|
@@ -159,7 +164,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
             :error_message => e.message,
             :class => e.class.name,
             :backtrace => e.backtrace
-            )
+          )
         end
       end
     end
@@ -197,11 +202,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         sniff(nodes)
       end
     end
-    
+
     def major_version(version_string)
       version_string.split('.').first.to_i
     end
-    
+
     def sniff(nodes)
       nodes.map do |id,info|
         # Skip master-only nodes
@@ -360,7 +365,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def update_urls(new_urls)
       return if new_urls.nil?
-      
+
       # Normalize URLs
       new_urls = new_urls.map(&method(:normalize_url))
 
@@ -388,14 +393,14 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
         logger.info? && logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
       end
-      
+
       # Run an inline healthcheck anytime URLs are updated
       # This guarantees that during startup / post-startup
       # sniffing we don't have idle periods waiting for the
       # periodic sniffer to allow new hosts to come online
-      healthcheck! 
+      healthcheck!
     end
-    
+
     def size
       @state_mutex.synchronize { @url_info.size }
     end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -177,6 +177,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def bulk_send(body_stream, batch_actions)
       params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
+
       response = @pool.post(@bulk_path, params, body_stream.string)
 
       @bulk_response_metrics.increment(response.code.to_s)

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -33,9 +33,9 @@ module LogStash; module Outputs; class ElasticSearch;
       end
 
       common_options[:bulk_path] = if params["bulk_path"]
-         dedup_slashes("/#{params["bulk_path"]}")
+        resolve_filter_path(dedup_slashes("/#{params["bulk_path"]}"))
       else
-         dedup_slashes("/#{params["path"]}/_bulk")
+        resolve_filter_path(dedup_slashes("/#{params["path"]}/_bulk"))
       end
 
       common_options[:sniffing_path] = if params["sniffing_path"]
@@ -197,5 +197,16 @@ module LogStash; module Outputs; class ElasticSearch;
     def self.dedup_slashes(url)
       url.gsub(/\/+/, "/")
     end
+
+    # Set a `filter_path` query parameter if it is not already set to be
+    # `filter_path=errors,items.*.error,items.*.status` to reduce the payload between Logstash and Elasticsearch
+    def self.resolve_filter_path(url)
+      return url if url.match?(/(?:[&|?])filter_path=/)
+      ("#{url}#{query_param_separator(url)}filter_path=errors,items.*.error,items.*.status")
+    end
+
+    def self.query_param_separator(url)
+      url.match?(/\?[^\s#]+/) ? '&' : '?'
+    end
   end
 end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -37,7 +37,7 @@ module LogStash; module Outputs; class ElasticSearch
       template_path = default_template_path(es_major_version, ecs_compatibility)
       read_template_file(template_path)
     rescue => e
-      fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
+      raise LogStash::ConfigurationError, "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
     end
 
     def self.install(client, template_endpoint, template_name, template, template_overwrite)
@@ -99,9 +99,11 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.read_template_file(template_path)
-      raise ArgumentError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
+      raise LogStash::ConfigurationError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
+    rescue => e
+      raise LogStash::ConfigurationError, "Failed to load template file '#{template_path}': #{e.message}"
     end
 
     def self.template_endpoint(plugin)

data/lib/logstash/outputs/elasticsearch.rb

@@ -322,12 +322,29 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
 
+    @shutdown_from_finish_register = Concurrent::AtomicBoolean.new(false)
     @after_successful_connection_thread = after_successful_connection do
       begin
         finish_register
         true # thread.value
+      rescue LogStash::ConfigurationError, LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        return e if pipeline_shutdown_requested?
+
+        # retry when 429
+        @logger.debug("Received a 429 status code during registration. Retrying..") && retry if too_many_requests?(e)
+
+        # shut down pipeline
+        if execution_context&.agent.respond_to?(:stop_pipeline)
+          details = { message: e.message, exception: e.class }
+          details[:backtrace] = e.backtrace if @logger.debug?
+          @logger.error("Failed to bootstrap. Pipeline \"#{execution_context.pipeline_id}\" is going to shut down", details)
+
+          @shutdown_from_finish_register.make_true
+          execution_context.agent.stop_pipeline(execution_context.pipeline_id)
+        end
+
+        e
       rescue => e
-        # we do not want to halt the thread with an exception as that has consequences for LS
         e # thread.value
       ensure
         @after_successful_connection_done.make_true
@@ -450,7 +467,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   private
 
   def stop_after_successful_connection_thread
-    @after_successful_connection_thread.join unless @after_successful_connection_thread.nil?
+    # avoid deadlock when finish_register calling execution_context.agent.stop_pipeline
+    # stop_pipeline triggers plugin close and the plugin close waits for after_successful_connection_thread to join
+    return if @shutdown_from_finish_register&.true?
+
+    @after_successful_connection_thread.join if @after_successful_connection_thread&.alive?
   end
 
   # Convert the event into a 3-tuple of action, params and event hash
@@ -599,6 +620,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     details = { message: e.message, exception: e.class, backtrace: e.backtrace }
     details[:body] = e.response_body if e.respond_to?(:response_body)
     @logger.error("Failed to install template", details)
+    raise e if register_termination_error?(e)
   end
 
   def setup_ecs_compatibility_related_defaults

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -407,5 +407,14 @@ module LogStash; module PluginMixins; module ElasticSearch
       return val if rest_keys.empty? || val == nil
       dig_value(val, *rest_keys)
     end
+
+    def register_termination_error?(e)
+      e.is_a?(LogStash::ConfigurationError) || e.is_a?(LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+
+    def too_many_requests?(e)
+      e.is_a?(LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError) &&
+        e.too_many_requests?
+    end
   end
 end; end; end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.18.0'
+  s.version         = '11.20.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/index_spec.rb

@@ -156,7 +156,7 @@ describe "indexing" do
   let(:config) { "not implemented" }
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::ElasticSearch.new(config) }
-  
+  let(:filter_path) { "filter_path=errors,items.*.error,items.*.status"}
   let(:es_url) { "http://#{get_host_port}" }
   let(:index_url) { "#{es_url}/#{index}" }
 
@@ -178,7 +178,7 @@ describe "indexing" do
     subject.do_close
   end
 
-  shared_examples "an indexer" do |secure|
+  shared_examples "an indexer" do |secure, expected_path|
     before(:each) do
       host_unreachable_error_class = LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError
       allow(host_unreachable_error_class).to receive(:new).with(any_args).and_wrap_original do |m, original, url|
@@ -212,13 +212,13 @@ describe "indexing" do
         expect(doc["_index"]).to eq(index)
       end
     end
-    
+
     it "sets the correct content-type header" do
       expected_manticore_opts = {:headers => {"Content-Type" => "application/json"}, :body => anything}
       if secure
         expected_manticore_opts = {
-          :headers => {"Content-Type" => "application/json"}, 
-          :body => anything, 
+          :headers => {"Content-Type" => "application/json"},
+          :body => anything,
           :auth => {
             :user => user,
             :password => password,
@@ -230,6 +230,20 @@ describe "indexing" do
         and_call_original
       subject.multi_receive(events)
     end
+
+    it "sets the bulk path URL and filter path parameter correctly" do
+      expect(subject.client.pool.adapter.client).to receive(:send).
+          with(anything, expected_path != nil ? expected_path : anything, anything).at_least(:once).and_call_original
+      subject.multi_receive(events)
+    end
+
+    it "receives a filtered response" do
+      expect(subject.client).to receive(:join_bulk_responses).
+          with([{"errors"=>false, "items"=>[{"index"=>{"status"=>201}}]}]).
+          and_call_original
+      subject.multi_receive([event])
+    end
+
   end
 
   shared_examples "PKIX path failure" do
@@ -269,6 +283,65 @@ describe "indexing" do
     it_behaves_like("an indexer")
   end
 
+  describe "an indexer with custom bulk path", :integration => true do
+    let(:bulk_path) { "/_bulk?routing=true"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}#{bulk_path}&#{filter_path}" }
+    end
+  end
+
+  describe "an indexer with filter path as second parameter", :integration => true do
+    let(:bulk_path) { "/_bulk?routing=true&#{filter_path}"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/#{bulk_path}" }
+    end
+  end
+
+  describe "an indexer with filter path as first  parameter", :integration => true do
+    let(:bulk_path) { "/_bulk?#{filter_path}&routing=true"}
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false,
+          "bulk_path" => bulk_path
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/#{bulk_path}" }
+    end
+  end
+
+  describe "an indexer with the standard bulk path", :integration => true do
+    let(:config) {
+      {
+          "hosts" => get_host_port,
+          "index" => index,
+          "http_compression" => false
+      }
+    }
+    it_behaves_like("an indexer", false) do
+      let (:expected_path) { "#{es_url}/_bulk?#{filter_path}" }
+    end
+
+  end
+
   describe "an indexer with no type value set (default to doc)", :integration => true do
     let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
     let(:config) {
@@ -296,7 +369,7 @@ describe "indexing" do
         "index" => index,
         "http_compression" => false
       }
-    end 
+    end
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 

data/spec/integration/outputs/no_es_on_startup_spec.rb

@@ -33,11 +33,11 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers before documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_raise(
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: before docs are sent"), 'http://test.es/'
     )
     subject.register
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_return(ESHelper.es_version)
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_call_original
     subject.multi_receive([event1, event2])
     @es.indices.refresh
     r = @es.search(index: 'logstash-*')
@@ -45,13 +45,13 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers after documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_raise(
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: after docs are sent"), 'http://test.es/'
     )
     subject.register
     Thread.new do
       sleep 4
-      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_return(ESHelper.es_version)
+      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_call_original
     end
     subject.multi_receive([event1, event2])
     @es.indices.refresh

data/spec/unit/http_client_builder_spec.rb

@@ -36,7 +36,17 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
       end
     end
 
-    describe "healthcheck_path" do
+    describe "bulk_path" do
+      let (:filter_path) {"filter_path=errors,items.*.error,items.*.status"}
+
+      shared_examples("filter_path added to bulk path appropriately") do
+        it "sets the bulk_path option to the expected bulk path" do
+          expect(described_class).to receive(:create_http_client) do |options|
+            expect(options[:bulk_path]).to eq(expected_bulk_path)
+          end
+          described_class.build(logger, hosts, options)
+          end
+      end
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
@@ -44,21 +54,31 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:options) { super().merge("path" => "/path") }
-          it "ignores the path setting" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq(bulk_path)
-            end
-            described_class.build(logger, hosts, options)
-          end
+          let(:expected_bulk_path) { "#{bulk_path}?#{filter_path}" }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
+        end
+
+        context "when setting a filter path as first parameter"  do
+          let (:filter_path) {"filter_path=error"}
+          let(:bulk_path) { "/meh?#{filter_path}&routing=true" }
+          let(:expected_bulk_path) { bulk_path }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
+        end
+
+        context "when setting a filter path as second parameter" do
+          let (:filter_path) {"filter_path=error"}
+          let(:bulk_path) { "/meh?routing=true&#{filter_path}" }
+          let(:expected_bulk_path) { bulk_path }
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
+
         context "when not using path" do
+          let(:expected_bulk_path) { "#{bulk_path}?#{filter_path}"}
 
-          it "uses the bulk_path setting" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq(bulk_path)
-            end
-            described_class.build(logger, hosts, options)
-          end
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
       end
 
@@ -66,25 +86,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
+          let(:expected_bulk_path) { "#{path}/_bulk?#{filter_path}"}
           let(:options) { super().merge("path" => path) }
-          it "sets bulk_path to path+_bulk" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq("#{path}/_bulk")
-            end
-            described_class.build(logger, hosts, options)
-          end
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
 
         context "when not using path" do
-          it "sets the bulk_path to _bulk" do
-            expect(described_class).to receive(:create_http_client) do |options|
-              expect(options[:bulk_path]).to eq("/_bulk")
-            end
-            described_class.build(logger, hosts, options)
-          end
+          let(:expected_bulk_path) { "/_bulk?#{filter_path}"}
+
+          it_behaves_like "filter_path added to bulk path appropriately"
         end
       end
     end
+
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.18.0
+  version: 11.20.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-25 00:00:00.000000000 Z
+date: 2023-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement