logstash-output-elasticsearch 11.14.1-java → 11.15.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6405648edffa1a591049d280196f4e7d0e4812f029daf782346e76e202f40b0
-  data.tar.gz: 2a8c1ae4c5cf07839aba4a7416305bae027b5461602c8518770f716683643e42
+  metadata.gz: 6ad6b80e4c6db7a54c8b7a830f6a922a7057b2e7585820db8a6c093b57db3800
+  data.tar.gz: 82cd893394fc7ac08ae7a79b2cd953f80fbf258ec2faad225300a2ce516ab208
 SHA512:
-  metadata.gz: 9d5f80600ff713cd5429b7c1fe3dde0c665918e1ff1dbf69e8ca2e5a71a6c5e083031ad8951eeb561c0007e305bf2e808884ffdc97e9cd13aaa068df8cdd3e77
-  data.tar.gz: 1f5e4fe699a2c7189d84215a5631a42b20f608600ad10a97e464e19429586b5949d0f1570d67535e748e2f2e24a43bd323418b4fa114b8896eb568894ee35f82
+  metadata.gz: b861d195377c3eeadf4489d21780be6627597aa0b93992bd7378b7cf680f6f99ac120ae342a4b48739ee6ef2652aa9238ee9fc08838cbed0f6dae0c887b06daa
+  data.tar.gz: 16091be406132dbb590ad76a0ba2e83cb0729717d8229b410bccbcabc660a7a26146fd2c7aaae6976f954bd17f1f763c6353552699ffd67173efb9917b913658

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.15.1
+ - Move async finish_register to bottom of register to avoid race condition [#1125](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1125)
+
+## 11.15.0
+ - Added the ability to negatively acknowledge the batch under processing if the plugin is blocked in a retry-error-loop and a shutdown is requested. [#1119](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1119)
+
 ## 11.14.1
  - [DOC] Fixed incorrect pull request link on the CHANGELOG `11.14.0` entry [#1122](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1122)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -313,6 +313,27 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     data_stream_enabled = data_stream_config?
 
     setup_template_manager_defaults(data_stream_enabled)
+    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
+
+    @dlq_codes = DOC_DLQ_CODES.to_set
+
+    if dlq_enabled?
+      check_dlq_custom_codes
+      @dlq_codes.merge(dlq_custom_codes)
+    else
+      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
+    end
+
+    setup_mapper_and_target(data_stream_enabled)
+
+    @bulk_request_metrics = metric.namespace(:bulk_requests)
+    @document_level_metrics = metric.namespace(:documents)
+
+    if ecs_compatibility == :v8
+      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
+                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
+    end
 
     @after_successful_connection_thread = after_successful_connection do
       begin
@@ -326,18 +347,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       end
     end
 
-    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
-
-    @dlq_codes = DOC_DLQ_CODES.to_set
-
-    if dlq_enabled?
-      check_dlq_custom_codes
-      @dlq_codes.merge(dlq_custom_codes)
-    else
-      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
-    end
+  end
 
+  def setup_mapper_and_target(data_stream_enabled)
     if data_stream_enabled
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
@@ -346,14 +358,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @event_mapper = -> (e) { event_action_tuple(e) }
       @event_target = -> (e) { e.sprintf(@index) }
     end
-
-    @bulk_request_metrics = metric.namespace(:bulk_requests)
-    @document_level_metrics = metric.namespace(:documents)
-
-    if ecs_compatibility == :v8
-      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
-                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
-    end
   end
 
   # @override post-register when ES connection established
@@ -431,7 +435,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   def wait_for_successful_connection
     after_successful_connection_done = @after_successful_connection_done
     return unless after_successful_connection_done
-    stoppable_sleep 1 until after_successful_connection_done.true?
+    stoppable_sleep 1 until (after_successful_connection_done.true? || pipeline_shutdown_requested?)
+
+    if pipeline_shutdown_requested?
+      logger.info "Aborting the batch due to shutdown request while waiting for connections to become live"
+      abort_batch_if_available!
+    end
 
     status = @after_successful_connection_thread && @after_successful_connection_thread.value
     if status.is_a?(Exception) # check if thread 'halted' with an error

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -159,6 +159,8 @@ module LogStash; module PluginMixins; module ElasticSearch
     def after_successful_connection(&block)
       Thread.new do
         sleep_interval = @retry_initial_interval
+        # in case of a pipeline's shutdown_requested?, the method #close shutdown also this thread
+        # so no need to explicitly handle it here and return an AbortedBatchException.
         until successful_connection? || @stopping.true?
           @logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
           sleep_interval = sleep_for_interval(sleep_interval)
@@ -191,8 +193,14 @@ module LogStash; module PluginMixins; module ElasticSearch
             @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
           end
         rescue => e
-          @logger.error("Encountered an unexpected error submitting a bulk request, will retry",
-                        message: e.message, exception: e.class, backtrace: e.backtrace)
+          if abort_batch_present? && e.instance_of?(org.logstash.execution.AbortedBatchException)
+            # if Logstash support abort of a batch and the batch is aborting,
+            # bubble up the exception so that the pipeline can handle it
+            raise e
+          else
+            @logger.error("Encountered an unexpected error submitting a bulk request, will retry",
+                          message: e.message, exception: e.class, backtrace: e.backtrace)
+          end
         end
 
         # Everything was a success!
@@ -331,6 +339,11 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
+        if pipeline_shutdown_requested?
+          # when any connection is available and a shutdown is requested
+          # the batch can be aborted, eventually for future retry.
+          abort_batch_if_available!
+        end
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
@@ -349,6 +362,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         end
 
         sleep_interval = sleep_for_interval(sleep_interval)
+        if pipeline_shutdown_requested?
+          # In case ES side changes access credentials and a pipeline reload is triggered
+          # this error becomes a retry on restart
+          abort_batch_if_available!
+        end
         retry
       rescue => e # Stuff that should never happen - print out full connection issues
         @logger.error(
@@ -363,6 +381,19 @@ module LogStash; module PluginMixins; module ElasticSearch
       end
     end
 
+    def pipeline_shutdown_requested?
+        return super if defined?(super) # since LS 8.1.0
+        execution_context&.pipeline&.shutdown_requested
+      end
+
+    def abort_batch_if_available!
+      raise org.logstash.execution.AbortedBatchException.new if abort_batch_present?
+    end
+
+    def abort_batch_present?
+      ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
+    end
+
     def dlq_enabled?
       # TODO there should be a better way to query if DLQ is enabled
       # See more in: https://github.com/elastic/logstash/issues/8064

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.14.1'
+  s.version         = '11.15.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -50,6 +50,104 @@ describe LogStash::Outputs::ElasticSearch do
     subject.close
   end
 
+  context "check aborting of a batch" do
+    context "on an unreachable ES instance" do
+      let(:events) { [ ::LogStash::Event.new("foo" => "bar1"), ::LogStash::Event.new("foo" => "bar2") ] }
+
+      let(:shutdown_value) { true }
+
+      let(:logger) { double("logger") }
+
+      before(:each) do
+        allow(subject).to receive(:logger).and_return(logger)
+        allow(logger).to receive(:info)
+
+        allow(subject).to receive(:pipeline_shutdown_requested?) do
+          shutdown_value
+        end
+      end
+
+      it "the #multi_receive abort while waiting on unreachable and a shutdown is requested" do
+        expect { subject.multi_receive(events) }.to raise_error(org.logstash.execution.AbortedBatchException)
+        expect(logger).to have_received(:info).with(/Aborting the batch due to shutdown request while waiting for connections to become live/i)
+      end
+    end
+
+    context "when a connected ES becomes unreachable" do
+#       let(:error) do
+#                 ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+#                   429, double("url").as_null_object, request_body, double("response body")
+#                 )
+#               end
+
+      shared_examples 'raise an abort error' do
+        let(:options) {
+          {
+            "index" => "my-index",
+            "hosts" => ["localhost","localhost:9202"],
+            "path" => "some-path",
+            "manage_template" => false
+          }
+        }
+
+        let(:manticore_urls) { subject.client.pool.urls }
+        let(:manticore_url) { manticore_urls.first }
+
+        let(:stub_http_client_pool!) do
+          [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+            allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+          end
+        end
+
+        let(:event) { ::LogStash::Event.new("foo" => "bar") }
+
+        let(:logger) { double("logger").as_null_object }
+        let(:response) { { :errors => [], :items => [] } }
+
+        let(:request_body) { double(:request_body, :bytesize => 1023) }
+
+        before(:each) do
+          bulk_param =  [["index", anything, event.to_hash]]
+
+          allow(subject).to receive(:logger).and_return(logger)
+
+          # fail consistently for ever
+          allow(subject.client).to receive(:bulk).with(bulk_param).and_raise(error)
+        end
+
+        it "should exit the retry with an abort exception if shutdown is requested" do
+          # execute in another thread because it blocks in a retry loop until the shutdown is triggered
+          th = Thread.new do
+            subject.multi_receive([event])
+          rescue org.logstash.execution.AbortedBatchException => e
+            # return exception's class so that it can be verified when retrieving the thread's value
+            e.class
+          end
+
+          # trigger the shutdown signal
+          allow(subject).to receive(:pipeline_shutdown_requested?) { true }
+
+          expect(th.value).to eq(org.logstash.execution.AbortedBatchException)
+        end
+      end
+
+      context "with 429 error" do
+        let(:error) do
+          ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+            429, double("url").as_null_object, request_body, double("response body")
+          )
+        end
+
+        it_behaves_like 'raise an abort error'
+      end
+
+      context "with 'no connections' error" do
+        let(:error) { ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError.new }
+
+        it_behaves_like 'raise an abort error'
+      end
+    end
+  end if LOGSTASH_VERSION >= '8.8'
 
   context "with an active instance" do
     let(:options) {
@@ -1358,6 +1456,42 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
     end
+
+    context 'during register/finish_register' do
+
+      let(:options) { { 'hosts' => '127.0.0.1:9999', 'data_stream' => 'true' } }
+      let(:es_version) { '8.7.0' } # DS default on LS 8.x
+
+      before do
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:maybe_create_rollover_alias)
+        allow(subject).to receive(:maybe_create_ilm_policy)
+        allow(subject).to receive(:build_client)
+      end
+
+      # this test could not have been done using latches as the fix to the race
+      # condition alters the order of the instructions in elasticsearch.rb
+      #
+      # the race condition happened when the @index was set to the datastream
+      # after `ilm_in_use?` is called but before `setup_ilm`
+      it 'doesn\'t have a race condition leading to resetting back to ILM' do
+        ilm_in_use = subject.method(:ilm_in_use?)
+        expect(subject).to receive(:ilm_in_use?) do |params|
+          ret = ilm_in_use.call
+          sleep 3
+          ret
+        end
+        setup_mapper_and_target = subject.method(:setup_mapper_and_target)
+        expect(subject).to receive(:setup_mapper_and_target).once do |params|
+          sleep 1
+          setup_mapper_and_target.call(params)
+        end
+        subject.register
+        sleep 6
+        expect(subject.index).to eq("logs-generic-default")
+      end
+
+    end
   end
 
   @private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.14.1
+  version: 11.15.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-22 00:00:00.000000000 Z
+date: 2023-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -356,7 +356,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.29
 signing_key:
 specification_version: 4
 summary: Stores logs in Elasticsearch