logstash-output-elasticsearch 11.19.0-java → 11.20.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0490f311461b39c49e3edaf045af9a7172dbd97da8bc456bf044a031c18e5afb'
-  data.tar.gz: fbf05190ed352c12f0948aecfa6ed08e760b03985b83aa2e6830167a00c431bc
+  metadata.gz: 3f3f97949dede5ce24e127df3a0cf4c809f1436d1ff34831cea6d0b873784968
+  data.tar.gz: cadc8d310da90f1cc853b24734520859101cd787efe29b5371394248303999f9
 SHA512:
-  metadata.gz: b50084b6fb4fd31e7592be7e35e23963dce4cf284763baa4c1c042d1f159f20cebe09c1013ace38a986dbf4c8ee3cc9ae4eda7f38042956bd10dbdfc0ad20114
-  data.tar.gz: 91643fa15c48a29a02659967c75e0b879f98af2a6e7f1582c83b141f67826337ccaa3d0513b1372266dd068cbccfcdbbe59a5fcf36ceff81318126dd1bdf64cc
+  metadata.gz: 9ef570b20433e6b3d69c24c9e402c39aa59a9f5556cb8c03f5643be65a7800b1a66b744be3095d11fe640a262902506fa78a7a10c3ea5b6ce1dfa81da98ea4e9
+  data.tar.gz: c323b23e45dfa795df06fdd206134ba340c5745529c690543bf94cfdb81526306c415723bb7a9cba759df658f38cbe6b6c62a4e6d1bbc6d777b224da830e8613

data/CHANGELOG.md

@@ -1,5 +1,11 @@
+## 11.20.1
+  - Doc: Replace `document_already_exist_exception` with `version_conflict_engine_exception` in the `silence_errors_in_log` setting example [#1159](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1159)
+
+## 11.20.0
+  - Changed the register to initiate pipeline shutdown upon bootstrap failure instead of simply logging the error [#1151](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1151)
+
 ## 11.19.0
-  - Added `filter_path` to bulk requests to reduce the size of responses from elasticsearch
+  - Added `filter_path` to bulk requests to reduce the size of responses from elasticsearch [#1154](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1154)
   
 ## 11.18.0
   - Added request header `Elastic-Api-Version` for serverless [#1147](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1147)

data/docs/index.asciidoc

@@ -988,12 +988,12 @@ if enabled, script is in charge of creating non-existent document (scripted upda
 
 Defines the list of Elasticsearch errors that you don't want to log.
 A useful example is when you want to skip all 409 errors
-which are `document_already_exists_exception`.
+which are `version_conflict_engine_exception`.
 
 [source,ruby]
     output {
       elasticsearch {
-        silence_errors_in_log => ["document_already_exists_exception"]
+        silence_errors_in_log => ["version_conflict_engine_exception"]
       }
     }
 

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -145,12 +145,12 @@ module LogStash module Outputs class ElasticSearch
     # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
-      fail 'no last_es_version' unless last_es_version # assert - should not happen
+      raise LogStash::ConfigurationError 'no last_es_version' unless last_es_version # assert - should not happen
       es_version = ::Gem::Version.create(last_es_version)
       if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
         @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
         # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
-        raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
+        raise LogStash::ConfigurationError, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
                                "(detected version #{es_version.version}), please upgrade your cluster"
       end
       es_version # return truthy

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -28,7 +28,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @response_code == 403
       end
 
+      def too_many_requests?
+        @response_code == 429
+      end
+
     end
+
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
 
@@ -69,7 +74,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @adapter = adapter
       @metric = options[:metric]
       @initial_urls = initial_urls
-      
+
       raise ArgumentError, "No URL Normalizer specified!" unless options[:url_normalizer]
       @url_normalizer = options[:url_normalizer]
       DEFAULT_OPTIONS.merge(options).tap do |merged|
@@ -159,7 +164,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
             :error_message => e.message,
             :class => e.class.name,
             :backtrace => e.backtrace
-            )
+          )
         end
       end
     end
@@ -197,11 +202,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         sniff(nodes)
       end
     end
-    
+
     def major_version(version_string)
       version_string.split('.').first.to_i
     end
-    
+
     def sniff(nodes)
       nodes.map do |id,info|
         # Skip master-only nodes
@@ -360,7 +365,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def update_urls(new_urls)
       return if new_urls.nil?
-      
+
       # Normalize URLs
       new_urls = new_urls.map(&method(:normalize_url))
 
@@ -388,14 +393,14 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
         logger.info? && logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
       end
-      
+
       # Run an inline healthcheck anytime URLs are updated
       # This guarantees that during startup / post-startup
       # sniffing we don't have idle periods waiting for the
       # periodic sniffer to allow new hosts to come online
-      healthcheck! 
+      healthcheck!
     end
-    
+
     def size
       @state_mutex.synchronize { @url_info.size }
     end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -37,7 +37,7 @@ module LogStash; module Outputs; class ElasticSearch
       template_path = default_template_path(es_major_version, ecs_compatibility)
       read_template_file(template_path)
     rescue => e
-      fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
+      raise LogStash::ConfigurationError, "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
     end
 
     def self.install(client, template_endpoint, template_name, template, template_overwrite)
@@ -99,9 +99,11 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.read_template_file(template_path)
-      raise ArgumentError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
+      raise LogStash::ConfigurationError, "Template file '#{template_path}' could not be found" unless ::File.exists?(template_path)
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
+    rescue => e
+      raise LogStash::ConfigurationError, "Failed to load template file '#{template_path}': #{e.message}"
     end
 
     def self.template_endpoint(plugin)

data/lib/logstash/outputs/elasticsearch.rb

@@ -322,12 +322,29 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
 
+    @shutdown_from_finish_register = Concurrent::AtomicBoolean.new(false)
     @after_successful_connection_thread = after_successful_connection do
       begin
         finish_register
         true # thread.value
+      rescue LogStash::ConfigurationError, LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        return e if pipeline_shutdown_requested?
+
+        # retry when 429
+        @logger.debug("Received a 429 status code during registration. Retrying..") && retry if too_many_requests?(e)
+
+        # shut down pipeline
+        if execution_context&.agent.respond_to?(:stop_pipeline)
+          details = { message: e.message, exception: e.class }
+          details[:backtrace] = e.backtrace if @logger.debug?
+          @logger.error("Failed to bootstrap. Pipeline \"#{execution_context.pipeline_id}\" is going to shut down", details)
+
+          @shutdown_from_finish_register.make_true
+          execution_context.agent.stop_pipeline(execution_context.pipeline_id)
+        end
+
+        e
       rescue => e
-        # we do not want to halt the thread with an exception as that has consequences for LS
         e # thread.value
       ensure
         @after_successful_connection_done.make_true
@@ -450,7 +467,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   private
 
   def stop_after_successful_connection_thread
-    @after_successful_connection_thread.join unless @after_successful_connection_thread.nil?
+    # avoid deadlock when finish_register calling execution_context.agent.stop_pipeline
+    # stop_pipeline triggers plugin close and the plugin close waits for after_successful_connection_thread to join
+    return if @shutdown_from_finish_register&.true?
+
+    @after_successful_connection_thread.join if @after_successful_connection_thread&.alive?
   end
 
   # Convert the event into a 3-tuple of action, params and event hash
@@ -599,6 +620,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     details = { message: e.message, exception: e.class, backtrace: e.backtrace }
     details[:body] = e.response_body if e.respond_to?(:response_body)
     @logger.error("Failed to install template", details)
+    raise e if register_termination_error?(e)
   end
 
   def setup_ecs_compatibility_related_defaults

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -149,7 +149,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         # Defines the list of Elasticsearch errors that you don't want to log.
         # A useful example is when you want to skip all 409 errors
-        # which are `document_already_exists_exception`.
+        # which are `version_conflict_engine_exception`.
         # Deprecates `failure_type_logging_whitelist`.
         :silence_errors_in_log => { :validate => :array, :default => [] },
 

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -407,5 +407,14 @@ module LogStash; module PluginMixins; module ElasticSearch
       return val if rest_keys.empty? || val == nil
       dig_value(val, *rest_keys)
     end
+
+    def register_termination_error?(e)
+      e.is_a?(LogStash::ConfigurationError) || e.is_a?(LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+
+    def too_many_requests?(e)
+      e.is_a?(LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError) &&
+        e.too_many_requests?
+    end
   end
 end; end; end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.19.0'
+  s.version         = '11.20.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/no_es_on_startup_spec.rb

@@ -33,11 +33,11 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers before documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_raise(
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: before docs are sent"), 'http://test.es/'
     )
     subject.register
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_return(ESHelper.es_version)
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_call_original
     subject.multi_receive([event1, event2])
     @es.indices.refresh
     r = @es.search(index: 'logstash-*')
@@ -45,13 +45,13 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers after documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_raise(
         ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: after docs are sent"), 'http://test.es/'
     )
     subject.register
     Thread.new do
       sleep 4
-      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_return(ESHelper.es_version)
+      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_root_path).with(any_args).and_call_original
     end
     subject.multi_receive([event1, event2])
     @es.indices.refresh

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -25,7 +25,7 @@ describe "whitelisting error types in expected behavior" do
           "create" => {
             "status" => 409, 
             "error" => {
-              "type" => "document_already_exists_exception",
+              "type" => "version_conflict_engine_exception",
               "reason" => "[shard] document already exists"
             }
           }
@@ -46,7 +46,7 @@ describe "whitelisting error types in expected behavior" do
   end
 
   describe "when failure logging is disabled for document exists error" do
-    let(:settings) { super().merge("silence_errors_in_log" => ["document_already_exists_exception"]) }
+    let(:settings) { super().merge("silence_errors_in_log" => ["version_conflict_engine_exception"]) }
 
     it "should log a failure on the action" do
       expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.19.0
+  version: 11.20.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-25 00:00:00.000000000 Z
+date: 2023-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement