logstash-output-elasticsearch 12.0.7-java → 12.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6da132cfd797a627e8a5fac299cc7b94a35750c3aec1ed3f046900ed7032c10f
-  data.tar.gz: 8a95530785c4fe01c075addd6a7d7b91ac486b8fcdf9259345e07ad51d372e92
+  metadata.gz: 2dc84ff1bfa23ba7aac5eea0cf36fc94fc0d3112878fda04fe2bf934fd42ccb8
+  data.tar.gz: b926a70cab6ba1d3415208512ee71fddf79ce8e46e9ab81d59354a1f79dd0cb5
 SHA512:
-  metadata.gz: c17d66090ac80e1af81d6cc4e83367c41c1c32645fc113045a77355d00483bcaf9ec78e55594ce94e7034ef285beac5b8463b92dcb57ba33e5b25c2cb12a48a8
-  data.tar.gz: 1b475b8b4f912715c114781dcb0d7a30190efd5cb5dc7afc2a6905ac372f19b53915ce0a0b75f57068225e73e9fbaa41fdbb027730c9bfd8be367da61f22a11c
+  metadata.gz: a3f18999365521054239b3d8316134dfb7c33e7f486fbe8e0de2467f25c127eebb42404b8ad649bb1aa2395464bdb1324b4006104ae031a73648e724093afa46
+  data.tar.gz: ab7ee85a0bd5cbee9e18dd0cd2f26be93d1da3857d206948d5a4b392066bfcad51b0ab0ebc3eb5c6418388782855d81b7584e3ec9fb8cc6cb6c71a913657b5ce

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 12.1.0
+ - Add drop_error_types config option to not retry after certain error types [#1228](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1228)
+
 ## 12.0.7
  - Support both, encoded and non encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223)
 

data/docs/index.asciidoc

@@ -373,6 +373,7 @@ Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-drop_error_types>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
@@ -644,6 +645,24 @@ If you don't set a value for this option:
 - for elasticsearch clusters 8.x: no value will be used;
 - for elasticsearch clusters 7.x: the value of '_doc' will be used.
 
+[id="plugins-{type}s-{plugin}-drop_error_types"]
+===== `drop_error_types`
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+Lists the set of error types for which individual bulk request actions will not be retried. Unless an individual - document level - action returns 409 or an error from this list, failures will be retried indefinitely.
+A warning message will be logged indicating that the action failed, unless the error type is
+listed in the <<plugins-{type}s-{plugin}-silence_errors_in_log>> config option.
+Note that the events are not added to the Dead Letter Queue (DLQ), regardless of whether it is enabled.
+
+[source,ruby]
+    output {
+      elasticsearch {
+        drop_error_types => ["index_closed_exception"]
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-ecs_compatibility"]
 ===== `ecs_compatibility`
 

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -204,6 +204,10 @@ module LogStash; module PluginMixins; module ElasticSearch
         # if enabled, failed index name interpolation events go into dead letter queue.
         :dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true },
 
+        # Failures on actions from a bulk request will not be retried for these error types; the events will be dropped.
+        # The events won't be added to the DLQ either.
+        :drop_error_types => { :validate => :string, :list => true, :default => [] },
+
         # Obsolete Settings
         :ssl => { :obsolete => "Set 'ssl_enabled' instead." },
         :ssl_certificate_verification => { :obsolete => "Set 'ssl_verification_mode' instead." },

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -278,16 +278,18 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         status = action_props["status"]
         error  = action_props["error"]
+        type = error["type"] if error
         action = actions[idx]
 
-        # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
+        # Retry logic: If it is success, we move on. If it is a failure, we have the following paths:
         # - For 409, we log and drop. there is nothing we can do
+        # - For any error types set in the 'drop_error_types' config, log and drop.
         # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
         # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
         if DOC_SUCCESS_CODES.include?(status)
           @document_level_metrics.increment(:successes)
           next
-        elsif DOC_CONFLICT_CODE == status
+        elsif DOC_CONFLICT_CODE == status || @drop_error_types.include?(type)
           @document_level_metrics.increment(:non_retryable_failures)
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.0.7'
+  s.version         = '12.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1499,6 +1499,80 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe 'drop_error_types' do
+
+    let(:error_type) { 'index_closed_exception' }
+
+    let(:options) { super().merge('drop_error_types' => [error_type]) }
+
+    let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+    let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+    let(:error_code) { 403 }
+
+    let(:event_action_tuples) { subject.map_events(events) }
+
+    let(:bulk_response) do
+      {
+        "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+        [{
+           "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status" => error_code,
+                     "error"=>{"type" => error_type, "reason" => "TEST" }
+           }
+         }]
+      }
+    end
+
+    before(:each) do
+      allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+    end
+
+    context 'DLQ is enabled' do
+
+      let(:options) { super().merge("dlq_custom_codes" => [403]) }
+
+      it 'does not write the event to the DLQ' do
+        expect(dlq_writer).not_to receive(:write)
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+
+    context 'DLQ is not enabled' do
+
+      before(:each) do
+        allow(subject).to receive(:dlq_enabled?).and_return(false)
+      end
+
+      it 'does not retry indexing the event' do
+        expect(subject).to receive(:submit).with(event_action_tuples).once.and_call_original
+        subject.send(:retrying_submit, event_action_tuples)
+      end
+    end
+
+    context 'the error type is not in `silence_errors_in_log`' do
+
+      it 'logs the error' do
+        expect(subject.logger).to receive(:warn).with(a_string_including("Failed action"), anything)
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+
+    context 'the error type is in `silence_errors_in_log`' do
+
+      let(:options) { super().merge('silence_errors_in_log' => [error_type]) }
+
+      before(:each) do
+        # ensure that neither warn nor info is called on the logger by using a test double
+        subject.instance_variable_set("@logger", double('logger'))
+      end
+
+      it 'does not log the error' do
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+  end
+
   describe "custom headers" do
     let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 12.0.7
+  version: 12.1.0
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2025-09-23 00:00:00.000000000 Z
+date: 2025-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore