logstash-output-elasticsearch 11.7.0-java → 11.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0002c4b7258cc2493f3ab5e1ddadf6dc3f3e6d51c3585af9c93f9d9a578cf01f
-  data.tar.gz: 1f4b7e4587ee5831ac32f499dba2e73324f5b00578db20369b46e2b53b0fa874
+  metadata.gz: 28dfa1d2a757a4cb14aa7972f13c13f78975a52db0cac8d39425fa6bb34a462c
+  data.tar.gz: 7a9b9e05f59e2d9024f8faf7d6d5faa58aeb123cb77fd4f89b88826787d5d4f5
 SHA512:
-  metadata.gz: a45147a1fc1f7d73bae911f05417d36150796edae9bd7622c11dbeb35bebbc938c3eddc7d7f65652743dfe94425ba4b1cc6905eaae4390116e2fa19d5c1f4be3
-  data.tar.gz: 2ad8fec97436ac571436407711793d8a97055f7a2fbe986ce9f75417d0ce414a01467c900c47c08e1caa7eb6ffd7884edcabedb12b3684c5b40ed543c040fc21
+  metadata.gz: 8ee227e4dcb1f1af9cb7d1baf11cabb1e4f1a8ab87781494ca71f2d3a9ad57e6635819827c00090f597f79d33d8724f1a2e5e4047f506a0fde144c0dc5d4d71a
+  data.tar.gz: 6ad8e717ade5b09f46cf4bab5d9749c782cf654dd3e69f79010a230237cd251b7fd034b18f42ed09ffb46aa729c8596eae62a3cb41a8b33c36a7c23e7084adcf

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.8.0
+ - Feature: Adds a new `dlq_custom_codes` option to customize DLQ codes [#1067](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1067)
+ 
 ## 11.7.0
  - Feature: deprecates the `failure_type_logging_whitelist` configuration option, renaming it `silence_errors_in_log` [#1068](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1068)
 

data/docs/index.asciidoc

@@ -196,6 +196,8 @@ processed at a later time. Often times, the offending field can be removed and
 re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
 happens, the problem is logged as a warning, and the event is dropped. See
 <<dead-letter-queues>> for more information about processing events in the DLQ.
+The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
+but should be used only in motivated cases.
 
 [id="plugins-{type}s-{plugin}-ilm"]
 ==== Index Lifecycle Management
@@ -317,6 +319,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
@@ -519,6 +522,17 @@ overwritten with a warning.
 The data stream type used to construct the data stream at index time.
 Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
 
+[id="plugins-{type}s-{plugin}-dlq_custom_codes"]
+===== `dlq_custom_codes`
+
+* Value type is <<number,number>>
+* Default value is `[]`.
+
+List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue.
+This list is an addition to the ordinary error codes considered for this feature, 400 and 404.
+It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+The option accepts a list of natural numbers corresponding to HTTP errors codes.
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -9,6 +9,7 @@ require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
 require "forwardable"
+require "set"
 
 # .Compatibility Note
 # [NOTE]
@@ -255,6 +256,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ILM policy to use, if undefined the default policy will be used.
   config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
 
+  # List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
+  # It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+  # The option accepts a list of natural numbers corresponding to HTTP errors codes.
+  config :dlq_custom_codes, :validate => :number, :list => true, :default => []
+
   attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
@@ -301,6 +307,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
+    @dlq_codes = DOC_DLQ_CODES.to_set
+
+    if dlq_enabled?
+      check_dlq_custom_codes
+      @dlq_codes.merge(dlq_custom_codes)
+    else
+      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
+    end
+
     if data_stream_config?
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
@@ -539,4 +554,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
   end
+
+  def check_dlq_custom_codes
+    intersection = dlq_custom_codes & DOC_DLQ_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are already defined as standard DLQ error codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & DOC_SUCCESS_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are success codes which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & [DOC_CONFLICT_CODE]
+    raise LogStash::ConfigurationError, "#{intersection} are error codes already defined as conflict which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+  end
 end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -268,7 +268,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           @document_level_metrics.increment(:non_retryable_failures)
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
-        elsif DOC_DLQ_CODES.include?(status)
+        elsif @dlq_codes.include?(status)
           handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.7.0'
+  s.version         = '11.8.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -260,6 +260,24 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
+    describe "when 'dlq_custom_codes'" do
+      let(:options) { super().merge('dlq_custom_codes' => [404]) }
+      let(:do_register) { false }
+
+      context "contains already defined codes" do
+        it "should raise a configuration error" do
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /are already defined as standard DLQ error codes/)
+        end
+      end
+
+      context "is configured but DLQ is not enabled" do
+        it "raise a configuration error" do
+          allow(subject).to receive(:dlq_enabled?).and_return(false)
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /configured while DLQ is not enabled/)
+        end
+      end
+    end if LOGSTASH_VERSION > '7.0'
+
     describe "#multi_receive" do
       let(:events) { [double("one"), double("two"), double("three")] }
       let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
@@ -807,7 +825,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context 'with response status 400' do
+    context 'with error response status' do
 
       let(:options) { super().merge 'document_id' => '%{foo}' }
 
@@ -815,11 +833,13 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
 
+      let(:error_code) { 400 }
+
       let(:bulk_response) do
         {
             "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
             [{
-                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status" => error_code,
                            "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
                   }
             }]
@@ -830,21 +850,34 @@ describe LogStash::Outputs::ElasticSearch do
         allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
       end
 
-      it "should write event to DLQ" do
-        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
-          expect( args.size ).to eql 2
+      shared_examples "should write event to DLQ" do
+        it "should write event to DLQ" do
+          expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+            expect( args.size ).to eql 2
+
+            event, reason = *args
+            expect( event ).to be_a LogStash::Event
+            expect( event ).to be events.first
+            expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
+            expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
 
-          event, reason = *args
-          expect( event ).to be_a LogStash::Event
-          expect( event ).to be events.first
-          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
-          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+            method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+          end.once
+
+          event_action_tuples = subject.map_events(events)
+          subject.send(:submit, event_action_tuples)
+        end
+      end
+
+      context "is one of the predefined codes" do
+        include_examples "should write event to DLQ"
+      end
 
-          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
-        end.once
+      context "when user customized dlq_custom_codes option" do
+        let(:error_code) { 403 }
+        let(:options) { super().merge 'dlq_custom_codes' => [error_code] }
 
-        event_action_tuples = subject.map_events(events)
-        subject.send(:submit, event_action_tuples)
+        include_examples "should write event to DLQ"
       end
 
     end if LOGSTASH_VERSION > '7.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.7.0
+  version: 11.8.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-30 00:00:00.000000000 Z
+date: 2022-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement