logstash-output-elasticsearch 11.7.0-java → 11.9.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0002c4b7258cc2493f3ab5e1ddadf6dc3f3e6d51c3585af9c93f9d9a578cf01f
-  data.tar.gz: 1f4b7e4587ee5831ac32f499dba2e73324f5b00578db20369b46e2b53b0fa874
+  metadata.gz: ad30f3f3af7faaebf3409ac23375e99531def7d30940330f0f617e007f341b1d
+  data.tar.gz: d06a9954211995b266d08cfbc6586164275d5d269ef5716ba18d4f7e3ca4cf5c
 SHA512:
-  metadata.gz: a45147a1fc1f7d73bae911f05417d36150796edae9bd7622c11dbeb35bebbc938c3eddc7d7f65652743dfe94425ba4b1cc6905eaae4390116e2fa19d5c1f4be3
-  data.tar.gz: 2ad8fec97436ac571436407711793d8a97055f7a2fbe986ce9f75417d0ce414a01467c900c47c08e1caa7eb6ffd7884edcabedb12b3684c5b40ed543c040fc21
+  metadata.gz: ad84072b2dcc3d3a567b18cc89bbd3390e93303e952188053a8f61cbefbddb0a8bf46453fb8df0b94cecaf29304452b85fbe7a822bb9ece0763c4278b514ff91
+  data.tar.gz: 348d53c674b7b2730ce918403e705ddbda5f9745ca2564e922d3762bff9de7a455ff871c269669dc1e74d5e0c5f89c7997221eca4c851a073b1540b2a8bbdc15

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.9.0
+ - Feature: force unresolved dynamic index names to be sent into DLQ. This feature could be explicitly disabled using `dlq_on_failed_indexname_interpolation` setting [#1084](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1084)
+
+## 11.8.0
+ - Feature: Adds a new `dlq_custom_codes` option to customize DLQ codes [#1067](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1067)
+ 
 ## 11.7.0
  - Feature: deprecates the `failure_type_logging_whitelist` configuration option, renaming it `silence_errors_in_log` [#1068](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1068)
 

data/docs/index.asciidoc

@@ -196,6 +196,8 @@ processed at a later time. Often times, the offending field can be removed and
 re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
 happens, the problem is logged as a warning, and the event is dropped. See
 <<dead-letter-queues>> for more information about processing events in the DLQ.
+The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
+but should be used only in motivated cases.
 
 [id="plugins-{type}s-{plugin}-ilm"]
 ==== Index Lifecycle Management
@@ -317,6 +319,8 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
@@ -519,6 +523,25 @@ overwritten with a warning.
 The data stream type used to construct the data stream at index time.
 Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
 
+[id="plugins-{type}s-{plugin}-dlq_custom_codes"]
+===== `dlq_custom_codes`
+
+* Value type is <<number,number>>
+* Default value is `[]`.
+
+List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue.
+This list is an addition to the ordinary error codes considered for this feature, 400 and 404.
+It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+The option accepts a list of natural numbers corresponding to HTTP errors codes.
+
+[id="plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"]
+===== `dlq_on_failed_indexname_interpolation`
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`.
+
+If enabled, failed index name interpolation events go into dead letter queue.
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -9,6 +9,7 @@ require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
 require "forwardable"
+require "set"
 
 # .Compatibility Note
 # [NOTE]
@@ -255,6 +256,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ILM policy to use, if undefined the default policy will be used.
   config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
 
+  # List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
+  # It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+  # The option accepts a list of natural numbers corresponding to HTTP errors codes.
+  config :dlq_custom_codes, :validate => :number, :list => true, :default => []
+
+  # if enabled, failed index name interpolation events go into dead letter queue.
+  config :dlq_on_failed_indexname_interpolation, :validate => :boolean, :default => true
+
   attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
@@ -301,6 +310,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
+    @dlq_codes = DOC_DLQ_CODES.to_set
+
+    if dlq_enabled?
+      check_dlq_custom_codes
+      @dlq_codes.merge(dlq_custom_codes)
+    else
+      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
+    end
+
     if data_stream_config?
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
@@ -347,11 +365,43 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Receive an array of events and immediately attempt to index them (no buffering)
   def multi_receive(events)
     wait_for_successful_connection if @after_successful_connection_done
-    retrying_submit map_events(events)
+    events_mapped = safe_interpolation_map_events(events)
+    retrying_submit(events_mapped.successful_events)
+    unless events_mapped.failed_events.empty?
+      @logger.error("Can't map some events, needs to be handled by DLQ #{events_mapped.failed_events}")
+      send_failed_resolutions_to_dlq(events_mapped.failed_events)
+    end
   end
 
+  # @param: Arrays of EventActionTuple
+  private
+  def send_failed_resolutions_to_dlq(failed_action_tuples)
+    failed_action_tuples.each do |action|
+      handle_dlq_status(action, "warn", "Could not resolve dynamic index")
+    end
+  end
+
+  MapEventsResult = Struct.new(:successful_events, :failed_events)
+
+  private
+  def safe_interpolation_map_events(events)
+    successful_events = [] # list of LogStash::Outputs::ElasticSearch::EventActionTuple
+    failed_events = [] # list of LogStash::Event
+    events.each do |event|
+       begin
+        successful_events << @event_mapper.call(event)
+       rescue IndexInterpolationError, e
+         action = event.sprintf(@action || 'index')
+         event_action_tuple = EventActionTuple.new(action, [], event)
+         failed_events << event_action_tuple
+       end
+    end
+    MapEventsResult.new(successful_events, failed_events)
+  end
+
+  public
   def map_events(events)
-    events.map(&@event_mapper)
+    safe_interpolation_map_events(events).successful_events
   end
 
   def wait_for_successful_connection
@@ -426,12 +476,24 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   end
 
+  class IndexInterpolationError < ArgumentError
+    attr_reader :bad_formatted_index
+
+    def initialize(bad_formatted_index)
+      super("Badly formatted index, after interpolation still contains placeholder: [#{bad_formatted_index}]")
+
+      @bad_formatted_index = bad_formatted_index
+    end
+  end
+
   # @return Hash (initial) parameters for given event
   # @private shared event params factory between index and data_stream mode
   def common_event_params(event)
+    sprintf_index = @event_target.call(event)
+    raise IndexInterpolationError, sprintf_index if sprintf_index.match(/%{.*?}/) && dlq_on_failed_indexname_interpolation
     params = {
         :_id => @document_id ? event.sprintf(@document_id) : nil,
-        :_index => @event_target.call(event),
+        :_index => sprintf_index,
         routing_field_name => @routing ? event.sprintf(@routing) : nil
     }
 
@@ -539,4 +601,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
   end
+
+  def check_dlq_custom_codes
+    intersection = dlq_custom_codes & DOC_DLQ_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are already defined as standard DLQ error codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & DOC_SUCCESS_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are success codes which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & [DOC_CONFLICT_CODE]
+    raise LogStash::ConfigurationError, "#{intersection} are error codes already defined as conflict which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+  end
 end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -206,19 +206,23 @@ module LogStash; module PluginMixins; module ElasticSearch
       doubled > @retry_max_interval ? @retry_max_interval : doubled
     end
 
-    def handle_dlq_status(message, action, status, response)
+    def handle_dlq_response(message, action, status, response)
+      _, action_params = action.event, [action[0], action[1], action[2]]
+      
+      # TODO: Change this to send a map with { :status => status, :action => action } in the future
+      detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
+      
+      log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
+      
+      handle_dlq_status(action, log_level, detailed_message)
+    end
+
+    def handle_dlq_status(action, log_level, message)
       # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
-        event, action = action.event, [action[0], action[1], action[2]]
-        # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
+        @dlq_writer.write(action.event, "#{message}")
       else
-        if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
-          level = :error
-        else
-          level = :warn
-        end
-        @logger.send level, message, status: status, action: action, response: response
+        @logger.send log_level, message
       end
     end
 
@@ -268,8 +272,8 @@ module LogStash; module PluginMixins; module ElasticSearch
           @document_level_metrics.increment(:non_retryable_failures)
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
-        elsif DOC_DLQ_CODES.include?(status)
-          handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
+        elsif @dlq_codes.include?(status)
+          handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next
         else

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.7.0'
+  s.version         = '11.9.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/index_spec.rb

@@ -1,5 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
+require 'cgi'
 
 describe "TARGET_BULK_BYTES", :integration => true do
   let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
@@ -45,16 +46,57 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 end
 
-describe "indexing" do
+def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); require 'open3'
+  cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
+  begin
+    out, err, status = Open3.capture3(cmd)
+  rescue Errno::ENOENT
+    fail "curl not available, make sure curl binary is installed and available on $PATH"
+  end
+
+  if status.success?
+    http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+    if http_status.strip[0].to_i > 2
+      error = (LogStash::Json.load(out)['error']) rescue nil
+      if error
+        if retrieve_err_payload
+          return error
+        else
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        end
+      else
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
+    end
+
+    LogStash::Json.load(out)
+  else
+    warn out
+    fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+  end
+end
+
+describe "indexing with sprintf resolution", :integration => true do
   let(:message) { "Hello from #{__FILE__}" }
   let(:event) { LogStash::Event.new("message" => message, "type" => type) }
-  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let (:index) { "%{[index_name]}_dynamic" }
   let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-  let(:event_count) { 1 + rand(2) }
-  let(:config) { "not implemented" }
+  let(:event_count) { 1 }
+  let(:user) { "simpleuser" }
+  let(:password) { "abc123" }
+  let(:config) do
+    {
+      "hosts" => [ get_host_port ],
+      "user" => user,
+      "password" => password,
+      "index" => index
+    }
+  end
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::ElasticSearch.new(config) }
-  
+
   let(:es_url) { "http://#{get_host_port}" }
   let(:index_url) { "#{es_url}/#{index}" }
 
@@ -63,33 +105,65 @@ describe "indexing" do
   let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
   let(:es_admin_pass) { 'elastic' }
 
-  def curl_and_get_json_response(url, method: :get); require 'open3'
-    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
-    begin
-      out, err, status = Open3.capture3(cmd)
-    rescue Errno::ENOENT
-      fail "curl not available, make sure curl binary is installed and available on $PATH"
-    end
+  let(:initial_events) { [] }
 
-    if status.success?
-      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+  let(:do_register) { true }
 
-      if http_status.strip[0].to_i > 2
-        error = (LogStash::Json.load(out)['error']) rescue nil
-        if error
-          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
-        else
-          warn out
-          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
-        end
-      end
+  before do
+    subject.register if do_register
+    subject.multi_receive(initial_events) if initial_events
+  end
 
-      LogStash::Json.load(out)
-    else
-      warn out
-      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+  after do
+    subject.do_close
+  end
+
+  let(:event) { LogStash::Event.new("message" => message, "type" => type, "index_name" => "test") }
+
+  it "should index successfully when field is resolved" do
+    expected_index_name = "test_dynamic"
+    subject.multi_receive(events)
+
+#     curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+    result = curl_and_get_json_response "#{es_url}/#{expected_index_name}"
+
+    expect(result[expected_index_name]).not_to be(nil)
+  end
+
+  context "when dynamic field doesn't resolve the index_name" do
+    let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+    let(:dlq_writer) { double('DLQ writer') }
+    before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
+
+    it "should doesn't create an index name with unresolved placeholders" do
+      expect(dlq_writer).to receive(:write).once.with(event, /Could not resolve dynamic index/)
+      subject.multi_receive(events)
+
+      escaped_index_name = CGI.escape("%{[index_name]}_dynamic")
+      result = curl_and_get_json_response "#{es_url}/#{escaped_index_name}", retrieve_err_payload: true
+      expect(result["root_cause"].first()["type"]).to eq("index_not_found_exception")
     end
   end
+end
+
+describe "indexing" do
+  let(:message) { "Hello from #{__FILE__}" }
+  let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 1 + rand(2) }
+  let(:config) { "not implemented" }
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+  
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) { "#{es_url}/#{index}" }
+
+  let(:curl_opts) { nil }
+
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
 
   let(:initial_events) { [] }
 

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -260,6 +260,24 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
+    describe "when 'dlq_custom_codes'" do
+      let(:options) { super().merge('dlq_custom_codes' => [404]) }
+      let(:do_register) { false }
+
+      context "contains already defined codes" do
+        it "should raise a configuration error" do
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /are already defined as standard DLQ error codes/)
+        end
+      end
+
+      context "is configured but DLQ is not enabled" do
+        it "raise a configuration error" do
+          allow(subject).to receive(:dlq_enabled?).and_return(false)
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /configured while DLQ is not enabled/)
+        end
+      end
+    end if LOGSTASH_VERSION > '7.0'
+
     describe "#multi_receive" do
       let(:events) { [double("one"), double("two"), double("three")] }
       let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
@@ -750,6 +768,7 @@ describe LogStash::Outputs::ElasticSearch do
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
     let(:options) { { "manage_template" => false } }
+    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
@@ -759,8 +778,7 @@ describe LogStash::Outputs::ElasticSearch do
           it 'should log at ERROR level' do
             subject.instance_variable_set(:@logger, double("logger").as_null_object)
             mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
-            subject.handle_dlq_status("Could not index event to Elasticsearch.",
-              [:action, :params, :event], :some_status, mock_response)
+            subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
           end
         end
 
@@ -768,10 +786,9 @@ describe LogStash::Outputs::ElasticSearch do
           it 'should log at WARN level' do
             logger = double("logger").as_null_object
             subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-            subject.handle_dlq_status("Could not index event to Elasticsearch.",
-              [:action, :params, :event], :some_status, mock_response)
+            subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
           end
         end
 
@@ -779,11 +796,10 @@ describe LogStash::Outputs::ElasticSearch do
           it 'should not fail, but just log a warning' do
             logger = double("logger").as_null_object
             subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
             mock_response = { 'index' => {} }
             expect do
-              subject.handle_dlq_status("Could not index event to Elasticsearch.",
-                [:action, :params, :event], :some_status, mock_response)
+              subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
             end.to_not raise_error
           end
         end
@@ -803,11 +819,11 @@ describe LogStash::Outputs::ElasticSearch do
         expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
         action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
-        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
+        subject.handle_dlq_response("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
 
-    context 'with response status 400' do
+    context 'with error response status' do
 
       let(:options) { super().merge 'document_id' => '%{foo}' }
 
@@ -815,11 +831,13 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
 
+      let(:error_code) { 400 }
+
       let(:bulk_response) do
         {
             "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
             [{
-                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status" => error_code,
                            "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
                   }
             }]
@@ -830,21 +848,34 @@ describe LogStash::Outputs::ElasticSearch do
         allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
       end
 
-      it "should write event to DLQ" do
-        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
-          expect( args.size ).to eql 2
+      shared_examples "should write event to DLQ" do
+        it "should write event to DLQ" do
+          expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+            expect( args.size ).to eql 2
+
+            event, reason = *args
+            expect( event ).to be_a LogStash::Event
+            expect( event ).to be events.first
+            expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
+            expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
 
-          event, reason = *args
-          expect( event ).to be_a LogStash::Event
-          expect( event ).to be events.first
-          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
-          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+            method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+          end.once
+
+          event_action_tuples = subject.map_events(events)
+          subject.send(:submit, event_action_tuples)
+        end
+      end
+
+      context "is one of the predefined codes" do
+        include_examples "should write event to DLQ"
+      end
 
-          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
-        end.once
+      context "when user customized dlq_custom_codes option" do
+        let(:error_code) { 403 }
+        let(:options) { super().merge 'dlq_custom_codes' => [error_code] }
 
-        event_action_tuples = subject.map_events(events)
-        subject.send(:submit, event_action_tuples)
+        include_examples "should write event to DLQ"
       end
 
     end if LOGSTASH_VERSION > '7.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.7.0
+  version: 11.9.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-30 00:00:00.000000000 Z
+date: 2022-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement