logstash-output-elasticsearch 11.6.0-java → 11.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd68d4f20c14d9cac712f5069acb4085abbd6ed2a3e7d88dc99e5337de810608
-  data.tar.gz: c88b315e6cdd40597773a47be79c174e0c4b6a8aef8b355cef2e1f5bd6cdb536
+  metadata.gz: 28dfa1d2a757a4cb14aa7972f13c13f78975a52db0cac8d39425fa6bb34a462c
+  data.tar.gz: 7a9b9e05f59e2d9024f8faf7d6d5faa58aeb123cb77fd4f89b88826787d5d4f5
 SHA512:
-  metadata.gz: 44254336c7076ca9bf5289b753f4e472f1186a639a8f0572057b865b975a1e1f258598e30c2778b64782993a80a79456a42b88ef4c0915b17312de63362935c8
-  data.tar.gz: 808cc047ba6a3020ab334c11dbc123e22777e62894ac39c5557563b670d7b3adb4aac5326e98d6e9c505cc83743a93ccbacefec7e8458d7d2b4651431654d451
+  metadata.gz: 8ee227e4dcb1f1af9cb7d1baf11cabb1e4f1a8ab87781494ca71f2d3a9ad57e6635819827c00090f597f79d33d8724f1a2e5e4047f506a0fde144c0dc5d4d71a
+  data.tar.gz: 6ad8e717ade5b09f46cf4bab5d9749c782cf654dd3e69f79010a230237cd251b7fd034b18f42ed09ffb46aa729c8596eae62a3cb41a8b33c36a7c23e7084adcf

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.8.0
+ - Feature: Adds a new `dlq_custom_codes` option to customize DLQ codes [#1067](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1067)
+ 
+## 11.7.0
+ - Feature: deprecates the `failure_type_logging_whitelist` configuration option, renaming it `silence_errors_in_log` [#1068](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1068)
+
 ## 11.6.0
  - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#1074](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1074)
 

data/docs/index.asciidoc

@@ -196,6 +196,8 @@ processed at a later time. Often times, the offending field can be removed and
 re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
 happens, the problem is logged as a warning, and the event is dropped. See
 <<dead-letter-queues>> for more information about processing events in the DLQ.
+The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
+but should be used only in motivated cases.
 
 [id="plugins-{type}s-{plugin}-ilm"]
 ==== Index Lifecycle Management
@@ -317,6 +319,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
@@ -332,6 +335,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-silence_errors_in_log>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-manage_template>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-parent>> |<<string,string>>|No
@@ -518,6 +522,17 @@ overwritten with a warning.
 The data stream type used to construct the data stream at index time.
 Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
 
+[id="plugins-{type}s-{plugin}-dlq_custom_codes"]
+===== `dlq_custom_codes`
+
+* Value type is <<number,number>>
+* Default value is `[]`.
+
+List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue.
+This list is an addition to the ordinary error codes considered for this feature, 400 and 404.
+It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+The option accepts a list of natural numbers corresponding to HTTP errors codes.
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
 
@@ -584,9 +599,7 @@ of this setting affects the _default_ values of:
   * Value type is <<array,array>>
   * Default value is `[]`
 
-Set the Elasticsearch errors in the whitelist that you don't want to log.
-A useful example is when you want to skip all 409 errors
-which are `document_already_exists_exception`.
+NOTE: Deprecated, refer to <<plugins-{type}s-{plugin}-silence_errors_in_log>>.
 
 [id="plugins-{type}s-{plugin}-custom_headers"]
 ===== `custom_headers`
@@ -743,8 +756,7 @@ Indexes may not contain uppercase characters.
 For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
 Logstash uses
 http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
-formats] for the index pattern from event timestamp.
-
+formats] and the `@timestamp` field of each event is being used as source for the date.
 
 [id="plugins-{type}s-{plugin}-keystore"]
 ===== `keystore`
@@ -963,6 +975,25 @@ Set variable name passed to script (scripted update)
 
 if enabled, script is in charge of creating non-existent document (scripted update)
 
+[id="plugins-{type}s-{plugin}-silence_errors_in_log"]
+===== `silence_errors_in_log`
+
+* Value type is <<array,array>>
+* Default value is `[]`
+
+Defines the list of Elasticsearch errors that you don't want to log.
+A useful example is when you want to skip all 409 errors
+which are `document_already_exists_exception`.
+
+[source,ruby]
+    output {
+      elasticsearch {
+        silence_errors_in_log => ["document_already_exists_exception"]
+      }
+    }
+
+NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
+
 [id="plugins-{type}s-{plugin}-sniffing"]
 ===== `sniffing`
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -9,6 +9,7 @@ require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
 require "forwardable"
+require "set"
 
 # .Compatibility Note
 # [NOTE]
@@ -255,6 +256,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ILM policy to use, if undefined the default policy will be used.
   config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
 
+  # List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
+  # It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
+  # The option accepts a list of natural numbers corresponding to HTTP errors codes.
+  config :dlq_custom_codes, :validate => :number, :list => true, :default => []
+
   attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
@@ -266,6 +272,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def register
+    if !failure_type_logging_whitelist.empty?
+      log_message = "'failure_type_logging_whitelist' is deprecated and in a future version of Elasticsearch " +
+        "output plugin will be removed, please use 'silence_errors_in_log' instead."
+      @deprecation_logger.deprecated log_message
+      @logger.warn log_message
+      @silence_errors_in_log = silence_errors_in_log | failure_type_logging_whitelist
+    end
+
     @after_successful_connection_done = Concurrent::AtomicBoolean.new(false)
     @stopping = Concurrent::AtomicBoolean.new(false)
 
@@ -293,6 +307,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
+    @dlq_codes = DOC_DLQ_CODES.to_set
+
+    if dlq_enabled?
+      check_dlq_custom_codes
+      @dlq_codes.merge(dlq_custom_codes)
+    else
+      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
+    end
+
     if data_stream_config?
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
@@ -531,4 +554,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
   end
+
+  def check_dlq_custom_codes
+    intersection = dlq_custom_codes & DOC_DLQ_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are already defined as standard DLQ error codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & DOC_SUCCESS_CODES
+    raise LogStash::ConfigurationError, "#{intersection} are success codes which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+
+    intersection = dlq_custom_codes & [DOC_CONFLICT_CODE]
+    raise LogStash::ConfigurationError, "#{intersection} are error codes already defined as conflict which cannot be redefined in dlq_custom_codes" unless intersection.empty?
+  end
 end

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -99,10 +99,14 @@ module LogStash; module PluginMixins; module ElasticSearch
         # a timeout occurs, the request will be retried.
         :timeout => { :validate => :number, :default => 60 },
 
-        # Set the Elasticsearch errors in the whitelist that you don't want to log.
+        # Deprecated, refer to `silence_errors_in_log`.
+        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+
+        # Defines the list of Elasticsearch errors that you don't want to log.
         # A useful example is when you want to skip all 409 errors
         # which are `document_already_exists_exception`.
-        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+        # Deprecates `failure_type_logging_whitelist`.
+        :silence_errors_in_log => { :validate => :array, :default => [] },
 
         # While the output tries to reuse connections efficiently we have a maximum.
         # This sets the maximum number of open connections the output will create.

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -268,7 +268,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           @document_level_metrics.increment(:non_retryable_failures)
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
-        elsif DOC_DLQ_CODES.include?(status)
+        elsif @dlq_codes.include?(status)
           handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next
@@ -284,7 +284,7 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
 
     def log_failure_type?(failure)
-      !failure_type_logging_whitelist.include?(failure["type"])
+      !silence_errors_in_log.include?(failure["type"])
     end
 
     # Rescue retryable errors during bulk submission

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.6.0'
+  s.version         = '11.8.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -260,6 +260,24 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
+    describe "when 'dlq_custom_codes'" do
+      let(:options) { super().merge('dlq_custom_codes' => [404]) }
+      let(:do_register) { false }
+
+      context "contains already defined codes" do
+        it "should raise a configuration error" do
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /are already defined as standard DLQ error codes/)
+        end
+      end
+
+      context "is configured but DLQ is not enabled" do
+        it "raise a configuration error" do
+          allow(subject).to receive(:dlq_enabled?).and_return(false)
+          expect{ subject.register }.to raise_error(LogStash::ConfigurationError, /configured while DLQ is not enabled/)
+        end
+      end
+    end if LOGSTASH_VERSION > '7.0'
+
     describe "#multi_receive" do
       let(:events) { [double("one"), double("two"), double("three")] }
       let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
@@ -807,7 +825,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context 'with response status 400' do
+    context 'with error response status' do
 
       let(:options) { super().merge 'document_id' => '%{foo}' }
 
@@ -815,11 +833,13 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
 
+      let(:error_code) { 400 }
+
       let(:bulk_response) do
         {
             "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
             [{
-                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status" => error_code,
                            "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
                   }
             }]
@@ -830,21 +850,34 @@ describe LogStash::Outputs::ElasticSearch do
         allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
       end
 
-      it "should write event to DLQ" do
-        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
-          expect( args.size ).to eql 2
+      shared_examples "should write event to DLQ" do
+        it "should write event to DLQ" do
+          expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+            expect( args.size ).to eql 2
+
+            event, reason = *args
+            expect( event ).to be_a LogStash::Event
+            expect( event ).to be events.first
+            expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
+            expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
 
-          event, reason = *args
-          expect( event ).to be_a LogStash::Event
-          expect( event ).to be events.first
-          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
-          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+            method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+          end.once
+
+          event_action_tuples = subject.map_events(events)
+          subject.send(:submit, event_action_tuples)
+        end
+      end
+
+      context "is one of the predefined codes" do
+        include_examples "should write event to DLQ"
+      end
 
-          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
-        end.once
+      context "when user customized dlq_custom_codes option" do
+        let(:error_code) { 403 }
+        let(:options) { super().merge 'dlq_custom_codes' => [error_code] }
 
-        event_action_tuples = subject.map_events(events)
-        subject.send(:submit, event_action_tuples)
+        include_examples "should write event to DLQ"
       end
 
     end if LOGSTASH_VERSION > '7.0'

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -45,8 +45,8 @@ describe "whitelisting error types in expected behavior" do
     end
   end
 
-  describe "when failure logging is disabled for docuemnt exists error" do
-    let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+  describe "when failure logging is disabled for document exists error" do
+    let(:settings) { super().merge("silence_errors_in_log" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
       expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.6.0
+  version: 11.8.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-24 00:00:00.000000000 Z
+date: 2022-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement