logstash-output-elasticsearch 12.0.6-java → 12.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed4936a4ade53b1bfabb21a6a217a2587ee38235a3219be044ca59a664fd25a1
-  data.tar.gz: c06797095dd77c26bd7c70d1d851fcc50ca8e2b4db35485e9f5c018941e54cd2
+  metadata.gz: 2dc84ff1bfa23ba7aac5eea0cf36fc94fc0d3112878fda04fe2bf934fd42ccb8
+  data.tar.gz: b926a70cab6ba1d3415208512ee71fddf79ce8e46e9ab81d59354a1f79dd0cb5
 SHA512:
-  metadata.gz: 163f1e9cd1127527b6ce207f18f451c6dfd7510c6f752739908d6804c3fa70b2fced78b22668f6388d02a92d9969dd7f47f079e0180abacf8f0305d56ad14441
-  data.tar.gz: 77dc391a2d5ef77f76d07a52b6f00375eb6b9ffb9c8dbb731c849b2bffd8fb25f819c3070a729a9f1c72e7ba4ff845359ba5109aa2e5eb1242461c4b0cab5719
+  metadata.gz: a3f18999365521054239b3d8316134dfb7c33e7f486fbe8e0de2467f25c127eebb42404b8ad649bb1aa2395464bdb1324b4006104ae031a73648e724093afa46
+  data.tar.gz: ab7ee85a0bd5cbee9e18dd0cd2f26be93d1da3857d206948d5a4b392066bfcad51b0ab0ebc3eb5c6418388782855d81b7584e3ec9fb8cc6cb6c71a913657b5ce

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 12.1.0
+ - Add drop_error_types config option to not retry after certain error types [#1228](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1228)
+
+## 12.0.7
+ - Support both, encoded and non encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223)
+
 ## 12.0.6
  - Add headers reporting uncompressed size and doc count for bulk requests [#1217](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1217)
 

data/docs/index.asciidoc

@@ -303,12 +303,17 @@ single request.
 ==== DNS Caching
 
 This plugin uses the JVM to lookup DNS entries and is subject to the value of
-https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
-a global setting for the JVM.
+https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/net/doc-files/net-properties.html#address-cache-heading[Address Cache settings]
+such as `networkaddress.cache.ttl` and `networkaddress.cache.negative.ttl`, global settings for the JVM.
 
 As an example, to set your DNS TTL to 1 second you would set
 the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
 
+The default value for `networkaddress.cache.ttl` depends on the JVM implementation,
+which is 30 seconds for the JDK bundled with Logstash. 
+The `networkaddress.cache.negative.ttl` setting, that controls how long Java caches
+the result of failed DNS lookups, defaults to 10 seconds.
+
 Keep in mind that a connection with keepalive enabled will
 not reevaluate its DNS value while the keepalive is in effect.
 
@@ -368,6 +373,7 @@ Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-drop_error_types>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
@@ -639,6 +645,24 @@ If you don't set a value for this option:
 - for elasticsearch clusters 8.x: no value will be used;
 - for elasticsearch clusters 7.x: the value of '_doc' will be used.
 
+[id="plugins-{type}s-{plugin}-drop_error_types"]
+===== `drop_error_types`
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+Lists the set of error types for which individual bulk request actions will not be retried. Unless an individual - document level - action returns 409 or an error from this list, failures will be retried indefinitely.
+A warning message will be logged indicating that the action failed, unless the error type is
+listed in the <<plugins-{type}s-{plugin}-silence_errors_in_log>> config option.
+Note that the events are not added to the Dead Letter Queue (DLQ), regardless of whether it is enabled.
+
+[source,ruby]
+    output {
+      elasticsearch {
+        drop_error_types => ["index_closed_exception"]
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-ecs_compatibility"]
 ===== `ecs_compatibility`
 

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -188,25 +188,37 @@ module LogStash; module Outputs; class ElasticSearch;
     def self.setup_api_key(logger, params)
       api_key = params["api_key"]
 
-      return {} unless (api_key && api_key.value)
+      return {} unless (api_key&.value)
 
-      { "Authorization" => "ApiKey " + Base64.strict_encode64(api_key.value) }
-    end
+      value = is_base64?(api_key.value) ?  api_key.value : Base64.strict_encode64(api_key.value)
 
-    private
-    def self.dedup_slashes(url)
-      url.gsub(/\/+/, "/")
+      { "Authorization" => "ApiKey #{value}" }
     end
 
-    # Set a `filter_path` query parameter if it is not already set to be
-    # `filter_path=errors,items.*.error,items.*.status` to reduce the payload between Logstash and Elasticsearch
-    def self.resolve_filter_path(url)
-      return url if url.match?(/(?:[&|?])filter_path=/)
-      ("#{url}#{query_param_separator(url)}filter_path=errors,items.*.error,items.*.status")
-    end
+    class << self
+      private
+      def dedup_slashes(url)
+        url.gsub(/\/+/, "/")
+      end
+
+      # Set a `filter_path` query parameter if it is not already set to be
+      # `filter_path=errors,items.*.error,items.*.status` to reduce the payload between Logstash and Elasticsearch
+      def resolve_filter_path(url)
+        return url if url.match?(/(?:[&|?])filter_path=/)
+        ("#{url}#{query_param_separator(url)}filter_path=errors,items.*.error,items.*.status")
+      end
 
-    def self.query_param_separator(url)
-      url.match?(/\?[^\s#]+/) ? '&' : '?'
+      def query_param_separator(url)
+        url.match?(/\?[^\s#]+/) ? '&' : '?'
+      end
+
+      def is_base64?(string)
+        begin
+          string == Base64.strict_encode64(Base64.strict_decode64(string))
+        rescue ArgumentError
+          false
+        end
+      end
     end
   end
 end; end; end

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -204,6 +204,10 @@ module LogStash; module PluginMixins; module ElasticSearch
         # if enabled, failed index name interpolation events go into dead letter queue.
         :dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true },
 
+        # Failures on actions from a bulk request will not be retried for these error types; the events will be dropped.
+        # The events won't be added to the DLQ either.
+        :drop_error_types => { :validate => :string, :list => true, :default => [] },
+
         # Obsolete Settings
         :ssl => { :obsolete => "Set 'ssl_enabled' instead." },
         :ssl_certificate_verification => { :obsolete => "Set 'ssl_verification_mode' instead." },

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -278,16 +278,18 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         status = action_props["status"]
         error  = action_props["error"]
+        type = error["type"] if error
         action = actions[idx]
 
-        # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
+        # Retry logic: If it is success, we move on. If it is a failure, we have the following paths:
         # - For 409, we log and drop. there is nothing we can do
+        # - For any error types set in the 'drop_error_types' config, log and drop.
         # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
         # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
         if DOC_SUCCESS_CODES.include?(status)
           @document_level_metrics.increment(:successes)
           next
-        elsif DOC_CONFLICT_CODE == status
+        elsif DOC_CONFLICT_CODE == status || @drop_error_types.include?(type)
           @document_level_metrics.increment(:non_retryable_failures)
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.0.6'
+  s.version         = '12.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/metrics_spec.rb

@@ -5,7 +5,10 @@ describe "metrics", :integration => true do
     require "logstash/outputs/elasticsearch"
     settings = {
       "manage_template" => false,
-      "hosts" => "#{get_host_port()}"
+      "hosts" => "#{get_host_port()}",
+      # write data to a random non templated index to ensure the bulk partially fails
+      # don't use streams like "logs-*" as those have failure stores enabled, causing the bulk to succeed instead
+      "index" => "custom_index_#{rand(10000)}"
     }
     plugin = LogStash::Outputs::ElasticSearch.new(settings)
   end

data/spec/unit/http_client_builder_spec.rb

@@ -26,6 +26,44 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     end
   end
 
+  describe "auth setup with api-key" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+
+    context "when api-key is not encoded (id:api-key)" do
+      let(:api_key) { "id:api-key" }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+      let(:api_key_header) { klass.setup_api_key(logger, options) }
+
+      it "returns the correct encoded api-key header" do
+        expected = "ApiKey #{Base64.strict_encode64(api_key)}"
+        expect(api_key_header["Authorization"]).to eql(expected)
+      end
+    end
+
+    context "when api-key is already encoded" do
+      let(:api_key) { Base64.strict_encode64("id:api-key") }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+      let(:api_key_header) { klass.setup_api_key(logger, options) }
+
+      it "returns the api-key header as is" do
+        expected = "ApiKey #{api_key}"
+        expect(api_key_header["Authorization"]).to eql(expected)
+      end
+    end
+  end
+
   describe "customizing action paths" do
     let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
     let(:options) { {"hosts" => hosts } }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1499,6 +1499,80 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe 'drop_error_types' do
+
+    let(:error_type) { 'index_closed_exception' }
+
+    let(:options) { super().merge('drop_error_types' => [error_type]) }
+
+    let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+    let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+    let(:error_code) { 403 }
+
+    let(:event_action_tuples) { subject.map_events(events) }
+
+    let(:bulk_response) do
+      {
+        "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+        [{
+           "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status" => error_code,
+                     "error"=>{"type" => error_type, "reason" => "TEST" }
+           }
+         }]
+      }
+    end
+
+    before(:each) do
+      allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+    end
+
+    context 'DLQ is enabled' do
+
+      let(:options) { super().merge("dlq_custom_codes" => [403]) }
+
+      it 'does not write the event to the DLQ' do
+        expect(dlq_writer).not_to receive(:write)
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+
+    context 'DLQ is not enabled' do
+
+      before(:each) do
+        allow(subject).to receive(:dlq_enabled?).and_return(false)
+      end
+
+      it 'does not retry indexing the event' do
+        expect(subject).to receive(:submit).with(event_action_tuples).once.and_call_original
+        subject.send(:retrying_submit, event_action_tuples)
+      end
+    end
+
+    context 'the error type is not in `silence_errors_in_log`' do
+
+      it 'logs the error' do
+        expect(subject.logger).to receive(:warn).with(a_string_including("Failed action"), anything)
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+
+    context 'the error type is in `silence_errors_in_log`' do
+
+      let(:options) { super().merge('silence_errors_in_log' => [error_type]) }
+
+      before(:each) do
+        # ensure that neither warn nor info is called on the logger by using a test double
+        subject.instance_variable_set("@logger", double('logger'))
+      end
+
+      it 'does not log the error' do
+        subject.send(:submit, event_action_tuples)
+      end
+    end
+  end
+
   describe "custom headers" do
     let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 12.0.6
+  version: 12.1.0
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2025-07-29 00:00:00.000000000 Z
+date: 2025-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore