logstash-filter-elasticsearch 3.13.0 → 3.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d55417f0b601e60aa33702b94eec237fc3ed47dc37e06bfa83b1f8b3f2c0884
-  data.tar.gz: 335f6c76502901f6b0021d8fd89be14b716c2c0f493e22c4d59bd865bcce91db
+  metadata.gz: aa249a04a85198ffab82e97a53f15c8c1ce3792b0d7f737962979aaccf0b8a78
+  data.tar.gz: 1501f7b558ead54e7701b0048b5f1f9847d250e96ebdf4cd2875000f733ad8c0
 SHA512:
-  metadata.gz: cd657fd129d3972aff051cf111e15f18b40861631e3d47249887023e3779d6c9cbd09b515dcb7beb5d6783b73f5fef69d3e57425115f100b6cf7576984da832b
-  data.tar.gz: 410cdfcd7857b6522f91549b8e57244624385f1dde1ddc393bbadb4a8c7468ea5e4df91697b9b5cb1221a1c99a0c4ea054a6a21a0babb52617bb0b54b71bd4bb
+  metadata.gz: c31a11c7588de85b8bbdb922c8938946a651230080202b2442dcea5fc3ef686e499aa74b3f10df9df172713b4b007e0e3ba539b7dff838fccb33da116c3aacbf
+  data.tar.gz: 3bcd227e3ba31712287bb27367959b8a5e032a7e9fa5a907f1ae7018e4422db6b3275bbcb3f88d83d6d623d5f015c2ae80a66a148e2bcb3a84181a801e486640

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.14.0
+  - Added support for configurable retries with new `retry_on_failure` and `retry_on_status` options [#160](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/160)
+
 ## 3.13.0
   - Added support for this plugin identifying itself to Elasticsearch with an SSL/TLS client certificate using a new `keystore` option [#162](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/162)
 

data/docs/index.asciidoc

@@ -142,6 +142,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-query_template>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-result_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_on_failure>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-result_on_status_>> |<<number,number list>>|No
 | <<plugins-{type}s-{plugin}-sort>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
@@ -330,11 +332,30 @@ the {ref}/query-dsl.html[Elasticsearch query documentation].
 [id="plugins-{type}s-{plugin}-result_size"]
 ===== `result_size` 
 
-  * Value type is <<number,number>>
-  * Default value is `1`
+* Value type is <<number,number>>
+* Default value is `1`
 
 How many results to return
 
+[id="plugins-{type}s-{plugin}-retry_on_failure"]
+===== `retry_on_failure`
+
+* Value type is <<number,number>>
+* Default value is `0` (retries disabled)
+
+How many times to retry an individual failed request.
+
+When enabled, retry requests that result in connection errors or an HTTP status code included in <<plugins-{type}s-{plugin}-retry_on_status>>
+
+[id="plugins-{type}s-{plugin}-retry_on_status"]
+===== `retry_on_status`
+
+* Value type is <<number,number list>>
+* Default value is an empty list `[]`
+
+Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<plugins-{type}s-{plugin}-retry_on_failure>>,
+
+
 [id="plugins-{type}s-{plugin}-sort"]
 ===== `sort` 
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -34,15 +34,23 @@ module LogStash
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options.update(ssl: true, ca_file: options[:ca_file]) if options[:ca_file]
         ssl_options.update(ssl: true, trust_strategy: options[:ssl_trust_strategy]) if options[:ssl_trust_strategy]
-
         if keystore
           ssl_options[:keystore] = keystore
           logger.debug("Keystore for client certificate", :keystore => keystore)
           ssl_options[:keystore_password] = keystore_password.value if keystore_password
         end
 
+        client_options = {
+                       hosts: hosts,
+             transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+           transport_options: transport_options,
+                         ssl: ssl_options,
+            retry_on_failure: options[:retry_on_failure],
+             retry_on_status: options[:retry_on_status]
+        }
+
         logger.info("New ElasticSearch filter client", :hosts => hosts)
-        @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
+        @client = ::Elasticsearch::Client.new(client_options)
       end
 
       def search(params)

data/lib/logstash/filters/elasticsearch.rb

@@ -82,6 +82,12 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
+  # How many times to retry on failure?
+  config :retry_on_failure, :validate => :number, :default => 0
+
+  # What status codes to retry on?
+  config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
+
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
@@ -215,6 +221,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :proxy => @proxy,
       :ssl => @ssl,
       :ca_file => @ca_file,
+      :retry_on_failure => @retry_on_failure,
+      :retry_on_status => @retry_on_status,
       :keystore => @keystore,
       :keystore_password => @keystore_password,
       :ssl_trust_strategy => trust_strategy_for_ca_trusted_fingerprint

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.13.0'
+  s.version         = '3.14.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -29,4 +29,3 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/filters/elasticsearch_spec.rb

@@ -301,6 +301,15 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
+    context 'with client-level retries' do
+      let(:config) do
+        super().merge(
+          "retry_on_failure" => 3,
+          "retry_on_status" => [500]
+        )
+      end
+    end
+
     context "if query is on nested field" do
       let(:config) do
         {
@@ -559,6 +568,28 @@ describe LogStash::Filters::Elasticsearch do
         end
       end
     end
+
+    describe "retry_on_failure" do
+      let(:config) { super().merge("retry_on_failure" => 3) }
+
+      it 'propagates to the client' do
+        plugin.register
+
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:retry_on_failure] ).to eq(3)
+      end
+    end
+
+    describe "retry_on_status" do
+      let(:config) { super().merge("retry_on_status" => [500, 502, 503, 504]) }
+
+      it 'propagates to the client' do
+        plugin.register
+
+        client = plugin.send(:get_client).client
+        expect( extract_transport(client).options[:retry_on_status] ).to eq([500, 502, 503, 504])
+      end
+    end
   end
 
   describe "ca_trusted_fingerprint" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.13.0
+  version: 3.14.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-26 00:00:00.000000000 Z
+date: 2023-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement