logstash-input-elasticsearch 4.7.1 → 4.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba97fe448682c96253a9809b3c8010e358d0044a0bb6888498653f05048369d5
-  data.tar.gz: f49d1231a4e82657a2128e27f522d8e93cdf48b955d57e5f8ed788d0e58babe7
+  metadata.gz: 74690fbfa461c4c9087181c3f95e4feb33f68e6398cc357a1c8d12d6bca8d837
+  data.tar.gz: 67425c9a092529e779e0c7693cc8afa9eaed0cfb188509026881d1972f970f3c
 SHA512:
-  metadata.gz: 24ec0b4f107494c0be8264769ae6ff5143e30e02c680f7158bc4a04d430b370a67963c82f3148fbf064e70e6f439e182fdbf9ec8e2cfe180fcdf0697d884f260
-  data.tar.gz: d0a0841f6b014fe6f0799c423095db4fc112469893b5ca628717db42edaeb271457d92fc79cd5e1c63399f06293bf9823e0911ed7cfa21c4b0c71f88279a09ec
+  metadata.gz: 63482d80b537ba41ca5d3ed34a1e35dc6550887747b7d50a7320564dbd5fcac3336519cae21c21363322616f6a1c66f803743e2246e2d2dc8adf2b03b0a23231
+  data.tar.gz: 98547555319b8d856ea58ed33b8236fcf15c2483594ca45c9c2e09791b4a20f6250ae7019e4e1be341f2000816a90e522b8c6fb21bec3b1aebc6b511cce2e09a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.8.0
+  - Added the ability to configure connection-, request-, and socket-timeouts with `connect_timeout_seconds`, `request_timeout_seconds`, and `socket_timeout_seconds` [#121](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/121)
+
 ## 4.7.1
   - [DOC] Updated sliced scroll link to resolve to correct location after doc structure change [#135](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/135)
   - [DOC] Added usage example of docinfo metadata [#98](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/98)

data/docs/index.asciidoc

@@ -90,6 +90,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -98,11 +99,13 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -149,6 +152,15 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
 For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
 
+[id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
+===== `connect_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
+Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
 
@@ -271,6 +283,16 @@ The query to be executed. Read the
 https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
 for more information.
 
+[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
+===== `request_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, for a single request to Elasticsearch.
+Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
+documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -334,6 +356,15 @@ instructions into the query.
 If enabled, SSL will be used when communicating with the Elasticsearch
 server (i.e. HTTPS will be used instead of plain HTTP).
 
+[id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
+===== `socket_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended.
+Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query.
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -135,6 +135,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Connection Timeout, in Seconds
+  config :connect_timeout_seconds, :validate => :positive_whole_number, :default => 10
+
+  # Request Timeout, in Seconds
+  config :request_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Socket Timeout, in Seconds
+  config :socket_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
   # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
@@ -189,6 +198,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
     hosts = setup_hosts
     ssl_options = setup_ssl
@@ -205,22 +217,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     )
   end
 
-  ##
-  # @override to handle proxy => '' as if none was set
-  # @param value [Array<Object>]
-  # @param validator [nil,Array,Symbol]
-  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
-  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
-  def self.validate_value(value, validator)
-    return super unless validator == :uri_or_empty
-
-    value = deep_replace(value)
-    value = hash_or_array(value)
 
-    return true, value.first if value.size == 1 && value.first.empty?
-
-    return super(value, :uri)
-  end
 
   def run(output_queue)
     if @schedule
@@ -439,4 +436,42 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  module URIOrEmptyValidator
+    ##
+    # @override to provide :uri_or_empty validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :uri_or_empty
+
+      value = deep_replace(value)
+      value = hash_or_array(value)
+
+      return true, value.first if value.size == 1 && value.first.empty?
+
+      return super(value, :uri)
+    end
+  end
+  extend(URIOrEmptyValidator)
+
+  module PositiveWholeNumberValidator
+    ##
+    # @override to provide :positive_whole_number validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :positive_whole_number
+
+      is_number, coerced_number = super(value, :number)
+
+      return [true, coerced_number.to_i] if is_number && coerced_number.denominator == 1 && coerced_number > 0
+
+      return [false, "Expected positive whole number, got `#{value.inspect}`"]
+    end
+  end
+  extend(PositiveWholeNumberValidator)
 end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.7.1'
+  s.version         = '4.8.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -640,6 +640,54 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
       end
     end
+
+    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+      let(:config_value) { fail NotImplementedError }
+      let(:config) { super().merge(config_name => config_value) }
+      {
+          :string   => 'banana',
+          :negative => -123,
+          :zero     => 0,
+      }.each do |value_desc, value|
+        let(:config_value) { value }
+        context "with an invalid #{value_desc} value" do
+          it 'prevents instantiation with a helpful message' do
+            expect(described_class.logger).to receive(:error).with(/Expected positive whole number/)
+            expect { described_class.new(config) }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+      end
+
+      context 'with a valid value' do
+        let(:config_value) { 17 }
+
+        it "instantiates the elasticsearch client with the timeout value set via #{manticore_transport_option} in the transport options" do
+          expect(Elasticsearch::Client).to receive(:new) do |new_elasticsearch_client_params|
+            # We rely on Manticore-specific transport options, fail early if we are using a different
+            # transport or are allowing the client to determine its own transport class.
+            expect(new_elasticsearch_client_params).to include(:transport_class)
+            expect(new_elasticsearch_client_params[:transport_class].name).to match(/\bManticore\b/)
+
+            expect(new_elasticsearch_client_params).to include(:transport_options)
+            transport_options = new_elasticsearch_client_params[:transport_options]
+            expect(transport_options).to include(manticore_transport_option)
+            expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+          end
+
+          plugin.register
+        end
+      end
+    end
+
+    context 'connect_timeout_seconds' do
+      include_examples('configurable timeout', 'connect_timeout_seconds', :connect_timeout)
+    end
+    context 'request_timeout_seconds' do
+      include_examples('configurable timeout', 'request_timeout_seconds', :request_timeout)
+    end
+    context 'socket_timeout_seconds' do
+      include_examples('configurable timeout', 'socket_timeout_seconds', :socket_timeout)
+    end
   end
 
   context "when scheduling" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.7.1
+  version: 4.8.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement