logstash-input-elasticsearch 4.6.1 → 4.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3fee8b4ee34b945816ffa2fe5588fcc6107b5c995aadb311a903863898fdfe2
-  data.tar.gz: '09644097ce9d8707ca6c629f72a439692b29bf23a0d536135d5fb8c4c9ee057e'
+  metadata.gz: f2739ba5eed141d75ad674402bb35485e0be50c6a7593556dbcda575120bbb54
+  data.tar.gz: 1affc8bfa4aad83cda6c59f0a6b3dcf7ca24a5d7e0e53cacc66f45b88d353ce1
 SHA512:
-  metadata.gz: c9dae7fc0d75b73a9624be4bd356cbd7019858e32f67542ec61d6114792756655f39acc49b57054a593d0cab32f28a2ed42dbe94856ca1d31d6413e91f68bf2c
-  data.tar.gz: e25e5fcaf0a3d8871bb3ddea91e1137ec88c380cd790b0e09912be2423f416a89650a23cf5921e816cb92cbfd22a8d57c7476f5f5f800aff420ef745dd55a161
+  metadata.gz: d4d2eb3be415ddf52ed7b17d06999fe93022004fcf218d11919c750e3b98f8a1ddc943d7d0911232a43fcc8f237a08ffcbd63ba5ec1dd59de9498061f4b4bf25
+  data.tar.gz: 74524dcd7cacd49a324049f80b5cabf5bd2a96c8c634ed15877139371cc8cb89bb38c074a576a0214a3785e218ee5dc0f0966b59de75394e679d8eef34ad14ed

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 4.8.1
+  - Fixed connection error when using multiple `slices`. [#133](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/133)
+
+## 4.8.0
+  - Added the ability to configure connection-, request-, and socket-timeouts with `connect_timeout_seconds`, `request_timeout_seconds`, and `socket_timeout_seconds` [#121](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/121)
+
+## 4.7.1
+  - [DOC] Updated sliced scroll link to resolve to correct location after doc structure change [#135](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/135)
+  - [DOC] Added usage example of docinfo metadata [#98](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/98)
+
+## 4.7.0
+  - Added api_key support [#131](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/131)
+
+## 4.6.2
+  - Added scroll clearing and better handling of scroll expiration [#128](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/128)
+
 ## 4.6.1
   - [DOC] Removed outdated compatibility notice [#124](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/124)
 

data/docs/index.asciidoc

@@ -68,6 +68,16 @@ Further documentation describing this syntax can be found
 https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
 
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
+
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
 
@@ -76,9 +86,11 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -87,11 +99,13 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -100,6 +114,16 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -128,6 +152,15 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
 For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
 
+[id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
+===== `connect_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
+Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
 
@@ -163,6 +196,19 @@ Example
       }
     }
 
+If set, you can use metadata information in the <<plugins-{type}s-{plugin}-add_field>> common option.
+
+Example
+[source, ruby]
+    input {
+      elasticsearch {
+        docinfo => true
+        add_field => {
+          identifier => %{[@metadata][_index]}:%{[@metadata][_type]}:%{[@metadata][_id]}"
+        }
+      }
+    }
+
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
@@ -237,6 +283,16 @@ The query to be executed. Read the
 https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
 for more information.
 
+[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
+===== `request_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, for a single request to Elasticsearch.
+Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
+documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -275,8 +331,8 @@ This allows you to set the maximum number of hits returned per scroll.
   * Sensible values range from 2 to about 8.
 
 In some cases, it is possible to improve overall throughput by consuming multiple
-distinct slices of a query simultaneously using the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
+distinct slices of a query simultaneously using
+https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#slice-scroll[sliced scrolls],
 especially if the pipeline is spending significant time waiting on Elasticsearch
 to provide results.
 
@@ -300,6 +356,15 @@ instructions into the query.
 If enabled, SSL will be used when communicating with the Elasticsearch
 server (i.e. HTTPS will be used instead of plain HTTP).
 
+[id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
+===== `socket_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended.
+Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query.
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 
@@ -315,4 +380,4 @@ empty string authentication will be disabled.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/elasticsearch.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
 require "base64"
+require_relative "patch"
 
 
 # .Compatibility Note
@@ -70,11 +71,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Port defaults to 9200
   config :hosts, :validate => :array
 
-  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-  #
-  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-  config :cloud_id, :validate => :string
-
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -140,11 +136,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Connection Timeout, in Seconds
+  config :connect_timeout_seconds, :validate => :positive_whole_number, :default => 10
+
+  # Request Timeout, in Seconds
+  config :request_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Socket Timeout, in Seconds
+  config :socket_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
   config :cloud_auth, :validate => :password
 
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
@@ -177,54 +191,34 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
-    transport_options = {}
-
+    validate_authentication
     fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
 
-    if @user && @password
-      token = Base64.strict_encode64("#{@user}:#{@password.value}")
-      transport_options[:headers] = { :Authorization => "Basic #{token}" }
-    end
 
-    fill_hosts_from_cloud_id
-    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    transport_options = {:headers => {}}
+    transport_options[:headers].merge!(setup_basic_auth(user, password))
+    transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
-    hosts = if @ssl
-      @hosts.map do |h|
-        host, port = h.split(":")
-        { :host => host, :scheme => 'https', :port => port }
-      end
-    else
-      @hosts
-    end
-    ssl_options = { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
-    ssl_options ||= {}
+    hosts = setup_hosts
+    ssl_options = setup_ssl
 
     @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
     transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
 
-    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options,
-                                        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
-                                        :ssl => ssl_options)
+    @client = Elasticsearch::Client.new(
+      :hosts => hosts,
+      :transport_options => transport_options,
+      :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+      :ssl => ssl_options
+    )
   end
 
-  ##
-  # @override to handle proxy => '' as if none was set
-  # @param value [Array<Object>]
-  # @param validator [nil,Array,Symbol]
-  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
-  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
-  def self.validate_value(value, validator)
-    return super unless validator == :uri_or_empty
-
-    value = deep_replace(value)
-    value = hash_or_array(value)
 
-    return true, value.first if value.size == 1 && value.first.empty?
-
-    return super(value, :uri)
-  end
 
   def run(output_queue)
     if @schedule
@@ -266,25 +260,41 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
 
     logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-    r = search_request(slice_options)
 
-    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
-    logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-
-    has_hits = r['hits']['hits'].any?
+    scroll_id = nil
+    begin
+      r = search_request(slice_options)
 
-    while has_hits && r['_scroll_id'] && !stop?
-      r = process_next_scroll(output_queue, r['_scroll_id'])
+      r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
       logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-      has_hits = r['has_hits']
+
+      has_hits = r['hits']['hits'].any?
+      scroll_id = r['_scroll_id']
+
+      while has_hits && scroll_id && !stop?
+        has_hits, scroll_id = process_next_scroll(output_queue, scroll_id)
+        logger.debug("Slice progress", slice_id: slice_id, slices: @slices) if logger.debug? && slice_id
+      end
+      logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    ensure
+      clear_scroll(scroll_id)
     end
-    logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
   end
 
+  ##
+  # @param output_queue [#<<]
+  # @param scroll_id [String]: a scroll id to resume
+  # @return [Array(Boolean,String)]: a tuple representing whether the response
+  #
   def process_next_scroll(output_queue, scroll_id)
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
-    {'has_hits' => r['hits']['hits'].any?, '_scroll_id' => r['_scroll_id']}
+    [r['hits']['hits'].any?, r['_scroll_id']]
+  rescue => e
+    # this will typically be triggered by a scroll timeout
+    logger.error("Scroll request error, aborting scroll", error: e.inspect)
+    # return no hits and original scroll_id so we can try to clear it
+    [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
@@ -313,39 +323,86 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     output_queue << event
   end
 
+  def clear_scroll(scroll_id)
+    @client.clear_scroll(scroll_id: scroll_id) if scroll_id
+  rescue => e
+    # ignore & log any clear_scroll errors
+    logger.warn("Ignoring clear_scroll exception", message: e.message)
+  end
+
   def scroll_request scroll_id
-    client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
+    @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
 
   def search_request(options)
-    client.search(options)
+    @client.search(options)
   end
 
-  attr_reader :client
-
   def hosts_default?(hosts)
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
 
-  def fill_hosts_from_cloud_id
-    return unless @cloud_id
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
 
-    if @hosts && !hosts_default?(@hosts)
-      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
-    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    end
+  end
+
+  def setup_ssl
+    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+  end
+
+  def setup_hosts
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    if @ssl
+      @hosts.map do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    else
+      @hosts
+    end
+  end
+
+  def setup_basic_auth(user, password)
+    return {} unless user && password && password.value
+
+    token = ::Base64.strict_encode64("#{user}:#{password.value}")
+    { Authorization: "Basic #{token}" }
+  end
+
+  def setup_api_key(api_key)
+    return {} unless (api_key && api_key.value)
+
+    token = ::Base64.strict_encode64(api_key.value)
+    { Authorization: "ApiKey #{token}" }
   end
 
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
-    if @user || @password
-      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
-    end
     @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
     params['user'], params['password'] = @user, @password
   end
 
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
   def parse_host_uri_from_cloud_id(cloud_id)
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'
@@ -380,4 +437,42 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  module URIOrEmptyValidator
+    ##
+    # @override to provide :uri_or_empty validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :uri_or_empty
+
+      value = deep_replace(value)
+      value = hash_or_array(value)
+
+      return true, value.first if value.size == 1 && value.first.empty?
+
+      return super(value, :uri)
+    end
+  end
+  extend(URIOrEmptyValidator)
+
+  module PositiveWholeNumberValidator
+    ##
+    # @override to provide :positive_whole_number validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :positive_whole_number
+
+      is_number, coerced_number = super(value, :number)
+
+      return [true, coerced_number.to_i] if is_number && coerced_number.denominator == 1 && coerced_number > 0
+
+      return [false, "Expected positive whole number, got `#{value.inspect}`"]
+    end
+  end
+  extend(PositiveWholeNumberValidator)
 end

data/lib/logstash/inputs/patch.rb

@@ -0,0 +1,48 @@
+if Gem.loaded_specs['elasticsearch-transport'].version >= Gem::Version.new("7.2.0")
+  # elasticsearch-transport versions prior to 7.2.0 suffered of a race condition on accessing
+  # the connection pool. This issue was fixed with https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
+   # This plugin, at the moment, is forced to use v5.x so we have to monkey patch the gem. When this requirement
+   # ceases, this patch could be removed.
+  puts "WARN remove the patch code into logstash-input-elasticsearch plugin"
+else
+  module Elasticsearch
+    module Transport
+      module Transport
+        module Connections
+          module Selector
+
+            # "Round-robin" selector strategy (default).
+            #
+            class RoundRobin
+              include Base
+
+              # @option arguments [Connections::Collection] :connections Collection with connections.
+              #
+              def initialize(arguments = {})
+                super
+                @mutex = Mutex.new
+                @current = nil
+              end
+
+              # Returns the next connection from the collection, rotating them in round-robin fashion.
+              #
+              # @return [Connections::Connection]
+              #
+              def select(options={})
+                @mutex.synchronize do
+                  conns = connections
+                  if @current && (@current < conns.size-1)
+                    @current += 1
+                  else
+                    @current = 0
+                  end
+                  conns[@current]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.6.1'
+  s.version         = '4.8.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -8,9 +8,13 @@ require "stud/temporary"
 require "time"
 require "date"
 
-describe LogStash::Inputs::Elasticsearch do
+class LogStash::Inputs::TestableElasticsearch < LogStash::Inputs::Elasticsearch
+  attr_reader :client
+end
+
+describe LogStash::Inputs::TestableElasticsearch do
 
-  let(:plugin) { LogStash::Inputs::Elasticsearch.new(config) }
+  let(:plugin) { LogStash::Inputs::TestableElasticsearch.new(config) }
   let(:queue) { Queue.new }
 
   it_behaves_like "an interruptible input plugin" do
@@ -32,6 +36,7 @@ describe LogStash::Inputs::Elasticsearch do
       }
       allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
       allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(esclient).to receive(:clear_scroll).and_return(nil)
     end
   end
 
@@ -76,6 +81,7 @@ describe LogStash::Inputs::Elasticsearch do
     expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
     expect(client).to receive(:search).with(any_args).and_return(response)
     expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(scroll_reponse)
+    expect(client).to receive(:clear_scroll).and_return(nil)
 
     event = input(config) do |pipeline, queue|
       queue.pop
@@ -257,6 +263,8 @@ describe LogStash::Inputs::Elasticsearch do
         expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
         plugin.register
 
+        expect(client).to receive(:clear_scroll).and_return(nil)
+
         # SLICE0 is a three-page scroll in which the last page is empty
         slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
         expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
@@ -360,6 +368,7 @@ describe LogStash::Inputs::Elasticsearch do
       expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
       expect(client).to receive(:search).with(any_args).and_return(response)
       allow(client).to receive(:scroll).with({ :body => {:scroll_id => "cXVlcnlUaGVuRmV0Y2g"}, :scroll => "1m" }).and_return(scroll_reponse)
+      allow(client).to receive(:clear_scroll).and_return(nil)
     end
 
     context 'when defining docinfo' do
@@ -405,6 +414,7 @@ describe LogStash::Inputs::Elasticsearch do
             "docinfo_target" => 'metadata_with_string'
         } }
         it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          expect(client).not_to receive(:clear_scroll)
           plugin.register
           expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
         end
@@ -573,7 +583,37 @@ describe LogStash::Inputs::Elasticsearch do
         let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
-          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:client)
+          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
         end
       end
     end if LOGSTASH_VERSION > '6.0'
@@ -600,6 +640,54 @@ describe LogStash::Inputs::Elasticsearch do
         end
       end
     end
+
+    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+      let(:config_value) { fail NotImplementedError }
+      let(:config) { super().merge(config_name => config_value) }
+      {
+          :string   => 'banana',
+          :negative => -123,
+          :zero     => 0,
+      }.each do |value_desc, value|
+        let(:config_value) { value }
+        context "with an invalid #{value_desc} value" do
+          it 'prevents instantiation with a helpful message' do
+            expect(described_class.logger).to receive(:error).with(/Expected positive whole number/)
+            expect { described_class.new(config) }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+      end
+
+      context 'with a valid value' do
+        let(:config_value) { 17 }
+
+        it "instantiates the elasticsearch client with the timeout value set via #{manticore_transport_option} in the transport options" do
+          expect(Elasticsearch::Client).to receive(:new) do |new_elasticsearch_client_params|
+            # We rely on Manticore-specific transport options, fail early if we are using a different
+            # transport or are allowing the client to determine its own transport class.
+            expect(new_elasticsearch_client_params).to include(:transport_class)
+            expect(new_elasticsearch_client_params[:transport_class].name).to match(/\bManticore\b/)
+
+            expect(new_elasticsearch_client_params).to include(:transport_options)
+            transport_options = new_elasticsearch_client_params[:transport_options]
+            expect(transport_options).to include(manticore_transport_option)
+            expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+          end
+
+          plugin.register
+        end
+      end
+    end
+
+    context 'connect_timeout_seconds' do
+      include_examples('configurable timeout', 'connect_timeout_seconds', :connect_timeout)
+    end
+    context 'request_timeout_seconds' do
+      include_examples('configurable timeout', 'request_timeout_seconds', :request_timeout)
+    end
+    context 'socket_timeout_seconds' do
+      include_examples('configurable timeout', 'socket_timeout_seconds', :socket_timeout)
+    end
   end
 
   context "when scheduling" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.6.1
+  version: 4.8.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -200,6 +200,7 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
+- lib/logstash/inputs/patch.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca/ca.crt