logstash-input-elasticsearch 4.6.2 → 4.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92c58eb709a67922d73d44a13cca7fc933968747feebb0c7afa81c04ea3543d2
-  data.tar.gz: 4354af2982b8af23492a8c1f4c063c21ce6e2d1bfc1e082a9de243ab1c575248
+  metadata.gz: 4091feeb0b3bf292cfb9afcde7496a72f95168eb570b21d29064ade174bb1352
+  data.tar.gz: cfd02af050bb495dceea16b4ffe5daa6828088d2bd7994639ee08374f87da69d
 SHA512:
-  metadata.gz: ba09a7c97593fb2b32cbd4c1f5c4448af2d6ed10c32908ea98b5d4bd54ec0bf11321ae30c48afa745be27bc6280c8fbac6c3f7e62dfa8d5887142dce04e3d0b3
-  data.tar.gz: 86e4b75f7937c4e66a23b5724902d35db23303d297d7d46cfb55092efc14d0141e708b73b2a72ee1f459f27bc4a9ee0f46d337de2b31b9f38675e25286b58778
+  metadata.gz: 873680bea22204e65d519d310f3952f26fd672ce336504e45e99b13988e2da2794b7c05f83b3e1b7a87f317eb51c079759f9df515dc99236577417ecfd379aa1
+  data.tar.gz: 85c66a665eb7a3b503ab7cec64ea2f24b0e16829c4b78a75560d69c6272b1861583dad41e04ea1157ce58435714a86eaa6f4d71b5a4a724edfb9604f77150b99

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 4.9.0
+  - Added `target` option, allowing the hit's source to target a specific field instead of being expanded at the root of the event. This allows the input to play nicer with the Elastic Common Schema when the input does not follow the schema. [#117](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/117)
+
+## 4.8.3
+  - [DOC] Fixed links to restructured Logstash-to-cloud docs [#139](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/139)
+
+## 4.8.2
+  - [DOC] Document the permissions required in secured clusters [#137](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/137)
+
+## 4.8.1
+  - Fixed connection error when using multiple `slices`. [#133](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/133)
+
+## 4.8.0
+  - Added the ability to configure connection-, request-, and socket-timeouts with `connect_timeout_seconds`, `request_timeout_seconds`, and `socket_timeout_seconds` [#121](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/121)
+
+## 4.7.1
+  - [DOC] Updated sliced scroll link to resolve to correct location after doc structure change [#135](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/135)
+  - [DOC] Added usage example of docinfo metadata [#98](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/98)
+
+## 4.7.0
+  - Added api_key support [#131](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/131)
+
 ## 4.6.2
   - Added scroll clearing and better handling of scroll expiration [#128](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/128)
 

data/CONTRIBUTORS

@@ -7,6 +7,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * João Duarte (jsvd)
 * Kurt Hurtado (kurtado)
+* Luca Belluccini (lucabelluccini)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)

data/docs/index.asciidoc

@@ -68,6 +68,21 @@ Further documentation describing this syntax can be found
 https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
 
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
+
+[id="plugins-{type}s-{plugin}-autz"]
+==== Authorization
+
+Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
+The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
 
@@ -76,9 +91,11 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -87,11 +104,14 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-target>> | https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -100,6 +120,19 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the
+Elasticsearch
+https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create
+API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -116,7 +149,9 @@ SSL Certificate Authority file in PEM encoded format, must also include any chai
 
 Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+For more info, check out the
+https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
+documentation]
 
 [id="plugins-{type}s-{plugin}-cloud_id"]
 ===== `cloud_id`
@@ -126,7 +161,18 @@ For more info, check out the https://www.elastic.co/guide/en/logstash/current/co
 
 Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+For more info, check out the
+https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
+documentation]
+
+[id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
+===== `connect_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
+Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
@@ -163,6 +209,19 @@ Example
       }
     }
 
+If set, you can use metadata information in the <<plugins-{type}s-{plugin}-add_field>> common option.
+
+Example
+[source, ruby]
+    input {
+      elasticsearch {
+        docinfo => true
+        add_field => {
+          identifier => %{[@metadata][_index]}:%{[@metadata][_type]}:%{[@metadata][_id]}"
+        }
+      }
+    }
+
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
@@ -237,6 +296,16 @@ The query to be executed. Read the
 https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
 for more information.
 
+[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
+===== `request_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, for a single request to Elasticsearch.
+Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
+documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -275,8 +344,8 @@ This allows you to set the maximum number of hits returned per scroll.
   * Sensible values range from 2 to about 8.
 
 In some cases, it is possible to improve overall throughput by consuming multiple
-distinct slices of a query simultaneously using the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
+distinct slices of a query simultaneously using
+https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#slice-scroll[sliced scrolls],
 especially if the pipeline is spending significant time waiting on Elasticsearch
 to provide results.
 
@@ -300,6 +369,28 @@ instructions into the query.
 If enabled, SSL will be used when communicating with the Elasticsearch
 server (i.e. HTTPS will be used instead of plain HTTP).
 
+[id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
+===== `socket_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended.
+Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query.
+
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+* Value type is https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference]
+* There is no default value for this setting.
+
+Without a `target`, events are created from each hit's `_source` at the root level.
+When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead.
+
+This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
+It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 
@@ -315,4 +406,4 @@ empty string authentication will be disabled.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/elasticsearch.rb

@@ -3,7 +3,9 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "base64"
+require_relative "patch"
 
 
 # .Compatibility Note
@@ -61,6 +63,8 @@ require "base64"
 #
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "elasticsearch"
 
   default :codec, "json"
@@ -70,11 +74,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Port defaults to 9200
   config :hosts, :validate => :array
 
-  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-  #
-  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-  config :cloud_id, :validate => :string
-
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -140,11 +139,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Connection Timeout, in Seconds
+  config :connect_timeout_seconds, :validate => :positive_whole_number, :default => 10
+
+  # Request Timeout, in Seconds
+  config :request_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Socket Timeout, in Seconds
+  config :socket_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
   config :cloud_auth, :validate => :password
 
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
@@ -161,6 +178,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
+  # If set, the _source of each hit will be added nested under the target instead of at the top-level
+  config :target, :validate => :field_reference
+
   def register
     require "elasticsearch"
     require "rufus/scheduler"
@@ -177,28 +197,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
-    transport_options = {}
-
+    validate_authentication
     fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
 
-    if @user && @password
-      token = Base64.strict_encode64("#{@user}:#{@password.value}")
-      transport_options[:headers] = { :Authorization => "Basic #{token}" }
-    end
 
-    fill_hosts_from_cloud_id
-    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    transport_options = {:headers => {}}
+    transport_options[:headers].merge!(setup_basic_auth(user, password))
+    transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
-    hosts = if @ssl
-      @hosts.map do |h|
-        host, port = h.split(":")
-        { :host => host, :scheme => 'https', :port => port }
-      end
-    else
-      @hosts
-    end
-    ssl_options = { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
-    ssl_options ||= {}
+    hosts = setup_hosts
+    ssl_options = setup_ssl
 
     @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
@@ -212,22 +224,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     )
   end
 
-  ##
-  # @override to handle proxy => '' as if none was set
-  # @param value [Array<Object>]
-  # @param validator [nil,Array,Symbol]
-  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
-  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
-  def self.validate_value(value, validator)
-    return super unless validator == :uri_or_empty
-
-    value = deep_replace(value)
-    value = hash_or_array(value)
 
-    return true, value.first if value.size == 1 && value.first.empty?
-
-    return super(value, :uri)
-  end
 
   def run(output_queue)
     if @schedule
@@ -307,7 +304,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def push_hit(hit, output_queue)
-    event = LogStash::Event.new(hit['_source'])
+    if @target.nil?
+      event = LogStash::Event.new(hit['_source'])
+    else
+      event = LogStash::Event.new
+      event.set(@target, hit['_source'])
+    end
 
     if @docinfo
       # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
@@ -351,25 +353,67 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
 
-  def fill_hosts_from_cloud_id
-    return unless @cloud_id
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
 
-    if @hosts && !hosts_default?(@hosts)
-      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
-    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    end
+  end
+
+  def setup_ssl
+    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+  end
+
+  def setup_hosts
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    if @ssl
+      @hosts.map do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    else
+      @hosts
+    end
+  end
+
+  def setup_basic_auth(user, password)
+    return {} unless user && password && password.value
+
+    token = ::Base64.strict_encode64("#{user}:#{password.value}")
+    { Authorization: "Basic #{token}" }
+  end
+
+  def setup_api_key(api_key)
+    return {} unless (api_key && api_key.value)
+
+    token = ::Base64.strict_encode64(api_key.value)
+    { Authorization: "ApiKey #{token}" }
   end
 
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
-    if @user || @password
-      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
-    end
     @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
     params['user'], params['password'] = @user, @password
   end
 
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
   def parse_host_uri_from_cloud_id(cloud_id)
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'
@@ -404,4 +448,42 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  module URIOrEmptyValidator
+    ##
+    # @override to provide :uri_or_empty validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :uri_or_empty
+
+      value = deep_replace(value)
+      value = hash_or_array(value)
+
+      return true, value.first if value.size == 1 && value.first.empty?
+
+      return super(value, :uri)
+    end
+  end
+  extend(URIOrEmptyValidator)
+
+  module PositiveWholeNumberValidator
+    ##
+    # @override to provide :positive_whole_number validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :positive_whole_number
+
+      is_number, coerced_number = super(value, :number)
+
+      return [true, coerced_number.to_i] if is_number && coerced_number.denominator == 1 && coerced_number > 0
+
+      return [false, "Expected positive whole number, got `#{value.inspect}`"]
+    end
+  end
+  extend(PositiveWholeNumberValidator)
 end

data/lib/logstash/inputs/patch.rb

@@ -0,0 +1,48 @@
+if Gem.loaded_specs['elasticsearch-transport'].version >= Gem::Version.new("7.2.0")
+  # elasticsearch-transport versions prior to 7.2.0 suffered of a race condition on accessing
+  # the connection pool. This issue was fixed with https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
+   # This plugin, at the moment, is forced to use v5.x so we have to monkey patch the gem. When this requirement
+   # ceases, this patch could be removed.
+  puts "WARN remove the patch code into logstash-input-elasticsearch plugin"
+else
+  module Elasticsearch
+    module Transport
+      module Transport
+        module Connections
+          module Selector
+
+            # "Round-robin" selector strategy (default).
+            #
+            class RoundRobin
+              include Base
+
+              # @option arguments [Connections::Collection] :connections Collection with connections.
+              #
+              def initialize(arguments = {})
+                super
+                @mutex = Mutex.new
+                @current = nil
+              end
+
+              # Returns the next connection from the collection, rotating them in round-robin fashion.
+              #
+              # @return [Connections::Connection]
+              #
+              def select(options={})
+                @mutex.synchronize do
+                  conns = connections
+                  if @current && (@current < conns.size-1)
+                    @current += 1
+                  else
+                    @current = 0
+                  end
+                  conns[@current]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.6.2'
+  s.version         = '4.9.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'

data/spec/inputs/elasticsearch_spec.rb

@@ -40,57 +40,91 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
   end
 
-  it "should retrieve json event from elasticseach" do
-    config = %q[
-      input {
-        elasticsearch {
-          hosts => ["localhost"]
-          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+  context 'creating events from Elasticsearch' do
+    let(:config) do
+      %q[
+        input {
+          elasticsearch {
+            hosts => ["localhost"]
+            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+          }
         }
+      ]
+    end
+
+    let(:mock_response) do
+      {
+          "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+          "took" => 27,
+          "timed_out" => false,
+          "_shards" => {
+              "total" => 169,
+              "successful" => 169,
+              "failed" => 0
+          },
+          "hits" => {
+              "total" => 1,
+              "max_score" => 1.0,
+              "hits" => [ {
+                              "_index" => "logstash-2014.10.12",
+                              "_type" => "logs",
+                              "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                              "_score" => 1.0,
+                              "_source" => { "message" => ["ohayo"] }
+                          } ]
+          }
       }
-    ]
-
-    response = {
-      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-      "took" => 27,
-      "timed_out" => false,
-      "_shards" => {
-        "total" => 169,
-        "successful" => 169,
-        "failed" => 0
-      },
-      "hits" => {
-        "total" => 1,
-        "max_score" => 1.0,
-        "hits" => [ {
-          "_index" => "logstash-2014.10.12",
-          "_type" => "logs",
-          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-          "_score" => 1.0,
-          "_source" => { "message" => ["ohayo"] }
-        } ]
+    end
+
+    let(:mock_scroll_response) do
+      {
+          "_scroll_id" => "r453Wc1jh0caLJhSDg",
+          "hits" => { "hits" => [] }
       }
-    }
+    end
 
-    scroll_reponse = {
-      "_scroll_id" => "r453Wc1jh0caLJhSDg",
-      "hits" => { "hits" => [] }
-    }
+    before(:each) do
+      client = Elasticsearch::Client.new
+      expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      expect(client).to receive(:search).with(any_args).and_return(mock_response)
+      expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      expect(client).to receive(:clear_scroll).and_return(nil)
+    end
 
-    client = Elasticsearch::Client.new
-    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-    expect(client).to receive(:search).with(any_args).and_return(response)
-    expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(scroll_reponse)
-    expect(client).to receive(:clear_scroll).and_return(nil)
+    it 'creates the events from the hits' do
+      event = input(config) do |pipeline, queue|
+        queue.pop
+      end
 
-    event = input(config) do |pipeline, queue|
-      queue.pop
+      expect(event).to be_a(LogStash::Event)
+      puts event.to_hash_with_metadata
+      expect(event.get("message")).to eql [ "ohayo" ]
     end
 
-    expect(event).to be_a(LogStash::Event)
-    expect(event.get("message")).to eql [ "ohayo" ]
-  end
+    context 'when a target is set' do
+      let(:config) do
+        %q[
+          input {
+            elasticsearch {
+              hosts => ["localhost"]
+              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+              target => "[@metadata][_source]"
+            }
+          }
+        ]
+      end
+
+      it 'creates the event using the target' do
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
 
+        expect(event).to be_a(LogStash::Event)
+        puts event.to_hash_with_metadata
+        expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
+      end
+    end
+  end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
   # to support sliced scrolling. The underlying implementation will spawn its own threads to consume, so we must be
@@ -583,7 +617,37 @@ describe LogStash::Inputs::TestableElasticsearch do
         let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
-          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:client)
+          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
         end
       end
     end if LOGSTASH_VERSION > '6.0'
@@ -610,6 +674,54 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
       end
     end
+
+    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+      let(:config_value) { fail NotImplementedError }
+      let(:config) { super().merge(config_name => config_value) }
+      {
+          :string   => 'banana',
+          :negative => -123,
+          :zero     => 0,
+      }.each do |value_desc, value|
+        let(:config_value) { value }
+        context "with an invalid #{value_desc} value" do
+          it 'prevents instantiation with a helpful message' do
+            expect(described_class.logger).to receive(:error).with(/Expected positive whole number/)
+            expect { described_class.new(config) }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+      end
+
+      context 'with a valid value' do
+        let(:config_value) { 17 }
+
+        it "instantiates the elasticsearch client with the timeout value set via #{manticore_transport_option} in the transport options" do
+          expect(Elasticsearch::Client).to receive(:new) do |new_elasticsearch_client_params|
+            # We rely on Manticore-specific transport options, fail early if we are using a different
+            # transport or are allowing the client to determine its own transport class.
+            expect(new_elasticsearch_client_params).to include(:transport_class)
+            expect(new_elasticsearch_client_params[:transport_class].name).to match(/\bManticore\b/)
+
+            expect(new_elasticsearch_client_params).to include(:transport_options)
+            transport_options = new_elasticsearch_client_params[:transport_options]
+            expect(transport_options).to include(manticore_transport_option)
+            expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+          end
+
+          plugin.register
+        end
+      end
+    end
+
+    context 'connect_timeout_seconds' do
+      include_examples('configurable timeout', 'connect_timeout_seconds', :connect_timeout)
+    end
+    context 'request_timeout_seconds' do
+      include_examples('configurable timeout', 'request_timeout_seconds', :request_timeout)
+    end
+    context 'socket_timeout_seconds' do
+      include_examples('configurable timeout', 'socket_timeout_seconds', :socket_timeout)
+    end
   end
 
   context "when scheduling" do

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.6.2
+  version: 4.9.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-02 00:00:00.000000000 Z
+date: 2020-12-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -200,6 +214,7 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
+- lib/logstash/inputs/patch.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca/ca.crt
@@ -230,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster