logstash-input-elasticsearch 4.7.0 → 4.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7719819282371ea4fbfc4ce20de33e30a47f60a3683eb865de44f46743b0df0
-  data.tar.gz: 0a73015d8f782597a939a326aebddc79bf260bdf1f02b543f00823f59bc74e27
+  metadata.gz: 70ea634a4cac65bd247f61462fc51679e7f331191879200061538f5996f3f832
+  data.tar.gz: c813467a5737330193d68c248e35cdc74fa1ba5150823e59530623ace7b62ed4
 SHA512:
-  metadata.gz: a2dea22533a17a4b47d72f5fe58ac5d1b819f58de71e6311d8dc2f6adac733be96bbf0748dcf4fdd5bca644dae15f755463c72313dff3e11ad805f1bebc350ad
-  data.tar.gz: bdebe2a507305e0fe0d788cce9e47acbc607de2992257a7d60f486c16126b7c2a4f7e47fd1a12be87eb2b57a1c9df9d5c9c19c9b2d919f5a0b23eda06b3a9f8c
+  metadata.gz: acd8b8807b116d1b926a5d286edce78b4aee0c3d37c86df18933f973c8684b870a63b70626bcc21bcc29f5cb0ec8420b3b1e64715df24eccd9d091f6b7669334
+  data.tar.gz: 6b2c2b47af0b30881120a36118c9dac6b77ff6d01a2f0db396cc218977ad361fd8b9a56966ba623135709d74ad1fc2c6640dddb80148ecb44f5d5f24aef06b83

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## 4.9.1
+  - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
+  - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)
+
+## 4.9.0
+  - Added `target` option, allowing the hit's source to target a specific field instead of being expanded at the root of the event. This allows the input to play nicer with the Elastic Common Schema when the input does not follow the schema. [#117](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/117)
+
+## 4.8.3
+  - [DOC] Fixed links to restructured Logstash-to-cloud docs [#139](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/139)
+
+## 4.8.2
+  - [DOC] Document the permissions required in secured clusters [#137](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/137)
+
+## 4.8.1
+  - Fixed connection error when using multiple `slices`. [#133](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/133)
+
+## 4.8.0
+  - Added the ability to configure connection-, request-, and socket-timeouts with `connect_timeout_seconds`, `request_timeout_seconds`, and `socket_timeout_seconds` [#121](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/121)
+
+## 4.7.1
+  - [DOC] Updated sliced scroll link to resolve to correct location after doc structure change [#135](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/135)
+  - [DOC] Added usage example of docinfo metadata [#98](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/98)
+
 ## 4.7.0
   - Added api_key support [#131](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/131)
 

data/CONTRIBUTORS

@@ -7,6 +7,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * João Duarte (jsvd)
 * Kurt Hurtado (kurtado)
+* Luca Belluccini (lucabelluccini)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)

data/README.md

@@ -1,7 +1,7 @@
 # Logstash Plugin
 
 [![Gem Version](https://badge.fury.io/rb/logstash-input-elasticsearch.svg)](https://badge.fury.io/rb/logstash-input-elasticsearch)
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-input-elasticsearch)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-input-elasticsearch)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -77,6 +77,11 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 * <<plugins-{type}s-{plugin}-cloud_auth>>
 * <<plugins-{type}s-{plugin}-api_key>>
 
+[id="plugins-{type}s-{plugin}-autz"]
+==== Authorization
+
+Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
+The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
@@ -90,6 +95,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -98,11 +104,14 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -119,7 +128,10 @@ input plugins.
 
 Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].
+Format is `id:api_key` where `id` and `api_key` are as returned by the
+Elasticsearch
+{ref}/security-api-create-api-key.html[Create
+API key API].
 
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
@@ -137,7 +149,8 @@ SSL Certificate Authority file in PEM encoded format, must also include any chai
 
 Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
 [id="plugins-{type}s-{plugin}-cloud_id"]
 ===== `cloud_id`
@@ -147,7 +160,17 @@ For more info, check out the https://www.elastic.co/guide/en/logstash/current/co
 
 Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
+
+[id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
+===== `connect_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
+Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
@@ -184,6 +207,19 @@ Example
       }
     }
 
+If set, you can use metadata information in the <<plugins-{type}s-{plugin}-add_field>> common option.
+
+Example
+[source, ruby]
+    input {
+      elasticsearch {
+        docinfo => true
+        add_field => {
+          identifier => "%{[@metadata][_index]}:%{[@metadata][_type]}:%{[@metadata][_id]}"
+        }
+      }
+    }
+
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
@@ -222,10 +258,9 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. See
-https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html[Multi Indices documentation]
-in the Elasticsearch documentation for more information on how to reference
-multiple indices.
+The index or alias to search. See {ref}/multi-index.html[Multi Indices
+documentation] in the Elasticsearch documentation for more information on how to
+reference multiple indices.
 
 
 [id="plugins-{type}s-{plugin}-password"]
@@ -254,9 +289,18 @@ environment variables e.g. `proxy => '${LS_PROXY:}'`.
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
 
-The query to be executed. Read the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
-for more information.
+The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL
+documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
+===== `request_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, for a single request to Elasticsearch.
+Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
+documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
 
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
@@ -296,8 +340,8 @@ This allows you to set the maximum number of hits returned per scroll.
   * Sensible values range from 2 to about 8.
 
 In some cases, it is possible to improve overall throughput by consuming multiple
-distinct slices of a query simultaneously using the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
+distinct slices of a query simultaneously using
+{ref}/paginate-search-results.html#slice-scroll[sliced scrolls],
 especially if the pipeline is spending significant time waiting on Elasticsearch
 to provide results.
 
@@ -321,6 +365,28 @@ instructions into the query.
 If enabled, SSL will be used when communicating with the Elasticsearch
 server (i.e. HTTPS will be used instead of plain HTTP).
 
+[id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
+===== `socket_timeout_seconds`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended.
+Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query.
+
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+* Value type is {logstash-ref}/field-references-deepdive.html[field reference]
+* There is no default value for this setting.
+
+Without a `target`, events are created from each hit's `_source` at the root level.
+When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead.
+
+This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
+It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -3,7 +3,9 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "base64"
+require_relative "patch"
 
 
 # .Compatibility Note
@@ -61,6 +63,8 @@ require "base64"
 #
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "elasticsearch"
 
   default :codec, "json"
@@ -135,6 +139,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Connection Timeout, in Seconds
+  config :connect_timeout_seconds, :validate => :positive_whole_number, :default => 10
+
+  # Request Timeout, in Seconds
+  config :request_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Socket Timeout, in Seconds
+  config :socket_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
   # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
@@ -165,6 +178,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
+  # If set, the _source of each hit will be added nested under the target instead of at the top-level
+  config :target, :validate => :field_reference
+
   def register
     require "elasticsearch"
     require "rufus/scheduler"
@@ -189,6 +205,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
     hosts = setup_hosts
     ssl_options = setup_ssl
@@ -205,22 +224,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     )
   end
 
-  ##
-  # @override to handle proxy => '' as if none was set
-  # @param value [Array<Object>]
-  # @param validator [nil,Array,Symbol]
-  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
-  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
-  def self.validate_value(value, validator)
-    return super unless validator == :uri_or_empty
 
-    value = deep_replace(value)
-    value = hash_or_array(value)
-
-    return true, value.first if value.size == 1 && value.first.empty?
-
-    return super(value, :uri)
-  end
 
   def run(output_queue)
     if @schedule
@@ -300,7 +304,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def push_hit(hit, output_queue)
-    event = LogStash::Event.new(hit['_source'])
+    if @target.nil?
+      event = LogStash::Event.new(hit['_source'])
+    else
+      event = LogStash::Event.new
+      event.set(@target, hit['_source'])
+    end
 
     if @docinfo
       # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
@@ -439,4 +448,42 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  module URIOrEmptyValidator
+    ##
+    # @override to provide :uri_or_empty validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :uri_or_empty
+
+      value = deep_replace(value)
+      value = hash_or_array(value)
+
+      return true, value.first if value.size == 1 && value.first.empty?
+
+      return super(value, :uri)
+    end
+  end
+  extend(URIOrEmptyValidator)
+
+  module PositiveWholeNumberValidator
+    ##
+    # @override to provide :positive_whole_number validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :positive_whole_number
+
+      is_number, coerced_number = super(value, :number)
+
+      return [true, coerced_number.to_i] if is_number && coerced_number.denominator == 1 && coerced_number > 0
+
+      return [false, "Expected positive whole number, got `#{value.inspect}`"]
+    end
+  end
+  extend(PositiveWholeNumberValidator)
 end

data/lib/logstash/inputs/patch.rb

@@ -0,0 +1,48 @@
+if Gem.loaded_specs['elasticsearch-transport'].version >= Gem::Version.new("7.2.0")
+  # elasticsearch-transport versions prior to 7.2.0 suffered of a race condition on accessing
+  # the connection pool. This issue was fixed with https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
+   # This plugin, at the moment, is forced to use v5.x so we have to monkey patch the gem. When this requirement
+   # ceases, this patch could be removed.
+  puts "WARN remove the patch code into logstash-input-elasticsearch plugin"
+else
+  module Elasticsearch
+    module Transport
+      module Transport
+        module Connections
+          module Selector
+
+            # "Round-robin" selector strategy (default).
+            #
+            class RoundRobin
+              include Base
+
+              # @option arguments [Connections::Collection] :connections Collection with connections.
+              #
+              def initialize(arguments = {})
+                super
+                @mutex = Mutex.new
+                @current = nil
+              end
+
+              # Returns the next connection from the collection, rotating them in round-robin fashion.
+              #
+              # @return [Connections::Connection]
+              #
+              def select(options={})
+                @mutex.synchronize do
+                  conns = connections
+                  if @current && (@current < conns.size-1)
+                    @current += 1
+                  else
+                    @current = 0
+                  end
+                  conns[@current]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.7.0'
+  s.version         = '4.9.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'

data/spec/inputs/elasticsearch_spec.rb

@@ -40,57 +40,91 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
   end
 
-  it "should retrieve json event from elasticseach" do
-    config = %q[
-      input {
-        elasticsearch {
-          hosts => ["localhost"]
-          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+  context 'creating events from Elasticsearch' do
+    let(:config) do
+      %q[
+        input {
+          elasticsearch {
+            hosts => ["localhost"]
+            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+          }
         }
+      ]
+    end
+
+    let(:mock_response) do
+      {
+          "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+          "took" => 27,
+          "timed_out" => false,
+          "_shards" => {
+              "total" => 169,
+              "successful" => 169,
+              "failed" => 0
+          },
+          "hits" => {
+              "total" => 1,
+              "max_score" => 1.0,
+              "hits" => [ {
+                              "_index" => "logstash-2014.10.12",
+                              "_type" => "logs",
+                              "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                              "_score" => 1.0,
+                              "_source" => { "message" => ["ohayo"] }
+                          } ]
+          }
       }
-    ]
-
-    response = {
-      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-      "took" => 27,
-      "timed_out" => false,
-      "_shards" => {
-        "total" => 169,
-        "successful" => 169,
-        "failed" => 0
-      },
-      "hits" => {
-        "total" => 1,
-        "max_score" => 1.0,
-        "hits" => [ {
-          "_index" => "logstash-2014.10.12",
-          "_type" => "logs",
-          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-          "_score" => 1.0,
-          "_source" => { "message" => ["ohayo"] }
-        } ]
+    end
+
+    let(:mock_scroll_response) do
+      {
+          "_scroll_id" => "r453Wc1jh0caLJhSDg",
+          "hits" => { "hits" => [] }
       }
-    }
+    end
 
-    scroll_reponse = {
-      "_scroll_id" => "r453Wc1jh0caLJhSDg",
-      "hits" => { "hits" => [] }
-    }
+    before(:each) do
+      client = Elasticsearch::Client.new
+      expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      expect(client).to receive(:search).with(any_args).and_return(mock_response)
+      expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      expect(client).to receive(:clear_scroll).and_return(nil)
+    end
 
-    client = Elasticsearch::Client.new
-    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-    expect(client).to receive(:search).with(any_args).and_return(response)
-    expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(scroll_reponse)
-    expect(client).to receive(:clear_scroll).and_return(nil)
+    it 'creates the events from the hits' do
+      event = input(config) do |pipeline, queue|
+        queue.pop
+      end
 
-    event = input(config) do |pipeline, queue|
-      queue.pop
+      expect(event).to be_a(LogStash::Event)
+      puts event.to_hash_with_metadata
+      expect(event.get("message")).to eql [ "ohayo" ]
     end
 
-    expect(event).to be_a(LogStash::Event)
-    expect(event.get("message")).to eql [ "ohayo" ]
-  end
+    context 'when a target is set' do
+      let(:config) do
+        %q[
+          input {
+            elasticsearch {
+              hosts => ["localhost"]
+              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+              target => "[@metadata][_source]"
+            }
+          }
+        ]
+      end
+
+      it 'creates the event using the target' do
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
 
+        expect(event).to be_a(LogStash::Event)
+        puts event.to_hash_with_metadata
+        expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
+      end
+    end
+  end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
   # to support sliced scrolling. The underlying implementation will spawn its own threads to consume, so we must be
@@ -640,6 +674,54 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
       end
     end
+
+    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+      let(:config_value) { fail NotImplementedError }
+      let(:config) { super().merge(config_name => config_value) }
+      {
+          :string   => 'banana',
+          :negative => -123,
+          :zero     => 0,
+      }.each do |value_desc, value|
+        let(:config_value) { value }
+        context "with an invalid #{value_desc} value" do
+          it 'prevents instantiation with a helpful message' do
+            expect(described_class.logger).to receive(:error).with(/Expected positive whole number/)
+            expect { described_class.new(config) }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+      end
+
+      context 'with a valid value' do
+        let(:config_value) { 17 }
+
+        it "instantiates the elasticsearch client with the timeout value set via #{manticore_transport_option} in the transport options" do
+          expect(Elasticsearch::Client).to receive(:new) do |new_elasticsearch_client_params|
+            # We rely on Manticore-specific transport options, fail early if we are using a different
+            # transport or are allowing the client to determine its own transport class.
+            expect(new_elasticsearch_client_params).to include(:transport_class)
+            expect(new_elasticsearch_client_params[:transport_class].name).to match(/\bManticore\b/)
+
+            expect(new_elasticsearch_client_params).to include(:transport_options)
+            transport_options = new_elasticsearch_client_params[:transport_options]
+            expect(transport_options).to include(manticore_transport_option)
+            expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+          end
+
+          plugin.register
+        end
+      end
+    end
+
+    context 'connect_timeout_seconds' do
+      include_examples('configurable timeout', 'connect_timeout_seconds', :connect_timeout)
+    end
+    context 'request_timeout_seconds' do
+      include_examples('configurable timeout', 'request_timeout_seconds', :request_timeout)
+    end
+    context 'socket_timeout_seconds' do
+      include_examples('configurable timeout', 'socket_timeout_seconds', :socket_timeout)
+    end
   end
 
   context "when scheduling" do

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.7.0
+  version: 4.9.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-03 00:00:00.000000000 Z
+date: 2021-01-21 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -200,6 +214,7 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
+- lib/logstash/inputs/patch.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca/ca.crt