logstash-input-elasticsearch 4.3.1 → 4.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a87e0f6689410ccaef80bdc7d83b61d414c1f72ba63ea4524b479723931d390a
-  data.tar.gz: 6bc8fc519e80e98ecdeeaf69d6043464eee30850bcd09a8fa245751c2500a79f
+  metadata.gz: 0255dad9a7c1cdd4b9feae3431528d84c6dca6665d76e01d49a8671cfd50b7ec
+  data.tar.gz: 4f62de728c1b6090338b96d0dc9ed751428bdb35290cc7fc7d677f005733aba9
 SHA512:
-  metadata.gz: 8c4ce2ec0b83e77be2022ecea02e716bc5d87d3c1d8f82f61e616b6e3725f8636e0e02f1035bb51feef75d1c00594c5cd2a53e52622878c7a438e309f42124e4
-  data.tar.gz: 1c5cfcf1f0a57c94dc03cb4ed83f968f8d66b372acdf96792db6da4dd8a11f12ece1c00fe3c9cdad3d4aa8c6035ed2be49be47152d4e166b3c7ab31a1ce7793e
+  metadata.gz: b3e71209504174bcab3cdd2f937d669dd830c8d9e140c3564f50b43abddd89a7b4dc31ad2bf77439a658429f3026e7f10370eb2870a461a3f86d74774c9588bc
+  data.tar.gz: aac7f02f589d34f167e6a71e0b0825408df1f24908c9c35289a8aec690c7b22c2cbfb94a6e101ab30e9749cc42c7aefda503064698e45b505127d5f3b77f7e2f

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 4.6.0
+  - Feat: added option to specify proxy for ES [#114](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/114)
+
+## 4.5.0
+  - Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
+
+## 4.4.0
+  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111) 
+
+## 4.3.3
+  - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
+
+## 4.3.2
+  - Fixed broken link to Elasticsearch Reference  [#106](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/106)
+
 ## 4.3.1
   - Fixed deeplink to Elasticsearch Reference  [#103](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/103)
 

data/docs/index.asciidoc

@@ -87,12 +87,15 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
@@ -113,8 +116,27 @@ input plugins.
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-SSL Certificate Authority file in PEM encoded format, must also
-include any chain certificates as necessary.
+SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
+
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
 
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
@@ -210,6 +232,16 @@ The password to use together with the username in the `user` option
 when authenticating to the Elasticsearch server. If set to an empty
 string authentication will be disabled.
 
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy.
+An empty string is treated as if proxy was not set, this is useful when using
+environment variables e.g. `proxy => '${LS_PROXY:}'`.
+
 [id="plugins-{type}s-{plugin}-query"]
 ===== `query` 
 
@@ -259,7 +291,7 @@ This allows you to set the maximum number of hits returned per scroll.
 
 In some cases, it is possible to improve overall throughput by consuming multiple
 distinct slices of a query simultaneously using the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#sliced-scroll[Sliced Scroll API],
+https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
 especially if the pipeline is spending significant time waiting on Elasticsearch
 to provide results.
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -2,8 +2,10 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/json"
+require "logstash/util/safe_uri"
 require "base64"
 
+
 # .Compatibility Note
 # [NOTE]
 # ================================================================================
@@ -68,6 +70,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Port defaults to 9200
   config :hosts, :validate => :array
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -133,6 +140,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+  config :cloud_auth, :validate => :password
+
+  # Set the address of a forward HTTP proxy.
+  config :proxy, :validate => :uri_or_empty
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -149,6 +164,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "elasticsearch"
     require "rufus/scheduler"
+    require "elasticsearch/transport/transport/http/manticore"
 
     @options = {
       :index => @index,
@@ -163,12 +179,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     transport_options = {}
 
+    fill_user_password_from_cloud_auth
+
     if @user && @password
       token = Base64.strict_encode64("#{@user}:#{@password.value}")
       transport_options[:headers] = { :Authorization => "Basic #{token}" }
     end
 
-    hosts = if @ssl then
+    fill_hosts_from_cloud_id
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+
+    hosts = if @ssl
       @hosts.map do |h|
         host, port = h.split(":")
         { :host => host, :scheme => 'https', :port => port }
@@ -176,14 +197,34 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     else
       @hosts
     end
+    ssl_options = { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
+    ssl_options ||= {}
 
-    if @ssl && @ca_file
-      transport_options[:ssl] = { :ca_file => @ca_file }
-    end
+    @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
-    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
+    transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
+
+    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options,
+                                        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+                                        :ssl => ssl_options)
   end
 
+  ##
+  # @override to handle proxy => '' as if none was set
+  # @param value [Array<Object>]
+  # @param validator [nil,Array,Symbol]
+  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+  def self.validate_value(value, validator)
+    return super unless validator == :uri_or_empty
+
+    value = deep_replace(value)
+    value = hash_or_array(value)
+
+    return true, value.first if value.size == 1 && value.first.empty?
+
+    return super(value, :uri)
+  end
 
   def run(output_queue)
     if @schedule
@@ -273,10 +314,70 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def scroll_request scroll_id
-    @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
+    client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
 
   def search_request(options)
-    @client.search(options)
+    client.search(options)
+  end
+
+  attr_reader :client
+
+  def hosts_default?(hosts)
+    hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
+  end
+
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
+  def fill_user_password_from_cloud_auth
+    return unless @cloud_auth
+
+    if @user || @password
+      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
+    end
+    @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+    params['user'], params['password'] = @user, @password
+  end
+
+  def parse_host_uri_from_cloud_id(cloud_id)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_id'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set hosts instead).'
+    end
+
+    begin
+      cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+    end
+    cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+    LogStash::Util::SafeURI.new(cloud_uri)
   end
+
+  def parse_user_password_from_cloud_auth(cloud_auth)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_auth'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set user/password instead).'
+    end
+
+    cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+    begin
+      cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+    end
+    [ cloud_auth.username, cloud_auth.password ]
+  end
+
 end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.3.1'
+  s.version         = '4.6.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
-  s.add_runtime_dependency 'elasticsearch', ['>= 5.0.3', '< 6.0.0']
+  s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'
 
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
   s.add_runtime_dependency 'rufus-scheduler'
+  s.add_runtime_dependency 'manticore', "~> 0.6"
+  s.add_runtime_dependency 'faraday', "~> 0.15.4"
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'

data/spec/es_helper.rb

@@ -0,0 +1,59 @@
+module ESHelper
+  def self.get_host_port
+    return "elasticsearch:9200" if ENV["INTEGRATION"] == "true" || ENV["SECURE_INTEGRATION"] == "true"
+    raise "This setting is only used for integration tests"
+  end
+
+  def self.get_client(options = {})
+    ssl_options = {}
+    hosts = [get_host_port]
+
+    if options[:ca_file]
+      ssl_options = { :ssl  => true, :ca_file => options[:ca_file] }
+      hosts.map! do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    end
+
+    transport_options = {}
+
+    if options[:user] && options[:password]
+      token = Base64.strict_encode64("#{options[:user]}:#{options[:password]}")
+      transport_options[:headers] = { :Authorization => "Basic #{token}" }
+    end
+
+    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options, :ssl => ssl_options,
+                                        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore)
+  end
+
+  def self.doc_type
+    if ESHelper.es_version_satisfies?(">=8")
+      nil
+    elsif ESHelper.es_version_satisfies?(">=7")
+      "_doc"
+    else
+      "doc"
+    end
+  end
+
+  def self.index_doc(es, params)
+    type = doc_type
+    params[:type] = doc_type unless type.nil?
+    es.index(params)
+  end
+
+  def self.es_version
+    ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+  end
+
+  def self.es_version_satisfies?(*requirement)
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    if es_version.nil?
+      puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
+      return false
+    end
+    es_release_version = Gem::Version.new(es_version).release
+    Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
+  end
+end

data/spec/fixtures/test_certs/ca/ca.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
+BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
+DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
+WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
+BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
+gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
+B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
+6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
+4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
+nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
+9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
+wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
+ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
+UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
+w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
+g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
+dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
+UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
+1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
+MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
+jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
+Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
+DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
+lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
+36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
+rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
+sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
+nIg9fQPvVw+fIHACZQ==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca/ca.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
+Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
+nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
+AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
+COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
+Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
+eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
+X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
+eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
+CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
+RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
+AQKCAgBVVGU6qk5i2xrkO5oHO+8YwOpfsBdJG7yIEsYamJhHveH3zW/6vpHIw7Eq
+G8UXRtnA2svqKqXp9YI0Wns71NNlvoi1bO3pP6IpH/PpFb9PdaEtB3/mC5HsFNXN
+svb3mecILC6E9In6XUHx5hWwQstXgTZcGVA1VfqnAGUgQ6goyTbAasRMkuM9+i0m
+I1e47XGF/69dVatCDvZBpJKMn2vMlvR3sYw4fP8zMiFtLPb4mq1OKerEX6Fz7zTl
+oh119+m5koXdEzso9jKO2UTz85XT2JKGcriO5/e3D4v/RcLNPk2+Ek+CavgJKGMQ
+WogqaHjTyu+wUm7omqA6VuGDLZqh1r0xYR+EXVMAudLjy7/NtAaE4MVOqVRs4WVd
+sGccyirkTosxlvK3/vTfsp0VQtreBbxO1maqR5od0aa36MrP4Sk5O07yB9GAthp8
+5qlqtiYaO2Hcq2KJjKPUGwXlAWFZtENQe+G/jy+gYVDwKRInK7f7HubZlAMwsq2S
+LSjtgvhqayAMsa7HoeevSVPLVdFb1IVkIw2jgMhXRgxmKa8WzbAUs124f9ey9z81
+si7w+qpZHq9LGChBjweTbd0abCianyRGHZIlDBE43XEcs3easxuHM6eOoJz0B7aj
+oCXBCo/6Zd0om4ll5jva2+VOH2wTkZk7OhPiGU2f4g7kTJNAAQKCAQEA7YT3UdjN
+HybAD3c/a5Kh17R4zCvymQkUWBs80ZaV9LlCYgie6aWlY6e+9m6vpDhY8dJZd+Zm
+hlAF3VitRLw3SQUEImEC1caS1q99o1eQxMGu+mk9OiibF9PzZwoPE6zt5EZ0I/Ha
+ifmf0Jn3xLyWF4qOKvO3gbWLknirDKffzNnWtFr4SQlEwtOR4m7IFDEz7e7RoGlv
+K1qEFyK1PCfR8GeVHXWkJ3udHJRIZlEtNNLkHzU4nCRRYTvQ4l67rD9Tj7CoLbH1
+2OGSSvAkg+1lTBBs6RXZvhVvLrJVtQTXR7Oi8Z3mi3iJu9oWYa/OFaJl4lAN9xTe
+QY0u0J1+AS5qAQKCAQEA2yUhO3rC1A7qHgbY4dAsx8f8yy9D0yCrI9OLnPQNF3ws
+4mC1fUS60+07u0FYkgU3zIDwdLj5nsxWjB4ciY4hCgwz7pNJWlowurrfTvQNlqvC
+m+Jrt1HYoaV+73mSj+rGv89HXWBW2I/1ED37BRoNB/YIMd/MUL8h0ubt3LIVaRow
+41DT3dM969zuw3Avpx1uXQdnijJ1kA3oHpJ756YLHri/Nv6K0hJmGAbMrHPRxuhY
+hYrxPJPVlp5mWUIjNkKoaNl3du3a6iVSbf7W15LxhAHmkKozxnhqoMldI6C8R548
+IKGyW4wo3GQvcEGPhgGnz2lswmvtx/6cWMv81b7sdwKCAQAXiC3sqPshk/hBUAIz
+iTKJqXKyX8RITzL5y7EJ3s56kYQ3LD16TpQFPJBZ3/t83KxLQRjmHSiZNAJwvKFm
+BvO/Q0T2lv/t6B+SL47WCQ3pwHqyioyrX2yGPtSFp+R4gZCMJkLJcOPC+b1QsIBw
+uHJyYKLXNJBAxJjQaS4hMdylcguokL66lbV/S/DPK0SdY4aOkzOnneqKtAwUMrcb
+/6H4HHsUkRwzYTbepv5JTM+axS4evWofZiW96Ww3kUUsupVvzgPLiy8dTrPswsAL
+ZhC8KYBw015gS8VZLgf5yEH/85c4MvmtZcLXnrLK+N0FHbLUajQH/8RJYFB8EK50
+NYIBAoIBAQCNO8/AIqz/uCEAew858U168BOm62492lcRvtvCqrLpSNkwiH1PH4V8
+4e7WDxZC/WPpw8u0niYaRr0cjqd7q4L1k8nAX2It/eRb4+XZX4aGbWn9xx3+xpvk
+CeHV+rcPU5MFuVPYBSfTaGvbLObjcdemItVDN2XZQGVPJA92ZbtQwlVxmv0AgIzu
+vrOOd3XusRQYlpYPRdfooF3RnjyGncea5BwwGDpliwALSg6MshQnqkSqxFIW5XwJ
+F0sFCY/h/3HEKStKFZ85uhX21/+kbYqDtinfYCIALkkiGMSTGptdWMiNi0iEob8P
+0u2T3lzeU6DQFrTGVIKpmxkuTUFEjEifAoIBAH4nyu4ei4u7sGUhSZ79egUbqBZR
+pjYblM8NB5UOAVmmbaswCWozsnsaBdIgymeokQXDPqIOwadP8IgGrgIxS5phdgvE
+CNepxQDoVTXYEecPjc0LL4Kb+urmJL7HEP9BteIkc+7l8b9USDhNlJeCXICoJKBu
+bNxgm60ZuoObx7h5APq9wC4x8Xj7AxQKu57Ied/tRFPCHW4UbhZhJfrnS2xTTk0u
+z71AS7akI/NPfm3nLviISZeDzTgYs6vLYr/j4JUlcw1z6UpX4DvNm/MULi7ItXP5
+yV2H8jpCdjAe+OoC3OHTuQ8FQR29y758HUY7iF8ruuqUSWxF7pfL/a27EMw=
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/test.crt

@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
+BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
+CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
+VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
+IVxXSvau2ZZ94rkQR2xNL8TLijBNx46mU+kCniy8X5r+LX9seGqdBhhTh/tCJzh8
+MCzMt2JIijSjVyw28iiCb8/669LMTp5lFlRKajj11jlIpIm3o+OHqUzYwcSOw8og
+p0A3nvAQ33Srghm/oAcT2umGrFyYXWT6PnGaEJRLUQn7LuHJnRLseCF2Cn/RzFK7
+/tiVVjImmQiVB3dE9fMR/pVJiO2v0COnWuG+/brXWrQIHk0AuD8pHc6Iw9iZODkc
+Ao53B41qbvqcbdXFN5XfL4tb+lkBuLioCX7j9zR44awvuj9hKfuqFOFTUBZL2RjV
+bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
+1GfnSVZBwO4MMd85taHNvitLnkEREjANSoPUuAJF3SKRHE9K8jUAzhyXflvgNNoM
+tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
+hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
+2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
+6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
+BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
+YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
+FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
+CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
+BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
+A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
+f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
+9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
+y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
+HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
+liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
+y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
+lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
+re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
+Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
+VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
+FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEAxiE/bM84TeR72dp4r5wg81X3WG26gYYBGz6aVj8hXFdK9q7Z
+ln3iuRBHbE0vxMuKME3HjqZT6QKeLLxfmv4tf2x4ap0GGFOH+0InOHwwLMy3YkiK
+NKNXLDbyKIJvz/rr0sxOnmUWVEpqOPXWOUikibej44epTNjBxI7DyiCnQDee8BDf
+dKuCGb+gBxPa6YasXJhdZPo+cZoQlEtRCfsu4cmdEux4IXYKf9HMUrv+2JVWMiaZ
+CJUHd0T18xH+lUmI7a/QI6da4b79utdatAgeTQC4PykdzojD2Jk4ORwCjncHjWpu
++pxt1cU3ld8vi1v6WQG4uKgJfuP3NHjhrC+6P2Ep+6oU4VNQFkvZGNVsUwqykYef
+K1BkX5r6Zzkffcb0eI8/eNkTYmOtaU5rL6PoibKUyCNxmYcCBpR7cRPUZ+dJVkHA
+7gwx3zm1oc2+K0ueQRESMA1Kg9S4AkXdIpEcT0ryNQDOHJd+W+A02gy3JzOhDn8v
+kE2LIDaH7VNTyNaJw0O2XX4K06yARUEEcDqn8geGwds1lY+ti0g72aGGInDVHnKs
+hNeeNljSs5shPDK397pJzdiehB+xIZoskQbRoNzREAsCGDOP5FjzTzrZEo7J93DW
+zDVGYkp0J/507OE/IOm9q68LGsvxRlBPNwGz6qnE5adkVFZ42wvCgOfoldcCAwEA
+AQKCAgA1FkOATCWx+T6WKMudgh/yE16q+vu2KMmzGxsPcOrnaxxS7JawlBpjq9D3
+W9coy8DDIJQPzNE+5cyr/+0+Akz+j3nUVy6C5h7RW/BWWjAuUMvyMa2WXQ3GcxJ/
+eDOtbnYxjTyjhEJvY2EC0hwMTUKJBAONu5PJW2rP19DuH8Gwmzai7GJzSGEbtRST
+0OYfHE6ioNCldce1eKpokaWtHvh41ySXJXUqwg4eIYC1ylmGfr0RwvXOLuBJPNkJ
+wBCOv51I0oragsT/J8Wkgn9zLZmw2DiF8+ZgqJSRPLyr0K1+rrX/Vj1WOQPU+3rh
+VWPP211A7A0qrRuePEbIcHtHP6KPUCepABL44K33zyyOydmnJ7vg3dsW7AN7+Y6O
+H4B24d1ogn4TJwzpZCfRvqJJVu2wsnzleng9PcpXyHhldB6S9h2fPpNqDUBvfxMv
+w/fGZ2ZpOeUKRfQ7VVR3XIWwFq/eDhzLicHipaoM+6gKeOZdJPAc0Ew5jvOXQSBD
+CYCM12a8gnEYd55NLo/fF3wX6Wdq/X6EbWW97gwtmmEqnhcZMxLdeMuKyli22JyX
+Ik7QIDsmPSWhCkI2JvQ+CAZZp6oMEKuSb7UqqfACQreIuxCUmPTZq/pAEUGSCZGP
+wnWqOk5jwxJ4d5TQm7g2RgPC6lTd7as1m4+JB8H1cNVpS2d0AQKCAQEA5tL9WjIK
+u6x1h4a4KPmOQ9B34GxmGM+P9/bQkkJsWYf5eG1MlYPAigAiN0PMYPHDweeROsOO
+rvmJiWXtmPMVFFSalHVotN6LMj400HhHObg7dADDYTBBGX6QuljxBX9jgUiKSOKO
+66ngXEyRdUmTAbral3UuSCFGcqG8Khd3taULO2q5JQLEioFT+Lr0UzHywVSJq06O
+k37aC3zpiQj4S/bJG4GOadYDIENq+gRCIU7Hn4pS1qtxLYMyhtNDzK0pVriYNj9T
+ydHUxSKZO6ogM4423wVKnKOa2Cj4rgKBDHGpJJ9R0ZhrTbAQOa8LgDy1P6aMlvH/
+t9bG/HClmuHrFwKCAQEA271gZpIeCFG/XYIbxO8Uh9wG2cHrt7idw0iFTW4PpOBk
+CGExq7WlkR29BkzxTDEXz5bSgDa8Q83cDf9EjSHVJYVGSYnodz7ZV8jZd2PUdCV1
+dL1kHV7vqz/KKxuCp7icCwzG9rQ1CjsTv8gBM3cN6DrZgw/2F+HjQpCkeyxc6KBR
+Q+167yaLvOv3W0BHdSywtiNDU48oSSBpEK2anh7ShjG3BaEr/gAqGsTvxjsl7zDg
+6MZFegeYPFIEH4ftvLZugPgd3NBg00CfsNRkjVWsH51y4gBy2ZL8d31Q2D2FI94s
+he57Trvs8t8Y9QHGTMCuUk9IwRBpgapmW+c6G65jQQKCAQB0IPquAjc8LIwE31aP
+5t4YaC2nj2HeYa8BBf/xVdHFSY/Ncs/w+RRJWb/dJhMRQxFF4QdEFVcX2RvFMK7V
+IJceX2JWBqvjM4O8h2dy6tCKzZG7zxZ9MxXNicuutUith2W8iY5pFPLqxdDHHw6f
+f6CiYivPv3jFeVhEG/LbCmuDy8FW5162rCnNtTtWDFkX8i077xhEQ4Wf11ZEKVgl
+RYoGTeboG8pWnQF9ne2YU8Qhlc0BC0qaDi8mwrcM9gVKWGRP6RdLU5kIFLWDaODH
+D9Sbm5UnpzXekME6t4JFqaTbaeO7NRyo4pI5x7aiDtsubVyS5WweFSqeh0QdhV8M
+CVWJAoIBAQCJ7OSFfVG8hxNG7lPf2PFaFZF3PXFncEoWOX7yixLmurIPUHKNBQdX
+fwMW4VTULjxz4IdgUvr41E47uu6cZ5ASbQUhQ57WumxR3ZAikFbaUGjZTcd5aA2n
+v/J1/F6WSBoqFWsNjy97rHlI95FJbIEeM1I0IeCmPPMY5RFY/w+SNI7NxFJVqiwr
++TDZ5g70TYjdymSIHmN7AwzvmwhiF5atBKeMsQ2b8R58jwCxvI6jBFsnwMv7PKkh
+s5lC8V8YBKp36UVVRLaB4x5ZL/etfwj7Dyj9EqsERm6R0ebc1ECtstbfekGLugmQ
+qNhRcTu3EXpZz8oq5NJUwVBef1TJ6zwBAoIBAQC7Oq5AdmLzYOX0AYUuT3Cjgl0u
+3Tq1c1uqlVaxQGjA3oqua4SR0+kvmRZbZGLpLAVnnwOjUEfUFBBYgP/5Mo/OiCkQ
+C8eWkSQKXy6OFy5wh4mbL5oJttKjM4ZoB0gpF31+tGOmrfJwacqEPnyZLKzkgBdG
+djVVKP+HH4XUdB2VXst8tvcif+VTrUsD1nuhGMOgbiHZkdx3eug05wfhnYvWljA+
+r/4xcq7TmKSoJqkb0OcOkhqjeZWleA6xRtEvHPGRzbEM67FVsVTxr/N9BX5tS9zu
+YLCNI3tTNsDV0Ac+4rl46rghQ/n2TNSEpwvA/pjytsdPXLOpoapWirrsEiXf
+-----END RSA PRIVATE KEY-----

data/spec/inputs/elasticsearch_spec.rb

@@ -375,15 +375,13 @@ describe LogStash::Inputs::Elasticsearch do
       end
 
       it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        metadata_field = 'metadata_with_hash'
-
         config_metadata_with_hash = %Q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_target => '#{metadata_field}'
+                docinfo_target => 'metadata_with_hash'
               }
             }
         ]
@@ -392,33 +390,23 @@ describe LogStash::Inputs::Elasticsearch do
           queue.pop
         end
 
-        expect(event.get("[#{metadata_field}][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[#{metadata_field}][_type]")).to eq('logs')
-        expect(event.get("[#{metadata_field}][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[#{metadata_field}][awesome]")).to eq("logstash")
+        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
       end
 
-      it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-        metadata_field = 'metadata_with_string'
-
-        config_metadata_with_string = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => '#{metadata_field}'
-              }
-            }
-        ]
-
-        pipeline = new_pipeline_from_string(config_metadata_with_string)
-        queue = Queue.new
-        pipeline.instance_eval do
-          @output_func = lambda { |event| queue << event }
+      context 'if the `docinfo_target` exist but is not of type hash' do
+        let (:config) { {
+            "hosts" => ["localhost"],
+            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            "docinfo" => true,
+            "docinfo_target" => 'metadata_with_string'
+        } }
+        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          plugin.register
+          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
         end
-
-        expect { pipeline.run }.to raise_error(Exception, /incompatible event/)
       end
 
       it "should move the document info to the @metadata field" do
@@ -516,6 +504,103 @@ describe LogStash::Inputs::Elasticsearch do
     end
   end
 
+  describe "client" do
+    let(:config) do
+      {
+
+      }
+    end
+    let(:plugin) { described_class.new(config) }
+    let(:event)  { LogStash::Event.new({}) }
+
+    describe "cloud.id" do
+      let(:valid_cloud_id) do
+        'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
+      end
+
+      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+
+      it "should set host(s)" do
+        plugin.register
+        client = plugin.send(:client)
+        expect( client.transport.hosts ).to eql [{
+                                                     :scheme => "https",
+                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                     :port => 9243,
+                                                     :path => "",
+                                                     :protocol => "https"
+                                                 }]
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
+        end
+      end
+
+      context 'hosts also set' do
+        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "cloud.auth" do
+      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+
+      it "should set authorization" do
+        plugin.register
+        client = plugin.send(:client)
+        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+        expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
+        end
+      end
+
+      context 'user also set' do
+        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "proxy" do
+      let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+
+      it "should set proxy" do
+        plugin.register
+        client = plugin.send(:client)
+        proxy = client.transport.options[:transport_options][:proxy]
+
+        expect( proxy ).to eql "http://localhost:1234"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+
+        it "should not set proxy" do
+          plugin.register
+          client = plugin.send(:client)
+
+          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+        end
+      end
+    end
+  end
+
   context "when scheduling" do
     let(:config) do
       {
@@ -525,39 +610,11 @@ describe LogStash::Inputs::Elasticsearch do
       }
     end
 
-    response = {
-      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-      "took" => 27,
-      "timed_out" => false,
-      "_shards" => {
-        "total" => 169,
-        "successful" => 169,
-        "failed" => 0
-      },
-      "hits" => {
-        "total" => 1,
-        "max_score" => 1.0,
-        "hits" => [ {
-          "_index" => "logstash-2014.10.12",
-          "_type" => "logs",
-          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-          "_score" => 1.0,
-          "_source" => { "message" => ["ohayo"] }
-        } ]
-      }
-    }
-
-    scroll_reponse = {
-      "_scroll_id" => "r453Wc1jh0caLJhSDg",
-      "hits" => { "hits" => [] }
-    }
-
     before do
       plugin.register
     end
 
     it "should properly schedule" do
-
       Timecop.travel(Time.new(2000))
       Timecop.scale(60)
       runner = Thread.new do
@@ -576,5 +633,4 @@ describe LogStash::Inputs::Elasticsearch do
     end
 
   end
-
 end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin"
+require "logstash/inputs/elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Inputs::Elasticsearch do
+
+  let(:config)   { { 'hosts' => [ESHelper.get_host_port],
+                     'index' => 'logs',
+                     'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+  let(:plugin) { described_class.new(config) }
+  let(:event)  { LogStash::Event.new({}) }
+  let(:client_options) {{}}
+
+  before(:each) do
+    @es = ESHelper.get_client(client_options)
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    10.times do
+      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+    end
+    @es.indices.refresh
+    plugin.register
+  end
+
+  after(:each) do
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  shared_examples 'an elasticsearch index plugin' do
+    it 'should retrieve json event from elasticsearch' do
+      queue = []
+      plugin.run(queue)
+      event = queue.pop
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("response")).to eql(404)
+    end
+  end
+
+  describe 'against an unsecured elasticsearch', :integration => true do
+    it_behaves_like 'an elasticsearch index plugin'
+  end
+
+  describe 'against a secured elasticsearch', :secure_integration => true do
+    let(:user) { 'simpleuser' }
+    let(:password) { 'abc123' }
+    let(:ca_file) { "spec/fixtures/test_certs/test.crt" }
+    let(:client_options) {{:ca_file => ca_file, :user => user, :password => password}}
+    let(:config)   { super.merge({
+                       'user' => user,
+                       'password' => password,
+                       'ssl' => true,
+                       'ca_file' => ca_file })
+    }
+    it_behaves_like 'an elasticsearch index plugin'
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.3.1
+  version: 4.6.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-01 00:00:00.000000000 Z
+date: 2020-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,9 +36,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -47,9 +44,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -134,6 +128,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.4
+  name: faraday
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -179,7 +201,13 @@ files:
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
 - logstash-input-elasticsearch.gemspec
+- spec/es_helper.rb
+- spec/fixtures/test_certs/ca/ca.crt
+- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -207,4 +235,10 @@ signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
+- spec/es_helper.rb
+- spec/fixtures/test_certs/ca/ca.crt
+- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb