logstash-input-elasticsearch 4.3.3 → 4.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72f28226e8297df8d4eda501ef9a99a67c11bab0574711f553797c678cd5e9a8
-  data.tar.gz: d141edd1c0664b1f77f4ddc757cda564781aeaa2ada1ebee828ac2f76268c1de
+  metadata.gz: 92c58eb709a67922d73d44a13cca7fc933968747feebb0c7afa81c04ea3543d2
+  data.tar.gz: 4354af2982b8af23492a8c1f4c063c21ce6e2d1bfc1e082a9de243ab1c575248
 SHA512:
-  metadata.gz: 468f91f896d037895c7479e7ae377cee60c4b0c22b05918695f35b6e56bdfa405524fd04d03440133ac40a71fa4ed00251ad27e38792f7c30a8b736ffaa186c9
-  data.tar.gz: 5ba240ae691f622ab6bf410b7603ebbb959ee5c50cddc90748dd26a1a349fcc36057e9147a15d0501ea1e4f0bbc2861f3042c0addc070568ec805cdd6668d999
+  metadata.gz: ba09a7c97593fb2b32cbd4c1f5c4448af2d6ed10c32908ea98b5d4bd54ec0bf11321ae30c48afa745be27bc6280c8fbac6c3f7e62dfa8d5887142dce04e3d0b3
+  data.tar.gz: 86e4b75f7937c4e66a23b5724902d35db23303d297d7d46cfb55092efc14d0141e708b73b2a72ee1f459f27bc4a9ee0f46d337de2b31b9f38675e25286b58778

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 4.6.2
+  - Added scroll clearing and better handling of scroll expiration [#128](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/128)
+
+## 4.6.1
+  - [DOC] Removed outdated compatibility notice [#124](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/124)
+
+## 4.6.0
+  - Feat: added option to specify proxy for ES [#114](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/114)
+
+## 4.5.0
+  - Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
+
+## 4.4.0
+  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111) 
+
 ## 4.3.3
   - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
 

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/docs/index.asciidoc

@@ -21,16 +21,6 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-.Compatibility Note
-[NOTE]
-================================================================================
-Starting with Elasticsearch 5.3, there's an {ref}/modules-http.html[HTTP setting]
-called `http.content_type.required`. If this option is set to `true`, and you
-are using Logstash 2.4 through 5.2, you need to update the Elasticsearch input
-plugin to version 4.0.2 or higher.
-
-================================================================================
-
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
 You can periodically schedule ingestion using a cron syntax 
@@ -87,12 +77,15 @@ This plugin supports the following configuration options plus the <
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
@@ -113,8 +106,27 @@ input plugins.
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-SSL Certificate Authority file in PEM encoded format, must also
-include any chain certificates as necessary.
+SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
+
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
 
 [id="plugins-{type}s-{plugin}-docinfo"]
 ===== `docinfo` 
@@ -152,11 +164,6 @@ Example
     }
 
 
-NOTE: Starting with Logstash 6.0, the `document_type` option is
-deprecated due to the
-https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
-It will be removed in the next major version of Logstash.
-
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 
@@ -210,6 +217,16 @@ The password to use together with the username in the `user` option
 when authenticating to the Elasticsearch server. If set to an empty
 string authentication will be disabled.
 
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy.
+An empty string is treated as if proxy was not set, this is useful when using
+environment variables e.g. `proxy => '${LS_PROXY:}'`.
+
 [id="plugins-{type}s-{plugin}-query"]
 ===== `query` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -2,8 +2,10 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/json"
+require "logstash/util/safe_uri"
 require "base64"
 
+
 # .Compatibility Note
 # [NOTE]
 # ================================================================================
@@ -68,6 +70,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Port defaults to 9200
   config :hosts, :validate => :array
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -133,6 +140,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+  config :cloud_auth, :validate => :password
+
+  # Set the address of a forward HTTP proxy.
+  config :proxy, :validate => :uri_or_empty
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -149,6 +164,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "elasticsearch"
     require "rufus/scheduler"
+    require "elasticsearch/transport/transport/http/manticore"
 
     @options = {
       :index => @index,
@@ -163,12 +179,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     transport_options = {}
 
+    fill_user_password_from_cloud_auth
+
     if @user && @password
       token = Base64.strict_encode64("#{@user}:#{@password.value}")
       transport_options[:headers] = { :Authorization => "Basic #{token}" }
     end
 
-    hosts = if @ssl then
+    fill_hosts_from_cloud_id
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+
+    hosts = if @ssl
       @hosts.map do |h|
         host, port = h.split(":")
         { :host => host, :scheme => 'https', :port => port }
@@ -176,14 +197,37 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     else
       @hosts
     end
+    ssl_options = { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
+    ssl_options ||= {}
 
-    if @ssl && @ca_file
-      transport_options[:ssl] = { :ca_file => @ca_file }
-    end
+    @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
-    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
+    transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
+
+    @client = Elasticsearch::Client.new(
+      :hosts => hosts,
+      :transport_options => transport_options,
+      :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+      :ssl => ssl_options
+    )
   end
 
+  ##
+  # @override to handle proxy => '' as if none was set
+  # @param value [Array<Object>]
+  # @param validator [nil,Array,Symbol]
+  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+  def self.validate_value(value, validator)
+    return super unless validator == :uri_or_empty
+
+    value = deep_replace(value)
+    value = hash_or_array(value)
+
+    return true, value.first if value.size == 1 && value.first.empty?
+
+    return super(value, :uri)
+  end
 
   def run(output_queue)
     if @schedule
@@ -225,25 +269,41 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
 
     logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-    r = search_request(slice_options)
 
-    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
-    logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-
-    has_hits = r['hits']['hits'].any?
+    scroll_id = nil
+    begin
+      r = search_request(slice_options)
 
-    while has_hits && r['_scroll_id'] && !stop?
-      r = process_next_scroll(output_queue, r['_scroll_id'])
+      r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
       logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
-      has_hits = r['has_hits']
+
+      has_hits = r['hits']['hits'].any?
+      scroll_id = r['_scroll_id']
+
+      while has_hits && scroll_id && !stop?
+        has_hits, scroll_id = process_next_scroll(output_queue, scroll_id)
+        logger.debug("Slice progress", slice_id: slice_id, slices: @slices) if logger.debug? && slice_id
+      end
+      logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    ensure
+      clear_scroll(scroll_id)
     end
-    logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
   end
 
+  ##
+  # @param output_queue [#<<]
+  # @param scroll_id [String]: a scroll id to resume
+  # @return [Array(Boolean,String)]: a tuple representing whether the response
+  #
   def process_next_scroll(output_queue, scroll_id)
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
-    {'has_hits' => r['hits']['hits'].any?, '_scroll_id' => r['_scroll_id']}
+    [r['hits']['hits'].any?, r['_scroll_id']]
+  rescue => e
+    # this will typically be triggered by a scroll timeout
+    logger.error("Scroll request error, aborting scroll", error: e.inspect)
+    # return no hits and original scroll_id so we can try to clear it
+    [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
@@ -272,6 +332,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     output_queue << event
   end
 
+  def clear_scroll(scroll_id)
+    @client.clear_scroll(scroll_id: scroll_id) if scroll_id
+  rescue => e
+    # ignore & log any clear_scroll errors
+    logger.warn("Ignoring clear_scroll exception", message: e.message)
+  end
+
   def scroll_request scroll_id
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
@@ -279,4 +346,62 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def search_request(options)
     @client.search(options)
   end
+
+  def hosts_default?(hosts)
+    hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
+  end
+
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
+  def fill_user_password_from_cloud_auth
+    return unless @cloud_auth
+
+    if @user || @password
+      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
+    end
+    @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+    params['user'], params['password'] = @user, @password
+  end
+
+  def parse_host_uri_from_cloud_id(cloud_id)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_id'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set hosts instead).'
+    end
+
+    begin
+      cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+    end
+    cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+    LogStash::Util::SafeURI.new(cloud_uri)
+  end
+
+  def parse_user_password_from_cloud_auth(cloud_auth)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_auth'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set user/password instead).'
+    end
+
+    cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+    begin
+      cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+    end
+    [ cloud_auth.username, cloud_auth.password ]
+  end
+
 end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.3.3'
+  s.version         = '4.6.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
   s.add_runtime_dependency 'rufus-scheduler'
+  s.add_runtime_dependency 'manticore', "~> 0.6"
+  s.add_runtime_dependency 'faraday', "~> 0.15.4"
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'

data/spec/es_helper.rb

@@ -1,11 +1,30 @@
 module ESHelper
   def self.get_host_port
-    return "http://elasticsearch:9200" if ENV["INTEGRATION"] == "true"
+    return "elasticsearch:9200" if ENV["INTEGRATION"] == "true" || ENV["SECURE_INTEGRATION"] == "true"
     raise "This setting is only used for integration tests"
   end
 
-  def self.get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+  def self.get_client(options = {})
+    ssl_options = {}
+    hosts = [get_host_port]
+
+    if options[:ca_file]
+      ssl_options = { :ssl  => true, :ca_file => options[:ca_file] }
+      hosts.map! do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    end
+
+    transport_options = {}
+
+    if options[:user] && options[:password]
+      token = Base64.strict_encode64("#{options[:user]}:#{options[:password]}")
+      transport_options[:headers] = { :Authorization => "Basic #{token}" }
+    end
+
+    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options, :ssl => ssl_options,
+                                        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore)
   end
 
   def self.doc_type

data/spec/fixtures/test_certs/ca/ca.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
+BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
+DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
+WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
+BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
+gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
+B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
+6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
+4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
+nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
+9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
+wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
+ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
+UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
+w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
+g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
+dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
+UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
+1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
+MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
+jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
+Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
+DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
+lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
+36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
+rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
+sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
+nIg9fQPvVw+fIHACZQ==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca/ca.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
+Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
+nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
+AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
+COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
+Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
+eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
+X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
+eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
+CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
+RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
+AQKCAgBVVGU6qk5i2xrkO5oHO+8YwOpfsBdJG7yIEsYamJhHveH3zW/6vpHIw7Eq
+G8UXRtnA2svqKqXp9YI0Wns71NNlvoi1bO3pP6IpH/PpFb9PdaEtB3/mC5HsFNXN
+svb3mecILC6E9In6XUHx5hWwQstXgTZcGVA1VfqnAGUgQ6goyTbAasRMkuM9+i0m
+I1e47XGF/69dVatCDvZBpJKMn2vMlvR3sYw4fP8zMiFtLPb4mq1OKerEX6Fz7zTl
+oh119+m5koXdEzso9jKO2UTz85XT2JKGcriO5/e3D4v/RcLNPk2+Ek+CavgJKGMQ
+WogqaHjTyu+wUm7omqA6VuGDLZqh1r0xYR+EXVMAudLjy7/NtAaE4MVOqVRs4WVd
+sGccyirkTosxlvK3/vTfsp0VQtreBbxO1maqR5od0aa36MrP4Sk5O07yB9GAthp8
+5qlqtiYaO2Hcq2KJjKPUGwXlAWFZtENQe+G/jy+gYVDwKRInK7f7HubZlAMwsq2S
+LSjtgvhqayAMsa7HoeevSVPLVdFb1IVkIw2jgMhXRgxmKa8WzbAUs124f9ey9z81
+si7w+qpZHq9LGChBjweTbd0abCianyRGHZIlDBE43XEcs3easxuHM6eOoJz0B7aj
+oCXBCo/6Zd0om4ll5jva2+VOH2wTkZk7OhPiGU2f4g7kTJNAAQKCAQEA7YT3UdjN
+HybAD3c/a5Kh17R4zCvymQkUWBs80ZaV9LlCYgie6aWlY6e+9m6vpDhY8dJZd+Zm
+hlAF3VitRLw3SQUEImEC1caS1q99o1eQxMGu+mk9OiibF9PzZwoPE6zt5EZ0I/Ha
+ifmf0Jn3xLyWF4qOKvO3gbWLknirDKffzNnWtFr4SQlEwtOR4m7IFDEz7e7RoGlv
+K1qEFyK1PCfR8GeVHXWkJ3udHJRIZlEtNNLkHzU4nCRRYTvQ4l67rD9Tj7CoLbH1
+2OGSSvAkg+1lTBBs6RXZvhVvLrJVtQTXR7Oi8Z3mi3iJu9oWYa/OFaJl4lAN9xTe
+QY0u0J1+AS5qAQKCAQEA2yUhO3rC1A7qHgbY4dAsx8f8yy9D0yCrI9OLnPQNF3ws
+4mC1fUS60+07u0FYkgU3zIDwdLj5nsxWjB4ciY4hCgwz7pNJWlowurrfTvQNlqvC
+m+Jrt1HYoaV+73mSj+rGv89HXWBW2I/1ED37BRoNB/YIMd/MUL8h0ubt3LIVaRow
+41DT3dM969zuw3Avpx1uXQdnijJ1kA3oHpJ756YLHri/Nv6K0hJmGAbMrHPRxuhY
+hYrxPJPVlp5mWUIjNkKoaNl3du3a6iVSbf7W15LxhAHmkKozxnhqoMldI6C8R548
+IKGyW4wo3GQvcEGPhgGnz2lswmvtx/6cWMv81b7sdwKCAQAXiC3sqPshk/hBUAIz
+iTKJqXKyX8RITzL5y7EJ3s56kYQ3LD16TpQFPJBZ3/t83KxLQRjmHSiZNAJwvKFm
+BvO/Q0T2lv/t6B+SL47WCQ3pwHqyioyrX2yGPtSFp+R4gZCMJkLJcOPC+b1QsIBw
+uHJyYKLXNJBAxJjQaS4hMdylcguokL66lbV/S/DPK0SdY4aOkzOnneqKtAwUMrcb
+/6H4HHsUkRwzYTbepv5JTM+axS4evWofZiW96Ww3kUUsupVvzgPLiy8dTrPswsAL
+ZhC8KYBw015gS8VZLgf5yEH/85c4MvmtZcLXnrLK+N0FHbLUajQH/8RJYFB8EK50
+NYIBAoIBAQCNO8/AIqz/uCEAew858U168BOm62492lcRvtvCqrLpSNkwiH1PH4V8
+4e7WDxZC/WPpw8u0niYaRr0cjqd7q4L1k8nAX2It/eRb4+XZX4aGbWn9xx3+xpvk
+CeHV+rcPU5MFuVPYBSfTaGvbLObjcdemItVDN2XZQGVPJA92ZbtQwlVxmv0AgIzu
+vrOOd3XusRQYlpYPRdfooF3RnjyGncea5BwwGDpliwALSg6MshQnqkSqxFIW5XwJ
+F0sFCY/h/3HEKStKFZ85uhX21/+kbYqDtinfYCIALkkiGMSTGptdWMiNi0iEob8P
+0u2T3lzeU6DQFrTGVIKpmxkuTUFEjEifAoIBAH4nyu4ei4u7sGUhSZ79egUbqBZR
+pjYblM8NB5UOAVmmbaswCWozsnsaBdIgymeokQXDPqIOwadP8IgGrgIxS5phdgvE
+CNepxQDoVTXYEecPjc0LL4Kb+urmJL7HEP9BteIkc+7l8b9USDhNlJeCXICoJKBu
+bNxgm60ZuoObx7h5APq9wC4x8Xj7AxQKu57Ied/tRFPCHW4UbhZhJfrnS2xTTk0u
+z71AS7akI/NPfm3nLviISZeDzTgYs6vLYr/j4JUlcw1z6UpX4DvNm/MULi7ItXP5
+yV2H8jpCdjAe+OoC3OHTuQ8FQR29y758HUY7iF8ruuqUSWxF7pfL/a27EMw=
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/test.crt

@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
+BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
+CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
+VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
+IVxXSvau2ZZ94rkQR2xNL8TLijBNx46mU+kCniy8X5r+LX9seGqdBhhTh/tCJzh8
+MCzMt2JIijSjVyw28iiCb8/669LMTp5lFlRKajj11jlIpIm3o+OHqUzYwcSOw8og
+p0A3nvAQ33Srghm/oAcT2umGrFyYXWT6PnGaEJRLUQn7LuHJnRLseCF2Cn/RzFK7
+/tiVVjImmQiVB3dE9fMR/pVJiO2v0COnWuG+/brXWrQIHk0AuD8pHc6Iw9iZODkc
+Ao53B41qbvqcbdXFN5XfL4tb+lkBuLioCX7j9zR44awvuj9hKfuqFOFTUBZL2RjV
+bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
+1GfnSVZBwO4MMd85taHNvitLnkEREjANSoPUuAJF3SKRHE9K8jUAzhyXflvgNNoM
+tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
+hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
+2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
+6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
+BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
+YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
+FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
+CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
+BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
+A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
+f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
+9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
+y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
+HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
+liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
+y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
+lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
+re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
+Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
+VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
+FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEAxiE/bM84TeR72dp4r5wg81X3WG26gYYBGz6aVj8hXFdK9q7Z
+ln3iuRBHbE0vxMuKME3HjqZT6QKeLLxfmv4tf2x4ap0GGFOH+0InOHwwLMy3YkiK
+NKNXLDbyKIJvz/rr0sxOnmUWVEpqOPXWOUikibej44epTNjBxI7DyiCnQDee8BDf
+dKuCGb+gBxPa6YasXJhdZPo+cZoQlEtRCfsu4cmdEux4IXYKf9HMUrv+2JVWMiaZ
+CJUHd0T18xH+lUmI7a/QI6da4b79utdatAgeTQC4PykdzojD2Jk4ORwCjncHjWpu
++pxt1cU3ld8vi1v6WQG4uKgJfuP3NHjhrC+6P2Ep+6oU4VNQFkvZGNVsUwqykYef
+K1BkX5r6Zzkffcb0eI8/eNkTYmOtaU5rL6PoibKUyCNxmYcCBpR7cRPUZ+dJVkHA
+7gwx3zm1oc2+K0ueQRESMA1Kg9S4AkXdIpEcT0ryNQDOHJd+W+A02gy3JzOhDn8v
+kE2LIDaH7VNTyNaJw0O2XX4K06yARUEEcDqn8geGwds1lY+ti0g72aGGInDVHnKs
+hNeeNljSs5shPDK397pJzdiehB+xIZoskQbRoNzREAsCGDOP5FjzTzrZEo7J93DW
+zDVGYkp0J/507OE/IOm9q68LGsvxRlBPNwGz6qnE5adkVFZ42wvCgOfoldcCAwEA
+AQKCAgA1FkOATCWx+T6WKMudgh/yE16q+vu2KMmzGxsPcOrnaxxS7JawlBpjq9D3
+W9coy8DDIJQPzNE+5cyr/+0+Akz+j3nUVy6C5h7RW/BWWjAuUMvyMa2WXQ3GcxJ/
+eDOtbnYxjTyjhEJvY2EC0hwMTUKJBAONu5PJW2rP19DuH8Gwmzai7GJzSGEbtRST
+0OYfHE6ioNCldce1eKpokaWtHvh41ySXJXUqwg4eIYC1ylmGfr0RwvXOLuBJPNkJ
+wBCOv51I0oragsT/J8Wkgn9zLZmw2DiF8+ZgqJSRPLyr0K1+rrX/Vj1WOQPU+3rh
+VWPP211A7A0qrRuePEbIcHtHP6KPUCepABL44K33zyyOydmnJ7vg3dsW7AN7+Y6O
+H4B24d1ogn4TJwzpZCfRvqJJVu2wsnzleng9PcpXyHhldB6S9h2fPpNqDUBvfxMv
+w/fGZ2ZpOeUKRfQ7VVR3XIWwFq/eDhzLicHipaoM+6gKeOZdJPAc0Ew5jvOXQSBD
+CYCM12a8gnEYd55NLo/fF3wX6Wdq/X6EbWW97gwtmmEqnhcZMxLdeMuKyli22JyX
+Ik7QIDsmPSWhCkI2JvQ+CAZZp6oMEKuSb7UqqfACQreIuxCUmPTZq/pAEUGSCZGP
+wnWqOk5jwxJ4d5TQm7g2RgPC6lTd7as1m4+JB8H1cNVpS2d0AQKCAQEA5tL9WjIK
+u6x1h4a4KPmOQ9B34GxmGM+P9/bQkkJsWYf5eG1MlYPAigAiN0PMYPHDweeROsOO
+rvmJiWXtmPMVFFSalHVotN6LMj400HhHObg7dADDYTBBGX6QuljxBX9jgUiKSOKO
+66ngXEyRdUmTAbral3UuSCFGcqG8Khd3taULO2q5JQLEioFT+Lr0UzHywVSJq06O
+k37aC3zpiQj4S/bJG4GOadYDIENq+gRCIU7Hn4pS1qtxLYMyhtNDzK0pVriYNj9T
+ydHUxSKZO6ogM4423wVKnKOa2Cj4rgKBDHGpJJ9R0ZhrTbAQOa8LgDy1P6aMlvH/
+t9bG/HClmuHrFwKCAQEA271gZpIeCFG/XYIbxO8Uh9wG2cHrt7idw0iFTW4PpOBk
+CGExq7WlkR29BkzxTDEXz5bSgDa8Q83cDf9EjSHVJYVGSYnodz7ZV8jZd2PUdCV1
+dL1kHV7vqz/KKxuCp7icCwzG9rQ1CjsTv8gBM3cN6DrZgw/2F+HjQpCkeyxc6KBR
+Q+167yaLvOv3W0BHdSywtiNDU48oSSBpEK2anh7ShjG3BaEr/gAqGsTvxjsl7zDg
+6MZFegeYPFIEH4ftvLZugPgd3NBg00CfsNRkjVWsH51y4gBy2ZL8d31Q2D2FI94s
+he57Trvs8t8Y9QHGTMCuUk9IwRBpgapmW+c6G65jQQKCAQB0IPquAjc8LIwE31aP
+5t4YaC2nj2HeYa8BBf/xVdHFSY/Ncs/w+RRJWb/dJhMRQxFF4QdEFVcX2RvFMK7V
+IJceX2JWBqvjM4O8h2dy6tCKzZG7zxZ9MxXNicuutUith2W8iY5pFPLqxdDHHw6f
+f6CiYivPv3jFeVhEG/LbCmuDy8FW5162rCnNtTtWDFkX8i077xhEQ4Wf11ZEKVgl
+RYoGTeboG8pWnQF9ne2YU8Qhlc0BC0qaDi8mwrcM9gVKWGRP6RdLU5kIFLWDaODH
+D9Sbm5UnpzXekME6t4JFqaTbaeO7NRyo4pI5x7aiDtsubVyS5WweFSqeh0QdhV8M
+CVWJAoIBAQCJ7OSFfVG8hxNG7lPf2PFaFZF3PXFncEoWOX7yixLmurIPUHKNBQdX
+fwMW4VTULjxz4IdgUvr41E47uu6cZ5ASbQUhQ57WumxR3ZAikFbaUGjZTcd5aA2n
+v/J1/F6WSBoqFWsNjy97rHlI95FJbIEeM1I0IeCmPPMY5RFY/w+SNI7NxFJVqiwr
++TDZ5g70TYjdymSIHmN7AwzvmwhiF5atBKeMsQ2b8R58jwCxvI6jBFsnwMv7PKkh
+s5lC8V8YBKp36UVVRLaB4x5ZL/etfwj7Dyj9EqsERm6R0ebc1ECtstbfekGLugmQ
+qNhRcTu3EXpZz8oq5NJUwVBef1TJ6zwBAoIBAQC7Oq5AdmLzYOX0AYUuT3Cjgl0u
+3Tq1c1uqlVaxQGjA3oqua4SR0+kvmRZbZGLpLAVnnwOjUEfUFBBYgP/5Mo/OiCkQ
+C8eWkSQKXy6OFy5wh4mbL5oJttKjM4ZoB0gpF31+tGOmrfJwacqEPnyZLKzkgBdG
+djVVKP+HH4XUdB2VXst8tvcif+VTrUsD1nuhGMOgbiHZkdx3eug05wfhnYvWljA+
+r/4xcq7TmKSoJqkb0OcOkhqjeZWleA6xRtEvHPGRzbEM67FVsVTxr/N9BX5tS9zu
+YLCNI3tTNsDV0Ac+4rl46rghQ/n2TNSEpwvA/pjytsdPXLOpoapWirrsEiXf
+-----END RSA PRIVATE KEY-----

data/spec/inputs/elasticsearch_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/devutils/rspec/shared_examples"
 require "logstash/inputs/elasticsearch"
 require "elasticsearch"
 require "timecop"
@@ -7,9 +8,13 @@ require "stud/temporary"
 require "time"
 require "date"
 
-describe LogStash::Inputs::Elasticsearch do
+class LogStash::Inputs::TestableElasticsearch < LogStash::Inputs::Elasticsearch
+  attr_reader :client
+end
+
+describe LogStash::Inputs::TestableElasticsearch do
 
-  let(:plugin) { LogStash::Inputs::Elasticsearch.new(config) }
+  let(:plugin) { LogStash::Inputs::TestableElasticsearch.new(config) }
   let(:queue) { Queue.new }
 
   it_behaves_like "an interruptible input plugin" do
@@ -31,6 +36,7 @@ describe LogStash::Inputs::Elasticsearch do
       }
       allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
       allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(esclient).to receive(:clear_scroll).and_return(nil)
     end
   end
 
@@ -75,13 +81,14 @@ describe LogStash::Inputs::Elasticsearch do
     expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
     expect(client).to receive(:search).with(any_args).and_return(response)
     expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(scroll_reponse)
+    expect(client).to receive(:clear_scroll).and_return(nil)
 
     event = input(config) do |pipeline, queue|
       queue.pop
     end
 
-    insist { event }.is_a?(LogStash::Event)
-    insist { event.get("message") } == [ "ohayo" ]
+    expect(event).to be_a(LogStash::Event)
+    expect(event.get("message")).to eql [ "ohayo" ]
   end
 
 
@@ -256,6 +263,8 @@ describe LogStash::Inputs::Elasticsearch do
         expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
         plugin.register
 
+        expect(client).to receive(:clear_scroll).and_return(nil)
+
         # SLICE0 is a three-page scroll in which the last page is empty
         slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
         expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
@@ -359,6 +368,7 @@ describe LogStash::Inputs::Elasticsearch do
       expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
       expect(client).to receive(:search).with(any_args).and_return(response)
       allow(client).to receive(:scroll).with({ :body => {:scroll_id => "cXVlcnlUaGVuRmV0Y2g"}, :scroll => "1m" }).and_return(scroll_reponse)
+      allow(client).to receive(:clear_scroll).and_return(nil)
     end
 
     context 'when defining docinfo' do
@@ -404,6 +414,7 @@ describe LogStash::Inputs::Elasticsearch do
             "docinfo_target" => 'metadata_with_string'
         } }
         it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          expect(client).not_to receive(:clear_scroll)
           plugin.register
           expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
         end
@@ -504,6 +515,103 @@ describe LogStash::Inputs::Elasticsearch do
     end
   end
 
+  describe "client" do
+    let(:config) do
+      {
+
+      }
+    end
+    let(:plugin) { described_class.new(config) }
+    let(:event)  { LogStash::Event.new({}) }
+
+    describe "cloud.id" do
+      let(:valid_cloud_id) do
+        'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
+      end
+
+      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+
+      it "should set host(s)" do
+        plugin.register
+        client = plugin.send(:client)
+        expect( client.transport.hosts ).to eql [{
+                                                     :scheme => "https",
+                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                     :port => 9243,
+                                                     :path => "",
+                                                     :protocol => "https"
+                                                 }]
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
+        end
+      end
+
+      context 'hosts also set' do
+        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "cloud.auth" do
+      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+
+      it "should set authorization" do
+        plugin.register
+        client = plugin.send(:client)
+        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+        expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
+        end
+      end
+
+      context 'user also set' do
+        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "proxy" do
+      let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+
+      it "should set proxy" do
+        plugin.register
+        client = plugin.send(:client)
+        proxy = client.transport.options[:transport_options][:proxy]
+
+        expect( proxy ).to eql "http://localhost:1234"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+
+        it "should not set proxy" do
+          plugin.register
+          client = plugin.send(:client)
+
+          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+        end
+      end
+    end
+  end
+
   context "when scheduling" do
     let(:config) do
       {
@@ -513,39 +621,11 @@ describe LogStash::Inputs::Elasticsearch do
       }
     end
 
-    response = {
-      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-      "took" => 27,
-      "timed_out" => false,
-      "_shards" => {
-        "total" => 169,
-        "successful" => 169,
-        "failed" => 0
-      },
-      "hits" => {
-        "total" => 1,
-        "max_score" => 1.0,
-        "hits" => [ {
-          "_index" => "logstash-2014.10.12",
-          "_type" => "logs",
-          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-          "_score" => 1.0,
-          "_source" => { "message" => ["ohayo"] }
-        } ]
-      }
-    }
-
-    scroll_reponse = {
-      "_scroll_id" => "r453Wc1jh0caLJhSDg",
-      "hits" => { "hits" => [] }
-    }
-
     before do
       plugin.register
     end
 
     it "should properly schedule" do
-
       Timecop.travel(Time.new(2000))
       Timecop.scale(60)
       runner = Thread.new do
@@ -564,5 +644,4 @@ describe LogStash::Inputs::Elasticsearch do
     end
 
   end
-
 end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,16 +4,17 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch, :integration => true do
+describe LogStash::Inputs::Elasticsearch do
 
   let(:config)   { { 'hosts' => [ESHelper.get_host_port],
                      'index' => 'logs',
                      'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
+  let(:client_options) {{}}
 
   before(:each) do
-    @es = ESHelper.get_client
+    @es = ESHelper.get_client(client_options)
     # Delete all templates first.
     # Clean ES of data before we start.
     @es.indices.delete_template(:name => "*")
@@ -31,8 +32,8 @@ describe LogStash::Inputs::Elasticsearch, :integration => true do
     @es.indices.delete(:index => "*") rescue nil
   end
 
-  describe 'smoke test' do
-    it "should retrieve json event from elasticseach" do
+  shared_examples 'an elasticsearch index plugin' do
+    it 'should retrieve json event from elasticsearch' do
       queue = []
       plugin.run(queue)
       event = queue.pop
@@ -40,4 +41,23 @@ describe LogStash::Inputs::Elasticsearch, :integration => true do
       expect(event.get("response")).to eql(404)
     end
   end
+
+  describe 'against an unsecured elasticsearch', :integration => true do
+    it_behaves_like 'an elasticsearch index plugin'
+  end
+
+  describe 'against a secured elasticsearch', :secure_integration => true do
+    let(:user) { 'simpleuser' }
+    let(:password) { 'abc123' }
+    let(:ca_file) { "spec/fixtures/test_certs/test.crt" }
+    let(:client_options) {{:ca_file => ca_file, :user => user, :password => password}}
+    let(:config)   { super.merge({
+                       'user' => user,
+                       'password' => password,
+                       'ssl' => true,
+                       'ca_file' => ca_file })
+    }
+    it_behaves_like 'an elasticsearch index plugin'
+  end
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.3.3
+  version: 4.6.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2020-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +128,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.4
+  name: faraday
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -174,6 +202,10 @@ files:
 - lib/logstash/inputs/elasticsearch.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
+- spec/fixtures/test_certs/ca/ca.crt
+- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.key
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -204,5 +236,9 @@ specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
+- spec/fixtures/test_certs/ca/ca.crt
+- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.key
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb