logstash-filter-elasticsearch 3.6.1 → 3.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54415da2d9bc4aa6d1d90ea0b65937aa4782c9a9614cafd2ac583d94c0780913
-  data.tar.gz: 44eea0bc6efd2a0a2a5415320a872e03cd3883e122e8d7627c3873241d47720c
+  metadata.gz: 886cfa121f2af1a9efdcfff9f3f2e97be3456be2aecbb668b89d3517d8087ffc
+  data.tar.gz: 0f0a568d1bf012cd0845a6d9cb5782a6460a32332a493790b84f010e68c06fcd
 SHA512:
-  metadata.gz: f148fcacc216cbbfca630e858ec93d85329487738dda440200ac2ea4f85253f2f4b6dc4a55cb859781be372a7ef15c727b5ded25015de1988302cd04e64de98e
-  data.tar.gz: '003910ce329cf1945fb3f92c2aef9f096588eb33c85e8716dbba0b65dd770e168e3960b1751553c2d251bc43b2a36dbc3eef339720acc9d40a08e1d1d36db402'
+  metadata.gz: 3e15a05bd9d013a2145ce983b2699f94c619276d93a675d49695ef8b8cf1d48ef88e6dfb4edac975d11a85bc57c0e769566b35ec44b8b2776fca8e35300af65d
+  data.tar.gz: 667279291114a2a93cbbd211e6ec7ff7533eada83dbfc73377d0a67b759ae773e7787f9b1a11518e9101302653cf5b8b356936a211521f9f635cc58646826730

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 3.9.3
+  - [DOC] Update links to use shared attributes [#144](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/144)
+
+## 3.9.2
+  - [DOC] Fixed links to restructured Logstash-to-cloud docs [#142](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/142)
+
+## 3.9.1
+  - [DOC] Document the permissions required in secured clusters [#140](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/140)
+  
+## 3.9.0
+  - Add support to define a proxy with the proxy config option [#134](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/134)
+
+## 3.8.0
+  - Added api_key support [#132](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/132)
+
+## 3.7.2
+  - [DOC] Removed outdated compatibility notice [#131](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/131)
+
+## 3.7.1
+  - Fix: solves an issue where non-ascii unicode values in a template were not handled correctly [#128](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/128)
+
+## 3.7.0
+  - Feat: support cloud_id / cloud_auth configuration [#122](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/122)
+
 ## 3.6.1
   - Loosen restrictions on Elasticsearch gem ([#120](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/120))
 

data/CONTRIBUTORS

@@ -9,6 +9,9 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
 * Adrian Solom (addrians)
+* Colin Surprenant (colinsurprenant)
+* Andres Rodriguez (roaksoax)
+* Luca Belluccini (lucabelluccini)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-elasticsearch)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -20,21 +20,12 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-.Compatibility Note
-[NOTE]
-================================================================================
-Starting with Elasticsearch 5.3, there's an {ref}/modules-http.html[HTTP setting]
-called `http.content_type.required`. If this option is set to `true`, and you
-are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
-plugin to version 3.1.1 or higher.
-
-================================================================================
-
 Search Elasticsearch for a previous log event and copy some fields from it
-into the current event.  Below are two complete examples of how this filter might
+into the current event. Below are two complete examples of how this filter might
 be used.
 
-The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
+The first example uses the legacy 'query' parameter where the user is limited to
+an Elasticsearch query_string.
 Whenever logstash receives an "end" event, it uses this elasticsearch
 filter to find the matching "start" event based on some operation identifier.
 Then it copies the `@timestamp` field from the "start" event into a new field on
@@ -111,6 +102,20 @@ Notice also that when you use `query_template`, the Logstash attributes `result_
 and `sort` will be ignored. They should be specified directly in the JSON
 template, as shown in the example above.
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
+
+[id="plugins-{type}s-{plugin}-autz"]
+==== Authorization
+
+Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
+The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
@@ -121,13 +126,17 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-query_template>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-result_size>> |<<number,number>>|No
@@ -160,6 +169,18 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires
+enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the
+Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -168,6 +189,28 @@ Example:
 
 SSL Certificate Authority file
 
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
+
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 
@@ -241,14 +284,26 @@ Field substitution (e.g. `index-name-%{date_field}`) is available
 
 Basic Auth - password
 
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy.
+An empty string is treated as if proxy was not set, and is useful when using
+environment variables e.g. `proxy => '${LS_PROXY:}'`.
+
 [id="plugins-{type}s-{plugin}-query"]
 ===== `query` 
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-Elasticsearch query string. Read the Elasticsearch query string documentation.
-for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+Elasticsearch query string. More information is available in the
+{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query
+string documentation].
+
 
 [id="plugins-{type}s-{plugin}-query_template"]
 ===== `query_template` 
@@ -256,8 +311,8 @@ for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
-for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+File path to elasticsearch query in DSL format. More information is available in
+the {ref}/query-dsl.html[Elasticsearch query documentation].
 
 [id="plugins-{type}s-{plugin}-result_size"]
 ===== `result_size` 
@@ -300,6 +355,5 @@ Tags the event on failure to look up previous log event information. This can be
 Basic Auth - username
 
 
-
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch.rb

@@ -3,15 +3,18 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require_relative "elasticsearch/client"
 require "logstash/json"
+require "logstash/util/safe_uri"
 java_import "java.util.concurrent.ConcurrentHashMap"
 
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
 
+  DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//localhost:9200")
+
   # List of elasticsearch hosts to use for querying.
-  config :hosts, :validate => :array,  :default => [ "localhost:9200" ]
-  
+  config :hosts, :validate => :array, :default => [ DEFAULT_HOST ]
+
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
@@ -42,6 +45,23 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
+  # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+  config :cloud_auth, :validate => :password
+
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
+  # Set the address of a forward HTTP proxy.
+  config :proxy, :validate => :uri_or_empty
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -59,6 +79,23 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   attr_reader :clients_pool
 
+  ##
+  # @override to handle proxy => '' as if none was set
+  # @param value [Array<Object>]
+  # @param validator [nil,Array,Symbol]
+  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+  def self.validate_value(value, validator)
+    return super unless validator == :uri_or_empty
+
+    value = deep_replace(value)
+    value = hash_or_array(value)
+
+    return true, value.first if value.size == 1 && value.first.empty?
+
+    return super(value, :uri)
+  end
+
   def register
     @clients_pool = java.util.concurrent.ConcurrentHashMap.new
 
@@ -67,10 +104,16 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       if File.zero?(@query_template)
         raise "template is empty"
       end
-      file = File.open(@query_template, "rb")
+      file = File.open(@query_template, 'r')
       @query_dsl = file.read
     end
 
+    validate_authentication
+    fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
+
+    @hosts = Array(@hosts).map { |host| host.to_s } # for ES client URI#to_s
+
     test_connection!
   end # def register
 
@@ -138,17 +181,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def filter
 
   private
+
   def client_options
     {
+      :user => @user,
+      :password => @password,
+      :api_key => @api_key,
+      :proxy => @proxy,
       :ssl => @ssl,
-      :hosts => @hosts,
       :ca_file => @ca_file,
-      :logger => @logger
     }
   end
 
   def new_client
-    LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
+    # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
+    # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
   end
 
   def get_client
@@ -188,6 +236,76 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     total
   end
 
+  def hosts_default?(hosts)
+    # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
+    hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(DEFAULT_HOST)
+  end
+
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
+
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
+    end
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    end
+  end
+
+  def fill_user_password_from_cloud_auth
+    return unless @cloud_auth
+
+    @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
+    params['user'], params['password'] = @user, @password
+  end
+
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
+  def parse_host_uri_from_cloud_id(cloud_id)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_id'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set hosts instead).'
+    end
+
+    begin
+      cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
+    end
+    cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
+    LogStash::Util::SafeURI.new(cloud_uri)
+  end
+
+  def parse_user_password_from_cloud_auth(cloud_auth)
+    begin # might not be available on older LS
+      require 'logstash/util/cloud_setting_auth'
+    rescue LoadError
+      raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
+          'please upgrade your installation (or set user/password instead).'
+    end
+
+    cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
+    begin
+      cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
+    rescue ArgumentError => e
+      raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
+    end
+    [ cloud_auth.username, cloud_auth.password ]
+  end
+
   def test_connection!
     get_client.client.ping
   end

data/lib/logstash/filters/elasticsearch/client.rb

@@ -10,23 +10,26 @@ module LogStash
 
       attr_reader :client
 
-      def initialize(user, password, options={})
-        ssl     = options.fetch(:ssl, false)
-        hosts   = options[:hosts]
-        @logger = options[:logger]
+      def initialize(logger, hosts, options = {})
+        ssl = options.fetch(:ssl, false)
+        user = options.fetch(:user, nil)
+        password = options.fetch(:password, nil)
+        api_key = options.fetch(:api_key, nil)
+        proxy = options.fetch(:proxy, nil)
 
-        transport_options = {}
-        if user && password
-          token = ::Base64.strict_encode64("#{user}:#{password.value}")
-          transport_options[:headers] = { Authorization: "Basic #{token}" }
-        end
+        transport_options = {:headers => {}}
+        transport_options[:headers].merge!(setup_basic_auth(user, password))
+        transport_options[:headers].merge!(setup_api_key(api_key))
+
+        logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
+        transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 
         hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
         ssl_options ||= {}
 
-        @logger.info("New ElasticSearch filter client", :hosts => hosts)
+        logger.info("New ElasticSearch filter client", :hosts => hosts)
         @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
       end
 
@@ -34,6 +37,21 @@ module LogStash
         @client.search(params)
       end
 
+      private
+
+      def setup_basic_auth(user, password)
+        return {} unless user && password && password.value
+
+        token = ::Base64.strict_encode64("#{user}:#{password.value}")
+        { Authorization: "Basic #{token}" }
+      end
+
+      def setup_api_key(api_key)
+        return {} unless (api_key && api_key.value)
+
+        token = ::Base64.strict_encode64(api_key.value)
+        { Authorization: "ApiKey #{token}" }
+      end
     end
   end
 end

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.6.1'
+  s.version         = '3.9.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -289,7 +289,168 @@ describe LogStash::Filters::Elasticsearch do
       end
 
     end
+  end
+
+  describe "client" do
+    let(:config) do
+      {
+          "query" => "response: unknown"
+      }
+    end
+    let(:plugin) { described_class.new(config) }
+    let(:event)  { LogStash::Event.new({}) }
+
+    before(:each) do
+      allow(plugin).to receive(:test_connection!)
+    end
+
+    after(:each) do
+      Thread.current[:filter_elasticsearch_client] = nil
+    end
+
+    describe "cloud.id" do
+      let(:valid_cloud_id) do
+        'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
+      end
+
+      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+
+      it "should set host(s)" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect( client.transport.hosts ).to eql [{
+                                                     :scheme => "https",
+                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                     :port => 9243,
+                                                     :path => "",
+                                                     :protocol => "https"
+                                                 }]
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
+        end
+      end
+
+      context 'hosts also set' do
+        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "cloud.auth" do
+      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+
+      it "should set authorization" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+        expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
+        end
+      end
+
+      context 'user also set' do
+        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:get_client).client
+          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "proxy" do
+      context 'valid' do
+        let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+
+        it "should set proxy" do
+          plugin.register
+          client = plugin.send(:get_client).client
+          proxy = client.transport.options[:transport_options][:proxy]
+
+          expect( proxy ).to eql "http://localhost:1234"
+        end
+      end
+
+      context 'invalid' do
+        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+
+        it "should not set proxy" do
+          plugin.register
+          client = plugin.send(:get_client).client
+
+          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+        end
+      end
+    end
   end
 
+  describe "query template" do
+    let(:config) do
+      {
+          "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template_unicode.json"),
+      }
+    end
+
+    let(:plugin) { described_class.new(config) }
+
+    let(:client) { double(:client) }
+
+    before(:each) do
+      allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      allow(plugin).to receive(:test_connection!)
+      plugin.register
+    end
+
+    it "should read and send non-ascii query" do
+      expect(client).to receive(:search).with(
+          :body => { "query" => { "terms" => { "lock" => [ "잠금", "uzávěr" ] } } },
+          :index => "")
+
+      plugin.filter(LogStash::Event.new)
+    end
+  end
 end

data/spec/filters/fixtures/query_template_unicode.json

@@ -0,0 +1,7 @@
+{
+  "query": {
+    "terms": {
+      "lock": [ "잠금",  "uzávěr" ]
+    }
+  }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.6.1
+  version: 3.9.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,7 @@ files:
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/query_template.json
+- spec/filters/fixtures/query_template_unicode.json
 - spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json
@@ -130,6 +131,7 @@ test_files:
 - spec/filters/elasticsearch_spec.rb
 - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/query_template.json
+- spec/filters/fixtures/query_template_unicode.json
 - spec/filters/fixtures/request_error.json
 - spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json