logstash-filter-elasticsearch 3.7.1 → 3.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3546d5b2082aa7d3f83c0a855c6fcebe9203c83bb6d5a7831da2237c9b5b2672
-  data.tar.gz: fa20e8e10194ebbe86b3ceea46f582d4a0c50c66122bd75968a4df149d848648
+  metadata.gz: 3264c43657eea13f942de4c005f0996dfc30e845cf7643fe374ec6918d400d83
+  data.tar.gz: bfdb2268299b87ed893d778411c450c1758977cd2e2e3c051e0679aa9467fd37
 SHA512:
-  metadata.gz: a4278733f509ed60089bcaae0e2d509acb66fd8b0ba3bb361d889b167991625e790a3d2422870a114ad3527f418ed2389b95fc7abee7334e8c2e7e07eb1dedc2
-  data.tar.gz: fffc8107d0ce39d3cc3d12669a0c13340f70fa5d1ec6807ea953b0f2d1f2049594372c6360ec87e7a8c2eb4b0682e85d0b906c6028e458ba7cf350ca45d5653d
+  metadata.gz: 759cb7f87175ac2894adc715e108399fd6d27af1cae91fdf91bd05aa5312678e9963fde3c36a25519b6eac3b53a326ed203583a1f22a94fdd66239014ddf2433
+  data.tar.gz: 5dda79137366f9d40d1fddf21b6eb92da6377187eb470d3e725b1cfb338bd1e2f2296263e67fec7f10bf7b9a4d611bcca243d393235727ffef200264da5ef4c9

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 3.9.4
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+    header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
+    [#148](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/148)
+  - Fix: default setting for `hosts` not working (since 3.7.0) GH-147
+  - Fix: mutating @hosts variable which leads to issues with multiple worker threads GH-129
+
+## 3.9.3
+  - [DOC] Update links to use shared attributes [#144](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/144)
+
+## 3.9.2
+  - [DOC] Fixed links to restructured Logstash-to-cloud docs [#142](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/142)
+
+## 3.9.1
+  - [DOC] Document the permissions required in secured clusters [#140](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/140)
+  
+## 3.9.0
+  - Add support to define a proxy with the proxy config option [#134](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/134)
+
+## 3.8.0
+  - Added api_key support [#132](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/132)
+
+## 3.7.2
+  - [DOC] Removed outdated compatibility notice [#131](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/131)
+
 ## 3.7.1
   - Fix: solves an issue where non-ascii unicode values in a template were not handled correctly [#128](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/128)
 

data/CONTRIBUTORS

@@ -9,6 +9,9 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
 * Adrian Solom (addrians)
+* Colin Surprenant (colinsurprenant)
+* Andres Rodriguez (roaksoax)
+* Luca Belluccini (lucabelluccini)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-elasticsearch)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -20,21 +20,12 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-.Compatibility Note
-[NOTE]
-================================================================================
-Starting with Elasticsearch 5.3, there's an {ref}/modules-http.html[HTTP setting]
-called `http.content_type.required`. If this option is set to `true`, and you
-are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
-plugin to version 3.1.1 or higher.
-
-================================================================================
-
 Search Elasticsearch for a previous log event and copy some fields from it
-into the current event.  Below are two complete examples of how this filter might
+into the current event. Below are two complete examples of how this filter might
 be used.
 
-The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
+The first example uses the legacy 'query' parameter where the user is limited to
+an Elasticsearch query_string.
 Whenever logstash receives an "end" event, it uses this elasticsearch
 filter to find the matching "start" event based on some operation identifier.
 Then it copies the `@timestamp` field from the "start" event into a new field on
@@ -111,6 +102,20 @@ Notice also that when you use `query_template`, the Logstash attributes `result_
 and `sort` will be ignored. They should be specified directly in the JSON
 template, as shown in the example above.
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
+
+[id="plugins-{type}s-{plugin}-autz"]
+==== Authorization
+
+Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
+The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
@@ -121,6 +126,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -130,6 +136,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-query_template>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-result_size>> |<<number,number>>|No
@@ -162,6 +169,18 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires
+enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the
+Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -170,6 +189,28 @@ Example:
 
 SSL Certificate Authority file
 
+[id="plugins-{type}s-{plugin}-cloud_auth"]
+===== `cloud_auth`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
+
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
+
+[id="plugins-{type}s-{plugin}-cloud_id"]
+===== `cloud_id`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+
+For more info, check out the
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
+
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 
@@ -243,14 +284,26 @@ Field substitution (e.g. `index-name-%{date_field}`) is available
 
 Basic Auth - password
 
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy.
+An empty string is treated as if proxy was not set, and is useful when using
+environment variables e.g. `proxy => '${LS_PROXY:}'`.
+
 [id="plugins-{type}s-{plugin}-query"]
 ===== `query` 
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-Elasticsearch query string. Read the Elasticsearch query string documentation.
-for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+Elasticsearch query string. More information is available in the
+{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query
+string documentation].
+
 
 [id="plugins-{type}s-{plugin}-query_template"]
 ===== `query_template` 
@@ -258,8 +311,8 @@ for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
-for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+File path to elasticsearch query in DSL format. More information is available in
+the {ref}/query-dsl.html[Elasticsearch query documentation].
 
 [id="plugins-{type}s-{plugin}-result_size"]
 ===== `result_size` 
@@ -301,27 +354,6 @@ Tags the event on failure to look up previous log event information. This can be
 
 Basic Auth - username
 
-[id="plugins-{type}s-{plugin}-cloud_auth"]
-===== `cloud_auth`
-
-  * Value type is <<password,password>>
-  * There is no default value for this setting.
-
-Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
-
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
-
-[id="plugins-{type}s-{plugin}-cloud_id"]
-===== `cloud_id`
-
-  * Value type is <<string,string>>
-  * There is no default value for this setting.
-
-Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-
-
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch.rb

@@ -1,24 +1,15 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require_relative "elasticsearch/client"
 require "logstash/json"
-require "logstash/util/safe_uri"
-java_import "java.util.concurrent.ConcurrentHashMap"
-
+require_relative "elasticsearch/client"
+require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
 
-  DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//localhost:9200")
-
   # List of elasticsearch hosts to use for querying.
-  config :hosts, :validate => :array, :default => [ DEFAULT_HOST ]
-
-  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-  #
-  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-  config :cloud_id, :validate => :string
+  config :hosts, :validate => :array, :default => [ 'localhost:9200' ]
 
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
@@ -50,11 +41,23 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
   config :cloud_auth, :validate => :password
 
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
+  # Set the address of a forward HTTP proxy.
+  config :proxy, :validate => :uri_or_empty
+
   # SSL
   config :ssl, :validate => :boolean, :default => false
 
@@ -72,6 +75,23 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   attr_reader :clients_pool
 
+  ##
+  # @override to handle proxy => '' as if none was set
+  # @param value [Array<Object>]
+  # @param validator [nil,Array,Symbol]
+  # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+  # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+  def self.validate_value(value, validator)
+    return super unless validator == :uri_or_empty
+
+    value = deep_replace(value)
+    value = hash_or_array(value)
+
+    return true, value.first if value.size == 1 && value.first.empty?
+
+    return super(value, :uri)
+  end
+
   def register
     @clients_pool = java.util.concurrent.ConcurrentHashMap.new
 
@@ -84,10 +104,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
-    fill_hosts_from_cloud_id
+    validate_authentication
     fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
 
-    @hosts = Array(@hosts).map { |host| host.to_s } # for ES client URI#to_s
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
   end # def register
@@ -95,7 +116,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def filter(event)
     matched = false
     begin
-      params = {:index => event.sprintf(@index) }
+      params = { :index => event.sprintf(@index) }
 
       if @query_dsl
         query = LogStash::Json.load(event.sprintf(@query_dsl))
@@ -156,19 +177,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def filter
 
   private
+
   def client_options
     {
+      :user => @user,
+      :password => @password,
+      :api_key => @api_key,
+      :proxy => @proxy,
       :ssl => @ssl,
-      :hosts => @hosts,
       :ca_file => @ca_file,
-      :logger => @logger
     }
   end
 
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
   end
 
   def get_client
@@ -209,30 +233,42 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def hosts_default?(hosts)
-    # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
-    hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(DEFAULT_HOST)
+    hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
-  def fill_hosts_from_cloud_id
-    return unless @cloud_id
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
 
-    if @hosts && !hosts_default?(@hosts)
-      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
+    end
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
     end
-    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
   end
 
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
-    if @user || @password
-      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
-    end
     @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
     params['user'], params['password'] = @user, @password
   end
 
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
   def parse_host_uri_from_cloud_id(cloud_id)
+    require 'logstash/util/safe_uri'
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'
     rescue LoadError

data/lib/logstash/filters/elasticsearch/client.rb

@@ -10,23 +10,26 @@ module LogStash
 
       attr_reader :client
 
-      def initialize(user, password, options={})
-        ssl     = options.fetch(:ssl, false)
-        hosts   = options[:hosts]
-        @logger = options[:logger]
-
-        transport_options = {}
-        if user && password
-          token = ::Base64.strict_encode64("#{user}:#{password.value}")
-          transport_options[:headers] = { Authorization: "Basic #{token}" }
-        end
-
-        hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
+      def initialize(logger, hosts, options = {})
+        ssl = options.fetch(:ssl, false)
+        user = options.fetch(:user, nil)
+        password = options.fetch(:password, nil)
+        api_key = options.fetch(:api_key, nil)
+        proxy = options.fetch(:proxy, nil)
+
+        transport_options = {:headers => {}}
+        transport_options[:headers].merge!(setup_basic_auth(user, password))
+        transport_options[:headers].merge!(setup_api_key(api_key))
+
+        logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
+        transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
+
+        hosts = hosts.map { |host| { host: host, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
         ssl_options ||= {}
 
-        @logger.info("New ElasticSearch filter client", :hosts => hosts)
+        logger.info("New ElasticSearch filter client", :hosts => hosts)
         @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
       end
 
@@ -34,6 +37,21 @@ module LogStash
         @client.search(params)
       end
 
+      private
+
+      def setup_basic_auth(user, password)
+        return {} unless user && password && password.value
+
+        token = ::Base64.strict_encode64("#{user}:#{password.value}")
+        { 'Authorization' => "Basic #{token}" }
+      end
+
+      def setup_api_key(api_key)
+        return {} unless (api_key && api_key.value)
+
+        token = ::Base64.strict_encode64(api_key.value)
+        { 'Authorization' => "ApiKey #{token}" }
+      end
     end
   end
 end

data/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+es_client_version = Gem.loaded_specs['elasticsearch-transport'].version
+if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Version.new('7.16')
+  # elasticsearch-transport 7.2.0 - 7.14.0 had a bug where setting http headers
+  #   ES::Client.new ..., transport_options: { headers: { 'Authorization' => ... } }
+  # would be lost https://github.com/elastic/elasticsearch-ruby/issues/1428
+  #
+  # NOTE: needs to be idempotent as input ES plugin might apply the same patch!
+  #
+  # @private
+  module Elasticsearch
+    module Transport
+      module Transport
+        module HTTP
+          class Manticore
+
+            def apply_headers(request_options, options)
+              headers = (options && options[:headers]) || {}
+              headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+              headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+              headers[ACCEPT_ENCODING] = GZIP if use_compression?
+              (request_options[:headers] ||= {}).merge!(headers) # this line was changed
+            end
+
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.7.1'
+  s.version         = '3.9.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 5.0.3"
+  s.add_runtime_dependency 'elasticsearch', ">= 5.0.5" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', "~> 0.6"
 
   s.add_development_dependency 'logstash-devutils'

data/spec/es_helper.rb

@@ -7,8 +7,12 @@ module ESHelper
     end
   end
 
-  def self.get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+  def self.get_client(credentials)
+    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
+    host, port = get_host_port.split(':')
+    host_opts = credentials.inject({}) { |h, (k, v)| h[k.to_sym] = v; h } # user: _, password: _
+    host_opts.merge! host: host, port: port, scheme: 'http'
+    Elasticsearch::Client.new(hosts: [host_opts], transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
   end
 
   def self.doc_type

data/spec/filters/elasticsearch_spec.rb

@@ -313,12 +313,12 @@ describe LogStash::Filters::Elasticsearch do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
       end
 
-      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+      let(:config) { super().merge({ 'cloud_id' => valid_cloud_id }) }
 
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:get_client).client
-        expect( client.transport.hosts ).to eql [{
+        expect( extract_transport(client).hosts ).to eql [{
                                                      :scheme => "https",
                                                      :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                      :port => 9243,
@@ -328,7 +328,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+        let(:config) { super().merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
@@ -336,7 +336,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'hosts also set' do
-        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+        let(:config) { super().merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
@@ -345,18 +345,18 @@ describe LogStash::Filters::Elasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "cloud.auth" do
-      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+      let(:config) { super().merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
 
       it "should set authorization" do
         plugin.register
         client = plugin.send(:get_client).client
-        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'invalid-format' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
@@ -364,13 +364,82 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'user also set' do
-        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
-          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
         end
       end
     end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:get_client).client
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super().merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "proxy" do
+      context 'valid' do
+        let(:config) { super().merge({ 'proxy' => 'http://localhost:1234' }) }
+
+        it "should set proxy" do
+          plugin.register
+          client = plugin.send(:get_client).client
+          proxy = extract_transport(client).options[:transport_options][:proxy]
+
+          expect( proxy ).to eql "http://localhost:1234"
+        end
+      end
+
+      context 'invalid' do
+        let(:config) { super().merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+
+        it "should not set proxy" do
+          plugin.register
+          client = plugin.send(:get_client).client
+
+          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
+        end
+      end
+    end
+  end
+
+  describe "defaults" do
+
+    let(:config) { Hash.new }
+    let(:plugin) { described_class.new(config) }
+
+    before { allow(plugin).to receive(:test_connection!) }
+
+    it "should set localhost:9200 as hosts" do
+      plugin.register
+      client = plugin.send(:get_client).client
+      expect( extract_transport(client).hosts ).to eql [{ :host => "localhost", :port => 9200, :protocol => "http"}]
+    end
   end
 
   describe "query template" do
@@ -398,4 +467,10 @@ describe LogStash::Filters::Elasticsearch do
       plugin.filter(LogStash::Event.new)
     end
   end
+
+  # @note can be removed once gem depends on elasticsearch >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
+  end
+
 end

data/spec/filters/integration/elasticsearch_spec.rb

@@ -6,21 +6,31 @@ require_relative "../../../spec/es_helper"
 
 describe LogStash::Filters::Elasticsearch, :integration => true do
 
+  ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
 
-  let(:config) do
+  let(:base_config) do
     {
-      "index" => 'logs',
-      "hosts" => [ESHelper.get_host_port],
-      "query" => "response: 404",
-      "sort" => "response",
-      "fields" => [ ["response", "code"] ],
+        "index" => 'logs',
+        "hosts" => [ESHelper.get_host_port],
+        "query" => "response: 404",
+        "sort" => "response",
+        "fields" => [ ["response", "code"] ],
     }
   end
+
+  let(:credentials) do
+    { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
+  end
+
+  let(:config) do
+    ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
+  end
+
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
 
   before(:each) do
-    @es = ESHelper.get_client
+    @es = ESHelper.get_client(ELASTIC_SECURITY_ENABLED ? credentials : {})
     # Delete all templates first.
     # Clean ES of data before we start.
     @es.indices.delete_template(:name => "*")
@@ -30,11 +40,10 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :this => 'that'})
     end
     @es.indices.refresh
-
-    plugin.register
   end
 
   it "should enhance the current event with new data" do
+    plugin.register
     plugin.filter(event)
     expect(event.get('code')).to eq(404)
   end
@@ -42,20 +51,28 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
   context "when retrieving a list of elements" do
 
     let(:config) do
-      {
-        "index" => 'logs',
-        "hosts" => [ESHelper.get_host_port],
-        "query" => "response: 404",
-        "fields" => [ ["response", "code"] ],
-        "sort" => "response",
-        "result_size" => 10
-      }
+      super().merge("fields" => [ ["response", "code"] ], "result_size" => 10)
     end
 
+    before { plugin.register }
+
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq([404]*10)
     end
 
   end
+
+  context "incorrect auth credentials" do
+
+    let(:config) do
+      super().reject { |key, _| key == 'password' }
+    end
+
+    it "should enhance the current event with new data" do
+      expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+    end
+
+  end if ELASTIC_SECURITY_ENABLED
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.9.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -35,15 +35,15 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.3
+        version: 5.0.5
   name: elasticsearch
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.3
+        version: 5.0.5
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: manticore
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -89,6 +89,7 @@ files:
 - docs/index.asciidoc
 - lib/logstash/filters/elasticsearch.rb
 - lib/logstash/filters/elasticsearch/client.rb
+- lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-filter-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/filters/elasticsearch_spec.rb
@@ -121,8 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events