logstash-filter-elasticsearch 3.9.3 → 3.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 886cfa121f2af1a9efdcfff9f3f2e97be3456be2aecbb668b89d3517d8087ffc
-  data.tar.gz: 0f0a568d1bf012cd0845a6d9cb5782a6460a32332a493790b84f010e68c06fcd
+  metadata.gz: 2fb1bc80bd95bd0e99591f07f0d2b0c76942d9a06618457e275679caeae9c0ec
+  data.tar.gz: 763fbbf17d77630c097bbbfa834842e8316830687a40a216704ee8902eb61e2a
 SHA512:
-  metadata.gz: 3e15a05bd9d013a2145ce983b2699f94c619276d93a675d49695ef8b8cf1d48ef88e6dfb4edac975d11a85bc57c0e769566b35ec44b8b2776fca8e35300af65d
-  data.tar.gz: 667279291114a2a93cbbd211e6ec7ff7533eada83dbfc73377d0a67b759ae773e7787f9b1a11518e9101302653cf5b8b356936a211521f9f635cc58646826730
+  metadata.gz: 764940be26047fae75ffc160a5a75831599073c10a2f2712620984ea6fec3e04107efb6fa8c1b7e302e55387ab6629f88e3a82a460543f7a9a90c46ebf3c3ac2
+  data.tar.gz: 6a68318f5028c52265c42c3f939f43619427306964bb70453a1ca60444113601b662c45c6eb19fc65912c63b652707463879b2caba9a67f2aa9686736cd42d13

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 3.11.0
+  - Feat: update Elasticsearch client to 7.14.0 [#150](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/150)
+
+## 3.10.0
+  - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#152](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/152)
+
+## 3.9.5
+  - Fixed SSL handshake hang indefinitely with proxy setup [#151](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/151)
+
+## 3.9.4
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+    header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
+    [#148](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/148)
+  - Fix: default setting for `hosts` not working (since 3.7.0) GH-147
+  - Fix: mutating @hosts variable which leads to issues with multiple worker threads GH-129
+
 ## 3.9.3
   - [DOC] Update links to use shared attributes [#144](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/144)
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -16,15 +16,17 @@ module LogStash
         password = options.fetch(:password, nil)
         api_key = options.fetch(:api_key, nil)
         proxy = options.fetch(:proxy, nil)
+        user_agent = options[:user_agent]
 
         transport_options = {:headers => {}}
         transport_options[:headers].merge!(setup_basic_auth(user, password))
         transport_options[:headers].merge!(setup_api_key(api_key))
+        transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
 
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 
-        hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
+        hosts = hosts.map { |host| { host: host, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
         ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
         ssl_options ||= {}
@@ -43,14 +45,14 @@ module LogStash
         return {} unless user && password && password.value
 
         token = ::Base64.strict_encode64("#{user}:#{password.value}")
-        { Authorization: "Basic #{token}" }
+        { 'Authorization' => "Basic #{token}" }
       end
 
       def setup_api_key(api_key)
         return {} unless (api_key && api_key.value)
 
         token = ::Base64.strict_encode64(api_key.value)
-        { Authorization: "ApiKey #{token}" }
+        { 'Authorization' => "ApiKey #{token}" }
       end
     end
   end

data/lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+es_client_version = Gem.loaded_specs['elasticsearch-transport'].version
+if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Version.new('7.16')
+  # elasticsearch-transport 7.2.0 - 7.14.0 had a bug where setting http headers
+  #   ES::Client.new ..., transport_options: { headers: { 'Authorization' => ... } }
+  # would be lost https://github.com/elastic/elasticsearch-ruby/issues/1428
+  #
+  # NOTE: needs to be idempotent as input ES plugin might apply the same patch!
+  #
+  # @private
+  module Elasticsearch
+    module Transport
+      module Transport
+        module HTTP
+          class Manticore
+
+            def apply_headers(request_options, options)
+              headers = (options && options[:headers]) || {}
+              headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+
+              # this code is necessary to grab the correct user-agent header
+              # when this method is invoked with apply_headers(@request_options, options)
+              # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114
+              transport_user_agent = nil
+              if (options && options[:transport_options] && options[:transport_options][:headers])
+                transport_headers = {}
+                transport_headers = options[:transport_options][:headers]
+                transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX)
+              end
+
+              headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header
+              headers[ACCEPT_ENCODING] = GZIP if use_compression?
+              (request_options[:headers] ||= {}).merge!(headers) # this line was changed
+            end
+
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/filters/elasticsearch.rb

@@ -1,19 +1,15 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require_relative "elasticsearch/client"
 require "logstash/json"
-require "logstash/util/safe_uri"
-java_import "java.util.concurrent.ConcurrentHashMap"
-
+require_relative "elasticsearch/client"
+require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
 
-  DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//localhost:9200")
-
   # List of elasticsearch hosts to use for querying.
-  config :hosts, :validate => :array, :default => [ DEFAULT_HOST ]
+  config :hosts, :validate => :array, :default => [ 'localhost:9200' ]
 
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
   # Field substitution (e.g. `index-name-%{date_field}`) is available
@@ -112,7 +108,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     fill_user_password_from_cloud_auth
     fill_hosts_from_cloud_id
 
-    @hosts = Array(@hosts).map { |host| host.to_s } # for ES client URI#to_s
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
   end # def register
@@ -120,7 +116,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def filter(event)
     matched = false
     begin
-      params = {:index => event.sprintf(@index) }
+      params = { :index => event.sprintf(@index) }
 
       if @query_dsl
         query = LogStash::Json.load(event.sprintf(@query_dsl))
@@ -180,6 +176,19 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end # def filter
 
+  # public only to be reuse in testing
+  def prepare_user_agent
+    os_name = java.lang.System.getProperty('os.name')
+    os_version = java.lang.System.getProperty('os.version')
+    os_arch = java.lang.System.getProperty('os.arch')
+    jvm_vendor = java.lang.System.getProperty('java.vendor')
+    jvm_version = java.lang.System.getProperty('java.version')
+
+    plugin_version = Gem.loaded_specs['logstash-filter-elasticsearch'].version
+    # example: logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-output-elasticsearch/11.0.1
+    "logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-#{@plugin_type}-#{config_name}/#{plugin_version}"
+  end
+
   private
 
   def client_options
@@ -196,7 +205,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
-    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
+    opts = client_options
+
+    opts[:user_agent] = prepare_user_agent
+
+    LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, opts)
   end
 
   def get_client
@@ -237,8 +250,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def hosts_default?(hosts)
-    # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
-    hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(DEFAULT_HOST)
+    hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
   def validate_authentication
@@ -273,6 +285,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def parse_host_uri_from_cloud_id(cloud_id)
+    require 'logstash/util/safe_uri'
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'
     rescue LoadError
@@ -307,6 +320,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def test_connection!
-    get_client.client.ping
+    begin
+      get_client.client.ping
+    rescue Elasticsearch::UnsupportedProductError
+      raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+    end
   end
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.9.3'
+  s.version         = '3.11.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,10 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 5.0.3"
-  s.add_runtime_dependency 'manticore', "~> 0.6"
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'manticore', ">= 0.7.1"
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/es_helper.rb

@@ -7,8 +7,12 @@ module ESHelper
     end
   end
 
-  def self.get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+  def self.get_client(credentials)
+    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
+    host, port = get_host_port.split(':')
+    host_opts = credentials.inject({}) { |h, (k, v)| h[k.to_sym] = v; h } # user: _, password: _
+    host_opts.merge! host: host, port: port, scheme: 'http'
+    Elasticsearch::Client.new(hosts: [host_opts], transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
   end
 
   def self.doc_type

data/spec/filters/elasticsearch_spec.rb

@@ -3,18 +3,43 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/plugin"
 require "logstash/filters/elasticsearch"
 require "logstash/json"
+require "cabin"
+require "webrick"
+require "uri"
 
 describe LogStash::Filters::Elasticsearch do
 
   context "registration" do
 
     let(:plugin) { LogStash::Plugin.lookup("filter", "elasticsearch").new({}) }
-    before do
-      allow(plugin).to receive(:test_connection!)
+
+    context "against authentic Elasticsearch" do
+      before do
+        allow(plugin).to receive(:test_connection!)
+      end
+      
+      it "should not raise an exception" do
+        expect {plugin.register}.to_not raise_error
+      end
     end
 
-    it "should not raise an exception" do
-      expect {plugin.register}.to_not raise_error
+    context "against not authentic Elasticsearch" do
+      let(:failing_client) do
+        client = double("client")
+        allow(client).to receive(:ping).and_raise Elasticsearch::UnsupportedProductError
+
+        client_wrapper = double("filter_client")
+        allow(client_wrapper).to receive(:client).and_return client
+        client_wrapper
+      end
+
+      before do
+        allow(plugin).to receive(:get_client).and_return(failing_client)
+      end
+
+      it "should raise ConfigurationError" do
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
     end
   end
 
@@ -291,6 +316,112 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  class StoppableServer
+
+    attr_reader :port
+
+    def initialize()
+      queue = Queue.new
+      @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+      @first_request = nil
+
+      @t = java.lang.Thread.new(
+        proc do
+          begin
+            @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                     :Logger => Cabin::Channel.get, # silence WEBrick logging
+                     :StartCallback => Proc.new {
+                           queue.push("started")
+                         }
+            @port = @server.config[:Port]
+            @server.mount_proc '/' do |req, res|
+              res.body = '''
+              {
+                  "name": "ce7ccfb438e8",
+                  "cluster_name": "docker-cluster",
+                  "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
+                  "version": {
+                      "number": "7.13.1",
+                      "build_flavor": "default",
+                      "build_type": "docker",
+                      "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
+                      "build_date": "2021-05-28T17:40:59.346932922Z",
+                      "build_snapshot": false,
+                      "lucene_version": "8.8.2",
+                      "minimum_wire_compatibility_version": "6.8.0",
+                      "minimum_index_compatibility_version": "6.0.0-beta1"
+                  },
+                  "tagline": "You Know, for Search"
+              }
+              '''
+              res.status = 200
+              res['Content-Type'] = 'application/json'
+              @first_request = req
+              @first_req_waiter.countDown()
+            end
+
+            @server.start
+          rescue => e
+            puts "Error in webserver thread #{e}"
+            # ignore
+          end
+        end
+      )
+      @t.daemon = true
+      @t.start
+      queue.pop # blocks until the server is up
+    end
+
+    def stop
+      @server.shutdown
+    end
+
+    def wait_receive_request
+      @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+      @first_request
+    end
+  end
+
+  describe "user-agent header" do
+    let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+    after :each do
+      webserver.stop
+    end
+
+    it "server should be started" do
+      require 'net/http'
+      response = nil
+      Net::HTTP.start('localhost', webserver.port) {|http|
+        response = http.request_get('/')
+      }
+      expect(response.code.to_i).to eq(200)
+    end
+
+    context "used by plugin" do
+      let(:config) do
+        {
+          "hosts" => ["localhost:#{webserver.port}"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" },
+          "docinfo_fields" => { "_index" => "es_index" },
+          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+        }
+      end
+      let(:plugin) { described_class.new(config) }
+      let(:event)  { LogStash::Event.new({}) }
+
+      it "client should sent the expect user-agent" do
+        plugin.register
+
+        request = webserver.wait_receive_request
+
+        expect(request.header['user-agent'].size).to eq(1)
+        expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-filter-elasticsearch\/\d*\.\d*\.\d*/)
+      end
+    end
+  end
+
   describe "client" do
     let(:config) do
       {
@@ -313,12 +444,12 @@ describe LogStash::Filters::Elasticsearch do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
       end
 
-      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+      let(:config) { super().merge({ 'cloud_id' => valid_cloud_id }) }
 
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:get_client).client
-        expect( client.transport.hosts ).to eql [{
+        expect( extract_transport(client).hosts ).to eql [{
                                                      :scheme => "https",
                                                      :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                      :port => 9243,
@@ -328,7 +459,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+        let(:config) { super().merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
@@ -336,7 +467,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'hosts also set' do
-        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+        let(:config) { super().merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
@@ -345,18 +476,18 @@ describe LogStash::Filters::Elasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "cloud.auth" do
-      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+      let(:config) { super().merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
 
       it "should set authorization" do
         plugin.register
         client = plugin.send(:get_client).client
-        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'invalid-format' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
@@ -364,7 +495,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context 'user also set' do
-        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -374,7 +505,7 @@ describe LogStash::Filters::Elasticsearch do
 
     describe "api_key" do
       context "without ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
@@ -382,18 +513,18 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context "with ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
 
         it "should set authorization" do
           plugin.register
           client = plugin.send(:get_client).client
-          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
           expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
         end
 
         context 'user also set' do
-          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+          let(:config) { super().merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
 
           it "should fail" do
             expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -404,30 +535,44 @@ describe LogStash::Filters::Elasticsearch do
 
     describe "proxy" do
       context 'valid' do
-        let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+        let(:config) { super().merge({ 'proxy' => 'http://localhost:1234' }) }
 
         it "should set proxy" do
           plugin.register
           client = plugin.send(:get_client).client
-          proxy = client.transport.options[:transport_options][:proxy]
+          proxy = extract_transport(client).options[:transport_options][:proxy]
 
           expect( proxy ).to eql "http://localhost:1234"
         end
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+        let(:config) { super().merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
 
         it "should not set proxy" do
           plugin.register
           client = plugin.send(:get_client).client
 
-          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
         end
       end
     end
   end
 
+  describe "defaults" do
+
+    let(:config) { Hash.new }
+    let(:plugin) { described_class.new(config) }
+
+    before { allow(plugin).to receive(:test_connection!) }
+
+    it "should set localhost:9200 as hosts" do
+      plugin.register
+      client = plugin.send(:get_client).client
+      expect( extract_transport(client).hosts ).to eql [{ :host => "localhost", :port => 9200, :protocol => "http"}]
+    end
+  end
+
   describe "query template" do
     let(:config) do
       {
@@ -453,4 +598,10 @@ describe LogStash::Filters::Elasticsearch do
       plugin.filter(LogStash::Event.new)
     end
   end
+
+  # @note can be removed once gem depends on elasticsearch >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
+  end
+
 end

data/spec/filters/integration/elasticsearch_spec.rb

@@ -6,21 +6,31 @@ require_relative "../../../spec/es_helper"
 
 describe LogStash::Filters::Elasticsearch, :integration => true do
 
+  ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
 
-  let(:config) do
+  let(:base_config) do
     {
-      "index" => 'logs',
-      "hosts" => [ESHelper.get_host_port],
-      "query" => "response: 404",
-      "sort" => "response",
-      "fields" => [ ["response", "code"] ],
+        "index" => 'logs',
+        "hosts" => [ESHelper.get_host_port],
+        "query" => "response: 404",
+        "sort" => "response",
+        "fields" => [ ["response", "code"] ],
     }
   end
+
+  let(:credentials) do
+    { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
+  end
+
+  let(:config) do
+    ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
+  end
+
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
 
   before(:each) do
-    @es = ESHelper.get_client
+    @es = ESHelper.get_client(ELASTIC_SECURITY_ENABLED ? credentials : {})
     # Delete all templates first.
     # Clean ES of data before we start.
     @es.indices.delete_template(:name => "*")
@@ -30,11 +40,10 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :this => 'that'})
     end
     @es.indices.refresh
-
-    plugin.register
   end
 
   it "should enhance the current event with new data" do
+    plugin.register
     plugin.filter(event)
     expect(event.get('code')).to eq(404)
   end
@@ -42,20 +51,28 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
   context "when retrieving a list of elements" do
 
     let(:config) do
-      {
-        "index" => 'logs',
-        "hosts" => [ESHelper.get_host_port],
-        "query" => "response: 404",
-        "fields" => [ ["response", "code"] ],
-        "sort" => "response",
-        "result_size" => 10
-      }
+      super().merge("fields" => [ ["response", "code"] ], "result_size" => 10)
     end
 
+    before { plugin.register }
+
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq([404]*10)
     end
 
   end
+
+  context "incorrect auth credentials" do
+
+    let(:config) do
+      super().reject { |key, _| key == 'password' }
+    end
+
+    it "should enhance the current event with new data" do
+      expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+    end
+
+  end if ELASTIC_SECURITY_ENABLED
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.9.3
+  version: 3.11.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-26 00:00:00.000000000 Z
+date: 2021-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.3
+        version: 7.14.0
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -43,21 +43,49 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.3
+        version: 7.14.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: 0.7.1
   name: manticore
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -89,6 +117,7 @@ files:
 - docs/index.asciidoc
 - lib/logstash/filters/elasticsearch.rb
 - lib/logstash/filters/elasticsearch/client.rb
+- lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-filter-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/filters/elasticsearch_spec.rb
@@ -121,8 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events