logstash-filter-elasticsearch 3.10.0 → 3.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e04ff9ca7407b8404764623b42f7ce8e754fe9ceafc6bde7cc064bc13c30723
-  data.tar.gz: 3b317df7d5bc1b11c912e8f9d6ec14c2ef260fa975412f2ebbf2baf84da547ba
+  metadata.gz: 7ecfb3d5b15acecc9b301e27f77f5170ead83708c2722db56324807e3663cc08
+  data.tar.gz: bd8798a9f82792afb79b1be85936bdf51967dd74c91d6b45bb24b7cadec16e1b
 SHA512:
-  metadata.gz: 6dd245e044d1c06882885e776ba64463c1dbfc93d3c80ad03ad4009b6a3b6246e5a166dc11c09cafb58f2b05b3a6e485741aef6a2137edc227081b1e06bf9e3c
-  data.tar.gz: b9b211e1bf50bcb18d028e5322e78eaa3e113a034550eaae4ee3550b941e74235b5ed672417759eb35709e46693f7f602b8106ce13beb17bc6937eacbd991514
+  metadata.gz: b76de8e2722b3b1c5cf11efd0a29cc827042e48d584215e14fb9272349bc5cb50aa04b763a52ff26800ad36f64f2c1870cab27a38dbf94776fde5a25f75a7e08
+  data.tar.gz: 9304e6e00443b13fe5888ae62d0f9c0610cb6917cf148b21373afae460602548172a61efa4fed287e085421c7e9ef26bcebaf7a8ec1aee70a12449942b226a3e

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.12.0
+  - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#158](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/158)
+
+## 3.11.1
+  -  Fix: hosts => "es_host:port" regression [#156](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/156)
+
+## 3.11.0
+  - Feat: update Elasticsearch client to 7.14.0 [#150](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/150)
+
 ## 3.10.0
   - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#152](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/152)
 

data/Gemfile

@@ -9,3 +9,6 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+gem 'manticore', ENV['MANTICORE_VERSION'] if ENV['MANTICORE_VERSION']
+gem 'elasticsearch', ENV['ELASTICSEARCH_VERSION'] if ENV['ELASTICSEARCH_VERSION']

data/docs/index.asciidoc

@@ -128,6 +128,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
@@ -189,6 +190,15 @@ Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
 
 SSL Certificate Authority file
 
+[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
+===== `ca_trusted_fingerprint`
+
+* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
+* There is no default value for this setting.
+* Use of this option _requires_ Logstash 8.3+
+
+The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
+
 [id="plugins-{type}s-{plugin}-cloud_auth"]
 ===== `cloud_auth`
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -26,10 +26,12 @@ module LogStash
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 
-        hosts = hosts.map { |host| { host: host, scheme: 'https' } } if ssl
+        hosts = setup_hosts(hosts, ssl)
+
+        ssl_options = {}
         # set ca_file even if ssl isn't on, since the host can be an https url
-        ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
-        ssl_options ||= {}
+        ssl_options.update(ssl: true, ca_file: options[:ca_file]) if options[:ca_file]
+        ssl_options.update(ssl: true, trust_strategy: options[:ssl_trust_strategy]) if options[:ssl_trust_strategy]
 
         logger.info("New ElasticSearch filter client", :hosts => hosts)
         @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
@@ -41,6 +43,17 @@ module LogStash
 
       private
 
+      def setup_hosts(hosts, ssl)
+        hosts.map do |h|
+          if h.start_with?('http:/', 'https:/')
+            h
+          else
+            host, port = h.split(':')
+            { host: host, port: port, scheme: (ssl ? 'https' : 'http') }
+          end
+        end
+      end
+
       def setup_basic_auth(user, password)
         return {} unless user && password && password.value
 

data/lib/logstash/filters/elasticsearch.rb

@@ -2,6 +2,8 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
+require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+
 require_relative "elasticsearch/client"
 require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
@@ -73,6 +75,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
+  # config :ca_trusted_fingerprint, :validate => :sha_256_hex
+  include LogStash::PluginMixins::CATrustedFingerprintSupport
+
   attr_reader :clients_pool
 
   ##
@@ -199,6 +204,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :proxy => @proxy,
       :ssl => @ssl,
       :ca_file => @ca_file,
+      :ssl_trust_strategy => trust_strategy_for_ca_trusted_fingerprint
     }
   end
 
@@ -320,6 +326,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def test_connection!
-    get_client.client.ping
+    begin
+      get_client.client.ping
+    rescue Elasticsearch::UnsupportedProductError
+      raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+    end
   end
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.10.0'
+  s.version         = '3.12.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 5.0.5" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
+  s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
 

data/spec/es_helper.rb

@@ -7,12 +7,26 @@ module ESHelper
     end
   end
 
-  def self.get_client(credentials)
-    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
-    host, port = get_host_port.split(':')
-    host_opts = credentials.inject({}) { |h, (k, v)| h[k.to_sym] = v; h } # user: _, password: _
-    host_opts.merge! host: host, port: port, scheme: 'http'
-    Elasticsearch::Client.new(hosts: [host_opts], transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
+  def self.curl_and_get_json_response(url, method: :get, args: nil); require 'open3'
+    cmd = "curl -s -v --show-error #{args} -X #{method.to_s.upcase} -k #{url}"
+    begin
+      out, err, status = Open3.capture3(cmd)
+    rescue Errno::ENOENT
+      fail "curl not available, make sure curl binary is installed and available on $PATH"
+    end
+
+    if status.success?
+      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      if http_status.strip[0].to_i > 2
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
+
+      LogStash::Json.load(out)
+    else
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+    end
   end
 
   def self.doc_type
@@ -25,12 +39,6 @@ module ESHelper
     end
   end
 
-  def self.index_doc(es, params)
-    type = doc_type
-    params[:type] = doc_type unless type.nil?
-    es.index(params)
-  end
-
   def self.es_version
     ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
   end

data/spec/filters/elasticsearch_spec.rb

@@ -9,15 +9,41 @@ require "uri"
 
 describe LogStash::Filters::Elasticsearch do
 
+  subject(:plugin) { described_class.new(config) }
+
+  let(:event)  { LogStash::Event.new({}) }
+
   context "registration" do
 
     let(:plugin) { LogStash::Plugin.lookup("filter", "elasticsearch").new({}) }
-    before do
-      allow(plugin).to receive(:test_connection!)
+
+    context "against authentic Elasticsearch" do
+      before do
+        allow(plugin).to receive(:test_connection!)
+      end
+      
+      it "should not raise an exception" do
+        expect {plugin.register}.to_not raise_error
+      end
     end
 
-    it "should not raise an exception" do
-      expect {plugin.register}.to_not raise_error
+    context "against not authentic Elasticsearch" do
+      let(:failing_client) do
+        client = double("client")
+        allow(client).to receive(:ping).and_raise Elasticsearch::UnsupportedProductError
+
+        client_wrapper = double("filter_client")
+        allow(client_wrapper).to receive(:client).and_return client
+        client_wrapper
+      end
+
+      before do
+        allow(plugin).to receive(:get_client).and_return(failing_client)
+      end
+
+      it "should raise ConfigurationError" do
+        expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+      end
     end
   end
 
@@ -31,8 +57,6 @@ describe LogStash::Filters::Elasticsearch do
         "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
       }
     end
-    let(:plugin) { described_class.new(config) }
-    let(:event)  { LogStash::Event.new({}) }
 
     let(:response) do
       LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
@@ -537,6 +561,38 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
+  describe "ca_trusted_fingerprint" do
+    let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
+    let(:config) { {"ca_trusted_fingerprint" => ca_trusted_fingerprint}}
+
+    subject(:plugin) { described_class.new(config) }
+
+    if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
+      context 'the generated trust_strategy' do
+        before(:each) { allow(plugin).to receive(:test_connection!) }
+
+        it 'is passed to the Manticore client' do
+          expect(Manticore::Client).to receive(:new)
+                                         .with(
+                                           a_hash_including(
+                                             ssl: a_hash_including(
+                                               trust_strategy: plugin.trust_strategy_for_ca_trusted_fingerprint
+                                             )
+                                           )
+                                         ).and_call_original
+          plugin.register
+
+          # the client is built lazily, so we need to get it explicitly
+          plugin.send(:get_client).client
+        end
+      end
+    else
+      it 'raises a configuration error' do
+        expect { plugin }.to raise_exception(LogStash::ConfigurationError, a_string_including("ca_trusted_fingerprint"))
+      end
+    end
+  end
+
   describe "defaults" do
 
     let(:config) { Hash.new }
@@ -547,7 +603,9 @@ describe LogStash::Filters::Elasticsearch do
     it "should set localhost:9200 as hosts" do
       plugin.register
       client = plugin.send(:get_client).client
-      expect( extract_transport(client).hosts ).to eql [{ :host => "localhost", :port => 9200, :protocol => "http"}]
+      hosts = extract_transport(client).hosts
+      expect( hosts.size ).to be 1
+      expect( hosts[0] ).to include(:host => "localhost", :port => 9200, :scheme => "http")
     end
   end
 

data/spec/filters/fixtures/test_certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
+8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
+PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
+OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
+oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
+YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
+AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
+MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
+G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
+DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
+sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
+NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
+emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
+-----END CERTIFICATE-----

data/spec/filters/fixtures/test_certs/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABAoIBABmBC0P6Ebegljkk
+lO26GdbOKvbfqulDS3mN5QMyXkUMopea03YzMnKUJriE+2O33a1mUcuDPWnLpYPK
+BTiQieYHlulNtY0Bzf+R69igRq9+1WpZftGnzrlu7NVxkOokRqWJv3546ilV7QZ0
+f9ngmu+tiN7hEnlBC8m613VMuGGb3czwbCizEVZxlZX0Dk2GExbH7Yf3NNs/aOP/
+8x6CqgL+rhrtOQ80xwRrOlEF8oSSjXCzypa3nFv21YO3J2lVo4BoIwnHgOzyz46A
+b37gekqXXajIYQ0HAB+NDgVoCRFFJ7Xe16mgB3DpyUpUJzwiMedJkeQ0TprIownQ
++1mPe9ECgYEA/K4jc0trr3sk8KtcZjOYdpvwrhEqSSGEPeGfFujZaKOb8PZ8PX6j
+MbCTV12nEgm8FEhZQ3azxLnO17gbJ2A+Ksm/IIwnTWlqvvMZD5qTQ7L3qZuCtbWQ
++EGC/H1SDjhiwvjHcXP61/tYL/peApBSoj0L4kC+U/VaNyvicudKk08CgYEAr46J
+4VJBJfZ4ZaUBRy53+fy+mknOfaj2wo8MnD3u+/x4YWTapqvDOPN2nJVtKlIsxbS4
+qCO+fzUV17YHlsQmGULNbtFuXWJkP/RcLVbe8VYg/6tmk0dJwNAe90flagX2KJov
+8eDX129nNpuUqrNNWsfeLmPmH6vUzpKlga+1zfUCgYBrbUHHJ96dmbZn2AMNtIvy
+iXP3HXcj5msJwB3aKJ8eHMkU1kaWAnwxiQfrkfaQ9bCP0v6YbyQY1IJ7NlvdDs7/
+dAydMtkW0WW/zyztdGN92d3vrx0QUiRTV87vt/wl7ZUXnZt1wcB5CPRCWaiUYHWx
+YlDmHW6N1XdIk5DQF0OegwKBgEt7S8k3Zo9+A5IgegYy8p7njsQjy8a3qTFJ9DAR
+aPmrOc8WX/SdkVihRXRZwxAZOOrgoyyYAcYL+xI+T9EBESh3UoC9R2ibb2MYG7Ha
+0gyN7a4/8eCNHCbs1QOZRAhr+8TFVqv28pbMbWJLToZ+hVns6Zikl0MyzFLtNoAm
+HlMpAoGBAIOkqnwwuRKhWprL59sdcJfWY26os9nvuDV4LoKFNEFLJhj2AA2/3UlV
+v85gqNSxnMNlHLZC9l2HZ3mKv/mfx1aikmFvyhJAnk5u0f9KkexmCPLjQzS5q3ba
+yFuxK2DXwN4x46RgQPFlLjOTCX0BG6rkEu4JdonF8ETSjoCtGEU8
+-----END RSA PRIVATE KEY-----

data/spec/filters/fixtures/test_certs/es.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
+VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
+1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
+Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
+CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
+2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
+LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
+tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
+MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
+YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
+7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
+f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
+Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
+LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
+qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
+qi02i4q6meHGcw==
+-----END CERTIFICATE-----

data/spec/filters/fixtures/test_certs/es.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5VV21nXpYzQJoQ
+buWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz36pcFw7UyF51
+/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/cMjNrUC7iP0dv
+fOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH/z07/mVKoBAa
+5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gRhQNM3zcKKsjE
+MomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABAoIBAQCm/VBDz41ImG7p
+yu3e6iMeFi7HW5SKdlRUS5dJbHT1uBWJAm/q8TbwvnUBVdsn9cKWY06QYDPQBjAy
+0LxRSIKivjyl+aIJDZbbEUXrmk/M0zT9rHtgSc2isM8ITH6IHw5q7lmNMPLYOu6T
+IMvfTDtADBOOTV/vF+/4NKf5GCUXVt1XTzLBFMK0p/ZoI7Fsw7fhH6FR12vk0xA4
+BEC4pwRbGfHo7P31ii0by8epkve93tF4IZuFmN92A84bN1z7Kc4TYaSbua2rgguz
+FzMyWpsTxr363HzCK1xOJb6JyJOiXbq4+j2oqtne3GIvyozJeiyKRgjLIMoe/LV7
+fPPc5wlhAoGBAOD3z0JH2eyR/1RHILFsWInH2nDbKHHuCjhFIL2XloeXsJkiJZ95
+BpdjExMZCqD44tPNRW/GgWKwoVwltm6zB0aq0aW/OfOzw6fhKt1W+go47L7Tpwap
+VQgy6BFXSueUKfQDlZEWV4E2gakf8vOl0/VRQExae/CeKf1suEedQaErAoGBAMWE
+LOmNDEU2NFqghfNBAFYyFJst3YnBmSmlL7W22+OsfSK/PhxnJbuNHxMgxpg9rieW
+tVyjuZRo/i7WLVm3uG+dK1RJ9t8Y6kpYkCRKpi9G8DBOj3PSulOybBr+fdRfW9mf
+8UmqOjOkrhxXPkchc9TY4EM7/1XeKvEidlIp0gvRAoGAAurz4zYvW2QhXaR2hhaT
+p2XSLXiKM8AUndo3rH3U0/lhrvrEZicZsMj2LF88xg20U27sIaD/eJo13Y4XqaPk
+ykPY6D9srv574SeIeMpx/8PxPiBcoDd+BNc0L1VkgVBoouORAwq5I9HjKKBjdEmI
+UDw3i0X5KYvDm6fXVAZ0HXUCgYBWc4To8KiXPqNpq2sVzrSkBaWJSmj2G7u7Q6b/
+RTs3is72v3gjHG6iiaE5URY7mnu4rjlRhAP9Vnsy6uHMrCJZEBTf/sPEYHZj9iGZ
+EOduOAF3U1tsmaaebbDtm8hdhSOBvITy9kQlSIZAt1r17Ulytz5pj0AySFzJUIkz
+a0SZkQKBgCWixtUxiK8PAdWhyS++90WJeJn8eqjuSAz+VMtFQFRRWDUbkiHvGMRu
+o/Hhk6zS46gSF2Evb1d26uUEenXnJlIp6YWzb0DLPrfy5P53kPA6YEvYq5MSAg3l
+DZOJUF+ko7cWXSZkeTIBH/jrGOdP4tTALZt6DNt+Gz7xwPO5tGgV
+-----END RSA PRIVATE KEY-----

data/spec/filters/integration/elasticsearch_spec.rb

@@ -7,11 +7,12 @@ require_relative "../../../spec/es_helper"
 describe LogStash::Filters::Elasticsearch, :integration => true do
 
   ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
+  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 
   let(:base_config) do
     {
         "index" => 'logs',
-        "hosts" => [ESHelper.get_host_port],
+        "hosts" => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"],
         "query" => "response: 404",
         "sort" => "response",
         "fields" => [ ["response", "code"] ],
@@ -19,27 +20,40 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
   end
 
   let(:credentials) do
-    { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
+    if SECURE_INTEGRATION
+      { 'user' => 'tests', 'password' => 'Tests123' } # added user
+    else
+      { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
+    end
   end
 
   let(:config) do
-    ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
+    config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
+    config = { 'ca_file' => ca_path }.merge(config) if SECURE_INTEGRATION
+    config
+  end
+
+  let(:ca_path) do
+    File.expand_path('../fixtures/test_certs/ca.crt', File.dirname(__FILE__))
   end
 
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
 
   before(:each) do
-    @es = ESHelper.get_client(ELASTIC_SECURITY_ENABLED ? credentials : {})
-    # Delete all templates first.
+    es_url = ESHelper.get_host_port
+    es_url = SECURE_INTEGRATION ? "https://#{es_url}" : "http://#{es_url}"
+    args = ELASTIC_SECURITY_ENABLED ? "-u #{credentials['user']}:#{credentials['password']}" : ''
     # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
+    # Delete all templates first.
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: args
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response "#{es_url}/_index/*", method: 'DELETE', args: args
+    doc_args = "#{args} -H 'Content-Type: application/json' -d '{\"response\": 404, \"this\":\"that\"}'"
     10.times do
-      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :this => 'that'})
+      ESHelper.curl_and_get_json_response "#{es_url}/logs/_doc", method: 'POST', args: doc_args
     end
-    @es.indices.refresh
+    ESHelper.curl_and_get_json_response "#{es_url}/_refresh", method: 'POST', args: args
   end
 
   it "should enhance the current event with new data" do
@@ -69,10 +83,23 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       super().reject { |key, _| key == 'password' }
     end
 
-    it "should enhance the current event with new data" do
+    it "fails to register plugin" do
       expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
     end
 
   end if ELASTIC_SECURITY_ENABLED
 
+  context 'setting host:port (and ssl)' do # reproduces GH-155
+
+    let(:config) do
+      super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION
+    end
+
+    it "works" do
+      expect { plugin.register }.to_not raise_error
+      plugin.filter(event)
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.10.0
+  version: 3.12.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-27 00:00:00.000000000 Z
+date: 2022-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.5
+        version: 7.14.0
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.5
+        version: 7.14.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-ca_trusted_fingerprint_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -128,6 +142,10 @@ files:
 - spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
+- spec/filters/fixtures/test_certs/ca.crt
+- spec/filters/fixtures/test_certs/ca.key
+- spec/filters/fixtures/test_certs/es.crt
+- spec/filters/fixtures/test_certs/es.key
 - spec/filters/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -164,4 +182,8 @@ test_files:
 - spec/filters/fixtures/request_size0_agg.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
+- spec/filters/fixtures/test_certs/ca.crt
+- spec/filters/fixtures/test_certs/ca.key
+- spec/filters/fixtures/test_certs/es.crt
+- spec/filters/fixtures/test_certs/es.key
 - spec/filters/integration/elasticsearch_spec.rb