logstash-integration-elastic_enterprise_search 2.2.1 → 3.0.1

data/spec/fixtures/certificates/root_ca.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDnyKWaGkRr3N8z
+7lG7cD49FZlk8p26AVG213xKZaj4VlfJfcQZQ8AoGXzOFI5ow6qhnChglT7tyngZ
+U16TZdY0Iosh1ZFvt7S3N00tamYQdAH1IUQPvygHPZvs66Ik3G/+NOFeGVQKYFkU
+iuxUgGJ13Q+lFMRSshDay0rf8ekRnFE73roHBveo8Pr7TWDB/eacphQe+RPEVB4V
+YILqv/J+PByzYxidFMdhY5rlMYyRk/SOsOqUcXVLw94yo2D42OIcNMSkpdpxxTH4
+2iInWeGgyWkMCDZV9aUWNcXdZ+zRLO0IZD3LxRJ0c/QSFjwrfIVoSkNQyMq7b9HP
+imsc3Ud+YsRQiYuRsMIeAVFNrIZxbbC61o///hWvM2loTai8uOhdxj72l2pCq3Vz
+I+i3rkX0x0/oFx00h686batF+bPHwBUuJpzf+KMJPCiV5Z916TnjHA1OrtBd6WH+
+ckFmL+6TwjzLk9mg9iYWVFnky9+mG6c2svgjn05KA++2r2rVsXQPx8DwppgDdEk5
+70ciu0EhP94oZlw0smtf/uhp/wVym9Z1IOjo3i19DKeF+DzPTD2ylSi/43ubICeK
+GG70ZT3eP2Nes58z/GXhxHFYHLUJcg6KjvaPWiv6KciGWwtR/XuT2aoShlppsZv6
+u5ckPWxYvhZK0+07yxyoCrLgGGO1XQIDAQABAoICAAVm/FEgrpqS/dLQibAkOBcl
+YztimkGFGVn5IMj9IJiVLpugJktudFKa8Xk+KmmPsfDtWlWR9PMG5pC0jD5EP99c
+NMD3oNmpPvmMPOzDj5hvk8IaYeM6Ed1FVtaJUXCpgKS8p8oRLg/F2h2r9UEs8t6L
+ZftzhfXld5YZ6edZVJTX/161XMGgLHKQF1F3Jruuxsq85jgolwja6w6Vfseo7KZ7
+kTGvLBoxK0p+T1RrOLsF50vSHM2T7Wd0K8AjZbW3HGIK6tBpmzNOpNEHiIdBRYyB
+VUT2Bgov3Qas/y0faHohoW6yI2yV7c2wDC7FvOZIkzVwkawW2f7PZuT+9WspTyq8
+GKsWfeBaCvzT62LHzJuLIs6kpiQbS5yYQu+9Q2FOCTpqaNHpsgA5MHUvTCKEgqa7
+H4CI055eVKgktaCt2j1ShcryYjlYGgJ0opssVxk7EUQD+H28MeJ9BOcvpRv2YjZn
+hRzK3QO2eJWKAbdibVnwBrFeBY007AXj+c8NONPxMUVybWZeCLMNJ/U0/yf6JAmZ
+HfkDZsGN0AlovrvYV/n/ngg3JPdWOHLIwPHjJOfDLx+YaJA7VEEF961HAno66nH+
+CZQClSZsxl3+76IMG7ZpNDddFGYYXC6zSyHbcoo/f1Nz+syG9zC/q6lutd6Y/Pkb
++FMx1HZMhTzWXDYbdrZRAoIBAQDzyIAHWE4J7f0rBe1YLNwtX2K3TO6UK8qVg4jT
+1BM7uyCgFe4qU2Yv69oVx3TMAefiPh6DxL6zZRbOYdFFPNWyxFxz129P2mRCTads
+trS+2+nF5F8114zy4vfxn9TC1SDWuGFmukUdvLWAmSJOjrQQn5OPoEZgiP5N8Oq2
+yxrE84qdUGBXEtTaF2RvqpfytB5hepTsDN7/GCUCpVPp+jTNFV7t7LaZIEWUEemD
+VssmdPg4SgV3CNujjuip+RmJr40YeuTk80IqrquOmNOU/mOkRHa/8cwBxmbaH0L2
+AzEAehJYisb0jk3cAm1kNA8VxhTvlPr6jpId47LEbwEGRYAZAoIBAQDzZjSz+1Bn
+AI/3sZM/QHhiXUfJ/iTdhFlzfcwAY4ZO6ksJS2tjTKwvABum7BTFaUyzwfx4Ffn8
+VyHNFBiIENn1VJhaAGUf4kbY/xSGcFOH+BAW+c6XygbcSTF7eFJpd4E116gbOOd0
+I/4ZWHNcbaBx/prj1rHXvt/ytvpjvJzWWAXYQq1mN9Kzf/StuQmaNWc7Tts6qazr
+9Ovq5GFnHHI/tSG7y5jjzJu+A4d+ES/1Xle6cdozYoQfC48h6TQJHLQehD8fV2Rh
+4abIYp43JBVaC1gNvRZYRDbNot6SzVq33GRgJdSIiwvaOUCFV0ynAldsuZiPGWzC
+VU4qsA8OoPflAoIBAQDIh5Qw9oVzC44mFsEOuBABTqYwK5tQzNjetg0WWivcBWin
+5oI71NCI+NQy+BfuPF8XLowBThSkErJ43+O5tXn/jCCUtZD9uaKhG6lt9rilCta4
+jst2ybFKlkLoQdZM0e4qUg7k5rI17OXMO6sJvb8GsY07JtbdT7MIP/WCmX2SptbN
+4rDszzPBll5qsIQ/A8S94Iyhw1Cnu5+uFnr5S3Slgj/NLNtjCDXTwAw6mmmPX8ZS
+3yixqtqpJsPiFWLZbEt+6JBieGtkGrxKFxJsS/mAxUR9TZPHngE6yV5WaXekccWn
+ibhwsKIBKQn81AnYmlDcZssw0ZgF+Cg0By9rjj35AoIBAQDmWTjRiKsr/+1AJk9L
+UjFcLhmNwbaT6fhaA5i4WXg3YSaIclb8UAi4JmsFC3IQ7jnH+XN9FpdSSAfWNQmO
+ZBxSQLA6pRsPFK+mF6oOHWoASTS4+cuZ/XmGqsJorpOX+qBPG3yd3rm1QYbt57at
+p5Zrq+BcdcgzfRImeyL2lv77H6zQYTIHhhcjB5rBelfrGFVNXLC/kOr1GALZdy/z
+TImXX03Ak8tq8y4FowNsvMlddt6htbQiYz7UvrTEDBbR9NdF9jRi7eszc0HLbQF8
+/+oxIIFB50ypVk1jSZtI5Y3QaYSZNdQxMR6TdRFp4BfMCrxASllmPFKgi+kBH0Pw
+vUg9AoIBAFzfj1wFuaa6xNyRpS5kde7fVLn3FXfD91noB+UNVEkmqzxPdhCW3OjO
+TGwdtME5pQ8RKMCGrCw0LHQvniAibtuzZQFY+NpYzusZghvoaj/A6bmCTiTGrTuh
+K2Qn+AdMttY2QxWrj217AXEkPzfIAlkJdF9DMH2k3v09EyqrW5ljVbfx5t6SXwK9
+i9RU4YNgDH5nf96iJd3k/ulFlJkUy1yqJCFk6JbDfYQXB0eOP3ORRH8vaOYbrDdT
+mhBJMrVZiMPVxDt24vF+JpLA9+PefhOoXzmAH/CUEn92sBJStgBNm9BnXT59NRxa
+kt/GbqpcVj20znJbbvFRlGLnSyk/9dA=
+-----END PRIVATE KEY-----

data/spec/fixtures/certificates/root_untrusted_ca.crt

@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIFzTCCA7WgAwIBAgIUEezrYIhX47jUGV7OZsBZF8jv1FswDQYJKoZIhvcNAQEL
+BQAwdjELMAkGA1UEBhMCRVMxFDASBgNVBAgMC1RoZSBEYXJrbmV0MRQwEgYDVQQH
+DAtUaGUgRGFya25ldDEUMBIGA1UECgwLTG9nc3Rhc2ggQ0ExETAPBgNVBAsMCExv
+Z3N0YXNoMRIwEAYDVQQDDAkxMjcuMC4wLjEwHhcNMjMxMDE2MTQ1MDQ2WhcNMjQx
+MDE1MTQ1MDQ2WjB2MQswCQYDVQQGEwJFUzEUMBIGA1UECAwLVGhlIERhcmtuZXQx
+FDASBgNVBAcMC1RoZSBEYXJrbmV0MRQwEgYDVQQKDAtMb2dzdGFzaCBDQTERMA8G
+A1UECwwITG9nc3Rhc2gxEjAQBgNVBAMMCTEyNy4wLjAuMTCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBAOfIpZoaRGvc3zPuUbtwPj0VmWTynboBUbbXfEpl
+qPhWV8l9xBlDwCgZfM4UjmjDqqGcKGCVPu3KeBlTXpNl1jQiiyHVkW+3tLc3TS1q
+ZhB0AfUhRA+/KAc9m+zroiTcb/404V4ZVApgWRSK7FSAYnXdD6UUxFKyENrLSt/x
+6RGcUTveugcG96jw+vtNYMH95pymFB75E8RUHhVgguq/8n48HLNjGJ0Ux2FjmuUx
+jJGT9I6w6pRxdUvD3jKjYPjY4hw0xKSl2nHFMfjaIidZ4aDJaQwINlX1pRY1xd1n
+7NEs7QhkPcvFEnRz9BIWPCt8hWhKQ1DIyrtv0c+KaxzdR35ixFCJi5Gwwh4BUU2s
+hnFtsLrWj//+Fa8zaWhNqLy46F3GPvaXakKrdXMj6LeuRfTHT+gXHTSHrzptq0X5
+s8fAFS4mnN/4owk8KJXln3XpOeMcDU6u0F3pYf5yQWYv7pPCPMuT2aD2JhZUWeTL
+36Ybpzay+COfTkoD77avatWxdA/HwPCmmAN0STnvRyK7QSE/3ihmXDSya1/+6Gn/
+BXKb1nUg6OjeLX0Mp4X4PM9MPbKVKL/je5sgJ4oYbvRlPd4/Y16znzP8ZeHEcVgc
+tQlyDoqO9o9aK/opyIZbC1H9e5PZqhKGWmmxm/q7lyQ9bFi+FkrT7TvLHKgKsuAY
+Y7VdAgMBAAGjUzBRMB0GA1UdDgQWBBT6mRpxJs4yBEmTQaLgKFmtajy+szAfBgNV
+HSMEGDAWgBT6mRpxJs4yBEmTQaLgKFmtajy+szAPBgNVHRMBAf8EBTADAQH/MA0G
+CSqGSIb3DQEBCwUAA4ICAQCxbhhE07kRLXsYSfmSRyKM0yn4MfNkCco8rm8lFOig
+ux9Cq49vEojLT2ohKrkap7aJ1ldQCxGPsndbcRuMu+Pge4BiYmRMhuvbXkj1dgMe
+ILsJ4j4cq5l4qH4o5pWE619gaOMmegBTiitslMyP3YR0GU42n17HPjiXpa5W7mJm
+Y7kVnHv+KIfPwLTqEQx23YjHIwUdTtc4SuiQr32F5o4+Yh/hig9KHhv61e9OrjQG
+hE/lpW8xjpzWDNPU3+bvIKwF5RIQ0t4IHU/RasMZZCyv5F9nBv59Z4pnzgp284iA
+0rCodNrCS+f6n76WjyRpAuunweOHE5zXsIXBQJhcGtd9ciKk8DiLkCXTWRpfgMfv
+D6TrH7AJZ6j+6jH7aDmar8QmQ+R2uD2srvrlU8hGYfA/RSr2Npp/kTfJ8IZOLTxk
+kP4uheroByjkXbYKlagLlCbHDGrmFi58zXHflT/v+3MU1G9J0aVVzpNETBo1bvTQ
++LBRoxwICXr9EGI501iuiQI6r9EkI6IpZo1vrz0k4GvekOLscda08pZTmVEkSNl6
+/c3k5BO5QoV57IDgg8f+csEKrCWFaPxj47Bhc+VOriSNuxD8ML/aJJ3JsXYurLmc
+fW13vjENTGpwx84D07vWpeXKcgAwAekXEYiQXoyxPHP2olyrvAX9sQZ6xQLI02OM
+AQ==
+-----END CERTIFICATE-----

data/spec/integration/outputs/elastic_app_search_spec.rb

@@ -1,11 +1,16 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/elastic_app_search"
-require "logstash/codecs/plain"
-require "logstash/event"
-require "json"
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/codecs/plain'
+require 'logstash/event'
+require 'json'
 
-describe "indexing against running App Search", :integration => true do
+describe 'indexing against running App Search', :integration => true do
+
+  require 'logstash/outputs/elastic_app_search'
+
+  let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
+  let(:private_api_key) { ENV['APP_SEARCH_PRIVATE_KEY'] }
+  let(:search_api_key) { ENV['APP_SEARCH_SEARCH_KEY'] }
 
   let(:engine_name) do
     (0...10).map { ('a'..'z').to_a[rand(26)] }.join
@@ -13,136 +18,76 @@ describe "indexing against running App Search", :integration => true do
 
   let(:config) do
     {
-      "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
-      "engine" => engine_name,
-      "url" => "http://enterprise_search:3002"
+      'api_key' => private_api_key,
+      'engine' => engine_name,
+      'url' => url
     }
   end
 
   subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
 
   before(:each) do
-    create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
+    create_engine(engine_name)
   end
 
   private
-  def create_engine(engine_name, host, api_key)
-    url = host + "/api/as/v1/engines"
-    resp = Faraday.post(url, "{\"name\": \"#{engine_name}\"}",
-      "Content-Type" => "application/json",
-      "Authorization" => "Bearer " + api_key)
-    expect(resp.status).to eq(200)
-  end
 
-  describe "search and private keys are configured" do
+  describe 'search and private keys are configured' do
     let(:api_key_settings) do
       {
-        :private => ENV['APP_SEARCH_PRIVATE_KEY'],
-        :search => ENV['APP_SEARCH_SEARCH_KEY']
+        :private => private_api_key,
+        :search => search_api_key
       }
     end
 
-    it "setup apikeys" do
-      expect(api_key_settings[:private]).to start_with("private-")
-      expect(api_key_settings[:search]).to start_with("search-")
+    it 'setup api keys' do
+      expect(api_key_settings[:private]).to start_with('private-')
+      expect(api_key_settings[:search]).to start_with('search-')
     end
   end
 
-  describe "register" do
-    let(:config) do
-      {
-        "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
-        "engine" => "%{engine_name_field}",
-        "url" => "http://enterprise_search:3002"
-      }
-    end
+  describe 'indexing' do
+    let(:config) { super().merge('ssl_verification_mode' => 'none') }
+    let(:total_property_keys) { %w[meta page total_pages] }
+    let(:register) { true }
 
-    context "when engine is defined in sprintf format" do
-      it "does not raise an error" do
-        expect { subject.register }.to_not raise_error
-      end
-    end
-  end
-
-  describe "indexing" do
-
-    before do
-      app_search_output.register
-    end
+    before(:each) { app_search_output.register if register }
 
-    describe "single event" do
-      let(:event) { LogStash::Event.new("message" => "an event to index") }
+    describe 'single event' do
+      let(:event) { LogStash::Event.new('message' => 'an event to index') }
 
-      it "should be indexed" do
+      it 'should be indexed' do
         app_search_output.multi_receive([event])
-
-        results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
-          attempt_response = execute_search_call(engine_name)
-          expect(attempt_response.status).to eq(200)
-          parsed_resp = JSON.parse(attempt_response.body)
-          expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
-          parsed_resp["results"]
-        end
-        expect(results.first.dig("message", "raw")).to eq "an event to index"
+        expect_indexed(engine_name, 1, total_property_keys)
       end
 
-      context "using sprintf-ed engine" do
-        let(:config) do
-          {
-            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
-            "engine" => "%{engine_name_field}",
-            "url" => "http://enterprise_search:3002"
-          }
-        end
-
-        let(:event) { LogStash::Event.new("message" => "an event to index", "engine_name_field" => engine_name) }
+      context 'using sprintf-ed engine' do
+        let(:config) { super().merge('engine' => '%{engine_name_field}') }
+        let(:event) { LogStash::Event.new('message' => 'an event to index', 'engine_name_field' => engine_name) }
 
-        it "should be indexed" do
+        it 'should be indexed' do
           app_search_output.multi_receive([event])
-
-          results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
-            attempt_response = execute_search_call(engine_name)
-            expect(attempt_response.status).to eq(200)
-            parsed_resp = JSON.parse(attempt_response.body)
-            expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
-            parsed_resp["results"]
-          end
-          expect(results.first.dig("message", "raw")).to eq "an event to index"
+          expect_indexed(engine_name, 1, total_property_keys)
         end
       end
     end
 
-    private
-    def execute_search_call(engine_name)
-      url = config["url"] + "/api/as/v1/engines/#{engine_name}/search"
-      resp = Faraday.post(url, '{"query": "event"}',
-            "Content-Type" => "application/json",
-            "Authorization" => "Bearer " + config["api_key"])
-    end
-
-    describe "multiple events" do
-      context "single static engine" do
+    describe 'multiple events' do
+      context 'single static engine' do
         let(:events) { generate_events(200) } #2 times the slice size used to batch
 
-        it "all should be indexed" do
+        it 'all should be indexed' do
           app_search_output.multi_receive(events)
-
           expect_indexed(engine_name, 200)
         end
       end
 
-      context "multiple sprintf engines" do
-        let(:config) do
-          {
-            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
-            "engine" => "%{engine_name_field}",
-            "url" => "http://enterprise_search:3002"
-          }
-        end
+      context 'with sprintf engines' do
+        let(:config) { super().merge('engine' => '%{engine_name_field}') }
 
-        it "all should be indexed" do
-         create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
-         create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
+        it 'all should be indexed' do
+         create_engine('testengin1')
+         create_engine('testengin2')
          events = generate_events(100, 'testengin1')
          events += generate_events(100, 'testengin2')
          events.shuffle!
@@ -155,26 +100,109 @@ describe "indexing against running App Search", :integration => true do
       end
     end
 
+    describe 'with ssl enabled using a self-signed certificate', :secure_integration => true do
+      let(:ca_cert) { 'spec/fixtures/certificates/root_ca.crt' }
+      let(:event_message) { 'an event to index with ssl enabled' }
+      let(:event) { LogStash::Event.new('message' => event_message) }
+
+      context 'and ssl_verification_mode set to `full`' do
+        let(:config) { super().merge('ssl_verification_mode' => 'full') }
+        let(:register) { false }
+
+        it 'should raise an error' do
+          allow(app_search_output).to receive(:check_connection!).and_return(nil)
+          app_search_output.register
+          app_search_output.instance_variable_set(:@retry_disabled, true)
+
+          expect { app_search_output.multi_receive([event]) }.to raise_error(/PKIX path/)
+        end
+      end
+
+      context 'and ssl_certificate_authorities set to a valid CA' do
+        let(:config) { super().merge('ssl_certificate_authorities' => ca_cert) }
+        it 'should be indexed' do
+          app_search_output.multi_receive([event])
+          expect_indexed(engine_name, 1, %w[meta page total_pages], event_message)
+        end
+      end
+
+      context 'and ssl_truststore_path set to a valid CA' do
+        let(:config) do
+          super().merge(
+            'ssl_truststore_path' => 'spec/fixtures/certificates/root_keystore.jks',
+            'ssl_truststore_password' => 'changeme'
+          )
+        end
+
+        it 'should be indexed' do
+          app_search_output.multi_receive([event])
+          expect_indexed(engine_name, 1, %w[meta page total_pages], event_message)
+        end
+      end
+
+      context 'and ssl_supported_protocols configured' do
+        let(:config) { super().merge('ssl_certificate_authorities' => ca_cert, 'ssl_supported_protocols' => 'TLSv1.3') }
+
+        it 'should be indexed' do
+          app_search_output.multi_receive([event])
+          expect_indexed(engine_name, 1, %w[meta page total_pages], event_message)
+        end
+      end
+
+      context 'and ssl_cipher_suites configured' do
+        let(:config) { super().merge('ssl_certificate_authorities' => ca_cert, 'ssl_cipher_suites' => 'TLS_AES_256_GCM_SHA384') }
+
+        it 'should be indexed' do
+          app_search_output.multi_receive([event])
+          expect_indexed(engine_name, 1, %w[meta page total_pages], event_message)
+        end
+      end
+    end
+
     private
-    def expect_indexed(engine_name, expected_docs_count)
+
+    def execute_search_call(engine_name)
+      faraday_client.post("#{url}/api/as/v1/engines/#{engine_name}/search",
+                          '{"query": "event"}',
+                          'Content-Type' => 'application/json',
+                          'Authorization' => "Bearer #{private_api_key}")
+    end
+
+    def expect_indexed(engine_name, total_expected, total_property_keys = %w[meta page total_results], message_prefix = 'an event to index')
       results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
         attempt_response = execute_search_call(engine_name)
         expect(attempt_response.status).to eq(200)
         parsed_resp = JSON.parse(attempt_response.body)
-        expect(parsed_resp.dig("meta", "page", "total_results")).to eq(expected_docs_count)
-        parsed_resp["results"]
+        expect(parsed_resp.dig(*total_property_keys)).to eq(total_expected)
+        parsed_resp['results']
       end
-      expect(results.first.dig("message", "raw")).to start_with("an event to index")
+
+      expect(results.first.dig('message', 'raw')).to start_with(message_prefix)
     end
 
     def generate_events(num_events, engine_name = nil)
       (1..num_events).map do |i|
         if engine_name
-          LogStash::Event.new("message" => "an event to index #{i}", "engine_name_field" => engine_name)
+          LogStash::Event.new('message' => "an event to index #{i}", 'engine_name_field' => engine_name)
         else
-          LogStash::Event.new("message" => "an event to index #{i}")
+          LogStash::Event.new('message' => "an event to index #{i}")
         end
       end
     end
   end
-end
+
+  private
+
+  def faraday_client
+    Faraday.new url, ssl: { verify: false }
+  end
+
+  def create_engine(engine_name)
+    resp = faraday_client.post("#{url}/api/as/v1/engines",
+                               "{\"name\": \"#{engine_name}\"}",
+                               'Content-Type' => 'application/json',
+                               'Authorization' => "Bearer #{private_api_key}")
+
+    expect(resp.status).to eq(200)
+  end
+end