logstash-output-elasticsearch 11.13.0-java → 11.14.0-java

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -45,35 +45,79 @@ module LogStash; module PluginMixins; module ElasticSearch
         # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
         # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
         # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
-        :ssl => { :validate => :boolean },
+        :ssl => { :validate => :boolean, :deprecated => "Set 'ssl_enabled' instead." },
+
+        # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
+        # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+        # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+        :ssl_enabled => { :validate => :boolean },
 
         # Option to validate the server's certificate. Disabling this severely compromises security.
         # For more information on disabling certificate verification please read
         # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-        :ssl_certificate_verification => { :validate => :boolean, :default => true },
+        :ssl_certificate_verification => { :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." },
+
+        # Options to verify the server's certificate.
+        # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+        # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+        # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+        :ssl_verification_mode => { :validate => %w[full none], :default => 'full' },
 
         # The .cer or .pem file to validate the server's certificate
-        :cacert => { :validate => :path },
+        :cacert => { :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." },
+
+        # The .cer or .pem files to validate the server's certificate
+        :ssl_certificate_authorities => { :validate => :path, :list => true },
 
         # One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
         :ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
 
         # The JKS truststore to validate the server's certificate.
         # Use either `:truststore` or `:cacert`
-        :truststore => { :validate => :path },
+        :truststore => { :validate => :path, :deprecated => "Set 'ssl_truststore_path' instead." },
+
+        # The JKS truststore to validate the server's certificate.
+        # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+        :ssl_truststore_path => { :validate => :path },
+
+        # The format of the truststore file. It must be either jks or pkcs12
+        :ssl_truststore_type => { :validate => %w[pkcs12 jks] },
+
+        # Set the truststore password
+        :truststore_password => { :validate => :password, :deprecated => "Use 'ssl_truststore_password' instead." },
 
         # Set the truststore password
-        :truststore_password => { :validate => :password },
+        :ssl_truststore_password => { :validate => :password },
 
         # The keystore used to present a certificate to the server.
         # It can be either .jks or .p12
-        :keystore => { :validate => :path },
+        :keystore => { :validate => :path, :deprecated => "Set 'ssl_keystore_path' instead." },
+
+        # The keystore used to present a certificate to the server.
+        # It can be either .jks or .p12
+        :ssl_keystore_path => { :validate => :path },
+
+        # The format of the keystore file. It must be either jks or pkcs12
+        :ssl_keystore_type => { :validate => %w[pkcs12 jks] },
 
         # Set the keystore password
-        :keystore_password => { :validate => :password },
+        :keystore_password => { :validate => :password, :deprecated => "Set 'ssl_keystore_password' instead." },
+
+        # Set the keystore password
+        :ssl_keystore_password => { :validate => :password },
 
         :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
 
+        # OpenSSL-style X.509 certificate certificate to authenticate the client
+        :ssl_certificate => { :validate => :path },
+
+        # OpenSSL-style RSA private key to authenticate the client
+        :ssl_key => { :validate => :path },
+
+        # The list of cipher suites to use, listed by priorities.
+        # Supported cipher suites vary depending on which version of Java is used.
+        :ssl_cipher_suites => { :validate => :string, :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -28,8 +28,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
       setup_hosts
 
-
-      params['ssl'] = effectively_ssl? unless params.include?('ssl')
+      params['ssl_enabled'] = effectively_ssl? unless params.include?('ssl_enabled')
 
       # inject the TrustStrategy from CATrustedFingerprintSupport
       if trust_strategy_for_ca_trusted_fingerprint
@@ -74,7 +73,7 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
 
     def effectively_ssl?
-      return @ssl unless @ssl.nil?
+      return @ssl_enabled unless @ssl_enabled.nil?
 
       hosts = Array(@hosts)
       return false if hosts.nil? || hosts.empty?

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.13.0'
+  s.version         = '11.14.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/integration/outputs/index_spec.rb

@@ -289,8 +289,8 @@ describe "indexing" do
         "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
-        "ssl" => true,
-        "cacert" => cacert,
+        "ssl_enabled" => true,
+        "ssl_certificate_authorities" => cacert,
         "index" => index
       }
     end 
@@ -302,7 +302,7 @@ describe "indexing" do
 
       context "when no keystore nor ca cert set and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge('ssl_certificate_verification' => false)
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
         end
 
         include_examples("an indexer", true)
@@ -311,9 +311,9 @@ describe "indexing" do
       context "when keystore is set and verification is disabled" do
         let(:config) do
           super().merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test.p12',
-              'keystore_password' => '1234567890'
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -322,10 +322,10 @@ describe "indexing" do
 
       context "when keystore has self-signed cert and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test_self_signed.p12',
-              'keystore_password' => '1234567890'
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -349,8 +349,8 @@ describe "indexing" do
         let(:config) do
           {
               "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
-              "ssl" => true,
-              "cacert" => "spec/fixtures/test_certs/test.crt",
+              "ssl_enabled" => true,
+              "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
               "index" => index
           }
         end
@@ -358,10 +358,10 @@ describe "indexing" do
         include_examples("an indexer", true)
       end
 
-      context "without providing `cacert`" do
+      context "without providing `ssl_certificate_authorities`" do
         let(:config) do
           super().tap do |c|
-            c.delete("cacert")
+            c.delete("ssl_certificate_authorities")
           end
         end
 
@@ -369,10 +369,10 @@ describe "indexing" do
       end
 
       if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
-        context "with `ca_trusted_fingerprint` instead of `cacert`" do
+        context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
           let(:config) do
             super().tap do |c|
-              c.delete("cacert")
+              c.delete("ssl_certificate_authorities")
               c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
             end
           end

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       {
           'hosts' => [ 'http://127.0.0.1:12345' ],
           'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
-          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
+          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl_enabled' => 'false'
       }
     end
 

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,4 +1,4 @@
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
     end
   end
 
-  describe "index template with ilm settings" do
+  context "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
 
-    describe "in version 8+" do
-      let(:file_path) { described_class.default_template_path(8) }
-      let(:template) { described_class.read_template_file(file_path)}
+    describe "with custom template" do
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('settings')).to be_falsey
+      describe "in version 8+" do
+        let(:file_path) { described_class.default_template_path(8) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('settings')).to be_falsey
+        end
+      end
+
+      describe "in version < 8" do
+        let(:file_path) { described_class.default_template_path(7) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('template')).to be_falsey
+        end
       end
     end
 
-    describe "in version < 8" do
-      let(:file_path) { described_class.default_template_path(7) }
-      let(:template) { described_class.read_template_file(file_path)}
+    context "resolve template setting" do
+      let(:plugin_settings) { super().merge({"template_api" => template_api}) }
+
+      describe "with composable template API" do
+        let(:template_api) { "composable" }
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('template')).to be_falsey
+        it 'resolves composable index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["template"]["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with legacy template API" do
+        let(:template_api) { "legacy" }
+
+        it 'resolves legacy index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with `template_api => 'auto'`" do
+        let(:template_api) { "auto" }
+
+        describe "with ES < 8 versions" do
+
+          it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["settings"]).not_to eq(nil)
+          end
+        end
+
+        describe "with ES >= 8 versions" do
+          it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["template"]["settings"]).not_to eq(nil)
+          end
+        end
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -699,9 +699,8 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-
-    context "With the 'ssl' option" do
-      let(:options) { {"ssl" => true}}
+    context "With the 'ssl_enabled' option" do
+      let(:options) { {"ssl_enabled" => true}}
 
       include_examples("an encrypted client connection")
     end
@@ -712,6 +711,81 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "SSL deprecated settings" do
+    let(:base_options) { {"ssl" => "true"} }
+
+    context "with client certificate" do
+      let(:do_register) { true }
+      let(:cacert) { Stud::Temporary.file.path }
+      let(:options) { base_options.merge(
+        "cacert" => cacert,
+        "ssl_certificate_verification" => false
+      ) }
+
+      after :each do
+        File.delete(cacert)
+      end
+
+      it "should map new configs into params" do
+        expect(subject.params).to match hash_including(
+                                          "ssl_enabled" => true,
+                                          "ssl_verification_mode" => "none",
+                                          "ssl_certificate_authorities" => [cacert]
+                                        )
+      end
+
+      it "should set new configs variables" do
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
+        expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
+      end
+    end
+
+    context "with java stores" do
+      let(:do_register) { true }
+      let(:keystore) { Stud::Temporary.file.path }
+      let(:truststore) { Stud::Temporary.file.path }
+      let(:options) { base_options.merge(
+        "keystore" => keystore,
+        "keystore_password" => "keystore",
+        "truststore" => truststore,
+        "truststore_password" => "truststore",
+        "ssl_certificate_verification" => true
+      ) }
+
+      let(:spy_http_client_builder!) do
+        allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
+        allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
+      end
+
+      after :each do
+        File.delete(keystore)
+        File.delete(truststore)
+      end
+
+      it "should map new configs into params" do
+        expect(subject.params).to match hash_including(
+                                          "ssl_enabled" => true,
+                                          "ssl_keystore_path" => keystore,
+                                          "ssl_truststore_path" => truststore,
+                                          "ssl_verification_mode" => "full"
+                                        )
+
+        expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
+        expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
+      end
+
+      it "should set new configs variables" do
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
+        expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
+        expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
+        expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
+        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
+      end
+    end
+  end
+
   describe "retry_on_conflict" do
     let(:num_retries) { 123 }
     let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
@@ -1093,12 +1167,12 @@ describe LogStash::Outputs::ElasticSearch do
       it 'adds the appropriate Authorization header to the manticore client' do
         expect(manticore_options[:headers]).to eq({ "Authorization" => base64_api_key })
       end
-      it 'is provides ssl=>true to the http client builder' do; aggregate_failures do
-        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl'=>true))
+      it 'is provides ssl_enabled=>true to the http client builder' do; aggregate_failures do
+        expect(described_class::HttpClientBuilder).to have_received(:build).with(anything, anything, hash_including('ssl_enabled'=>true))
       end; end
     end
 
-    context "when set without ssl => true" do
+    context "when set without ssl_enabled => true" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "api_key" => api_key } }
 
@@ -1114,14 +1188,14 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context "when set without ssl specified but with an https host" do
+    context "when set without ssl_enabled specified but with an https host" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["https://some.host.com"], "api_key" => api_key } }
 
       it_behaves_like 'secure api-key authenticated client'
     end
 
-    context "when set without ssl specified but with an http host`" do
+    context "when set without ssl_enabled specified but with an http host`" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "hosts" => ["http://some.host.com"], "api_key" => api_key } }
 
@@ -1130,9 +1204,9 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
-    context "when set with `ssl => false`" do
+    context "when set with `ssl_enabled => false`" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => "false", "api_key" => api_key } }
+      let(:options) { { "ssl_enabled" => "false", "api_key" => api_key } }
 
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /requires SSL\/TLS/
@@ -1142,13 +1216,13 @@ describe LogStash::Outputs::ElasticSearch do
     context "when set" do
       let(:options) { { "api_key" => ::LogStash::Util::Password.new(api_key) } }
 
-      context "with ssl => true" do
-        let(:options) { super().merge("ssl" => true) }
+      context "with ssl_enabled => true" do
+        let(:options) { super().merge("ssl_enabled" => true) }
         it_behaves_like 'secure api-key authenticated client'
       end
 
-      context "with ssl => false" do
-        let(:options) { super().merge("ssl" => "false") }
+      context "with ssl_enabled => false" do
+        let(:options) { super().merge("ssl_enabled" => "false") }
 
         let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
         it "should raise a configuration error" do
@@ -1156,7 +1230,7 @@ describe LogStash::Outputs::ElasticSearch do
         end
       end
 
-      context "without ssl specified" do
+      context "without ssl_enabled specified" do
         context "with an https host" do
           let(:options) { super().merge("hosts" => ["https://some.host.com"]) }
           it_behaves_like 'secure api-key authenticated client'
@@ -1180,7 +1254,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context 'user also set' do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => true, "api_key" => api_key, 'user' => 'another' } }
+      let(:options) { { "ssl_enabled" => true, "api_key" => api_key, 'user' => 'another' } }
 
       it "should fail" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -1189,7 +1263,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context 'cloud_auth also set' do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
-      let(:options) { { "ssl" => true, "api_key" => api_key, 'cloud_auth' => 'foobar' } }
+      let(:options) { { "ssl_enabled" => true, "api_key" => api_key, 'cloud_auth' => 'foobar' } }
 
       it "should fail" do
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/