logstash-output-elasticsearch 11.12.4-java → 11.15.9-java

data/spec/integration/outputs/index_spec.rb

@@ -289,8 +289,8 @@ describe "indexing" do
         "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
-        "ssl" => true,
-        "cacert" => cacert,
+        "ssl_enabled" => true,
+        "ssl_certificate_authorities" => cacert,
         "index" => index
       }
     end 
@@ -302,7 +302,7 @@ describe "indexing" do
 
       context "when no keystore nor ca cert set and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge('ssl_certificate_verification' => false)
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge('ssl_verification_mode' => 'none')
         end
 
         include_examples("an indexer", true)
@@ -311,9 +311,9 @@ describe "indexing" do
       context "when keystore is set and verification is disabled" do
         let(:config) do
           super().merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test.p12',
-              'keystore_password' => '1234567890'
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -322,10 +322,10 @@ describe "indexing" do
 
       context "when keystore has self-signed cert and verification is disabled" do
         let(:config) do
-          super().tap { |config| config.delete('cacert') }.merge(
-              'ssl_certificate_verification' => false,
-              'keystore' => 'spec/fixtures/test_certs/test_self_signed.p12',
-              'keystore_password' => '1234567890'
+          super().tap { |config| config.delete('ssl_certificate_authorities') }.merge(
+              'ssl_verification_mode' => 'none',
+              'ssl_keystore_path' => 'spec/fixtures/test_certs/test_self_signed.p12',
+              'ssl_keystore_password' => '1234567890'
           )
         end
 
@@ -349,8 +349,8 @@ describe "indexing" do
         let(:config) do
           {
               "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
-              "ssl" => true,
-              "cacert" => "spec/fixtures/test_certs/test.crt",
+              "ssl_enabled" => true,
+              "ssl_certificate_authorities" => "spec/fixtures/test_certs/test.crt",
               "index" => index
           }
         end
@@ -358,10 +358,10 @@ describe "indexing" do
         include_examples("an indexer", true)
       end
 
-      context "without providing `cacert`" do
+      context "without providing `ssl_certificate_authorities`" do
         let(:config) do
           super().tap do |c|
-            c.delete("cacert")
+            c.delete("ssl_certificate_authorities")
           end
         end
 
@@ -369,10 +369,10 @@ describe "indexing" do
       end
 
       if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
-        context "with `ca_trusted_fingerprint` instead of `cacert`" do
+        context "with `ca_trusted_fingerprint` instead of `ssl_certificate_authorities`" do
           let(:config) do
             super().tap do |c|
-              c.delete("cacert")
+              c.delete("ssl_certificate_authorities")
               c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
             end
           end

data/spec/integration/outputs/templates_spec.rb

@@ -23,7 +23,9 @@ describe "index template expected behavior", :integration => true do
 
     elasticsearch_client.indices.delete_template(:name => '*')
     # This can fail if there are no indexes, ignore failure.
-    elasticsearch_client.indices.delete(:index => '*') rescue nil
+    elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
+    # Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
+    elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
   end
 
   context 'with ecs_compatibility => disabled' do
@@ -48,19 +50,19 @@ describe "index template expected behavior", :integration => true do
 
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
-        r = @es.search(index: 'logstash-*')
+        r = @es.search(index: 'logstash*')
         expect(r).to have_hits(8)
       end
     end
 
     it "permits phrase searching on string fields" do
-      results = @es.search(:q => "message:\"sample message\"")
+      results = @es.search(index: 'logstash*', q: "message:\"sample message\"")
       expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
     end
 
     it "numbers dynamically map to a numeric type and permit range queries" do
-      results = @es.search(:q => "somevalue:[5 TO 105]")
+      results = @es.search(index: 'logstash*', q: "somevalue:[5 TO 105]")
       expect(results).to have_hits(2)
 
       values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
@@ -70,22 +72,22 @@ describe "index template expected behavior", :integration => true do
     end
 
     it "does not create .keyword field for top-level message field" do
-      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      results = @es.search(index: 'logstash*', q: "message.keyword:\"sample message here\"")
       expect(results).to have_hits(0)
     end
 
     it "creates .keyword field for nested message fields" do
-      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      results = @es.search(index: 'logstash*', q: "somemessage.message.keyword:\"sample nested message here\"")
       expect(results).to have_hits(1)
     end
 
     it "creates .keyword field from any string field which is not_analyzed" do
-      results = @es.search(:q => "country.keyword:\"us\"")
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"us\"")
       expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
 
       # partial or terms should not work.
-      results = @es.search(:q => "country.keyword:\"u\"")
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"u\"")
       expect(results).to have_hits(0)
     end
 
@@ -94,7 +96,7 @@ describe "index template expected behavior", :integration => true do
     end
 
     it "aggregate .keyword results correctly " do
-      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      results = @es.search(index: 'logstash*', body: { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
       terms = results["buckets"].collect { |b| b["key"] }
 
       expect(terms).to include("us")

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       {
           'hosts' => [ 'http://127.0.0.1:12345' ],
           'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
-          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
+          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl_enabled' => 'false'
       }
     end
 
@@ -164,7 +164,9 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
         'data_stream_dataset' => 'any',
         'data_stream_namespace' => 'any',
         'data_stream_sync_fields' => true,
-        'data_stream_auto_routing' => true)
+        'data_stream_auto_routing' => true,
+        'dlq_custom_codes' => [ 404 ],
+        'dlq_on_failed_indexname_interpolation' => false)
       }
 
       it 'should enable data-streams by default' do

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -50,6 +50,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     describe "healthcheck url handling" do
       let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+      let(:success_response) { double("Response", :code => 200) }
+
       before(:example) do
         expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
           expect(url.path).to be_empty
@@ -60,6 +62,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         it "performs the healthcheck to the root" do
           expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
+
+            success_response
           end
           expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
@@ -71,6 +75,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         it "performs the healthcheck to the healthcheck_path" do
           expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
+
+            success_response
           end
           expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
@@ -197,9 +203,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
                                  "build_flavor" => 'default'}
                                })
       end
+      let(:success_response) { double("head_req", :code => 200)}
 
       before :each do
-        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
         allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
       end
       let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
@@ -225,7 +232,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         u,m = subject.get_connection
 
         # The resurrectionist will call this to check on the backend
-        response = double("response")
+        response = double("response", :code => 200)
         expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
 
         subject.return_connection(u)

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,4 +1,4 @@
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
     end
   end
 
-  describe "index template with ilm settings" do
+  context "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
 
-    describe "in version 8+" do
-      let(:file_path) { described_class.default_template_path(8) }
-      let(:template) { described_class.read_template_file(file_path)}
+    describe "with custom template" do
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('settings')).to be_falsey
+      describe "in version 8+" do
+        let(:file_path) { described_class.default_template_path(8) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('settings')).to be_falsey
+        end
+      end
+
+      describe "in version < 8" do
+        let(:file_path) { described_class.default_template_path(7) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('template')).to be_falsey
+        end
       end
     end
 
-    describe "in version < 8" do
-      let(:file_path) { described_class.default_template_path(7) }
-      let(:template) { described_class.read_template_file(file_path)}
+    context "resolve template setting" do
+      let(:plugin_settings) { super().merge({"template_api" => template_api}) }
+
+      describe "with composable template API" do
+        let(:template_api) { "composable" }
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('template')).to be_falsey
+        it 'resolves composable index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["template"]["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with legacy template API" do
+        let(:template_api) { "legacy" }
+
+        it 'resolves legacy index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with `template_api => 'auto'`" do
+        let(:template_api) { "auto" }
+
+        describe "with ES < 8 versions" do
+
+          it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["settings"]).not_to eq(nil)
+          end
+        end
+
+        describe "with ES >= 8 versions" do
+          it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["template"]["settings"]).not_to eq(nil)
+          end
+        end
       end
     end
   end