logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

data/spec/integration/outputs/data_stream_spec.rb

@@ -0,0 +1,61 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+describe "data streams", :integration => true do
+
+  let(:ds_name) { "logs-#{ds_dataset}-default" }
+  let(:ds_dataset) { 'integration_test' }
+
+  let(:options) do
+    { "data_stream" => 'true', "data_stream_dataset" => ds_dataset, "hosts" => get_host_port() }
+  end
+
+  subject { LogStash::Outputs::ElasticSearch.new(options) }
+
+  before :each do
+    @es = get_client
+    @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
+
+    es_version = @es.info['version']['number']
+    if Gem::Version.create(es_version) < Gem::Version.create('7.9.0')
+      skip "ES version #{es_version} does not support data-streams"
+    end
+  end
+
+  it "creates a new document" do
+    subject.register
+    subject.multi_receive([LogStash::Event.new("message" => "MSG 111")])
+
+    @es.indices.refresh
+
+    Stud::try(3.times) do
+      r = @es.search(index: ds_name)
+
+      expect( r['hits']['total']['value'] ).to eq 1
+      doc = r['hits']['hits'].first
+      expect( doc['_source'] ).to include "message"=>"MSG 111"
+      expect( doc['_source'] ).to include "data_stream"=>{"dataset"=>ds_dataset, "type"=>"logs", "namespace"=>"default"}
+    end
+  end
+
+  context "with document_id" do
+
+    let(:document_id) { '1234567890' }
+    let(:options) { super().merge("document_id" => document_id) }
+
+    it "creates a new document" do
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "foo")])
+
+      @es.indices.refresh
+
+      Stud::try(3.times) do
+        r = @es.search(index: ds_name, body: { query: { match: { _id: document_id } } })
+        expect( r['hits']['total']['value'] ).to eq 1
+        doc = r['hits']['hits'].first
+        expect( doc['_source'] ).to include "message"=>"foo"
+      end
+    end
+
+  end
+end

data/spec/integration/outputs/delete_spec.rb

@@ -2,64 +2,62 @@ require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
 
-if ESHelper.es_version_satisfies?(">= 2")
-  describe "Versioned delete", :integration => true do
-    require "logstash/outputs/elasticsearch"
+describe "Versioned delete", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  let(:es) { get_client }
+
+  before :each do
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    es.indices.delete(:index => "*") rescue nil
+    es.indices.refresh
+  end
 
-    let(:es) { get_client }
+  context "when delete only" do
+    subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
-    before :each do
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      es.indices.delete(:index => "*") rescue nil
-      es.indices.refresh
+    before do
+      subject.register
     end
 
-    context "when delete only" do
-      subject { LogStash::Outputs::ElasticSearch.new(settings) }
-
-      before do
-        subject.register
-      end
-
-      let(:settings) do
-        {
-          "manage_template" => true,
-          "index" => "logstash-delete",
-          "template_overwrite" => true,
-          "hosts" => get_host_port(),
-          "document_id" => "%{my_id}",
-          "version" => "%{my_version}",
-          "version_type" => "external",
-          "action" => "%{my_action}"
-        }
-      end
-
-      it "should ignore non-monotonic external version updates" do
-        id = "ev2"
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r['_version']).to eq(99)
-        expect(r['_source']['message']).to eq('foo')
+    let(:settings) do
+      {
+        "manage_template" => true,
+        "index" => "logstash-delete",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "document_id" => "%{my_id}",
+        "version" => "%{my_version}",
+        "version_type" => "external",
+        "action" => "%{my_action}"
+      }
+    end
 
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-        r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r2['_version']).to eq(99)
-        expect(r2['_source']['message']).to eq('foo')
-      end
+    it "should ignore non-monotonic external version updates" do
+      id = "ev2"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
+
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
+      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r2['_version']).to eq(99)
+      expect(r2['_source']['message']).to eq('foo')
+    end
 
-      it "should commit monotonic external version updates" do
-        id = "ev3"
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
-        expect(r['_version']).to eq(99)
-        expect(r['_source']['message']).to eq('foo')
+    it "should commit monotonic external version updates" do
+      id = "ev3"
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      expect(r['_version']).to eq(99)
+      expect(r['_source']['message']).to eq('foo')
 
-        subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-        expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
+      subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
+      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
     end
   end
 end

data/spec/integration/outputs/ilm_spec.rb

@@ -111,9 +111,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
     template = get_template(@es, "logstash")
     expect(template).to have_index_pattern("logstash-*")
-    if ESHelper.es_version_satisfies?(">= 2")
-      expect(get_template_settings(template)['index']['lifecycle']).to be_nil
-    end
+    expect(get_template_settings(template)['index']['lifecycle']).to be_nil
   end
 
   context 'with an existing policy that will roll over' do
@@ -163,9 +161,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
       template = get_template(@es, template_name)
       expect(template).to have_index_pattern("logstash-*")
-      if ESHelper.es_version_satisfies?(">= 2")
-        expect(get_template_settings(template)['index']['lifecycle']).to be_nil
-      end
+      expect(get_template_settings(template)['index']['lifecycle']).to be_nil
     end
   end
 end
@@ -198,12 +194,16 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
     let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
 
     it 'should raise a configuration error' do
+      # TODO should be refactored not to rely on plugin internals
+      finish_register = subject.method(:finish_register)
+      expect(subject).to receive(:finish_register)
       expect do
         begin
           subject.register
-          sleep(1)
+          finish_register.call
+          sleep(1.5) # wait_for_successful_connection (for the thread to raise)
         ensure
-          subject.stop_template_installer
+          subject.send :stop_after_successful_connection_thread
         end
       end.to raise_error(LogStash::ConfigurationError)
     end
@@ -227,319 +227,307 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
 
 end
 
-if ESHelper.es_version_satisfies?("<= 6.5")
-  describe 'Pre-ILM versions of Elasticsearch', :integration => true do
-    it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management'
-  end
-end
+describe 'Elasticsearch has index lifecycle management enabled', :integration => true do
 
-if ESHelper.es_version_satisfies?(">= 6.6")
-  describe 'OSS Elasticsearch', :distribution => 'oss', :integration => true do
-    it_behaves_like 'an Elasticsearch instance that does not support index lifecycle management'
-  end
+  DEFAULT_INTERVAL = '600s'
 
-  describe 'Elasticsearch has index lifecycle management enabled', :distribution => 'xpack', :integration => true do
+  let (:ilm_enabled) { true }
 
-    DEFAULT_INTERVAL = '600s'
-
-    let (:ilm_enabled) { true }
-
-    let (:settings) {
-      {
-          "ilm_enabled" => ilm_enabled,
-          "hosts" => "#{get_host_port()}"
-      }
+  let (:settings) {
+    {
+        "ilm_enabled" => ilm_enabled,
+        "hosts" => "#{get_host_port()}"
     }
-    let (:small_max_doc_policy) { max_docs_policy(3) }
-    let (:large_max_doc_policy) { max_docs_policy(1000000) }
-    let (:expected_index) { elasticsearch_output_plugin.default_ilm_rollover_alias }
-
-    subject(:elasticsearch_output_plugin) { LogStash::Outputs::ElasticSearch.new(settings) }
-
-    before :each do
-      # Delete all templates first.
-      require "elasticsearch"
-
-      # Clean ES of data before we start.
-      @es = get_client
-      clean(@es)
-      # Set the poll interval for lifecycle management to be short so changes get picked up in time.
-      set_cluster_settings(@es,  {
-          "persistent" => {
-          "indices.lifecycle.poll_interval" => "1s"
-          }
-      })
-    end
+  }
+  let (:small_max_doc_policy) { max_docs_policy(3) }
+  let (:large_max_doc_policy) { max_docs_policy(1000000) }
+  let (:expected_index) { elasticsearch_output_plugin.default_ilm_rollover_alias }
 
-    after :each do
-      # Set poll interval back to default
-      set_cluster_settings(@es,  {
-          "persistent" => {
-              "indices.lifecycle.poll_interval" => DEFAULT_INTERVAL
-          }
-      })
-      clean(@es)
-    end
+  subject(:elasticsearch_output_plugin) { LogStash::Outputs::ElasticSearch.new(settings) }
 
-    context 'with ilm enabled' do
-      let (:ilm_enabled) { true }
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
 
-      context 'when using the default policy' do
-        context 'with a custom pattern' do
-          let (:settings) { super().merge("ilm_pattern" => "000001")}
-          it 'should create a rollover alias' do
-            expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
-            subject.register
-            sleep(1)
-            expect(@es.indices.exists_alias(name: "logstash")).to be_truthy
-            expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
-          end
-        end
+    # Clean ES of data before we start.
+    @es = get_client
+    clean(@es)
+    # Set the poll interval for lifecycle management to be short so changes get picked up in time.
+    set_cluster_settings(@es,  {
+        "persistent" => {
+        "indices.lifecycle.poll_interval" => "1s"
+        }
+    })
+  end
 
-        it 'should install it if it is not present' do
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-          subject.register
-          sleep(1)
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error
-        end
+  after :each do
+    # Set poll interval back to default
+    set_cluster_settings(@es,  {
+        "persistent" => {
+            "indices.lifecycle.poll_interval" => DEFAULT_INTERVAL
+        }
+    })
+    clean(@es)
+  end
+
+  context 'with ilm enabled' do
+    let (:ilm_enabled) { true }
 
-        it 'should create the default rollover alias' do
+    context 'when using the default policy' do
+      context 'with a custom pattern' do
+        let (:settings) { super().merge("ilm_pattern" => "000001")}
+        it 'should create a rollover alias' do
           expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(name: "logstash")).to be_truthy
-          expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
+          expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
         end
+      end
 
-        it 'should ingest into a single index' do
-          subject.register
-          subject.multi_receive([
-                                    LogStash::Event.new("message" => "sample message here"),
-                                    LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-                                    LogStash::Event.new("somevalue" => 100),
-                                ])
-
-          sleep(6)
-
-          subject.multi_receive([
-                                    LogStash::Event.new("country" => "us"),
-                                    LogStash::Event.new("country" => "at"),
-                                    LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-                                ])
-
-          @es.indices.refresh
-
-          # Wait or fail until everything's indexed.
-          Stud::try(20.times) do
-            r = @es.search(index: "logstash-*")
-            expect(r).to have_hits(6)
-          end
-          indexes_written = @es.search(index: "logstash-*")['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
-            index_written = x['_index']
-            res[index_written] += 1
-          end
-
-          expect(indexes_written.count).to eq(1)
-          expect(indexes_written["logstash-#{todays_date}-000001"]).to eq(6)
-        end
+      it 'should install it if it is not present' do
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        subject.register
+        sleep(1)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error
       end
 
-      context 'when not using the default policy' do
-        let (:ilm_policy_name) {"logstash-policy-small"}
-        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
-        let (:policy) { small_max_doc_policy }
+      it 'should create the default rollover alias' do
+        expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
+        subject.register
+        sleep(1)
+        expect(@es.indices.exists_alias(name: "logstash")).to be_truthy
+        expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
+      end
 
-        before do
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-          put_policy(@es,ilm_policy_name, policy)
-        end
+      it 'should ingest into a single index' do
+        subject.register
+        subject.multi_receive([
+                                  LogStash::Event.new("message" => "sample message here"),
+                                  LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+                                  LogStash::Event.new("somevalue" => 100),
+                              ])
 
-        it 'should not install the default policy if it is not used' do
-          subject.register
-          sleep(1)
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        sleep(6)
+
+        subject.multi_receive([
+                                  LogStash::Event.new("country" => "us"),
+                                  LogStash::Event.new("country" => "at"),
+                                  LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+                              ])
+
+        @es.indices.refresh
+
+        # Wait or fail until everything's indexed.
+        Stud::try(20.times) do
+          r = @es.search(index: "logstash-*")
+          expect(r).to have_hits(6)
+        end
+        indexes_written = @es.search(index: "logstash-*")['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
+          index_written = x['_index']
+          res[index_written] += 1
         end
+
+        expect(indexes_written.count).to eq(1)
+        expect(indexes_written["logstash-#{todays_date}-000001"]).to eq(6)
       end
+    end
 
-      context 'when using a time based policy' do
-        let (:ilm_policy_name) {"logstash-policy-time"}
-        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
-        let (:policy) { max_age_policy("1d") }
+    context 'when not using the default policy' do
+      let (:ilm_policy_name) {"logstash-policy-small"}
+      let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
+      let (:policy) { small_max_doc_policy }
 
-        before do
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-          put_policy(@es,ilm_policy_name, policy)
-        end
+      before do
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        put_policy(@es,ilm_policy_name, policy)
+      end
 
-        it 'should not install the default policy if it is not used' do
-          subject.register
-          sleep(1)
-          expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-        end
+      it 'should not install the default policy if it is not used' do
+        subject.register
+        sleep(1)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
+    end
 
-      context 'with the default template' do
-        it 'should create the rollover alias' do
-          expect(@es.indices.exists_alias(name: expected_index)).to be_falsey
-          subject.register
-          sleep(1)
-          expect(@es.indices.exists_alias(name: expected_index)).to be_truthy
-          expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001")
-        end
+    context 'when using a time based policy' do
+      let (:ilm_policy_name) {"logstash-policy-time"}
+      let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
+      let (:policy) { max_age_policy("1d") }
 
-        it 'should write the ILM settings into the template' do
-          subject.register
-          sleep(1)
+      before do
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        put_policy(@es,ilm_policy_name, policy)
+      end
 
-          template = get_template(@es, "logstash")
-          expect(template).to have_index_pattern("logstash-*")
-          expect(get_template_settings(template)['index']['lifecycle']['name']).to eq("logstash-policy")
-          expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq("logstash")
-        end
+      it 'should not install the default policy if it is not used' do
+        subject.register
+        sleep(1)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+    end
 
-        it_behaves_like 'an ILM enabled Logstash'
+    context 'with the default template' do
+      it 'should create the rollover alias' do
+        expect(@es.indices.exists_alias(name: expected_index)).to be_falsey
+        subject.register
+        sleep(1)
+        expect(@es.indices.exists_alias(name: expected_index)).to be_truthy
+        expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001")
       end
 
-      context 'with a set index and a custom index pattern' do
-        if ESHelper.es_version_satisfies?(">= 8.0")
-          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
-        elsif ESHelper.es_version_satisfies?(">= 7.0")
-          let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
-        else
-          let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
-        end
+      it 'should write the ILM settings into the template' do
+        subject.register
+        sleep(1)
 
-        let (:settings) { super().merge("template" => template,
-                                      "index" => "overwrite-4")}
+        template = get_template(@es, "logstash")
+        expect(template).to have_index_pattern("logstash-*")
+        expect(get_template_settings(template)['index']['lifecycle']['name']).to eq("logstash-policy")
+        expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq("logstash")
+      end
 
-        it 'should not overwrite the index patterns' do
-          subject.register
-          sleep(1)
+      it_behaves_like 'an ILM enabled Logstash'
+    end
 
-          template = get_template(@es, "logstash")
-          expect(template).to have_index_pattern("overwrite-*")
-        end
+    context 'with a set index and a custom index pattern' do
+      if ESHelper.es_version_satisfies?(">= 8.0")
+        let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+      elsif ESHelper.es_version_satisfies?(">= 7.0")
+        let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
+      else
+        let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
       end
 
+      let (:settings) { super().merge("template" => template,
+                                    "index" => "overwrite-4")}
 
-      context 'with a custom template' do
-        let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
-        let (:index) { ilm_rollover_alias }
-        let(:expected_index) { index }
-        let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
-                                      "template" => template,
-                                      "ilm_rollover_alias" => ilm_rollover_alias)}
+      it 'should not overwrite the index patterns' do
+        subject.register
+        sleep(1)
 
+        template = get_template(@es, "logstash")
+        expect(template).to have_index_pattern("overwrite-*")
+      end
+    end
 
-        if ESHelper.es_version_satisfies?(">= 8.0")
-          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
-        elsif ESHelper.es_version_satisfies?(">= 7.0")
-          let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
-        else
-          let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
-        end
-        let (:ilm_enabled) { true }
-        let (:ilm_policy_name) { "logstash-policy-custom-policy" }
-        let (:policy) { small_max_doc_policy }
 
-        before :each do
-          put_policy(@es,ilm_policy_name, policy)
-        end
+    context 'with a custom template' do
+      let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
+      let (:index) { ilm_rollover_alias }
+      let(:expected_index) { index }
+      let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
+                                    "template" => template,
+                                    "ilm_rollover_alias" => ilm_rollover_alias)}
 
-        it_behaves_like 'an ILM enabled Logstash'
 
-        it 'should create the rollover alias' do
+      if ESHelper.es_version_satisfies?(">= 8.0")
+        let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+      elsif ESHelper.es_version_satisfies?(">= 7.0")
+        let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
+      else
+        let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
+      end
+      let (:ilm_enabled) { true }
+      let (:ilm_policy_name) { "logstash-policy-custom-policy" }
+      let (:policy) { small_max_doc_policy }
+
+      before :each do
+        put_policy(@es,ilm_policy_name, policy)
+      end
+
+      it_behaves_like 'an ILM enabled Logstash'
+
+      it 'should create the rollover alias' do
+        expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey
+        subject.register
+        sleep(1)
+        expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
+        expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
+      end
+
+      context 'when the custom rollover alias already exists' do
+        it 'should ignore the already exists error' do
           expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey
+          put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias)
+          expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
           subject.register
           sleep(1)
-          expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
           expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
         end
 
-        context 'when the custom rollover alias already exists' do
-          it 'should ignore the already exists error' do
-            expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey
-            put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias)
-            expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
-            subject.register
-            sleep(1)
-            expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
-          end
+      end
 
-        end
+      it 'should write the ILM settings into the template' do
+        subject.register
+        sleep(1)
+
+        template = get_template(@es, ilm_rollover_alias)
+        expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
+        expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
+        expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
+      end
+
+      context 'with a different template_name' do
+        let (:template_name) { "logstash_custom_template_name" }
+        let (:settings) { super().merge('template_name' => template_name)}
+
+        it_behaves_like 'an ILM enabled Logstash'
 
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-
-          template = get_template(@es, ilm_rollover_alias)
+          template = get_template(@es, template_name)
           expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
           expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
           expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
         end
-
-        context 'with a different template_name' do
-          let (:template_name) { "logstash_custom_template_name" }
-          let (:settings) { super().merge('template_name' => template_name)}
-
-          it_behaves_like 'an ILM enabled Logstash'
-
-          it 'should write the ILM settings into the template' do
-            subject.register
-            sleep(1)
-            template = get_template(@es, template_name)
-            expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
-            expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
-            expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
-          end
-        end
-
       end
+
     end
+  end
 
-    context 'when ilm_enabled is set to "auto"' do
-      let (:ilm_enabled) { 'auto' }
+  context 'when ilm_enabled is set to "auto"' do
+    let (:ilm_enabled) { 'auto' }
 
-      if ESHelper.es_version_satisfies?(">=7.0")
-        context 'when Elasticsearch is version 7 or above' do
-          it_behaves_like 'an ILM enabled Logstash'
-        end
+    if ESHelper.es_version_satisfies?(">=7.0")
+      context 'when Elasticsearch is version 7 or above' do
+        it_behaves_like 'an ILM enabled Logstash'
       end
+    end
 
-      if ESHelper.es_version_satisfies?('< 7.0')
-        context 'when Elasticsearch is version 7 or below' do
-          it_behaves_like 'an ILM disabled Logstash'
-        end
+    if ESHelper.es_version_satisfies?('< 7.0')
+      context 'when Elasticsearch is version 7 or below' do
+        it_behaves_like 'an ILM disabled Logstash'
       end
     end
+  end
 
-    context 'when ilm_enabled is the default' do
-      let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
+  context 'when ilm_enabled is the default' do
+    let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
 
-      if ESHelper.es_version_satisfies?(">=7.0")
-        context 'when Elasticsearch is version 7 or above' do
-          it_behaves_like 'an ILM enabled Logstash'
-        end
+    if ESHelper.es_version_satisfies?(">=7.0")
+      context 'when Elasticsearch is version 7 or above' do
+        it_behaves_like 'an ILM enabled Logstash'
       end
+    end
 
-      if ESHelper.es_version_satisfies?('< 7.0')
-        context 'when Elasticsearch is version 7 or below' do
-          it_behaves_like 'an ILM disabled Logstash'
-        end
+    if ESHelper.es_version_satisfies?('< 7.0')
+      context 'when Elasticsearch is version 7 or below' do
+        it_behaves_like 'an ILM disabled Logstash'
       end
     end
+  end
 
-    context 'with ilm disabled' do
-      let (:settings) { super().merge('ilm_enabled' => false )}
-
-      it_behaves_like 'an ILM disabled Logstash'
-    end
+  context 'with ilm disabled' do
+    let (:settings) { super().merge('ilm_enabled' => false )}
 
-    context 'with ilm disabled using a string' do
-      let (:settings) { super().merge('ilm_enabled' => 'false' )}
+    it_behaves_like 'an ILM disabled Logstash'
+  end
 
-      it_behaves_like 'an ILM disabled Logstash'
-    end
+  context 'with ilm disabled using a string' do
+    let (:settings) { super().merge('ilm_enabled' => 'false' )}
 
+    it_behaves_like 'an ILM disabled Logstash'
   end
-end
+
+end