logstash-output-elasticsearch 9.3.2-java → 9.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ae86a048dbbbc543991c0f0940002f81c13ef3619a3ab0dcfbd73030b4992be
-  data.tar.gz: 8ccb17687bc962888d4a0f2eb94d62a29f27ec7460078deb98f4a789ef9ac13f
+  metadata.gz: 7a6c544ed5e43a7c722849336fedcd792b7f843e3ed12de12a2898d019a56084
+  data.tar.gz: 9db3b25912b6d104752a6eb73cc4d12fe900a41b7fad369720fbb56cf0d087cf
 SHA512:
-  metadata.gz: 89be3b3bf2a12d130ae7d6c8da1f9961d827b12814901c180a9caa966ca15bb7b8ae2a1df0b79906edd48edaca7888ed9dd8f6c132a5b52170641a777fcd0e54
-  data.tar.gz: 890f65b47189d95c7087095e7f1a0cfad288499428de0002cd260f40a67788b43a9f4b7c6a643d9428ee25cd5e326fe336191bda6e674f5eea264b21f413c97f
+  metadata.gz: f785bc20eaad291c9a5eb30e4aee4737ce55dbcc879267b71fe0a1bc13e938410374080c9c28094ae28406894890a7710b04fa9eea5aab1ec854efd18db5aae2
+  data.tar.gz: 50aef5de97548888460009d7148530b65b9b5ea62ae49bc7ee4ec31935e51bb7d4a870690d23015738682072c9c5fadac2390af19a633a5702f6dd1aef60793c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.4.0
+  - Added 'auto' setting for ILM [#841](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/841)
+
 ## 9.3.2
   - Fixed sniffing support for 7.x [#827](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/827)
 

data/docs/index.asciidoc

@@ -94,9 +94,8 @@ happens, the problem is logged as a warning, and the event is dropped. See
 <<dead-letter-queues>> for more information about processing events in the DLQ.
 
 [id="plugins-{type}s-{plugin}-ilm"]
-==== Index Lifecycle Management (Beta)
+==== Index Lifecycle Management
 
-beta[]
 
 [NOTE]
 The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
@@ -104,18 +103,22 @@ The Index Lifecycle Management feature requires plugin version `9.3.1` or higher
 [NOTE]
 This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
 
-Logstash can use the {ref}/index-lifecycle-management.html[Index Lifecycle Management] to automate the management of indices over time.
+Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle Management] to automate the management of indices over time.
 
-To configure the Elasticsearch output to use Index Lifecycle Management, set the `ilm_enabled` flag to true in the output definition:
+The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting.
+The `ilm_enabled` flag can be set to `true`, `false` (default) or `auto`.
 
-[source,ruby]
-    output {
-      elasticsearch {
-        ilm_enabled => true
-      }
-    }
+The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting.
+The `ilm_enabled` flag can be set to `true`, `false` (default) or `auto`.
 
-This will overwrite the index settings and adjust the Logstash template to write the necessary settings for the template
+Setting `ilm_enabled` to `auto` will automatically detect whether the
+Elasticsearch instance is version `7.0.0` and above and has Index Lifecycle Management enabled, and will enable its usage if so.
+
+Setting `ilm_enabled` to `true` will enable the Index Lifecycle Management feature if it is available on the
+Elasticsearch cluster - the plugin will not start if `ilm_enabled` is set to `true` and the Elasticsearch cluster does
+not support ILM.
+
+Enabling ILM support will overwrite the index settings and adjust the Logstash template to write the necessary settings for the template
 to support index lifecycle management, including the index policy and rollover alias to be used.
 
 Logstash will create a rollover alias for the indices to be written to, including a pattern for how the actual indices will be named, and unless an ILM policy that already exists has been specified,
@@ -133,9 +136,9 @@ See config below for an example:
     output {
       elasticsearch {
         ilm_enabled => true
-        ilm_rollover_alias: "custom"
-        ilm_pattern: "000001"
-        ilm_policy: "custom_policy"
+        ilm_rollover_alias => "custom"
+        ilm_pattern => "000001"
+        ilm_policy => "custom_policy"
       }
     }
 
@@ -192,7 +195,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-ilm_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ilm_enabled>> |<<string,string>>, one of `["true", "false", "auto"]`|No
 | <<plugins-{type}s-{plugin}-ilm_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ilm_policy>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ilm_rollover_alias>> |<<string,string>>|No
@@ -374,10 +377,16 @@ Enable gzip compression on requests. Note that response compression is on by def
 [id="plugins-{type}s-{plugin}-ilm_enabled"]
 ===== `ilm_enabled`
 
-  * Value type is <<boolean,boolean>>
+  * Value can be any of: `true`, `false`, `auto`
   * Default value is `false`
 
-Setting this flag to `true` will enable indices to be managed by the Index Lifecycle Management feature in Elasticsearch.
+The default setting of `false` will disable the Index Lifecycle Management feature.
+
+Setting this flag to `true` will enable Index Lifecycle Management feature, if the Elasticsearch cluster supports it. This is required
+to enable Index Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`.
+
+Setting this flag to `auto` will automatically enable the Index Lifecycle Management feature, if the Elasticsearch
+ cluster is running Elasticsearch version `7.0.0` or higher with the ILM feature enabled, and disable it otherwise.
 
 NOTE: This feature requires a Basic License or above to be installed on an Elasticsearch cluster version 6.6.0 or later
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -47,7 +47,6 @@ module LogStash; module Outputs; class ElasticSearch;
           sleep_interval = next_sleep_interval(sleep_interval)
         end
         if successful_connection?
-          verify_ilm_readiness if ilm_enabled?
           install_template
           setup_ilm if ilm_enabled?
         end
@@ -349,6 +348,10 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
+    def default_index?(index)
+      @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
+    end
+
     def dlq_enabled?
       # TODO there should be a better way to query if DLQ is enabled
       # See more in: https://github.com/elastic/logstash/issues/8064

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -5,6 +5,7 @@ module LogStash; module Outputs; class ElasticSearch
 
     DEFAULT_INDEX_NAME = "logstash-%{+YYYY.MM.dd}"
     DEFAULT_POLICY = "logstash-policy"
+    DEFAULT_ROLLOVER_ALIAS = 'logstash'
 
     def self.included(mod)
       # The index to write events to. This can be dynamic using the `%{foo}` syntax.
@@ -146,10 +147,10 @@ module LogStash; module Outputs; class ElasticSearch
       # ILM configurations (beta)
       # -----
       # Flag for enabling Index Lifecycle Management integration.
-      mod.config :ilm_enabled, :validate => :boolean, :default => false
+      mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => false
 
       # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
-      mod.config :ilm_rollover_alias, :validate => :string, :default => 'logstash'
+      mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS
 
       # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
       # {now/d} is date math, and will insert the appropriate value automatically.

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -5,41 +5,66 @@ module LogStash; module Outputs; class ElasticSearch
 
     def setup_ilm
       return unless ilm_enabled?
-      @logger.info("Using Index lifecycle management - this feature is currently in beta.")
-      @logger.warn "Overwriting supplied index name with rollover alias #{@ilm_rollover_alias}" if @index != LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
-      @index = ilm_rollover_alias
+      if default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
+        logger.warn("Overwriting supplied index #{@index} with rollover alias #{@ilm_rollover_alias}") unless default_index?(@index)
+        @index = @ilm_rollover_alias
+        maybe_create_rollover_alias
+        maybe_create_ilm_policy
+      end
+    end
 
-      maybe_create_rollover_alias
-      maybe_create_ilm_policy
+    def default_rollover_alias?(rollover_alias)
+      rollover_alias == LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS
     end
 
     def ilm_enabled?
-      @ilm_enabled
+      return @ilm_actually_enabled if defined?(@ilm_actually_enabled)
+      @ilm_actually_enabled =
+        begin
+          if @ilm_enabled == 'auto'
+            if ilm_on_by_default?
+              ilm_ready, error = ilm_ready?
+              if !ilm_ready
+                @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - #{error}")
+                false
+              else
+                true
+              end
+            else
+              @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
+              false
+            end
+          elsif @ilm_enabled.to_s == 'true'
+            ilm_ready, error = ilm_ready?
+            raise LogStash::ConfigurationError,"Index Lifecycle Management is set to enabled in Logstash, but cannot be used - #{error}"  unless ilm_ready
+            true
+          else
+            false
+          end
+        end
     end
 
-    def verify_ilm_readiness
-      return unless ilm_enabled?
+    def ilm_on_by_default?
+      maximum_seen_major_version >= 7
+    end
 
+    def ilm_ready?
       # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature
       # available and enabled.
       begin
         xpack = client.get_xpack_info
-        features = xpack["features"]
+        features = xpack.nil? || xpack.empty? ? nil : xpack["features"]
         ilm = features.nil? ? nil : features["ilm"]
-        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
-        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not available in your Elasticsearch cluster" unless ilm['available']
-        raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not enabled in your Elasticsearch cluster" unless ilm['enabled']
-
-        unless ilm_policy_default? || client.ilm_policy_exists?(ilm_policy)
-          raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance"
-        end
-
+        return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" if features.nil? || ilm.nil?
+        return false, "Index Lifecycle management is not available in your Elasticsearch cluster" unless ilm['available']
+        return false, "Index Lifecycle management is not enabled in your Elasticsearch cluster" unless ilm['enabled']
+        return true, nil
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible
         if e.response_code == 404
-          raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster"
+          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
         elsif e.response_code == 400
-          raise LogStash::ConfigurationError, "Index Lifecycle management is enabled in logstash, but not installed on your Elasticsearch cluster"
+          return false, "Index Lifecycle management is not installed on your Elasticsearch cluster"
         else
           raise e
         end
@@ -53,8 +78,10 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def maybe_create_ilm_policy
-      if ilm_policy_default? && !client.ilm_policy_exists?(ilm_policy)
-        client.ilm_policy_put(ilm_policy, policy_payload)
+      if ilm_policy_default?
+          client.ilm_policy_put(ilm_policy, policy_payload) unless client.ilm_policy_exists?(ilm_policy)
+      else
+        raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" unless client.ilm_policy_exists?(ilm_policy)
       end
     end
 

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -3,7 +3,13 @@ module LogStash; module Outputs; class ElasticSearch
     # To be mixed into the elasticsearch plugin base
     def self.install_template(plugin)
       return unless plugin.manage_template
-      plugin.logger.info("Using mapping template from", :path => plugin.template)
+      if plugin.template.nil?
+        plugin.logger.info("Using default mapping template")
+      else
+        plugin.logger.info("Using mapping template from", :path => plugin.template)
+      end
+
+
       template = get_template(plugin.template, plugin.maximum_seen_major_version)
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_enabled?
       plugin.logger.info("Attempting to install template", :manage_template => template)
@@ -23,7 +29,6 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.add_ilm_settings_to_template(plugin, template)
-			plugin.logger.info("Overwriting index patterns, as ILM is enabled.")
       # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
       # definition - remove any existing definition of 'template'
       template.delete('template') if template.include?('template')
@@ -48,7 +53,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def self.read_template_file(template_path)
-      raise ArgumentError, "Template file '#{@template_path}' could not be found!" unless ::File.exists?(template_path)
+      raise ArgumentError, "Template file '#{template_path}' could not be found!" unless ::File.exists?(template_path)
       template_data = ::IO.read(template_path)
       LogStash::Json.load(template_data)
     end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.3.2'
+  s.version         = '9.4.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -67,6 +67,13 @@ module ESHelper
     end
   end
 
+  RSpec::Matchers.define :have_index_pattern do |expected|
+    match do |actual|
+      test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
+      test_against.include?(expected)
+    end
+  end
+
 
   def self.es_version_satisfies?(*requirement)
     es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
@@ -125,6 +132,38 @@ module ESHelper
       false
     end
   end
+
+  def max_docs_policy(max_docs)
+  {
+    "policy" => {
+      "phases"=> {
+        "hot" => {
+          "actions" => {
+            "rollover" => {
+              "max_docs" => max_docs
+            }
+          }
+        }
+      }
+    }
+  }
+  end
+
+  def max_age_policy(max_age)
+  {
+    "policy" => {
+      "phases"=> {
+        "hot" => {
+          "actions" => {
+            "rollover" => {
+              "max_age" => max_age
+            }
+          }
+        }
+      }
+    }
+  }
+  end
 end
 
 RSpec.configure do |config|

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -60,7 +60,11 @@ if ESHelper.es_version_satisfies?(">= 5")
     end
 
     it "sets the correct content-encoding header and body is compressed" do
-      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_template/, anything).and_call_original
+      # Allow xpack endpoint to be checked\
+      allow(subject.client.pool.adapter.client).to receive(:send).at_least(:once).with(anything, /_template/, anything).and_call_original
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_xpack/, anything).and_call_original
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /logstash/, anything).and_call_original
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_ilm/, anything).and_call_original
       expect(subject.client.pool.adapter.client).to receive(:send).
         with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
         and_call_original

data/spec/integration/outputs/ilm_spec.rb

@@ -1,47 +1,14 @@
 require_relative "../../../spec/es_spec_helper"
 
-shared_examples_for 'an Elasticsearch instance that does not support index lifecycle management' do
-  require "logstash/outputs/elasticsearch"
-
-  let (:ilm_enabled) { false }
-  let (:settings) {
-    {
-        "ilm_enabled" => ilm_enabled,
-        "hosts" => "#{get_host_port()}"
-    }
-  }
-
-  before :each do
-    require "elasticsearch"
-
-    # Clean ES of data before we start.
-    @es = get_client
-    clean(@es)
-  end
-
-  after :each do
-    clean(@es)
-  end
-
-  subject { LogStash::Outputs::ElasticSearch.new(settings) }
-
-  context 'when ilm is enabled in Logstash' do
-    let (:ilm_enabled) { true }
+shared_examples_for 'an ILM enabled Logstash' do
 
-    it 'should raise a configuration error' do
-      expect do
-        begin
-          subject.register
-          sleep(1)
-        ensure
-          subject.stop_template_installer
-        end
-      end.to raise_error(LogStash::ConfigurationError)
-    end
-  end
+  context 'with a policy with a maximum number of documents' do
+    let (:policy) { small_max_doc_policy }
+    let (:ilm_policy_name) { "custom-policy"}
+    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
 
-  context 'when ilm is disabled in Logstash' do
-    it 'should index documents normally' do
+    it 'should rollover when the policy max docs is reached' do
+      put_policy(@es,ilm_policy_name, policy)
       subject.register
 
       subject.multi_receive([
@@ -58,31 +25,38 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
                                 LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
                             ])
 
+      sleep(6)
+
+      subject.multi_receive([
+                                LogStash::Event.new("country" => "uk"),
+                                LogStash::Event.new("country" => "fr"),
+                                LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] })
+                            ])
+
       @es.indices.refresh
 
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
         r = @es.search
-        expect(r).to have_hits(6)
+        expect(r).to have_hits(9)
       end
       indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
         index_written = x['_index']
         res[index_written] += 1
       end
-      expect(indexes_written.count).to eq(1)
+      expect(indexes_written.count).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000002"]).to eq(3)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000003"]).to eq(3)
     end
   end
 
-end
-
-shared_examples_for 'an ILM enabled Logstash' do
-
-  context 'with a policy with a maximum number of documents' do
-    let (:policy) { small_max_doc_policy }
+  context 'with a policy where the maximum number of documents is not reached' do
+    let (:policy) { large_max_doc_policy }
     let (:ilm_policy_name) { "custom-policy"}
     let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
 
-    it 'should rollover when the policy max docs is reached' do
+    it 'should ingest into a single index when max docs is not reached' do
       put_policy(@es,ilm_policy_name, policy)
       subject.register
 
@@ -100,41 +74,53 @@ shared_examples_for 'an ILM enabled Logstash' do
                                 LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
                             ])
 
-      sleep(6)
-
-      subject.multi_receive([
-                                LogStash::Event.new("country" => "uk"),
-                                LogStash::Event.new("country" => "fr"),
-                                LogStash::Event.new("geoip" => { "location" => [ 0.1, 1.0 ] })
-                            ])
-
       @es.indices.refresh
 
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
         r = @es.search
-        expect(r).to have_hits(9)
+        expect(r).to have_hits(6)
       end
       indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
         index_written = x['_index']
         res[index_written] += 1
       end
-      expect(indexes_written.count).to eq(3)
-      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(3)
-      expect(indexes_written["#{expected_index}-#{todays_date}-000002"]).to eq(3)
-      expect(indexes_written["#{expected_index}-#{todays_date}-000003"]).to eq(3)
+      expect(indexes_written.count).to eq(1)
+      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(6)
     end
   end
+end
 
-  context 'with a policy where the maximum number of documents is not reached' do
-    let (:policy) { large_max_doc_policy }
-    let (:ilm_policy_name) { "custom-policy"}
+shared_examples_for 'an ILM disabled Logstash' do
+  it 'should not create a rollover alias' do
+    expect(@es.get_alias).to be_empty
+    subject.register
+    sleep(1)
+    expect(@es.get_alias).to be_empty
+  end
+
+  it 'should not install the default policy' do
+    subject.register
+    sleep(1)
+    expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+  end
+
+  it 'should not write the ILM settings into the template' do
+    subject.register
+    sleep(1)
+    expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
+    if ESHelper.es_version_satisfies?(">= 2")
+      expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil
+    end
+  end
+
+  context 'with an existing policy that will roll over' do
+    let (:policy) { small_max_doc_policy }
+    let (:ilm_policy_name) { "3_docs"}
     let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
 
-    it 'should ingest into a single index when max docs is not reached' do
-      put_policy(@es,ilm_policy_name, policy)
+    it 'should not roll over indices' do
       subject.register
-
       subject.multi_receive([
                                 LogStash::Event.new("message" => "sample message here"),
                                 LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
@@ -161,11 +147,82 @@ shared_examples_for 'an ILM enabled Logstash' do
         res[index_written] += 1
       end
       expect(indexes_written.count).to eq(1)
-      expect(indexes_written["#{expected_index}-#{todays_date}-000001"]).to eq(6)
+      expect(indexes_written.values.first).to eq(6)
+    end
+  end
+
+  context 'with a custom template name' do
+    let (:template_name) { "custom_template_name" }
+    let (:settings) { super.merge('template_name' => template_name)}
+
+    it 'should not write the ILM settings into the template' do
+      subject.register
+      sleep(1)
+
+      expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("logstash-*")
+      if ESHelper.es_version_satisfies?(">= 2")
+        expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']).to be_nil
+      end
     end
   end
 end
 
+shared_examples_for 'an Elasticsearch instance that does not support index lifecycle management' do
+  require "logstash/outputs/elasticsearch"
+
+  let (:ilm_enabled) { false }
+  let (:settings) {
+    {
+        "hosts" => "#{get_host_port()}"
+    }
+  }
+
+  before :each do
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    clean(@es)
+  end
+
+  after :each do
+    clean(@es)
+  end
+
+  subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+  context 'when ilm is enabled in Logstash' do
+    let (:settings) { super.merge!({ 'ilm_enabled' => true }) }
+
+    it 'should raise a configuration error' do
+      expect do
+        begin
+          subject.register
+          sleep(1)
+        ensure
+          subject.stop_template_installer
+        end
+      end.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context 'when ilm is disabled in Logstash' do
+    let (:settings) { super.merge!({ 'ilm_enabled' => false }) }
+
+    it_behaves_like 'an ILM disabled Logstash'
+  end
+
+  context 'when ilm is set to auto in Logstash' do
+    let (:settings) { super.merge!({ 'ilm_enabled' => 'auto' }) }
+
+    it_behaves_like 'an ILM disabled Logstash'
+  end
+
+  context 'when ilm is not set in Logstash' do
+    it_behaves_like 'an ILM disabled Logstash'
+  end
+
+end
 
 if ESHelper.es_version_satisfies?("<= 6.5")
   describe 'Pre-ILM versions of Elasticsearch', :integration => true do
@@ -179,9 +236,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
   end
 
   describe 'Elasticsearch has index lifecycle management enabled', :distribution => 'xpack', :integration => true do
+
     DEFAULT_INTERVAL = '600s'
 
-    require "logstash/outputs/elasticsearch"
     let (:ilm_enabled) { true }
 
     let (:settings) {
@@ -190,38 +247,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           "hosts" => "#{get_host_port()}"
       }
     }
-    let (:policy) { small_max_doc_policy }
-
-
-    let (:small_max_doc_policy) {
-      {"policy" => {
-          "phases"=> {
-              "hot" => {
-                  "actions" => {
-                      "rollover" => {
-                          "max_docs" => "3"
-                      }
-                  }
-              }
-          }
-      }}
-    }
-
-    let (:large_max_doc_policy) {
-      {"policy" => {
-          "phases"=> {
-              "hot" => {
-                  "actions" => {
-                      "rollover" => {
-                          "max_docs" => "1000000"
-                      }
-                  }
-              }
-          }
-      }}
-    }
-
-
+    let (:small_max_doc_policy) { max_docs_policy(3) }
+    let (:large_max_doc_policy) { max_docs_policy(1000000) }
+    let (:expected_index) { LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS }
 
     subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
@@ -250,11 +278,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
       clean(@es)
     end
 
-
     context 'with ilm enabled' do
       let (:ilm_enabled) { true }
 
-
       context 'when using the default policy' do
         context 'with a custom pattern' do
           let (:settings) { super.merge("ilm_pattern" => "000001")}
@@ -267,7 +293,6 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           end
         end
 
-
         it 'should install it if it is not present' do
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
           subject.register
@@ -283,10 +308,8 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
         end
 
-
         it 'should ingest into a single index' do
           subject.register
-
           subject.multi_receive([
                                     LogStash::Event.new("message" => "sample message here"),
                                     LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
@@ -321,18 +344,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"new_one"}
         let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
-        let (:policy) {{
-            "policy" => {
-                "phases"=> {
-                    "hot" => {
-                        "actions" => {
-                            "rollover" => {
-                                "max_docs" => "3"
-                            }
-                        }
-                    }
-                }
-            }}}
+        let (:policy) { small_max_doc_policy }
 
         before do
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
@@ -349,18 +361,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
       context 'when using a time based policy' do
         let (:ilm_policy_name) {"new_one"}
         let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
-        let (:policy) {{
-            "policy" => {
-                "phases"=> {
-                    "hot" => {
-                        "actions" => {
-                            "rollover" => {
-                                "max_age" => "1d"
-                            }
-                        }
-                    }
-                }
-            }}}
+        let (:policy) { max_age_policy("1d") }
 
         before do
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
@@ -373,9 +374,8 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
         end
       end
-      context 'with the default template' do
-        let(:expected_index) { "logstash" }
 
+      context 'with the default template' do
         it 'should create the rollover alias' do
           expect(@es.indices.exists_alias(index: expected_index)).to be_falsey
           subject.register
@@ -387,7 +387,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"])
+          expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
           expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['name']).to eq("logstash-policy")
           expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['rollover_alias']).to eq("logstash")
         end
@@ -411,7 +411,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         end
         let (:ilm_enabled) { true }
         let (:ilm_policy_name) { "custom-policy" }
-
+        let (:policy) { small_max_doc_policy }
 
         before :each do
           put_policy(@es,ilm_policy_name, policy)
@@ -442,7 +442,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"])
+          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]).to have_index_pattern("#{ilm_rollover_alias}-*")
           expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
           expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
         end
@@ -456,7 +456,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           it 'should write the ILM settings into the template' do
             subject.register
             sleep(1)
-            expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["#{ilm_rollover_alias}-*"])
+            expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("#{ilm_rollover_alias}-*")
             expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
             expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
           end
@@ -465,78 +465,49 @@ if ESHelper.es_version_satisfies?(">= 6.6")
       end
     end
 
-    context 'with ilm disabled' do
-      let (:ilm_enabled) { false }
+    context 'when ilm_enabled is set to "auto"' do
+      let (:ilm_enabled) { 'auto' }
 
-      it 'should not create a rollover alias' do
-        expect(@es.get_alias).to be_empty
-        subject.register
-        sleep(1)
-        expect(@es.get_alias).to be_empty
-      end
-
-      it 'should not install the default policy' do
-        subject.register
-        sleep(1)
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      if ESHelper.es_version_satisfies?(">=7.0")
+        context 'when Elasticsearch is version 7 or above' do
+          it_behaves_like 'an ILM enabled Logstash'
+        end
       end
 
-      it 'should not write the ILM settings into the template' do
-        subject.register
-        sleep(1)
-        expect(@es.indices.get_template(name: "logstash")["logstash"]["index_patterns"]).to eq(["logstash-*"])
-        expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil
+      if ESHelper.es_version_satisfies?('< 7.0')
+        context 'when Elasticsearch is version 7 or below' do
+          it_behaves_like 'an ILM disabled Logstash'
+        end
       end
+    end
 
-      context 'with an existing policy that will roll over' do
-        let (:policy) { small_max_doc_policy }
-        let (:ilm_policy_name) { "3_docs"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
-
-        it 'should not roll over indices' do
-          subject.register
-          subject.multi_receive([
-                                    LogStash::Event.new("message" => "sample message here"),
-                                    LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-                                    LogStash::Event.new("somevalue" => 100),
-                                ])
-
-          sleep(6)
-
-          subject.multi_receive([
-                                    LogStash::Event.new("country" => "us"),
-                                    LogStash::Event.new("country" => "at"),
-                                    LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-                                ])
-
-          @es.indices.refresh
+    context 'when ilm_enabled is the default' do
+      let (:settings) { super.tap{|x|x.delete('ilm_enabled')}}
 
-          # Wait or fail until everything's indexed.
-          Stud::try(20.times) do
-            r = @es.search
-            expect(r).to have_hits(6)
-          end
-          indexes_written = @es.search['hits']['hits'].each_with_object(Hash.new(0)) do |x, res|
-            index_written = x['_index']
-            res[index_written] += 1
-          end
-          expect(indexes_written.count).to eq(1)
-          expect(indexes_written.values.first).to eq(6)
+      if ESHelper.es_version_satisfies?(">=7.0")
+        context 'when Elasticsearch is version 7 or above' do
+          it_behaves_like 'an ILM disabled Logstash'
         end
       end
 
-      context 'with a custom template name' do
-        let (:template_name) { "custom_template_name" }
-        let (:settings) { super.merge('template_name' => template_name)}
-
-        it 'should not write the ILM settings into the template' do
-          subject.register
-          sleep(1)
-          expect(@es.indices.get_template(name: template_name)[template_name]["index_patterns"]).to eq(["logstash-*"])
-          expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']).to be_nil
+      if ESHelper.es_version_satisfies?('< 7.0')
+        context 'when Elasticsearch is version 7 or below' do
+          it_behaves_like 'an ILM disabled Logstash'
         end
       end
+    end
+
+    context 'with ilm disabled' do
+      let (:settings) { super.merge('ilm_enabled' => false )}
 
+      it_behaves_like 'an ILM disabled Logstash'
     end
+
+    context 'with ilm disabled using a string' do
+      let (:settings) { super.merge('ilm_enabled' => 'false' )}
+
+      it_behaves_like 'an ILM disabled Logstash'
+    end
+
   end
 end

data/spec/integration/outputs/index_spec.rb

@@ -100,7 +100,13 @@ describe "indexing" do
           }}
       end
       # Allow template to be checked for existence/installed
-      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_template/, anything).and_call_original
+      allow(subject.client.pool.adapter.client).to receive(:send).at_least(:once).with(anything, /_template/, anything).and_call_original
+      # Allow xpack endpoint to be checked\
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_xpack/, anything).and_call_original
+      # Allow ilm policy to be checked for existence/installed
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /_ilm/, anything).and_call_original
+      # Allow write alias to be checked for existence/installed
+      allow(subject.client.pool.adapter.client).to receive(:send).with(anything, /logstash/, anything).and_call_original
       expect(subject.client.pool.adapter.client).to receive(:send).
         with(anything, anything, expected_manticore_opts).at_least(:once).
         and_call_original

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -19,6 +19,7 @@ describe LogStash::Outputs::ElasticSearch do
       allow(subject.client.pool).to receive(:start_sniffer)
       allow(subject.client.pool).to receive(:healthcheck!)
       allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
+      allow(subject.client).to receive(:get_xpack_info)
       subject.register
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -15,6 +15,7 @@ describe "whitelisting error types in expected behavior" do
     subject.register
 
     allow(subject.client).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject.client).to receive(:get_xpack_info)
     allow(subject.client).to receive(:bulk).and_return(
       {
         "errors" => true,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.3.2
+  version: 9.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-02-05 00:00:00.000000000 Z
+date: 2019-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement