logstash-output-elasticsearch 12.0.1-java → 12.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e4412a465fe0bd0f91b4c6ac11e4fdaf30ead8eff0888f22e0b030af9b70207
-  data.tar.gz: 36da192cd2b436f94a58c4e413e388db129c55eaa3def6b0a4840df79fe1a35b
+  metadata.gz: f5b6e330ff67352acd72041d917885768b47d2808bce5d116e555d61eb89e6b5
+  data.tar.gz: f450de9ab1542645aea2072702252d0c1ecba00110ebbf035571319e83dd1468
 SHA512:
-  metadata.gz: 7e71766366f3d934883f47df25b3632a460cac2c868f0bc5994f18acec09426b5e30769fe3332722a85940421b9309fdad45dfd259ae5fa725766112f2c2d75d
-  data.tar.gz: 27a0ad1475ad90efb62ebd3dd657ec9a6681eb8404b4891bb4bb0bcef6b8de48d6c906ce7f8faed42f511c1167f8833a65c4e4b8b82d0b7015c2c0bb1a2ee7f7
+  metadata.gz: 14a820573d8d16ea69ad91e281c36a0c4c4aac863053596a50c72b065803f7e2afa3f849f662d185e2baebd089605c8dc77d445985ddf7a43afceb7e9d20b1be
+  data.tar.gz: 58b5159e08b621602b892f3ea7d7911f47f1236144e35980582e6dda511edd81350d850c3ae73abde6cd298c664f737ba0a64de61a5c35a963e40bec28f692f2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 12.0.3
+ - Demote connection log entry from WARN to INFO connection during register phase [#1211](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1211)
+
+## 12.0.2
+ - Properly handle http code 413 (Payload Too Large) [#1199](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199)
+
 ## 12.0.1
  - Remove irrelevant log warning about elastic stack version [#1200](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1200)
 

data/docs/index.asciidoc

@@ -196,7 +196,22 @@ This plugin uses the Elasticsearch bulk API to optimize its imports into Elastic
 either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
 request are handled differently than error codes for individual documents.
 
-HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
+
+HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely,
+including 413 (Payload Too Large) responses.
+
+If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead:
+
+[source,ruby]
+-----
+output {
+  elasticsearch {
+    hosts => ["localhost:9200"]
+    dlq_custom_codes => [413]  # Send 413 errors to DLQ instead of retrying
+  }
+-----
+
+This will capture oversized payloads in the DLQ for analysis rather than retrying them.
 
 The following document errors are handled as follows:
 
@@ -612,8 +627,7 @@ Elasticsearch with the same ID.
 
 NOTE: This option is deprecated due to the
 https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal
-of types in Elasticsearch 6.0]. It will be removed in the next major version of
-Logstash.
+of types in Elasticsearch 6.0].
 
 NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`.
 
@@ -622,9 +636,7 @@ similar events to the same 'type'. String expansion `%{foo}` works here.
 If you don't set a value for this option:
 
 - for elasticsearch clusters 8.x: no value will be used;
-- for elasticsearch clusters 7.x: the value of '_doc' will be used;
-- for elasticsearch clusters 6.x: the value of 'doc' will be used;
-- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
+- for elasticsearch clusters 7.x: the value of '_doc' will be used.
 
 [id="plugins-{type}s-{plugin}-ecs_compatibility"]
 ===== `ecs_compatibility`
@@ -1039,8 +1051,6 @@ NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
 
 This setting asks Elasticsearch for the list of all cluster nodes and adds them
 to the hosts list.
-For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will
-be added to the hosts list, excluding master-only nodes.
 
 [id="plugins-{type}s-{plugin}-sniffing_delay"]
 ===== `sniffing_delay`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -127,7 +127,6 @@ module LogStash module Outputs class ElasticSearch
           value.to_s == 'true'
         when 'manage_template'
           value.to_s == 'false'
-        when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
               shared_params.include?(name) ||

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -76,11 +76,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
       end
 
-      # 404s are excluded because they are valid codes in the case of
-      # template installation. We might need a better story around this later
-      # but for our current purposes this is correct
       code = resp.code
-      if code < 200 || code > 299 && code != 404
+      if code < 200 || code > 299 # assume anything not 2xx is an error that the layer above needs to interpret
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -52,7 +52,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     ROOT_URI_PATH = '/'.freeze
     LICENSE_PATH = '/_license'.freeze
 
-    VERSION_6_TO_7 = ::Gem::Requirement.new([">= 6.0.0", "< 7.0.0"])
     VERSION_7_TO_7_14 = ::Gem::Requirement.new([">= 7.0.0", "< 7.14.0"])
 
     DEFAULT_OPTIONS = {
@@ -253,13 +252,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def health_check_request(url)
       logger.debug("Running health check to see if an Elasticsearch connection is working",
                    :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
-      begin
-        response = perform_request_to_url(url, :head, @healthcheck_path)
-        return response, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
-        return nil, e
-      end
+      response = perform_request_to_url(url, :head, @healthcheck_path)
+      return response, nil
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
+      return nil, e
     end
 
     def healthcheck!(register_phase = true)
@@ -289,7 +286,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           raise root_bad_code_err if root_bad_code_err
 
           # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
+          if register_phase
+            logger.info("Connected to ES instance", url: url.sanitized.to_s)
+          else
+            logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
+          end
 
           # We check its ES version
           es_version, build_flavor = parse_es_version(root_response)
@@ -312,13 +313,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def get_root_path(url, params={})
-      begin
-        resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
-        return resp, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
-        return nil, e
-      end
+      resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
+      return resp, nil
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
+      return nil, e
     end
 
     def test_serverless_connection(url, root_response)
@@ -550,11 +549,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       return false if version_info['version'].nil?
 
       version = ::Gem::Version.new(version_info["version"]['number'])
-      return false if version < ::Gem::Version.new('6.0.0')
+      return false if version < ::Gem::Version.new('7.0.0')
 
-      if VERSION_6_TO_7.satisfied_by?(version)
-        return valid_tagline?(version_info)
-      elsif VERSION_7_TO_7_14.satisfied_by?(version)
+      if VERSION_7_TO_7_14.satisfied_by?(version)
         build_flavor = version_info["version"]['build_flavor']
         return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
       else

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -182,22 +182,20 @@ module LogStash; module Outputs; class ElasticSearch;
     def bulk_send(body_stream, batch_actions)
       params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
 
-      response = @pool.post(@bulk_path, params, body_stream.string)
-
-      @bulk_response_metrics.increment(response.code.to_s)
-
-      case response.code
-      when 200 # OK
-        LogStash::Json.load(response.body)
-      when 413 # Payload Too Large
+      begin
+        response = @pool.post(@bulk_path, params, body_stream.string)
+        @bulk_response_metrics.increment(response.code.to_s)
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        @bulk_response_metrics.increment(e.response_code.to_s)
+        raise e unless e.response_code == 413
+        # special handling for 413, treat it as a document level issue
         logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
-        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
-      else
-        url = ::LogStash::Util::SafeURI.new(response.final_url)
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-          response.code, url, body_stream.to_s, response.body
-        )
+        return emulate_batch_error_response(batch_actions, 413, 'payload_too_large')
+      rescue => e # it may be a network issue instead, re-raise
+        raise e
       end
+
+      LogStash::Json.load(response.body)
     end
 
     def emulate_batch_error_response(actions, http_code, reason)
@@ -411,6 +409,9 @@ module LogStash; module Outputs; class ElasticSearch;
     def exists?(path, use_get=false)
       response = use_get ? @pool.get(path) : @pool.head(path)
       response.code >= 200 && response.code <= 299
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      return false if e.response_code == 404
+      raise e
     end
 
     def template_exists?(template_endpoint, name)
@@ -421,6 +422,8 @@ module LogStash; module Outputs; class ElasticSearch;
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      raise e unless e.response_code == 404
     end
 
     # ILM methods
@@ -432,17 +435,15 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      begin
-        @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
-        logger.info("Created rollover alias", name: alias_name)
-        # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        if e.response_code == 400
-            logger.info("Rollover alias already exists, skipping", name: alias_name)
-            return
-        end
-        raise e
+      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+      logger.info("Created rollover alias", name: alias_name)
+      # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      if e.response_code == 400
+        logger.info("Rollover alias already exists, skipping", name: alias_name)
+        return
       end
+      raise e
     end
 
     def get_xpack_info

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -21,13 +21,7 @@ module LogStash; module Outputs; class ElasticSearch
               "Serverless Elasticsearch cluster does not support Index Lifecycle Management.") if @ilm_enabled == 'auto'
             false
           elsif @ilm_enabled == 'auto'
-            if ilm_on_by_default?
-              ilm_alias_set?
-            else
-              @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`."\
-                " Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
-              false
-            end
+            ilm_alias_set?
           elsif @ilm_enabled.to_s == 'true'
             ilm_alias_set?
           else
@@ -42,10 +36,6 @@ module LogStash; module Outputs; class ElasticSearch
       default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
     end
 
-    def ilm_on_by_default?
-      maximum_seen_major_version >= 7
-    end
-
     def default_index?(index)
       index == @default_index
     end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -47,7 +47,7 @@ module LogStash; module Outputs; class ElasticSearch
     def self.add_ilm_settings_to_template(plugin, template)
       # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
       # definition - remove any existing definition of 'template'
-      template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8
+      template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version == 7
       template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
       settings = resolve_template_settings(plugin, template)
       if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])

data/lib/logstash/outputs/elasticsearch.rb

@@ -14,39 +14,16 @@ require "set"
 # .Compatibility Note
 # [NOTE]
 # ================================================================================
-# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
-# called `http.content_type.required`. If this option is set to `true`, and you
-# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
-# plugin to version 6.2.5 or higher.
-#
-# ================================================================================
 #
 # This plugin is the recommended method of storing logs in Elasticsearch.
 # If you plan on using the Kibana web interface, you'll want to use this output.
 #
-# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
-# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
-# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-# to upgrade Logstash in lock-step.
+# This output only speaks the HTTP protocol.
 #
 # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
 #
-# ==== Template management for Elasticsearch 5.x
-# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
-# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
-# behavior.
-#
-# ** Users installing ES 5.x and LS 5.x **
-# This change will not affect you and you will continue to use the ES defaults.
-#
-# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
-# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
-# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
-# the new template is installed.
-#
 # ==== Retry Policy
 #
-# The retry policy has changed significantly in the 2.2.0 release.
 # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
 # either partial or total failures.
 #
@@ -129,8 +106,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # - delete: deletes a document by id (An id is required for this action)
   # - create: indexes a document, fails if a document by that id already exists in the index.
   # - update: updates a document by id. Update has a special case where you can upsert -- update a
-  #   document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
-  #   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
+  #   document if not already present. See the `upsert` option.
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
@@ -148,7 +124,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   config :document_type,
     :validate => :string,
-    :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
+    :deprecated => "Document types were deprecated in Elasticsearch 7.0, and no longer configurable since 8.0. You should avoid this feature."
 
   # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
   # Logstash's startup if one with the name `template_name` does not already exist.
@@ -483,7 +459,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         join_value = event.get(@join_field)
         parent_value = event.sprintf(@parent)
         event.set(@join_field, { "name" => join_value, "parent" => parent_value })
-        params[routing_field_name] = event.sprintf(@parent)
+        params[:routing] = event.sprintf(@parent)
       else
         params[:parent] = event.sprintf(@parent)
       end
@@ -495,7 +471,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
       params[:_script] = event.sprintf(@script) if @script != ""
-      params[retry_on_conflict_action_name] = @retry_on_conflict
+      params[:retry_on_conflict] = @retry_on_conflict
     end
 
     event_control = event.get("[@metadata][_ingest_document]")
@@ -552,7 +528,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     params = {
         :_id => resolve_document_id(event, event_id),
         :_index => resolve_index!(event, event_index),
-        routing_field_name => resolve_routing(event, event_routing)
+        :routing => resolve_routing(event, event_routing)
     }
 
     target_pipeline = resolve_pipeline(event, event_pipeline)
@@ -615,16 +591,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     require "logstash/outputs/elasticsearch/#{name}"
   end
 
-  def retry_on_conflict_action_name
-    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
-  end
-
-  def routing_field_name
-    :routing
-  end
-
   # Determine the correct value for the 'type' field for the given event
-  DEFAULT_EVENT_TYPE_ES6 = "doc".freeze
   DEFAULT_EVENT_TYPE_ES7 = "_doc".freeze
 
   def get_event_type(event)
@@ -633,9 +600,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
              event.sprintf(@document_type)
            else
              major_version = maximum_seen_major_version
-             if major_version == 6
-               DEFAULT_EVENT_TYPE_ES6
-             elsif major_version == 7
+             if major_version == 7
                DEFAULT_EVENT_TYPE_ES7
              else
                nil
@@ -653,9 +618,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    # always set type for ES 6
-    # for ES 7 only set it if the user defined it
-    (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
+    # never use event type unless
+    # ES is 7.x and the user defined it
+    maximum_seen_major_version == 7 && @document_type
   end
 
   def install_template

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.0.1'
+  s.version         = '12.0.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -1,15 +1,18 @@
 require_relative './spec_helper'
 
 require 'elasticsearch'
-require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
-require_relative "support/elasticsearch/api/actions/get_alias"
-require_relative "support/elasticsearch/api/actions/put_alias"
-require_relative "support/elasticsearch/api/actions/get_ilm_policy"
-require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
 require 'cabin'
 
+# remove this condition and support package once plugin starts consuming elasticsearch-ruby v8 client
+# in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available
+unless elastic_ruby_v8_client_available?
+  require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
+  require_relative "support/elasticsearch/api/actions/get_ilm_policy"
+  require_relative "support/elasticsearch/api/actions/put_ilm_policy"
+end
+
 module ESHelper
   def get_host_port
     if ENV["INTEGRATION"] == "true"
@@ -20,8 +23,12 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
-      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+    if elastic_ruby_v8_client_available?
+      Elasticsearch::Client.new(:hosts => [get_host_port])
+    else
+      Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+        allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+      end
     end
   end
 
@@ -30,8 +37,6 @@ module ESHelper
       nil
     elsif ESHelper.es_version_satisfies?(">=7")
       "_doc"
-    else
-      "doc"
     end
   end
 
@@ -70,7 +75,7 @@ module ESHelper
   end
 
   RSpec::Matchers.define :have_hits do |expected|
-    hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total)
+    hits_count_path = %w(hits total value)
 
     match do |actual|
       @actual_hits_count = actual&.dig(*hits_count_path)
@@ -128,31 +133,36 @@ module ESHelper
   end
 
   def get_policy(client, policy_name)
-    client.get_ilm_policy(name: policy_name)
+    if elastic_ruby_v8_client_available?
+      client.index_lifecycle_management.get_lifecycle(policy: policy_name)
+    else
+      client.get_ilm_policy(name: policy_name)
+    end
   end
 
   def put_policy(client, policy_name, policy)
-    client.put_ilm_policy({:name => policy_name, :body=> policy})
-  end
-
-  def put_alias(client, the_alias, index)
-    body = {
-        "aliases" => {
-            index => {
-                "is_write_index"=>  true
-            }
-        }
-    }
-    client.put_alias({name: the_alias, body: body})
+    if elastic_ruby_v8_client_available?
+      client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy})
+    else
+      client.put_ilm_policy({:name => policy_name, :body=> policy})
+    end
   end
 
   def clean_ilm(client)
-    client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key)  if key =~ /logstash-policy/ }
+    if elastic_ruby_v8_client_available?
+      client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ }
+    else
+      client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ }
+    end
   end
 
   def supports_ilm?(client)
     begin
-      client.get_ilm_policy
+      if elastic_ruby_v8_client_available?
+        client.index_lifecycle_management.get_lifecycle
+      else
+        client.get_ilm_policy
+      end
       true
     rescue
       false
@@ -214,8 +224,6 @@ module ESHelper
       template['template']['mappings']
     elsif ESHelper.es_version_satisfies?(">=7")
       template['mappings']
-    else
-      template['mappings']["_default_"]
     end
   end
 end

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -14,7 +14,7 @@ end
     let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
 
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+    let(:type) { "_doc" }
     let(:event_count) { 10000 + rand(500) }
     # mix the events with valid and invalid UTF-8 payloads
     let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
@@ -59,10 +59,10 @@ end
         response = http_client.get("#{index_url}/_search?q=*&size=1000")
         result = LogStash::Json.load(response.body)
         result["hits"]["hits"].each do |doc|
-          if ESHelper.es_version_satisfies?("< 8")
-            expect(doc["_type"]).to eq(type)
-          else
+          if ESHelper.es_version_satisfies?(">= 8")
             expect(doc).not_to include("_type")
+          else
+            expect(doc["_type"]).to eq(type)
           end
           expect(doc["_index"]).to eq(index)
         end
@@ -78,4 +78,4 @@ end
 
     it_behaves_like("an indexer")
   end
-end
+end

data/spec/integration/outputs/delete_spec.rb

@@ -39,12 +39,12 @@ describe "Versioned delete", :integration => true do
     it "should ignore non-monotonic external version updates" do
       id = "ev2"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r2 = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
       expect(r2['_version']).to eq(99)
       expect(r2['_source']['message']).to eq('foo')
     end
@@ -52,12 +52,12 @@ describe "Versioned delete", :integration => true do
     it "should commit monotonic external version updates" do
       id = "ev3"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect { es.get(:index => 'logstash-delete', :id => id, :refresh => true) }.to raise_error(get_expected_error_class)
     end
   end
 end