logstash-output-elasticsearch 11.22.13-java → 12.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db023d3f7c10e52c9a7fdd9e2480ccd0f988c977357d86b3980d0698a21f5b10
-  data.tar.gz: f7d2ca8aa8dbb5acf848d5c20cca9911a6530fc5296d9765cc26df96df7554c0
+  metadata.gz: 983d99de3a0dcd5e58fb123e01ad8ee4c2396ce15fd565dda439e43c182b32e6
+  data.tar.gz: 8733e1a9b256b36e9f08a9be0803ca667817d7fe9f1e4cd1e090fe95882eb245
 SHA512:
-  metadata.gz: cd23a9523896c06623ec91a33ec820363af6df32149bf0d445ab1218e5b344668398b09cbcdc5b88b39efa4771d79f9de52f80d968105d991ce369cbafd634bc
-  data.tar.gz: e63f6a22b9eb7f308c164ba9e4d63f6685305f82d9e1ef9d4e86d7c6fedb1d19d9de4fd6949b82d534f5745c33d75ec35ae6b036fcc3b8efc2f41977a6948e07
+  metadata.gz: f9626da6b7d428b17a16b2874a8758e260e84af265636fabea8e35dec0777ecdeae1b662910d2e065eccc4742ef706765d87b6cf475a141544a78347c69953de
+  data.tar.gz: 7997cfb3b851130a0504c0907761aba37ea31ec35deb7fc4de19eaf7f1ae00cee9df7b2575d59ace2b08c19ad7710460c534467bb79f7019c3a125478d0de5e2

data/CHANGELOG.md

@@ -1,11 +1,14 @@
-## 11.22.13
- - Add headers reporting uncompressed size and doc count for bulk requests [#1217](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1217)
-
-## 11.22.12
- - Properly handle http code 413 (Payload Too Large) [#1199](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199)
-
-## 11.22.11
- - Remove irrelevant log warning about elastic stack version [#1202](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202)
+## 12.0.0
+  - SSL settings that were marked deprecated in version `11.14.0` are now marked obsolete, and will prevent the plugin from starting.
+  - These settings are:
+    - `cacert`, which should be replaced by `ssl_certificate_authorities`
+    - `keystore`, which should be replaced by `ssl_keystore_path`
+    - `keystore_password`, which should be replaced by `ssl_keystore_password`
+    - `ssl`, which should be replaced by `ssl_enabled`
+    - `ssl_certificate_verification`, which should be replaced by `ssl_verification_mode`
+    - `truststore`, which should be replaced by `ssl_truststore_path`
+    - `truststore_password`, which should be replaced by `ssl_truststore_password`
+    - [#1197](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1197)
 
 ## 11.22.10
  - Add `x-elastic-product-origin` header to Elasticsearch requests [#1195](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1195)

data/docs/index.asciidoc

@@ -196,22 +196,7 @@ This plugin uses the Elasticsearch bulk API to optimize its imports into Elastic
 either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
 request are handled differently than error codes for individual documents.
 
-
-HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely,
-including 413 (Payload Too Large) responses.
-
-If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead:
-
-[source,ruby]
------
-output {
-  elasticsearch {
-    hosts => ["localhost:9200"]
-    dlq_custom_codes => [413]  # Send 413 errors to DLQ instead of retrying
-  }
------
-
-This will capture oversized payloads in the DLQ for analysis rather than retrying them.
+HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
 
 The following document errors are handled as follows:
 
@@ -340,8 +325,10 @@ When a string value on an event contains one or more byte sequences that are not
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Output Configuration Options
 
-This plugin supports the following configuration options plus the
-<<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
+This plugin supports these configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed.
+Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -456,7 +443,7 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
   * There is no default value for this setting.
 
 Authenticate using Elasticsearch API key.
-Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>.
+Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
@@ -1339,98 +1326,24 @@ https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support
 blog] and {ref}/docs-index_.html#_version_types[Version types] in the
 Elasticsearch documentation.
 
-[id="plugins-{type}s-{plugin}-deprecated-options"]
-==== Elasticsearch Output Deprecated Configuration Options
-
-This plugin supports the following deprecated configurations.
+[id="plugins-{type}s-{plugin}-obsolete-options"]
+==== Elasticsearch Output Obsolete Configuration Options
 
-WARNING: Deprecated options are subject to removal in future releases.
+WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced.
+The plugin will fail to start if it contains any of these obsolete options.
 
-[cols="<,<,<",options="header",]
+[cols="<,<",options="header",]
 |=======================================================================
-|Setting|Input type|Replaced by
-| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
-| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
-| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
-| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
-| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_truststore_path>>
-| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_truststore_password>>
+|Setting|Replaced by
+| cacert | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| keystore | <<plugins-{type}s-{plugin}-ssl_keystore_path>>
+| keystore_password | <<plugins-{type}s-{plugin}-ssl_keystore_password>>
+| ssl | <<plugins-{type}s-{plugin}-ssl_enabled>>
+| ssl_certificate_verification | <<plugins-{type}s-{plugin}-ssl_verification_mode>>
+| truststore | <<plugins-{type}s-{plugin}-ssl_truststore_path>>
+| truststore_password | <<plugins-{type}s-{plugin}-ssl_truststore_password>>
 |=======================================================================
 
-
-[id="plugins-{type}s-{plugin}-cacert"]
-===== `cacert`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
-
-* Value type is a list of <<path,path>>
-* There is no default value for this setting.
-
-The .cer or .pem file to validate the server's certificate.
-
-[id="plugins-{type}s-{plugin}-keystore"]
-===== `keystore`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-The keystore used to present a certificate to the server.
-It can be either .jks or .p12
-
-NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
-
-[id="plugins-{type}s-{plugin}-keystore_password"]
-===== `keystore_password`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
-
-* Value type is <<password,password>>
-* There is no default value for this setting.
-
-Set the keystore password
-
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
-
-* Value type is <<boolean,boolean>>
-* There is no default value for this setting.
-
-Enable SSL/TLS secured communication to Elasticsearch cluster.
-Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
-If no explicit protocol is specified plain HTTP will be used.
-
-[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
-===== `ssl_certificate_verification`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `true`
-
-Option to validate the server's certificate. Disabling this severely compromises security.
-For more information on disabling certificate verification please read
-https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-
-[id="plugins-{type}s-{plugin}-truststore"]
-===== `truststore`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_path>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-The truststore to validate the server's certificate.
-It can be either `.jks` or `.p12`.
-Use either `:truststore` or `:cacert`.
-
-[id="plugins-{type}s-{plugin}-truststore_password"]
-===== `truststore_password`
-deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_password>>]
-
-* Value type is <<password,password>>
-* There is no default value for this setting.
-
-Set the truststore password
-
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -76,8 +76,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
       end
 
+      # 404s are excluded because they are valid codes in the case of
+      # template installation. We might need a better story around this later
+      # but for our current purposes this is correct
       code = resp.code
-      if code < 200 || code > 299 # assume anything not 2xx is an error that the layer above needs to interpret
+      if code < 200 || code > 299 && code != 404
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -253,11 +253,13 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def health_check_request(url)
       logger.debug("Running health check to see if an Elasticsearch connection is working",
                    :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
-      response = perform_request_to_url(url, :head, @healthcheck_path)
-      return response, nil
-    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-      logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
-      return nil, e
+      begin
+        response = perform_request_to_url(url, :head, @healthcheck_path)
+        return response, nil
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
+        return nil, e
+      end
     end
 
     def healthcheck!(register_phase = true)
@@ -310,11 +312,13 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def get_root_path(url, params={})
-      resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
-      return resp, nil
-    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-      logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
-      return nil, e
+      begin
+        resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
+        return resp, nil
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
+        return nil, e
+      end
     end
 
     def test_serverless_connection(url, root_response)
@@ -511,13 +515,20 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       major = major_version(version)
       if @maximum_seen_major_version.nil?
         @logger.info("Elasticsearch version determined (#{version})", es_version: major)
-        @maximum_seen_major_version = major
+        set_maximum_seen_major_version(major)
       elsif major > @maximum_seen_major_version
         warn_on_higher_major_version(major, url)
         @maximum_seen_major_version = major
       end
     end
 
+    def set_maximum_seen_major_version(major)
+      if major >= 6
+        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", es_version: major)
+      end
+      @maximum_seen_major_version = major
+    end
+
     def warn_on_higher_major_version(major, url)
       @logger.warn("Detected a node with a higher major version than previously observed, " +
                    "this could be the result of an Elasticsearch cluster upgrade",

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -21,8 +21,7 @@ module LogStash; module Outputs; class ElasticSearch;
   # We wound up agreeing that a number greater than 10 MiB and less than 100MiB
   # made sense. We picked one on the lowish side to not use too much heap.
   TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
-  EVENT_COUNT_HEADER = "X-Elastic-Event-Count".freeze
-  UNCOMPRESSED_LENGTH_HEADER = "X-Elastic-Uncompressed-Request-Length".freeze
+
 
   class HttpClient
     attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
@@ -144,11 +143,7 @@ module LogStash; module Outputs; class ElasticSearch;
                        :payload_size => stream_writer.pos,
                        :content_length => body_stream.size,
                        :batch_offset => (index + 1 - batch_actions.size))
-          headers = {
-            EVENT_COUNT_HEADER => batch_actions.size.to_s,
-            UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s
-          }
-          bulk_responses << bulk_send(body_stream, batch_actions, headers)
+          bulk_responses << bulk_send(body_stream, batch_actions)
           body_stream.truncate(0) && body_stream.seek(0)
           stream_writer = gzip_writer(body_stream) if compression_level?
           batch_actions.clear
@@ -164,14 +159,7 @@ module LogStash; module Outputs; class ElasticSearch;
                    :payload_size => stream_writer.pos,
                    :content_length => body_stream.size,
                    :batch_offset => (actions.size - batch_actions.size))
-
-      if body_stream.size > 0
-        headers = {
-          EVENT_COUNT_HEADER => batch_actions.size.to_s,
-          UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s
-        }
-        bulk_responses << bulk_send(body_stream, batch_actions, headers)
-      end
+      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
 
       body_stream.close unless compression_level?
       join_bulk_responses(bulk_responses)
@@ -191,23 +179,25 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream, batch_actions, headers = {})
-      params = compression_level? ? {:headers => headers.merge("Content-Encoding" => "gzip") } : { :headers => headers }
+    def bulk_send(body_stream, batch_actions)
+      params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
 
-      begin
-        response = @pool.post(@bulk_path, params, body_stream.string)
-        @bulk_response_metrics.increment(response.code.to_s)
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        @bulk_response_metrics.increment(e.response_code.to_s)
-        raise e unless e.response_code == 413
-        # special handling for 413, treat it as a document level issue
+      response = @pool.post(@bulk_path, params, body_stream.string)
+
+      @bulk_response_metrics.increment(response.code.to_s)
+
+      case response.code
+      when 200 # OK
+        LogStash::Json.load(response.body)
+      when 413 # Payload Too Large
         logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
-        return emulate_batch_error_response(batch_actions, 413, 'payload_too_large')
-      rescue => e # it may be a network issue instead, re-raise
-        raise e
+        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
+      else
+        url = ::LogStash::Util::SafeURI.new(response.final_url)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+          response.code, url, body_stream.to_s, response.body
+        )
       end
-
-      LogStash::Json.load(response.body)
     end
 
     def emulate_batch_error_response(actions, http_code, reason)
@@ -421,9 +411,6 @@ module LogStash; module Outputs; class ElasticSearch;
     def exists?(path, use_get=false)
       response = use_get ? @pool.get(path) : @pool.head(path)
       response.code >= 200 && response.code <= 299
-    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-      return false if e.response_code == 404
-      raise e
     end
 
     def template_exists?(template_endpoint, name)
@@ -434,8 +421,6 @@ module LogStash; module Outputs; class ElasticSearch;
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
-    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-      raise e unless e.response_code == 404
     end
 
     # ILM methods
@@ -447,15 +432,17 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
-      logger.info("Created rollover alias", name: alias_name)
-      # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
-    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-      if e.response_code == 400
-        logger.info("Rollover alias already exists, skipping", name: alias_name)
-        return
+      begin
+        @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+        logger.info("Created rollover alias", name: alias_name)
+        # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        if e.response_code == 400
+            logger.info("Rollover alias already exists, skipping", name: alias_name)
+            return
+        end
+        raise e
       end
-      raise e
     end
 
     def get_xpack_info

data/lib/logstash/outputs/elasticsearch.rb

@@ -275,7 +275,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   def initialize(*params)
     super
     setup_ecs_compatibility_related_defaults
-    setup_ssl_params!
     setup_compression_level!
   end
 
@@ -694,52 +693,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     end
   end
 
-  def setup_ssl_params!
-    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
-      normalize.with_deprecated_alias(:ssl)
-    end
-
-    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
-      normalize.with_deprecated_mapping(:cacert) do |cacert|
-        [cacert]
-      end
-    end
-
-    @ssl_keystore_path =  normalize_config(:ssl_keystore_path) do |normalize|
-      normalize.with_deprecated_alias(:keystore)
-    end
-
-    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
-      normalize.with_deprecated_alias(:keystore_password)
-    end
-
-    @ssl_truststore_path = normalize_config(:ssl_truststore_path) do |normalize|
-      normalize.with_deprecated_alias(:truststore)
-    end
-
-    @ssl_truststore_password =  normalize_config(:ssl_truststore_password) do |normalize|
-      normalize.with_deprecated_alias(:truststore_password)
-    end
-
-    @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
-      normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
-        if ssl_certificate_verification == true
-          "full"
-        else
-          "none"
-        end
-      end
-    end
-
-    params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil?
-    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
-    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
-    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
-    params['ssl_truststore_path'] = @ssl_truststore_path unless @ssl_truststore_path.nil?
-    params['ssl_truststore_password'] = @ssl_truststore_password unless @ssl_truststore_password.nil?
-    params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
-  end
-
   def setup_compression_level!
     @compression_level = normalize_config(:compression_level) do |normalize|
       normalize.with_deprecated_mapping(:http_compression) do |http_compression|

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -43,40 +43,23 @@ module LogStash; module PluginMixins; module ElasticSearch
         # urls that already have query strings, the one specified here will be appended.
         :parameters => { :validate => :hash },
 
-        # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
-        # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
-        # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
-        :ssl => { :validate => :boolean, :deprecated => "Set 'ssl_enabled' instead." },
-
         # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
         # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
         # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
         :ssl_enabled => { :validate => :boolean },
 
-        # Option to validate the server's certificate. Disabling this severely compromises security.
-        # For more information on disabling certificate verification please read
-        # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-        :ssl_certificate_verification => { :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." },
-
         # Options to verify the server's certificate.
         # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
         # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
         # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
         :ssl_verification_mode => { :validate => %w[full none], :default => 'full' },
 
-        # The .cer or .pem file to validate the server's certificate
-        :cacert => { :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." },
-
         # The .cer or .pem files to validate the server's certificate
         :ssl_certificate_authorities => { :validate => :path, :list => true },
 
         # One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
         :ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
 
-        # The JKS truststore to validate the server's certificate.
-        # Use either `:truststore` or `:cacert`
-        :truststore => { :validate => :path, :deprecated => "Set 'ssl_truststore_path' instead." },
-
         # The JKS truststore to validate the server's certificate.
         # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
         :ssl_truststore_path => { :validate => :path },
@@ -84,16 +67,9 @@ module LogStash; module PluginMixins; module ElasticSearch
         # The format of the truststore file. It must be either jks or pkcs12
         :ssl_truststore_type => { :validate => %w[pkcs12 jks] },
 
-        # Set the truststore password
-        :truststore_password => { :validate => :password, :deprecated => "Use 'ssl_truststore_password' instead." },
-
         # Set the truststore password
         :ssl_truststore_password => { :validate => :password },
 
-        # The keystore used to present a certificate to the server.
-        # It can be either .jks or .p12
-        :keystore => { :validate => :path, :deprecated => "Set 'ssl_keystore_path' instead." },
-
         # The keystore used to present a certificate to the server.
         # It can be either .jks or .p12
         :ssl_keystore_path => { :validate => :path },
@@ -101,9 +77,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         # The format of the keystore file. It must be either jks or pkcs12
         :ssl_keystore_type => { :validate => %w[pkcs12 jks] },
 
-        # Set the keystore password
-        :keystore_password => { :validate => :password, :deprecated => "Set 'ssl_keystore_password' instead." },
-
         # Set the keystore password
         :ssl_keystore_password => { :validate => :password },
 
@@ -229,7 +202,17 @@ module LogStash; module PluginMixins; module ElasticSearch
         :dlq_custom_codes => { :validate => :number, :list => true, :default => [] },
 
         # if enabled, failed index name interpolation events go into dead letter queue.
-        :dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true }
+        :dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true },
+
+        # Obsolete Settings
+        :ssl => { :obsolete => "Set 'ssl_enabled' instead." },
+        :ssl_certificate_verification => { :obsolete => "Set 'ssl_verification_mode' instead." },
+        :cacert => { :obsolete => "Set 'ssl_certificate_authorities' instead." },
+        :truststore => { :obsolete => "Set 'ssl_truststore_path' instead." },
+        :keystore => { :obsolete => "Set 'ssl_keystore_path' instead." },
+        # Leave :validate to ensure obfuscation of sensitive setting for passwords
+        :truststore_password => { :validate => :password, :obsolete => "Use 'ssl_truststore_password' instead." },
+        :keystore_password => { :validate => :password, :obsolete => "Set 'ssl_keystore_password' instead." }
     }.freeze
 
     def self.included(base)
@@ -243,3 +226,4 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
   end
 end; end; end
+

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.13'
+  s.version         = '12.0.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -1,18 +1,15 @@
 require_relative './spec_helper'
 
 require 'elasticsearch'
+require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
+require_relative "support/elasticsearch/api/actions/get_alias"
+require_relative "support/elasticsearch/api/actions/put_alias"
+require_relative "support/elasticsearch/api/actions/get_ilm_policy"
+require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
 require 'cabin'
 
-# remove this condition and support package once plugin starts consuming elasticsearch-ruby v8 client
-# in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available
-unless elastic_ruby_v8_client_available?
-  require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
-  require_relative "support/elasticsearch/api/actions/get_ilm_policy"
-  require_relative "support/elasticsearch/api/actions/put_ilm_policy"
-end
-
 module ESHelper
   def get_host_port
     if ENV["INTEGRATION"] == "true"
@@ -23,12 +20,8 @@ module ESHelper
   end
 
   def get_client
-    if elastic_ruby_v8_client_available?
-      Elasticsearch::Client.new(:hosts => [get_host_port])
-    else
-      Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
-        allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
-      end
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
     end
   end
 
@@ -135,36 +128,31 @@ module ESHelper
   end
 
   def get_policy(client, policy_name)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.get_lifecycle(policy: policy_name)
-    else
-      client.get_ilm_policy(name: policy_name)
-    end
+    client.get_ilm_policy(name: policy_name)
   end
 
   def put_policy(client, policy_name, policy)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy})
-    else
-      client.put_ilm_policy({:name => policy_name, :body=> policy})
-    end
+    client.put_ilm_policy({:name => policy_name, :body=> policy})
+  end
+
+  def put_alias(client, the_alias, index)
+    body = {
+        "aliases" => {
+            index => {
+                "is_write_index"=>  true
+            }
+        }
+    }
+    client.put_alias({name: the_alias, body: body})
   end
 
   def clean_ilm(client)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ }
-    else
-      client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ }
-    end
+    client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key)  if key =~ /logstash-policy/ }
   end
 
   def supports_ilm?(client)
     begin
-      if elastic_ruby_v8_client_available?
-        client.index_lifecycle_management.get_lifecycle
-      else
-        client.get_ilm_policy
-      end
+      client.get_ilm_policy
       true
     rescue
       false

data/spec/fixtures/test_certs/GENERATED_AT

@@ -1 +1 @@
-2025-07-22T11:15:03+01:00
+2024-06-25T21:50:58+01:00