logstash-input-elasticsearch 4.16.0 → 4.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c5466b50a56ec047ac20dc019d9e24862756b518613b63edf44583900a090e2
-  data.tar.gz: 6b3915bd640318ebd6fa4b87f676d7e43d4fba7fe54893efdb6eb39f70481da8
+  metadata.gz: 9cdbda91c48e11353b49426f1d9c01e461698e507506563848045560d05421d0
+  data.tar.gz: 367351221d9028775410a4ef272de5bf6bad66ab82d91aedacb91ef462c5ff56
 SHA512:
-  metadata.gz: 930202045f30125060cce1f3d45b22c0336976a6cdcbf5931f4b98410785bd425cf665224edfe4994953ad2d53cdaa994b31be9db86fd05f538a722ff788ba97
-  data.tar.gz: 8cf1f45481575f653867831ab38fb7efd286c12432b2afc2c6940c32d4eb91f913dd7b89c284ed4e11e9205536cc75b318f399b92136777f84ac6dec75d2de2d
+  metadata.gz: 88071a99d4ee00c356c2729cdb0d873e7b0ecb067f02e3ccc76c83ec86166d4b4c1daa036d6943a5913339b524e69ecf3f741d17cd871cdba8cf19f59a371016
+  data.tar.gz: 1b20527dd5f3f3ca00bed248b6cdb57dc81d816f56d3f3d08c393fec4fdcf37f57f8ba61dec957efe03cf6f8695426ef544780e32d7b6818671ae0f0d6930b90

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 4.17.1
+  - Fix: scroll slice high memory consumption [#189](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/189)
+
+## 4.17.0
+  - Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185)
+    - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme
+    - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client
+    - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate`
+    - `ssl_truststore_path`: The JKS truststore to validate the server's certificate
+    - `ssl_truststore_type`: The format of the truststore file
+    - `ssl_truststore_password`: The truststore password
+    - `ssl_keystore_path`: The keystore used to present a certificate to the server
+    - `ssl_keystore_type`: The format of the keystore file
+    - `ssl_keystore_password`: The keystore password
+    - `ssl_cipher_suites`: The list of cipher suites to use
+    - `ssl_supported_protocols`: Supported protocols with versions
+  - Reviewed and deprecated SSL settings to comply with Logstash's naming convention
+    - Deprecated `ssl` in favor of `ssl_enabled`
+    - Deprecated `ca_file` in favor of `ssl_certificate_authorities`
+    - Deprecated `ssl_certificate_verification` in favor of `ssl_verification_mode`
+
 ## 4.16.0
   - Added `ssl_certificate_verification` option to control SSL certificate verification [#180](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/180)
 

data/docs/index.asciidoc

@@ -96,13 +96,12 @@ TIP: Set the `target` option to avoid potential schema conflicts.
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -121,8 +120,19 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-retries>> | <<number,number>>|No
@@ -140,21 +150,13 @@ input plugins.
   * Value type is <<password,password>>
   * There is no default value for this setting.
 
-Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<plugins-{type}s-{plugin}-ssl_enabled>> option.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch
 {ref}/security-api-create-api-key.html[Create
 API key API].
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file` 
-
-  * Value type is <<path,path>>
-  * There is no default value for this setting.
-
-SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
-
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`
 
@@ -351,6 +353,7 @@ documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a l
 The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
 The default is 0 (no retry). This value should be equal to or greater than zero.
 
+NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
@@ -406,28 +409,138 @@ NOTE: The Elasticsearch manual indicates that there can be _negative_ performanc
 If the `slices` parameter is left unset, the plugin will _not_ inject slice
 instructions into the query.
 
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl` 
+[id="plugins-{type}s-{plugin}-ssl_certificate"]
+===== `ssl_certificate`
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
+
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_key>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
+===== `ssl_certificate_authorities`
+
+  * Value type is a list of <<path,path>>
+  * There is no default value for this setting
+
+The `.cer` or `.pem` files to validate the server's certificate.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
+===== `ssl_cipher_suites`
+  * Value type is a list of <<string,string>>
+  * There is no default value for this setting
+
+The list of cipher suites to use, listed by priorities.
+Supported cipher suites vary depending on the Java and protocol versions.
+
+[id="plugins-{type}s-{plugin}-ssl_enabled"]
+===== `ssl_enabled`
 
   * Value type is <<boolean,boolean>>
-  * Default value is `false`
+  * There is no default value for this setting.
 
-If enabled, SSL will be used when communicating with the Elasticsearch
-server (i.e. HTTPS will be used instead of plain HTTP).
+Enable SSL/TLS secured communication to Elasticsearch cluster.
+Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
+If no explicit protocol is specified plain HTTP will be used.
 
-[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
-===== `ssl_certificate_verification`
+[id="plugins-{type}s-{plugin}-ssl_key"]
+===== `ssl_key`
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
 
-* Value type is <<boolean,boolean>>
-* Default value is `true`
+OpenSSL-style RSA private key that corresponds to the <<plugins-{type}s-{plugin}-ssl_certificate>>.
 
-Option to validate the server's certificate. Disabling this severely compromises security.
-When certificate validation is disabled, this plugin implicitly trusts the machine
-resolved at the given address without validating its proof-of-identity.
-In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
-man-in-the-middle or other compromised infrastructure.
-More information on the importance of certificate verification:
-**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
+===== `ssl_keystore_password`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the keystore password
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
+===== `ssl_keystore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The keystore used to present a certificate to the server.
+It can be either `.jks` or `.p12`
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
+===== `ssl_keystore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the keystore filename.
+
+The format of the keystore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+  `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
+===== `ssl_truststore_password`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the truststore password.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
+===== `ssl_truststore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The truststore to validate the server's certificate.
+It can be either .jks or .p12.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
+===== `ssl_truststore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the truststore filename.
+
+The format of the truststore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
+===== `ssl_verification_mode`
+
+  * Value can be any of: `full`, `none`
+  * Default value is `full`
+
+Defines how to verify the certificates presented by another party in the TLS connection:
+
+`full` validates that the server certificate has an issue date that’s within
+the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and
+has a hostname or IP address that matches the names within the certificate.
+
+`none` performs no certificate validation.
+
+WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
 [id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
 ===== `socket_timeout_seconds`
@@ -463,6 +576,55 @@ option when authenticating to the Elasticsearch server. If set to an
 empty string authentication will be disabled.
 
 
+[id="plugins-{type}s-{plugin}-deprecated-options"]
+==== Elasticsearch Input deprecated configuration options
+
+This plugin supports the following deprecated configurations.
+
+WARNING: Deprecated options are subject to removal in future releases.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting|Input type|Replaced by
+| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
+| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
+|=======================================================================
+
+[id="plugins-{type}s-{plugin}-ca_file"]
+===== `ca_file`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+If enabled, SSL will be used when communicating with the Elasticsearch
+server (i.e. HTTPS will be used instead of plain HTTP).
+
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
+===== `ssl_certificate_verification`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`
+
+Option to validate the server's certificate. Disabling this severely compromises security.
+When certificate validation is disabled, this plugin implicitly trusts the machine
+resolved at the given address without validating its proof-of-identity.
+In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
+man-in-the-middle or other compromised infrastructure.
+More information on the importance of certificate verification:
+**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/elasticsearch.rb

@@ -9,6 +9,7 @@ require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/scheduler"
+require "logstash/plugin_mixins/normalize_config_support"
 require "base64"
 require 'logstash/helpers/loggable_try'
 
@@ -82,6 +83,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::Scheduler
 
+  include LogStash::PluginMixins::NormalizeConfigSupport
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -185,15 +188,60 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config :proxy, :validate => :uri_or_empty
 
   # SSL
-  config :ssl, :validate => :boolean, :default => false
+  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
+
+  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
+  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
+
+  # OpenSSL-style X.509 certificate certificate to authenticate the client
+  config :ssl_certificate, :validate => :path
 
-  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary 
-  config :ca_file, :validate => :path
+  # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
+  config :ssl_certificate_authorities, :validate => :path, :list => true
 
   # Option to validate the server's certificate. Disabling this severely compromises security.
   # For more information on the importance of certificate verification please read
   # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-  config :ssl_certificate_verification, :validate => :boolean, :default => true
+  config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead."
+
+  # The list of cipher suites to use, listed by priorities.
+  # Supported cipher suites vary depending on which version of Java is used.
+  config :ssl_cipher_suites, :validate => :string, :list => true
+
+  # SSL
+  config :ssl_enabled, :validate => :boolean
+
+  # OpenSSL-style RSA private key to authenticate the client
+  config :ssl_key, :validate => :path
+
+  # Set the keystore password
+  config :ssl_keystore_password, :validate => :password
+
+  # The keystore used to present a certificate to the server.
+  # It can be either .jks or .p12
+  config :ssl_keystore_path, :validate => :path
+
+  # The format of the keystore file. It must be either jks or pkcs12
+  config :ssl_keystore_type, :validate => %w[pkcs12 jks]
+
+  # Supported protocols with versions.
+  config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true
+
+  # Set the truststore password
+  config :ssl_truststore_password, :validate => :password
+
+  # The JKS truststore to validate the server's certificate.
+  # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+  config :ssl_truststore_path, :validate => :path
+
+  # The format of the truststore file. It must be either jks or pkcs12
+  config :ssl_truststore_type, :validate => %w[pkcs12 jks]
+
+  # Options to verify the server's certificate.
+  # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+  # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+  # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+  config :ssl_verification_mode, :validate => %w[full none], :default => 'full'
 
   # Schedule of when to periodically run statement, in Cron format
   # for example: "* * * * *" (execute query every minute, on the minute)
@@ -208,6 +256,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
+  attr_reader :pipeline_id
+
   def initialize(params={})
     super(params)
 
@@ -219,6 +269,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "rufus/scheduler"
 
+    @pipeline_id = execution_context&.pipeline_id || 'main'
+
+    fill_hosts_from_cloud_id
+    setup_ssl_params!
+
     @options = {
       :index => @index,
       :scroll => @scroll,
@@ -234,8 +289,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     validate_authentication
     fill_user_password_from_cloud_auth
-    fill_hosts_from_cloud_id
-
 
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
@@ -246,7 +299,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
     hosts = setup_hosts
-    ssl_options = setup_ssl
+    ssl_options = setup_client_ssl
 
     @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
@@ -277,20 +330,22 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
     if @slices.nil? || @slices <= 1
-      success, events = retryable_slice
-      success && events.each { |event| output_queue << event }
-      return
+      return retryable(JOB_NAME) do
+        do_run_slice(output_queue)
+      end
     end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
-    slice_results = parallel_slice # array of tuple(ok, events)
 
-    # insert events to queue if all slices success
-    if slice_results.all?(&:first)
-      slice_results.flat_map { |success, events| events }
-                  .each { |event| output_queue << event }
-    end
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
+        retryable(JOB_NAME) do
+          do_run_slice(output_queue, slice_id)
+        end
+      end
+    end.map(&:join)
 
     logger.trace("#{@slices} slices completed")
   end
@@ -298,42 +353,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def retryable(job_name, &block)
     begin
       stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
-      output = stud_try.try((@retries + 1).times) { yield }
-      [true, output]
+      stud_try.try((@retries + 1).times) { yield }
     rescue => e
       error_details = {:message => e.message, :cause => e.cause}
       error_details[:backtrace] = e.backtrace if logger.debug?
       logger.error("Tried #{job_name} unsuccessfully", error_details)
-      [false, nil]
     end
   end
 
-
-  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
-  def parallel_slice
-    pipeline_id = execution_context&.pipeline_id || 'main'
-    @slices.times.map do |slice_id|
-      Thread.new do
-        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
-        retryable_slice(slice_id)
-      end
-    end.map do |t|
-      t.join
-      t.value
-    end
-  end
-
-  # @param scroll_id [Integer]
-  # @return (ok, events) [Boolean, Array(Logstash::Event)]
-  def retryable_slice(slice_id=nil)
-    retryable(JOB_NAME) do
-      output = []
-      do_run_slice(output, slice_id)
-      output
-    end
-  end
-
-
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
@@ -416,6 +443,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
 
+  def effectively_ssl?
+    return true if @ssl_enabled
+
+    hosts = Array(@hosts)
+    return false if hosts.nil? || hosts.empty?
+
+    hosts.all? { |host| host && host.to_s.start_with?("https") }
+  end
+
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth
@@ -426,26 +462,113 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
 
-    if @api_key && @api_key.value && @ssl != true
-      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    if @api_key && @api_key.value && @ssl_enabled != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl_enabled => true` option")
     end
   end
 
-  def setup_ssl
+  def setup_client_ssl
     ssl_options = {}
+    ssl_options[:ssl] = true if @ssl_enabled
 
-    ssl_options[:ssl] = true if @ssl
-    ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
-    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
-    if @ssl && !@ssl_certificate_verification
-      logger.warn "You have enabled encryption but DISABLED certificate verification, " +
-                    "to make sure your data is secure remove `ssl_certificate_verification => false`"
-      ssl_options[:verify] = :disable
+    unless @ssl_enabled
+      # Keep it backward compatible with the deprecated `ssl` option
+      ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint if original_params.include?('ssl')
+      return ssl_options
+    end
+
+    ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path')
+
+    if ssl_certificate_authorities && ssl_truststore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate'
+    end
+
+    if ssl_certificate && ssl_keystore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates'
+    end
+
+    if ssl_certificate_authorities&.any?
+      raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
+      ssl_options[:ca_file] = ssl_certificate_authorities.first
+    end
+
+    if ssl_truststore_path
+      ssl_options[:truststore] = ssl_truststore_path
+      ssl_options[:truststore_type] = params["ssl_truststore_type"] if params.include?("ssl_truststore_type")
+      ssl_options[:truststore_password] = params["ssl_truststore_password"].value if params.include?("ssl_truststore_password")
+    end
+
+    if ssl_keystore_path
+      ssl_options[:keystore] = ssl_keystore_path
+      ssl_options[:keystore_type] = params["ssl_keystore_type"] if params.include?("ssl_keystore_type")
+      ssl_options[:keystore_password] = params["ssl_keystore_password"].value if params.include?("ssl_keystore_password")
+    end
+
+    ssl_key = params["ssl_key"]
+    if ssl_certificate
+      raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key
+      ssl_options[:client_cert] = ssl_certificate
+      ssl_options[:client_key] = ssl_key
+    elsif !ssl_key.nil?
+      raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"'
+    end
+
+    ssl_verification_mode = params["ssl_verification_mode"]
+    unless ssl_verification_mode.nil?
+      case ssl_verification_mode
+        when 'none'
+          logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                        "to make sure your data is secure set `ssl_verification_mode => full`"
+          ssl_options[:verify] = :disable
+        else
+          ssl_options[:verify] = :strict
+      end
     end
 
+    ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites")
+
+    protocols = params['ssl_supported_protocols']
+    ssl_options[:protocols] = protocols if protocols&.any?
+    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+
     ssl_options
   end
 
+  def setup_ssl_params!
+    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
+      normalize.with_deprecated_alias(:ssl)
+    end
+
+    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
+    infer_ssl_enabled_from_hosts
+
+    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
+      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
+        [ca_file]
+      end
+    end
+
+    @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
+      normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
+        if ssl_certificate_verification == true
+          "full"
+        else
+          "none"
+        end
+      end
+    end
+
+    params['ssl_enabled'] = @ssl_enabled
+    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
+    params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
+  end
+
+  def infer_ssl_enabled_from_hosts
+    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
+
+    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
+  end
+
   def setup_hosts
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
     @hosts.map do |h|
@@ -453,7 +576,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
         h
       else
         host, port = h.split(':')
-        { host: host, port: port, scheme: (@ssl ? 'https' : 'http') }
+        { host: host, port: port, scheme: (@ssl_enabled ? 'https' : 'http') }
       end
     end
   end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.16.0'
+  s.version         = '4.17.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'

data/spec/inputs/elasticsearch_spec.rb

@@ -204,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -215,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -414,18 +414,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
           plugin.register
 
-          expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).twice.and_return(nil)
 
-          # SLICE0 is a three-page scroll in which the second page throw exception
+          # SLICE0 is a three-page scroll
           slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
           allow(client).to receive(:ping)
 
-          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          # SLICE1 is a two-page scroll in which the last page throws exception
           slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
 
           synchronize_method!(plugin, :scroll_request)
           synchronize_method!(plugin, :search_request)
@@ -433,18 +434,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         let(:client) { Elasticsearch::Client.new }
 
-        it 'does not insert event to queue' do
-          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
-            slice0, slice1 = m.call
-            expect(slice0[0]).to be_falsey
-            expect(slice1[0]).to be_truthy
-            expect(slice1[1].size).to eq(4) # four items from SLICE1
-            [slice0, slice1]
+        it 'insert event to queue without waiting other slices' do
+          expect(plugin).to receive(:do_run_slice).twice.and_wrap_original do |m, *args|
+            q = args[0]
+            slice_id = args[1]
+            if slice_id == 0
+              m.call(*args)
+              expect(q.size).to eq(3)
+            else
+              sleep(1)
+              m.call(*args)
+            end
           end
 
           queue = Queue.new
           plugin.run(queue)
-          expect(queue.size).to eq(0)
+          expect(queue.size).to eq(5)
         end
       end
     end
@@ -681,7 +686,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
 
       context "with ssl" do
-        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
 
         it "should set authorization" do
           plugin.register
@@ -700,9 +705,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         end
         
         context 'ssl verification disabled' do
-          let(:config) { super().merge({ 'ssl_certificate_verification' => false }) }
+          let(:config) { super().merge({ 'ssl_verification_mode' => 'none' }) }
           it 'should warn data security risk' do
-            expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure remove `ssl_certificate_verification => false`")
+            expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure set `ssl_verification_mode => full`")
             plugin.register
           end
         end

data/spec/inputs/elasticsearch_ssl_spec.rb

@@ -0,0 +1,265 @@
+require 'stud/temporary'
+require "elasticsearch"
+
+describe "SSL options" do
+  let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") }
+  let(:hosts) {["localhost"]}
+  let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } }
+
+  subject do
+    require "logstash/inputs/elasticsearch"
+    LogStash::Inputs::Elasticsearch.new(settings)
+  end
+
+  before do
+    allow(es_client_double).to receive(:close)
+    allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
+    allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
+  end
+
+  after do
+    subject.close
+  end
+
+  context "when ssl_enabled is" do
+    context "true and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "false and cloud_id resolved host is https" do
+      let(:settings) {{
+        "ssl_enabled" => false,
+        "hosts" => [],
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+  end
+
+  context "when neither ssl nor ssl_enabled is set" do
+    let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } }
+
+    context "and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and there is https hosts" do
+      let(:hosts) { %w[https://sec-es01 https://sec-es01] }
+
+      it "should infer the ssl_enabled value to true" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "and hosts have no scheme defined" do
+      let(:hosts) { %w[es01 es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and cloud_id resolved host is https" do
+      let(:settings) {{
+        "hosts" => [],
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+  end
+
+  context "when ssl_verification_mode" do
+    context "is set to none" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => "none",
+      ) }
+
+      it "should print a warning" do
+        expect(subject.logger).to receive(:warn).with(/You have enabled encryption but DISABLED certificate verification/).at_least(:once)
+        allow(subject.logger).to receive(:warn).with(any_args)
+
+        subject.register
+      end
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :disable)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+
+    context "is set to full" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => 'full',
+      ) }
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :strict)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+  end
+
+  context "with the conflicting configs" do
+    context "ssl_certificate_authorities and ssl_truststore_path set" do
+      let(:ssl_truststore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_truststore_path" => ssl_truststore_path,
+        "ssl_certificate_authorities" => ssl_certificate_authorities_path
+      ) }
+
+      after :each do
+        File.delete(ssl_truststore_path)
+        File.delete(ssl_certificate_authorities_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate_authorities\/ca_file" or "ssl_truststore_path"/)
+      end
+    end
+
+    context "ssl_certificate and ssl_keystore_path set" do
+      let(:ssl_keystore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_certificate" => ssl_certificate_path,
+        "ssl_keystore_path" => ssl_keystore_path
+      ) }
+
+      after :each do
+        File.delete(ssl_keystore_path)
+        File.delete(ssl_certificate_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate" or "ssl_keystore_path\/keystore"/)
+      end
+    end
+  end
+
+  context "when configured with Java store files" do
+    let(:ssl_truststore_path) { Stud::Temporary.file.path }
+    let(:ssl_keystore_path) { Stud::Temporary.file.path }
+
+    after :each do
+      File.delete(ssl_truststore_path)
+      File.delete(ssl_keystore_path)
+    end
+
+    let(:settings) { super().merge(
+      "ssl_truststore_path" => ssl_truststore_path,
+      "ssl_truststore_type" => "jks",
+      "ssl_truststore_password" => "foo",
+      "ssl_keystore_path" => ssl_keystore_path,
+      "ssl_keystore_type" => "jks",
+      "ssl_keystore_password" => "bar",
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :ssl => true,
+                                      :keystore => ssl_keystore_path,
+                                      :keystore_type => "jks",
+                                      :keystore_password => "bar",
+                                      :truststore => ssl_truststore_path,
+                                      :truststore_type => "jks",
+                                      :truststore_password => "foo",
+                                      :verify => :strict,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+  end
+
+  context "when configured with certificate files" do
+    let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+    let(:ssl_certificate_path) { Stud::Temporary.file.path }
+    let(:ssl_key_path) { Stud::Temporary.file.path }
+    let(:settings) { super().merge(
+      "ssl_certificate_authorities" => [ssl_certificate_authorities_path],
+      "ssl_certificate" => ssl_certificate_path,
+      "ssl_key" => ssl_key_path,
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    after :each do
+      File.delete(ssl_certificate_authorities_path)
+      File.delete(ssl_certificate_path)
+      File.delete(ssl_key_path)
+    end
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :ssl => true,
+                                      :ca_file => ssl_certificate_authorities_path,
+                                      :client_cert => ssl_certificate_path,
+                                      :client_key => ssl_key_path,
+                                      :verify => :strict,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+
+    context "and only the ssl_certificate is set" do
+      let(:settings) { super().reject { |k| "ssl_key".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Using an "ssl_certificate" requires an "ssl_key"/)
+      end
+    end
+
+    context "and only the ssl_key is set" do
+      let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /An "ssl_certificate" is required when using an "ssl_key"/)
+      end
+    end
+  end
+end
+

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -95,13 +95,13 @@ describe LogStash::Inputs::Elasticsearch do
     end
 
     context 'with ca_file' do
-      let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) }
+      let(:config) { super().merge('ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file) }
       it_behaves_like 'secured_elasticsearch'
     end
 
     context 'with `ca_trusted_fingerprint`' do
       let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp }
-      let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
+      let(:config) { super().merge('ssl_enabled' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
 
       if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
         it_behaves_like 'secured_elasticsearch'
@@ -125,11 +125,11 @@ describe LogStash::Inputs::Elasticsearch do
 
   context 'setting host:port (and ssl)', secure_integration: true do
 
-    let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
+    let(:client_options) { { :ssl_certificate_authorities => ca_file, :user => user, :password => password } }
 
     let(:config) do
       config = super().merge "hosts" => [ESHelper.get_host_port]
-      config.merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file)
+      config.merge('user' => user, 'password' => password, 'ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file)
     end
 
     it_behaves_like 'an elasticsearch index plugin'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.16.0
+  version: 4.17.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2023-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +114,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-normalize_config_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -267,6 +281,7 @@ files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -301,4 +316,5 @@ test_files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb