logstash-input-elasticsearch 4.15.0 → 4.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9379a31460bf649615c038b1b35d207f6a6db869bb1a27e963da70e278ae6bcd
-  data.tar.gz: f62823b8ddfb587ce9614a1d131be1bce2186dbd859287b1107757917a754c7e
+  metadata.gz: 43e3c8e44ba8bd1ce4065a1410b26a63d621180d0354b4b4aa8f3891962d9986
+  data.tar.gz: c9f6d3b53d1bb90c9ad77eab86b92818128f310aa95a80e2908bdc5b981a780f
 SHA512:
-  metadata.gz: c675ed6a5d1a4a104313611e4895171c6461149d0415c5ad4c276e42b5b99672584ae38ddc37f3e32521074727051c68f0d10a37b77064d0cea6ea03f041a3b9
-  data.tar.gz: beba1b98be77880e6e3712cebdd0474d68c33556a86236e828a3332d2e812b187e920e78f6fb006b27b82269faa0e0654568d050f7057ae7db48da7fd9a20e31
+  metadata.gz: 5799d6ef93349a2ca9a6408bba0bbaa78f5c985137a4d126c70efe53122b7eac4d1fcda48aeb51f629651ea04e4457e9bb9705b760744b0ff22cd462f617c7b8
+  data.tar.gz: f0057cebd50dc79156d8f05be48341f009366fc9057df0f61d50327a3cecf0a95ebccd05bed120173ab429caab295d5b71f25edc04232fd36ecf862ad58bd974

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 4.17.0
+  - Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185)
+    - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme
+    - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client
+    - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate`
+    - `ssl_truststore_path`: The JKS truststore to validate the server's certificate
+    - `ssl_truststore_type`: The format of the truststore file
+    - `ssl_truststore_password`: The truststore password
+    - `ssl_keystore_path`: The keystore used to present a certificate to the server
+    - `ssl_keystore_type`: The format of the keystore file
+    - `ssl_keystore_password`: The keystore password
+    - `ssl_cipher_suites`: The list of cipher suites to use
+    - `ssl_supported_protocols`: Supported protocols with versions
+  - Reviewed and deprecated SSL settings to comply with Logstash's naming convention
+    - Deprecated `ssl` in favor of `ssl_enabled`
+    - Deprecated `ca_file` in favor of `ssl_certificate_authorities`
+    - Deprecated `ssl_certificate_verification` in favor of `ssl_verification_mode`
+
+## 4.16.0
+  - Added `ssl_certificate_verification` option to control SSL certificate verification [#180](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/180)
+
 ## 4.15.0
   - Feat: add `retries` option. allow retry for failing query [#179](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/179)
 

data/docs/index.asciidoc

@@ -96,13 +96,12 @@ TIP: Set the `target` option to avoid potential schema conflicts.
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -121,7 +120,19 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-retries>> | <<number,number>>|No
@@ -139,21 +150,13 @@ input plugins.
   * Value type is <<password,password>>
   * There is no default value for this setting.
 
-Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<plugins-{type}s-{plugin}-ssl_enabled>> option.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch
 {ref}/security-api-create-api-key.html[Create
 API key API].
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file` 
-
-  * Value type is <<path,path>>
-  * There is no default value for this setting.
-
-SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
-
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`
 
@@ -405,14 +408,138 @@ NOTE: The Elasticsearch manual indicates that there can be _negative_ performanc
 If the `slices` parameter is left unset, the plugin will _not_ inject slice
 instructions into the query.
 
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl` 
+[id="plugins-{type}s-{plugin}-ssl_certificate"]
+===== `ssl_certificate`
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
+
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_key>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
+===== `ssl_certificate_authorities`
+
+  * Value type is a list of <<path,path>>
+  * There is no default value for this setting
+
+The `.cer` or `.pem` files to validate the server's certificate.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
+===== `ssl_cipher_suites`
+  * Value type is a list of <<string,string>>
+  * There is no default value for this setting
+
+The list of cipher suites to use, listed by priorities.
+Supported cipher suites vary depending on the Java and protocol versions.
+
+[id="plugins-{type}s-{plugin}-ssl_enabled"]
+===== `ssl_enabled`
 
   * Value type is <<boolean,boolean>>
-  * Default value is `false`
+  * There is no default value for this setting.
 
-If enabled, SSL will be used when communicating with the Elasticsearch
-server (i.e. HTTPS will be used instead of plain HTTP).
+Enable SSL/TLS secured communication to Elasticsearch cluster.
+Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
+If no explicit protocol is specified plain HTTP will be used.
+
+[id="plugins-{type}s-{plugin}-ssl_key"]
+===== `ssl_key`
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+OpenSSL-style RSA private key that corresponds to the <<plugins-{type}s-{plugin}-ssl_certificate>>.
+
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
+===== `ssl_keystore_password`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the keystore password
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
+===== `ssl_keystore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The keystore used to present a certificate to the server.
+It can be either `.jks` or `.p12`
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
+===== `ssl_keystore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the keystore filename.
+
+The format of the keystore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+  `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
+===== `ssl_truststore_password`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the truststore password.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
+===== `ssl_truststore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The truststore to validate the server's certificate.
+It can be either .jks or .p12.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
+===== `ssl_truststore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the truststore filename.
+
+The format of the truststore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
+===== `ssl_verification_mode`
+
+  * Value can be any of: `full`, `none`
+  * Default value is `full`
+
+Defines how to verify the certificates presented by another party in the TLS connection:
+
+`full` validates that the server certificate has an issue date that’s within
+the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and
+has a hostname or IP address that matches the names within the certificate.
+
+`none` performs no certificate validation.
+
+WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
 [id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
 ===== `socket_timeout_seconds`
@@ -448,6 +575,55 @@ option when authenticating to the Elasticsearch server. If set to an
 empty string authentication will be disabled.
 
 
+[id="plugins-{type}s-{plugin}-deprecated-options"]
+==== Elasticsearch Input deprecated configuration options
+
+This plugin supports the following deprecated configurations.
+
+WARNING: Deprecated options are subject to removal in future releases.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting|Input type|Replaced by
+| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
+| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
+|=======================================================================
+
+[id="plugins-{type}s-{plugin}-ca_file"]
+===== `ca_file`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+If enabled, SSL will be used when communicating with the Elasticsearch
+server (i.e. HTTPS will be used instead of plain HTTP).
+
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
+===== `ssl_certificate_verification`
+deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`
+
+Option to validate the server's certificate. Disabling this severely compromises security.
+When certificate validation is disabled, this plugin implicitly trusts the machine
+resolved at the given address without validating its proof-of-identity.
+In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
+man-in-the-middle or other compromised infrastructure.
+More information on the importance of certificate verification:
+**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/elasticsearch.rb

@@ -9,6 +9,7 @@ require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/scheduler"
+require "logstash/plugin_mixins/normalize_config_support"
 require "base64"
 require 'logstash/helpers/loggable_try'
 
@@ -82,6 +83,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::Scheduler
 
+  include LogStash::PluginMixins::NormalizeConfigSupport
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -185,10 +188,60 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config :proxy, :validate => :uri_or_empty
 
   # SSL
-  config :ssl, :validate => :boolean, :default => false
+  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
+
+  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
+  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
+
+  # OpenSSL-style X.509 certificate certificate to authenticate the client
+  config :ssl_certificate, :validate => :path
+
+  # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
+  config :ssl_certificate_authorities, :validate => :path, :list => true
+
+  # Option to validate the server's certificate. Disabling this severely compromises security.
+  # For more information on the importance of certificate verification please read
+  # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+  config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead."
+
+  # The list of cipher suites to use, listed by priorities.
+  # Supported cipher suites vary depending on which version of Java is used.
+  config :ssl_cipher_suites, :validate => :string, :list => true
+
+  # SSL
+  config :ssl_enabled, :validate => :boolean
+
+  # OpenSSL-style RSA private key to authenticate the client
+  config :ssl_key, :validate => :path
+
+  # Set the keystore password
+  config :ssl_keystore_password, :validate => :password
+
+  # The keystore used to present a certificate to the server.
+  # It can be either .jks or .p12
+  config :ssl_keystore_path, :validate => :path
+
+  # The format of the keystore file. It must be either jks or pkcs12
+  config :ssl_keystore_type, :validate => %w[pkcs12 jks]
+
+  # Supported protocols with versions.
+  config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true
+
+  # Set the truststore password
+  config :ssl_truststore_password, :validate => :password
+
+  # The JKS truststore to validate the server's certificate.
+  # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+  config :ssl_truststore_path, :validate => :path
 
-  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary 
-  config :ca_file, :validate => :path
+  # The format of the truststore file. It must be either jks or pkcs12
+  config :ssl_truststore_type, :validate => %w[pkcs12 jks]
+
+  # Options to verify the server's certificate.
+  # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+  # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+  # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+  config :ssl_verification_mode, :validate => %w[full none], :default => 'full'
 
   # Schedule of when to periodically run statement, in Cron format
   # for example: "* * * * *" (execute query every minute, on the minute)
@@ -214,6 +267,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "rufus/scheduler"
 
+    fill_hosts_from_cloud_id
+    setup_ssl_params!
+
     @options = {
       :index => @index,
       :scroll => @scroll,
@@ -229,8 +285,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     validate_authentication
     fill_user_password_from_cloud_auth
-    fill_hosts_from_cloud_id
-
 
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
@@ -241,7 +295,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
 
     hosts = setup_hosts
-    ssl_options = setup_ssl
+    ssl_options = setup_client_ssl
 
     @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
@@ -411,6 +465,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
 
+  def effectively_ssl?
+    return true if @ssl_enabled
+
+    hosts = Array(@hosts)
+    return false if hosts.nil? || hosts.empty?
+
+    hosts.all? { |host| host && host.to_s.start_with?("https") }
+  end
+
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth
@@ -421,21 +484,113 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
 
-    if @api_key && @api_key.value && @ssl != true
-      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    if @api_key && @api_key.value && @ssl_enabled != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl_enabled => true` option")
     end
   end
 
-  def setup_ssl
+  def setup_client_ssl
     ssl_options = {}
+    ssl_options[:ssl] = true if @ssl_enabled
+
+    unless @ssl_enabled
+      # Keep it backward compatible with the deprecated `ssl` option
+      ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint if original_params.include?('ssl')
+      return ssl_options
+    end
+
+    ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path')
+
+    if ssl_certificate_authorities && ssl_truststore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate'
+    end
 
-    ssl_options[:ssl] = true if @ssl
-    ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
+    if ssl_certificate && ssl_keystore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates'
+    end
+
+    if ssl_certificate_authorities&.any?
+      raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
+      ssl_options[:ca_file] = ssl_certificate_authorities.first
+    end
+
+    if ssl_truststore_path
+      ssl_options[:truststore] = ssl_truststore_path
+      ssl_options[:truststore_type] = params["ssl_truststore_type"] if params.include?("ssl_truststore_type")
+      ssl_options[:truststore_password] = params["ssl_truststore_password"].value if params.include?("ssl_truststore_password")
+    end
+
+    if ssl_keystore_path
+      ssl_options[:keystore] = ssl_keystore_path
+      ssl_options[:keystore_type] = params["ssl_keystore_type"] if params.include?("ssl_keystore_type")
+      ssl_options[:keystore_password] = params["ssl_keystore_password"].value if params.include?("ssl_keystore_password")
+    end
+
+    ssl_key = params["ssl_key"]
+    if ssl_certificate
+      raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key
+      ssl_options[:client_cert] = ssl_certificate
+      ssl_options[:client_key] = ssl_key
+    elsif !ssl_key.nil?
+      raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"'
+    end
+
+    ssl_verification_mode = params["ssl_verification_mode"]
+    unless ssl_verification_mode.nil?
+      case ssl_verification_mode
+        when 'none'
+          logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                        "to make sure your data is secure set `ssl_verification_mode => full`"
+          ssl_options[:verify] = :disable
+        else
+          ssl_options[:verify] = :strict
+      end
+    end
+
+    ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites")
+
+    protocols = params['ssl_supported_protocols']
+    ssl_options[:protocols] = protocols if protocols&.any?
     ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
 
     ssl_options
   end
 
+  def setup_ssl_params!
+    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
+      normalize.with_deprecated_alias(:ssl)
+    end
+
+    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
+    infer_ssl_enabled_from_hosts
+
+    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
+      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
+        [ca_file]
+      end
+    end
+
+    @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
+      normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
+        if ssl_certificate_verification == true
+          "full"
+        else
+          "none"
+        end
+      end
+    end
+
+    params['ssl_enabled'] = @ssl_enabled
+    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
+    params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
+  end
+
+  def infer_ssl_enabled_from_hosts
+    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
+
+    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
+  end
+
   def setup_hosts
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
     @hosts.map do |h|
@@ -443,7 +598,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
         h
       else
         host, port = h.split(':')
-        { host: host, port: port, scheme: (@ssl ? 'https' : 'http') }
+        { host: host, port: port, scheme: (@ssl_enabled ? 'https' : 'http') }
       end
     end
   end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.15.0'
+  s.version         = '4.17.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'

data/spec/inputs/elasticsearch_spec.rb

@@ -681,7 +681,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
 
       context "with ssl" do
-        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
 
         it "should set authorization" do
           plugin.register
@@ -698,6 +698,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
             expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
           end
         end
+        
+        context 'ssl verification disabled' do
+          let(:config) { super().merge({ 'ssl_verification_mode' => 'none' }) }
+          it 'should warn data security risk' do
+            expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure set `ssl_verification_mode => full`")
+            plugin.register
+          end
+        end
       end
     end if LOGSTASH_VERSION > '6.0'
 

data/spec/inputs/elasticsearch_ssl_spec.rb

@@ -0,0 +1,265 @@
+require 'stud/temporary'
+require "elasticsearch"
+
+describe "SSL options" do
+  let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") }
+  let(:hosts) {["localhost"]}
+  let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } }
+
+  subject do
+    require "logstash/inputs/elasticsearch"
+    LogStash::Inputs::Elasticsearch.new(settings)
+  end
+
+  before do
+    allow(es_client_double).to receive(:close)
+    allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
+    allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
+  end
+
+  after do
+    subject.close
+  end
+
+  context "when ssl_enabled is" do
+    context "true and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "false and cloud_id resolved host is https" do
+      let(:settings) {{
+        "ssl_enabled" => false,
+        "hosts" => [],
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+  end
+
+  context "when neither ssl nor ssl_enabled is set" do
+    let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } }
+
+    context "and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and there is https hosts" do
+      let(:hosts) { %w[https://sec-es01 https://sec-es01] }
+
+      it "should infer the ssl_enabled value to true" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "and hosts have no scheme defined" do
+      let(:hosts) { %w[es01 es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and cloud_id resolved host is https" do
+      let(:settings) {{
+        "hosts" => [],
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+  end
+
+  context "when ssl_verification_mode" do
+    context "is set to none" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => "none",
+      ) }
+
+      it "should print a warning" do
+        expect(subject.logger).to receive(:warn).with(/You have enabled encryption but DISABLED certificate verification/).at_least(:once)
+        allow(subject.logger).to receive(:warn).with(any_args)
+
+        subject.register
+      end
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :disable)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+
+    context "is set to full" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => 'full',
+      ) }
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :strict)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+  end
+
+  context "with the conflicting configs" do
+    context "ssl_certificate_authorities and ssl_truststore_path set" do
+      let(:ssl_truststore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_truststore_path" => ssl_truststore_path,
+        "ssl_certificate_authorities" => ssl_certificate_authorities_path
+      ) }
+
+      after :each do
+        File.delete(ssl_truststore_path)
+        File.delete(ssl_certificate_authorities_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate_authorities\/ca_file" or "ssl_truststore_path"/)
+      end
+    end
+
+    context "ssl_certificate and ssl_keystore_path set" do
+      let(:ssl_keystore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_certificate" => ssl_certificate_path,
+        "ssl_keystore_path" => ssl_keystore_path
+      ) }
+
+      after :each do
+        File.delete(ssl_keystore_path)
+        File.delete(ssl_certificate_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate" or "ssl_keystore_path\/keystore"/)
+      end
+    end
+  end
+
+  context "when configured with Java store files" do
+    let(:ssl_truststore_path) { Stud::Temporary.file.path }
+    let(:ssl_keystore_path) { Stud::Temporary.file.path }
+
+    after :each do
+      File.delete(ssl_truststore_path)
+      File.delete(ssl_keystore_path)
+    end
+
+    let(:settings) { super().merge(
+      "ssl_truststore_path" => ssl_truststore_path,
+      "ssl_truststore_type" => "jks",
+      "ssl_truststore_password" => "foo",
+      "ssl_keystore_path" => ssl_keystore_path,
+      "ssl_keystore_type" => "jks",
+      "ssl_keystore_password" => "bar",
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :ssl => true,
+                                      :keystore => ssl_keystore_path,
+                                      :keystore_type => "jks",
+                                      :keystore_password => "bar",
+                                      :truststore => ssl_truststore_path,
+                                      :truststore_type => "jks",
+                                      :truststore_password => "foo",
+                                      :verify => :strict,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+  end
+
+  context "when configured with certificate files" do
+    let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+    let(:ssl_certificate_path) { Stud::Temporary.file.path }
+    let(:ssl_key_path) { Stud::Temporary.file.path }
+    let(:settings) { super().merge(
+      "ssl_certificate_authorities" => [ssl_certificate_authorities_path],
+      "ssl_certificate" => ssl_certificate_path,
+      "ssl_key" => ssl_key_path,
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    after :each do
+      File.delete(ssl_certificate_authorities_path)
+      File.delete(ssl_certificate_path)
+      File.delete(ssl_key_path)
+    end
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :ssl => true,
+                                      :ca_file => ssl_certificate_authorities_path,
+                                      :client_cert => ssl_certificate_path,
+                                      :client_key => ssl_key_path,
+                                      :verify => :strict,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+
+    context "and only the ssl_certificate is set" do
+      let(:settings) { super().reject { |k| "ssl_key".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Using an "ssl_certificate" requires an "ssl_key"/)
+      end
+    end
+
+    context "and only the ssl_key is set" do
+      let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /An "ssl_certificate" is required when using an "ssl_key"/)
+      end
+    end
+  end
+end
+

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -95,13 +95,13 @@ describe LogStash::Inputs::Elasticsearch do
     end
 
     context 'with ca_file' do
-      let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) }
+      let(:config) { super().merge('ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file) }
       it_behaves_like 'secured_elasticsearch'
     end
 
     context 'with `ca_trusted_fingerprint`' do
       let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp }
-      let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
+      let(:config) { super().merge('ssl_enabled' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
 
       if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
         it_behaves_like 'secured_elasticsearch'
@@ -125,11 +125,11 @@ describe LogStash::Inputs::Elasticsearch do
 
   context 'setting host:port (and ssl)', secure_integration: true do
 
-    let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
+    let(:client_options) { { :ssl_certificate_authorities => ca_file, :user => user, :password => password } }
 
     let(:config) do
       config = super().merge "hosts" => [ESHelper.get_host_port]
-      config.merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file)
+      config.merge('user' => user, 'password' => password, 'ssl_enabled' => true, 'ssl_certificate_authorities' => ca_file)
     end
 
     it_behaves_like 'an elasticsearch index plugin'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.15.0
+  version: 4.17.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-08 00:00:00.000000000 Z
+date: 2023-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +114,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-normalize_config_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -267,6 +281,7 @@ files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -301,4 +316,5 @@ test_files:
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb