logstash-filter-elasticsearch 3.14.0 → 3.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa249a04a85198ffab82e97a53f15c8c1ce3792b0d7f737962979aaccf0b8a78
-  data.tar.gz: 1501f7b558ead54e7701b0048b5f1f9847d250e96ebdf4cd2875000f733ad8c0
+  metadata.gz: 4a80d8c888e4c85fad00861c48191031b12827b75d4f7adf2f82d3be8023ef4c
+  data.tar.gz: ecf9e2cf9c3d020bc5ece06d2b430bef230f60f2505847a534556fd95d72094f
 SHA512:
-  metadata.gz: c31a11c7588de85b8bbdb922c8938946a651230080202b2442dcea5fc3ef686e499aa74b3f10df9df172713b4b007e0e3ba539b7dff838fccb33da116c3aacbf
-  data.tar.gz: 3bcd227e3ba31712287bb27367959b8a5e032a7e9fa5a907f1ae7018e4422db6b3275bbcb3f88d83d6d623d5f015c2ae80a66a148e2bcb3a84181a801e486640
+  metadata.gz: c02acdc7b189b40b9d53f32374c982a0239a2be8a5e6d10f4ec62ee99771822e3a8b0345e432f5fd488eaed02e8f9207b82b4226e29f310bbbfc2f786eafd6bc
+  data.tar.gz: 25cfab8d4069d7e30c5f1dd113857c24ce8c43ef7ed566b66230c64ead218398e28f504b7c0f5d864a8a6174cbf817e1f76faa22d996f15cfa0e298d3ed6e334

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## 3.15.1
+  - Fixes a regression introduced in 3.15.0 which could prevent a connection from being established to Elasticsearch in some SSL configurations
+
+## 3.15.0
+  - Added SSL settings for: [#168](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/168)
+    - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme
+    - `ssl_certificate`: OpenSSL-style X.509 certificate file to authenticate the client
+    - `ssl_key`: OpenSSL-style RSA private key that corresponds to the `ssl_certificate`
+    - `ssl_truststore_path`: The JKS truststore to validate the server's certificate
+    - `ssl_truststore_type`: The format of the truststore file
+    - `ssl_truststore_password`: The truststore password
+    - `ssl_keystore_path`: The keystore used to present a certificate to the server
+    - `ssl_keystore_type`: The format of the keystore file
+    - `ssl_keystore_password`: The keystore password
+    - `ssl_cipher_suites`: The list of cipher suites to use
+    - `ssl_supported_protocols`: Supported protocols with versions
+    - `ssl_verification_mode`: Defines how to verify the certificates presented by another party in the TLS connection
+  - Reviewed and deprecated SSL settings to comply with Logstash's naming convention
+    - Deprecated `ssl` in favor of `ssl_enabled`
+    - Deprecated `ca_file` in favor of `ssl_certificate_authorities`
+    - Deprecated `keystore` in favor of `ssl_keystore_path`
+    - Deprecated `keystore_password` in favor of `ssl_keystore_password`
+
 ## 3.14.0
   - Added support for configurable retries with new `retry_on_failure` and `retry_on_status` options [#160](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/160)
 

data/docs/index.asciidoc

@@ -121,14 +121,13 @@ The `monitoring` permission at cluster level is necessary to perform periodic co
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -143,11 +142,22 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-query_template>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-result_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_on_failure>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-result_on_status_>> |<<number,number list>>|No
+| <<plugins-{type}s-{plugin}-retry_on_status>> |<<number,number list>>|No
 | <<plugins-{type}s-{plugin}-sort>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
-| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|__Deprecated__
+| <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
@@ -182,19 +192,11 @@ Example:
   * There is no default value for this setting.
 
 Authenticate using Elasticsearch API key. Note that this option also requires
-enabling the `ssl` option.
+enabling the <<plugins-{type}s-{plugin}-ssl_enabled>> option.
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file` 
-
-  * Value type is <<path,path>>
-  * There is no default value for this setting.
-
-SSL Certificate Authority file
-
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`
 
@@ -364,30 +366,140 @@ Which HTTP Status codes to consider for retries (in addition to connection error
 
 Comma-delimited list of `<field>:<direction>` pairs that define the sort order
 
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl` 
+[id="plugins-{type}s-{plugin}-ssl_certificate"]
+===== `ssl_certificate`
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
+SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
 
-SSL
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_key>> is set.
 
-[id="plugins-{type}s-{plugin}-keystore"]
-===== `keystore` 
+[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
+===== `ssl_certificate_authorities`
+
+  * Value type is a list of <<path,path>>
+  * There is no default value for this setting
+
+The .cer or .pem files to validate the server's certificate.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
+===== `ssl_cipher_suites`
+  * Value type is a list of <<string,string>>
+  * There is no default value for this setting
+
+The list of cipher suites to use, listed by priorities.
+Supported cipher suites vary depending on the Java and protocol versions.
+
+
+[id="plugins-{type}s-{plugin}-ssl_enabled"]
+===== `ssl_enabled`
+
+  * Value type is <<boolean,boolean>>
+  * There is no default value for this setting.
+
+Enable SSL/TLS secured communication to Elasticsearch cluster.
+Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
+If no explicit protocol is specified plain HTTP will be used.
 
+[id="plugins-{type}s-{plugin}-ssl_key"]
+===== `ssl_key`
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-The keystore used to present a certificate to the server. It can be either .jks or .p12
+OpenSSL-style RSA private key that corresponds to the <<plugins-{type}s-{plugin}-ssl_certificate>>.
 
-[id="plugins-{type}s-{plugin}-keystore_password"]
-===== `keystore_password` 
+NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate>> is set.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
+===== `ssl_keystore_password`
 
   * Value type is <<password,password>>
   * There is no default value for this setting.
 
 Set the keystore password
 
+[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
+===== `ssl_keystore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The keystore used to present a certificate to the server.
+It can be either `.jks` or `.p12`
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
+===== `ssl_keystore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the keystore filename.
+
+The format of the keystore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+  `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
+===== `ssl_truststore_password`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the truststore password
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
+===== `ssl_truststore_path`
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The truststore to validate the server's certificate.
+It can be either `.jks` or `.p12`.
+
+NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
+===== `ssl_truststore_type`
+
+  * Value can be any of: `jks`, `pkcs12`
+  * If not provided, the value will be inferred from the truststore filename.
+
+The format of the truststore file. It must be either `jks` or `pkcs12`.
+
+[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
+===== `ssl_verification_mode`
+
+  * Value can be any of: `full`, `none`
+  * Default value is `full`
+
+Defines how to verify the certificates presented by another party in the TLS connection:
+
+`full` validates that the server certificate has an issue date that’s within
+the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and
+has a hostname or IP address that matches the names within the certificate.
+
+`none` performs no certificate validation.
+
+WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+
 [id="plugins-{type}s-{plugin}-tag_on_failure"]
 ===== `tag_on_failure` 
 
@@ -405,5 +517,57 @@ Tags the event on failure to look up previous log event information. This can be
 Basic Auth - username
 
 
+[id="plugins-{type}s-{plugin}-deprecated-options"]
+==== Elasticsearch Filter Deprecated Configuration Options
+
+This plugin supports the following deprecated configurations.
+
+WARNING: Deprecated options are subject to removal in future releases.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting|Input type|Replaced by
+| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
+| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
+|=======================================================================
+
+[id="plugins-{type}s-{plugin}-ca_file"]
+===== `ca_file`
+deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+SSL Certificate Authority file
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl`
+deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+SSL
+
+[id="plugins-{type}s-{plugin}-keystore"]
+===== `keystore`
+deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+The keystore used to present a certificate to the server. It can be either .jks or .p12
+
+[id="plugins-{type}s-{plugin}-keystore_password"]
+===== `keystore_password`
+deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
+
+* Value type is <<password,password>>
+* There is no default value for this setting.
+
+Set the keystore password
+
+
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch/client.rb

@@ -11,9 +11,6 @@ module LogStash
       attr_reader :client
 
       def initialize(logger, hosts, options = {})
-        ssl = options.fetch(:ssl, false)
-        keystore = options.fetch(:keystore, nil)
-        keystore_password = options.fetch(:keystore_password, nil)
         user = options.fetch(:user, nil)
         password = options.fetch(:password, nil)
         api_key = options.fetch(:api_key, nil)
@@ -28,17 +25,10 @@ module LogStash
         logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
         transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
 
-        hosts = setup_hosts(hosts, ssl)
+        ssl_options = options.fetch(:ssl, { :enabled => false })
+        ssl_enabled = ssl_options.fetch(:enabled, false)
 
-        ssl_options = {}
-        # set ca_file even if ssl isn't on, since the host can be an https url
-        ssl_options.update(ssl: true, ca_file: options[:ca_file]) if options[:ca_file]
-        ssl_options.update(ssl: true, trust_strategy: options[:ssl_trust_strategy]) if options[:ssl_trust_strategy]
-        if keystore
-          ssl_options[:keystore] = keystore
-          logger.debug("Keystore for client certificate", :keystore => keystore)
-          ssl_options[:keystore_password] = keystore_password.value if keystore_password
-        end
+        hosts = setup_hosts(hosts, ssl_enabled)
 
         client_options = {
                        hosts: hosts,
@@ -59,13 +49,14 @@ module LogStash
 
       private
 
-      def setup_hosts(hosts, ssl)
+      def setup_hosts(hosts, ssl_enabled)
+        hosts = Array(hosts).map { |host| host.to_s } # potential SafeURI#to_s
         hosts.map do |h|
           if h.start_with?('http:/', 'https:/')
             h
           else
             host, port = h.split(':')
-            { host: host, port: port, scheme: (ssl ? 'https' : 'http') }
+            { host: host, port: port, scheme: (ssl_enabled ? 'https' : 'http') }
           end
         end
       end

data/lib/logstash/filters/elasticsearch.rb

@@ -3,6 +3,7 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+require "logstash/plugin_mixins/normalize_config_support"
 
 require_relative "elasticsearch/client"
 require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
@@ -61,17 +62,62 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config :proxy, :validate => :uri_or_empty
 
   # SSL
-  config :ssl, :validate => :boolean, :default => false
+  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
 
   # SSL Certificate Authority file
-  config :ca_file, :validate => :path
+  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
 
   # The keystore used to present a certificate to the server.
   # It can be either .jks or .p12
-  config :keystore, :validate => :path
+  config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead."
 
   # Set the keystore password
-  config :keystore_password, :validate => :password
+  config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead."
+
+  # OpenSSL-style X.509 certificate certificate to authenticate the client
+  config :ssl_certificate, :validate => :path
+
+  # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
+  config :ssl_certificate_authorities, :validate => :path, :list => true
+
+  # The list of cipher suites to use, listed by priorities.
+  # Supported cipher suites vary depending on which version of Java is used.
+  config :ssl_cipher_suites, :validate => :string, :list => true
+
+  # SSL
+  config :ssl_enabled, :validate => :boolean
+
+  # OpenSSL-style RSA private key to authenticate the client
+  config :ssl_key, :validate => :path
+
+  # Set the keystore password
+  config :ssl_keystore_password, :validate => :password
+
+  # The keystore used to present a certificate to the server.
+  # It can be either .jks or .p12
+  config :ssl_keystore_path, :validate => :path
+
+  # The format of the keystore file. It must be either jks or pkcs12
+  config :ssl_keystore_type, :validate => %w[pkcs12 jks]
+
+  # Supported protocols with versions.
+  config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true
+
+  # Set the truststore password
+  config :ssl_truststore_password, :validate => :password
+
+  # The JKS truststore to validate the server's certificate.
+  # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
+  config :ssl_truststore_path, :validate => :path
+
+  # The format of the truststore file. It must be either jks or pkcs12
+  config :ssl_truststore_type, :validate => %w[pkcs12 jks]
+
+  # Options to verify the server's certificate.
+  # "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
+  # chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
+  # "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
+  config :ssl_verification_mode, :validate => %w[full none], :default => 'full'
 
   # Whether results should be sorted or not
   config :enable_sort, :validate => :boolean, :default => true
@@ -91,6 +137,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
+  include LogStash::PluginMixins::NormalizeConfigSupport
+
   attr_reader :clients_pool
 
   ##
@@ -122,13 +170,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
-    if @keystore_password && !@keystore
-      fail "`keystore_password` was provided, without a `keystore`"
-    end
-
+    fill_hosts_from_cloud_id
+    setup_ssl_params!
     validate_authentication
     fill_user_password_from_cloud_auth
-    fill_hosts_from_cloud_id
 
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
@@ -219,16 +264,82 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :password => @password,
       :api_key => @api_key,
       :proxy => @proxy,
-      :ssl => @ssl,
-      :ca_file => @ca_file,
+      :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
-      :retry_on_status => @retry_on_status,
-      :keystore => @keystore,
-      :keystore_password => @keystore_password,
-      :ssl_trust_strategy => trust_strategy_for_ca_trusted_fingerprint
+      :retry_on_status => @retry_on_status
     }
   end
 
+  def client_ssl_options
+    ssl_options = {}
+    ssl_options[:enabled] = @ssl_enabled
+
+    # If the deprecated `ssl` option was explicitly provided, it keeps the same behavior
+    # setting up all the client SSL configs even if ssl => false. Otherwise, it should use
+    # the @ssl_enabled value as it was either explicitly set by the `ssl_enabled` option or
+    # inferred from the hosts scheme.
+    return ssl_options unless @ssl_enabled || original_params.include?('ssl')
+
+    ssl_options[:enabled] = true
+    ssl_certificate_authorities, ssl_truststore_path, ssl_certificate, ssl_keystore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path', 'ssl_certificate', 'ssl_keystore_path')
+
+    if ssl_certificate_authorities && ssl_truststore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities/ca_file" or "ssl_truststore_path" when configuring the CA certificate'
+    end
+
+    if ssl_certificate && ssl_keystore_path
+      raise LogStash::ConfigurationError, 'Use either "ssl_certificate" or "ssl_keystore_path/keystore" when configuring client certificates'
+    end
+
+    if ssl_certificate_authorities&.any?
+      raise LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
+      ssl_options[:ca_file] = ssl_certificate_authorities.first
+    end
+
+    setup_client_ssl_store(ssl_options, 'truststore', ssl_truststore_path)
+    setup_client_ssl_store(ssl_options, 'keystore', ssl_keystore_path)
+    logger.debug("Keystore for client certificate", :keystore => ssl_keystore_path) if ssl_keystore_path
+
+    ssl_key = params["ssl_key"]
+    if ssl_certificate
+      raise LogStash::ConfigurationError, 'Using an "ssl_certificate" requires an "ssl_key"' unless ssl_key
+      ssl_options[:client_cert] = ssl_certificate
+      ssl_options[:client_key] = ssl_key
+    elsif !ssl_key.nil?
+      raise LogStash::ConfigurationError, 'An "ssl_certificate" is required when using an "ssl_key"'
+    end
+
+    ssl_verification_mode = params["ssl_verification_mode"]
+    unless ssl_verification_mode.nil?
+      case ssl_verification_mode
+        when 'none'
+          logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                        "to make sure your data is secure set `ssl_verification_mode => full`"
+          ssl_options[:verify] = :disable
+        else
+          # Manticore's :default maps to Apache HTTP Client's DefaultHostnameVerifier,
+          # which is the modern STRICT verifier that replaces the deprecated StrictHostnameVerifier
+          ssl_options[:verify] = :default
+      end
+    end
+
+    ssl_options[:cipher_suites] = params["ssl_cipher_suites"] if params.include?("ssl_cipher_suites")
+    protocols = params['ssl_supported_protocols']
+    ssl_options[:protocols] = protocols if protocols&.any?
+    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+
+    ssl_options
+  end
+
+  # @param kind is a string [truststore|keystore]
+  def setup_client_ssl_store(ssl_options, kind, store_path)
+    if store_path
+      ssl_options[kind.to_sym] = store_path
+      ssl_options["#{kind}_type".to_sym] = params["ssl_#{kind}_type"] if params.include?("ssl_#{kind}_type")
+      ssl_options["#{kind}_password".to_sym] = params["ssl_#{kind}_password"].value if params.include?("ssl_#{kind}_password")
+    end
+  end
+
   def new_client
     # NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
     # and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
@@ -290,7 +401,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
 
-    if @api_key && @api_key.value && @ssl != true
+    if @api_key && @api_key.value && @ssl_enabled != true
       raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
     end
   end
@@ -353,4 +464,48 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
     end
   end
+
+  def setup_ssl_params!
+    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
+      normalize.with_deprecated_alias(:ssl)
+    end
+
+    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
+    infer_ssl_enabled_from_hosts
+
+    @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
+      normalize.with_deprecated_alias(:keystore)
+    end
+
+    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
+      normalize.with_deprecated_alias(:keystore_password)
+    end
+
+    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
+      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
+        [ca_file]
+      end
+    end
+
+    params['ssl_enabled'] = @ssl_enabled
+    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
+    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
+    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
+  end
+
+  def infer_ssl_enabled_from_hosts
+    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
+
+    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
+  end
+
+  def effectively_ssl?
+    return true if @ssl_enabled
+
+    hosts = Array(@hosts)
+    return false if hosts.nil? || hosts.empty?
+
+    hosts.all? { |host| host && host.to_s.start_with?("https") }
+  end
+
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.14.0'
+  s.version         = '3.15.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,8 +24,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', ">= 7.14.0" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
-
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/elasticsearch_spec.rb

@@ -524,7 +524,7 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context "with ssl" do
-        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
 
         it "should set authorization" do
           plugin.register
@@ -594,7 +594,7 @@ describe LogStash::Filters::Elasticsearch do
 
   describe "ca_trusted_fingerprint" do
     let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
-    let(:config) { {"ca_trusted_fingerprint" => ca_trusted_fingerprint}}
+    let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint}}
 
     subject(:plugin) { described_class.new(config) }
 
@@ -630,8 +630,9 @@ describe LogStash::Filters::Elasticsearch do
 
     let(:config) do
       {
-        'keystore' => keystore_path,
-        'keystore_password' => keystore_password,
+        'hosts' => 'https://localhost:9200',
+        'ssl_keystore_path' => keystore_path,
+        'ssl_keystore_password' => keystore_password,
       }
     end
 

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -0,0 +1,264 @@
+require 'stud/temporary'
+require "elasticsearch"
+require "logstash/codecs/base"
+
+describe "SSL options" do
+  let(:es_client_double) { double("Elasticsearch::Client #{self.inspect}") }
+  let(:hosts) {["localhost"]}
+  let(:settings) { { "ssl_enabled" => true, "hosts" => hosts } }
+
+  subject do
+    require "logstash/filters/elasticsearch"
+    LogStash::Filters::Elasticsearch.new(settings)
+  end
+
+  before do
+    allow(es_client_double).to receive(:close)
+    allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
+    allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
+  end
+
+  after do
+    subject.close
+  end
+
+  context "when ssl_enabled is" do
+    context "true and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "false and cloud_id resolved host is https" do
+      let(:settings) {{
+        "ssl_enabled" => false,
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should not infer the ssl_enabled value" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+  end
+
+  context "when neither ssl nor ssl_enabled is set" do
+    let(:settings) { super().reject { |k| %w[ssl ssl_enabled].include?(k) } }
+
+    context "and there is no https hosts" do
+      let(:hosts) { %w[http://es01 http://es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and there is https hosts" do
+      let(:hosts) { %w[https://sec-es01 https://sec-es01] }
+
+      it "should infer the ssl_enabled value to true" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+
+    context "and hosts have no scheme defined" do
+      let(:hosts) { %w[es01 es01] }
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(false)
+        expect(subject.params).to match hash_including("ssl_enabled" => false)
+      end
+    end
+
+    context "and cloud_id resolved host is https" do
+      let(:settings) {{
+        "cloud_id" => "sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=="
+      }}
+
+      it "should infer the ssl_enabled value to false" do
+        subject.register
+        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
+        expect(subject.params).to match hash_including("ssl_enabled" => true)
+      end
+    end
+  end
+
+  context "when ssl_verification_mode" do
+    context "is set to none" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => "none",
+      ) }
+
+      it "should print a warning" do
+        expect(subject.logger).to receive(:warn).with(/You have enabled encryption but DISABLED certificate verification/).at_least(:once)
+        allow(subject.logger).to receive(:warn).with(any_args)
+
+        subject.register
+      end
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+
+    context "is set to full" do
+      let(:settings) { super().merge(
+        "ssl_verification_mode" => 'full',
+      ) }
+
+      it "should pass the flag to the ES client" do
+        expect(::Elasticsearch::Client).to receive(:new) do |args|
+          expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :default)
+        end.and_return(es_client_double)
+
+        subject.register
+      end
+    end
+  end
+
+  context "with the conflicting configs" do
+    context "ssl_certificate_authorities and ssl_truststore_path set" do
+      let(:ssl_truststore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_truststore_path" => ssl_truststore_path,
+        "ssl_certificate_authorities" => ssl_certificate_authorities_path
+      ) }
+
+      after :each do
+        File.delete(ssl_truststore_path)
+        File.delete(ssl_certificate_authorities_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate_authorities\/ca_file" or "ssl_truststore_path"/)
+      end
+    end
+
+    context "ssl_certificate and ssl_keystore_path set" do
+      let(:ssl_keystore_path) { Stud::Temporary.file.path }
+      let(:ssl_certificate_path) { Stud::Temporary.file.path }
+      let(:settings) { super().merge(
+        "ssl_certificate" => ssl_certificate_path,
+        "ssl_keystore_path" => ssl_keystore_path
+      ) }
+
+      after :each do
+        File.delete(ssl_keystore_path)
+        File.delete(ssl_certificate_path)
+      end
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Use either "ssl_certificate" or "ssl_keystore_path\/keystore"/)
+      end
+    end
+  end
+
+  context "when configured with Java store files" do
+    let(:ssl_truststore_path) { Stud::Temporary.file.path }
+    let(:ssl_keystore_path) { Stud::Temporary.file.path }
+
+    after :each do
+      File.delete(ssl_truststore_path)
+      File.delete(ssl_keystore_path)
+    end
+
+    let(:settings) { super().merge(
+      "ssl_truststore_path" => ssl_truststore_path,
+      "ssl_truststore_type" => "jks",
+      "ssl_truststore_password" => "foo",
+      "ssl_keystore_path" => ssl_keystore_path,
+      "ssl_keystore_type" => "jks",
+      "ssl_keystore_password" => "bar",
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :enabled => true,
+                                      :keystore => ssl_keystore_path,
+                                      :keystore_type => "jks",
+                                      :keystore_password => "bar",
+                                      :truststore => ssl_truststore_path,
+                                      :truststore_type => "jks",
+                                      :truststore_password => "foo",
+                                      :verify => :default,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+  end
+
+  context "when configured with certificate files" do
+    let(:ssl_certificate_authorities_path) { Stud::Temporary.file.path }
+    let(:ssl_certificate_path) { Stud::Temporary.file.path }
+    let(:ssl_key_path) { Stud::Temporary.file.path }
+    let(:settings) { super().merge(
+      "ssl_certificate_authorities" => [ssl_certificate_authorities_path],
+      "ssl_certificate" => ssl_certificate_path,
+      "ssl_key" => ssl_key_path,
+      "ssl_verification_mode" => "full",
+      "ssl_cipher_suites" => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+      "ssl_supported_protocols" => ["TLSv1.3"]
+    ) }
+
+    after :each do
+      File.delete(ssl_certificate_authorities_path)
+      File.delete(ssl_certificate_path)
+      File.delete(ssl_key_path)
+    end
+
+    it "should pass the parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to match hash_including(
+                                      :enabled => true,
+                                      :ca_file => ssl_certificate_authorities_path,
+                                      :client_cert => ssl_certificate_path,
+                                      :client_key => ssl_key_path,
+                                      :verify => :default,
+                                      :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
+                                      :protocols => ["TLSv1.3"],
+                                    )
+      end.and_return(es_client_double)
+
+      subject.register
+    end
+
+    context "and only the ssl_certificate is set" do
+      let(:settings) { super().reject { |k| "ssl_key".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Using an "ssl_certificate" requires an "ssl_key"/)
+      end
+    end
+
+    context "and only the ssl_key is set" do
+      let(:settings) { super().reject { |k| "ssl_certificate".eql?(k) } }
+
+      it "should raise a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /An "ssl_certificate" is required when using an "ssl_key"/)
+      end
+    end
+  end
+end
+

data/spec/filters/integration/elasticsearch_spec.rb

@@ -29,7 +29,7 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
 
   let(:config) do
     config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
-    config = { 'ca_file' => ca_path }.merge(config) if SECURE_INTEGRATION
+    config = { 'ssl_certificate_authorities' => ca_path }.merge(config) if SECURE_INTEGRATION
     config
   end
 
@@ -92,7 +92,7 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
   context 'setting host:port (and ssl)' do # reproduces GH-155
 
     let(:config) do
-      super().merge "hosts" => [ESHelper.get_host_port], "ssl" => SECURE_INTEGRATION
+      super().merge "hosts" => [ESHelper.get_host_port], "ssl_enabled" => SECURE_INTEGRATION
     end
 
     it "works" do
@@ -110,9 +110,9 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       let(:config) do
         super().merge(
           "hosts" => [ESHelper.get_host_port],
-          "keystore" => keystore_path,
-          "keystore_password" => keystore_password,
-          "ssl" => true,
+          "ssl_keystore_path" => keystore_path,
+          "ssl_keystore_password" => keystore_password,
+          "ssl_enabled" => true,
           "fields" => { "this" => "contents", "response" => "four-oh-four" }
         )
       end
@@ -132,7 +132,7 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
 
         let(:config) do
           bc = super()
-          bc.delete('ca_file')
+          bc.delete('ssl_certificate_authorities')
           bc.merge({
             'ca_trusted_fingerprint' => ca_trusted_fingerprint,
             'fields' => { "this" => "contents", "response" => "four-oh-four" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.14.0
+  version: 3.15.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-23 00:00:00.000000000 Z
+date: 2023-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-normalize_config_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -135,6 +149,7 @@ files:
 - logstash-filter-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/elasticsearch_ssl_spec.rb
 - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/generate_test_certs.openssl.cnf
 - spec/filters/fixtures/generate_test_certs.sh
@@ -180,13 +195,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events
 test_files:
 - spec/es_helper.rb
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/elasticsearch_ssl_spec.rb
 - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
 - spec/filters/fixtures/generate_test_certs.openssl.cnf
 - spec/filters/fixtures/generate_test_certs.sh