logstash-output-elasticsearch 11.4.1-java → 11.7.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1ce0504aa5082dc3fc958b72019f243c6cad0174bb883612c25eb5d7f8aedfe
-  data.tar.gz: 4d28c34d89b5a70a02c10ebd3b8d425111d3902bc58fb9e428acadee4a471646
+  metadata.gz: 0002c4b7258cc2493f3ab5e1ddadf6dc3f3e6d51c3585af9c93f9d9a578cf01f
+  data.tar.gz: 1f4b7e4587ee5831ac32f499dba2e73324f5b00578db20369b46e2b53b0fa874
 SHA512:
-  metadata.gz: d1d0680cf639e01b6c2a2a3b8eb87a2ad525764d006fb3cbfadc06c845925d9f1055075265d7327794e326b1a9df67308d85ebefafb38297bd2121499ecc2279
-  data.tar.gz: 285bafc2ffa2ae726bdafdb3176c193c7cd20979625e59cd7526445ae5243137e6dad37cc64208bb3494fc45afe44d913f7b7b4188fcf6b25faf1e9fb6a04d06
+  metadata.gz: a45147a1fc1f7d73bae911f05417d36150796edae9bd7622c11dbeb35bebbc938c3eddc7d7f65652743dfe94425ba4b1cc6905eaae4390116e2fa19d5c1f4be3
+  data.tar.gz: 2ad8fec97436ac571436407711793d8a97055f7a2fbe986ce9f75417d0ce414a01467c900c47c08e1caa7eb6ffd7884edcabedb12b3684c5b40ed543c040fc21

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 11.7.0
+ - Feature: deprecates the `failure_type_logging_whitelist` configuration option, renaming it `silence_errors_in_log` [#1068](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1068)
+
+## 11.6.0
+ - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#1074](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1074)
+
+## 11.5.0
+ - Feat: add ssl_supported_protocols option [#1055](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1055)
+
+## 11.4.2
+ - [DOC] Add `v8` to supported values for ecs_compatiblity defaults [#1059](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1059)
+
 ## 11.4.1
  - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
    - the underlying changes include latest HttpClient (4.5.13)

data/docs/index.asciidoc

@@ -307,6 +307,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-bulk_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
@@ -331,6 +332,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-silence_errors_in_log>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-manage_template>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-parent>> |<<string,string>>|No
@@ -355,6 +357,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
@@ -421,6 +424,15 @@ this defaults to a concatenation of the path parameter and "_bulk"
 
 The .cer or .pem file to validate the server's certificate.
 
+[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
+===== `ca_trusted_fingerprint`
+
+* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
+* There is no default value for this setting.
+* Use of this option _requires_ Logstash 8.3+
+
+The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
+
 [id="plugins-{type}s-{plugin}-cloud_auth"]
 ===== `cloud_auth`
 
@@ -554,7 +566,7 @@ If you don't set a value for this option:
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: does not provide ECS-compatible templates
-** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema
+** `v1`,`v8`: Elastic Common Schema-compliant behavior
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.
@@ -573,9 +585,7 @@ of this setting affects the _default_ values of:
   * Value type is <<array,array>>
   * Default value is `[]`
 
-Set the Elasticsearch errors in the whitelist that you don't want to log.
-A useful example is when you want to skip all 409 errors
-which are `document_already_exists_exception`.
+NOTE: Deprecated, refer to <<plugins-{type}s-{plugin}-silence_errors_in_log>>.
 
 [id="plugins-{type}s-{plugin}-custom_headers"]
 ===== `custom_headers`
@@ -732,8 +742,7 @@ Indexes may not contain uppercase characters.
 For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
 Logstash uses
 http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
-formats] for the index pattern from event timestamp.
-
+formats] and the `@timestamp` field of each event is being used as source for the date.
 
 [id="plugins-{type}s-{plugin}-keystore"]
 ===== `keystore`
@@ -952,6 +961,25 @@ Set variable name passed to script (scripted update)
 
 if enabled, script is in charge of creating non-existent document (scripted update)
 
+[id="plugins-{type}s-{plugin}-silence_errors_in_log"]
+===== `silence_errors_in_log`
+
+* Value type is <<array,array>>
+* Default value is `[]`
+
+Defines the list of Elasticsearch errors that you don't want to log.
+A useful example is when you want to skip all 409 errors
+which are `document_already_exists_exception`.
+
+[source,ruby]
+    output {
+      elasticsearch {
+        silence_errors_in_log => ["document_already_exists_exception"]
+      }
+    }
+
+NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
+
 [id="plugins-{type}s-{plugin}-sniffing"]
 ===== `sniffing`
 
@@ -1004,6 +1032,23 @@ Option to validate the server's certificate. Disabling this severely compromises
 For more information on disabling certificate verification please read
 https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`
 
@@ -1018,8 +1063,8 @@ If not set, the included template will be used.
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
-  ** ECS Compatibility disabled: `logstash`
-  ** ECS Compatibility enabled: `ecs-logstash`
+    ** ECS Compatibility disabled: `logstash`
+    ** ECS Compatibility enabled: `ecs-logstash`
 
 
 This configuration option defines how the template is named inside Elasticsearch.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -283,11 +283,11 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def client_settings
-      @options[:client_settings] || {}
+      @_client_settings ||= @options[:client_settings] || {}
     end
 
     def ssl_options
-      client_settings.fetch(:ssl, {})
+      @_ssl_options ||= client_settings.fetch(:ssl, {})
     end
 
     def http_compression

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -132,11 +132,18 @@ module LogStash; module Outputs; class ElasticSearch;
         ssl_options[:keystore] = keystore
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
+
       if !params["ssl_certificate_verification"]
         logger.warn "You have enabled encryption but DISABLED certificate verification, " +
                     "to make sure your data is secure remove `ssl_certificate_verification => false`"
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
+
+      ssl_options[:trust_strategy] = params["ssl_trust_strategy"] if params.include?("ssl_trust_strategy")
+
+      protocols = params['ssl_supported_protocols']
+      ssl_options[:protocols] = protocols if protocols && protocols.any?
+
       { ssl: ssl_options }
     end
 

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json

@@ -4,7 +4,7 @@
   ],
   "mappings": {
     "_meta": {
-      "version": "8.0.0-dev"
+      "version": "8.0.1"
     },
     "date_detection": false,
     "dynamic_templates": [

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json

@@ -15,7 +15,7 @@
     },
     "mappings": {
       "_meta": {
-        "version": "8.0.0-dev"
+        "version": "8.0.1"
       },
       "date_detection": false,
       "dynamic_templates": [

data/lib/logstash/outputs/elasticsearch.rb

@@ -266,6 +266,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def register
+    if !failure_type_logging_whitelist.empty?
+      log_message = "'failure_type_logging_whitelist' is deprecated and in a future version of Elasticsearch " +
+        "output plugin will be removed, please use 'silence_errors_in_log' instead."
+      @deprecation_logger.deprecated log_message
+      @logger.warn log_message
+      @silence_errors_in_log = silence_errors_in_log | failure_type_logging_whitelist
+    end
+
     @after_successful_connection_done = Concurrent::AtomicBoolean.new(false)
     @stopping = Concurrent::AtomicBoolean.new(false)
 

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -1,3 +1,6 @@
+
+require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+
 module LogStash; module PluginMixins; module ElasticSearch
   module APIConfigs
 
@@ -52,6 +55,9 @@ module LogStash; module PluginMixins; module ElasticSearch
         # The .cer or .pem file to validate the server's certificate
         :cacert => { :validate => :path },
 
+        # One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
+        :ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
+
         # The JKS truststore to validate the server's certificate.
         # Use either `:truststore` or `:cacert`
         :truststore => { :validate => :path },
@@ -66,6 +72,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
+        :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting
@@ -91,10 +99,14 @@ module LogStash; module PluginMixins; module ElasticSearch
         # a timeout occurs, the request will be retried.
         :timeout => { :validate => :number, :default => 60 },
 
-        # Set the Elasticsearch errors in the whitelist that you don't want to log.
+        # Deprecated, refer to `silence_errors_in_log`.
+        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+
+        # Defines the list of Elasticsearch errors that you don't want to log.
         # A useful example is when you want to skip all 409 errors
         # which are `document_already_exists_exception`.
-        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+        # Deprecates `failure_type_logging_whitelist`.
+        :silence_errors_in_log => { :validate => :array, :default => [] },
 
         # While the output tries to reuse connections efficiently we have a maximum.
         # This sets the maximum number of open connections the output will create.
@@ -161,7 +173,13 @@ module LogStash; module PluginMixins; module ElasticSearch
     }.freeze
 
     def self.included(base)
-      CONFIG_PARAMS.each { |name, opts| base.config(name, opts) }
+      CONFIG_PARAMS.each do |name, opts|
+        if opts.kind_of?(Module)
+          base.include(opts)
+        else
+          base.config(name, opts)
+        end
+      end
     end
   end
 end; end; end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -27,6 +27,11 @@ module LogStash; module PluginMixins; module ElasticSearch
       fill_hosts_from_cloud_id
       setup_hosts
 
+      # inject the TrustStrategy from CATrustedFingerprintSupport
+      if trust_strategy_for_ca_trusted_fingerprint
+        params["ssl_trust_strategy"] = trust_strategy_for_ca_trusted_fingerprint
+      end
+
       params["metric"] = metric
       if @proxy.eql?('')
         @logger.warn "Supplied proxy setting (proxy => '') has no effect"
@@ -165,7 +170,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
       sleep_interval = @retry_initial_interval
 
-      while submit_actions && submit_actions.length > 0
+      while submit_actions && submit_actions.size > 0
 
         # We retry with whatever is didn't succeed
         begin
@@ -279,7 +284,7 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
 
     def log_failure_type?(failure)
-      !failure_type_logging_whitelist.include?(failure["type"])
+      !silence_errors_in_log.include?(failure["type"])
     end
 
     # Rescue retryable errors during bulk submission

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.4.1'
-
+  s.version         = '11.7.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -26,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/fixtures/test_certs/ca.crt

@@ -1,32 +1,29 @@
 -----BEGIN CERTIFICATE-----
-MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
-BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
-DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
-WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
-BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
-AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
-gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
-B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
-6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
-4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
-nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
-9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
-wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
-ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
-UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
-w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
-g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
-dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
-Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
-UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
-1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
-MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
-jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
-Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
-DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
-lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
-36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
-rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
-sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
-nIg9fQPvVw+fIHACZQ==
+MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQEFBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
+BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
+DAZSb290Q0EwHhcNMjIwNTIzMTcyODU1WhcNMjMwNTIzMTcyODU1WjBMMQswCQYD
+VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
+TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
+AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh
+5er5JF2iomYOHiD5JncYr4YmRQKuYfD6B1WI5FuQthD/OlA1/RHqtbY27J33SaO6
+6ro8gal7vjHrXKQkefVYRwdfO6DqqbhV6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQq
+GZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju4jPhp+43BkGwEimI5km7jNC1nwjiHtxD
+sY/s93AKa/vLktXKUK5nA3jjJOhAbRTVnbOAgxFt0YbX98xW/aUqscgBUVs9J/My
+TRMwVKJ7Vsmth1PdJQksUASuzESlSPl09dMjTQ+MXzJDt0JvX8SIJPmbBng78MSa
+CUhpOZiii1l2mBfPWejx20I/SMCUNmzbwm2w9JD50Jv2iX4l4ge4H1CIK1/orW1p
+dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv
+5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx
+r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP
+LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ
+KoZIhvcNAQEFBQADggIBAAGUkKT6GwoOOqPT7/FTdjU7h6q2vAaevd/TbYOBjhMw
+XNVpmuIE/r9mXF5lR1MuMebUXIWrrthXeX0TqucQzsJI+pCNugQP0HyUNF83S4l9
+G/0xvL2iYx7ftkMtje/NNiCUMpaXxulHi94fx4Kbivihlga6f8OF4+wNmIatb5bp
+SnLE/CsE3vLrwPZgcROXhKy8ESAI4mLclOn86nOXbIunFRNxFHis/dQOxX+CfkPp
+CDJv10jiaG9HCcGppNzDfxP0+v67RU2zTsCktEIILYBGTBBi5jczbtbtM0L/VCIA
+AoJTGWkKtPUesAuthPaHsOAXUSnNYakf4PEyJF6g9mIiFyeosGNhgNcA6coKsX+6
+pzS2pr+X2TiuNMGTCayFFIDpLvr99pPbf1yq2IBkEn09uZHLS/xyDxYtNaJAhbUh
+JuszjjjfHDHVTnDykyIoTzfeLICFKoMRL0rUedljqYuI0QAic6rgn68dkfYK8zzy
+IjRK5wZ4rM94xcEQfJSDxusJSPlCPTN4oe6A5HCaHe4GKYihiGKlOMGWkCxwYVa5
+nl88TNh2xG6y+ZZMQDQJdRBwmJ/i+rDRTxHGuemQka5bZH8PRZGBYUiIRVS7N8px
+Y1ITp+FdSlJAm41UGChuF8Our31AqZYvLNRWAvLJRhR/kNM9HMeURz7zI/KKYhlA
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.der.sha256

@@ -0,0 +1 @@
+3e1c908fb2d7f1634643bb75462119c55a7cc392cd1877dd91d9f15f87e86757

data/spec/fixtures/test_certs/renew.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+cd "$(dirname "$0")"
+
+openssl x509 -x509toreq -in ca.crt -signkey ca.key -out ca.csr
+openssl x509 -req -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
+openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
+
+openssl x509 -x509toreq -in test.crt -signkey test.key -out test.csr
+openssl x509 -req -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr
+openssl x509 -in test.crt -outform der | sha256sum | awk '{print $1}' > test.der.sha256
+openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12

data/spec/fixtures/test_certs/test.crt

@@ -1,36 +1,30 @@
 -----BEGIN CERTIFICATE-----
-MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
-MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
-BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
-CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
-VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
-AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
-IVxXSvau2ZZ94rkQR2xNL8TLijBNx46mU+kCniy8X5r+LX9seGqdBhhTh/tCJzh8
-MCzMt2JIijSjVyw28iiCb8/669LMTp5lFlRKajj11jlIpIm3o+OHqUzYwcSOw8og
-p0A3nvAQ33Srghm/oAcT2umGrFyYXWT6PnGaEJRLUQn7LuHJnRLseCF2Cn/RzFK7
-/tiVVjImmQiVB3dE9fMR/pVJiO2v0COnWuG+/brXWrQIHk0AuD8pHc6Iw9iZODkc
-Ao53B41qbvqcbdXFN5XfL4tb+lkBuLioCX7j9zR44awvuj9hKfuqFOFTUBZL2RjV
-bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
-1GfnSVZBwO4MMd85taHNvitLnkEREjANSoPUuAJF3SKRHE9K8jUAzhyXflvgNNoM
-tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
-hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
-2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
-6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
-BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
-YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
-FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
-CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
-BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
-A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
-f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
-9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
-y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
-HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
-liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
-y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
-lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
-re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
-Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
-VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
-FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
+MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQEFBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
+BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
+DAZSb290Q0EwHhcNMjIwNTIzMTcyODU1WhcNMjMwNTIzMTcyODU1WjBTMQswCQYD
+VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
+TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2
+rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAszLdi
+SIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdAN57w
+EN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7YlVYy
+JpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKOdweN
+am76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxTCrKR
+h58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn50lW
+QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO
+fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe
+cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3
+cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID
+AQABMA0GCSqGSIb3DQEBBQUAA4ICAQAhg0y7SfTv2RIcU8tsvSGOpXM6KPx111eJ
+pWrJTEZBCieCUhkonmlUifZHjV6B4d1OiS3GBXP0iAWff3Pb40co8AR4Brhne7Bd
+xkD8TKReJ/sfeKDsr3enLxFrmcxWCD5x9b6ybl7aotzP1S286rPpehE3QKJM3L1Z
+tRZik7pE3Iju4PpnvfaOAoJup9+v9Y6ySMKcMY19b/izM9VPwF+hllFQ31bibCRz
+Mqa1o9k27e1MQEH7LpGcUBY18fofb2Ie3Y+wzfXm/xG/JrXxgRD/rpyBapCM6jcZ
+C11mj2st+0/9pj4trhq39fj7f3+GWvOY2kZj9x/05gXcFmeaVOnZr/njcQfLd9K7
+2WD1tgr4fTgG8H3UOUMfw5u+pGfAeky1mgHwkjNT6H9PDtoi3lh4y/CmspSSv6t7
+szbaKZUsxXz49hLt8q4IrtHrzqVa3Jk5YXt3GAFlXP1ZnwV5/fvltFNrvpWeUjTn
+IR9CLcYTV9gsLVq7OKFAwelBmcBbbyRoQdqFeoePhv6Frw9mDBoyYoZ8oMmg20to
+in9VrxtbDjw9qaSY58kGNj1cKV5eUnKOi9v0gDjrVyKVuesnDeOmoi25/YvBbBA5
+TKgMUwSmJ2P5p6W4h0ftV/Nyy1Hx/rwJ7ZcvUJCtwgCNOeXw9e61Ys+C2ruLSPuh
+wRncxHmbiw==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.der.sha256

@@ -0,0 +1 @@
+dca380f330bdf3d4b242b3c48d541c4698eaffa0d532316b27e6080443e601b5

data/spec/integration/outputs/index_spec.rb

@@ -60,26 +60,63 @@ describe "indexing" do
 
   let(:curl_opts) { nil }
 
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
   def curl_and_get_json_response(url, method: :get); require 'open3'
+    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
     begin
-      stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+      out, err, status = Open3.capture3(cmd)
     rescue Errno::ENOENT
       fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
     if status.success?
-      LogStash::Json.load(stdout)
+      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+      if http_status.strip[0].to_i > 2
+        error = (LogStash::Json.load(out)['error']) rescue nil
+        if error
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        else
+          warn out
+          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+        end
+      end
+
+      LogStash::Json.load(out)
     else
-      fail "curl failed: #{status}\n  #{stdout}"
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
     end
   end
 
+  let(:initial_events) { [] }
+
+  let(:do_register) { true }
+
   before do
-    subject.register
-    subject.multi_receive([])
+    subject.register if do_register
+    subject.multi_receive(initial_events) if initial_events
   end
-  
+
+  after do
+    subject.do_close
+  end
+
   shared_examples "an indexer" do |secure|
+    before(:each) do
+      host_unreachable_error_class = LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError
+      allow(host_unreachable_error_class).to receive(:new).with(any_args).and_wrap_original do |m, original, url|
+        if original.message.include?("PKIX path building failed")
+          $stderr.puts "Client not connecting due to PKIX path building failure; " +
+                         "shutting plugin down to prevent infinite retries"
+          subject.close # premature shutdown to prevent infinite retry
+        end
+        m.call(original, url)
+      end
+    end
+
     it "ships events" do
       subject.multi_receive(events)
 
@@ -121,6 +158,32 @@ describe "indexing" do
     end
   end
 
+  shared_examples "PKIX path failure" do
+    let(:do_register) { false }
+    let(:host_unreachable_error_class) { LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError }
+
+    before(:each) do
+      limit_execution
+    end
+
+    let(:limit_execution) do
+      Thread.new { sleep 5; subject.close }
+    end
+
+    it 'fails to establish TLS' do
+      allow(host_unreachable_error_class).to receive(:new).with(any_args).and_call_original.at_least(:once)
+
+      subject.register
+      limit_execution.join
+
+      sleep 1
+
+      expect(host_unreachable_error_class).to have_received(:new).at_least(:once) do |original, url|
+        expect(original.message).to include("PKIX path building failed")
+      end
+    end
+  end
+
   describe "an indexer with custom index_type", :integration => true do
     let(:config) {
       {
@@ -146,17 +209,17 @@ describe "indexing" do
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
     let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
-    let(:es_url) {"https://elasticsearch:9200"}
+    let(:es_url) { "https://#{get_host_port}" }
     let(:config) do
       {
-        "hosts" => ["elasticsearch:9200"],
+        "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
         "ssl" => true,
         "cacert" => cacert,
         "index" => index
       }
-    end
+    end 
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 
@@ -197,6 +260,8 @@ describe "indexing" do
 
     else
 
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password
+
       it_behaves_like("an indexer", true)
 
       describe "with a password requiring escaping" do
@@ -219,6 +284,63 @@ describe "indexing" do
         include_examples("an indexer", true)
       end
 
+      context "without providing `cacert`" do
+        let(:config) do
+          super().tap do |c|
+            c.delete("cacert")
+          end
+        end
+
+        it_behaves_like("PKIX path failure")
+      end
+
+      if Gem::Version.new(LOGSTASH_VERSION) >= Gem::Version.new("8.3.0")
+        context "with `ca_trusted_fingerprint` instead of `cacert`" do
+          let(:config) do
+            super().tap do |c|
+              c.delete("cacert")
+              c.update("ca_trusted_fingerprint" => ca_trusted_fingerprint)
+            end
+          end
+          let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/test.der.sha256").chomp }
+
+
+          it_behaves_like("an indexer", true)
+
+          context 'with an invalid `ca_trusted_fingerprint`' do
+            let(:ca_trusted_fingerprint) { super().reverse }
+
+            it_behaves_like("PKIX path failure")
+          end
+        end
+      end
+
+      context 'with enforced TLSv1.3 protocol' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] }
+
+        it_behaves_like("an indexer", true)
+      end
+
+      context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] }
+
+        let(:initial_events) { nil }
+
+        it "does not ship events" do
+          curl_and_get_json_response index_url, method: :put # make sure index exists
+          Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
+          sleep 2.5
+
+          curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+          result = curl_and_get_json_response "#{index_url}/_count?q=*"
+          cur_count = result["count"]
+          expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
+          # [Manticore::ClientProtocolException] Received fatal alert: protocol_version
+        end
+
+      end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3'
+
     end
 
   end

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -33,7 +33,7 @@ describe "SSL option" do
     
     it "should pass the flag to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
-        expect(args[:ssl]).to eq(:enabled => true, :verify => :disable)
+        expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable)
       end.and_return(manticore_double)
       
       subject.register

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -45,8 +45,8 @@ describe "whitelisting error types in expected behavior" do
     end
   end
 
-  describe "when failure logging is disabled for docuemnt exists error" do
-    let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+  describe "when failure logging is disabled for document exists error" do
+    let(:settings) { super().merge("silence_errors_in_log" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
       expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.4.1
+  version: 11.7.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-19 00:00:00.000000000 Z
+date: 2022-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +98,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-ca_trusted_fingerprint_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -245,8 +259,11 @@ files:
 - spec/fixtures/template-with-policy-es7x.json
 - spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.der.sha256
 - spec/fixtures/test_certs/test.key
 - spec/fixtures/test_certs/test.p12
 - spec/fixtures/test_certs/test_invalid.crt
@@ -327,8 +344,11 @@ test_files:
 - spec/fixtures/template-with-policy-es7x.json
 - spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
+- spec/fixtures/test_certs/test.der.sha256
 - spec/fixtures/test_certs/test.key
 - spec/fixtures/test_certs/test.p12
 - spec/fixtures/test_certs/test_invalid.crt