logstash-output-elasticsearch 11.4.2-java → 11.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5aa752898bbe2a9417fc15ec7fb1e9a55a804a03c84cef30a936206d5b97005
-  data.tar.gz: 89ec0c8d8ff45e4ef4892c6b2a2a5de3f108d07443f925a897e058e79987fcba
+  metadata.gz: 7eab2dc342c636e2d5df4e2f9bbd776b446cf110a0803c11fc919fcbbd2f2d83
+  data.tar.gz: 565ed3031b685f8541853043d486bda0cff8aa7da2c6f2733e0e2b43345de099
 SHA512:
-  metadata.gz: 633594b5c0010ca14c42231f8f1851bfdadd17819b61ea75f90d89c5db805d20d1a809ade265aedd432d80138965bdca18ec66819f387f69452f10d07a2ddb88
-  data.tar.gz: 9b4b8afaf80c0a9eebe7f7443c4fba5073c73823de87259641d373be62b9f52eb72e527d28d1dd4aca724abb487e13d1f49eff16f5ea0d528ab2ac7722522317
+  metadata.gz: 1fbb452170d276531d9b202538d9cf0a23b834053309c818aaaff6f75dd41ecd3d787394234c295639d95d2fe763d1063424cff56b598d1ba16e2fe92374df44
+  data.tar.gz: 6b7f17fa679306e65ec94a048ef2205ec9f884cbd11fe1adf861dbb1eccaea752105c3bf149a4fe08f1cdb3c6189200d612dc4cece52ac937eb78d599b188926

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 11.5.0
+ - Feat: add ssl_supported_protocols option [#1055](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1055)
+
 ## 11.4.2
- - Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169)
+ - [DOC] Add `v8` to supported values for ecs_compatiblity defaults [#1059](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1059)
 
 ## 11.4.1
  - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)

data/docs/index.asciidoc

@@ -293,11 +293,6 @@ index level and `monitoring` permissions at cluster level. The `monitoring`
 permission at cluster level is necessary to perform periodic connectivity
 checks.
 
-[id="plugins-{type}s-{plugin}-handling-non-utf-8"]
-==== Handling non UTF-8 data
-
-This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8.
-When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`).
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Output Configuration Options
@@ -360,6 +355,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
@@ -559,7 +555,7 @@ If you don't set a value for this option:
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: does not provide ECS-compatible templates
-** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema
+** `v1`,`v8`: Elastic Common Schema-compliant behavior
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.
@@ -1009,6 +1005,23 @@ Option to validate the server's certificate. Disabling this severely compromises
 For more information on disabling certificate verification please read
 https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
+
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`
 
@@ -1023,8 +1036,8 @@ If not set, the included template will be used.
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
-  ** ECS Compatibility disabled: `logstash`
-  ** ECS Compatibility enabled: `ecs-logstash`
+    ** ECS Compatibility disabled: `logstash`
+    ** ECS Compatibility enabled: `ecs-logstash`
 
 
 This configuration option defines how the template is named inside Elasticsearch.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -127,9 +127,6 @@ module LogStash; module Outputs; class ElasticSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-
-        as_json.scrub! # ensure generated JSON is valid UTF-8
-
         if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
           stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
           logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
@@ -286,11 +283,11 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def client_settings
-      @options[:client_settings] || {}
+      @_client_settings ||= @options[:client_settings] || {}
     end
 
     def ssl_options
-      client_settings.fetch(:ssl, {})
+      @_ssl_options ||= client_settings.fetch(:ssl, {})
     end
 
     def http_compression

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -132,11 +132,16 @@ module LogStash; module Outputs; class ElasticSearch;
         ssl_options[:keystore] = keystore
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
+
       if !params["ssl_certificate_verification"]
         logger.warn "You have enabled encryption but DISABLED certificate verification, " +
                     "to make sure your data is secure remove `ssl_certificate_verification => false`"
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
+
+      protocols = params['ssl_supported_protocols']
+      ssl_options[:protocols] = protocols if protocols && protocols.any?
+
       { ssl: ssl_options }
     end
 

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -66,6 +66,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
+        :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.4.2'
-
+  s.version         = '11.5.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/test_certs/ca.crt

@@ -1,31 +1,32 @@
 -----BEGIN CERTIFICATE-----
-MIIFRTCCAy2gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
-MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
-BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMEwx
-CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
-VQQKDAVNeUxhYjEPMA0GA1UEAwwGUm9vdENBMIICIjANBgkqhkiG9w0BAQEFAAOC
-Ag8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
-Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
-nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
-AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
-COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
-Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
-eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
-X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
-eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
-CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
-RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
-AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoVp0nHjq6mJ/UGuFhnSK
-7yjbPyswDQYJKoZIhvcNAQELBQADggIBADeI0gRfzF5zyhCCmtlA44L1fo3BEYtm
-0Deq2W6bsW9zakYT7fivCUbt0VtS0jzhAlbaLfksJk+Emg+kpsy/CdFr8nk9jlpN
-XJMKwjQIVGYDDlJsP9VDw2m+HhUugEpx09QCREaCWCDtobE9bd42VloPd0u3VWZx
-u5rSrZCy5wjFeq3dPEFPyDTfV6p1kXkwy9pdG3ww0rw0dqj5OqcGhLyG8b7q7LLP
-aFVuwjnutrBJtDNOsvbD/dyybQVj3hF1IpVZLwoFQ3ji43/X3NJ5jdo/WrwwdwbH
-NE6YxNlsIrql4sJFeHzNukTxcC5wHcOLtc6E6dhe7uK+5rRD75+odz85rPLsBTPt
-zje8OWyxO1cGol7inpKUsO9ITFQdDC9iyRjvDRouwneT+XeVwW6BmAKg4DvXdLtQ
-oQNLlMbbZWskvWoI4amXwSpzWmn4mRK3e4IPWFju64QRvkEExFynvC69jeih5Llh
-JU3OjLobG4wKOQHB7w1UjiPJNsxGB0P1zug6ztTBLjsvJoxdJuIGkCsI7g12cKNR
-PbXp8453klofFpMNGlCP8dns92Qu20pPvYJcWiC9k42nYu3FHDMbAfSrFU90RNS6
-hRWkpXZOMIY1GdudgsxanDtJgFRbkPzwNkMcBQ1538f9U4MrJ9vPsuKx4bYbZ//8
-pIe/z3b1V9zV
+MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
+BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
+DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
+WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
+BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
+gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
+B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
+6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
+4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
+nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
+9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
+wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
+ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
+UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
+w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
+g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
+dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
+UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
+1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
+MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
+jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
+Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
+DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
+lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
+36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
+rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
+sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
+nIg9fQPvVw+fIHACZQ==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.crt

@@ -1,7 +1,7 @@
 -----BEGIN CERTIFICATE-----
-MIIF1jCCA76gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
 MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
-BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMFMx
+BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
 CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
 VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
 AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
@@ -15,20 +15,22 @@ bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
 tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
 hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
 2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
-6JXXAgMBAAGjgbswgbgwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJ
-YIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0
-ZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYE
-FG+/PTIbDpPPZTFsEsAbLNUy1wxQMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0
-iu8o2z8rMA0GCSqGSIb3DQEBCwUAA4ICAQB6Fptgev/wPHn36lamR9RCBpqvh/Xg
-4nSnUfT1EVkEI7vUKkC/+/XeRTfnc14Yclo3uiYBY7QM953FhXOmlH0Rx1wwZSME
-ZzyO+1TWNVR2QI7/MhpXdKpqHhNLNTYKFa602ZTA5NXa9XDJ8T3kjbuAKiNwcuu7
-RzRK0o3KOWe0uohuI6bgcpgpKqcxrbtUkrPOWAn+XK7JQVSEV1kst4uDr1S+041c
-T+NHFQarDXw5g/Y0pHnU42dPyDvT/RrcBbzbe7qylFTbjKro/uFqPAayHlLMnGZp
-ZthGAPJJ0pDzBBxGtUmwOnat/HKWe2UN/A3Xqmy0Za7adhrYGj0IaZ0SNblNYDKy
-OjWtfOwLQrD9NEIhBXBE3KS5uyXP8kZESnHtIN0b7SMyP7lUxTQ/Rh8U7S6QD9AI
-gc+AfBx8UO+fdW3Z4VK+DU0NGRP+127w0yiEz1OtfS87dWBQq+kk2Hlc9TfFeG0b
-rckM4v54X8uTlnNfPo+RYRkMAXB/fnx2vVIyxTxFPo4bAnTFA6vnIz8rrRO+Vmh3
-oOclhE+ZemZ3BEC+Tgpvb4XD8pvDLnNOYr8tuDjN9jNNDofxplbsjI44wnb1v3HX
-M8Xe3P1WfqpWZniRmtVi/CWjkPy2JeC6zW4dn7JdkfEFHGsHQZrEZyrVrtVFPRFo
-hbgPGNWCxOhWzg==
+6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
+BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
+YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
+FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
+CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
+BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
+A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
+f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
+9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
+y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
+HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
+liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
+y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
+lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
+re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
+Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
+VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
+FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
 -----END CERTIFICATE-----

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -10,12 +10,10 @@ end
 
 describe "indexing with http_compression turned on", :integration => true do
   let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-  let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
   let(:index) { 10.times.collect { rand(10).to_s }.join("") }
   let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
   let(:event_count) { 10000 + rand(500) }
-  # mix the events with valid and invalid UTF-8 payloads
-  let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
+  let(:events) { event_count.times.map { event }.to_a }
   let(:config) {
     {
       "hosts" => get_host_port,

data/spec/integration/outputs/index_spec.rb

@@ -60,25 +60,48 @@ describe "indexing" do
 
   let(:curl_opts) { nil }
 
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
   def curl_and_get_json_response(url, method: :get); require 'open3'
+    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
     begin
-      stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+      out, err, status = Open3.capture3(cmd)
     rescue Errno::ENOENT
       fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
     if status.success?
-      LogStash::Json.load(stdout)
+      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+      if http_status.strip[0].to_i > 2
+        error = (LogStash::Json.load(out)['error']) rescue nil
+        if error
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        else
+          warn out
+          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+        end
+      end
+
+      LogStash::Json.load(out)
     else
-      fail "curl failed: #{status}\n  #{stdout}"
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
     end
   end
 
+  let(:initial_events) { [] }
+
   before do
     subject.register
-    subject.multi_receive([])
+    subject.multi_receive(initial_events) if initial_events
   end
-  
+
+  after do
+    subject.do_close
+  end
+
   shared_examples "an indexer" do |secure|
     it "ships events" do
       subject.multi_receive(events)
@@ -146,17 +169,17 @@ describe "indexing" do
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
     let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
-    let(:es_url) {"https://elasticsearch:9200"}
+    let(:es_url) { "https://#{get_host_port}" }
     let(:config) do
       {
-        "hosts" => ["elasticsearch:9200"],
+        "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
         "ssl" => true,
         "cacert" => cacert,
         "index" => index
       }
-    end
+    end 
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 
@@ -197,6 +220,8 @@ describe "indexing" do
 
     else
 
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password
+
       it_behaves_like("an indexer", true)
 
       describe "with a password requiring escaping" do
@@ -219,6 +244,32 @@ describe "indexing" do
         include_examples("an indexer", true)
       end
 
+      context 'with enforced TLSv1.3 protocol' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] }
+
+        it_behaves_like("an indexer", true)
+      end
+
+      context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] }
+
+        let(:initial_events) { nil }
+
+        it "does not ship events" do
+          curl_and_get_json_response index_url, method: :put # make sure index exists
+          Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
+          sleep 2.5
+
+          curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+          result = curl_and_get_json_response "#{index_url}/_count?q=*"
+          cur_count = result["count"]
+          expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
+          # [Manticore::ClientProtocolException] Received fatal alert: protocol_version
+        end
+
+      end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3'
+
     end
 
   end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -243,14 +243,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
           end
         end
 
-        context "with multiple messages" do
-          let(:message_head) { "Spacecraft message" }
-          let(:message_tail) { "byte sequence" }
-          let(:invalid_utf_8_message) { "contains invalid \xAC" }
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
           let(:actions) { [
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}],
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}],
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
           ]}
           it "executes one bulk_send operation" do
             allow(subject).to receive(:join_bulk_responses)
@@ -260,7 +258,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
           context "if one exceeds TARGET_BULK_BYTES" do
             let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-            let(:message_head) { "a" * (target_bulk_bytes + 1) }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
             it "executes two bulk_send operations" do
               allow(subject).to receive(:join_bulk_responses)
               expect(subject).to receive(:bulk_send).twice

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -33,7 +33,7 @@ describe "SSL option" do
     
     it "should pass the flag to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
-        expect(args[:ssl]).to eq(:enabled => true, :verify => :disable)
+        expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable)
       end.and_return(manticore_double)
       
       subject.register

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.4.2
+  version: 11.5.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-20 00:00:00.000000000 Z
+date: 2022-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -246,7 +246,6 @@ files:
 - spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
 - spec/fixtures/test_certs/test.key
 - spec/fixtures/test_certs/test.p12
@@ -329,7 +328,6 @@ test_files:
 - spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
 - spec/fixtures/test_certs/test.key
 - spec/fixtures/test_certs/test.p12

data/spec/fixtures/test_certs/renew.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-cd "$(dirname "$0")"
-
-openssl x509 -x509toreq -copy_extensions copyall -in ca.crt -signkey ca.key -out ca.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
-
-openssl x509 -x509toreq -copy_extensions copyall -in test.crt -signkey test.key -out test.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr
-openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12