logstash-output-elasticsearch 11.4.0-java → 11.4.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3932944b80f3ae81063657ab9255cb9504624d1c6c0634c547b69b38d4a9b4a9
-  data.tar.gz: b850f5abbe8ae00b62e379d6f7c90802172602f23a3ab0e3f67b1a4d8319bbea
+  metadata.gz: a5aa752898bbe2a9417fc15ec7fb1e9a55a804a03c84cef30a936206d5b97005
+  data.tar.gz: 89ec0c8d8ff45e4ef4892c6b2a2a5de3f108d07443f925a897e058e79987fcba
 SHA512:
-  metadata.gz: 5bc64de78f7ca0808e377595774f63b874d753801bbee8722bcd58af6a556e9c8baae683359179b65ee5a282c321c40dbeda838a9caacb5fc7ff14e7fe63e294
-  data.tar.gz: 0ff0a9731418cb7858e51109c8392d4a7692629d85141eedfa7019d99bbc12d7b9f016bd21d976f390ba5d745e039be6998742107b0eded2525a8c4dc7d0bf5f
+  metadata.gz: 633594b5c0010ca14c42231f8f1851bfdadd17819b61ea75f90d89c5db805d20d1a809ade265aedd432d80138965bdca18ec66819f387f69452f10d07a2ddb88
+  data.tar.gz: 9b4b8afaf80c0a9eebe7f7443c4fba5073c73823de87259641d373be62b9f52eb72e527d28d1dd4aca724abb487e13d1f49eff16f5ea0d528ab2ac7722522317

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 11.4.2
+ - Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169)
+
+## 11.4.1
+ - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
+   - the underlying changes include latest HttpClient (4.5.13)
+   - resolves an old issue with `ssl_certificate_verification => false` still doing some verification logic
+
 ## 11.4.0
  - Updates ECS templates [#1062](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1062)
    - Updates v1 templates to 1.12.1 for use with Elasticsearch 7.x and 8.x

data/Gemfile

@@ -12,4 +12,5 @@ end
 
 if RUBY_VERSION == "1.9.3"
   gem 'rake', '12.2.1'
-end
+end
+

data/docs/index.asciidoc

@@ -293,6 +293,11 @@ index level and `monitoring` permissions at cluster level. The `monitoring`
 permission at cluster level is necessary to perform periodic connectivity
 checks.
 
+[id="plugins-{type}s-{plugin}-handling-non-utf-8"]
+==== Handling non UTF-8 data
+
+This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8.
+When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`).
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Output Configuration Options

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -127,6 +127,9 @@ module LogStash; module Outputs; class ElasticSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
+
+        as_json.scrub! # ensure generated JSON is valid UTF-8
+
         if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
           stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
           logger.debug("Sending partial bulk request for batch with one or more actions remaining.",

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -133,12 +133,9 @@ module LogStash; module Outputs; class ElasticSearch;
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
       if !params["ssl_certificate_verification"]
-        logger.warn [
-                       "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
-                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
-                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
-                     ].join("\n")
-        ssl_options[:verify] = false
+        logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                    "to make sure your data is secure remove `ssl_certificate_verification => false`"
+        ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
       { ssl: ssl_options }
     end

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json

@@ -4,7 +4,7 @@
   ],
   "mappings": {
     "_meta": {
-      "version": "8.0.0-dev"
+      "version": "8.0.1"
     },
     "date_detection": false,
     "dynamic_templates": [

data/lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json

@@ -15,7 +15,7 @@
     },
     "mappings": {
       "_meta": {
-        "version": "8.0.0-dev"
+        "version": "8.0.1"
       },
       "date_detection": false,
       "dynamic_templates": [

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.4.0'
+  s.version         = '11.4.2'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.add_runtime_dependency "manticore", '>= 0.7.1', '< 1.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.8.0', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'

data/spec/fixtures/test_certs/ca.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFRTCCAy2gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
+BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMEwx
+CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
+VQQKDAVNeUxhYjEPMA0GA1UEAwwGUm9vdENBMIICIjANBgkqhkiG9w0BAQEFAAOC
+Ag8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
+Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
+nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
+AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
+COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
+Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
+eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
+X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
+eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
+CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
+RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
+AaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUoVp0nHjq6mJ/UGuFhnSK
+7yjbPyswDQYJKoZIhvcNAQELBQADggIBADeI0gRfzF5zyhCCmtlA44L1fo3BEYtm
+0Deq2W6bsW9zakYT7fivCUbt0VtS0jzhAlbaLfksJk+Emg+kpsy/CdFr8nk9jlpN
+XJMKwjQIVGYDDlJsP9VDw2m+HhUugEpx09QCREaCWCDtobE9bd42VloPd0u3VWZx
+u5rSrZCy5wjFeq3dPEFPyDTfV6p1kXkwy9pdG3ww0rw0dqj5OqcGhLyG8b7q7LLP
+aFVuwjnutrBJtDNOsvbD/dyybQVj3hF1IpVZLwoFQ3ji43/X3NJ5jdo/WrwwdwbH
+NE6YxNlsIrql4sJFeHzNukTxcC5wHcOLtc6E6dhe7uK+5rRD75+odz85rPLsBTPt
+zje8OWyxO1cGol7inpKUsO9ITFQdDC9iyRjvDRouwneT+XeVwW6BmAKg4DvXdLtQ
+oQNLlMbbZWskvWoI4amXwSpzWmn4mRK3e4IPWFju64QRvkEExFynvC69jeih5Llh
+JU3OjLobG4wKOQHB7w1UjiPJNsxGB0P1zug6ztTBLjsvJoxdJuIGkCsI7g12cKNR
+PbXp8453klofFpMNGlCP8dns92Qu20pPvYJcWiC9k42nYu3FHDMbAfSrFU90RNS6
+hRWkpXZOMIY1GdudgsxanDtJgFRbkPzwNkMcBQ1538f9U4MrJ9vPsuKx4bYbZ//8
+pIe/z3b1V9zV
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/renew.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+cd "$(dirname "$0")"
+
+openssl x509 -x509toreq -copy_extensions copyall -in ca.crt -signkey ca.key -out ca.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
+
+openssl x509 -x509toreq -copy_extensions copyall -in test.crt -signkey test.key -out test.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in test.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out test.crt && rm test.csr
+openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12

data/spec/fixtures/test_certs/test.crt

@@ -1,7 +1,7 @@
 -----BEGIN CERTIFICATE-----
-MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MIIF1jCCA76gAwIBAgIBATANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
 MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
-BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
+BgNVBAMMBlJvb3RDQTAeFw0yNDAzMTkyMjM3MTdaFw0yNTAzMTkyMjM3MTdaMFMx
 CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
 VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
 AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
@@ -15,22 +15,20 @@ bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
 tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
 hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
 2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
-6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
-BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
-YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
-FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
-CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
-BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
-A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
-f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
-9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
-y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
-HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
-liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
-y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
-lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
-re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
-Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
-VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
-FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
+6JXXAgMBAAGjgbswgbgwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJ
+YIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0
+ZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYE
+FG+/PTIbDpPPZTFsEsAbLNUy1wxQMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0
+iu8o2z8rMA0GCSqGSIb3DQEBCwUAA4ICAQB6Fptgev/wPHn36lamR9RCBpqvh/Xg
+4nSnUfT1EVkEI7vUKkC/+/XeRTfnc14Yclo3uiYBY7QM953FhXOmlH0Rx1wwZSME
+ZzyO+1TWNVR2QI7/MhpXdKpqHhNLNTYKFa602ZTA5NXa9XDJ8T3kjbuAKiNwcuu7
+RzRK0o3KOWe0uohuI6bgcpgpKqcxrbtUkrPOWAn+XK7JQVSEV1kst4uDr1S+041c
+T+NHFQarDXw5g/Y0pHnU42dPyDvT/RrcBbzbe7qylFTbjKro/uFqPAayHlLMnGZp
+ZthGAPJJ0pDzBBxGtUmwOnat/HKWe2UN/A3Xqmy0Za7adhrYGj0IaZ0SNblNYDKy
+OjWtfOwLQrD9NEIhBXBE3KS5uyXP8kZESnHtIN0b7SMyP7lUxTQ/Rh8U7S6QD9AI
+gc+AfBx8UO+fdW3Z4VK+DU0NGRP+127w0yiEz1OtfS87dWBQq+kk2Hlc9TfFeG0b
+rckM4v54X8uTlnNfPo+RYRkMAXB/fnx2vVIyxTxFPo4bAnTFA6vnIz8rrRO+Vmh3
+oOclhE+ZemZ3BEC+Tgpvb4XD8pvDLnNOYr8tuDjN9jNNDofxplbsjI44wnb1v3HX
+M8Xe3P1WfqpWZniRmtVi/CWjkPy2JeC6zW4dn7JdkfEFHGsHQZrEZyrVrtVFPRFo
+hbgPGNWCxOhWzg==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/test_invalid.crt

@@ -0,0 +1,36 @@
+-----BEGIN CERTIFICATE-----
+MIIGVzCCBD+gAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
+MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
+BgNVBAMMBlJvb3RDQTAeFw0yMjAxMTgwNTA0MDZaFw0yNTAxMTgwNTA0MDZaMFIx
+CzAJBgNVBAYTAkxTMQswCQYDVQQIDAJOQTESMBAGA1UEBwwJRVMgT3V0cHV0MREw
+DwYDVQQKDAhMb2dzdGFzaDEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAunKYkXLoAC14bdIveIZlImtrp2rkQ5Ugz/vGVWFJ
+YatJh9x5dmqJQeO1gpGayNedbwDI/50CTDoqwCz4aunMKpgQdKHryjeuR1zqElhC
+QLZtAkzM/pCkTnMYvXDfJrBZcSWP+ddlkH8ffmN0Fxf3LsKln+K5A4hASrVGSKYw
++eNDV7yIxdhthqD0xNRuw/j3lSLxwBbwKOs7Mh+xmdMa4vs3AKJvG9LdTm7xdHtD
+4rkQAA7TRQOR5pl+eDICRnNkGVzgPMdf2kM94ZU7TI1zUMqV1uPNE05Vps14kuWq
+Z84r8ecExCo6mQxrQ1M7Y2UBGa5NM9kb+UP1famANadEOlS5kAGEtcpHnh1WK+98
+4mxtMQJHIOOASde/coA3mZ4Oa8Z0Hzy3fsNvD9ieLo2V8yQN9UAQshbB1BEuOaRr
+9wQKT4jlnCinZ4UU6FpEJ7NIiZ9wBKqNbN8iySPYmRihOj6BDuEQB/W/K54KQB5n
+ctT6MsoTyTFgqe3Zn0owMrGCsSDJsvUcaAz7ZsnesoiFtKsYyRZAWJVcLO5R0FxZ
+YjCGk9stspLT1cqmJ8VbMLhiW7T4ZP+sZj3B+aSZWnS5r78I4G9sB9swDMJMjNRP
+O15zvQKIRluULhm9WhPET78Iy9Qg1zoiGdnzxBSZ9Sh8+yb57cCRD4da7eLne4Sb
+uU8CAwEAAaOCATwwggE4MAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMG
+CWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNh
+dGUwHQYDVR0OBBYEFPqrHNv/8Io82JmzcNWBdaRmftvdMIGIBgNVHSMEgYAwfoAU
+oVp0nHjq6mJ/UGuFhnSK7yjbPyuhUKROMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQI
+DAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVNeUxhYjEPMA0GA1UEAwwG
+Um9vdENBghRT5Ucn3UmwssDUYlgL5RyE1/DeETAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3
+DQEBCwUAA4ICAQDHt/3rR3rzESrZNNLIU+jbW6Vsfcr22mghhbdoU+pnkZB8Spzp
+AUqgmSlOOuz/3OMLPP64sj+gm9kiU177uBoqJUsnaAaV0LNvp+EIWIdskw0OGU2X
+uOslZK5EdYGqGgRlElnohPRXcFXwsh//QJYDmNnnC3Fk+ZjZQKjH4q4Ur8ECPqit
+wVnRXqlKfLRjWWzvTgoPTAN42KHP2R7xxIHdV+cXH6riLLvtkWcGJbfoQaSuNOvc
+GAIoQc2YmIiVsGZ82n9Ww2zO9ByqF+KfGgIFDTCp1CzpfLKLLhzMv/p4n2zf/BOb
+MCJJfljOewqmzMo48Wj2vk/46IAGl5uA6PnDwa0LNgomA9c6loDcYEOsCr69xL+D
+7GL/Jzm9HaTE97lRGVwoKBG0hKabFEfOueKB6Oab8bVTuY99kbbRaFZs7I9QYVQy
+eY38YJv5kN3yAjFclO39R8cAngqecbQDZ7xTl7dF7CvrpAoNI8olL01Kjy/+vfX4
+WAMO9YONnVPwwB05voRZfKErVKi9iwAWa5m9DdtE5QdfjcsXZkITW2CF2skSgujw
+BS8P8Z+HccKa+qEPQM+eBWjrlzlx/XC2iXKE1w4zZL0wRgY7+W4dIMgJmPmV/Gue
+5wSnZtMueBdyKy+xFrJcszoakKg/JfSesKfnVLC+60EL2FQqbrGCEiTp2A==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/test_invalid.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAunKYkXLoAC14bdIveIZlImtrp2rkQ5Ugz/vGVWFJYatJh9x5
+dmqJQeO1gpGayNedbwDI/50CTDoqwCz4aunMKpgQdKHryjeuR1zqElhCQLZtAkzM
+/pCkTnMYvXDfJrBZcSWP+ddlkH8ffmN0Fxf3LsKln+K5A4hASrVGSKYw+eNDV7yI
+xdhthqD0xNRuw/j3lSLxwBbwKOs7Mh+xmdMa4vs3AKJvG9LdTm7xdHtD4rkQAA7T
+RQOR5pl+eDICRnNkGVzgPMdf2kM94ZU7TI1zUMqV1uPNE05Vps14kuWqZ84r8ecE
+xCo6mQxrQ1M7Y2UBGa5NM9kb+UP1famANadEOlS5kAGEtcpHnh1WK+984mxtMQJH
+IOOASde/coA3mZ4Oa8Z0Hzy3fsNvD9ieLo2V8yQN9UAQshbB1BEuOaRr9wQKT4jl
+nCinZ4UU6FpEJ7NIiZ9wBKqNbN8iySPYmRihOj6BDuEQB/W/K54KQB5nctT6MsoT
+yTFgqe3Zn0owMrGCsSDJsvUcaAz7ZsnesoiFtKsYyRZAWJVcLO5R0FxZYjCGk9st
+spLT1cqmJ8VbMLhiW7T4ZP+sZj3B+aSZWnS5r78I4G9sB9swDMJMjNRPO15zvQKI
+RluULhm9WhPET78Iy9Qg1zoiGdnzxBSZ9Sh8+yb57cCRD4da7eLne4SbuU8CAwEA
+AQKCAgEAjd1oLTOrR4dRnO5S5HgON9RYg3iX2rx7zQUb8hcmSMSWHGQrn2iMq6/h
+iknT5oH47l9AkQ4ck3em07bkBiWDDd3Kl5Qk3ybi2rXd/LtfHfIwm5FoYUIZYv+8
+kVhy0vNX+vnDxSyZwQqZIgf2ayP3FoNQlolikUd4899ebSTXGyxLL1TrFO8K12dd
+Fu80oLStXbLLE4fgkKH71rW64vp3+MGBJmBx/k/ByE4uNp0cGEUL4Z9mLZr7xuSA
+EGCszoI6Zfn/PP/O9To4uY5dInB2j8C1pl6KyefO0C6DUfKiaH29fkBzpfcjwxRq
+AT4xb/2IisvPPmYyHMYS4ty4CxsMk7Cu5q+DlmRMa7ShOOdSAw+1k7yUl3rodXuu
+48k806s1cpoowo1aiNYMlZ6scQ0GtYU9/KfeWyfzO3QzgNguyAXifZNepU6qumUw
+S3670/1O/7KfpUCd8zgGmbHXlm6lhGj6ZHRDVDvp9/RKkqySbTGuWAXRgK3iLvEg
+26KTBtqW6WMx0XsZf7T2CDfXqdL+XzDrYNl9GB5rqgCjBUmsaK9QdwDGw4oFY3+k
+9KzBYj7wE7/W9vjBNBoWg/821McHJyYDaFCdOyC0YkppZKNENMcUkNE1noXQB7w5
+ib0NPWum//H+DT+F2iKOzP7N5f4XpHUSRnpYQ3C3d3Jt7n9NRoECggEBAObPgZd/
+ZnrgZtt++tMAsG8wEfjPS4GM/vbAdNJFruT1zamwIQv4ba/7/m2UO2nIpS1T6vBR
+MeVjsaueNe2K/9iRbXWKidTj38EnWCeEQQek9+bCjXqW01WjF+fomXYGSOj2RUWx
+5z7CcsfFe2N4yQxZEv9ynf2PZsA90F5FGcm7RvputhcY79Oyq8/baQmT2JpJMbHT
+X3J1usCQyIBjaGRErh0rSwXKcNL09Z1P5t4TDkZ+bctzO3s/0qdgS0E3keilyE4L
+cKe+hoGl8CsCaUBSmzY5evuLVChoZi6Pwlk84+CxFm5O9l505Bmphb22vZXII3ZS
+k7zDIJEBRzN+Sy8CggEBAM7LqE0yy56c55itsWQOKAVYyObn7HdwylQvd3OaEDhG
+L5MFFdif/GeLHcNzv2eRnd2n7cyjDfHR7znw9H0/xlLlloxonGiGR3XXCGexQYBY
+lJPkeCQRPAqNWXxUKsSPaChXJROsXRy6G2dgcdruM88z9iKnyeu8Ky++y8r6DLrm
+niTcRdqzz2HiGbnUyYdH7mg3IVT5LZsXxNyLAN+t4r0LmeZkijCC5xUd1brOxEAM
+h36qJZ1XXAGmhPgYW31VAqyDZ4oL9PGXEU/MCrxp8xHflbBOWBLm80gYmbZzxvvK
+4aGdXYWRmlEI3MS+HsvtcGXh4WOE1nLSU3cvDDIBa+ECggEBAKO+hvORoIR/+rix
+hwR2srTO55EajzijbKZltvsOEJvCfltp5qf1YOu+3Kukw9myTOyxYjWHhNx2M6/L
+F/sj54oe8ga3eD0eRLllTjcKro/byztcvr4/jkJs6CLQcz62CrerL03YfnOZw5BS
+W80f/ZHTB5VOHSOrvnuX3uFiKH9ja2FzdZ9BQ7NuSFG4GPaAeuRKFQVRLZ+oQgsn
+K/dZjs/Dobpz4k+DZTNkMXOfIexenHwKaZ4ya/puNuYjfIASCmOAaXBk0VFP62DC
+9nWsyjql4BNCCCu4lsXr+sIBnyFr/0aCm6U6Q7KTPtet2oHSyQEf8XiZ8NGzpMD6
+pSa07GUCggEBAJc0JCAmJuoX0eM6BT2ieDLIo0TqiWUf7GC3wECfgoKTFxAJpNqa
+yCQxfRa/WFFzEJnUwrRg+L47AQ89lpbJ/cn5IyYRC2QF4tRP4U5oNfuRSToF0K6W
+h28zwR1+MTM9pCvy5CJJYl+x2H2y8CzjBLDZTnwycRrToEQt3rbQNGSoYTOUd+Mc
+nGL2vla5No2a08ARp2aJN4ZyT7fuTzo9207c36+tDbnAzRQMl40ayDYIsz2zTQOJ
+r7VpO0poDOVMNkNLZXZznarUCY1uJN91HIySDdI2xoEzquipTCMy7miHBIl2Fb6n
+Is1jjICyfrQfLZLhITryExcroGnB1cnubiECggEAdOeyL1Q0MC59nbZ3E0OwCSMe
+HAEbcEp8J6nyyxJ+VVSVSmh5SIEajSh5PPkrzzqKkVo6LSXvHeiwvx5R302RBIew
+GZ41oUJA5ApPWAiZo/pusRR0nnClBXPSqFOb7uYuDTXuma2rWcN3xt4L1zGbSMS7
+de3Wsp6kVv3q2uF/2fv78BwpwaRQaDcmw269iSHjMAtVbBm2QIkN4FI+4CCUdn7H
+plUAW5CXWtAHRKFJwjF/I8PYAlyvGb4i8ll/RK5hworj1VrPgnKIg2JP8LcAm1a4
+9sk2DRryU+7PZBWgqaXeTfoDUzdcKmWx/jbGPP7Wlvh3Ao6ityQ0pauWZKzSFg==
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/test_self_signed.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdzCCA1+gAwIBAgIUNtmhsOlqXXLvFvsvImAvs9ZrnFkwDQYJKoZIhvcNAQEL
+BQAwSzELMAkGA1UEBhMCTFMxCzAJBgNVBAgMAk5BMQswCQYDVQQHDAJFUzERMA8G
+A1UECgwITG9nc3Rhc2gxDzANBgNVBAMMBmNsaWVudDAeFw0yMjAxMTgxMzIwMDNa
+Fw0yNTAxMTgxMzIwMDNaMEsxCzAJBgNVBAYTAkxTMQswCQYDVQQIDAJOQTELMAkG
+A1UEBwwCRVMxETAPBgNVBAoMCExvZ3N0YXNoMQ8wDQYDVQQDDAZjbGllbnQwggIi
+MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDT+NYvZzi7y2RHvklPQ7BYV478
+A+nN/ncakJ0JjmdekwDhUjb8VO/YxN68FyLargI89hsa9Aw20GRZrtibUYfl2qZy
+CW5Gydl/t2evCbO2mIcObImsagp/wNbPfopX8efU1aK3KPSlES0qDX6oB2hl9Afx
+oeJY+NBlg1xhVWC8/WAKxG/me9XPYhsOSVYR8UiA2RkPnhSND6dqqR+KGUdhxcZH
+rB2Y/fHyy2uAgErWguNSvVRudy7yZn5eAIhV2cI+710SOGPIXPbMPzdJBiNFNhaK
+QSCze+lC7xJg2lYhR+2H3DofhftemPMLPtFauVUe1xm5rP/hrVSbZuNN8DEffpAO
+whhgsDDmkAh8oY+OTSWCdjKHrir+Dbqo/OfmMK5tkC12LP2e45nvGKLrJGsHtBeP
++l3edtYLy0lvKVXvF/3krujTQYrA04Gqb9JKsvUSFarlhiY3dmJ1+na3kCNRu5Ei
+oFlvr8uWJLeEFTll6ahTGkCnWZ5bHP1NNYD4JRwTNtujArtWxEEomoiyjgw3RfHE
+pGlSaljnCvP9OX72zfrpfbQJx937CEt22jVTyKls4TPvkX+bKXHUVNG0xDlMcUK/
++DdTDLxD+j0gBJofSD/uWeWd8VhwSG7jY5PimUhQxwrHouyKbktF3useuElH/3wn
+pDy0hPKbgZmATChOBQIDAQABo1MwUTAdBgNVHQ4EFgQUKj3kW3xSP4/7D9djf8DM
+XdAG0e8wHwYDVR0jBBgwFoAUKj3kW3xSP4/7D9djf8DMXdAG0e8wDwYDVR0TAQH/
+BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAbNsBp/QUTXNKblQUPsjK97BDHVLs
+pdcHAabLd5Cwveb+0RyqUoMGz+XmtOiCS4bX5uLnNbjiFQK1zMyen6RBmdfZ4+nv
+/Uh5XMQx3oZCCrmXbTEsFjlSgQuQlTGlaJBZXKwN7KmBgqvsNEPRwteDPOLuQ4DP
+4zc/6euJDeHcrqTYEF4sYB5srkkL4A0Kw3ZaukopKMpdEaAzgqeQdQ5AqbKlfiiI
+/XJxomGLQ7LWY9KWYeZZ3RhdN/mEREFYHZ+OkptPcKqjQimIq6JulHVQJar3A3ec
+zaVsyT4UkDDn0nf8N/D9mjbCuO3Pd/8EHgqRzxMvglkCkDElrucrXjECK6SBpmgu
+nCIvEIJxlHqLicBAQxdv6/N+UILZ37Wg1MyoIkzQA1j6lkw8kMLi0OIStZ6COOVA
+6x00SDH7GwkWEWJCGN7v8xOTd4ftN3Srhsai0wMyNtA39Wi3GiAWo8XwtfrtkWwk
+7zmWYZu/Q98WAKSv7V5UYkCCzI1R9L2ZRBYtl94kQyfCzwVAyw7+x+z2mFm9hKIo
+g4Gc2UVOgMdYeghNwst7GUmCOJGo1hAr36UtZgDV4PnM1V41GAFNff+XArdZTpgu
+dEEhFp2ITlM/+9c68EvFMUhoi8M0GBv7IPwOE70f4blkBvTnNLQqe0bRfwB4FUcI
+8X4SIgxrmMAlKIc=
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/test_self_signed.key

@@ -0,0 +1,54 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIJnDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQI+GmeuhQKe3oCAggA
+MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECBMDAiAm75L9BIIJSAuO3VTZO5tn
+F16ZpsL/9CW03G78wIGfiDE3Z4cWGfV/b+0eLUiPEPTWXwsT2dKOK1ih7yWecOLY
+Bb1jE/RJGd6SdAvE7Ur8xyEQhDgVbybcn0hubjQj4koP/k7HwcVVyR4cyXUCKcsr
+105LS4lagX3RI6zQCpK47dKbBj3YpZKN8CZPM03SEnmFh+QMDoXAegP2941hIdvL
+UO+VY5UpCe1VweBaMAXS6xWSk+LCQ/rlvR7vOTFkXqqC9KybkmrwN8hP2DPi218P
+XdQ720tP5gfhYLZtWoEQGrSgoJ7P4we3YyTMtCMmBZLuk5zk/OfwvX7Z+RlxX2rN
+TQxn2DlOc5anEG23r3Opv/ETm4awPgpdYYm7by3MJEY0lnOTZRuze8LcW6NjvyVd
+oi7geDPmVdvs5AQTV4aSBNfd9xvaq5NWiFPTbtnK17FqWSnWYlmrGJLaWvmSSBdZ
+4BLKElaU9v5uOmCCExaTpx4r+RcNZIdfAggCpcThwJuLtS1Kt0VhDvW5vkUbEq7h
+lhObPkUQfCEOgcNgo97ww1E4GxJEth6TN1hP78pcDYg7/Lb4p4DzrBTHH2IFYVmW
+cd195YeJIHOt96LWQuslv1HK24Of4+8luR/aLEfT1pGjyvm+5ShsgklEv21YVLJZ
+a4NKw8VNL5Y0ErlgN5RXd0n4MAdQHfpHSSUmcW3WNkC607OiS5f4DOAwoyWu8Roa
+9RaHpvJcQEtT1g0qCmLPoaqXOXqXJL+/ayEPXtaTv9wfUYgYpQdaB1b88/USneCx
+yUpSnI1Sn4HiZiaNzh+ZpHf0qBDdwPslRXFjtBefoOfJidklTTMQ4kEx4j6Uh2lM
+u2EEq0/kldydIUKhqpNdvOBmsylAzOoeWX/PQrVneRXZwe18aZb4QbEtPdB+Lobf
+oqRSObI98o9B7joXDJ891GnlsiNg+JvKBgUR9hce14EV8wU25/DiuFr6Mm8yUv3f
+UaGf8Cz6H8poj8djBoSF5vrVxu9Ucxkt4thN6J7/OXCnCCOU0vV+FUQqE23FPadh
+VK9MMA3cQKPiv+d4HNJriThLEgj2s+5xxZ2QXMl1gGNmMEHclIOA2BOSqoq8VkKS
+BpLQEZHMyfkRHzB+bclJnkAFLb2Vh/y3SkOgFggJFIRq0kL7g5PYrOwS6qRgHnNZ
+tw+hdTMWBLMwLNgMs1rWkEvTxvT//mDRKFmOV/rlG3xozb6OKB+7O60Tx1N5o3oP
+KH7qD3bzJ8JyIg85oJJHqOvvMlkoFk4qhL0l8N79EQy/+bOKpQeOxhWDuIpd0sCA
+FJjVEBFlYVI25ZO0pjFYvFQKd8IwfA2DpnXX7DRLZmzUvNG7mfkhmzKzqjwAw3GY
+RKfRdYF3OMo6/QcKDE3xl/x7XyepAnFlTEQzHHD6o+uhWEvX0+7McY2YAmPrXUjf
+bC+au6vp6945FoBe1AcqbQb2ZdJqQq9F1bgi0QwWhh/JlQfvRMz3PqcYct/92Wvr
+Fq6P+awwEq0V2XvOU376f5qC2TPOoEyErCHj+m5zUTezP6rmeO6G1txf1qdVfvcS
+yqU5iOyQZnf++ObJCsV3HILu3FyOVKrjplFYF7VzLwcNw/ulUK4d2LaZX99WTTJA
+H18OG3x0Y0OqLyWKkO9Pl5WkCW8v8IvocDwVl3KsZI6m6JE+92t+IDI2p27hiN8+
+PzoOQ51EGY/nmtDevaxAy4HcOxXYQJV3gZrAOOlBe+7KwPw1mhR/BW64y7JZaCsj
+m7CTWbu/tt+xio+PGJ4woj5K5zKDKkP2O46shJRS5/03r3EDrfySgYvfT1M2y9Aw
+tGdFQhF8tMSRUYWVPaj8dBH/cRoFvOSI4ARek/TbbJO0XHYPV+rf76MA8VpE6EFK
+BskLXLsgxti3sm5p/6D6tg6iC7efBkEebtKmjMvLK38Td1h8aVRZ1tVtJj7K/hYw
+Zp2WQaEwpZskgVGXvN+fMN33VUNTYqNS6jTYwPY9OyN1lwoaxw4yAhJolLKp9qJZ
+SF7CdYFbao6pgBj5/pzTwANPeRtm5M2gzrNrLgPHjM5w43R3mC6L2qG47Apw8DnI
+HhyfzWbnS4UNP64yZoeIY7QpykGcOsR3wO5qjg9hH3WFOgWMciOzOKn3LnhcA05I
+C8a4W+xYEcWYfRG7oTBpMKgswWz+++Ho09MuDbkDO3WSwmaKw+dU+ACJc0L6Rov2
+wEe0vE4vNugvUbS15ST9Z3zKZuEnMVDw8u5qFLDRz9tEhF/wgi4O9W4k/Qy6Ib42
+SUJxbLxPRC8w/CnzRa6xHPxzDcfYQbwnNG+hq5PPgn8xyoIjNJatfigo9o3THRbv
+wwDzJEWEzhew9MVYz0Re09KZrCi4BqCNCMwkAM8nOdPu38+MO+0DylqgIY+0u/zt
+HbfKl+cbImCuRWzUZFzb6lo3uEJeUZa0UjrxykYCupzMNJ15ezXYxSsWgH58ah/0
+6uqw0I8XwJM6mWY7BQvhixUXmeOgRBFBaNaLBe/bvvEideoEAXgLWw5ID40PZvh4
+4nRtMSOdEJFt6EGLnNl+WCC/f9B/NXHKx2yTzXam52UEe0SDnDcDca3v4z54U2MY
+v3qozFf8GSM1EFAcEHRDfCLj+/gXgOHK2DFcMc3YwGJUfiSbz2LfGzFG34nezJDA
+TtRFdRlW4ctqUKA4CkGbbZlJ6v9WFAg72BD5OognB035vtpdvfLOiUfUeflskMTt
+Uyci5s5sxw8TWRIpaXwk1pnLcivXC0TSVnzi1HhD6pNsejkBXBeQPxB6qMbWUhQb
+RDzRqbe4z0/IjG6uAXhacXw087iPjPN7+xtZJWwCaQjWGuNn2Fs76CJRrWlt/DF9
+PgaBlpul8EuNyMqZWKaehln0zBvH5Y177BXlrEeTpuUo5/kPWj2jEu51jfe+xIW8
+3RejLknCS39KjOV794ImLw8B1WlCwrfajVnXwgga8fCY6KGz7u3Prqq9irhRfycO
+pAl7Rja/fb/1yBuQHrUU1lgYIXqb262lebMhLlDHntc+J8Vjk++UuP2WjPNeztp2
+H4VF5NxMKJU8gScaN67FUtFYst5cz2aJuDHxbitFHKgoxNFYHVMjY26X/kmZVnUb
+aNCsaa8YbPPUDmHf2rJjIlK+PZLvLlENFbWDxaRC7/ab39Za9DTD9zH4aut6gw3Y
+q/eu/hnuc3qa580zbpgcAg==
+-----END ENCRYPTED PRIVATE KEY-----

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -10,10 +10,12 @@ end
 
 describe "indexing with http_compression turned on", :integration => true do
   let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
   let(:index) { 10.times.collect { rand(10).to_s }.join("") }
   let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
   let(:event_count) { 10000 + rand(500) }
-  let(:events) { event_count.times.map { event }.to_a }
+  # mix the events with valid and invalid UTF-8 payloads
+  let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
   let(:config) {
     {
       "hosts" => get_host_port,

data/spec/integration/outputs/index_spec.rb

@@ -46,7 +46,8 @@ describe "TARGET_BULK_BYTES", :integration => true do
 end
 
 describe "indexing" do
-  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:message) { "Hello from #{__FILE__}" }
+  let(:event) { LogStash::Event.new("message" => message, "type" => type) }
   let(:index) { 10.times.collect { rand(10).to_s }.join("") }
   let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
   let(:event_count) { 1 + rand(2) }
@@ -55,10 +56,22 @@ describe "indexing" do
   subject { LogStash::Outputs::ElasticSearch.new(config) }
   
   let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
-  let(:http_client_options) { {} }
-  let(:http_client) do
-    Manticore::Client.new(http_client_options)
+  let(:index_url) { "#{es_url}/#{index}" }
+
+  let(:curl_opts) { nil }
+
+  def curl_and_get_json_response(url, method: :get); require 'open3'
+    begin
+      stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+    rescue Errno::ENOENT
+      fail "curl not available, make sure curl binary is installed and available on $PATH"
+    end
+
+    if status.success?
+      LogStash::Json.load(stdout)
+    else
+      fail "curl failed: #{status}\n  #{stdout}"
+    end
   end
 
   before do
@@ -70,16 +83,16 @@ describe "indexing" do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      curl_and_get_json_response "#{es_url}/_refresh", method: :post
 
-      response = http_client.get("#{index_url}/_count?q=*")
-      result = LogStash::Json.load(response.body)
+      result = curl_and_get_json_response "#{index_url}/_count?q=*"
       cur_count = result["count"]
       expect(cur_count).to eq(event_count)
 
-      response = http_client.get("#{index_url}/_search?q=*&size=1000")
-      result = LogStash::Json.load(response.body)
+      result = curl_and_get_json_response "#{index_url}/_search?q=*&size=1000"
       result["hits"]["hits"].each do |doc|
+        expect(doc["_source"]["message"]).to eq(message)
+
         if ESHelper.es_version_satisfies?("< 8")
           expect(doc["_type"]).to eq(type)
         else
@@ -132,7 +145,7 @@ describe "indexing" do
   describe "a secured indexer", :secure_integration => true do
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
-    let(:cacert) { "spec/fixtures/test_certs/test.crt" }
+    let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
     let(:es_url) {"https://elasticsearch:9200"}
     let(:config) do
       {
@@ -140,42 +153,73 @@ describe "indexing" do
         "user" => user,
         "password" => password,
         "ssl" => true,
-        "cacert" => "spec/fixtures/test_certs/test.crt",
+        "cacert" => cacert,
         "index" => index
       }
     end
-    let(:http_client_options) do
-      {
-        :auth => {
-          :user => user,
-          :password => password
-        }, 
-        :ssl => {
-          :enabled => true,
-          :ca_file => cacert
-        }
-      }
-    end
-    it_behaves_like("an indexer", true)
-    
-    describe "with a password requiring escaping" do
-      let(:user) { "f@ncyuser" }
-      let(:password) { "ab%12#" }
-      
-      include_examples("an indexer", true)
-    end
-    
-    describe "with a user/password requiring escaping in the URL" do
-      let(:config) do
-        {
-          "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
-          "ssl" => true,
-          "cacert" => "spec/fixtures/test_certs/test.crt",
-          "index" => index
-        }
+
+    let(:curl_opts) { "-u #{user}:#{password}" }
+
+    if ENV['ES_SSL_KEY_INVALID'] == 'true' # test_invalid.crt (configured in ES) has SAN: DNS:localhost
+      # javax.net.ssl.SSLPeerUnverifiedException: Host name 'elasticsearch' does not match the certificate subject ...
+
+      context "when no keystore nor ca cert set and verification is disabled" do
+        let(:config) do
+          super().tap { |config| config.delete('cacert') }.merge('ssl_certificate_verification' => false)
+        end
+
+        include_examples("an indexer", true)
+      end
+
+      context "when keystore is set and verification is disabled" do
+        let(:config) do
+          super().merge(
+              'ssl_certificate_verification' => false,
+              'keystore' => 'spec/fixtures/test_certs/test.p12',
+              'keystore_password' => '1234567890'
+          )
+        end
+
+        include_examples("an indexer", true)
+      end
+
+      context "when keystore has self-signed cert and verification is disabled" do
+        let(:config) do
+          super().tap { |config| config.delete('cacert') }.merge(
+              'ssl_certificate_verification' => false,
+              'keystore' => 'spec/fixtures/test_certs/test_self_signed.p12',
+              'keystore_password' => '1234567890'
+          )
+        end
+
+        include_examples("an indexer", true)
+      end
+
+    else
+
+      it_behaves_like("an indexer", true)
+
+      describe "with a password requiring escaping" do
+        let(:user) { "f@ncyuser" }
+        let(:password) { "ab%12#" }
+
+        include_examples("an indexer", true)
       end
-      
-      include_examples("an indexer", true)
+
+      describe "with a user/password requiring escaping in the URL" do
+        let(:config) do
+          {
+              "hosts" => ["https://#{CGI.escape(user)}:#{CGI.escape(password)}@elasticsearch:9200"],
+              "ssl" => true,
+              "cacert" => "spec/fixtures/test_certs/test.crt",
+              "index" => index
+          }
+        end
+
+        include_examples("an indexer", true)
+      end
+
     end
+
   end
 end

data/spec/integration/outputs/no_es_on_startup_spec.rb

@@ -12,13 +12,13 @@ describe "elasticsearch is down on startup", :integration => true do
                                            "template_overwrite" => true,
                                            "hosts" => get_host_port(),
                                            "retry_max_interval" => 64,
-                                           "retry_initial_interval" => 2
+                                           "retry_initial_interval" => 2,
+                                           'ecs_compatibility' => 'disabled'
                                        })
   }
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean ES of data before we start.
@@ -33,7 +33,9 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers before documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+        ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: before docs are sent"), 'http://test.es/'
+    )
     subject.register
     allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_return(ESHelper.es_version)
     subject.multi_receive([event1, event2])
@@ -43,7 +45,9 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should ingest events when Elasticsearch recovers after documents are sent' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_es_version).and_raise(
+        ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: after docs are sent"), 'http://test.es/'
+    )
     subject.register
     Thread.new do
       sleep 4
@@ -56,11 +60,13 @@ describe "elasticsearch is down on startup", :integration => true do
   end
 
   it 'should get cluster_uuid when Elasticsearch recovers from license check failure' do
-    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_raise(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).and_raise(
+        ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new StandardError.new("TEST: docs are sent"), 'http://test.es/_license'
+    )
     subject.register
     Thread.new do
       sleep 4
-      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_call_original
+      allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).and_call_original
     end
     subject.multi_receive([event1, event2])
     @es.indices.refresh

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -243,12 +243,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
           end
         end
 
-        context "with two messages" do
-          let(:message1) { "hey" }
-          let(:message2) { "you" }
+        context "with multiple messages" do
+          let(:message_head) { "Spacecraft message" }
+          let(:message_tail) { "byte sequence" }
+          let(:invalid_utf_8_message) { "contains invalid \xAC" }
           let(:actions) { [
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}],
           ]}
           it "executes one bulk_send operation" do
             allow(subject).to receive(:join_bulk_responses)
@@ -258,7 +260,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
           context "if one exceeds TARGET_BULK_BYTES" do
             let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            let(:message_head) { "a" * (target_bulk_bytes + 1) }
             it "executes two bulk_send operations" do
               allow(subject).to receive(:join_bulk_responses)
               expect(subject).to receive(:bulk_send).twice

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -33,7 +33,7 @@ describe "SSL option" do
     
     it "should pass the flag to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
-        expect(args[:ssl]).to eq(:enabled => true, :verify => false)
+        expect(args[:ssl]).to eq(:enabled => true, :verify => :disable)
       end.and_return(manticore_double)
       
       subject.register

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.4.0
+  version: 11.4.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-13 00:00:00.000000000 Z
+date: 2024-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -244,10 +244,18 @@ files:
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
 - spec/fixtures/template-with-policy-es8x.json
-- spec/fixtures/test_certs/ca/ca.crt
-- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
 - spec/fixtures/test_certs/test.key
+- spec/fixtures/test_certs/test.p12
+- spec/fixtures/test_certs/test_invalid.crt
+- spec/fixtures/test_certs/test_invalid.key
+- spec/fixtures/test_certs/test_invalid.p12
+- spec/fixtures/test_certs/test_self_signed.crt
+- spec/fixtures/test_certs/test_self_signed.key
+- spec/fixtures/test_certs/test_self_signed.p12
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/data_stream_spec.rb
@@ -319,10 +327,18 @@ test_files:
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
 - spec/fixtures/template-with-policy-es8x.json
-- spec/fixtures/test_certs/ca/ca.crt
-- spec/fixtures/test_certs/ca/ca.key
+- spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/renew.sh
 - spec/fixtures/test_certs/test.crt
 - spec/fixtures/test_certs/test.key
+- spec/fixtures/test_certs/test.p12
+- spec/fixtures/test_certs/test_invalid.crt
+- spec/fixtures/test_certs/test_invalid.key
+- spec/fixtures/test_certs/test_invalid.p12
+- spec/fixtures/test_certs/test_self_signed.crt
+- spec/fixtures/test_certs/test_self_signed.key
+- spec/fixtures/test_certs/test_self_signed.p12
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/data_stream_spec.rb

data/spec/fixtures/test_certs/ca/ca.crt

@@ -1,32 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
-BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
-DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
-WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
-BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
-AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
-gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
-B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
-6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
-4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
-nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
-9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
-wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
-ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
-UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
-w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
-g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
-dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
-Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
-UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
-1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
-MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
-jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
-Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
-DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
-lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
-36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
-rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
-sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
-nIg9fQPvVw+fIHACZQ==
------END CERTIFICATE-----