logstash-output-elasticsearch 12.0.4-java → 12.0.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d05f307c49bf3e1060a89278d32ac100b073de8b76dc2e84b4cd2e9dd40efdf
-  data.tar.gz: e8d09363e761fc2902a4f345e12cf754bec11de3ab2de662be1a2f77af03b888
+  metadata.gz: ed4936a4ade53b1bfabb21a6a217a2587ee38235a3219be044ca59a664fd25a1
+  data.tar.gz: c06797095dd77c26bd7c70d1d851fcc50ca8e2b4db35485e9f5c018941e54cd2
 SHA512:
-  metadata.gz: c66b31e7d1074d02b78c14274bde382ba53f74d73e634715a397c81fe4e29820e5ddcfb4b20c9339be772fe5efa820cfa8c5c127821245a7fa2ddd98c7391a87
-  data.tar.gz: d1e4a02566e65b1388e82cce6134126dfd2f9106a9ed2f2daacf7b354450a7cd6ca8f36b4b4400a1f70d404dc375d32fbaeb8b7061b365258386955c2afa48d0
+  metadata.gz: 163f1e9cd1127527b6ce207f18f451c6dfd7510c6f752739908d6804c3fa70b2fced78b22668f6388d02a92d9969dd7f47f079e0180abacf8f0305d56ad14441
+  data.tar.gz: 77dc391a2d5ef77f76d07a52b6f00375eb6b9ffb9c8dbb731c849b2bffd8fb25f819c3070a729a9f1c72e7ba4ff845359ba5109aa2e5eb1242461c4b0cab5719

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 12.0.6
+ - Add headers reporting uncompressed size and doc count for bulk requests [#1217](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1217)
+
+## 12.0.5
+ - [DOC] Fix link to Logstash DLQ docs [#1214](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1214)
+
 ## 12.0.4
  - [DOC] Update Cloud terminology [#1212](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1212)
 

data/docs/index.asciidoc

@@ -190,7 +190,7 @@ Example:
     }
 
 
-==== Retry Policy
+==== Retry policy
 
 The retry policy has changed significantly in the 8.1.1 release.
 This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
@@ -223,7 +223,7 @@ Note that 409 exceptions are no longer retried. Please set a higher `retry_on_co
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
 
 [id="plugins-{type}s-{plugin}-dlq-policy"]
-==== DLQ Policy
+==== DLQ policy
 
 Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately
 mapping errors cannot be handled without human intervention and without looking
@@ -231,8 +231,8 @@ at the field that caused the mapping mismatch. If the DLQ is enabled, the
 original events causing the mapping errors are stored in a file that can be
 processed at a later time. Often times, the offending field can be removed and
 re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
-happens, the problem is logged as a warning, and the event is dropped. See
-<<dead-letter-queues>> for more information about processing events in the DLQ.
+happens, the problem is logged as a warning, and the event is dropped. 
+Check out https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html[Dead letter queues (DLQ)] for more information about processing events in the DLQ.
 The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
 but should be used only in motivated cases.
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -21,7 +21,8 @@ module LogStash; module Outputs; class ElasticSearch;
   # We wound up agreeing that a number greater than 10 MiB and less than 100MiB
   # made sense. We picked one on the lowish side to not use too much heap.
   TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
-
+  EVENT_COUNT_HEADER = "X-Elastic-Event-Count".freeze
+  UNCOMPRESSED_LENGTH_HEADER = "X-Elastic-Uncompressed-Request-Length".freeze
 
   class HttpClient
     attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
@@ -143,7 +144,11 @@ module LogStash; module Outputs; class ElasticSearch;
                        :payload_size => stream_writer.pos,
                        :content_length => body_stream.size,
                        :batch_offset => (index + 1 - batch_actions.size))
-          bulk_responses << bulk_send(body_stream, batch_actions)
+          headers = {
+            EVENT_COUNT_HEADER => batch_actions.size.to_s,
+            UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s
+          }
+          bulk_responses << bulk_send(body_stream, batch_actions, headers)
           body_stream.truncate(0) && body_stream.seek(0)
           stream_writer = gzip_writer(body_stream) if compression_level?
           batch_actions.clear
@@ -159,7 +164,14 @@ module LogStash; module Outputs; class ElasticSearch;
                    :payload_size => stream_writer.pos,
                    :content_length => body_stream.size,
                    :batch_offset => (actions.size - batch_actions.size))
-      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
+
+      if body_stream.size > 0
+        headers = {
+          EVENT_COUNT_HEADER => batch_actions.size.to_s,
+          UNCOMPRESSED_LENGTH_HEADER => stream_writer.pos.to_s
+        }
+        bulk_responses << bulk_send(body_stream, batch_actions, headers)
+      end
 
       body_stream.close unless compression_level?
       join_bulk_responses(bulk_responses)
@@ -179,8 +191,8 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream, batch_actions)
-      params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
+    def bulk_send(body_stream, batch_actions, headers = {})
+      params = compression_level? ? {:headers => headers.merge("Content-Encoding" => "gzip") } : { :headers => headers }
 
       begin
         response = @pool.post(@bulk_path, params, body_stream.string)

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.0.4'
+  s.version         = '12.0.6'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/test_certs/GENERATED_AT

@@ -1 +1 @@
-2024-06-25T21:50:58+01:00
+2025-07-22T11:15:03+01:00

data/spec/fixtures/test_certs/ca.crt

@@ -1,29 +1,32 @@
 -----BEGIN CERTIFICATE-----
-MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
-BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
-DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBMMQswCQYD
-VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
-TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
-AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh
-5er5JF2iomYOHiD5JncYr4YmRQKuYfD6B1WI5FuQthD/OlA1/RHqtbY27J33SaO6
-6ro8gal7vjHrXKQkefVYRwdfO6DqqbhV6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQq
-GZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju4jPhp+43BkGwEimI5km7jNC1nwjiHtxD
-sY/s93AKa/vLktXKUK5nA3jjJOhAbRTVnbOAgxFt0YbX98xW/aUqscgBUVs9J/My
-TRMwVKJ7Vsmth1PdJQksUASuzESlSPl09dMjTQ+MXzJDt0JvX8SIJPmbBng78MSa
-CUhpOZiii1l2mBfPWejx20I/SMCUNmzbwm2w9JD50Jv2iX4l4ge4H1CIK1/orW1p
-dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv
-5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx
-r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP
-LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ
-KoZIhvcNAQELBQADggIBABym9LMyS9W9lvpcH4OK1YLfBPJwrhZ+4keiriY4zWOo
-pB+v2Q35neMMXSlTDpeIwPdMkqsh8VZprOWURF80JGvpJ6fBfi05rCDWp/ol1ZKi
-snCA+dE2zDK7Z3+F0MbakT5oBi5WgkXSvRvlJEJ/gBD7WC1wq0kxCMK+M5w2RPAT
-nnV/iozNBkwExxyJA7BpS6F/v0XjwK7fm5Kpql7zKlh5piZ2IVU0B60Sqskcb2mU
-90+1r9T06ekIW/Iz1jd5RWYziu0nbmDeKeKvGAICNU+evYXW+/5kKecMLuEvDCgS
-ssbt/Hb510uLHhxfhN4SbvBl2zADsLC+2arf2ATIwD8ZXDDs04ayBsejV0ZwVrTZ
-ExKqAys+B3tuIHGRqL5VukdmH6g6oJziYueohPBCOuSOzDd0FhppF4uXZS8DReSg
-KieO2ZYfiA1gVRiY6jPx+r7J9I5kSS1gwr/e3zHJHa79ijMB1SSIswQUmgSMkwGh
-sNyDNI9ZxgJan3v7kVargMt2LiNcXvVyTzPSYSXcY7SoebfpMprVIG7vZ9TZf+Uu
-FQeOfxdLFuGTnpFrYmvOD3OIKfODlY5t+TNICg7A3eTUXeJPcdBBnuVCiQU6TCB5
-H+69K5w54Q6a70sHZU1IWsGT8XtbUizPNQky+LAFsE/5oUnCqtypeEu4srcZK53x
+MIIFdTCCA12gAwIBAgIUDITbsLT9hKser0ZzBZsxqgaZdWswDQYJKoZIhvcNAQEL
+BQAwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
+DjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MB4XDTI1MDcyMjEwMTUwM1oX
+DTM1MDcyMDEwMTUwM1owSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD
+VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MIICIjAN
+BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3n
+z6C9w63tD+w4S9wNAmCTNn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjk
+W5C2EP86UDX9Eeq1tjbsnfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwy
+LLwXNDnlMymgzflxa6+gAzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn
+7jcGQbASKYjmSbuM0LWfCOIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CD
+EW3Rhtf3zFb9pSqxyAFRWz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNN
+D4xfMkO3Qm9fxIgk+ZsGeDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0
+kPnQm/aJfiXiB7gfUIgrX+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8
+hPCaZ+l+T1z/JY98Luy0eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuC
+fREAGiNGgZ2GFjdAOKmDCZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO
++WbApbcXIwOHBgv/NfsnRzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfz
+Iokr0sEBStgKMncCAwEAAaNTMFEwHQYDVR0OBBYEFKFadJx46upif1BrhYZ0iu8o
+2z8rMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0iu8o2z8rMA8GA1UdEwEB/wQF
+MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJi4FwYJz/RotoUpfrLZFf69RoI01Fje
+8ITt8SR1Dx/1GTPEuqVVfx0EYtOoH6Gg3FwgSQ9GHRDIa1vkHY5S+FUSOW3pCoZE
+/kaLu9bmFxn+GntghvQEor+LzODuZKLXupaGcu1tA4fzyuI4jglVD2sGZtLk//CT
+Hd4tOWXo5k1Fj0jMnJq+2Htr8yBeSAO5ZNsvtAjOUU6pfDEwL9bgRzlKKFQQMUYo
+6x1FvRDRXWjpzB/H+OSqOaoNLEB9FfEl8I7nn6uTenr5WxjPAOpwjZl9ObB/95xM
+p91abKbLQLev5I8npM9G3C/n01l3IzRs7DNHqGJTZO7frGhicD7/jNa+tkSioeJ2
+fIMqgDOvQE+gMxs19zw1tsI3+kqX7+ptTkU4Lan5V5ZKGfU8xtcVIlyRk5/yDUI5
+1dfQVubs6z07s6De2qa92LFz9l8sT6QuVer+c/wPPhBdMwbzcHyUJIBjFaBpxH86
+F7Mr5Zr/+qcbHglAHow1lBqdZzimqGd1koqFRat/pFUFh0iqktMmpl+ZUCjyoQEX
+93j8aMU2UQjYM8NJDE2aRculo9OEoqERYFM2m3nHvrtE7iZgddryLNH7ZmC1EquX
+MhZJ26GuZ2U4b9dAX858WTv0q1EF5S8KObMlxMU7IDk+cWlSD+puWliwfUKoTR/4
+JErSfjCSaRqh
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.der.sha256

@@ -1 +1 @@
-8b23238088af65cbae6ee9c23821068d896ec1dad081e2a1035ff70866943247
+d403930d5296f1515aadd3f730757e7719188b63a276687a3475128b746e4340

data/spec/fixtures/test_certs/renew.sh

@@ -3,8 +3,7 @@
 set -e
 cd "$(dirname "$0")"
 
-openssl x509 -x509toreq -in ca.crt -signkey ca.key -out ca.csr
-openssl x509 -req -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
+openssl req -x509 -new -nodes -key ca.key -subj "/C=PT/ST=NA/L=Lisbon/O=MyLab/CN=root" -sha256 -days 3650 -out ca.crt
 openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
 
 openssl x509 -x509toreq -in test.crt -signkey test.key -out test.csr
@@ -13,4 +12,4 @@ openssl x509 -in test.crt -outform der | sha256sum | awk '{print $1}' > test.der
 openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12
 
 # output ISO8601 timestamp to file
-date -Iseconds > GENERATED_AT
+date -Iseconds > GENERATED_AT

data/spec/fixtures/test_certs/test.crt

@@ -1,30 +1,31 @@
 -----BEGIN CERTIFICATE-----
-MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
-BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
-DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBTMQswCQYD
-VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
-TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA
-A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2
-rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAszLdi
-SIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdAN57w
-EN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7YlVYy
-JpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKOdweN
-am76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxTCrKR
-h58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn50lW
-QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO
-fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe
-cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3
-cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID
-AQABMA0GCSqGSIb3DQEBCwUAA4ICAQAlB7YFw7e1pzYz55Kqa91juTWP5XiCn59n
-J0YKM++vdHqy224HZb9jGtJjZ+0Wod4cwiOVWm+5hLs4IrzfGuXFZEFx/VWP3SDq
-4F3IJJXQkc7jSNrL6IR92xRDSB+yFZZI6FFsnaKMT2fZELndPVFnH+oclc8ZZoyz
-2H/r1CT4yYx7YclAWUqq8Ci3J82qUeeM8Xj9fzGFKy6oCoRsApQb4qb4DoQ1TbZC
-b8gWxHj8l4izul1MtTzSkoMb0Ot50vMoT69m1hDz5H4wF6KuAZUAgM9LQWNHJCkt
-hlOXvqFTHF+y+bvK+hGs976xViq3HA45M3+5Psv0+fdoHgYQJvd23yt8CM0rGfv3
-P+34HlLCW+FdWiazmo+tl5YmtGs6pYuAEp2z5pmUO2l2CutFmv4xBOvXF+rZOzxY
-Q0ackJtflnDC/Tlq2qAldY3Oa8nyI3UIaMUcqHemwm5KpDjc0XF2J1qCoSrMxD8+
-L8HdvUYlh3DIFgJIG1DlTtfQO+RwrVi9+NBBGAsforla9HJDO/POiv7O9hED71u+
-pev8flmULeisMeYqeiL55jyS/+45VaF7t36FMyiP3zXANwbHZMvzVobEsXAuzPOt
-pVNo/EpszrdBe9JWt1GrFLY9c14FmWG8cAWpcwRH0ofhJPPvEB7usFVWCSduOAbA
-Zytzb+8iSw==
+MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL
+MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL
+BgNVBAMMBHJvb3QwHhcNMjUwNzIyMTAxNTAzWhcNMjYwNzIyMTAxNTAzWjBTMQsw
+CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE
+CgwFTXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFc
+V0r2rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAs
+zLdiSIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdA
+N57wEN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7Y
+lVYyJpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKO
+dweNam76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxT
+CrKRh58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn
+50lWQcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcn
+M6EOfy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYi
+cNUecqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkS
+jsn3cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV
+1wIDAQABo0IwQDAdBgNVHQ4EFgQUb789MhsOk89lMWwSwBss1TLXDFAwHwYDVR0j
+BBgwFoAUoVp0nHjq6mJ/UGuFhnSK7yjbPyswDQYJKoZIhvcNAQELBQADggIBAI+G
+NKZ3s3m+/R4mH3M84gGWPE1joC2bLavYYLZjKnZv18o6fHX0IW/8v5hd5Df3SP5u
+vhjC88bewiKVHldqkC6ju9rbZxQynhFZGXbN9zLvFMZGkfRH5vB2Y13ZWBdWhq5L
+cRxpRk6WlwaSy0Ed4F12u9ERmhMOtSZhqAnNJBeVraOHeGlcMZXJdZkeyxkdcZ4y
+YJcrI8Da0dMxILgIuc9ZCynAItRAjMw1/3wjlx0Cyxif10ct+EFiP6Zv/gzoo05v
+tNeqOCrxAqAcwrS1u4q/KAKySiEIyxyU1nEI/g53nALwoQhFsRVqVXNAoy7xu37y
+o+lvs98rkq/NkkbBvRBPdcF/BYNtesRxKja/QAEvslyZfyICL9oqsuPPEB2nHtXa
+mWntT2NaXyr1FWCxHaXfZQOxSwco3vTk7HLuNug2wxIc/hewkLlk5NCRkAYfTlan
+gLhZ3vBej4oA8cdpODMb8SrYhqKTeX8E+ulHVS0paY0kszAGK2x2kHqRGNXUlfoB
+Ax0etGudHhgtTCAmUgJDyQNLkvBKHYQJ2V/Wv/xej7wXKkACNKlRORl8zcnbVErd
+GM/ibfqNIPIo8dP2EDycSV6vIICqkxpCZZObNjfgKa0UN03qYi7xREhhEehXgU8H
+IO9w2pG7ReiO2E+bLIs0Zh1+2IwlM1EM/eqbq+Gi
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.der.sha256

@@ -1 +1 @@
-80329a197063dea8cf7905d10d221648bbdbc05b8fb1d4c2e384b831bc6590df
+386ae6ef809d20ddfcc7ca68f480e82007c031b365c86cc58922cf1bd7238f89

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -36,7 +36,9 @@ end
       {
         "Content-Encoding" => "gzip",
         "Content-Type" => "application/json",
-        'x-elastic-product-origin' => 'logstash-output-elasticsearch'
+        'x-elastic-product-origin' => 'logstash-output-elasticsearch',
+        'X-Elastic-Event-Count' => anything,
+        'X-Elastic-Uncompressed-Request-Length' => anything,
       }
     }
 

data/spec/integration/outputs/index_spec.rb

@@ -215,12 +215,22 @@ describe "indexing" do
 
     it "sets the correct content-type header" do
       expected_manticore_opts = {
-        :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'},
+        :headers => {
+          "Content-Type" => "application/json",
+          'x-elastic-product-origin' => 'logstash-output-elasticsearch',
+          'X-Elastic-Event-Count' => anything,
+          'X-Elastic-Uncompressed-Request-Length' => anything
+        },
         :body => anything
       }
       if secure
         expected_manticore_opts = {
-          :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'},
+          :headers => {
+            "Content-Type" => "application/json",
+            'x-elastic-product-origin' => 'logstash-output-elasticsearch',
+            'X-Elastic-Event-Count' => anything,
+            'X-Elastic-Uncompressed-Request-Length' => anything
+          },
           :body => anything,
           :auth => {
             :user => user,

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -270,6 +270,83 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
        end
      end
+    context "the 'user-agent' header" do
+      let(:pool) { double("pool") }
+      let(:compression_level) { 6 }
+      let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) }
+      let(:actions) { [
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}],
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}],
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}],
+      ]}
+      let(:message_1) { "hello" }
+      let(:message_2_size) { 1_000 }
+      let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 }
+      let(:message_3_size) { 1_000 }
+      let(:message_3) { "m" * message_3_size }
+      let(:messages_size) { message_1.size + message_2.size + message_3.size }
+      let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action
+
+      let(:response) do
+        response = double("response")
+        allow(response).to receive(:code).and_return(response)
+        allow(response).to receive(:body).and_return({"errors" => false}.to_json)
+        response
+      end
+
+      before(:each) do
+        subject.instance_variable_set("@pool", pool)
+      end
+
+      it "carries bulk request's uncompressed size" do
+        expect(pool).to receive(:post) do |path, params, body|
+          headers = params.fetch(:headers, {})
+          expect(headers["X-Elastic-Event-Count"]).to eq("3")
+          expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
+        end.and_return(response)
+
+        subject.send(:bulk, actions)
+      end
+      context "without compression" do
+        let(:compression_level) { 0 }
+        it "carries bulk request's uncompressed size" do
+          expect(pool).to receive(:post) do |path, params, body|
+            headers = params.fetch(:headers, {})
+            expect(headers["X-Elastic-Event-Count"]).to eq("3")
+            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
+          end.and_return(response)
+          subject.send(:bulk, actions)
+        end
+      end
+
+      context "with compressed messages over 20MB" do
+        let(:message_2_size) { 21_000_000 }
+        it "carries bulk request's uncompressed size" do
+          # only the first, tiny, message is sent first
+          expect(pool).to receive(:post) do |path, params, body|
+            headers = params.fetch(:headers, {})
+            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s
+            expect(headers["X-Elastic-Event-Count"]).to eq("1")
+          end.and_return(response)
+
+          # huge message_2 is sent afterwards alone
+          expect(pool).to receive(:post) do |path, params, body|
+            headers = params.fetch(:headers, {})
+            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s
+            expect(headers["X-Elastic-Event-Count"]).to eq("1")
+          end.and_return(response)
+
+          # finally medium message_3 is sent alone as well
+          expect(pool).to receive(:post) do |path, params, body|
+            headers = params.fetch(:headers, {})
+            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s
+            expect(headers["X-Elastic-Event-Count"]).to eq("1")
+          end.and_return(response)
+
+          subject.send(:bulk, actions)
+        end
+      end
+    end
   end
 
   describe "sniffing" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -770,7 +770,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
       before(:each) do
-        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
+        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array), instance_of(Hash)) do |stream, actions, headers|
           expect( stream.string ).to include '"foo":"bar1"'
           expect( stream.string ).to include '"foo":"bar2"'
         end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 12.0.4
+  version: 12.0.6
 platform: java
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-07-07 00:00:00.000000000 Z
+date: 2025-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: manticore
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-  name: manticore
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,6 +30,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-  name: stud
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -51,6 +50,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,7 +59,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -71,12 +70,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -85,12 +84,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-deprecation_logger_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-deprecation_logger_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -99,12 +98,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ca_trusted_fingerprint_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-ca_trusted_fingerprint_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -113,12 +112,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-normalize_config_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-normalize_config_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -127,12 +126,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -141,12 +140,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -155,12 +154,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: flores
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -169,12 +168,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-  name: cabin
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -183,12 +182,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
+  name: webrick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webrick
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -197,12 +196,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webmock
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -211,12 +210,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-collection_matchers
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -225,12 +224,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: elasticsearch
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -341,7 +340,6 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -356,8 +354,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Stores logs in Elasticsearch
 test_files: