logstash-output-elasticsearch 11.22.16-java → 12.0.0-java

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -241,6 +241,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         @dlq_writer.write(event, "#{detailed_message}")
       else
         log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
+
         @logger.public_send(log_level, message, status: status, action: action_params, response: response)
       end
     end
@@ -272,7 +273,6 @@ module LogStash; module PluginMixins; module ElasticSearch
       end
 
       actions_to_retry = []
-      dlq_routed_stats = {}
       responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
@@ -294,11 +294,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         elsif @dlq_codes.include?(status)
           handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:dlq_routed)
-          if dlq_routed_stats.key?(status)
-            dlq_routed_stats[status][:count] += 1
-          else
-            dlq_routed_stats[status] = { :count => 1, :sample_event => { :action => action, :response => response } }
-          end
           next
         else
           # only log what the user whitelisted
@@ -308,11 +303,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         end
       end
 
-      if @dlq_writer && !dlq_routed_stats.empty?
-        total = dlq_routed_stats.values.sum { |entry| entry[:count] }
-        @logger.warn("Events could not be indexed and routing to DLQ", :count => total, :dlq_routed_stats => dlq_routed_stats)
-      end
-
       actions_to_retry
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = ::File.read('version').split("\n").first
+  s.version         = '12.0.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.platform = RUBY_PLATFORM
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/es_spec_helper.rb

@@ -1,18 +1,15 @@
 require_relative './spec_helper'
 
 require 'elasticsearch'
+require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
+require_relative "support/elasticsearch/api/actions/get_alias"
+require_relative "support/elasticsearch/api/actions/put_alias"
+require_relative "support/elasticsearch/api/actions/get_ilm_policy"
+require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
 require 'cabin'
 
-# remove this condition and support package once plugin starts consuming elasticsearch-ruby v8 client
-# in elasticsearch-ruby v7, ILM APIs were in a separate xpack gem, now directly available
-unless elastic_ruby_v8_client_available?
-  require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
-  require_relative "support/elasticsearch/api/actions/get_ilm_policy"
-  require_relative "support/elasticsearch/api/actions/put_ilm_policy"
-end
-
 module ESHelper
   def get_host_port
     if ENV["INTEGRATION"] == "true"
@@ -23,12 +20,8 @@ module ESHelper
   end
 
   def get_client
-    if elastic_ruby_v8_client_available?
-      Elasticsearch::Client.new(:hosts => [get_host_port])
-    else
-      Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
-        allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
-      end
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
     end
   end
 
@@ -135,36 +128,31 @@ module ESHelper
   end
 
   def get_policy(client, policy_name)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.get_lifecycle(policy: policy_name)
-    else
-      client.get_ilm_policy(name: policy_name)
-    end
+    client.get_ilm_policy(name: policy_name)
   end
 
   def put_policy(client, policy_name, policy)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.put_lifecycle({:policy => policy_name, :body=> policy})
-    else
-      client.put_ilm_policy({:name => policy_name, :body=> policy})
-    end
+    client.put_ilm_policy({:name => policy_name, :body=> policy})
+  end
+
+  def put_alias(client, the_alias, index)
+    body = {
+        "aliases" => {
+            index => {
+                "is_write_index"=>  true
+            }
+        }
+    }
+    client.put_alias({name: the_alias, body: body})
   end
 
   def clean_ilm(client)
-    if elastic_ruby_v8_client_available?
-      client.index_lifecycle_management.get_lifecycle.each_key { |key| client.index_lifecycle_management.delete_lifecycle(policy: key) if key =~ /logstash-policy/ }
-    else
-      client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ }
-    end
+    client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key)  if key =~ /logstash-policy/ }
   end
 
   def supports_ilm?(client)
     begin
-      if elastic_ruby_v8_client_available?
-        client.index_lifecycle_management.get_lifecycle
-      else
-        client.get_ilm_policy
-      end
+      client.get_ilm_policy
       true
     rescue
       false

data/spec/fixtures/test_certs/GENERATED_AT

@@ -1 +1 @@
-2025-07-22T11:15:03+01:00
+2024-06-25T21:50:58+01:00

data/spec/fixtures/test_certs/ca.crt

@@ -1,32 +1,29 @@
 -----BEGIN CERTIFICATE-----
-MIIFdTCCA12gAwIBAgIUDITbsLT9hKser0ZzBZsxqgaZdWswDQYJKoZIhvcNAQEL
-BQAwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
-DjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MB4XDTI1MDcyMjEwMTUwM1oX
-DTM1MDcyMDEwMTUwM1owSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD
-VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MIICIjAN
-BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3n
-z6C9w63tD+w4S9wNAmCTNn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjk
-W5C2EP86UDX9Eeq1tjbsnfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwy
-LLwXNDnlMymgzflxa6+gAzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn
-7jcGQbASKYjmSbuM0LWfCOIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CD
-EW3Rhtf3zFb9pSqxyAFRWz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNN
-D4xfMkO3Qm9fxIgk+ZsGeDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0
-kPnQm/aJfiXiB7gfUIgrX+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8
-hPCaZ+l+T1z/JY98Luy0eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuC
-fREAGiNGgZ2GFjdAOKmDCZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO
-+WbApbcXIwOHBgv/NfsnRzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfz
-Iokr0sEBStgKMncCAwEAAaNTMFEwHQYDVR0OBBYEFKFadJx46upif1BrhYZ0iu8o
-2z8rMB8GA1UdIwQYMBaAFKFadJx46upif1BrhYZ0iu8o2z8rMA8GA1UdEwEB/wQF
-MAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJi4FwYJz/RotoUpfrLZFf69RoI01Fje
-8ITt8SR1Dx/1GTPEuqVVfx0EYtOoH6Gg3FwgSQ9GHRDIa1vkHY5S+FUSOW3pCoZE
-/kaLu9bmFxn+GntghvQEor+LzODuZKLXupaGcu1tA4fzyuI4jglVD2sGZtLk//CT
-Hd4tOWXo5k1Fj0jMnJq+2Htr8yBeSAO5ZNsvtAjOUU6pfDEwL9bgRzlKKFQQMUYo
-6x1FvRDRXWjpzB/H+OSqOaoNLEB9FfEl8I7nn6uTenr5WxjPAOpwjZl9ObB/95xM
-p91abKbLQLev5I8npM9G3C/n01l3IzRs7DNHqGJTZO7frGhicD7/jNa+tkSioeJ2
-fIMqgDOvQE+gMxs19zw1tsI3+kqX7+ptTkU4Lan5V5ZKGfU8xtcVIlyRk5/yDUI5
-1dfQVubs6z07s6De2qa92LFz9l8sT6QuVer+c/wPPhBdMwbzcHyUJIBjFaBpxH86
-F7Mr5Zr/+qcbHglAHow1lBqdZzimqGd1koqFRat/pFUFh0iqktMmpl+ZUCjyoQEX
-93j8aMU2UQjYM8NJDE2aRculo9OEoqERYFM2m3nHvrtE7iZgddryLNH7ZmC1EquX
-MhZJ26GuZ2U4b9dAX858WTv0q1EF5S8KObMlxMU7IDk+cWlSD+puWliwfUKoTR/4
-JErSfjCSaRqh
+MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
+BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
+DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBMMQswCQYD
+VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
+TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
+AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh
+5er5JF2iomYOHiD5JncYr4YmRQKuYfD6B1WI5FuQthD/OlA1/RHqtbY27J33SaO6
+6ro8gal7vjHrXKQkefVYRwdfO6DqqbhV6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQq
+GZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju4jPhp+43BkGwEimI5km7jNC1nwjiHtxD
+sY/s93AKa/vLktXKUK5nA3jjJOhAbRTVnbOAgxFt0YbX98xW/aUqscgBUVs9J/My
+TRMwVKJ7Vsmth1PdJQksUASuzESlSPl09dMjTQ+MXzJDt0JvX8SIJPmbBng78MSa
+CUhpOZiii1l2mBfPWejx20I/SMCUNmzbwm2w9JD50Jv2iX4l4ge4H1CIK1/orW1p
+dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv
+5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx
+r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP
+LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ
+KoZIhvcNAQELBQADggIBABym9LMyS9W9lvpcH4OK1YLfBPJwrhZ+4keiriY4zWOo
+pB+v2Q35neMMXSlTDpeIwPdMkqsh8VZprOWURF80JGvpJ6fBfi05rCDWp/ol1ZKi
+snCA+dE2zDK7Z3+F0MbakT5oBi5WgkXSvRvlJEJ/gBD7WC1wq0kxCMK+M5w2RPAT
+nnV/iozNBkwExxyJA7BpS6F/v0XjwK7fm5Kpql7zKlh5piZ2IVU0B60Sqskcb2mU
+90+1r9T06ekIW/Iz1jd5RWYziu0nbmDeKeKvGAICNU+evYXW+/5kKecMLuEvDCgS
+ssbt/Hb510uLHhxfhN4SbvBl2zADsLC+2arf2ATIwD8ZXDDs04ayBsejV0ZwVrTZ
+ExKqAys+B3tuIHGRqL5VukdmH6g6oJziYueohPBCOuSOzDd0FhppF4uXZS8DReSg
+KieO2ZYfiA1gVRiY6jPx+r7J9I5kSS1gwr/e3zHJHa79ijMB1SSIswQUmgSMkwGh
+sNyDNI9ZxgJan3v7kVargMt2LiNcXvVyTzPSYSXcY7SoebfpMprVIG7vZ9TZf+Uu
+FQeOfxdLFuGTnpFrYmvOD3OIKfODlY5t+TNICg7A3eTUXeJPcdBBnuVCiQU6TCB5
+H+69K5w54Q6a70sHZU1IWsGT8XtbUizPNQky+LAFsE/5oUnCqtypeEu4srcZK53x
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.der.sha256

@@ -1 +1 @@
-d403930d5296f1515aadd3f730757e7719188b63a276687a3475128b746e4340
+8b23238088af65cbae6ee9c23821068d896ec1dad081e2a1035ff70866943247

data/spec/fixtures/test_certs/renew.sh

@@ -3,7 +3,8 @@
 set -e
 cd "$(dirname "$0")"
 
-openssl req -x509 -new -nodes -key ca.key -subj "/C=PT/ST=NA/L=Lisbon/O=MyLab/CN=root" -sha256 -days 3650 -out ca.crt
+openssl x509 -x509toreq -in ca.crt -signkey ca.key -out ca.csr
+openssl x509 -req -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
 openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
 
 openssl x509 -x509toreq -in test.crt -signkey test.key -out test.csr
@@ -12,4 +13,4 @@ openssl x509 -in test.crt -outform der | sha256sum | awk '{print $1}' > test.der
 openssl pkcs12 -export -inkey test.key -in test.crt -passout "pass:1234567890" -out test.p12
 
 # output ISO8601 timestamp to file
-date -Iseconds > GENERATED_AT
+date -Iseconds > GENERATED_AT

data/spec/fixtures/test_certs/test.crt

@@ -1,31 +1,30 @@
 -----BEGIN CERTIFICATE-----
-MIIFWjCCA0KgAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL
-MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL
-BgNVBAMMBHJvb3QwHhcNMjUwNzIyMTAxNTAzWhcNMjYwNzIyMTAxNTAzWjBTMQsw
-CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE
-CgwFTXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEB
-AQUAA4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFc
-V0r2rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAs
-zLdiSIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdA
-N57wEN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7Y
-lVYyJpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKO
-dweNam76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxT
-CrKRh58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn
-50lWQcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcn
-M6EOfy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYi
-cNUecqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkS
-jsn3cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV
-1wIDAQABo0IwQDAdBgNVHQ4EFgQUb789MhsOk89lMWwSwBss1TLXDFAwHwYDVR0j
-BBgwFoAUoVp0nHjq6mJ/UGuFhnSK7yjbPyswDQYJKoZIhvcNAQELBQADggIBAI+G
-NKZ3s3m+/R4mH3M84gGWPE1joC2bLavYYLZjKnZv18o6fHX0IW/8v5hd5Df3SP5u
-vhjC88bewiKVHldqkC6ju9rbZxQynhFZGXbN9zLvFMZGkfRH5vB2Y13ZWBdWhq5L
-cRxpRk6WlwaSy0Ed4F12u9ERmhMOtSZhqAnNJBeVraOHeGlcMZXJdZkeyxkdcZ4y
-YJcrI8Da0dMxILgIuc9ZCynAItRAjMw1/3wjlx0Cyxif10ct+EFiP6Zv/gzoo05v
-tNeqOCrxAqAcwrS1u4q/KAKySiEIyxyU1nEI/g53nALwoQhFsRVqVXNAoy7xu37y
-o+lvs98rkq/NkkbBvRBPdcF/BYNtesRxKja/QAEvslyZfyICL9oqsuPPEB2nHtXa
-mWntT2NaXyr1FWCxHaXfZQOxSwco3vTk7HLuNug2wxIc/hewkLlk5NCRkAYfTlan
-gLhZ3vBej4oA8cdpODMb8SrYhqKTeX8E+ulHVS0paY0kszAGK2x2kHqRGNXUlfoB
-Ax0etGudHhgtTCAmUgJDyQNLkvBKHYQJ2V/Wv/xej7wXKkACNKlRORl8zcnbVErd
-GM/ibfqNIPIo8dP2EDycSV6vIICqkxpCZZObNjfgKa0UN03qYi7xREhhEehXgU8H
-IO9w2pG7ReiO2E+bLIs0Zh1+2IwlM1EM/eqbq+Gi
+MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
+BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
+DAZSb290Q0EwHhcNMjQwNjI1MjA1MDU4WhcNMjUwNjI1MjA1MDU4WjBTMQswCQYD
+VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
+TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2
+rtmWfeK5EEdsTS/Ey4owTceOplPpAp4svF+a/i1/bHhqnQYYU4f7Qic4fDAszLdi
+SIo0o1csNvIogm/P+uvSzE6eZRZUSmo49dY5SKSJt6Pjh6lM2MHEjsPKIKdAN57w
+EN90q4IZv6AHE9rphqxcmF1k+j5xmhCUS1EJ+y7hyZ0S7Hghdgp/0cxSu/7YlVYy
+JpkIlQd3RPXzEf6VSYjtr9Ajp1rhvv2611q0CB5NALg/KR3OiMPYmTg5HAKOdweN
+am76nG3VxTeV3y+LW/pZAbi4qAl+4/c0eOGsL7o/YSn7qhThU1AWS9kY1WxTCrKR
+h58rUGRfmvpnOR99xvR4jz942RNiY61pTmsvo+iJspTII3GZhwIGlHtxE9Rn50lW
+QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO
+fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe
+cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3
+cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID
+AQABMA0GCSqGSIb3DQEBCwUAA4ICAQAlB7YFw7e1pzYz55Kqa91juTWP5XiCn59n
+J0YKM++vdHqy224HZb9jGtJjZ+0Wod4cwiOVWm+5hLs4IrzfGuXFZEFx/VWP3SDq
+4F3IJJXQkc7jSNrL6IR92xRDSB+yFZZI6FFsnaKMT2fZELndPVFnH+oclc8ZZoyz
+2H/r1CT4yYx7YclAWUqq8Ci3J82qUeeM8Xj9fzGFKy6oCoRsApQb4qb4DoQ1TbZC
+b8gWxHj8l4izul1MtTzSkoMb0Ot50vMoT69m1hDz5H4wF6KuAZUAgM9LQWNHJCkt
+hlOXvqFTHF+y+bvK+hGs976xViq3HA45M3+5Psv0+fdoHgYQJvd23yt8CM0rGfv3
+P+34HlLCW+FdWiazmo+tl5YmtGs6pYuAEp2z5pmUO2l2CutFmv4xBOvXF+rZOzxY
+Q0ackJtflnDC/Tlq2qAldY3Oa8nyI3UIaMUcqHemwm5KpDjc0XF2J1qCoSrMxD8+
+L8HdvUYlh3DIFgJIG1DlTtfQO+RwrVi9+NBBGAsforla9HJDO/POiv7O9hED71u+
+pev8flmULeisMeYqeiL55jyS/+45VaF7t36FMyiP3zXANwbHZMvzVobEsXAuzPOt
+pVNo/EpszrdBe9JWt1GrFLY9c14FmWG8cAWpcwRH0ofhJPPvEB7usFVWCSduOAbA
+Zytzb+8iSw==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/test.der.sha256

@@ -1 +1 @@
-386ae6ef809d20ddfcc7ca68f480e82007c031b365c86cc58922cf1bd7238f89
+80329a197063dea8cf7905d10d221648bbdbc05b8fb1d4c2e384b831bc6590df

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -36,9 +36,7 @@ end
       {
         "Content-Encoding" => "gzip",
         "Content-Type" => "application/json",
-        'x-elastic-product-origin' => 'logstash-output-elasticsearch',
-        'X-Elastic-Event-Count' => anything,
-        'X-Elastic-Uncompressed-Request-Length' => anything,
+        'x-elastic-product-origin' => 'logstash-output-elasticsearch'
       }
     }
 

data/spec/integration/outputs/delete_spec.rb

@@ -39,12 +39,12 @@ describe "Versioned delete", :integration => true do
     it "should ignore non-monotonic external version updates" do
       id = "ev2"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
+      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r2['_version']).to eq(99)
       expect(r2['_source']['message']).to eq('foo')
     end
@@ -52,12 +52,12 @@ describe "Versioned delete", :integration => true do
     it "should commit monotonic external version updates" do
       id = "ev3"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :id => id, :refresh => true)
+      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :id => id, :refresh => true) }.to raise_error(get_expected_error_class)
+      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
     end
   end
 end

data/spec/integration/outputs/ilm_spec.rb

@@ -102,7 +102,7 @@ shared_examples_for 'an ILM disabled Logstash' do
   it 'should not install the default policy' do
     subject.register
     sleep(1)
-    expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+    expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
   end
 
   it 'should not write the ILM settings into the template' do
@@ -282,12 +282,12 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
           subject.register
           sleep(1)
           expect(@es.indices.exists_alias(name: "logstash")).to be_truthy
-          expect(@es.indices.get_alias(name: "logstash")).to include("logstash-000001")
+          expect(@es.get_alias(name: "logstash")).to include("logstash-000001")
         end
       end
 
       it 'should install it if it is not present' do
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
         subject.register
         sleep(1)
         expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.not_to raise_error
@@ -298,7 +298,7 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
         subject.register
         sleep(1)
         expect(@es.indices.exists_alias(name: "logstash")).to be_truthy
-        expect(@es.indices.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
+        expect(@es.get_alias(name: "logstash")).to include("logstash-#{todays_date}-000001")
       end
 
       it 'should ingest into a single index' do
@@ -340,14 +340,14 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
       let (:policy) { small_max_doc_policy }
 
       before do
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
         put_policy(@es,ilm_policy_name, policy)
       end
 
       it 'should not install the default policy if it is not used' do
         subject.register
         sleep(1)
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
     end
 
@@ -357,14 +357,14 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
       let (:policy) { max_age_policy("1d") }
 
       before do
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
         put_policy(@es,ilm_policy_name, policy)
       end
 
       it 'should not install the default policy if it is not used' do
         subject.register
         sleep(1)
-        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(get_expected_error_class)
+        expect{get_policy(@es, LogStash::Outputs::ElasticSearch::DEFAULT_POLICY)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
     end
 
@@ -374,7 +374,7 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
         subject.register
         sleep(1)
         expect(@es.indices.exists_alias(name: expected_index)).to be_truthy
-        expect(@es.indices.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001")
+        expect(@es.get_alias(name: expected_index)).to include("#{expected_index}-#{todays_date}-000001")
       end
 
       it 'should write the ILM settings into the template' do
@@ -443,18 +443,17 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
         subject.register
         sleep(1)
         expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
-        expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
+        expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
       end
 
       context 'when the custom rollover alias already exists' do
         it 'should ignore the already exists error' do
           expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_falsey
-          @es.indices.create(index: "#{ilm_rollover_alias}-#{todays_date}-000001")
-          @es.indices.put_alias(name: ilm_rollover_alias, index: "#{ilm_rollover_alias}-#{todays_date}-000001")
+          put_alias(@es, "#{ilm_rollover_alias}-#{todays_date}-000001", ilm_rollover_alias)
           expect(@es.indices.exists_alias(name: ilm_rollover_alias)).to be_truthy
           subject.register
           sleep(1)
-          expect(@es.indices.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
+          expect(@es.get_alias(name: ilm_rollover_alias)).to include("#{ilm_rollover_alias}-#{todays_date}-000001")
         end
 
       end
@@ -533,8 +532,3 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
   end
 
 end
-
-def get_expected_error_class
-  return Elastic::Transport::Transport::Errors::NotFound if elastic_ruby_v8_client_available?
-  Elasticsearch::Transport::Transport::Errors::NotFound
-end

data/spec/integration/outputs/index_spec.rb

@@ -215,22 +215,12 @@ describe "indexing" do
 
     it "sets the correct content-type header" do
       expected_manticore_opts = {
-        :headers => {
-          "Content-Type" => "application/json",
-          'x-elastic-product-origin' => 'logstash-output-elasticsearch',
-          'X-Elastic-Event-Count' => anything,
-          'X-Elastic-Uncompressed-Request-Length' => anything
-        },
+        :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'},
         :body => anything
       }
       if secure
         expected_manticore_opts = {
-          :headers => {
-            "Content-Type" => "application/json",
-            'x-elastic-product-origin' => 'logstash-output-elasticsearch',
-            'X-Elastic-Event-Count' => anything,
-            'X-Elastic-Uncompressed-Request-Length' => anything
-          },
+          :headers => {"Content-Type" => "application/json", 'x-elastic-product-origin' => 'logstash-output-elasticsearch'},
           :body => anything,
           :auth => {
             :user => user,

data/spec/integration/outputs/index_version_spec.rb

@@ -36,11 +36,11 @@ describe "Versioned indexing", :integration => true do
 
       it "should default to ES version" do
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :id => '123', :refresh => true)
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r["_version"]).to eq(1)
         expect(r["_source"]["message"]).to eq('foo')
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
-        r2 = es.get(:index => 'logstash-index', :id => '123', :refresh => true)
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r2["_version"]).to eq(2)
         expect(r2["_source"]["message"]).to eq('foobar')
       end  
@@ -63,7 +63,7 @@ describe "Versioned indexing", :integration => true do
       it "should respect the external version" do
         id = "ev1"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :id => id, :refresh => true)
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
       end
@@ -71,12 +71,12 @@ describe "Versioned indexing", :integration => true do
       it "should ignore non-monotonic external version updates" do
         id = "ev2"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :id => id, :refresh => true)
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true)
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(99)
         expect(r2["_source"]["message"]).to eq('foo')
       end
@@ -84,12 +84,12 @@ describe "Versioned indexing", :integration => true do
       it "should commit monotonic external version updates" do
         id = "ev3"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :id => id, :refresh => true)
+        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :id => id, :refresh => true)
+        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(100)
         expect(r2["_source"]["message"]).to eq('foo')
       end

data/spec/integration/outputs/metrics_spec.rb

@@ -5,10 +5,7 @@ describe "metrics", :integration => true do
     require "logstash/outputs/elasticsearch"
     settings = {
       "manage_template" => false,
-      "hosts" => "#{get_host_port()}",
-      # write data to a random non templated index to ensure the bulk partially fails
-      # don't use streams like "logs-*" as those have failure stores enabled, causing the bulk to succeed instead
-      "index" => "custom_index_#{rand(10000)}"
+      "hosts" => "#{get_host_port()}"
     }
     plugin = LogStash::Outputs::ElasticSearch.new(settings)
   end

data/spec/integration/outputs/painless_update_spec.rb

@@ -22,12 +22,11 @@ describe "Update actions using painless scripts", :integration => true, :update_
     # This can fail if there are no indexes, ignore failure.
     @es.indices.delete(:index => "*") rescue nil
     @es.index(
-      {
-        :index => 'logstash-update',
-        :id => '123',
-        :body => { :message => 'Test', :counter => 1 },
-        :refresh => true
-      })
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
     @es.indices.refresh
   end
 
@@ -47,7 +46,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
         subject = get_es_output(plugin_parameters)
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 4 )])
-        r = @es.get(:index => 'logstash-update', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
         expect(r["_source"]["counter"]).to eq(5)
       end
     end
@@ -58,7 +57,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
       subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
       expect(r["_source"]["message"]).to eq('upsert message')
     end
 
@@ -66,7 +65,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
       subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
       expect(r["_source"]["message"]).to eq('sample message here')
     end
 
@@ -83,7 +82,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
         subject.register
 
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
         expect(r["_source"]["message"]).to eq('upsert message')
       end
 
@@ -92,7 +91,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 1)])
         @es.indices.refresh
-        r = @es.get(:index => 'logstash-update', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
         expect(r["_source"]["counter"]).to eq(1)
       end
     end