logstash-input-elasticsearch 4.9.0 → 4.10.0

data/spec/es_helper.rb

@@ -1,30 +1,31 @@
 module ESHelper
   def self.get_host_port
-    return "elasticsearch:9200" if ENV["INTEGRATION"] == "true" || ENV["SECURE_INTEGRATION"] == "true"
-    raise "This setting is only used for integration tests"
+    if ENV["INTEGRATION"] == "true" || ENV["SECURE_INTEGRATION"] == "true"
+      "elasticsearch:9200"
+    else
+      "localhost:9200" # for local running integration specs outside docker
+    end
   end
 
-  def self.get_client(options = {})
-    ssl_options = {}
-    hosts = [get_host_port]
+  def self.get_client(options)
+    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
+    host, port = get_host_port.split(':')
+    host_opts = { host: host, port: port, scheme: 'http' }
+    ssl_opts = {}
 
     if options[:ca_file]
-      ssl_options = { :ssl  => true, :ca_file => options[:ca_file] }
-      hosts.map! do |h|
-        host, port = h.split(":")
-        { :host => host, :scheme => 'https', :port => port }
-      end
+      ssl_opts = { ca_file: options[:ca_file], version: 'TLSv1.2', verify: false }
+      host_opts[:scheme] = 'https'
     end
 
-    transport_options = {}
-
     if options[:user] && options[:password]
-      token = Base64.strict_encode64("#{options[:user]}:#{options[:password]}")
-      transport_options[:headers] = { :Authorization => "Basic #{token}" }
+      host_opts[:user] = options[:user]
+      host_opts[:password] = options[:password]
     end
 
-    @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options, :ssl => ssl_options,
-                                        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore)
+    Elasticsearch::Client.new(hosts: [host_opts],
+                              transport_options: { ssl: ssl_opts },
+                              transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
   end
 
   def self.doc_type

data/spec/fixtures/test_certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
+8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
+PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
+OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
+oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
+YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
+AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
+MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
+G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
+DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
+sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
+NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
+emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABAoIBABmBC0P6Ebegljkk
+lO26GdbOKvbfqulDS3mN5QMyXkUMopea03YzMnKUJriE+2O33a1mUcuDPWnLpYPK
+BTiQieYHlulNtY0Bzf+R69igRq9+1WpZftGnzrlu7NVxkOokRqWJv3546ilV7QZ0
+f9ngmu+tiN7hEnlBC8m613VMuGGb3czwbCizEVZxlZX0Dk2GExbH7Yf3NNs/aOP/
+8x6CqgL+rhrtOQ80xwRrOlEF8oSSjXCzypa3nFv21YO3J2lVo4BoIwnHgOzyz46A
+b37gekqXXajIYQ0HAB+NDgVoCRFFJ7Xe16mgB3DpyUpUJzwiMedJkeQ0TprIownQ
++1mPe9ECgYEA/K4jc0trr3sk8KtcZjOYdpvwrhEqSSGEPeGfFujZaKOb8PZ8PX6j
+MbCTV12nEgm8FEhZQ3azxLnO17gbJ2A+Ksm/IIwnTWlqvvMZD5qTQ7L3qZuCtbWQ
++EGC/H1SDjhiwvjHcXP61/tYL/peApBSoj0L4kC+U/VaNyvicudKk08CgYEAr46J
+4VJBJfZ4ZaUBRy53+fy+mknOfaj2wo8MnD3u+/x4YWTapqvDOPN2nJVtKlIsxbS4
+qCO+fzUV17YHlsQmGULNbtFuXWJkP/RcLVbe8VYg/6tmk0dJwNAe90flagX2KJov
+8eDX129nNpuUqrNNWsfeLmPmH6vUzpKlga+1zfUCgYBrbUHHJ96dmbZn2AMNtIvy
+iXP3HXcj5msJwB3aKJ8eHMkU1kaWAnwxiQfrkfaQ9bCP0v6YbyQY1IJ7NlvdDs7/
+dAydMtkW0WW/zyztdGN92d3vrx0QUiRTV87vt/wl7ZUXnZt1wcB5CPRCWaiUYHWx
+YlDmHW6N1XdIk5DQF0OegwKBgEt7S8k3Zo9+A5IgegYy8p7njsQjy8a3qTFJ9DAR
+aPmrOc8WX/SdkVihRXRZwxAZOOrgoyyYAcYL+xI+T9EBESh3UoC9R2ibb2MYG7Ha
+0gyN7a4/8eCNHCbs1QOZRAhr+8TFVqv28pbMbWJLToZ+hVns6Zikl0MyzFLtNoAm
+HlMpAoGBAIOkqnwwuRKhWprL59sdcJfWY26os9nvuDV4LoKFNEFLJhj2AA2/3UlV
+v85gqNSxnMNlHLZC9l2HZ3mKv/mfx1aikmFvyhJAnk5u0f9KkexmCPLjQzS5q3ba
+yFuxK2DXwN4x46RgQPFlLjOTCX0BG6rkEu4JdonF8ETSjoCtGEU8
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/client/ls.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDMTCCAhmgAwIBAgIUCJ5+zdYJIlL04EOwC0tqVbZYKQUwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
+VQQDEwJsczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMFY71j9Proa
+x95bWoBQOcc0ncqVvQO+Tk/Do4ynoe64pOJEf3txh1viHbnzFUGf+NF5WW6trlZ2
+ErP81vy7Ds5J5ngXjjdzOOGsTs9+l7KkqPfUwXqGoldWFtr//9mkLvWpd8uPvVtO
+dRnpcjQSlHHEB/HaqxkyBAvHLv1Fi1jTgIgn32NEM2mlCJ3M8OVfO9pqlO/6gjjs
+Miow8zZqtczeuv0JPu7V5xPDrcX0xh0kZdpH4gSh9r314TwZXFCofNUbkZrPV+Q9
+XJs64NlBjJZkd5qwKeXujZRV4eVvAOTlp+4Nh4eDqXE323s5yiPuov0StvNrbh3d
+rcg+IM+RTdMCAwEAAaNiMGAwHQYDVR0OBBYEFGnXHJEJ+LGmQTqDNr50C8FE8pcF
+MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBMGA1UdEQQMMAqCCGxv
+Z3N0YXNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAI08C6IeweN3LrEq
+ZauDVoiE2IdA1/nN3sxl+wL2xfauv1nctxej9TYR3mNoWiacgbbfJkPMCSIYk2Vc
+G396yLiGC1V96FfonnKfr3tKd0BiijTu3u5pOTgNNf5n4TZaTHTYmuKPtWoXyuLR
+QbH3jdgq9aq/9bwK0E5FOmuv6LGatnKzLf56aHjzerSZCnRw7V/1J/Yj3cy6TB1l
+9Lc7IAk4dGyrgwfKCuZOSzAtCWpOA/FfqCqMSuW6lrZ1zAXnk6VI3RkmBCuLE6kj
+aAjwORJHyiwBsMQbaYcQaXXjhguS+iHrnWdR0DFs9gHJSTuf6EoOeygdtMDutb2O
+4xfHWG8=
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/client/ls.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwVjvWP0+uhrH3ltagFA5xzSdypW9A75OT8OjjKeh7rik4kR/
+e3GHW+IdufMVQZ/40XlZbq2uVnYSs/zW/LsOzknmeBeON3M44axOz36XsqSo99TB
+eoaiV1YW2v//2aQu9al3y4+9W051GelyNBKUccQH8dqrGTIEC8cu/UWLWNOAiCff
+Y0QzaaUInczw5V872mqU7/qCOOwyKjDzNmq1zN66/Qk+7tXnE8OtxfTGHSRl2kfi
+BKH2vfXhPBlcUKh81RuRms9X5D1cmzrg2UGMlmR3mrAp5e6NlFXh5W8A5OWn7g2H
+h4OpcTfbeznKI+6i/RK282tuHd2tyD4gz5FN0wIDAQABAoIBAGLHVvC14PgfeoEl
+VuU7F2moffzj5z8kWMnzf3j6o4ZcmxBmQmMEq0zMBrfbcr6mRe5u+rvKy8isZf3C
+bOuNfZDyvGYaUrQNj7/r0g+78zB3Y0PKVFaOth28g8y7ATFl6f/j5qn+85TUTotA
+cvIbk+9TYWO0fblPjjWeO2l1wC1ObV0pFrXVzthFTQk0oxl3Gyq54P5rPp7QPOdM
+jB0lSAhvkEEA7nIZ0dF1zm3RuByahSQC45kYEp5TCC8SDwO3WCJ32Q/dhPw0y9Go
+gbIEY9QBFc8Rn3HDYGTSRPVVDM3HctDqzCihWOgFQlv718yCAZYq43fdOcy+6BY/
+UT3xRhECgYEA5wWcYD8bY5x0fk2ubxKJTE4Fm2HvUrexvLX00yF3sX9XqrsiBBC4
+14mB2yFEKQL+M/ZqLNV9IfJhfRmBffoRvf9lAuHhUrw4wGijERoO9xkmS3lGvzw+
+hqLIMnd986wyWC8FdU+dStDg8PC97xK4bg0tKYo9jmKf3kb0Ov2UWQ0CgYEA1kCK
+w30P3ZgYakXUjbwv3KoRXIVm6cIqgvm8iPh7B0GeTUCIgGBSNzJ8LVts/m6V7xtU
+RJ8RKw2gqqUTiFqrRKefr0APFv0YeVgZ7zK2EGLrL2rFWNZZQqA97Mh/5CwFh+zs
+1/IYuqRUVDaLq0sAPcZA5KEoO91eChBVK/RAyl8CgYAqgVPGOZY2e6DLZEuF0ClG
+yswpTJmV5IplKC1Fc1DsbXuZxBh8Gv+HWJt1z+cUjKJsuRfL6/O7/TaGp9y1av88
+r/LL1vd4G31tmVL3YI4EVLJBDK1Bnjn615RyBJ496R7SLsSYUu+jxk68xe6MQCuC
+xBXdILw2qFq1sORavjE/OQKBgQCJlwNGDX9t2Cn9vYCF0Q+Pjyv9FbKEdevlFsor
+0B76BvrJM6M1hiXmSqaSXj89mfjxh8RzGQ/mbSb7z20eyNNqEJes7N+D7N+Vta1Z
+/mALX+sXFWNM7MJ/1fZOpGf1OQwIQW/MMi4NVlDNkAXb6BtskG/GI3R6FWw53ElG
+I+Kj0wKBgQDUCfiYYSA9wYug53uvE0LirZwMQjWTbqxMO/S3H0Be8rf+BLo+YY5X
+fF6L/NaKxNZzs2Uu6+xYLXhSMimiv8412t7qnjK5DLwTZfnj7EuNxd44cZ4cXDPx
+JtD9GgPJ5V3js1gA2usPnKLYPw5Pp3ayZn1q4Tg9A5cextKnyOV/Fg==
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/es.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
+VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
+1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
+Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
+CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
+2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
+LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
+tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
+MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
+YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
+7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
+f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
+Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
+LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
+qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
+qi02i4q6meHGcw==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/es.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5VV21nXpYzQJoQ
+buWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz36pcFw7UyF51
+/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/cMjNrUC7iP0dv
+fOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH/z07/mVKoBAa
+5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gRhQNM3zcKKsjE
+MomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABAoIBAQCm/VBDz41ImG7p
+yu3e6iMeFi7HW5SKdlRUS5dJbHT1uBWJAm/q8TbwvnUBVdsn9cKWY06QYDPQBjAy
+0LxRSIKivjyl+aIJDZbbEUXrmk/M0zT9rHtgSc2isM8ITH6IHw5q7lmNMPLYOu6T
+IMvfTDtADBOOTV/vF+/4NKf5GCUXVt1XTzLBFMK0p/ZoI7Fsw7fhH6FR12vk0xA4
+BEC4pwRbGfHo7P31ii0by8epkve93tF4IZuFmN92A84bN1z7Kc4TYaSbua2rgguz
+FzMyWpsTxr363HzCK1xOJb6JyJOiXbq4+j2oqtne3GIvyozJeiyKRgjLIMoe/LV7
+fPPc5wlhAoGBAOD3z0JH2eyR/1RHILFsWInH2nDbKHHuCjhFIL2XloeXsJkiJZ95
+BpdjExMZCqD44tPNRW/GgWKwoVwltm6zB0aq0aW/OfOzw6fhKt1W+go47L7Tpwap
+VQgy6BFXSueUKfQDlZEWV4E2gakf8vOl0/VRQExae/CeKf1suEedQaErAoGBAMWE
+LOmNDEU2NFqghfNBAFYyFJst3YnBmSmlL7W22+OsfSK/PhxnJbuNHxMgxpg9rieW
+tVyjuZRo/i7WLVm3uG+dK1RJ9t8Y6kpYkCRKpi9G8DBOj3PSulOybBr+fdRfW9mf
+8UmqOjOkrhxXPkchc9TY4EM7/1XeKvEidlIp0gvRAoGAAurz4zYvW2QhXaR2hhaT
+p2XSLXiKM8AUndo3rH3U0/lhrvrEZicZsMj2LF88xg20U27sIaD/eJo13Y4XqaPk
+ykPY6D9srv574SeIeMpx/8PxPiBcoDd+BNc0L1VkgVBoouORAwq5I9HjKKBjdEmI
+UDw3i0X5KYvDm6fXVAZ0HXUCgYBWc4To8KiXPqNpq2sVzrSkBaWJSmj2G7u7Q6b/
+RTs3is72v3gjHG6iiaE5URY7mnu4rjlRhAP9Vnsy6uHMrCJZEBTf/sPEYHZj9iGZ
+EOduOAF3U1tsmaaebbDtm8hdhSOBvITy9kQlSIZAt1r17Ulytz5pj0AySFzJUIkz
+a0SZkQKBgCWixtUxiK8PAdWhyS++90WJeJn8eqjuSAz+VMtFQFRRWDUbkiHvGMRu
+o/Hhk6zS46gSF2Evb1d26uUEenXnJlIp6YWzb0DLPrfy5P53kPA6YEvYq5MSAg3l
+DZOJUF+ko7cWXSZkeTIBH/jrGOdP4tTALZt6DNt+Gz7xwPO5tGgV
+-----END RSA PRIVATE KEY-----

data/spec/inputs/elasticsearch_spec.rb

@@ -8,13 +8,11 @@ require "stud/temporary"
 require "time"
 require "date"
 
-class LogStash::Inputs::TestableElasticsearch < LogStash::Inputs::Elasticsearch
-  attr_reader :client
-end
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
-describe LogStash::Inputs::TestableElasticsearch do
+describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
-  let(:plugin) { LogStash::Inputs::TestableElasticsearch.new(config) }
+  let(:plugin) { described_class.new(config) }
   let(:queue) { Queue.new }
 
   it_behaves_like "an interruptible input plugin" do
@@ -40,7 +38,13 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
   end
 
-  context 'creating events from Elasticsearch' do
+
+  ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+    before(:each) do
+      allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+    end
+
     let(:config) do
       %q[
         input {
@@ -97,7 +101,6 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       expect(event).to be_a(LogStash::Event)
-      puts event.to_hash_with_metadata
       expect(event.get("message")).to eql [ "ohayo" ]
     end
 
@@ -120,10 +123,10 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
 
         expect(event).to be_a(LogStash::Event)
-        puts event.to_hash_with_metadata
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
       end
     end
+
   end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
@@ -135,6 +138,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           'query' => "#{LogStash::Json.dump(query)}",
           'slices' => slices,
           'docinfo' => true, # include ids
+          'docinfo_target' => '[@metadata]'
       }
     end
     let(:query) do
@@ -166,7 +170,7 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
     context 'without slices directive' do
-      let(:config) { super.tap { |h| h.delete('slices') } }
+      let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
         expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
@@ -405,127 +409,140 @@ describe LogStash::Inputs::TestableElasticsearch do
       allow(client).to receive(:clear_scroll).and_return(nil)
     end
 
-    context 'when defining docinfo' do
-      let(:config_metadata) do
-        %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-              }
-            }
-        ]
-      end
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
 
-      it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        config_metadata_with_hash = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'metadata_with_hash'
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      context 'with docinfo enabled' do
+        let(:config_metadata) do
+          %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                }
               }
-            }
-        ]
-
-        event = input(config_metadata_with_hash) do |pipeline, queue|
-          queue.pop
+          ]
         end
 
-        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
-      end
-
-      context 'if the `docinfo_target` exist but is not of type hash' do
-        let (:config) { {
-            "hosts" => ["localhost"],
-            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-            "docinfo" => true,
-            "docinfo_target" => 'metadata_with_string'
-        } }
-        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-          expect(client).not_to receive(:clear_scroll)
-          plugin.register
-          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+        it "provides document info under metadata" do
+          event = input(config_metadata) do |pipeline, queue|
+            queue.pop
+          end
+
+          if ecs_select.active_mode == :disabled
+            expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][_type]")).to eq('logs')
+            expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          else
+            expect(event.get("[@metadata][input][elasticsearch][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][input][elasticsearch][_type]")).to eq('logs')
+            expect(event.get("[@metadata][input][elasticsearch][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          end
         end
-      end
 
-      it "should move the document info to the @metadata field" do
-        event = input(config_metadata) do |pipeline, queue|
-          queue.pop
+        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+          config_metadata_with_hash = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'metadata_with_hash'
+                }
+              }
+          ]
+
+          event = input(config_metadata_with_hash) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_type]")).to eq('logs')
-        expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        context 'if the `docinfo_target` exist but is not of type hash' do
+          let (:config) { {
+              "hosts" => ["localhost"],
+              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+              "docinfo" => true,
+              "docinfo_target" => 'metadata_with_string'
+          } }
+          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+            expect(client).not_to receive(:clear_scroll)
+            plugin.register
+            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+          end
+        end
 
-      it 'should move the document information to the specified field' do
-        config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'meta'
+        it 'should move the document information to the specified field' do
+          config = %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'meta'
+                }
               }
-            }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+          ]
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[meta][_type]")).to eq('logs')
+          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[meta][_type]")).to eq('logs')
-        expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        it "allows to specify which fields from the document info to save to metadata" do
+          fields = ["_index"]
+          config = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_fields => #{fields}
+                }
+              }]
+
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+          expect(meta_base.keys).to eq(fields)
+        end
 
-      it "should allow to specify which fields from the document info to save to the @metadata field" do
-        fields = ["_index"]
-        config = %Q[
+        it 'should be able to reference metadata fields in `add_field` decorations' do
+          config = %q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_fields => #{fields}
-              }
-            }]
-
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
-
-        expect(event.get("@metadata").keys).to eq(fields)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_id]")).to eq(nil)
-      end
-
-      it 'should be able to reference metadata fields in `add_field` decorations' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              docinfo => true
-              add_field => {
-                'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                add_field => {
+                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                }
               }
             }
-          }
-        ]
+          ]
 
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+        end if ecs_select.active_mode == :disabled
 
-        expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
       end
+
     end
 
     context "when not defining the docinfo" do
@@ -542,9 +559,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           queue.pop
         end
 
-        expect(event.get("[@metadata][_index]")).to eq(nil)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_id]")).to eq(nil)
+        expect(event.get("[@metadata]")).to be_empty
       end
     end
   end
@@ -563,22 +578,22 @@ describe LogStash::Inputs::TestableElasticsearch do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
       end
 
-      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+      let(:config) { super().merge({ 'cloud_id' => valid_cloud_id }) }
 
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        expect( client.transport.hosts ).to eql [{
-                                                     :scheme => "https",
-                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
-                                                     :port => 9243,
-                                                     :path => "",
-                                                     :protocol => "https"
-                                                 }]
+        expect( extract_transport(client).hosts ).to eql [{
+                                                              :scheme => "https",
+                                                              :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                              :port => 9243,
+                                                              :path => "",
+                                                              :protocol => "https"
+                                                          }]
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+        let(:config) { super().merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
@@ -586,7 +601,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context 'hosts also set' do
-        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+        let(:config) { super().merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
@@ -595,18 +610,18 @@ describe LogStash::Inputs::TestableElasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "cloud.auth" do
-      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+      let(:config) { super().merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
 
       it "should set authorization" do
         plugin.register
         client = plugin.send(:client)
-        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'invalid-format' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
@@ -614,7 +629,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context 'user also set' do
-        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -624,7 +639,7 @@ describe LogStash::Inputs::TestableElasticsearch do
 
     describe "api_key" do
       context "without ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
@@ -632,18 +647,18 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context "with ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
 
         it "should set authorization" do
           plugin.register
           client = plugin.send(:client)
-          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
           expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
         end
 
         context 'user also set' do
-          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+          let(:config) { super().merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
 
           it "should fail" do
             expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -653,24 +668,24 @@ describe LogStash::Inputs::TestableElasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "proxy" do
-      let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+      let(:config) { super().merge({ 'proxy' => 'http://localhost:1234' }) }
 
       it "should set proxy" do
         plugin.register
         client = plugin.send(:client)
-        proxy = client.transport.options[:transport_options][:proxy]
+        proxy = extract_transport(client).options[:transport_options][:proxy]
 
         expect( proxy ).to eql "http://localhost:1234"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+        let(:config) { super().merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
 
         it "should not set proxy" do
           plugin.register
           client = plugin.send(:client)
 
-          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
         end
       end
     end
@@ -756,4 +771,10 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
   end
+
+  # @note can be removed once we depends on elasticsearch gem >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
+  end
+
 end