logstash-input-elasticsearch 4.21.0 → 4.21.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf89e42e7146cb59e938fed4a2a2d352bc72bf5c73c59d2a674be724c686a543
-  data.tar.gz: 9b0b0b80d4a3754dac6e5d4f2c179dbd1be2ee6d6ce6c2b1fc619c2369a808aa
+  metadata.gz: 70af2192f555f8afff4ef2f96072f2b215a2039207dfa12a9449f507f7b13f7b
+  data.tar.gz: 73621246eccfd1fbb385be5e9ca5ef9a071cdb64008cb539a1e80a08c7a0ed34
 SHA512:
-  metadata.gz: 87223b2b656aee8d27c9f0ea92151370f3eb46b1851040054f0fa74e99ab4882278b28c85d4f8290d211ef793ab3eab9141b9c3ab343bd80cc5b9b9e7f0ceb11
-  data.tar.gz: 10db333c4a67f69fb695bede43feb8f89692609afd74c40de3babf9b56580c18ea1d839865c66e9ec5ade759825309aecd84a2d269805583e8053b48bb563af9
+  metadata.gz: bbc5c842d77204339e0bb64174f98ffb8bb1728957a1f64d1f83e1f5bad27ad76fc24f44b23a64d23247b26a806cfee7cbd52a16ea34e5490f1355bcdbb98303
+  data.tar.gz: 7b258f80ca64e5dd16593a65d7326a5f3695f840cbf32fdeac9363a6a19d4747de9135065a7b940602cd77f43a02910b74d667761184ccb846a864e128334a20

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.21.2
+  - Add elastic-transport client support used in elasticsearch-ruby 8.x [#225](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/225)
+
+## 4.21.1
+  - Fix: prevent plugin crash when hits contain illegal structure [#183](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183)
+    - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
+
 ## 4.21.0
   - Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217)
 

data/docs/index.asciidoc

@@ -93,6 +93,16 @@ The plugin logs a warning when ECS is enabled and `target` isn't set.
 
 TIP: Set the `target` option to avoid potential schema conflicts.
 
+[id="plugins-{type}s-{plugin}-failure-handling"]
+==== Failure handling
+
+When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit.
+
+Common causes are:
+
+ - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace.
+ - When <<plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<plugins-{type}s-{plugin}-target>> and <<plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -13,9 +13,7 @@ require "logstash/plugin_mixins/normalize_config_support"
 require "base64"
 
 require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
-require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
-require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
+require "manticore"
 
 # .Compatibility Note
 # [NOTE]
@@ -323,7 +321,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     @client_options = {
       :hosts => hosts,
       :transport_options => transport_options,
-      :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+      :transport_class => get_transport_client_class,
       :ssl => ssl_options
     }
 
@@ -353,21 +351,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This can be called externally from the query_executor
   public
   def push_hit(hit, output_queue, root_field = '_source')
-    event = targeted_event_factory.new_event hit[root_field]
-    set_docinfo_fields(hit, event) if @docinfo
+    event = event_from_hit(hit, root_field)
     decorate(event)
     output_queue << event
   end
 
+  def event_from_hit(hit, root_field)
+    event = targeted_event_factory.new_event hit[root_field]
+    set_docinfo_fields(hit, event) if @docinfo
+
+    event
+  rescue => e
+    serialized_hit = hit.to_json
+    logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit)
+    return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
+  end
+
   def set_docinfo_fields(hit, event)
     # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
     docinfo_target = event.get(@docinfo_target) || {}
 
     unless docinfo_target.is_a?(Hash)
-      @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
-
-      # TODO: (colin) I am not sure raising is a good strategy here?
-      raise Exception.new("Elasticsearch input: incompatible event")
+      # expect error to be handled by `#event_from_hit`
+      fail RuntimeError, "Incompatible event; unable to merge docinfo fields into docinfo_target=`#{@docinfo_target}`"
     end
 
     @docinfo_fields.each do |field|
@@ -670,6 +676,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
                       end
   end
 
+  def get_transport_client_class
+    # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
+    # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
+    # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
+    # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
+    require "elasticsearch/transport/transport/http/manticore"
+    require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
+    require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
+    ::Elasticsearch::Transport::Transport::HTTP::Manticore
+  rescue ::LoadError
+    require "elastic/transport/transport/http/manticore"
+    ::Elastic::Transport::Transport::HTTP::Manticore
+  end
+
   module URIOrEmptyValidator
     ##
     # @override to provide :uri_or_empty validator

data/logstash-input-elasticsearch.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.21.0'
+  s.version         = '4.21.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.homepage        = "https://elastic.co/logstash"
   s.require_paths = ["lib"]
 
   # Files
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 7.17.9'
+  s.add_runtime_dependency 'elasticsearch', '>= 7.17.9', '< 9'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
 

data/spec/fixtures/test_certs/GENERATED_AT

@@ -0,0 +1 @@
+2024-12-26T22:27:15+00:00

data/spec/fixtures/test_certs/ca.crt

@@ -1,20 +1,19 @@
 -----BEGIN CERTIFICATE-----
-MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
-VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
-8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
-PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
-OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
-oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
-YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
-AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
-MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
-hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
-G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
-DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
-sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
-NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
-emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
+MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
+aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
+AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
+lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
+p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
+E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
+IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
+T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.der.sha256

@@ -1 +1 @@
-195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283
+b1e955819b0d14f64f863adb103c248ddacf2e17bea48d04ee4b57c64814ccc4

data/spec/fixtures/test_certs/es.chain.crt

@@ -0,0 +1,38 @@
+-----BEGIN CERTIFICATE-----
+MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
+V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
+36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
+MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
+/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
+hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
+BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
+K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
+CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
+T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
+byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
+UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
+avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
+aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
+AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
+lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
+p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
+E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
+IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
+T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/es.crt

@@ -1,20 +1,19 @@
 -----BEGIN CERTIFICATE-----
-MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
-VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
-1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
-Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
-CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
-2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
-LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
-tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
-MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
-YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
-7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
-f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
-Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
-LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
-qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
-qi02i4q6meHGcw==
+MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
+V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
+36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
+MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
+/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
+hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
+BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
+K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
+CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
+T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
+byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
+UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
+avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/renew.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+cd "$(dirname "$0")"
+
+openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
+openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
+
+openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
+cat es.crt ca.crt > es.chain.crt
+
+# output ISO8601 timestamp to file
+date -Iseconds > GENERATED_AT

data/spec/inputs/elasticsearch_spec.rb

@@ -21,6 +21,13 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:es_version) { "7.5.0" }
   let(:cluster_info) { {"version" => {"number" => es_version, "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
 
+  def elastic_ruby_v8_client_available?
+    Elasticsearch::Transport
+    false
+  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
+    true
+  end
+
   before(:each) do
     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
     allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
@@ -79,9 +86,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         before do
           allow(Elasticsearch::Client).to receive(:new).and_return(es_client)
-          allow(es_client).to receive(:info).and_raise(
-            Elasticsearch::Transport::Transport::Errors::BadRequest.new
-          )
+          if elastic_ruby_v8_client_available?
+            allow(es_client).to receive(:info).and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
+          else
+            allow(es_client).to receive(:info).and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
+          end
         end
 
         it "raises an exception" do
@@ -653,11 +662,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         context 'if the `docinfo_target` exist but is not of type hash' do
           let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
           let(:do_register) { false }
+          let(:mock_queue) { double('Queue', :<< => nil) }
+          let(:hit) { response.dig('hits', 'hits').first }
+
+          it 'emits a tagged event with JSON-serialized event in [event][original]' do
+            allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
 
-          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
-            expect(client).not_to receive(:clear_scroll)
             plugin.register
-            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+            plugin.run(mock_queue)
+
+            expect(mock_queue).to have_received(:<<) do |event|
+              expect(event).to be_a_kind_of LogStash::Event
+
+              expect(event.get('tags')).to include("_elasticsearch_input_failure")
+              expect(event.get('[event][original]')).to be_a_kind_of String
+              expect(JSON.load(event.get('[event][original]'))).to eq hit
+            end
+
+            expect(plugin.logger)
+              .to have_received(:warn).with(
+                a_string_including("Event creation error, original data now in [event][original] field"),
+                a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
+                                 :data    => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
           end
 
         end
@@ -714,8 +740,13 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-
-        expect( client.transport.instance_variable_get(:@seeds) ).to eql [{
+        target_field = :@seeds
+        begin
+          Elasticsearch::Transport::Client
+        rescue
+          target_field = :@hosts
+        end
+        expect( client.transport.instance_variable_get(target_field) ).to eql [{
                                                                               :scheme => "https",
                                                                               :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                                               :port => 9243,
@@ -1235,6 +1266,88 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
   end
 
+  context '#push_hit' do
+    let(:config) do
+      {
+        'docinfo' => true, # include ids
+        'docinfo_target' => '[@metadata][docinfo]'
+      }
+    end
+
+    let(:hit) do
+      JSON.load(<<~EOJSON)
+        {
+          "_index" : "test_bulk_index_2",
+          "_type" : "_doc",
+          "_id" : "sHe6A3wBesqF7ydicQvG",
+          "_score" : 1.0,
+          "_source" : {
+            "@timestamp" : "2021-09-20T15:02:02.557Z",
+            "message" : "ping",
+            "@version" : "17",
+            "sequence" : 7,
+            "host" : {
+              "name" : "maybe.local",
+              "ip" : "127.0.0.1"
+            }
+          }
+        }
+      EOJSON
+    end
+
+    let(:mock_queue) { double('queue', :<< => nil) }
+
+    it 'pushes a generated event to the queue' do
+      plugin.send(:push_hit, hit, mock_queue)
+      expect(mock_queue).to have_received(:<<) do |event|
+        expect(event).to be_a_kind_of LogStash::Event
+
+        # fields overriding defaults
+        expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
+        expect(event.get('@version')).to eq("17")
+
+        # structure from hit's _source
+        expect(event.get('message')).to eq("ping")
+        expect(event.get('sequence')).to eq(7)
+        expect(event.get('[host][name]')).to eq("maybe.local")
+        expect(event.get('[host][ip]')).to eq("127.0.0.1")
+
+        # docinfo fields
+        expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
+        expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
+        expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
+      end
+    end
+
+    context 'when event creation fails' do
+      before(:each) do
+        allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
+
+        allow(plugin.event_factory).to receive(:new_event).and_call_original
+        allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
+      end
+
+      it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
+        plugin.send(:push_hit, hit, mock_queue)
+
+        expect(mock_queue).to have_received(:<<) do |event|
+          expect(event).to be_a_kind_of LogStash::Event
+
+          expect(event.get('tags')).to include("_elasticsearch_input_failure")
+          expect(event.get('[event][original]')).to be_a_kind_of String
+          expect(JSON.load(event.get('[event][original]'))).to eq hit
+        end
+
+        expect(plugin.logger)
+          .to have_received(:warn).with(
+            a_string_including("Event creation error, original data now in [event][original] field"),
+            a_hash_including(:message => a_string_including('intentional'),
+                             :data    => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
+
+      end
+    end
+  end
+
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch, :integration => true do
+describe LogStash::Inputs::Elasticsearch do
 
   SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.21.0
+  version: 4.21.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-18 00:00:00.000000000 Z
+date: 2025-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +92,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -100,6 +103,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -277,16 +283,19 @@ files:
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
+- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
+- spec/fixtures/test_certs/renew.sh
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb
-homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+homepage: https://elastic.co/logstash
 licenses:
 - Apache License (2.0)
 metadata:
@@ -313,11 +322,14 @@ specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
+- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
+- spec/fixtures/test_certs/renew.sh
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb