logstash-input-elasticsearch 4.20.5 → 4.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eaa000218cac721012284803638cb681c0eccf02107c34b03e597998a2e9eee3
-  data.tar.gz: eef84d9f24f1346b1d4793f6b10f200e790739f63d588888bae1bcdae33ec51f
+  metadata.gz: ba4a906467e97c729acee2b28f29350b27ed838bc9d3e8ea0b7cd4a83b4dd06e
+  data.tar.gz: 2b0c263a32bbcaa0e2d88bb2a070072e95c1a80f6d9794ee267c703e34bed3af
 SHA512:
-  metadata.gz: a9ed0836e937e4ceb9726d05be77e39a10cc11189bbf0cffb829e4445f5e6948e339187c2f088d1d9408657c26318375756d974299667fdf86d62b77ba6f00d5
-  data.tar.gz: 3e919b494b24d4d61a98222e27310a6d0e2f9d1d2b6209b2879c80199dd4c69e39b789a2d6e7503e07fe3097d50261f78d0c4ab7300bd48d6cf79a5d28c66c01
+  metadata.gz: ff1d841fbd5cbe0469a704131bd55b206c418243e87b3afac4e4d0d3ef26af6e926148323ec6c939fd48ea53bf4e0d4fcdb09ea1015d2d815863db917ff322c5
+  data.tar.gz: 66fbdcc42a51b8859be13df832d7d5e5c2a71e53c8995c490d64316590f74d277d6ed73d8b366a580fe5dfb085205999d0488368e1c62c37dc83ad3a74e08fad

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.21.1
+  - Fix: prevent plugin crash when hits contain illegal structure [#183](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183)
+    - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
+
+## 4.21.0
+  - Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217)
+
 ## 4.20.5
   - Add `x-elastic-product-origin` header to Elasticsearch requests [#211](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211)
 

data/docs/index.asciidoc

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
-You can periodically schedule ingestion using a cron syntax 
+You can periodically schedule ingestion using a cron syntax
 (see `schedule` setting) or run the query one time to load
 data into Logstash.
 
@@ -93,6 +93,16 @@ The plugin logs a warning when ECS is enabled and `target` isn't set.
 
 TIP: Set the `target` option to avoid potential schema conflicts.
 
+[id="plugins-{type}s-{plugin}-failure-handling"]
+==== Failure handling
+
+When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit.
+
+Common causes are:
+
+ - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace.
+ - When <<plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<plugins-{type}s-{plugin}-target>> and <<plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 
@@ -106,6 +116,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -199,8 +210,18 @@ For more info, check out the
 The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
 Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
+[id="plugins-{type}s-{plugin}-custom_headers"]
+===== `custom_headers`
+
+  * Value type is <<hash,hash>>
+  * Default value is empty
+
+Pass a set of key value pairs as the headers sent in each request to an elasticsearch node.
+The headers will be used for any kind of request.
+These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
-===== `docinfo` 
+===== `docinfo`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -251,7 +272,7 @@ Example
 
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
-===== `docinfo_fields` 
+===== `docinfo_fields`
 
   * Value type is <<array,array>>
   * Default value is `["_index", "_type", "_id"]`
@@ -262,7 +283,7 @@ option lists the metadata fields to save in the current event. See
 more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
-===== `docinfo_target` 
+===== `docinfo_target`
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
@@ -286,7 +307,7 @@ this option names the field under which to store the metadata fields as subfield
 Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
-===== `hosts` 
+===== `hosts`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -296,18 +317,18 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
 9200.
 
 [id="plugins-{type}s-{plugin}-index"]
-===== `index` 
+===== `index`
 
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. 
+The index or alias to search.
 Check out {ref}/api-conventions.html#api-multi-index[Multi Indices
 documentation] in the Elasticsearch documentation for info on
 referencing multiple indices.
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * Value type is <<password,password>>
   * There is no default value for this setting.
@@ -327,7 +348,7 @@ An empty string is treated as if proxy was not set, this is useful when using
 environment variables e.g. `proxy => '${LS_PROXY:}'`.
 
 [id="plugins-{type}s-{plugin}-query"]
-===== `query` 
+===== `query`
 
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
@@ -375,7 +396,7 @@ The default is 0 (no retry). This value should be equal to or greater than zero.
 NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
-===== `schedule` 
+===== `schedule`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -387,7 +408,7 @@ There is no schedule by default. If no schedule is given, then the statement is
 exactly once.
 
 [id="plugins-{type}s-{plugin}-scroll"]
-===== `scroll` 
+===== `scroll`
 
   * Value type is <<string,string>>
   * Default value is `"1m"`
@@ -410,7 +431,7 @@ The query requires at least one `sort` field, as described in the <<plugins-{typ
 `scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended.
 
 [id="plugins-{type}s-{plugin}-size"]
-===== `size` 
+===== `size`
 
   * Value type is <<number,number>>
   * Default value is `1000`
@@ -598,7 +619,7 @@ It is also possible to target an entry in the event's metadata, which will be av
 
 
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.

data/lib/logstash/inputs/elasticsearch.rb

@@ -24,9 +24,9 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
 # called `http.content_type.required`. If this option is set to `true`, and you
 # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch input
 # plugin to version 4.0.2 or higher.
-# 
+#
 # ================================================================================
-# 
+#
 # Read from an Elasticsearch cluster, based on search query results.
 # This is useful for replaying test logs, reindexing, etc.
 # It also supports periodically scheduling lookup enrichments
@@ -166,6 +166,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html
   config :docinfo_fields, :validate => :array, :default => ['_index', '_type', '_id']
 
+  # Custom headers for Elasticsearch requests
+  config :custom_headers, :validate => :hash, :default => {}
+
   # Basic Auth - username
   config :user, :validate => :string
 
@@ -305,6 +308,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
     transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
+    transport_options[:headers].merge!(@custom_headers) unless @custom_headers.empty?
     transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
     transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
@@ -349,21 +353,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This can be called externally from the query_executor
   public
   def push_hit(hit, output_queue, root_field = '_source')
-    event = targeted_event_factory.new_event hit[root_field]
-    set_docinfo_fields(hit, event) if @docinfo
+    event = event_from_hit(hit, root_field)
     decorate(event)
     output_queue << event
   end
 
+  def event_from_hit(hit, root_field)
+    event = targeted_event_factory.new_event hit[root_field]
+    set_docinfo_fields(hit, event) if @docinfo
+
+    event
+  rescue => e
+    serialized_hit = hit.to_json
+    logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit)
+    return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
+  end
+
   def set_docinfo_fields(hit, event)
     # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
     docinfo_target = event.get(@docinfo_target) || {}
 
     unless docinfo_target.is_a?(Hash)
-      @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
-
-      # TODO: (colin) I am not sure raising is a good strategy here?
-      raise Exception.new("Elasticsearch input: incompatible event")
+      # expect error to be handled by `#event_from_hit`
+      fail RuntimeError, "Incompatible event; unable to merge docinfo fields into docinfo_target=`#{@docinfo_target}`"
     end
 
     @docinfo_fields.each do |field|

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.20.5'
+  s.version         = '4.21.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/test_certs/GENERATED_AT

@@ -0,0 +1 @@
+2024-12-26T22:27:15+00:00

data/spec/fixtures/test_certs/ca.crt

@@ -1,20 +1,19 @@
 -----BEGIN CERTIFICATE-----
-MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
-VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
-8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
-PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
-OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
-oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
-YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
-AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
-MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
-hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
-G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
-DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
-sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
-NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
-emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
+MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
+aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
+AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
+lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
+p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
+E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
+IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
+T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.der.sha256

@@ -1 +1 @@
-195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283
+b1e955819b0d14f64f863adb103c248ddacf2e17bea48d04ee4b57c64814ccc4

data/spec/fixtures/test_certs/es.chain.crt

@@ -0,0 +1,38 @@
+-----BEGIN CERTIFICATE-----
+MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
+V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
+36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
+MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
+/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
+hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
+BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
+K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
+CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
+T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
+byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
+UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
+avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
+aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
+AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
+lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
+p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
+E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
+IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
+T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/es.crt

@@ -1,20 +1,19 @@
 -----BEGIN CERTIFICATE-----
-MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
-VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
-1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
-Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
-CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
-2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
-LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
-tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
-MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
-YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
-7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
-f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
-Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
-LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
-qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
-qi02i4q6meHGcw==
+MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
+dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
+MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
+V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
+36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
+MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
+/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
+hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
+BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
+K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
+wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
+CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
+T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
+byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
+UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
+avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
 -----END CERTIFICATE-----

data/spec/fixtures/test_certs/renew.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+cd "$(dirname "$0")"
+
+openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
+openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
+
+openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
+openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
+cat es.crt ca.crt > es.chain.crt
+
+# output ISO8601 timestamp to file
+date -Iseconds > GENERATED_AT

data/spec/inputs/elasticsearch_spec.rb

@@ -103,6 +103,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
         end
       end
+
+      context "with custom headers" do
+        let(:config) do
+          {
+            "schedule" => "* * * * * UTC",
+            "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
+          }
+        end
+
+
+        it "sets custom headers" do
+          plugin.register
+          client = plugin.send(:client)
+          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including(config["custom_headers"])
+        end
+      end
     end
 
     context "retry" do
@@ -637,11 +653,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         context 'if the `docinfo_target` exist but is not of type hash' do
           let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
           let(:do_register) { false }
+          let(:mock_queue) { double('Queue', :<< => nil) }
+          let(:hit) { response.dig('hits', 'hits').first }
+
+          it 'emits a tagged event with JSON-serialized event in [event][original]' do
+            allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
 
-          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
-            expect(client).not_to receive(:clear_scroll)
             plugin.register
-            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+            plugin.run(mock_queue)
+
+            expect(mock_queue).to have_received(:<<) do |event|
+              expect(event).to be_a_kind_of LogStash::Event
+
+              expect(event.get('tags')).to include("_elasticsearch_input_failure")
+              expect(event.get('[event][original]')).to be_a_kind_of String
+              expect(JSON.load(event.get('[event][original]'))).to eq hit
+            end
+
+            expect(plugin.logger)
+              .to have_received(:warn).with(
+                a_string_including("Event creation error, original data now in [event][original] field"),
+                a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
+                                 :data    => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
           end
 
         end
@@ -1219,6 +1252,88 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
   end
 
+  context '#push_hit' do
+    let(:config) do
+      {
+        'docinfo' => true, # include ids
+        'docinfo_target' => '[@metadata][docinfo]'
+      }
+    end
+
+    let(:hit) do
+      JSON.load(<<~EOJSON)
+        {
+          "_index" : "test_bulk_index_2",
+          "_type" : "_doc",
+          "_id" : "sHe6A3wBesqF7ydicQvG",
+          "_score" : 1.0,
+          "_source" : {
+            "@timestamp" : "2021-09-20T15:02:02.557Z",
+            "message" : "ping",
+            "@version" : "17",
+            "sequence" : 7,
+            "host" : {
+              "name" : "maybe.local",
+              "ip" : "127.0.0.1"
+            }
+          }
+        }
+      EOJSON
+    end
+
+    let(:mock_queue) { double('queue', :<< => nil) }
+
+    it 'pushes a generated event to the queue' do
+      plugin.send(:push_hit, hit, mock_queue)
+      expect(mock_queue).to have_received(:<<) do |event|
+        expect(event).to be_a_kind_of LogStash::Event
+
+        # fields overriding defaults
+        expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
+        expect(event.get('@version')).to eq("17")
+
+        # structure from hit's _source
+        expect(event.get('message')).to eq("ping")
+        expect(event.get('sequence')).to eq(7)
+        expect(event.get('[host][name]')).to eq("maybe.local")
+        expect(event.get('[host][ip]')).to eq("127.0.0.1")
+
+        # docinfo fields
+        expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
+        expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
+        expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
+      end
+    end
+
+    context 'when event creation fails' do
+      before(:each) do
+        allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
+
+        allow(plugin.event_factory).to receive(:new_event).and_call_original
+        allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
+      end
+
+      it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
+        plugin.send(:push_hit, hit, mock_queue)
+
+        expect(mock_queue).to have_received(:<<) do |event|
+          expect(event).to be_a_kind_of LogStash::Event
+
+          expect(event.get('tags')).to include("_elasticsearch_input_failure")
+          expect(event.get('[event][original]')).to be_a_kind_of String
+          expect(JSON.load(event.get('[event][original]'))).to eq hit
+        end
+
+        expect(plugin.logger)
+          .to have_received(:warn).with(
+            a_string_including("Event creation error, original data now in [event][original] field"),
+            a_hash_including(:message => a_string_including('intentional'),
+                             :data    => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
+
+      end
+    end
+  end
+
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch, :integration => true do
+describe LogStash::Inputs::Elasticsearch do
 
   SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.20.5
+  version: 4.21.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-05 00:00:00.000000000 Z
+date: 2025-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -277,11 +277,14 @@ files:
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
+- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
+- spec/fixtures/test_certs/renew.sh
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
@@ -313,11 +316,14 @@ specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
+- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
+- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
+- spec/fixtures/test_certs/renew.sh
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb