logstash-input-elasticsearch 4.21.0 → 4.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bf89e42e7146cb59e938fed4a2a2d352bc72bf5c73c59d2a674be724c686a543
4
- data.tar.gz: 9b0b0b80d4a3754dac6e5d4f2c179dbd1be2ee6d6ce6c2b1fc619c2369a808aa
3
+ metadata.gz: ba4a906467e97c729acee2b28f29350b27ed838bc9d3e8ea0b7cd4a83b4dd06e
4
+ data.tar.gz: 2b0c263a32bbcaa0e2d88bb2a070072e95c1a80f6d9794ee267c703e34bed3af
5
5
  SHA512:
6
- metadata.gz: 87223b2b656aee8d27c9f0ea92151370f3eb46b1851040054f0fa74e99ab4882278b28c85d4f8290d211ef793ab3eab9141b9c3ab343bd80cc5b9b9e7f0ceb11
7
- data.tar.gz: 10db333c4a67f69fb695bede43feb8f89692609afd74c40de3babf9b56580c18ea1d839865c66e9ec5ade759825309aecd84a2d269805583e8053b48bb563af9
6
+ metadata.gz: ff1d841fbd5cbe0469a704131bd55b206c418243e87b3afac4e4d0d3ef26af6e926148323ec6c939fd48ea53bf4e0d4fcdb09ea1015d2d815863db917ff322c5
7
+ data.tar.gz: 66fbdcc42a51b8859be13df832d7d5e5c2a71e53c8995c490d64316590f74d277d6ed73d8b366a580fe5dfb085205999d0488368e1c62c37dc83ad3a74e08fad
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 4.21.1
2
+ - Fix: prevent plugin crash when hits contain illegal structure [#183](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183)
3
+ - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
4
+
1
5
  ## 4.21.0
2
6
  - Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217)
3
7
 
data/docs/index.asciidoc CHANGED
@@ -93,6 +93,16 @@ The plugin logs a warning when ECS is enabled and `target` isn't set.
93
93
 
94
94
  TIP: Set the `target` option to avoid potential schema conflicts.
95
95
 
96
+ [id="plugins-{type}s-{plugin}-failure-handling"]
97
+ ==== Failure handling
98
+
99
+ When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit.
100
+
101
+ Common causes are:
102
+
103
+ - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace.
104
+ - When <<plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<plugins-{type}s-{plugin}-target>> and <<plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict.
105
+
96
106
  [id="plugins-{type}s-{plugin}-options"]
97
107
  ==== Elasticsearch Input configuration options
98
108
 
@@ -353,21 +353,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
353
353
  # This can be called externally from the query_executor
354
354
  public
355
355
  def push_hit(hit, output_queue, root_field = '_source')
356
- event = targeted_event_factory.new_event hit[root_field]
357
- set_docinfo_fields(hit, event) if @docinfo
356
+ event = event_from_hit(hit, root_field)
358
357
  decorate(event)
359
358
  output_queue << event
360
359
  end
361
360
 
361
+ def event_from_hit(hit, root_field)
362
+ event = targeted_event_factory.new_event hit[root_field]
363
+ set_docinfo_fields(hit, event) if @docinfo
364
+
365
+ event
366
+ rescue => e
367
+ serialized_hit = hit.to_json
368
+ logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit)
369
+ return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
370
+ end
371
+
362
372
  def set_docinfo_fields(hit, event)
363
373
  # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
364
374
  docinfo_target = event.get(@docinfo_target) || {}
365
375
 
366
376
  unless docinfo_target.is_a?(Hash)
367
- @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
368
-
369
- # TODO: (colin) I am not sure raising is a good strategy here?
370
- raise Exception.new("Elasticsearch input: incompatible event")
377
+ # expect error to be handled by `#event_from_hit`
378
+ fail RuntimeError, "Incompatible event; unable to merge docinfo fields into docinfo_target=`#{@docinfo_target}`"
371
379
  end
372
380
 
373
381
  @docinfo_fields.each do |field|
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-elasticsearch'
4
- s.version = '4.21.0'
4
+ s.version = '4.21.1'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Reads query results from an Elasticsearch cluster"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -0,0 +1 @@
1
+ 2024-12-26T22:27:15+00:00
@@ -1,20 +1,19 @@
1
1
  -----BEGIN CERTIFICATE-----
2
- MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
3
- BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
4
- cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
5
- VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
6
- ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
7
- 8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
8
- PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
9
- OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
10
- oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
11
- YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
12
- AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
13
- MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
14
- hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
15
- G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
16
- DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
17
- sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
18
- NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
19
- emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
2
+ MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
3
+ dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
4
+ MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
5
+ aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
6
+ AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
7
+ QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
8
+ 90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
9
+ 99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
10
+ i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
11
+ m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
12
+ BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
13
+ AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
14
+ lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
15
+ p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
16
+ E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
17
+ IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
18
+ T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
20
19
  -----END CERTIFICATE-----
@@ -1 +1 @@
1
- 195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283
1
+ b1e955819b0d14f64f863adb103c248ddacf2e17bea48d04ee4b57c64814ccc4
@@ -0,0 +1,38 @@
1
+ -----BEGIN CERTIFICATE-----
2
+ MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
3
+ dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
4
+ MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
5
+ 9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
6
+ V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
7
+ 36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
8
+ MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
9
+ /z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
10
+ hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
11
+ BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
12
+ K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
13
+ wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
14
+ CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
15
+ T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
16
+ byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
17
+ UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
18
+ avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
19
+ -----END CERTIFICATE-----
20
+ -----BEGIN CERTIFICATE-----
21
+ MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
22
+ dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
23
+ MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
24
+ aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
25
+ AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
26
+ QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
27
+ 90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
28
+ 99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
29
+ i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
30
+ m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
31
+ BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
32
+ AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
33
+ lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
34
+ p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
35
+ E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
36
+ IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
37
+ T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
38
+ -----END CERTIFICATE-----
@@ -1,20 +1,19 @@
1
1
  -----BEGIN CERTIFICATE-----
2
- MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
3
- BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
4
- cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
5
- VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
6
- 1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
7
- Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
8
- CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
9
- 2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
10
- LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
11
- tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
12
- MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
13
- YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
14
- 7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
15
- f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
16
- Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
17
- LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
18
- qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
19
- qi02i4q6meHGcw==
2
+ MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
3
+ dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
4
+ MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
5
+ 9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
6
+ V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
7
+ 36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
8
+ MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
9
+ /z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
10
+ hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
11
+ BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
12
+ K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
13
+ wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
14
+ CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
15
+ T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
16
+ byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
17
+ UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
18
+ avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
20
19
  -----END CERTIFICATE-----
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+ cd "$(dirname "$0")"
5
+
6
+ openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
7
+ openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
8
+ openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
9
+
10
+ openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
11
+ openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
12
+ cat es.crt ca.crt > es.chain.crt
13
+
14
+ # output ISO8601 timestamp to file
15
+ date -Iseconds > GENERATED_AT
@@ -653,11 +653,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
653
653
  context 'if the `docinfo_target` exist but is not of type hash' do
654
654
  let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
655
655
  let(:do_register) { false }
656
+ let(:mock_queue) { double('Queue', :<< => nil) }
657
+ let(:hit) { response.dig('hits', 'hits').first }
658
+
659
+ it 'emits a tagged event with JSON-serialized event in [event][original]' do
660
+ allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
656
661
 
657
- it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
658
- expect(client).not_to receive(:clear_scroll)
659
662
  plugin.register
660
- expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
663
+ plugin.run(mock_queue)
664
+
665
+ expect(mock_queue).to have_received(:<<) do |event|
666
+ expect(event).to be_a_kind_of LogStash::Event
667
+
668
+ expect(event.get('tags')).to include("_elasticsearch_input_failure")
669
+ expect(event.get('[event][original]')).to be_a_kind_of String
670
+ expect(JSON.load(event.get('[event][original]'))).to eq hit
671
+ end
672
+
673
+ expect(plugin.logger)
674
+ .to have_received(:warn).with(
675
+ a_string_including("Event creation error, original data now in [event][original] field"),
676
+ a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
677
+ :data => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
661
678
  end
662
679
 
663
680
  end
@@ -1235,6 +1252,88 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
1235
1252
  end
1236
1253
  end
1237
1254
 
1255
+ context '#push_hit' do
1256
+ let(:config) do
1257
+ {
1258
+ 'docinfo' => true, # include ids
1259
+ 'docinfo_target' => '[@metadata][docinfo]'
1260
+ }
1261
+ end
1262
+
1263
+ let(:hit) do
1264
+ JSON.load(<<~EOJSON)
1265
+ {
1266
+ "_index" : "test_bulk_index_2",
1267
+ "_type" : "_doc",
1268
+ "_id" : "sHe6A3wBesqF7ydicQvG",
1269
+ "_score" : 1.0,
1270
+ "_source" : {
1271
+ "@timestamp" : "2021-09-20T15:02:02.557Z",
1272
+ "message" : "ping",
1273
+ "@version" : "17",
1274
+ "sequence" : 7,
1275
+ "host" : {
1276
+ "name" : "maybe.local",
1277
+ "ip" : "127.0.0.1"
1278
+ }
1279
+ }
1280
+ }
1281
+ EOJSON
1282
+ end
1283
+
1284
+ let(:mock_queue) { double('queue', :<< => nil) }
1285
+
1286
+ it 'pushes a generated event to the queue' do
1287
+ plugin.send(:push_hit, hit, mock_queue)
1288
+ expect(mock_queue).to have_received(:<<) do |event|
1289
+ expect(event).to be_a_kind_of LogStash::Event
1290
+
1291
+ # fields overriding defaults
1292
+ expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
1293
+ expect(event.get('@version')).to eq("17")
1294
+
1295
+ # structure from hit's _source
1296
+ expect(event.get('message')).to eq("ping")
1297
+ expect(event.get('sequence')).to eq(7)
1298
+ expect(event.get('[host][name]')).to eq("maybe.local")
1299
+ expect(event.get('[host][ip]')).to eq("127.0.0.1")
1300
+
1301
+ # docinfo fields
1302
+ expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
1303
+ expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
1304
+ expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
1305
+ end
1306
+ end
1307
+
1308
+ context 'when event creation fails' do
1309
+ before(:each) do
1310
+ allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
1311
+
1312
+ allow(plugin.event_factory).to receive(:new_event).and_call_original
1313
+ allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
1314
+ end
1315
+
1316
+ it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
1317
+ plugin.send(:push_hit, hit, mock_queue)
1318
+
1319
+ expect(mock_queue).to have_received(:<<) do |event|
1320
+ expect(event).to be_a_kind_of LogStash::Event
1321
+
1322
+ expect(event.get('tags')).to include("_elasticsearch_input_failure")
1323
+ expect(event.get('[event][original]')).to be_a_kind_of String
1324
+ expect(JSON.load(event.get('[event][original]'))).to eq hit
1325
+ end
1326
+
1327
+ expect(plugin.logger)
1328
+ .to have_received(:warn).with(
1329
+ a_string_including("Event creation error, original data now in [event][original] field"),
1330
+ a_hash_including(:message => a_string_including('intentional'),
1331
+ :data => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
1332
+
1333
+ end
1334
+ end
1335
+ end
1336
+
1238
1337
  # @note can be removed once we depends on elasticsearch gem >= 6.x
1239
1338
  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
1240
1339
  client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
@@ -4,7 +4,7 @@ require "logstash/plugin"
4
4
  require "logstash/inputs/elasticsearch"
5
5
  require_relative "../../../spec/es_helper"
6
6
 
7
- describe LogStash::Inputs::Elasticsearch, :integration => true do
7
+ describe LogStash::Inputs::Elasticsearch do
8
8
 
9
9
  SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
10
10
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.21.0
4
+ version: 4.21.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-12-18 00:00:00.000000000 Z
11
+ date: 2025-01-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -277,11 +277,14 @@ files:
277
277
  - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
278
278
  - logstash-input-elasticsearch.gemspec
279
279
  - spec/es_helper.rb
280
+ - spec/fixtures/test_certs/GENERATED_AT
280
281
  - spec/fixtures/test_certs/ca.crt
281
282
  - spec/fixtures/test_certs/ca.der.sha256
282
283
  - spec/fixtures/test_certs/ca.key
284
+ - spec/fixtures/test_certs/es.chain.crt
283
285
  - spec/fixtures/test_certs/es.crt
284
286
  - spec/fixtures/test_certs/es.key
287
+ - spec/fixtures/test_certs/renew.sh
285
288
  - spec/inputs/elasticsearch_spec.rb
286
289
  - spec/inputs/elasticsearch_ssl_spec.rb
287
290
  - spec/inputs/integration/elasticsearch_spec.rb
@@ -313,11 +316,14 @@ specification_version: 4
313
316
  summary: Reads query results from an Elasticsearch cluster
314
317
  test_files:
315
318
  - spec/es_helper.rb
319
+ - spec/fixtures/test_certs/GENERATED_AT
316
320
  - spec/fixtures/test_certs/ca.crt
317
321
  - spec/fixtures/test_certs/ca.der.sha256
318
322
  - spec/fixtures/test_certs/ca.key
323
+ - spec/fixtures/test_certs/es.chain.crt
319
324
  - spec/fixtures/test_certs/es.crt
320
325
  - spec/fixtures/test_certs/es.key
326
+ - spec/fixtures/test_certs/renew.sh
321
327
  - spec/inputs/elasticsearch_spec.rb
322
328
  - spec/inputs/elasticsearch_ssl_spec.rb
323
329
  - spec/inputs/integration/elasticsearch_spec.rb