logstash-input-elasticsearch 4.21.0 → 4.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/docs/index.asciidoc +10 -0
- data/lib/logstash/inputs/elasticsearch.rb +14 -6
- data/logstash-input-elasticsearch.gemspec +1 -1
- data/spec/fixtures/test_certs/GENERATED_AT +1 -0
- data/spec/fixtures/test_certs/ca.crt +17 -18
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/es.chain.crt +38 -0
- data/spec/fixtures/test_certs/es.crt +17 -18
- data/spec/fixtures/test_certs/renew.sh +15 -0
- data/spec/inputs/elasticsearch_spec.rb +102 -3
- data/spec/inputs/integration/elasticsearch_spec.rb +1 -1
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba4a906467e97c729acee2b28f29350b27ed838bc9d3e8ea0b7cd4a83b4dd06e
|
4
|
+
data.tar.gz: 2b0c263a32bbcaa0e2d88bb2a070072e95c1a80f6d9794ee267c703e34bed3af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ff1d841fbd5cbe0469a704131bd55b206c418243e87b3afac4e4d0d3ef26af6e926148323ec6c939fd48ea53bf4e0d4fcdb09ea1015d2d815863db917ff322c5
|
7
|
+
data.tar.gz: 66fbdcc42a51b8859be13df832d7d5e5c2a71e53c8995c490d64316590f74d277d6ed73d8b366a580fe5dfb085205999d0488368e1c62c37dc83ad3a74e08fad
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 4.21.1
|
2
|
+
- Fix: prevent plugin crash when hits contain illegal structure [#183](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183)
|
3
|
+
- When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
|
4
|
+
|
1
5
|
## 4.21.0
|
2
6
|
- Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217)
|
3
7
|
|
data/docs/index.asciidoc
CHANGED
@@ -93,6 +93,16 @@ The plugin logs a warning when ECS is enabled and `target` isn't set.
|
|
93
93
|
|
94
94
|
TIP: Set the `target` option to avoid potential schema conflicts.
|
95
95
|
|
96
|
+
[id="plugins-{type}s-{plugin}-failure-handling"]
|
97
|
+
==== Failure handling
|
98
|
+
|
99
|
+
When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit.
|
100
|
+
|
101
|
+
Common causes are:
|
102
|
+
|
103
|
+
- When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace.
|
104
|
+
- When <<plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<plugins-{type}s-{plugin}-target>> and <<plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict.
|
105
|
+
|
96
106
|
[id="plugins-{type}s-{plugin}-options"]
|
97
107
|
==== Elasticsearch Input configuration options
|
98
108
|
|
@@ -353,21 +353,29 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
353
353
|
# This can be called externally from the query_executor
|
354
354
|
public
|
355
355
|
def push_hit(hit, output_queue, root_field = '_source')
|
356
|
-
event =
|
357
|
-
set_docinfo_fields(hit, event) if @docinfo
|
356
|
+
event = event_from_hit(hit, root_field)
|
358
357
|
decorate(event)
|
359
358
|
output_queue << event
|
360
359
|
end
|
361
360
|
|
361
|
+
def event_from_hit(hit, root_field)
|
362
|
+
event = targeted_event_factory.new_event hit[root_field]
|
363
|
+
set_docinfo_fields(hit, event) if @docinfo
|
364
|
+
|
365
|
+
event
|
366
|
+
rescue => e
|
367
|
+
serialized_hit = hit.to_json
|
368
|
+
logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit)
|
369
|
+
return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
|
370
|
+
end
|
371
|
+
|
362
372
|
def set_docinfo_fields(hit, event)
|
363
373
|
# do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
|
364
374
|
docinfo_target = event.get(@docinfo_target) || {}
|
365
375
|
|
366
376
|
unless docinfo_target.is_a?(Hash)
|
367
|
-
|
368
|
-
|
369
|
-
# TODO: (colin) I am not sure raising is a good strategy here?
|
370
|
-
raise Exception.new("Elasticsearch input: incompatible event")
|
377
|
+
# expect error to be handled by `#event_from_hit`
|
378
|
+
fail RuntimeError, "Incompatible event; unable to merge docinfo fields into docinfo_target=`#{@docinfo_target}`"
|
371
379
|
end
|
372
380
|
|
373
381
|
@docinfo_fields.each do |field|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.21.
|
4
|
+
s.version = '4.21.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1 @@
|
|
1
|
+
2024-12-26T22:27:15+00:00
|
@@ -1,20 +1,19 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
|
2
|
+
MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
|
3
|
+
dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
|
4
|
+
MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
|
5
|
+
aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
|
6
|
+
AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
|
7
|
+
QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
|
8
|
+
90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
|
9
|
+
99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
|
10
|
+
i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
|
11
|
+
m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
|
12
|
+
BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
|
13
|
+
AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
|
14
|
+
lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
|
15
|
+
p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
|
16
|
+
E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
|
17
|
+
IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
|
18
|
+
T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
|
20
19
|
-----END CERTIFICATE-----
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
b1e955819b0d14f64f863adb103c248ddacf2e17bea48d04ee4b57c64814ccc4
|
@@ -0,0 +1,38 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
|
3
|
+
dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
|
4
|
+
MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
|
5
|
+
9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
|
6
|
+
V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
|
7
|
+
36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
|
8
|
+
MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
|
9
|
+
/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
|
10
|
+
hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
|
11
|
+
BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
|
12
|
+
K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
|
13
|
+
wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
|
14
|
+
CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
|
15
|
+
T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
|
16
|
+
byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
|
17
|
+
UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
|
18
|
+
avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
|
19
|
+
-----END CERTIFICATE-----
|
20
|
+
-----BEGIN CERTIFICATE-----
|
21
|
+
MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
|
22
|
+
dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
|
23
|
+
MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
|
24
|
+
aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
|
25
|
+
AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
|
26
|
+
QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
|
27
|
+
90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
|
28
|
+
99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
|
29
|
+
i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
|
30
|
+
m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
|
31
|
+
BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
|
32
|
+
AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
|
33
|
+
lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
|
34
|
+
p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
|
35
|
+
E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
|
36
|
+
IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
|
37
|
+
T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
|
38
|
+
-----END CERTIFICATE-----
|
@@ -1,20 +1,19 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
qi02i4q6meHGcw==
|
2
|
+
MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
|
3
|
+
dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
|
4
|
+
MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
|
5
|
+
9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
|
6
|
+
V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
|
7
|
+
36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
|
8
|
+
MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
|
9
|
+
/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
|
10
|
+
hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
|
11
|
+
BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
|
12
|
+
K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
|
13
|
+
wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
|
14
|
+
CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
|
15
|
+
T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
|
16
|
+
byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
|
17
|
+
UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
|
18
|
+
avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
|
20
19
|
-----END CERTIFICATE-----
|
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
set -e
|
4
|
+
cd "$(dirname "$0")"
|
5
|
+
|
6
|
+
openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
|
7
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
|
8
|
+
openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
|
9
|
+
|
10
|
+
openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
|
11
|
+
openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
|
12
|
+
cat es.crt ca.crt > es.chain.crt
|
13
|
+
|
14
|
+
# output ISO8601 timestamp to file
|
15
|
+
date -Iseconds > GENERATED_AT
|
@@ -653,11 +653,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
653
653
|
context 'if the `docinfo_target` exist but is not of type hash' do
|
654
654
|
let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
|
655
655
|
let(:do_register) { false }
|
656
|
+
let(:mock_queue) { double('Queue', :<< => nil) }
|
657
|
+
let(:hit) { response.dig('hits', 'hits').first }
|
658
|
+
|
659
|
+
it 'emits a tagged event with JSON-serialized event in [event][original]' do
|
660
|
+
allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
|
656
661
|
|
657
|
-
it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
|
658
|
-
expect(client).not_to receive(:clear_scroll)
|
659
662
|
plugin.register
|
660
|
-
|
663
|
+
plugin.run(mock_queue)
|
664
|
+
|
665
|
+
expect(mock_queue).to have_received(:<<) do |event|
|
666
|
+
expect(event).to be_a_kind_of LogStash::Event
|
667
|
+
|
668
|
+
expect(event.get('tags')).to include("_elasticsearch_input_failure")
|
669
|
+
expect(event.get('[event][original]')).to be_a_kind_of String
|
670
|
+
expect(JSON.load(event.get('[event][original]'))).to eq hit
|
671
|
+
end
|
672
|
+
|
673
|
+
expect(plugin.logger)
|
674
|
+
.to have_received(:warn).with(
|
675
|
+
a_string_including("Event creation error, original data now in [event][original] field"),
|
676
|
+
a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
|
677
|
+
:data => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
|
661
678
|
end
|
662
679
|
|
663
680
|
end
|
@@ -1235,6 +1252,88 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
1235
1252
|
end
|
1236
1253
|
end
|
1237
1254
|
|
1255
|
+
context '#push_hit' do
|
1256
|
+
let(:config) do
|
1257
|
+
{
|
1258
|
+
'docinfo' => true, # include ids
|
1259
|
+
'docinfo_target' => '[@metadata][docinfo]'
|
1260
|
+
}
|
1261
|
+
end
|
1262
|
+
|
1263
|
+
let(:hit) do
|
1264
|
+
JSON.load(<<~EOJSON)
|
1265
|
+
{
|
1266
|
+
"_index" : "test_bulk_index_2",
|
1267
|
+
"_type" : "_doc",
|
1268
|
+
"_id" : "sHe6A3wBesqF7ydicQvG",
|
1269
|
+
"_score" : 1.0,
|
1270
|
+
"_source" : {
|
1271
|
+
"@timestamp" : "2021-09-20T15:02:02.557Z",
|
1272
|
+
"message" : "ping",
|
1273
|
+
"@version" : "17",
|
1274
|
+
"sequence" : 7,
|
1275
|
+
"host" : {
|
1276
|
+
"name" : "maybe.local",
|
1277
|
+
"ip" : "127.0.0.1"
|
1278
|
+
}
|
1279
|
+
}
|
1280
|
+
}
|
1281
|
+
EOJSON
|
1282
|
+
end
|
1283
|
+
|
1284
|
+
let(:mock_queue) { double('queue', :<< => nil) }
|
1285
|
+
|
1286
|
+
it 'pushes a generated event to the queue' do
|
1287
|
+
plugin.send(:push_hit, hit, mock_queue)
|
1288
|
+
expect(mock_queue).to have_received(:<<) do |event|
|
1289
|
+
expect(event).to be_a_kind_of LogStash::Event
|
1290
|
+
|
1291
|
+
# fields overriding defaults
|
1292
|
+
expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
|
1293
|
+
expect(event.get('@version')).to eq("17")
|
1294
|
+
|
1295
|
+
# structure from hit's _source
|
1296
|
+
expect(event.get('message')).to eq("ping")
|
1297
|
+
expect(event.get('sequence')).to eq(7)
|
1298
|
+
expect(event.get('[host][name]')).to eq("maybe.local")
|
1299
|
+
expect(event.get('[host][ip]')).to eq("127.0.0.1")
|
1300
|
+
|
1301
|
+
# docinfo fields
|
1302
|
+
expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
|
1303
|
+
expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
|
1304
|
+
expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
|
1305
|
+
end
|
1306
|
+
end
|
1307
|
+
|
1308
|
+
context 'when event creation fails' do
|
1309
|
+
before(:each) do
|
1310
|
+
allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
|
1311
|
+
|
1312
|
+
allow(plugin.event_factory).to receive(:new_event).and_call_original
|
1313
|
+
allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
|
1314
|
+
end
|
1315
|
+
|
1316
|
+
it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
|
1317
|
+
plugin.send(:push_hit, hit, mock_queue)
|
1318
|
+
|
1319
|
+
expect(mock_queue).to have_received(:<<) do |event|
|
1320
|
+
expect(event).to be_a_kind_of LogStash::Event
|
1321
|
+
|
1322
|
+
expect(event.get('tags')).to include("_elasticsearch_input_failure")
|
1323
|
+
expect(event.get('[event][original]')).to be_a_kind_of String
|
1324
|
+
expect(JSON.load(event.get('[event][original]'))).to eq hit
|
1325
|
+
end
|
1326
|
+
|
1327
|
+
expect(plugin.logger)
|
1328
|
+
.to have_received(:warn).with(
|
1329
|
+
a_string_including("Event creation error, original data now in [event][original] field"),
|
1330
|
+
a_hash_including(:message => a_string_including('intentional'),
|
1331
|
+
:data => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
|
1332
|
+
|
1333
|
+
end
|
1334
|
+
end
|
1335
|
+
end
|
1336
|
+
|
1238
1337
|
# @note can be removed once we depends on elasticsearch gem >= 6.x
|
1239
1338
|
def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
|
1240
1339
|
client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
|
@@ -4,7 +4,7 @@ require "logstash/plugin"
|
|
4
4
|
require "logstash/inputs/elasticsearch"
|
5
5
|
require_relative "../../../spec/es_helper"
|
6
6
|
|
7
|
-
describe LogStash::Inputs::Elasticsearch
|
7
|
+
describe LogStash::Inputs::Elasticsearch do
|
8
8
|
|
9
9
|
SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
|
10
10
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.21.
|
4
|
+
version: 4.21.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-01-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -277,11 +277,14 @@ files:
|
|
277
277
|
- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
|
278
278
|
- logstash-input-elasticsearch.gemspec
|
279
279
|
- spec/es_helper.rb
|
280
|
+
- spec/fixtures/test_certs/GENERATED_AT
|
280
281
|
- spec/fixtures/test_certs/ca.crt
|
281
282
|
- spec/fixtures/test_certs/ca.der.sha256
|
282
283
|
- spec/fixtures/test_certs/ca.key
|
284
|
+
- spec/fixtures/test_certs/es.chain.crt
|
283
285
|
- spec/fixtures/test_certs/es.crt
|
284
286
|
- spec/fixtures/test_certs/es.key
|
287
|
+
- spec/fixtures/test_certs/renew.sh
|
285
288
|
- spec/inputs/elasticsearch_spec.rb
|
286
289
|
- spec/inputs/elasticsearch_ssl_spec.rb
|
287
290
|
- spec/inputs/integration/elasticsearch_spec.rb
|
@@ -313,11 +316,14 @@ specification_version: 4
|
|
313
316
|
summary: Reads query results from an Elasticsearch cluster
|
314
317
|
test_files:
|
315
318
|
- spec/es_helper.rb
|
319
|
+
- spec/fixtures/test_certs/GENERATED_AT
|
316
320
|
- spec/fixtures/test_certs/ca.crt
|
317
321
|
- spec/fixtures/test_certs/ca.der.sha256
|
318
322
|
- spec/fixtures/test_certs/ca.key
|
323
|
+
- spec/fixtures/test_certs/es.chain.crt
|
319
324
|
- spec/fixtures/test_certs/es.crt
|
320
325
|
- spec/fixtures/test_certs/es.key
|
326
|
+
- spec/fixtures/test_certs/renew.sh
|
321
327
|
- spec/inputs/elasticsearch_spec.rb
|
322
328
|
- spec/inputs/elasticsearch_ssl_spec.rb
|
323
329
|
- spec/inputs/integration/elasticsearch_spec.rb
|