logstash-input-beats 6.0.11-java → 6.1.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/README.md +1 -1
- data/VERSION +1 -1
- data/docs/agent.asciidoc +320 -0
- data/docs/index.asciidoc +40 -5
- data/lib/logstash-input-beats_jars.rb +1 -1
- data/lib/logstash/inputs/beats.rb +86 -34
- data/lib/logstash/inputs/beats/decoded_event_transform.rb +10 -1
- data/lib/logstash/inputs/beats/event_transform_common.rb +2 -2
- data/lib/logstash/inputs/beats/message_listener.rb +30 -5
- data/logstash-input-beats.gemspec +1 -0
- data/spec/inputs/beats/decoded_event_transform_spec.rb +4 -3
- data/spec/inputs/beats/event_transform_common_spec.rb +2 -1
- data/spec/inputs/beats/message_listener_spec.rb +82 -54
- data/spec/inputs/beats/raw_event_transform_spec.rb +2 -1
- data/spec/inputs/beats_spec.rb +38 -13
- data/spec/integration/filebeat_spec.rb +9 -9
- data/spec/integration/logstash_forwarder_spec.rb +1 -1
- data/spec/support/logstash_test.rb +7 -0
- data/spec/support/shared_examples.rb +39 -33
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/{6.0.11/logstash-input-beats-6.0.11.jar → 6.1.1/logstash-input-beats-6.1.1.jar} +0 -0
- metadata +18 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6def71232c684cd74e5be5913809fbfa34a5922f600e433174ebe323525d7936
|
4
|
+
data.tar.gz: 27f0ebc4bd488cc3ccd64cec8a64194eae87484f6af55d02a00fae95dcaae45c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10b2fdc15611ca30aaba8fafb79b53588188788a96ca6a906fbef53c2c9d6f950e56ba53e0c961c1edb09c3d85a70f91f4b58b7979cf9b77f9894e25220e1733
|
7
|
+
data.tar.gz: dfc2a89ffba6de5955d2bfd6c8c0862d5d594d4762a4a608da64d85017e0b46086bc0028b3c89717d19839a36c7261348ab498123f2caa313e060ee7ffe2811a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,21 @@
|
|
1
|
+
## 6.1.1
|
2
|
+
- [DOC] Enhanced ECS compatibility information for ease of use and readability
|
3
|
+
[#413](https://github.com/logstash-plugins/logstash-input-beats/pull/413)
|
4
|
+
|
5
|
+
## 6.1.0
|
6
|
+
- ECS compatibility enablement. Adds `ecs_compatibility` setting to declare the level of ECS compatibility (`disabled` or `v1`) at plugin level. When `disabled`, the plugin behaves like before, while `v1` does a rename of
|
7
|
+
`host` and `@metadata.ip_address` event fields. [#404](https://github.com/logstash-plugins/logstash-input-beats/pull/404)
|
8
|
+
|
9
|
+
## 6.0.14
|
10
|
+
- Feat: log + unwrap generic SSL context exceptions [#405](https://github.com/logstash-plugins/logstash-input-beats/pull/405)
|
11
|
+
|
12
|
+
## 6.0.13
|
13
|
+
- [DOC] Update links to use shared attributes
|
14
|
+
|
15
|
+
## 6.0.12
|
16
|
+
- Fix: log error when SSL context building fails [#402](https://github.com/logstash-plugins/logstash-input-beats/pull/402).
|
17
|
+
We've also made sure to log messages on configuration errors as LS 7.8/7.9 only prints details when level set to debug.
|
18
|
+
|
1
19
|
## 6.0.11
|
2
20
|
- Updated jackson databind and Netty dependencies. Additionally, this release removes the dependency on `tcnative` +
|
3
21
|
`boringssl`, using JVM supplied ciphers instead. This may result in fewer ciphers being available if the JCE
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[![Travis Build Status](https://travis-ci.
|
3
|
+
[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-beats.svg)](https://travis-ci.com/logstash-plugins/logstash-input-beats)
|
4
4
|
|
5
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
6
6
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
6.
|
1
|
+
6.1.1
|
data/docs/agent.asciidoc
ADDED
@@ -0,0 +1,320 @@
|
|
1
|
+
:plugin: agent
|
2
|
+
:type: input
|
3
|
+
:default_codec: plain
|
4
|
+
|
5
|
+
///////////////////////////////////////////
|
6
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
7
|
+
///////////////////////////////////////////
|
8
|
+
|
9
|
+
// Copied from Beats generated plugin output.
|
10
|
+
// Not actively generated at this time!
|
11
|
+
|
12
|
+
////
|
13
|
+
:version: %VERSION%
|
14
|
+
:release_date: %RELEASE_DATE%
|
15
|
+
:changelog_url: %CHANGELOG_URL%
|
16
|
+
:include_path: ../../../../logstash/docs/include
|
17
|
+
////
|
18
|
+
|
19
|
+
///////////////////////////////////////////
|
20
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
21
|
+
///////////////////////////////////////////
|
22
|
+
|
23
|
+
[id="plugins-{type}s-{plugin}"]
|
24
|
+
|
25
|
+
=== Agent input plugin
|
26
|
+
|
27
|
+
include::{include_path}/plugin_header.asciidoc[]
|
28
|
+
|
29
|
+
==== Description
|
30
|
+
|
31
|
+
This input plugin enables Logstash to receive events from the
|
32
|
+
https://www.elastic.co/products/beats[Elastic Beats] framework.
|
33
|
+
|
34
|
+
The following example shows how to configure Logstash to listen on port
|
35
|
+
5044 for incoming Beats connections and to index into Elasticsearch.
|
36
|
+
|
37
|
+
[source,logstash]
|
38
|
+
-----
|
39
|
+
|
40
|
+
input {
|
41
|
+
beats {
|
42
|
+
port => 5044
|
43
|
+
}
|
44
|
+
}
|
45
|
+
|
46
|
+
output {
|
47
|
+
elasticsearch {
|
48
|
+
hosts => ["http://localhost:9200"]
|
49
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}" <1>
|
50
|
+
}
|
51
|
+
}
|
52
|
+
-----
|
53
|
+
<1> `%{[@metadata][beat]}` sets the first part of the index name to the value
|
54
|
+
of the `beat` metadata field and `%{[@metadata][version]}` sets the second part to
|
55
|
+
the Beat's version. For example:
|
56
|
+
metricbeat-7.4.0.
|
57
|
+
|
58
|
+
Events indexed into Elasticsearch with the Logstash configuration shown here
|
59
|
+
will be similar to events directly indexed by Beats into Elasticsearch.
|
60
|
+
|
61
|
+
NOTE: If ILM is not being used, set `index` to
|
62
|
+
`%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}` instead so
|
63
|
+
Logstash creates an index per day, based on the `@timestamp` value of the events
|
64
|
+
coming from Beats.
|
65
|
+
|
66
|
+
IMPORTANT: If you are shipping events that span multiple lines, you need to use
|
67
|
+
the {filebeat-ref}/multiline-examples.html[configuration options available in
|
68
|
+
Filebeat] to handle multiline events before sending the event data to Logstash.
|
69
|
+
You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec
|
70
|
+
plugin] to handle multiline events. Doing so will result in the failure to start
|
71
|
+
Logstash.
|
72
|
+
|
73
|
+
[id="plugins-{type}s-{plugin}-versioned-indexes"]
|
74
|
+
==== Versioned Beats Indices
|
75
|
+
|
76
|
+
To minimize the impact of future schema changes on your existing indices and
|
77
|
+
mappings in Elasticsearch, configure the Elasticsearch output to write to
|
78
|
+
versioned indices. The pattern that you specify for the `index` setting
|
79
|
+
controls the index name:
|
80
|
+
|
81
|
+
[source,yaml]
|
82
|
+
----
|
83
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
84
|
+
----
|
85
|
+
|
86
|
+
`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of
|
87
|
+
the `beat` metadata field, for example, `filebeat`.
|
88
|
+
`%{[@metadata][version]}`:: Sets the second part of the name to the Beat
|
89
|
+
version, for example, +{logstash_version}+.
|
90
|
+
`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the
|
91
|
+
Logstash `@timestamp` field.
|
92
|
+
|
93
|
+
This configuration results in daily index names like
|
94
|
+
+filebeat-{logstash_version}-{localdate}+.
|
95
|
+
|
96
|
+
|
97
|
+
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
98
|
+
==== Event Metadata and the Elastic Common Schema (ECS)
|
99
|
+
When decoding `beats` events, this plugin adds two fields related to the event: the deprecated `host`
|
100
|
+
which contains the `hostname` provided by beats and the `ip_address` containing the remote address
|
101
|
+
of the client's connection. When <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is
|
102
|
+
enabled these are now moved in ECS compatible namespace.
|
103
|
+
|
104
|
+
[id="plugins-{type}s-{plugin}-options"]
|
105
|
+
==== Agent Input Configuration Options
|
106
|
+
|
107
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
108
|
+
|
109
|
+
[cols="<,<,<",options="header",]
|
110
|
+
|=======================================================================
|
111
|
+
|Setting |Input type|Required
|
112
|
+
| <<plugins-{type}s-{plugin}-add_hostname>> |<<boolean,boolean>>|No
|
113
|
+
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
114
|
+
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
115
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
116
|
+
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
117
|
+
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
|
118
|
+
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
|
119
|
+
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
|
120
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No
|
121
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |<<array,array>>|No
|
122
|
+
| <<plugins-{type}s-{plugin}-ssl_handshake_timeout>> |<<number,number>>|No
|
123
|
+
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
124
|
+
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
125
|
+
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
126
|
+
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|No
|
127
|
+
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|No
|
128
|
+
| <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|No
|
129
|
+
|=======================================================================
|
130
|
+
|
131
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
132
|
+
input plugins.
|
133
|
+
|
134
|
+
|
135
|
+
|
136
|
+
[id="plugins-{type}s-{plugin}-add_hostname"]
|
137
|
+
===== `add_hostname`
|
138
|
+
|
139
|
+
deprecated[6.0.0, The default value has been changed to `false`. In 7.0.0 this setting will be removed]
|
140
|
+
|
141
|
+
* Value type is <<boolean,boolean>>
|
142
|
+
* Default value is `false`
|
143
|
+
|
144
|
+
Flag to determine whether to add `host` field to event using the value supplied by the beat in the `hostname` field.
|
145
|
+
|
146
|
+
|
147
|
+
[id="plugins-{type}s-{plugin}-cipher_suites"]
|
148
|
+
===== `cipher_suites`
|
149
|
+
|
150
|
+
* Value type is <<array,array>>
|
151
|
+
* Default value is `java.lang.String[TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
|
152
|
+
|
153
|
+
The list of ciphers suite to use, listed by priorities.
|
154
|
+
|
155
|
+
[id="plugins-{type}s-{plugin}-client_inactivity_timeout"]
|
156
|
+
===== `client_inactivity_timeout`
|
157
|
+
|
158
|
+
* Value type is <<number,number>>
|
159
|
+
* Default value is `60`
|
160
|
+
|
161
|
+
Close Idle clients after X seconds of inactivity.
|
162
|
+
|
163
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
164
|
+
===== `ecs_compatibility`
|
165
|
+
|
166
|
+
* Value type is <<string,string>>
|
167
|
+
* Supported values are:
|
168
|
+
** `disabled`: unstructured connection metadata added at root level
|
169
|
+
** `v1`: structured connection metadata added under ECS compliant namespaces
|
170
|
+
* Default value depends on which version of Logstash is running:
|
171
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
172
|
+
** Otherwise, the default value is `disabled`.
|
173
|
+
|
174
|
+
Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
|
175
|
+
The value of this setting affects the keys for the Beats connection's metadata on the event:
|
176
|
+
|
177
|
+
.Metadata Location by `ecs_compatibility` value
|
178
|
+
[cols="<l,<l,e,<e"]
|
179
|
+
|=======================================================================
|
180
|
+
|`disabled` |`v1` |Availability |Description
|
181
|
+
|
182
|
+
|[host] |[@metadata][input][beats][host][name] |Always |Name or address of the beat host
|
183
|
+
|[@metadata][ip_address] |[@metadata][input][beats][host][ip] |Always |IP address of the Beats client
|
184
|
+
|[@metadata][tls_peer][status] | [@metadata][tls_peer][status] | When SSL related fields are populated | Contains "verified"/"unverified" labels in `disabled`, `true`/`false` in `v1`
|
185
|
+
|[@metadata][tls_peer][protocol] | [@metadata][input][beats][tls][version_protocol] | When SSL status is "verified" | Contains the TLS version used (e.g. `TLSv1.2`)
|
186
|
+
|[@metadata][tls_peer][subject] | [@metadata][input][beats][tls][client][subject] | When SSL status is "verified" | Contains the identity name of the remote end (e.g. `CN=artifacts-no-kpi.elastic.co`)
|
187
|
+
|[@metadata][tls_peer][cipher_suite] | [@metadata][input][beats][tls][cipher] | When SSL status is "verified" | Contains the name of cipher suite used (e.g. `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`)
|
188
|
+
|=======================================================================
|
189
|
+
|
190
|
+
[id="plugins-{type}s-{plugin}-host"]
|
191
|
+
===== `host`
|
192
|
+
|
193
|
+
* Value type is <<string,string>>
|
194
|
+
* Default value is `"0.0.0.0"`
|
195
|
+
|
196
|
+
The IP address to listen on.
|
197
|
+
|
198
|
+
[id="plugins-{type}s-{plugin}-include_codec_tag"]
|
199
|
+
===== `include_codec_tag`
|
200
|
+
|
201
|
+
* Value type is <<boolean,boolean>>
|
202
|
+
* Default value is `true`
|
203
|
+
|
204
|
+
|
205
|
+
|
206
|
+
[id="plugins-{type}s-{plugin}-port"]
|
207
|
+
===== `port`
|
208
|
+
|
209
|
+
* This is a required setting.
|
210
|
+
* Value type is <<number,number>>
|
211
|
+
* There is no default value for this setting.
|
212
|
+
|
213
|
+
The port to listen on.
|
214
|
+
|
215
|
+
[id="plugins-{type}s-{plugin}-ssl"]
|
216
|
+
===== `ssl`
|
217
|
+
|
218
|
+
* Value type is <<boolean,boolean>>
|
219
|
+
* Default value is `false`
|
220
|
+
|
221
|
+
Events are by default sent in plain text. You can
|
222
|
+
enable encryption by setting `ssl` to true and configuring
|
223
|
+
the `ssl_certificate` and `ssl_key` options.
|
224
|
+
|
225
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
226
|
+
===== `ssl_certificate`
|
227
|
+
|
228
|
+
* Value type is <<path,path>>
|
229
|
+
* There is no default value for this setting.
|
230
|
+
|
231
|
+
SSL certificate to use.
|
232
|
+
|
233
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
234
|
+
===== `ssl_certificate_authorities`
|
235
|
+
|
236
|
+
* Value type is <<array,array>>
|
237
|
+
* Default value is `[]`
|
238
|
+
|
239
|
+
Validate client certificates against these authorities.
|
240
|
+
You can define multiple files or paths. All the certificates will
|
241
|
+
be read and added to the trust store. You need to configure the `ssl_verify_mode`
|
242
|
+
to `peer` or `force_peer` to enable the verification.
|
243
|
+
|
244
|
+
|
245
|
+
[id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
|
246
|
+
===== `ssl_handshake_timeout`
|
247
|
+
|
248
|
+
* Value type is <<number,number>>
|
249
|
+
* Default value is `10000`
|
250
|
+
|
251
|
+
Time in milliseconds for an incomplete ssl handshake to timeout
|
252
|
+
|
253
|
+
[id="plugins-{type}s-{plugin}-ssl_key"]
|
254
|
+
===== `ssl_key`
|
255
|
+
|
256
|
+
* Value type is <<path,path>>
|
257
|
+
* There is no default value for this setting.
|
258
|
+
|
259
|
+
SSL key to use.
|
260
|
+
NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.0/apps/pkcs8.html[OpenSSL]
|
261
|
+
for more information.
|
262
|
+
|
263
|
+
[id="plugins-{type}s-{plugin}-ssl_key_passphrase"]
|
264
|
+
===== `ssl_key_passphrase`
|
265
|
+
|
266
|
+
* Value type is <<password,password>>
|
267
|
+
* There is no default value for this setting.
|
268
|
+
|
269
|
+
SSL key passphrase to use.
|
270
|
+
|
271
|
+
[id="plugins-{type}s-{plugin}-ssl_verify_mode"]
|
272
|
+
===== `ssl_verify_mode`
|
273
|
+
|
274
|
+
* Value can be any of: `none`, `peer`, `force_peer`
|
275
|
+
* Default value is `"none"`
|
276
|
+
|
277
|
+
By default the server doesn't do any client verification.
|
278
|
+
|
279
|
+
`peer` will make the server ask the client to provide a certificate.
|
280
|
+
If the client provides a certificate, it will be validated.
|
281
|
+
|
282
|
+
`force_peer` will make the server ask the client to provide a certificate.
|
283
|
+
If the client doesn't provide a certificate, the connection will be closed.
|
284
|
+
|
285
|
+
This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
|
286
|
+
|
287
|
+
[id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
|
288
|
+
===== `ssl_peer_metadata`
|
289
|
+
|
290
|
+
* Value type is <<boolean,boolean>>
|
291
|
+
* Default value is `false`
|
292
|
+
|
293
|
+
Enables storing client certificate information in event's metadata.
|
294
|
+
|
295
|
+
This option is only valid when `ssl_verify_mode` is set to `peer` or `force_peer`.
|
296
|
+
|
297
|
+
[id="plugins-{type}s-{plugin}-tls_max_version"]
|
298
|
+
===== `tls_max_version`
|
299
|
+
|
300
|
+
* Value type is <<number,number>>
|
301
|
+
* Default value is `1.2`
|
302
|
+
|
303
|
+
The maximum TLS version allowed for the encrypted connections. The value must be the one of the following:
|
304
|
+
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
305
|
+
|
306
|
+
[id="plugins-{type}s-{plugin}-tls_min_version"]
|
307
|
+
===== `tls_min_version`
|
308
|
+
|
309
|
+
* Value type is <<number,number>>
|
310
|
+
* Default value is `1`
|
311
|
+
|
312
|
+
The minimum TLS version allowed for the encrypted connections. The value must be one of the following:
|
313
|
+
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
314
|
+
|
315
|
+
|
316
|
+
|
317
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
318
|
+
include::{include_path}/{type}.asciidoc[]
|
319
|
+
|
320
|
+
:default_codec!:
|
data/docs/index.asciidoc
CHANGED
@@ -56,11 +56,12 @@ NOTE: If ILM is not being used, set `index` to
|
|
56
56
|
Logstash creates an index per day, based on the `@timestamp` value of the events
|
57
57
|
coming from Beats.
|
58
58
|
|
59
|
-
IMPORTANT: If you are shipping events that span multiple lines, you need to
|
60
|
-
|
61
|
-
before sending the event data to Logstash.
|
62
|
-
{logstash-ref}/plugins-codecs-multiline.html[Multiline codec
|
63
|
-
result in the failure to start
|
59
|
+
IMPORTANT: If you are shipping events that span multiple lines, you need to use
|
60
|
+
the {filebeat-ref}/multiline-examples.html[configuration options available in
|
61
|
+
Filebeat] to handle multiline events before sending the event data to Logstash.
|
62
|
+
You cannot use the {logstash-ref}/plugins-codecs-multiline.html[Multiline codec
|
63
|
+
plugin] to handle multiline events. Doing so will result in the failure to start
|
64
|
+
Logstash.
|
64
65
|
|
65
66
|
[id="plugins-{type}s-{plugin}-versioned-indexes"]
|
66
67
|
==== Versioned Beats Indices
|
@@ -85,6 +86,26 @@ Logstash `@timestamp` field.
|
|
85
86
|
This configuration results in daily index names like
|
86
87
|
+filebeat-{logstash_version}-{localdate}+.
|
87
88
|
|
89
|
+
|
90
|
+
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
91
|
+
==== Event Metadata and the Elastic Common Schema (ECS)
|
92
|
+
When decoding `beats` events, this plugin adds two fields related to the event: the deprecated `host`
|
93
|
+
which contains the `hostname` provided by beats and the `ip_address` containing the remote address
|
94
|
+
of the client's connection. When <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is
|
95
|
+
enabled these are now moved in ECS compatible namespace. Here's how <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> affects output.
|
96
|
+
|
97
|
+
[cols="<l,<l,e,<e"]
|
98
|
+
|=======================================================================
|
99
|
+
|ECS disabled |ECS v1 |Availability |Description
|
100
|
+
|
101
|
+
|[host] |[@metadata][input][beats][host][name] |Always |Name or address of the beat host
|
102
|
+
|[@metadata][ip_address] |[@metadata][input][beats][host][ip] |Always |IP address of the Beats client
|
103
|
+
|[@metadata][tls_peer][status] | [@metadata][tls_peer][status] | When SSL related fields are populated | Contains "verified"/"unverified" labels in `disabled`, `true`/`false` in `v1`
|
104
|
+
|[@metadata][tls_peer][protocol] | [@metadata][input][beats][tls][version_protocol] | When SSL status is "verified" | Contains the TLS version used (e.g. `TLSv1.2`)
|
105
|
+
|[@metadata][tls_peer][subject] | [@metadata][input][beats][tls][client][subject] | When SSL status is "verified" | Contains the identity name of the remote end (e.g. `CN=artifacts-no-kpi.elastic.co`)
|
106
|
+
|[@metadata][tls_peer][cipher_suite] | [@metadata][input][beats][tls][cipher] | When SSL status is "verified" | Contains the name of cipher suite used (e.g. `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`)
|
107
|
+
|=======================================================================
|
108
|
+
|
88
109
|
[id="plugins-{type}s-{plugin}-options"]
|
89
110
|
==== Beats Input Configuration Options
|
90
111
|
|
@@ -96,6 +117,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
96
117
|
| <<plugins-{type}s-{plugin}-add_hostname>> |<<boolean,boolean>>|No
|
97
118
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
98
119
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
120
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
99
121
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
100
122
|
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
|
101
123
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
|
@@ -143,6 +165,19 @@ The list of ciphers suite to use, listed by priorities.
|
|
143
165
|
|
144
166
|
Close Idle clients after X seconds of inactivity.
|
145
167
|
|
168
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
169
|
+
===== `ecs_compatibility`
|
170
|
+
|
171
|
+
* Value type is <<string,string>>
|
172
|
+
* Supported values are:
|
173
|
+
** `disabled`: unstructured connection metadata added at root level
|
174
|
+
** `v1`: structured connection metadata added under ECS compliant namespaces
|
175
|
+
* Default value depends on which version of Logstash is running:
|
176
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
177
|
+
** Otherwise, the default value is `disabled`.
|
178
|
+
|
179
|
+
Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
|
180
|
+
|
146
181
|
[id="plugins-{type}s-{plugin}-host"]
|
147
182
|
===== `host`
|
148
183
|
|
@@ -8,4 +8,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
|
|
8
8
|
require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.4')
|
9
9
|
require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
|
10
10
|
require_jar('org.apache.logging.log4j', 'log4j-api', '2.11.1')
|
11
|
-
require_jar('org.logstash.beats', 'logstash-input-beats', '6.
|
11
|
+
require_jar('org.logstash.beats', 'logstash-input-beats', '6.1.1')
|
@@ -5,6 +5,7 @@ require "logstash/timestamp"
|
|
5
5
|
require "logstash/codecs/multiline"
|
6
6
|
require "logstash/util"
|
7
7
|
require "logstash-input-beats_jars"
|
8
|
+
require "logstash/plugin_mixins/ecs_compatibility_support"
|
8
9
|
require_relative "beats/patch"
|
9
10
|
|
10
11
|
# This input plugin enables Logstash to receive events from the
|
@@ -49,6 +50,9 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
49
50
|
require "logstash/inputs/beats/message_listener"
|
50
51
|
require "logstash/inputs/beats/tls"
|
51
52
|
|
53
|
+
# adds ecs_compatibility config which could be :disabled or :v1
|
54
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled,:v1)
|
55
|
+
|
52
56
|
config_name "beats"
|
53
57
|
|
54
58
|
default :codec, "plain"
|
@@ -121,6 +125,8 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
121
125
|
# Beats handler executor thread
|
122
126
|
config :executor_threads, :validate => :number, :default => LogStash::Config::CpuCoreStrategy.maximum
|
123
127
|
|
128
|
+
attr_reader :field_hostname, :field_hostip
|
129
|
+
|
124
130
|
def register
|
125
131
|
# For Logstash 2.4 we need to make sure that the logger is correctly set for the
|
126
132
|
# java classes before actually loading them.
|
@@ -131,27 +137,39 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
131
137
|
LogStash::Logger.setup_log4j(@logger)
|
132
138
|
end
|
133
139
|
|
134
|
-
if
|
135
|
-
@
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
+
if @ssl
|
141
|
+
if @ssl_key.nil? || @ssl_key.empty?
|
142
|
+
configuration_error "ssl_key => is a required setting when ssl => true is configured"
|
143
|
+
end
|
144
|
+
if @ssl_certificate.nil? || @ssl_certificate.empty?
|
145
|
+
configuration_error "ssl_certificate => is a required setting when ssl => true is configured"
|
146
|
+
end
|
140
147
|
|
141
|
-
|
142
|
-
|
143
|
-
|
148
|
+
if require_certificate_authorities? && !client_authentification?
|
149
|
+
configuration_error "ssl_certificate_authorities => is a required setting when ssl_verify_mode => '#{@ssl_verify_mode}' is configured"
|
150
|
+
end
|
144
151
|
|
145
|
-
|
146
|
-
|
152
|
+
if client_authentication_metadata? && !require_certificate_authorities?
|
153
|
+
configuration_error "Configuring ssl_peer_metadata => true requires ssl_verify_mode => to be configured with 'peer' or 'force_peer'"
|
154
|
+
end
|
155
|
+
else
|
156
|
+
@logger.warn("configured ssl_certificate => #{@ssl_certificate.inspect} will not be used") if @ssl_certificate
|
157
|
+
@logger.warn("configured ssl_key => #{@ssl_key.inspect} will not be used") if @ssl_key
|
147
158
|
end
|
148
159
|
|
149
160
|
# Logstash 6.x breaking change (introduced with 4.0.0 of this gem)
|
150
161
|
if @codec.kind_of? LogStash::Codecs::Multiline
|
151
|
-
|
162
|
+
configuration_error "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html"
|
152
163
|
end
|
153
164
|
|
154
|
-
|
165
|
+
# define ecs name mapping
|
166
|
+
@field_hostname = ecs_select[disabled: "host", v1: "[@metadata][input][beats][host][name]"]
|
167
|
+
@field_hostip = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
|
168
|
+
@field_tls_protocol_version = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
|
169
|
+
@field_tls_peer_subject = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
|
170
|
+
@field_tls_cipher = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
|
171
|
+
|
172
|
+
@logger.info("Starting input listener", :address => "#{@host}:#{@port}")
|
155
173
|
|
156
174
|
@server = create_server
|
157
175
|
end # def register
|
@@ -159,37 +177,20 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
159
177
|
def create_server
|
160
178
|
server = org.logstash.beats.Server.new(@host, @port, @client_inactivity_timeout, @executor_threads)
|
161
179
|
if @ssl
|
162
|
-
|
163
|
-
begin
|
164
|
-
ssl_context_builder = org.logstash.netty.SslContextBuilder.new(@ssl_certificate, @ssl_key, @ssl_key_passphrase.nil? ? nil : @ssl_key_passphrase.value)
|
165
|
-
.setProtocols(convert_protocols)
|
166
|
-
.setCipherSuites(normalized_ciphers)
|
167
|
-
rescue java.lang.IllegalArgumentException => e
|
168
|
-
raise LogStash::ConfigurationError, e
|
169
|
-
end
|
170
|
-
|
171
|
-
|
180
|
+
ssl_context_builder = new_ssl_context_builder
|
172
181
|
if client_authentification?
|
173
|
-
if @ssl_verify_mode
|
182
|
+
if @ssl_verify_mode == "force_peer"
|
174
183
|
ssl_context_builder.setVerifyMode(org.logstash.netty.SslContextBuilder::SslClientVerifyMode::FORCE_PEER)
|
175
|
-
elsif @ssl_verify_mode
|
184
|
+
elsif @ssl_verify_mode == "peer"
|
176
185
|
ssl_context_builder.setVerifyMode(org.logstash.netty.SslContextBuilder::SslClientVerifyMode::VERIFY_PEER)
|
177
186
|
end
|
178
187
|
ssl_context_builder.setCertificateAuthorities(@ssl_certificate_authorities)
|
179
188
|
end
|
180
|
-
server.setSslHandlerProvider(
|
189
|
+
server.setSslHandlerProvider(new_ssl_handshake_provider(ssl_context_builder))
|
181
190
|
end
|
182
191
|
server
|
183
192
|
end
|
184
193
|
|
185
|
-
def ssl_configured?
|
186
|
-
!(@ssl_certificate.nil? || @ssl_key.nil?)
|
187
|
-
end
|
188
|
-
|
189
|
-
def target_codec_on_field?
|
190
|
-
!@target_codec_on_field.empty?
|
191
|
-
end
|
192
|
-
|
193
194
|
def run(output_queue)
|
194
195
|
message_listener = MessageListener.new(output_queue, self)
|
195
196
|
@server.setMessageListener(message_listener)
|
@@ -200,6 +201,14 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
200
201
|
@server.stop unless @server.nil?
|
201
202
|
end
|
202
203
|
|
204
|
+
def ssl_configured?
|
205
|
+
!(@ssl_certificate.nil? || @ssl_key.nil?)
|
206
|
+
end
|
207
|
+
|
208
|
+
def target_codec_on_field?
|
209
|
+
!@target_codec_on_field.empty?
|
210
|
+
end
|
211
|
+
|
203
212
|
def client_authentification?
|
204
213
|
@ssl_certificate_authorities && @ssl_certificate_authorities.size > 0
|
205
214
|
end
|
@@ -216,6 +225,32 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
216
225
|
@ssl_verify_mode == "force_peer" || @ssl_verify_mode == "peer"
|
217
226
|
end
|
218
227
|
|
228
|
+
private
|
229
|
+
|
230
|
+
def new_ssl_handshake_provider(ssl_context_builder)
|
231
|
+
begin
|
232
|
+
org.logstash.netty.SslHandlerProvider.new(ssl_context_builder.build_context, @ssl_handshake_timeout)
|
233
|
+
rescue java.lang.IllegalArgumentException => e
|
234
|
+
@logger.error("SSL configuration invalid", error_details(e))
|
235
|
+
raise LogStash::ConfigurationError, e
|
236
|
+
rescue java.lang.Exception => e # java.security.GeneralSecurityException
|
237
|
+
@logger.error("SSL configuration failed", error_details(e, true))
|
238
|
+
raise e
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def new_ssl_context_builder
|
243
|
+
passphrase = @ssl_key_passphrase.nil? ? nil : @ssl_key_passphrase.value
|
244
|
+
begin
|
245
|
+
org.logstash.netty.SslContextBuilder.new(@ssl_certificate, @ssl_key, passphrase)
|
246
|
+
.setProtocols(convert_protocols)
|
247
|
+
.setCipherSuites(normalized_ciphers)
|
248
|
+
rescue java.lang.IllegalArgumentException => e
|
249
|
+
@logger.error("SSL configuration invalid", error_details(e))
|
250
|
+
raise LogStash::ConfigurationError, e
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
219
254
|
def normalized_ciphers
|
220
255
|
@cipher_suites.map(&:upcase)
|
221
256
|
end
|
@@ -223,4 +258,21 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
223
258
|
def convert_protocols
|
224
259
|
TLS.get_supported(@tls_min_version..@tls_max_version).map(&:name)
|
225
260
|
end
|
261
|
+
|
262
|
+
def configuration_error(message)
|
263
|
+
@logger.error message
|
264
|
+
raise LogStash::ConfigurationError, message
|
265
|
+
end
|
266
|
+
|
267
|
+
def error_details(e, trace = false)
|
268
|
+
error_details = { :exception => e.class, :message => e.message }
|
269
|
+
error_details[:backtrace] = e.backtrace if trace || @logger.debug?
|
270
|
+
cause = e.cause
|
271
|
+
if cause && e != cause
|
272
|
+
error_details[:cause] = { :exception => cause.class, :message => cause.message }
|
273
|
+
error_details[:cause][:backtrace] = cause.backtrace if trace || @logger.debug?
|
274
|
+
end
|
275
|
+
error_details
|
276
|
+
end
|
277
|
+
|
226
278
|
end
|