logstash-input-beats 6.4.4-java → 6.6.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/VERSION +1 -1
- data/docs/index.asciidoc +157 -29
- data/lib/logstash/inputs/beats/message_listener.rb +6 -3
- data/lib/logstash/inputs/beats.rb +208 -55
- data/lib/logstash-input-beats_jars.rb +1 -1
- data/lib/tasks/test.rake +1 -1
- data/logstash-input-beats.gemspec +2 -0
- data/spec/inputs/beats_spec.rb +353 -29
- data/spec/integration/filebeat_spec.rb +4 -4
- data/spec/integration/logstash_forwarder_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -1
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/{6.4.4/logstash-input-beats-6.4.4.jar → 6.6.0/logstash-input-beats-6.6.0.jar} +0 -0
- metadata +31 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 92357d87addd898c82f05a241738a7b83debb31e2ae55197f5c2dd9003a75781
|
4
|
+
data.tar.gz: fbfd41f7cc59c46d1ca43a8b3a761b9c1dff1026d23aae5bcf5f9e2eb1b3c94f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 33330e2fe60093cde6c177483e74e2137064a232ad8beff1faa682433f342cb079817c831d6f22c8356509f9c51f6d53ee63cc7bf554e1123a1b3d48e375ea9a
|
7
|
+
data.tar.gz: 48d537c69e2bdcef820336dd55f8ec292a643503beaac9b594d4d41e1cb411567fc6274b7d17f6eca607bd83288c459beeb0d27265bd7d6be4997fbad8768c07
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
## 6.6.0
|
2
|
+
- Reviewed and deprecated SSL settings to comply with Logstash's naming convention [#470](https://github.com/logstash-plugins/logstash-input-beats/pull/470)
|
3
|
+
- Deprecated `ssl` in favor of `ssl_enabled`
|
4
|
+
- Deprecated `ssl_verify_mode` in favor of `ssl_client_authentication`
|
5
|
+
|
6
|
+
## 6.5.0
|
7
|
+
- An enrichment `enrich` option added to control ECS passthrough. `ssl_peer_metadata` and `include_codec_tag` configurations are deprecated and can be managed through the `enrich` [#464](https://github.com/logstash-plugins/logstash-input-beats/pull/464)
|
8
|
+
|
1
9
|
## 6.4.4
|
2
10
|
- Updates Netty dependency to 4.1.87 [#466](https://github.com/logstash-plugins/logstash-input-beats/pull/466)
|
3
11
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
6.
|
1
|
+
6.6.0
|
data/docs/index.asciidoc
CHANGED
@@ -143,26 +143,65 @@ endif::[]
|
|
143
143
|
|
144
144
|
|
145
145
|
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
146
|
-
==== Event
|
146
|
+
==== Event enrichment and the Elastic Common Schema (ECS)
|
147
147
|
|
148
|
-
When decoding {plugin-uc} events, this plugin
|
149
|
-
|
150
|
-
`ip_address` containing the remote address of the client's connection. When
|
151
|
-
<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled
|
152
|
-
these are now moved in ECS compatible namespace. Here's how
|
153
|
-
<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> affects
|
154
|
-
output.
|
148
|
+
When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing.
|
149
|
+
You can use the <<plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories.
|
155
150
|
|
156
|
-
|
151
|
+
The location of these enrichment fields depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled:
|
152
|
+
|
153
|
+
- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure.
|
154
|
+
- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema.
|
155
|
+
|
156
|
+
|
157
|
+
.`source_metadata`
|
158
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
159
|
+
|=======================================================================
|
160
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
161
|
+
|
162
|
+
|[@metadata][input][beats][host][name]
|
163
|
+
|[host]
|
164
|
+
|Name or address of the {plugin-singular} host
|
165
|
+
|
166
|
+
|[@metadata][input][beats][host][ip]
|
167
|
+
|[@metadata][ip_address]
|
168
|
+
|IP address of the {plugin-uc} client that connected to this input
|
169
|
+
|=======================================================================
|
170
|
+
|
171
|
+
.`ssl_peer_metadata`
|
172
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
173
|
+
|=======================================================================
|
174
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
175
|
+
|
176
|
+
|[@metadata][tls_peer][status]
|
177
|
+
|[@metadata][tls_peer][status]
|
178
|
+
|Contains "verified" or "unverified" label; available when SSL is enabled.
|
179
|
+
|
180
|
+
|[@metadata][input][beats][tls][version_protocol]
|
181
|
+
|[@metadata][tls_peer][protocol]
|
182
|
+
|Contains the TLS version used (such as `TLSv1.2`); available when SSL status is "verified"
|
183
|
+
|
184
|
+
|[@metadata][input][beats][tls][client][subject]
|
185
|
+
|[@metadata][tls_peer][subject]
|
186
|
+
|Contains the identity name of the remote end (such as `CN=artifacts-no-kpi.elastic.co`); available when SSL status is "verified"
|
187
|
+
|
188
|
+
|[@metadata][input][beats][tls][cipher]
|
189
|
+
|[@metadata][tls_peer][cipher_suite]
|
190
|
+
|Contains the name of cipher suite used (such as `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`); available when SSL status is "verified"
|
191
|
+
|=======================================================================
|
192
|
+
|
193
|
+
.`codec_metadata`
|
194
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
157
195
|
|=======================================================================
|
158
|
-
|ECS `
|
159
|
-
|
160
|
-
|[
|
161
|
-
|[
|
162
|
-
|
|
163
|
-
|
164
|
-
|[
|
165
|
-
|
|
196
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
197
|
+
|
198
|
+
|[tag]
|
199
|
+
|[tag]
|
200
|
+
|Contains `beats_input_codec_XXX_applied` where `XXX` is the name of the codec
|
201
|
+
|
202
|
+
|[event][original]
|
203
|
+
e|N/A
|
204
|
+
|When ECS is enabled, even if `[event][original]` field does not already exist on the event being processed, this plugin's *default codec* ensures that the field is populated using the bytes as-processed.
|
166
205
|
|=======================================================================
|
167
206
|
|
168
207
|
[id="plugins-{type}s-{plugin}-options"]
|
@@ -177,19 +216,23 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
177
216
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|__Deprecated__
|
178
217
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
179
218
|
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
219
|
+
| <<plugins-{type}s-{plugin}-enrich>> |<<string,string>>|No
|
180
220
|
| <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
|
181
221
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
182
|
-
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|
|
222
|
+
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|__Deprecated__
|
183
223
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
|
184
|
-
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|
|
224
|
+
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|__Deprecated__
|
185
225
|
| <<plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No
|
186
226
|
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |<<array,array>>|No
|
227
|
+
| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |<<array,array>>|No
|
228
|
+
| <<plugins-{type}s-{plugin}-ssl_client_authentication>> |<<string,string>>, one of `["none", "optional", "required"]`|No
|
229
|
+
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
187
230
|
| <<plugins-{type}s-{plugin}-ssl_handshake_timeout>> |<<number,number>>|No
|
188
231
|
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
189
232
|
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
190
|
-
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|
|
233
|
+
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|__Deprecated__
|
191
234
|
| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<array,array>>|No
|
192
|
-
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|
|
235
|
+
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|__Deprecated__
|
193
236
|
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|__Deprecated__
|
194
237
|
| <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|__Deprecated__
|
195
238
|
|=======================================================================
|
@@ -239,14 +282,72 @@ Close Idle clients after X seconds of inactivity.
|
|
239
282
|
|
240
283
|
Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
|
241
284
|
|
285
|
+
[id="plugins-{type}s-{plugin}-enrich"]
|
286
|
+
===== `enrich`
|
287
|
+
|
288
|
+
* Value type is <<string,string>>
|
289
|
+
** A <<list,list>> can also be provided
|
290
|
+
** Configures which enrichments are applied to each event
|
291
|
+
** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments.
|
292
|
+
** Supported values are:
|
293
|
+
+
|
294
|
+
[cols="2l,5"]
|
295
|
+
|=======================================================================
|
296
|
+
|Enrichment | Description
|
297
|
+
|
298
|
+
| codec_metadata | Information about how the codec transformed a sequence of bytes into
|
299
|
+
this Event, such as _which_ codec was used. Also, if no <<codec>> is
|
300
|
+
explicitly specified, _excluding_ `codec_metadata` from `enrich` will
|
301
|
+
disable `ecs_compatibility` for this plugin.
|
302
|
+
| source_metadata | Information about the _source_ of the event, such as the IP address
|
303
|
+
of the inbound connection this input received the event from and the
|
304
|
+
name of the Logstash host that processed the event
|
305
|
+
| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from,
|
306
|
+
such as identity information from the SSL client certificate that was
|
307
|
+
presented when establishing a connection to this input
|
308
|
+
| all | _alias_ to include _all_ available enrichments (including additional
|
309
|
+
enrichments introduced in future versions of this plugin)
|
310
|
+
| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_
|
311
|
+
defining <<codec>> with this option will not disable the `ecs_compatibility`,
|
312
|
+
instead it relies on pipeline or codec `ecs_compatibility` configuration.
|
313
|
+
|=======================================================================
|
314
|
+
|
315
|
+
|
316
|
+
**Example:**
|
317
|
+
|
318
|
+
This configuration disables _all_ enrichments:
|
319
|
+
|
320
|
+
["source",subs="attributes"]
|
321
|
+
--------------------------------------------------
|
322
|
+
input {
|
323
|
+
beats {
|
324
|
+
port => 5044
|
325
|
+
enrich => none
|
326
|
+
}
|
327
|
+
}
|
328
|
+
--------------------------------------------------
|
329
|
+
|
330
|
+
Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others):
|
331
|
+
|
332
|
+
|
333
|
+
["source",subs="attributes"]
|
334
|
+
--------------------------------------------------
|
335
|
+
input {
|
336
|
+
beats {
|
337
|
+
port => 5044
|
338
|
+
enrich => [source_metadata, ssl_peer_metadata]
|
339
|
+
}
|
340
|
+
}
|
341
|
+
--------------------------------------------------
|
342
|
+
|
242
343
|
[id="plugins-{type}s-{plugin}-executor_threads"]
|
243
344
|
===== `executor_threads`
|
244
345
|
|
245
346
|
* Value type is <<number,number>>
|
246
347
|
* Default value is equal to the number of CPU cores (1 executor thread per CPU core).
|
247
348
|
|
248
|
-
The number of threads to be used to process incoming
|
249
|
-
By default, the
|
349
|
+
The number of threads to be used to process incoming {plugin-uc} requests.
|
350
|
+
By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores.
|
250
351
|
These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management.
|
251
352
|
Parsing the Lumberjack protocol is offloaded to a dedicated thread pool.
|
252
353
|
|
@@ -268,6 +369,8 @@ The IP address to listen on.
|
|
268
369
|
[id="plugins-{type}s-{plugin}-include_codec_tag"]
|
269
370
|
===== `include_codec_tag`
|
270
371
|
|
372
|
+
deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
|
373
|
+
|
271
374
|
* Value type is <<boolean,boolean>>
|
272
375
|
* Default value is `true`
|
273
376
|
|
@@ -284,6 +387,7 @@ The port to listen on.
|
|
284
387
|
|
285
388
|
[id="plugins-{type}s-{plugin}-ssl"]
|
286
389
|
===== `ssl`
|
390
|
+
deprecated[6.6.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
|
287
391
|
|
288
392
|
* Value type is <<boolean,boolean>>
|
289
393
|
* Default value is `false`
|
@@ -308,8 +412,8 @@ SSL certificate to use.
|
|
308
412
|
|
309
413
|
Validate client certificates against these authorities.
|
310
414
|
You can define multiple files or paths. All the certificates will
|
311
|
-
be read and added to the trust store. You need to configure the
|
312
|
-
to `
|
415
|
+
be read and added to the trust store. You need to configure the <<plugins-{type}s-{plugin}-ssl_client_authentication>>
|
416
|
+
to `optional` or `required` to enable the verification.
|
313
417
|
|
314
418
|
[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
|
315
419
|
===== `ssl_cipher_suites`
|
@@ -322,6 +426,27 @@ This default list applies for OpenJDK 11.0.14 and higher.
|
|
322
426
|
For older JDK versions, the default list includes only suites supported by that version.
|
323
427
|
For example, the ChaCha20 family of ciphers is not supported in older versions.
|
324
428
|
|
429
|
+
[id="plugins-{type}s-{plugin}-ssl_client_authentication"]
|
430
|
+
===== `ssl_client_authentication`
|
431
|
+
|
432
|
+
* Value can be any of: `none`, `optional`, `required`
|
433
|
+
* Default value is `"none"`
|
434
|
+
|
435
|
+
Controls the server's behavior in regard to requesting a certificate from client connections:
|
436
|
+
`required` forces a client to present a certificate, while `optional` requests a client certificate
|
437
|
+
but the client is not required to present one. Defaults to `none`, which disables the client authentication.
|
438
|
+
|
439
|
+
NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> is set.
|
440
|
+
|
441
|
+
[id="plugins-{type}s-{plugin}-ssl_enabled"]
|
442
|
+
===== `ssl_enabled`
|
443
|
+
|
444
|
+
* Value type is <<boolean,boolean>>
|
445
|
+
* Default value is `false`
|
446
|
+
|
447
|
+
Events are by default sent in plain text. You can enable encryption by setting `ssl_enabled` to true and configuring
|
448
|
+
the <<plugins-{type}s-{plugin}-ssl_certificate>> and <<plugins-{type}s-{plugin}-ssl_key>> options.
|
449
|
+
|
325
450
|
[id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
|
326
451
|
===== `ssl_handshake_timeout`
|
327
452
|
|
@@ -357,12 +482,14 @@ SSL key passphrase to use.
|
|
357
482
|
[id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
|
358
483
|
===== `ssl_peer_metadata`
|
359
484
|
|
485
|
+
deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
|
486
|
+
|
360
487
|
* Value type is <<boolean,boolean>>
|
361
488
|
* Default value is `false`
|
362
489
|
|
363
490
|
Enables storing client certificate information in event's metadata.
|
364
491
|
|
365
|
-
This option is only valid when
|
492
|
+
This option is only valid when <<plugins-{type}s-{plugin}-ssl_client_authentication>> is set to `optional` or `required`.
|
366
493
|
|
367
494
|
[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
|
368
495
|
===== `ssl_supported_protocols`
|
@@ -383,11 +510,13 @@ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.
|
|
383
510
|
|
384
511
|
[id="plugins-{type}s-{plugin}-ssl_verify_mode"]
|
385
512
|
===== `ssl_verify_mode`
|
513
|
+
deprecated[6.6.0, Replaced by <<plugins-{type}s-{plugin}-ssl_client_authentication>>]
|
386
514
|
|
387
515
|
* Value can be any of: `none`, `peer`, `force_peer`
|
388
516
|
* Default value is `"none"`
|
389
517
|
|
390
|
-
By default the server doesn't do any client verification.
|
518
|
+
By default, the server doesn't do any client verification. If the <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
|
519
|
+
is configured, and no value or `none` is provided for this option, it defaults to `force_peer` instead of `none`.
|
391
520
|
|
392
521
|
`peer` will make the server ask the client to provide a certificate.
|
393
522
|
If the client provides a certificate, it will be validated.
|
@@ -395,7 +524,7 @@ If the client provides a certificate, it will be validated.
|
|
395
524
|
`force_peer` will make the server ask the client to provide a certificate.
|
396
525
|
If the client doesn't provide a certificate, the connection will be closed.
|
397
526
|
|
398
|
-
This option needs to be used with
|
527
|
+
This option needs to be used with <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> and a defined list of CAs.
|
399
528
|
|
400
529
|
[id="plugins-{type}s-{plugin}-tls_max_version"]
|
401
530
|
===== `tls_max_version`
|
@@ -416,7 +545,6 @@ The minimum TLS version allowed for the encrypted connections.
|
|
416
545
|
The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3
|
417
546
|
|
418
547
|
|
419
|
-
|
420
548
|
[id="plugins-{type}s-{plugin}-common-options"]
|
421
549
|
include::{include_path}/{type}.asciidoc[]
|
422
550
|
|
@@ -32,11 +32,14 @@ module LogStash module Inputs class Beats
|
|
32
32
|
|
33
33
|
def onNewMessage(ctx, message)
|
34
34
|
hash = message.getData
|
35
|
-
ip_address = ip_address(ctx)
|
36
35
|
|
37
|
-
|
38
|
-
|
36
|
+
if @input.include_source_metadata?
|
37
|
+
ip_address = ip_address(ctx)
|
38
|
+
unless ip_address.nil? || hash['@metadata'].nil?
|
39
|
+
set_nested(hash, @input.field_hostip, ip_address)
|
40
|
+
end
|
39
41
|
end
|
42
|
+
|
40
43
|
target_field = extract_target_field(hash)
|
41
44
|
|
42
45
|
extract_tls_peer(hash, ctx)
|