logstash-input-beats 6.4.4-java → 6.5.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/VERSION +1 -1
- data/docs/index.asciidoc +123 -22
- data/lib/logstash/inputs/beats/message_listener.rb +6 -3
- data/lib/logstash/inputs/beats.rb +58 -3
- data/lib/logstash-input-beats_jars.rb +1 -1
- data/lib/tasks/test.rake +1 -1
- data/logstash-input-beats.gemspec +1 -0
- data/spec/inputs/beats_spec.rb +181 -0
- data/spec/spec_helper.rb +0 -1
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/{6.4.4/logstash-input-beats-6.4.4.jar → 6.5.0/logstash-input-beats-6.5.0.jar} +0 -0
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1801dd24702dc3b6d751e679ecdeb78b34b2ffc23ad5ec2e236f2642ba4a0ddd
|
4
|
+
data.tar.gz: '0879bc47974cef2918e5c2725458bca543f2beab38e9c8f523ba848e468eb0cd'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d779990717562cb6db36423821e3471c0e8f582a06738a248c64346c54e1e0f38cea789146941dde2c2c784e2ca1daaac82418c9ba3cbd6029a7fd5b2643f323
|
7
|
+
data.tar.gz: e6c6c0164ff7c827e54ad51d71832c2b79eed3ad44452ec7cc9dad86475faf25c1127756425dbb1a1783eb8d7427b280acfdfcd8088b957b848b024f40e51b47
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 6.5.0
|
2
|
+
- An enrichment `enrich` option added to control ECS passthrough. `ssl_peer_metadata` and `include_codec_tag` configurations are deprecated and can be managed through the `enrich` [#464](https://github.com/logstash-plugins/logstash-input-beats/pull/464)
|
3
|
+
|
1
4
|
## 6.4.4
|
2
5
|
- Updates Netty dependency to 4.1.87 [#466](https://github.com/logstash-plugins/logstash-input-beats/pull/466)
|
3
6
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
6.
|
1
|
+
6.5.0
|
data/docs/index.asciidoc
CHANGED
@@ -143,26 +143,65 @@ endif::[]
|
|
143
143
|
|
144
144
|
|
145
145
|
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
146
|
-
==== Event
|
146
|
+
==== Event enrichment and the Elastic Common Schema (ECS)
|
147
147
|
|
148
|
-
When decoding {plugin-uc} events, this plugin
|
149
|
-
|
150
|
-
`ip_address` containing the remote address of the client's connection. When
|
151
|
-
<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled
|
152
|
-
these are now moved in ECS compatible namespace. Here's how
|
153
|
-
<<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> affects
|
154
|
-
output.
|
148
|
+
When decoding {plugin-uc} events, this plugin enriches each event with metadata about the event's source, making this information available during further processing.
|
149
|
+
You can use the <<plugins-{type}s-{plugin}-enrich>> option to activate or deactivate individual enrichment categories.
|
155
150
|
|
156
|
-
|
151
|
+
The location of these enrichment fields depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled:
|
152
|
+
|
153
|
+
- When ECS compatibility is _enabled_, enrichment fields are added in an ECS-compatible structure.
|
154
|
+
- When ECS compatibility is _disabled_, enrichment fields are added in a way that is backward-compatible with this plugin, but is known to clash with the Elastic Common Schema.
|
155
|
+
|
156
|
+
|
157
|
+
.`source_metadata`
|
158
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
159
|
+
|=======================================================================
|
160
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
161
|
+
|
162
|
+
|[@metadata][input][beats][host][name]
|
163
|
+
|[host]
|
164
|
+
|Name or address of the {plugin-singular} host
|
165
|
+
|
166
|
+
|[@metadata][input][beats][host][ip]
|
167
|
+
|[@metadata][ip_address]
|
168
|
+
|IP address of the {plugin-uc} client that connected to this input
|
169
|
+
|=======================================================================
|
170
|
+
|
171
|
+
.`ssl_peer_metadata`
|
172
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
173
|
+
|=======================================================================
|
174
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
175
|
+
|
176
|
+
|[@metadata][tls_peer][status]
|
177
|
+
|[@metadata][tls_peer][status]
|
178
|
+
|Contains "verified" or "unverified" label; available when SSL is enabled.
|
179
|
+
|
180
|
+
|[@metadata][input][beats][tls][version_protocol]
|
181
|
+
|[@metadata][tls_peer][protocol]
|
182
|
+
|Contains the TLS version used (such as `TLSv1.2`); available when SSL status is "verified"
|
183
|
+
|
184
|
+
|[@metadata][input][beats][tls][client][subject]
|
185
|
+
|[@metadata][tls_peer][subject]
|
186
|
+
|Contains the identity name of the remote end (such as `CN=artifacts-no-kpi.elastic.co`); available when SSL status is "verified"
|
187
|
+
|
188
|
+
|[@metadata][input][beats][tls][cipher]
|
189
|
+
|[@metadata][tls_peer][cipher_suite]
|
190
|
+
|Contains the name of cipher suite used (such as `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`); available when SSL status is "verified"
|
191
|
+
|=======================================================================
|
192
|
+
|
193
|
+
.`codec_metadata`
|
194
|
+
[cols="<l,<l,<e",caption="Enrichment category:"]
|
157
195
|
|=======================================================================
|
158
|
-
|ECS `
|
159
|
-
|
160
|
-
|[
|
161
|
-
|[
|
162
|
-
|
|
163
|
-
|
164
|
-
|[
|
165
|
-
|
|
196
|
+
|ECS `v1`, `v8` |ECS `disabled` |Description
|
197
|
+
|
198
|
+
|[tag]
|
199
|
+
|[tag]
|
200
|
+
|Contains `beats_input_codec_XXX_applied` where `XXX` is the name of the codec
|
201
|
+
|
202
|
+
|[event][original]
|
203
|
+
e|N/A
|
204
|
+
|When ECS is enabled, even if `[event][original]` field does not already exist on the event being processed, this plugin's *default codec* ensures that the field is populated using the bytes as-processed.
|
166
205
|
|=======================================================================
|
167
206
|
|
168
207
|
[id="plugins-{type}s-{plugin}-options"]
|
@@ -177,9 +216,10 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
177
216
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|__Deprecated__
|
178
217
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
179
218
|
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
219
|
+
| <<plugins-{type}s-{plugin}-enrich>> |<<string,string>>|No
|
180
220
|
| <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
|
181
221
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
182
|
-
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|
|
222
|
+
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|__Deprecated__
|
183
223
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
|
184
224
|
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
|
185
225
|
| <<plugins-{type}s-{plugin}-ssl_certificate>> |a valid filesystem path|No
|
@@ -187,7 +227,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
187
227
|
| <<plugins-{type}s-{plugin}-ssl_handshake_timeout>> |<<number,number>>|No
|
188
228
|
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
189
229
|
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
190
|
-
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|
|
230
|
+
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|__Deprecated__
|
191
231
|
| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<array,array>>|No
|
192
232
|
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
193
233
|
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|__Deprecated__
|
@@ -239,14 +279,72 @@ Close Idle clients after X seconds of inactivity.
|
|
239
279
|
|
240
280
|
Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
|
241
281
|
|
282
|
+
[id="plugins-{type}s-{plugin}-enrich"]
|
283
|
+
===== `enrich`
|
284
|
+
|
285
|
+
* Value type is <<string,string>>
|
286
|
+
** A <<list,list>> can also be provided
|
287
|
+
** Configures which enrichments are applied to each event
|
288
|
+
** Default value is `[codec_metadata, source_metadata]` that may be extended in future versions of this plugin to include additional enrichments.
|
289
|
+
** Supported values are:
|
290
|
+
+
|
291
|
+
[cols="2l,5"]
|
292
|
+
|=======================================================================
|
293
|
+
|Enrichment | Description
|
294
|
+
|
295
|
+
| codec_metadata | Information about how the codec transformed a sequence of bytes into
|
296
|
+
this Event, such as _which_ codec was used. Also, if no <<codec>> is
|
297
|
+
explicitly specified, _excluding_ `codec_metadata` from `enrich` will
|
298
|
+
disable `ecs_compatibility` for this plugin.
|
299
|
+
| source_metadata | Information about the _source_ of the event, such as the IP address
|
300
|
+
of the inbound connection this input received the event from and the
|
301
|
+
name of the Logstash host that processed the event
|
302
|
+
| ssl_peer_metadata | Detailed information about the _SSL peer_ we received the event from,
|
303
|
+
such as identity information from the SSL client certificate that was
|
304
|
+
presented when establishing a connection to this input
|
305
|
+
| all | _alias_ to include _all_ available enrichments (including additional
|
306
|
+
enrichments introduced in future versions of this plugin)
|
307
|
+
| none | _alias_ to _exclude_ all available enrichments. Note that, _explicitly_
|
308
|
+
defining <<codec>> with this option will not disable the `ecs_compatibility`,
|
309
|
+
instead it relies on pipeline or codec `ecs_compatibility` configuration.
|
310
|
+
|=======================================================================
|
311
|
+
|
312
|
+
|
313
|
+
**Example:**
|
314
|
+
|
315
|
+
This configuration disables _all_ enrichments:
|
316
|
+
|
317
|
+
["source",subs="attributes"]
|
318
|
+
--------------------------------------------------
|
319
|
+
input {
|
320
|
+
beats {
|
321
|
+
port => 5044
|
322
|
+
enrich => none
|
323
|
+
}
|
324
|
+
}
|
325
|
+
--------------------------------------------------
|
326
|
+
|
327
|
+
Or, to explicitly enable _only_ `source_metadata` and `ssl_peer_metadata` (disabling all others):
|
328
|
+
|
329
|
+
|
330
|
+
["source",subs="attributes"]
|
331
|
+
--------------------------------------------------
|
332
|
+
input {
|
333
|
+
beats {
|
334
|
+
port => 5044
|
335
|
+
enrich => [source_metadata, ssl_peer_metadata]
|
336
|
+
}
|
337
|
+
}
|
338
|
+
--------------------------------------------------
|
339
|
+
|
242
340
|
[id="plugins-{type}s-{plugin}-executor_threads"]
|
243
341
|
===== `executor_threads`
|
244
342
|
|
245
343
|
* Value type is <<number,number>>
|
246
344
|
* Default value is equal to the number of CPU cores (1 executor thread per CPU core).
|
247
345
|
|
248
|
-
The number of threads to be used to process incoming
|
249
|
-
By default, the
|
346
|
+
The number of threads to be used to process incoming {plugin-uc} requests.
|
347
|
+
By default, the {plugin-uc} input creates a number of threads equal to the number of CPU cores.
|
250
348
|
These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management.
|
251
349
|
Parsing the Lumberjack protocol is offloaded to a dedicated thread pool.
|
252
350
|
|
@@ -268,6 +366,8 @@ The IP address to listen on.
|
|
268
366
|
[id="plugins-{type}s-{plugin}-include_codec_tag"]
|
269
367
|
===== `include_codec_tag`
|
270
368
|
|
369
|
+
deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
|
370
|
+
|
271
371
|
* Value type is <<boolean,boolean>>
|
272
372
|
* Default value is `true`
|
273
373
|
|
@@ -357,6 +457,8 @@ SSL key passphrase to use.
|
|
357
457
|
[id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
|
358
458
|
===== `ssl_peer_metadata`
|
359
459
|
|
460
|
+
deprecated[6.5.0, Replaced by <<plugins-{type}s-{plugin}-enrich>>]
|
461
|
+
|
360
462
|
* Value type is <<boolean,boolean>>
|
361
463
|
* Default value is `false`
|
362
464
|
|
@@ -416,7 +518,6 @@ The minimum TLS version allowed for the encrypted connections.
|
|
416
518
|
The value must be one of the following: 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3
|
417
519
|
|
418
520
|
|
419
|
-
|
420
521
|
[id="plugins-{type}s-{plugin}-common-options"]
|
421
522
|
include::{include_path}/{type}.asciidoc[]
|
422
523
|
|
@@ -32,11 +32,14 @@ module LogStash module Inputs class Beats
|
|
32
32
|
|
33
33
|
def onNewMessage(ctx, message)
|
34
34
|
hash = message.getData
|
35
|
-
ip_address = ip_address(ctx)
|
36
35
|
|
37
|
-
|
38
|
-
|
36
|
+
if @input.include_source_metadata?
|
37
|
+
ip_address = ip_address(ctx)
|
38
|
+
unless ip_address.nil? || hash['@metadata'].nil?
|
39
|
+
set_nested(hash, @input.field_hostip, ip_address)
|
40
|
+
end
|
39
41
|
end
|
42
|
+
|
40
43
|
target_field = extract_target_field(hash)
|
41
44
|
|
42
45
|
extract_tls_peer(hash, ctx)
|
@@ -6,6 +6,7 @@ require "logstash/codecs/multiline"
|
|
6
6
|
require "logstash/util"
|
7
7
|
require "logstash-input-beats_jars"
|
8
8
|
require "logstash/plugin_mixins/ecs_compatibility_support"
|
9
|
+
require 'logstash/plugin_mixins/plugin_factory_support'
|
9
10
|
require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
10
11
|
require_relative "beats/patch"
|
11
12
|
|
@@ -58,6 +59,8 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
58
59
|
|
59
60
|
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
60
61
|
|
62
|
+
include LogStash::PluginMixins::PluginFactorySupport
|
63
|
+
|
61
64
|
config_name "beats"
|
62
65
|
|
63
66
|
default :codec, "plain"
|
@@ -104,9 +107,9 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
104
107
|
|
105
108
|
# Enables storing client certificate information in event's metadata. You need
|
106
109
|
# to configure the `ssl_verify_mode` to `peer` or `force_peer` to enable this.
|
107
|
-
config :ssl_peer_metadata, :validate => :boolean, :default => false
|
110
|
+
config :ssl_peer_metadata, :validate => :boolean, :default => false, :deprecated => "use `enrich` option to configure which enrichments to perform"
|
108
111
|
|
109
|
-
config :include_codec_tag, :validate => :boolean, :default => true
|
112
|
+
config :include_codec_tag, :validate => :boolean, :default => true, :deprecated => "use `enrich` option to configure which enrichments to perform"
|
110
113
|
|
111
114
|
# Time in milliseconds for an incomplete ssl handshake to timeout
|
112
115
|
config :ssl_handshake_timeout, :validate => :number, :default => 10000
|
@@ -136,8 +139,22 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
136
139
|
# 1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
137
140
|
config :tls_max_version, :validate => :number, :default => TLS.max.version, :deprecated => "Set 'ssl_supported_protocols' instead."
|
138
141
|
|
142
|
+
ENRICH_DEFAULTS = {
|
143
|
+
'source_metadata' => true,
|
144
|
+
'codec_metadata' => true,
|
145
|
+
'ssl_peer_metadata' => false,
|
146
|
+
}.freeze
|
147
|
+
|
148
|
+
ENRICH_ALL = ENRICH_DEFAULTS.keys.freeze
|
149
|
+
ENRICH_DEFAULT = ENRICH_DEFAULTS.select { |_,v| v }.keys.freeze
|
150
|
+
ENRICH_NONE = ['none'].freeze
|
151
|
+
ENRICH_ALIASES = %w(none all)
|
152
|
+
|
153
|
+
config :enrich, :validate => (ENRICH_ALL | ENRICH_ALIASES), :list => true
|
154
|
+
|
139
155
|
attr_reader :field_hostname, :field_hostip
|
140
156
|
attr_reader :field_tls_protocol_version, :field_tls_peer_subject, :field_tls_cipher
|
157
|
+
attr_reader :include_source_metadata
|
141
158
|
|
142
159
|
def register
|
143
160
|
# For Logstash 2.4 we need to make sure that the logger is correctly set for the
|
@@ -189,6 +206,23 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
189
206
|
@logger.warn("configured ssl_key => #{@ssl_key.inspect} will not be used") if @ssl_key
|
190
207
|
end
|
191
208
|
|
209
|
+
active_enrichments = resolve_enriches
|
210
|
+
|
211
|
+
@include_source_metadata = active_enrichments.include?('source_metadata')
|
212
|
+
@include_codec_tag = original_params.include?('include_codec_tag') ? params['include_codec_tag'] : active_enrichments.include?('codec_metadata')
|
213
|
+
@ssl_peer_metadata = original_params.include?('ssl_peer_metadata') ? params['ssl_peer_metadata'] : active_enrichments.include?('ssl_peer_metadata')
|
214
|
+
|
215
|
+
# intentionally ask users to provide codec when they want to use the codec metadata
|
216
|
+
# second layer enrich is also a controller, provide enrich => ['codec_metadata' or/with 'source_metadata'] with codec if you override event original
|
217
|
+
unless active_enrichments.include?('codec_metadata')
|
218
|
+
if original_params.include?('codec')
|
219
|
+
@logger.warn("An explicit `codec` is specified but `enrich` does not include `codec_metadata`. ECS compatibility will remain aligned on the pipeline or codec's `ecs_compatibility` (enabled by default).")
|
220
|
+
else
|
221
|
+
@codec = plugin_factory.codec('plain').new('ecs_compatibility' => 'disabled')
|
222
|
+
@logger.debug('Disabling `ecs_compatibility` for the default codec since `enrich` configuration does not include `codec_metadata` and no explicit codec is set.')
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
192
226
|
# Logstash 6.x breaking change (introduced with 4.0.0 of this gem)
|
193
227
|
if @codec.kind_of? LogStash::Codecs::Multiline
|
194
228
|
configuration_error "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html"
|
@@ -246,7 +280,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
246
280
|
end
|
247
281
|
|
248
282
|
def client_authentication_metadata?
|
249
|
-
@ssl_peer_metadata && ssl_configured? && client_authentification?
|
283
|
+
@ssl_peer_metadata && ssl_configured? && client_authentification?
|
250
284
|
end
|
251
285
|
|
252
286
|
def client_authentication_required?
|
@@ -257,6 +291,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
257
291
|
@ssl_verify_mode == "force_peer" || @ssl_verify_mode == "peer"
|
258
292
|
end
|
259
293
|
|
294
|
+
def include_source_metadata?
|
295
|
+
return @include_source_metadata
|
296
|
+
end
|
297
|
+
|
260
298
|
private
|
261
299
|
|
262
300
|
def new_ssl_handshake_provider(ssl_context_builder)
|
@@ -303,4 +341,21 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
303
341
|
error_details
|
304
342
|
end
|
305
343
|
|
344
|
+
def resolve_enriches
|
345
|
+
deprecated_flags_provided = %w(ssl_peer_metadata include_codec_tag) & original_params.keys
|
346
|
+
if deprecated_flags_provided.any? && original_params.include?('enrich')
|
347
|
+
raise LogStash::ConfigurationError, "both `enrich` and (deprecated) #{deprecated_flags_provided.join(',')} were provided; use only `enrich`"
|
348
|
+
end
|
349
|
+
|
350
|
+
aliases_provided = ENRICH_ALIASES & (@enrich || [])
|
351
|
+
if aliases_provided.any? && @enrich.size > 1
|
352
|
+
raise LogStash::ConfigurationError, "when an alias is provided to `enrich`, it must be the only value given (got: #{@enrich.inspect}, including #{aliases_provided.size > 1 ? 'aliases' : 'alias'} #{aliases_provided.join(',')})"
|
353
|
+
end
|
354
|
+
|
355
|
+
return ENRICH_ALL if aliases_provided.include?('all')
|
356
|
+
return ENRICH_NONE if aliases_provided.include?('none')
|
357
|
+
return ENRICH_DEFAULT unless original_params.include?('enrich')
|
358
|
+
|
359
|
+
return @enrich
|
360
|
+
end
|
306
361
|
end
|
@@ -8,4 +8,4 @@ require_jar('io.netty', 'netty-transport', '4.1.87.Final')
|
|
8
8
|
require_jar('io.netty', 'netty-handler', '4.1.87.Final')
|
9
9
|
require_jar('io.netty', 'netty-transport-native-unix-common', '4.1.87.Final')
|
10
10
|
require_jar('org.javassist', 'javassist', '3.24.0-GA')
|
11
|
-
require_jar('org.logstash.beats', 'logstash-input-beats', '6.
|
11
|
+
require_jar('org.logstash.beats', 'logstash-input-beats', '6.5.0')
|
data/lib/tasks/test.rake
CHANGED
@@ -28,7 +28,7 @@ namespace :test do
|
|
28
28
|
end
|
29
29
|
|
30
30
|
namespace :setup do
|
31
|
-
desc "Download
|
31
|
+
desc "Download latest stable version of Logstash-forwarder"
|
32
32
|
task :lsf do
|
33
33
|
destination = File.join(VENDOR_PATH, "logstash-forwarder")
|
34
34
|
FileUtils.rm_rf(destination)
|
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
|
|
29
29
|
s.add_runtime_dependency 'jar-dependencies', '~> 0.3', '>= 0.3.4'
|
30
30
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
|
31
31
|
s.add_runtime_dependency 'logstash-mixin-event_support', '~>1.0'
|
32
|
+
s.add_runtime_dependency 'logstash-mixin-plugin_factory_support', '~>1.0'
|
32
33
|
|
33
34
|
s.add_development_dependency "flores", "~>0.0.6"
|
34
35
|
s.add_development_dependency "rspec"
|
data/spec/inputs/beats_spec.rb
CHANGED
@@ -200,6 +200,187 @@ describe LogStash::Inputs::Beats do
|
|
200
200
|
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
|
201
201
|
end
|
202
202
|
end
|
203
|
+
|
204
|
+
context "enrich configuration" do
|
205
|
+
# We define a shared example for each enrichment type that can independently
|
206
|
+
# validate whether that enrichment is effectively enabled or disabled.
|
207
|
+
# - "#{enrichment} enabled"
|
208
|
+
# - "#{enrichment} disabled"
|
209
|
+
|
210
|
+
let(:registered_plugin) { plugin.tap(&:register) }
|
211
|
+
|
212
|
+
shared_examples "source_metadata enabled" do
|
213
|
+
it "is configured to enrich source metadata" do
|
214
|
+
expect(registered_plugin.include_source_metadata).to be true
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
shared_examples "source_metadata disabled" do
|
219
|
+
it "is configured to NOT enrich source metadata" do
|
220
|
+
expect(registered_plugin.include_source_metadata).to be false
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
shared_examples "include codec tag" do
|
225
|
+
it "is configured to include the codec tag" do
|
226
|
+
expect(registered_plugin.include_codec_tag).to be true
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
shared_examples "exclude codec tag" do
|
231
|
+
it "is configured to NOT include the codec tag" do
|
232
|
+
expect(registered_plugin.include_codec_tag).to be false
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
shared_examples "default codec configured to avoid metadata" do
|
237
|
+
it "configures the default codec to NOT enrich codec metadata" do
|
238
|
+
fail("spec setup error: not compatible with explicitly-given codec") if config.include?('codec')
|
239
|
+
# note: disabling ECS is an _implementation detail_ of how we prevent
|
240
|
+
# the codec from enriching the event with [event][original]
|
241
|
+
expect(registered_plugin.codec.original_params).to include('ecs_compatibility' => 'disabled')
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
shared_examples "codec is untouched" do
|
246
|
+
it "does NOT configure the codec to avoid enriching codec metadata" do
|
247
|
+
# note: disabling ECS is an _implementation detail_ of how we prevent
|
248
|
+
# the codec from enriching the event with [event][original], so we ensure
|
249
|
+
# the absence of the setting.
|
250
|
+
expect(registered_plugin.codec.original_params).to_not include('ecs_compatibility')
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
shared_examples "codec_metadata enabled" do
|
255
|
+
include_examples "include codec tag"
|
256
|
+
include_examples "codec is untouched"
|
257
|
+
end
|
258
|
+
|
259
|
+
shared_examples "codec_metadata disabled" do
|
260
|
+
include_examples "exclude codec tag"
|
261
|
+
include_examples "default codec configured to avoid metadata"
|
262
|
+
|
263
|
+
context "with an explicitly-provided codec" do
|
264
|
+
let(:config) { super().merge("codec" => "plain") }
|
265
|
+
|
266
|
+
include_examples "exclude codec tag"
|
267
|
+
include_examples "codec is untouched"
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
shared_examples "ssl_peer_metadata enabled" do
|
272
|
+
it "is configured to enrich ssl_peer_metadata" do
|
273
|
+
expect(registered_plugin.ssl_peer_metadata).to be_truthy
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
shared_examples "ssl_peer_metadata disabled" do
|
278
|
+
it "is configured to NOT enrich ssl_peer_metadata" do
|
279
|
+
expect(registered_plugin.ssl_peer_metadata).to be_falsey
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
shared_examples "reject deprecated enrichment flags" do
|
284
|
+
context "with deprecated `ssl_peer_metadata`" do
|
285
|
+
let(:config) { super().merge("ssl_peer_metadata" => true) }
|
286
|
+
it 'rejects the configuration with a helpful error message' do
|
287
|
+
expect { plugin.register }.to raise_exception(LogStash::ConfigurationError, "both `enrich` and (deprecated) ssl_peer_metadata were provided; use only `enrich`")
|
288
|
+
end
|
289
|
+
end
|
290
|
+
context "with deprecated `include_codec_tag`" do
|
291
|
+
let(:config) { super().merge("include_codec_tag" => false) }
|
292
|
+
it 'rejects the configuration with a helpful error message' do
|
293
|
+
expect { plugin.register }.to raise_exception(LogStash::ConfigurationError, "both `enrich` and (deprecated) include_codec_tag were provided; use only `enrich`")
|
294
|
+
end
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
context "when `enrich` is NOT provided" do
|
299
|
+
# validate defaults
|
300
|
+
include_examples "codec_metadata enabled"
|
301
|
+
include_examples "source_metadata enabled"
|
302
|
+
include_examples "ssl_peer_metadata disabled"
|
303
|
+
|
304
|
+
# validate interaction with deprecated settings
|
305
|
+
context "with deprecated `ssl_peer_metadata => true`" do
|
306
|
+
let(:config) { super().merge("ssl_peer_metadata" => true) }
|
307
|
+
|
308
|
+
# intended delta
|
309
|
+
include_examples "ssl_peer_metadata enabled"
|
310
|
+
|
311
|
+
# ensure no side-effects
|
312
|
+
include_examples "codec_metadata enabled"
|
313
|
+
include_examples "source_metadata enabled"
|
314
|
+
end
|
315
|
+
|
316
|
+
context "with deprecated `include_codec_tag => false`" do
|
317
|
+
let(:config) { super().merge("include_codec_tag" => false) }
|
318
|
+
|
319
|
+
# intended delta
|
320
|
+
include_examples "exclude codec tag"
|
321
|
+
include_examples "codec is untouched"
|
322
|
+
|
323
|
+
# ensure no side-effects
|
324
|
+
include_examples "source_metadata enabled"
|
325
|
+
include_examples "ssl_peer_metadata disabled"
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
# validate aliases
|
330
|
+
context "alias resolution" do
|
331
|
+
context "with alias `enrich => all`" do
|
332
|
+
let(:config) { super().merge("enrich" => "all") }
|
333
|
+
|
334
|
+
include_examples "codec_metadata enabled"
|
335
|
+
include_examples "source_metadata enabled"
|
336
|
+
include_examples "ssl_peer_metadata enabled"
|
337
|
+
|
338
|
+
include_examples "reject deprecated enrichment flags"
|
339
|
+
end
|
340
|
+
|
341
|
+
context "with alias `enrich => none`" do
|
342
|
+
let(:config) { super().merge("enrich" => "none") }
|
343
|
+
|
344
|
+
include_examples "codec_metadata disabled"
|
345
|
+
include_examples "source_metadata disabled"
|
346
|
+
include_examples "ssl_peer_metadata disabled"
|
347
|
+
|
348
|
+
include_examples "reject deprecated enrichment flags"
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
available_enrichments = %w(
|
353
|
+
codec_metadata
|
354
|
+
source_metadata
|
355
|
+
ssl_peer_metadata
|
356
|
+
)
|
357
|
+
shared_examples "enrich activations" do |enrich_arg|
|
358
|
+
activated = Array(enrich_arg)
|
359
|
+
context "with `enrich => #{enrich_arg}`" do
|
360
|
+
let(:config) { super().merge("enrich" => enrich_arg) }
|
361
|
+
|
362
|
+
available_enrichments.each do |enrichment|
|
363
|
+
include_examples "#{enrichment} #{activated.include?(enrichment) ? 'enabled' : 'disabled'}"
|
364
|
+
end
|
365
|
+
|
366
|
+
include_examples "reject deprecated enrichment flags"
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
# ensure explicit empty-list does not activate defaults
|
371
|
+
include_examples "enrich activations", []
|
372
|
+
|
373
|
+
# ensure single enrichment does not activate others
|
374
|
+
available_enrichments.each do |single_active_enrichment|
|
375
|
+
include_examples "enrich activations", single_active_enrichment # single
|
376
|
+
include_examples "enrich activations", [single_active_enrichment] # list-of-one
|
377
|
+
end
|
378
|
+
|
379
|
+
# ensure any combination of two enrichment categories activates only those two
|
380
|
+
available_enrichments.combination(2) do |active_enrichments|
|
381
|
+
include_examples "enrich activations", active_enrichments
|
382
|
+
end
|
383
|
+
end
|
203
384
|
end
|
204
385
|
|
205
386
|
context "tls meta-data" do
|
data/spec/spec_helper.rb
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-beats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.5.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-01-
|
11
|
+
date: 2023-01-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -134,6 +134,20 @@ dependencies:
|
|
134
134
|
- - "~>"
|
135
135
|
- !ruby/object:Gem::Version
|
136
136
|
version: '1.0'
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
requirement: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - "~>"
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '1.0'
|
143
|
+
name: logstash-mixin-plugin_factory_support
|
144
|
+
prerelease: false
|
145
|
+
type: :runtime
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - "~>"
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: '1.0'
|
137
151
|
- !ruby/object:Gem::Dependency
|
138
152
|
requirement: !ruby/object:Gem::Requirement
|
139
153
|
requirements:
|
@@ -299,7 +313,7 @@ files:
|
|
299
313
|
- vendor/jar-dependencies/io/netty/netty-transport-native-unix-common/4.1.87.Final/netty-transport-native-unix-common-4.1.87.Final.jar
|
300
314
|
- vendor/jar-dependencies/io/netty/netty-transport/4.1.87.Final/netty-transport-4.1.87.Final.jar
|
301
315
|
- vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
|
302
|
-
- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.
|
316
|
+
- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.5.0/logstash-input-beats-6.5.0.jar
|
303
317
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
304
318
|
licenses:
|
305
319
|
- Apache License (2.0)
|