logstash-input-beats 3.1.24-java → 5.1.9-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +111 -14
- data/Gemfile +3 -2
- data/LICENSE +1 -1
- data/VERSION +1 -1
- data/docs/index.asciidoc +74 -41
- data/lib/logstash/inputs/beats/event_transform_common.rb +1 -0
- data/lib/logstash/inputs/beats/message_listener.rb +63 -5
- data/lib/logstash/inputs/beats.rb +31 -21
- data/lib/logstash-input-beats_jars.rb +9 -9
- data/logstash-input-beats.gemspec +2 -2
- data/spec/inputs/beats/message_listener_spec.rb +103 -12
- data/spec/inputs/beats_spec.rb +15 -26
- data/spec/integration/logstash_forwarder_spec.rb +0 -1
- data/spec/support/client_process_helpers.rb +6 -4
- data/spec/support/helpers.rb +8 -1
- data/spec/support/shared_examples.rb +111 -16
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-annotations/2.9.9/jackson-annotations-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.9/jackson-core-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.9.3/jackson-databind-2.9.9.3.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.9/jackson-module-afterburner-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.30.Final/netty-all-4.1.30.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.12.Final/netty-tcnative-boringssl-static-2.0.12.Final.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar +0 -0
- data/vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/5.1.9/logstash-input-beats-5.1.9.jar +0 -0
- metadata +22 -14
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-annotations/2.7.5/jackson-annotations-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.7.5/jackson-core-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.7.5/jackson-databind-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.7.5/jackson-module-afterburner-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.3.Final/netty-all-4.1.3.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/1.1.33.Fork23/netty-tcnative-boringssl-static-1.1.33.Fork23.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
- data/vendor/jar-dependencies/org/javassist/javassist/3.20.0-GA/javassist-3.20.0-GA.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/3.1.24/logstash-input-beats-3.1.24.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d9a0e0074ea17e1cc3771b987e5adf153ac7bac0ed260be4fa32aebcaf864bb8
|
4
|
+
data.tar.gz: 2d3359740c3bc9cd0142120e0edbbb27e92ae92b982440f6eb9f6bc1228903c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d5c2934d262dba8b5dd23a35a190a255c870919ddbed710aa3db7a06be3a9fc99ccb28c16c54dbdf574c0a13d1867782dab4bdbd93459179098749a4a45d029
|
7
|
+
data.tar.gz: db8cf9e01ab5159a6e36211d2a19449f2da44f0ca9fed86fb838e9f3f2560dd16a3b235c7383643860a138d9d8fa91e9ec3f3cfde27b47eec876fd4a10ffa689
|
data/CHANGELOG.md
CHANGED
@@ -1,28 +1,125 @@
|
|
1
|
-
##
|
2
|
-
-
|
1
|
+
## 5.1.9
|
2
|
+
- Backport [#366](https://github.com/logstash-plugins/logstash-input-beats/pull/366) and [#368](https://github.com/logstash-plugins/logstash-input-beats/pull/368) from 6.x
|
3
3
|
|
4
|
-
##
|
5
|
-
-
|
4
|
+
## 5.1.8
|
5
|
+
- Loosen jar-dependencies manager gem dependency to allow plugin to work with JRubies that include a later version.
|
6
6
|
|
7
|
-
##
|
7
|
+
## 5.1.7
|
8
|
+
- Updated jar dependencies to reflect newer releases
|
9
|
+
|
10
|
+
## 5.1.6
|
11
|
+
- Docs: Fixed broken link by removing extra space. [#347](https://github.com/logstash-plugins/logstash-input-beats/pull/347)
|
12
|
+
|
13
|
+
## 5.1.5
|
14
|
+
- Docs: Fixed section ID that was causing doc build errors in the versioned
|
15
|
+
plugin docs. [#346](https://github.com/logstash-plugins/logstash-input-beats/pull/346)
|
16
|
+
|
17
|
+
## 5.1.4
|
18
|
+
- Added `add_hostname` flag to enable/disable the population of the `host` field from the beats.hostname field [#340](https://github.com/logstash-plugins/logstash-input-beats/pull/340)
|
19
|
+
|
20
|
+
## 5.1.3
|
21
|
+
- Fixed handling of batches where the sequence numbers do not start with 1 [#342](https://github.com/logstash-plugins/logstash-input-beats/pull/342)
|
22
|
+
|
23
|
+
## 5.1.2
|
24
|
+
- Changed project to use gradle version 4.8.1. [#334](https://github.com/logstash-plugins/logstash-input-beats/pull/334)
|
25
|
+
- This is an internal, non user-impacting, change to use a more modern version of gradle for building the plugin.
|
26
|
+
|
27
|
+
## 5.1.1
|
28
|
+
- Docs: Add more detail about creating versioned indexes for Beats data
|
29
|
+
|
30
|
+
## 5.1.0
|
31
|
+
- Added ssl_peer_metadata option. [#327](https://github.com/logstash-plugins/logstash-input-beats/pull/327)
|
32
|
+
- Fixed ssl_verify_mode => peer. [#326](https://github.com/logstash-plugins/logstash-input-beats/pull/326)
|
33
|
+
|
34
|
+
## 5.0.16
|
35
|
+
- [#289](https://github.com/logstash-plugins/logstash-input-beats/pull/289#issuecomment-394072063) Re-initialise Netty worker group on plugin restart
|
36
|
+
|
37
|
+
## 5.0.15
|
38
|
+
- [Ensure that context is available before trace is made](https://github.com/logstash-plugins/logstash-input-beats/pull/319/files)
|
39
|
+
|
40
|
+
## 5.0.14
|
41
|
+
- Update jackson deps to 2.9.5
|
42
|
+
|
43
|
+
## 5.0.13
|
44
|
+
- Fix broken 5.0.12 release
|
45
|
+
|
46
|
+
## 5.0.12
|
47
|
+
- Docs: Set the default_codec doc attribute.
|
48
|
+
|
49
|
+
## 5.0.11
|
50
|
+
- Ensure that the keep-alive is sent for ALL pending batches when the pipeline is blocked, not only the batches attempting to write to the queue. #310
|
51
|
+
|
52
|
+
## 5.0.10
|
53
|
+
- Update jackson deps to 2.9.4
|
54
|
+
|
55
|
+
## 5.0.9
|
56
|
+
- Improvements to back pressure handling and memory management #299
|
57
|
+
|
58
|
+
## 5.0.8
|
59
|
+
- Update jackson deps to 2.9.1
|
60
|
+
|
61
|
+
## 5.0.7
|
62
|
+
- Docs: Deprecate `document_type` option
|
63
|
+
|
64
|
+
## 5.0.6
|
65
|
+
- Re-order Netty pipeline to avoid NullPointerExceptions in KeepAliveHandler when Logstash is under load
|
66
|
+
- Improve exception logging
|
67
|
+
- Upgrade to Netty 4.1.18 with tcnative 2.0.7
|
68
|
+
|
69
|
+
## 5.0.5
|
70
|
+
- Better handle case when remoteAddress is nil to reduce amount of warning messages in logs #269
|
71
|
+
|
72
|
+
## 5.0.4
|
73
|
+
- Fix an issue with `close_wait` connection and making sure the keep alive are send back to the client all the time. #272
|
74
|
+
|
75
|
+
## 5.0.3
|
76
|
+
- Update gemspec summary
|
77
|
+
|
78
|
+
## 5.0.2
|
79
|
+
- Change IdleState strategy from `READER_IDLE` to `ALL_IDLE` #262
|
80
|
+
- Additional context when logging from the BeatsHandler #261
|
81
|
+
- Remove the `LoggingHandler` from the handler stack to reduce noise in the log.
|
82
|
+
|
83
|
+
## 5.0.1
|
84
|
+
- Fix some documentation issues
|
85
|
+
|
86
|
+
## 5.0.0
|
87
|
+
- Mark deprecated congestion_threshold and target_field_for_codec as obsolete
|
88
|
+
|
89
|
+
## 4.0.5
|
8
90
|
- Additional default cipher PR#242
|
91
|
+
- Fix logging from Java
|
9
92
|
|
10
|
-
##
|
11
|
-
-
|
93
|
+
## 4.0.4
|
94
|
+
- Documentation fixes
|
12
95
|
|
13
|
-
##
|
14
|
-
-
|
96
|
+
## 4.0.3
|
97
|
+
- Include remote ip_address in metadata. #180
|
98
|
+
- Require Java 8 #221
|
99
|
+
- Fix ability to set SSL protocols #228
|
15
100
|
|
16
|
-
##
|
17
|
-
|
101
|
+
## 4.0.2
|
102
|
+
- Relax version of concurrent-ruby to `~> 1.0` #216
|
18
103
|
|
19
|
-
##
|
20
|
-
-
|
104
|
+
## 4.0.1
|
105
|
+
- Breaking change: Logstash will no longer start when multiline codec is used with the Beats input plugin #201
|
21
106
|
|
22
|
-
##
|
107
|
+
## 4.0.0
|
108
|
+
- Version yanked from RubyGems for packaging issues
|
109
|
+
|
110
|
+
## 3.1.19
|
111
|
+
- Fix ability to set SSL protocols #228
|
112
|
+
|
113
|
+
## 3.1.18
|
114
|
+
- Relax version of concurrent-ruby to ~> 1.0 #216
|
115
|
+
|
116
|
+
## 3.1.17
|
23
117
|
- Docs: Add note indicating that the multiline codec should not be used with the Beats input plugin
|
24
118
|
- Deprecate warning for multiline codec with the Beats input plugin
|
25
119
|
|
120
|
+
## 3.1.16
|
121
|
+
- Version yanked from RubyGems for packaging issues
|
122
|
+
|
26
123
|
## 3.1.15
|
27
124
|
- DEBUG: Add information about the remote when an exception is catched #192
|
28
125
|
|
data/Gemfile
CHANGED
@@ -2,9 +2,10 @@ source 'https://rubygems.org'
|
|
2
2
|
|
3
3
|
gemspec
|
4
4
|
|
5
|
-
logstash_path = "../../logstash"
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
6
7
|
|
7
|
-
if Dir.exist?(logstash_path) &&
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
8
9
|
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
9
10
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
10
11
|
end
|
data/LICENSE
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
5.1.9
|
data/docs/index.asciidoc
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
:plugin: beats
|
2
2
|
:type: input
|
3
|
+
:default_codec: plain
|
3
4
|
|
4
5
|
///////////////////////////////////////////
|
5
6
|
START - GENERATED VARIABLES, DO NOT EDIT!
|
@@ -24,7 +25,7 @@ This input plugin enables Logstash to receive events from the
|
|
24
25
|
https://www.elastic.co/products/beats[Elastic Beats] framework.
|
25
26
|
|
26
27
|
The following example shows how to configure Logstash to listen on port
|
27
|
-
5044 for incoming Beats connections and to index into Elasticsearch
|
28
|
+
5044 for incoming Beats connections and to index into Elasticsearch.
|
28
29
|
|
29
30
|
[source,ruby]
|
30
31
|
------------------------------------------------------------------------------
|
@@ -38,23 +39,48 @@ output {
|
|
38
39
|
elasticsearch {
|
39
40
|
hosts => "localhost:9200"
|
40
41
|
manage_template => false
|
41
|
-
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
42
|
-
document_type => "%{[@metadata][type]}"
|
42
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" <1>
|
43
|
+
document_type => "%{[@metadata][type]}" <2>
|
43
44
|
}
|
44
45
|
}
|
45
46
|
------------------------------------------------------------------------------
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
<1> Specifies the index to write events to. See <<plugins-{type}s-{plugin}-versioned-indexes>> for
|
48
|
+
more about this setting.
|
49
|
+
<2> Starting with Logstash 6.0, the `document_type` option is
|
50
|
+
deprecated due to the
|
51
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
|
52
|
+
It will be removed in the next major version of Logstash. If you are running
|
53
|
+
Logstash 6.0 or later, do not set `document_type` in your configuration because
|
54
|
+
Logstash sets the type to `doc` by default.
|
51
55
|
|
52
56
|
IMPORTANT: If you are shipping events that span multiple lines, you need to
|
53
57
|
use the https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html[configuration options available in Filebeat] to handle multiline events
|
54
58
|
before sending the event data to Logstash. You cannot use the
|
55
|
-
|
59
|
+
{logstash-ref}/plugins-codecs-multiline.html[Multiline codec plugin] to handle multiline events. Doing so will
|
56
60
|
result in the failure to start Logstash.
|
57
61
|
|
62
|
+
[id="plugins-{type}s-{plugin}-versioned-indexes"]
|
63
|
+
==== Versioned Beats Indices
|
64
|
+
|
65
|
+
To minimize the impact of future schema changes on your existing indices and
|
66
|
+
mappings in Elasticsearch, configure the Elasticsearch output to write to
|
67
|
+
versioned indices. The pattern that you specify for the `index` setting
|
68
|
+
controls the index name:
|
69
|
+
|
70
|
+
[source,yaml]
|
71
|
+
----
|
72
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
73
|
+
----
|
74
|
+
|
75
|
+
`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of
|
76
|
+
the `beat` metadata field, for example, `filebeat`.
|
77
|
+
`%{[@metadata][version]}`:: Sets the second part of the name to the Beat
|
78
|
+
version, for example, +{logstash_version}+.
|
79
|
+
`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the
|
80
|
+
Logstash `@timestamp` field.
|
81
|
+
|
82
|
+
This configuration results in daily index names like
|
83
|
+
+filebeat-{logstash_version}-{localdate}+.
|
58
84
|
|
59
85
|
[id="plugins-{type}s-{plugin}-options"]
|
60
86
|
==== Beats Input Configuration Options
|
@@ -64,6 +90,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
64
90
|
[cols="<,<,<",options="header",]
|
65
91
|
|=======================================================================
|
66
92
|
|Setting |Input type|Required
|
93
|
+
| <<plugins-{type}s-{plugin}-add_hostname>> |<<boolean,boolean>>|No
|
67
94
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
68
95
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
69
96
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
@@ -76,6 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
76
103
|
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
77
104
|
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
78
105
|
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
106
|
+
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|No
|
79
107
|
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|No
|
80
108
|
| <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|No
|
81
109
|
|=======================================================================
|
@@ -85,8 +113,20 @@ input plugins.
|
|
85
113
|
|
86
114
|
|
87
115
|
|
116
|
+
[id="plugins-{type}s-{plugin}-add_hostname"]
|
117
|
+
===== `add_hostname`
|
118
|
+
|
119
|
+
added[5.1.4, Field was added to allow users to control whether or not the `host` field is automatically added to events.]
|
120
|
+
deprecated[5.1.4, In future versions of this plugin, this setting will be removed, and the 'hosts' field will not be added to events.]
|
121
|
+
|
122
|
+
* Value type is <<boolean,boolean>>
|
123
|
+
* Default value is `true`
|
124
|
+
|
125
|
+
Flag to determine whether to add `host` field to event using the value supplied by the beat in the `hostname` field.
|
126
|
+
|
127
|
+
|
88
128
|
[id="plugins-{type}s-{plugin}-cipher_suites"]
|
89
|
-
===== `cipher_suites`
|
129
|
+
===== `cipher_suites`
|
90
130
|
|
91
131
|
* Value type is <<array,array>>
|
92
132
|
* Default value is `java.lang.String[TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
|
@@ -94,25 +134,15 @@ input plugins.
|
|
94
134
|
The list of ciphers suite to use, listed by priorities.
|
95
135
|
|
96
136
|
[id="plugins-{type}s-{plugin}-client_inactivity_timeout"]
|
97
|
-
===== `client_inactivity_timeout`
|
137
|
+
===== `client_inactivity_timeout`
|
98
138
|
|
99
139
|
* Value type is <<number,number>>
|
100
140
|
* Default value is `60`
|
101
141
|
|
102
142
|
Close Idle clients after X seconds of inactivity.
|
103
143
|
|
104
|
-
[id="plugins-{type}s-{plugin}-congestion_threshold"]
|
105
|
-
===== `congestion_threshold` (DEPRECATED)
|
106
|
-
|
107
|
-
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
108
|
-
* Value type is <<number,number>>
|
109
|
-
* Default value is `5`
|
110
|
-
|
111
|
-
The number of seconds before we raise a timeout.
|
112
|
-
This option is useful to control how much time to wait if something is blocking the pipeline.
|
113
|
-
|
114
144
|
[id="plugins-{type}s-{plugin}-host"]
|
115
|
-
===== `host`
|
145
|
+
===== `host`
|
116
146
|
|
117
147
|
* Value type is <<string,string>>
|
118
148
|
* Default value is `"0.0.0.0"`
|
@@ -120,7 +150,7 @@ This option is useful to control how much time to wait if something is blocking
|
|
120
150
|
The IP address to listen on.
|
121
151
|
|
122
152
|
[id="plugins-{type}s-{plugin}-include_codec_tag"]
|
123
|
-
===== `include_codec_tag`
|
153
|
+
===== `include_codec_tag`
|
124
154
|
|
125
155
|
* Value type is <<boolean,boolean>>
|
126
156
|
* Default value is `true`
|
@@ -128,7 +158,7 @@ The IP address to listen on.
|
|
128
158
|
|
129
159
|
|
130
160
|
[id="plugins-{type}s-{plugin}-port"]
|
131
|
-
===== `port`
|
161
|
+
===== `port`
|
132
162
|
|
133
163
|
* This is a required setting.
|
134
164
|
* Value type is <<number,number>>
|
@@ -137,7 +167,7 @@ The IP address to listen on.
|
|
137
167
|
The port to listen on.
|
138
168
|
|
139
169
|
[id="plugins-{type}s-{plugin}-ssl"]
|
140
|
-
===== `ssl`
|
170
|
+
===== `ssl`
|
141
171
|
|
142
172
|
* Value type is <<boolean,boolean>>
|
143
173
|
* Default value is `false`
|
@@ -147,7 +177,7 @@ enable encryption by setting `ssl` to true and configuring
|
|
147
177
|
the `ssl_certificate` and `ssl_key` options.
|
148
178
|
|
149
179
|
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
150
|
-
===== `ssl_certificate`
|
180
|
+
===== `ssl_certificate`
|
151
181
|
|
152
182
|
* Value type is <<path,path>>
|
153
183
|
* There is no default value for this setting.
|
@@ -155,19 +185,19 @@ the `ssl_certificate` and `ssl_key` options.
|
|
155
185
|
SSL certificate to use.
|
156
186
|
|
157
187
|
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
158
|
-
===== `ssl_certificate_authorities`
|
188
|
+
===== `ssl_certificate_authorities`
|
159
189
|
|
160
190
|
* Value type is <<array,array>>
|
161
191
|
* Default value is `[]`
|
162
192
|
|
163
|
-
Validate client certificates against these authorities.
|
193
|
+
Validate client certificates against these authorities.
|
164
194
|
You can define multiple files or paths. All the certificates will
|
165
195
|
be read and added to the trust store. You need to configure the `ssl_verify_mode`
|
166
196
|
to `peer` or `force_peer` to enable the verification.
|
167
197
|
|
168
198
|
|
169
199
|
[id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
|
170
|
-
===== `ssl_handshake_timeout`
|
200
|
+
===== `ssl_handshake_timeout`
|
171
201
|
|
172
202
|
* Value type is <<number,number>>
|
173
203
|
* Default value is `10000`
|
@@ -175,7 +205,7 @@ to `peer` or `force_peer` to enable the verification.
|
|
175
205
|
Time in milliseconds for an incomplete ssl handshake to timeout
|
176
206
|
|
177
207
|
[id="plugins-{type}s-{plugin}-ssl_key"]
|
178
|
-
===== `ssl_key`
|
208
|
+
===== `ssl_key`
|
179
209
|
|
180
210
|
* Value type is <<path,path>>
|
181
211
|
* There is no default value for this setting.
|
@@ -185,7 +215,7 @@ NOTE: This key need to be in the PKCS8 format, you can convert it with https://w
|
|
185
215
|
for more information.
|
186
216
|
|
187
217
|
[id="plugins-{type}s-{plugin}-ssl_key_passphrase"]
|
188
|
-
===== `ssl_key_passphrase`
|
218
|
+
===== `ssl_key_passphrase`
|
189
219
|
|
190
220
|
* Value type is <<password,password>>
|
191
221
|
* There is no default value for this setting.
|
@@ -193,14 +223,14 @@ for more information.
|
|
193
223
|
SSL key passphrase to use.
|
194
224
|
|
195
225
|
[id="plugins-{type}s-{plugin}-ssl_verify_mode"]
|
196
|
-
===== `ssl_verify_mode`
|
226
|
+
===== `ssl_verify_mode`
|
197
227
|
|
198
228
|
* Value can be any of: `none`, `peer`, `force_peer`
|
199
229
|
* Default value is `"none"`
|
200
230
|
|
201
231
|
By default the server doesn't do any client verification.
|
202
232
|
|
203
|
-
`peer` will make the server ask the client to provide a certificate.
|
233
|
+
`peer` will make the server ask the client to provide a certificate.
|
204
234
|
If the client provides a certificate, it will be validated.
|
205
235
|
|
206
236
|
`force_peer` will make the server ask the client to provide a certificate.
|
@@ -208,17 +238,18 @@ If the client doesn't provide a certificate, the connection will be closed.
|
|
208
238
|
|
209
239
|
This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
|
210
240
|
|
211
|
-
[id="plugins-{type}s-{plugin}-
|
212
|
-
===== `
|
241
|
+
[id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
|
242
|
+
===== `ssl_peer_metadata`
|
213
243
|
|
214
|
-
*
|
215
|
-
*
|
216
|
-
* Default value is `"message"`
|
244
|
+
* Value type is <<boolean,boolean>>
|
245
|
+
* Default value is `false`
|
217
246
|
|
218
|
-
|
247
|
+
Enables storing client certificate information in event's metadata.
|
248
|
+
|
249
|
+
This option is only valid when `ssl_verify_mode` is set to `peer` or `force_peer`.
|
219
250
|
|
220
251
|
[id="plugins-{type}s-{plugin}-tls_max_version"]
|
221
|
-
===== `tls_max_version`
|
252
|
+
===== `tls_max_version`
|
222
253
|
|
223
254
|
* Value type is <<number,number>>
|
224
255
|
* Default value is `1.2`
|
@@ -227,7 +258,7 @@ The maximum TLS version allowed for the encrypted connections. The value must be
|
|
227
258
|
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
228
259
|
|
229
260
|
[id="plugins-{type}s-{plugin}-tls_min_version"]
|
230
|
-
===== `tls_min_version`
|
261
|
+
===== `tls_min_version`
|
231
262
|
|
232
263
|
* Value type is <<number,number>>
|
233
264
|
* Default value is `1`
|
@@ -239,3 +270,5 @@ The minimum TLS version allowed for the encrypted connections. The value must be
|
|
239
270
|
|
240
271
|
[id="plugins-{type}s-{plugin}-common-options"]
|
241
272
|
include::{include_path}/{type}.asciidoc[]
|
273
|
+
|
274
|
+
:default_codec!:
|
@@ -12,6 +12,7 @@ module LogStash module Inputs class Beats
|
|
12
12
|
# Copies the beat.hostname field into the host field unless
|
13
13
|
# the host field is already defined
|
14
14
|
def copy_beat_hostname(event)
|
15
|
+
return unless @input.add_hostname
|
15
16
|
host = event.get("[beat][hostname]")
|
16
17
|
|
17
18
|
if host && event.get("host").nil?
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "thread_safe"
|
3
3
|
require "logstash-input-beats_jars"
|
4
|
+
import "javax.net.ssl.SSLPeerUnverifiedException"
|
4
5
|
import "org.logstash.beats.MessageListener"
|
5
6
|
|
6
7
|
module LogStash module Inputs class Beats
|
@@ -10,7 +11,7 @@ module LogStash module Inputs class Beats
|
|
10
11
|
FILEBEAT_LOG_LINE_FIELD = "message".freeze
|
11
12
|
LSF_LOG_LINE_FIELD = "line".freeze
|
12
13
|
|
13
|
-
ConnectionState = Struct.new(:ctx, :codec)
|
14
|
+
ConnectionState = Struct.new(:ctx, :codec, :ip_address)
|
14
15
|
|
15
16
|
attr_reader :logger, :input, :connections_list
|
16
17
|
|
@@ -27,10 +28,14 @@ module LogStash module Inputs class Beats
|
|
27
28
|
end
|
28
29
|
|
29
30
|
def onNewMessage(ctx, message)
|
30
|
-
hash = message.getData
|
31
|
+
hash = message.getData
|
32
|
+
ip_address = ip_address(ctx)
|
31
33
|
|
34
|
+
hash['@metadata']['ip_address'] = ip_address unless ip_address.nil? || hash['@metadata'].nil?
|
32
35
|
target_field = extract_target_field(hash)
|
33
36
|
|
37
|
+
extract_tls_peer(hash, ctx)
|
38
|
+
|
34
39
|
if target_field.nil?
|
35
40
|
event = LogStash::Event.new(hash)
|
36
41
|
@nocodec_transformer.transform(event)
|
@@ -58,9 +63,9 @@ module LogStash module Inputs class Beats
|
|
58
63
|
# This is mostly due to a bad certificate or keys, running Logstash in debug mode will show more information
|
59
64
|
if cause.is_a?(Java::JavaLang::IllegalArgumentException)
|
60
65
|
if input.logger.debug?
|
61
|
-
input.logger.error("Looks like you either have an invalid key or your private key was not in PKCS8 format.")
|
66
|
+
input.logger.error("Looks like you either have a bad certificate, an invalid key or your private key was not in PKCS8 format.", :exception => cause)
|
62
67
|
else
|
63
|
-
input.logger.error("Looks like you either have an invalid key or your private key was not in PKCS8 format."
|
68
|
+
input.logger.error("Looks like you either have a bad certificate, an invalid key or your private key was not in PKCS8 format.")
|
64
69
|
end
|
65
70
|
else
|
66
71
|
input.logger.warn("Error when creating a connection", :exception => cause.to_s)
|
@@ -77,8 +82,29 @@ module LogStash module Inputs class Beats
|
|
77
82
|
connections_list[ctx].codec
|
78
83
|
end
|
79
84
|
|
85
|
+
def ip_address(ctx)
|
86
|
+
return if connections_list[ctx].nil?
|
87
|
+
connections_list[ctx].ip_address
|
88
|
+
end
|
89
|
+
|
80
90
|
def register_connection(ctx)
|
81
|
-
connections_list[ctx] = ConnectionState.new(ctx, input.codec.
|
91
|
+
connections_list[ctx] = ConnectionState.new(ctx, input.codec.clone, ip_address_from_ctx(ctx))
|
92
|
+
end
|
93
|
+
|
94
|
+
def ip_address_from_ctx(ctx)
|
95
|
+
begin
|
96
|
+
remote_address = ctx.channel.remoteAddress
|
97
|
+
# Netty allows remoteAddress to be nil, which can cause a lot of log entries - see
|
98
|
+
# https://github.com/logstash-plugins/logstash-input-beats/issues/269
|
99
|
+
if remote_address.nil?
|
100
|
+
input.logger.debug("Cannot retrieve remote IP address for beats input - remoteAddress is nil")
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
remote_address.getAddress.getHostAddress
|
104
|
+
rescue => e # This should not happen, but should not block the beats input
|
105
|
+
input.logger.warn("Could not retrieve remote IP address for beats input.", :error => e)
|
106
|
+
nil
|
107
|
+
end
|
82
108
|
end
|
83
109
|
|
84
110
|
def unregister_connection(ctx)
|
@@ -96,6 +122,38 @@ module LogStash module Inputs class Beats
|
|
96
122
|
end
|
97
123
|
end
|
98
124
|
|
125
|
+
def extract_tls_peer(hash, ctx)
|
126
|
+
if @input.client_authentication_metadata?
|
127
|
+
tls_session = ctx.channel().pipeline().get("ssl-handler").engine().getSession()
|
128
|
+
tls_verified = true
|
129
|
+
|
130
|
+
if not @input.client_authentication_required?
|
131
|
+
# throws SSLPeerUnverifiedException if unverified
|
132
|
+
begin
|
133
|
+
tls_session.getPeerCertificates()
|
134
|
+
rescue SSLPeerUnverifiedException => e
|
135
|
+
tls_verified = false
|
136
|
+
if input.logger.debug?
|
137
|
+
input.logger.debug("SSL peer unverified. This is normal with 'peer' verification and client does not presents a certificate.", :exception => e)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
if tls_verified
|
143
|
+
hash['@metadata']['tls_peer'] = {
|
144
|
+
:status => "verified",
|
145
|
+
:protocol => tls_session.getProtocol(),
|
146
|
+
:subject => tls_session.getPeerPrincipal().getName(),
|
147
|
+
:cipher_suite => tls_session.getCipherSuite()
|
148
|
+
}
|
149
|
+
else
|
150
|
+
hash['@metadata']['tls_peer'] = {
|
151
|
+
:status => "unverified"
|
152
|
+
}
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
99
157
|
def extract_target_field(hash)
|
100
158
|
if from_filebeat?(hash)
|
101
159
|
hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s
|