logstash-input-beats 3.1.24-java → 5.1.9-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +111 -14
- data/Gemfile +3 -2
- data/LICENSE +1 -1
- data/VERSION +1 -1
- data/docs/index.asciidoc +74 -41
- data/lib/logstash/inputs/beats/event_transform_common.rb +1 -0
- data/lib/logstash/inputs/beats/message_listener.rb +63 -5
- data/lib/logstash/inputs/beats.rb +31 -21
- data/lib/logstash-input-beats_jars.rb +9 -9
- data/logstash-input-beats.gemspec +2 -2
- data/spec/inputs/beats/message_listener_spec.rb +103 -12
- data/spec/inputs/beats_spec.rb +15 -26
- data/spec/integration/logstash_forwarder_spec.rb +0 -1
- data/spec/support/client_process_helpers.rb +6 -4
- data/spec/support/helpers.rb +8 -1
- data/spec/support/shared_examples.rb +111 -16
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-annotations/2.9.9/jackson-annotations-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.9/jackson-core-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.9.3/jackson-databind-2.9.9.3.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.9/jackson-module-afterburner-2.9.9.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.30.Final/netty-all-4.1.30.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/2.0.12.Final/netty-tcnative-boringssl-static-2.0.12.Final.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar +0 -0
- data/vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/5.1.9/logstash-input-beats-5.1.9.jar +0 -0
- metadata +22 -14
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-annotations/2.7.5/jackson-annotations-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.7.5/jackson-core-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.7.5/jackson-databind-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.7.5/jackson-module-afterburner-2.7.5.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.3.Final/netty-all-4.1.3.Final.jar +0 -0
- data/vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/1.1.33.Fork23/netty-tcnative-boringssl-static-1.1.33.Fork23.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar +0 -0
- data/vendor/jar-dependencies/org/javassist/javassist/3.20.0-GA/javassist-3.20.0-GA.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/3.1.24/logstash-input-beats-3.1.24.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d9a0e0074ea17e1cc3771b987e5adf153ac7bac0ed260be4fa32aebcaf864bb8
|
4
|
+
data.tar.gz: 2d3359740c3bc9cd0142120e0edbbb27e92ae92b982440f6eb9f6bc1228903c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d5c2934d262dba8b5dd23a35a190a255c870919ddbed710aa3db7a06be3a9fc99ccb28c16c54dbdf574c0a13d1867782dab4bdbd93459179098749a4a45d029
|
7
|
+
data.tar.gz: db8cf9e01ab5159a6e36211d2a19449f2da44f0ca9fed86fb838e9f3f2560dd16a3b235c7383643860a138d9d8fa91e9ec3f3cfde27b47eec876fd4a10ffa689
|
data/CHANGELOG.md
CHANGED
@@ -1,28 +1,125 @@
|
|
1
|
-
##
|
2
|
-
-
|
1
|
+
## 5.1.9
|
2
|
+
- Backport [#366](https://github.com/logstash-plugins/logstash-input-beats/pull/366) and [#368](https://github.com/logstash-plugins/logstash-input-beats/pull/368) from 6.x
|
3
3
|
|
4
|
-
##
|
5
|
-
-
|
4
|
+
## 5.1.8
|
5
|
+
- Loosen jar-dependencies manager gem dependency to allow plugin to work with JRubies that include a later version.
|
6
6
|
|
7
|
-
##
|
7
|
+
## 5.1.7
|
8
|
+
- Updated jar dependencies to reflect newer releases
|
9
|
+
|
10
|
+
## 5.1.6
|
11
|
+
- Docs: Fixed broken link by removing extra space. [#347](https://github.com/logstash-plugins/logstash-input-beats/pull/347)
|
12
|
+
|
13
|
+
## 5.1.5
|
14
|
+
- Docs: Fixed section ID that was causing doc build errors in the versioned
|
15
|
+
plugin docs. [#346](https://github.com/logstash-plugins/logstash-input-beats/pull/346)
|
16
|
+
|
17
|
+
## 5.1.4
|
18
|
+
- Added `add_hostname` flag to enable/disable the population of the `host` field from the beats.hostname field [#340](https://github.com/logstash-plugins/logstash-input-beats/pull/340)
|
19
|
+
|
20
|
+
## 5.1.3
|
21
|
+
- Fixed handling of batches where the sequence numbers do not start with 1 [#342](https://github.com/logstash-plugins/logstash-input-beats/pull/342)
|
22
|
+
|
23
|
+
## 5.1.2
|
24
|
+
- Changed project to use gradle version 4.8.1. [#334](https://github.com/logstash-plugins/logstash-input-beats/pull/334)
|
25
|
+
- This is an internal, non user-impacting, change to use a more modern version of gradle for building the plugin.
|
26
|
+
|
27
|
+
## 5.1.1
|
28
|
+
- Docs: Add more detail about creating versioned indexes for Beats data
|
29
|
+
|
30
|
+
## 5.1.0
|
31
|
+
- Added ssl_peer_metadata option. [#327](https://github.com/logstash-plugins/logstash-input-beats/pull/327)
|
32
|
+
- Fixed ssl_verify_mode => peer. [#326](https://github.com/logstash-plugins/logstash-input-beats/pull/326)
|
33
|
+
|
34
|
+
## 5.0.16
|
35
|
+
- [#289](https://github.com/logstash-plugins/logstash-input-beats/pull/289#issuecomment-394072063) Re-initialise Netty worker group on plugin restart
|
36
|
+
|
37
|
+
## 5.0.15
|
38
|
+
- [Ensure that context is available before trace is made](https://github.com/logstash-plugins/logstash-input-beats/pull/319/files)
|
39
|
+
|
40
|
+
## 5.0.14
|
41
|
+
- Update jackson deps to 2.9.5
|
42
|
+
|
43
|
+
## 5.0.13
|
44
|
+
- Fix broken 5.0.12 release
|
45
|
+
|
46
|
+
## 5.0.12
|
47
|
+
- Docs: Set the default_codec doc attribute.
|
48
|
+
|
49
|
+
## 5.0.11
|
50
|
+
- Ensure that the keep-alive is sent for ALL pending batches when the pipeline is blocked, not only the batches attempting to write to the queue. #310
|
51
|
+
|
52
|
+
## 5.0.10
|
53
|
+
- Update jackson deps to 2.9.4
|
54
|
+
|
55
|
+
## 5.0.9
|
56
|
+
- Improvements to back pressure handling and memory management #299
|
57
|
+
|
58
|
+
## 5.0.8
|
59
|
+
- Update jackson deps to 2.9.1
|
60
|
+
|
61
|
+
## 5.0.7
|
62
|
+
- Docs: Deprecate `document_type` option
|
63
|
+
|
64
|
+
## 5.0.6
|
65
|
+
- Re-order Netty pipeline to avoid NullPointerExceptions in KeepAliveHandler when Logstash is under load
|
66
|
+
- Improve exception logging
|
67
|
+
- Upgrade to Netty 4.1.18 with tcnative 2.0.7
|
68
|
+
|
69
|
+
## 5.0.5
|
70
|
+
- Better handle case when remoteAddress is nil to reduce amount of warning messages in logs #269
|
71
|
+
|
72
|
+
## 5.0.4
|
73
|
+
- Fix an issue with `close_wait` connection and making sure the keep alive are send back to the client all the time. #272
|
74
|
+
|
75
|
+
## 5.0.3
|
76
|
+
- Update gemspec summary
|
77
|
+
|
78
|
+
## 5.0.2
|
79
|
+
- Change IdleState strategy from `READER_IDLE` to `ALL_IDLE` #262
|
80
|
+
- Additional context when logging from the BeatsHandler #261
|
81
|
+
- Remove the `LoggingHandler` from the handler stack to reduce noise in the log.
|
82
|
+
|
83
|
+
## 5.0.1
|
84
|
+
- Fix some documentation issues
|
85
|
+
|
86
|
+
## 5.0.0
|
87
|
+
- Mark deprecated congestion_threshold and target_field_for_codec as obsolete
|
88
|
+
|
89
|
+
## 4.0.5
|
8
90
|
- Additional default cipher PR#242
|
91
|
+
- Fix logging from Java
|
9
92
|
|
10
|
-
##
|
11
|
-
-
|
93
|
+
## 4.0.4
|
94
|
+
- Documentation fixes
|
12
95
|
|
13
|
-
##
|
14
|
-
-
|
96
|
+
## 4.0.3
|
97
|
+
- Include remote ip_address in metadata. #180
|
98
|
+
- Require Java 8 #221
|
99
|
+
- Fix ability to set SSL protocols #228
|
15
100
|
|
16
|
-
##
|
17
|
-
|
101
|
+
## 4.0.2
|
102
|
+
- Relax version of concurrent-ruby to `~> 1.0` #216
|
18
103
|
|
19
|
-
##
|
20
|
-
-
|
104
|
+
## 4.0.1
|
105
|
+
- Breaking change: Logstash will no longer start when multiline codec is used with the Beats input plugin #201
|
21
106
|
|
22
|
-
##
|
107
|
+
## 4.0.0
|
108
|
+
- Version yanked from RubyGems for packaging issues
|
109
|
+
|
110
|
+
## 3.1.19
|
111
|
+
- Fix ability to set SSL protocols #228
|
112
|
+
|
113
|
+
## 3.1.18
|
114
|
+
- Relax version of concurrent-ruby to ~> 1.0 #216
|
115
|
+
|
116
|
+
## 3.1.17
|
23
117
|
- Docs: Add note indicating that the multiline codec should not be used with the Beats input plugin
|
24
118
|
- Deprecate warning for multiline codec with the Beats input plugin
|
25
119
|
|
120
|
+
## 3.1.16
|
121
|
+
- Version yanked from RubyGems for packaging issues
|
122
|
+
|
26
123
|
## 3.1.15
|
27
124
|
- DEBUG: Add information about the remote when an exception is catched #192
|
28
125
|
|
data/Gemfile
CHANGED
@@ -2,9 +2,10 @@ source 'https://rubygems.org'
|
|
2
2
|
|
3
3
|
gemspec
|
4
4
|
|
5
|
-
logstash_path = "../../logstash"
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
6
7
|
|
7
|
-
if Dir.exist?(logstash_path) &&
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
8
9
|
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
9
10
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
10
11
|
end
|
data/LICENSE
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
5.1.9
|
data/docs/index.asciidoc
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
:plugin: beats
|
2
2
|
:type: input
|
3
|
+
:default_codec: plain
|
3
4
|
|
4
5
|
///////////////////////////////////////////
|
5
6
|
START - GENERATED VARIABLES, DO NOT EDIT!
|
@@ -24,7 +25,7 @@ This input plugin enables Logstash to receive events from the
|
|
24
25
|
https://www.elastic.co/products/beats[Elastic Beats] framework.
|
25
26
|
|
26
27
|
The following example shows how to configure Logstash to listen on port
|
27
|
-
5044 for incoming Beats connections and to index into Elasticsearch
|
28
|
+
5044 for incoming Beats connections and to index into Elasticsearch.
|
28
29
|
|
29
30
|
[source,ruby]
|
30
31
|
------------------------------------------------------------------------------
|
@@ -38,23 +39,48 @@ output {
|
|
38
39
|
elasticsearch {
|
39
40
|
hosts => "localhost:9200"
|
40
41
|
manage_template => false
|
41
|
-
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
|
42
|
-
document_type => "%{[@metadata][type]}"
|
42
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" <1>
|
43
|
+
document_type => "%{[@metadata][type]}" <2>
|
43
44
|
}
|
44
45
|
}
|
45
46
|
------------------------------------------------------------------------------
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
<1> Specifies the index to write events to. See <<plugins-{type}s-{plugin}-versioned-indexes>> for
|
48
|
+
more about this setting.
|
49
|
+
<2> Starting with Logstash 6.0, the `document_type` option is
|
50
|
+
deprecated due to the
|
51
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
|
52
|
+
It will be removed in the next major version of Logstash. If you are running
|
53
|
+
Logstash 6.0 or later, do not set `document_type` in your configuration because
|
54
|
+
Logstash sets the type to `doc` by default.
|
51
55
|
|
52
56
|
IMPORTANT: If you are shipping events that span multiple lines, you need to
|
53
57
|
use the https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html[configuration options available in Filebeat] to handle multiline events
|
54
58
|
before sending the event data to Logstash. You cannot use the
|
55
|
-
|
59
|
+
{logstash-ref}/plugins-codecs-multiline.html[Multiline codec plugin] to handle multiline events. Doing so will
|
56
60
|
result in the failure to start Logstash.
|
57
61
|
|
62
|
+
[id="plugins-{type}s-{plugin}-versioned-indexes"]
|
63
|
+
==== Versioned Beats Indices
|
64
|
+
|
65
|
+
To minimize the impact of future schema changes on your existing indices and
|
66
|
+
mappings in Elasticsearch, configure the Elasticsearch output to write to
|
67
|
+
versioned indices. The pattern that you specify for the `index` setting
|
68
|
+
controls the index name:
|
69
|
+
|
70
|
+
[source,yaml]
|
71
|
+
----
|
72
|
+
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
|
73
|
+
----
|
74
|
+
|
75
|
+
`%{[@metadata][beat]}`:: Sets the first part of the index name to the value of
|
76
|
+
the `beat` metadata field, for example, `filebeat`.
|
77
|
+
`%{[@metadata][version]}`:: Sets the second part of the name to the Beat
|
78
|
+
version, for example, +{logstash_version}+.
|
79
|
+
`%{+YYYY.MM.dd}`:: Sets the third part of the name to a date based on the
|
80
|
+
Logstash `@timestamp` field.
|
81
|
+
|
82
|
+
This configuration results in daily index names like
|
83
|
+
+filebeat-{logstash_version}-{localdate}+.
|
58
84
|
|
59
85
|
[id="plugins-{type}s-{plugin}-options"]
|
60
86
|
==== Beats Input Configuration Options
|
@@ -64,6 +90,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
64
90
|
[cols="<,<,<",options="header",]
|
65
91
|
|=======================================================================
|
66
92
|
|Setting |Input type|Required
|
93
|
+
| <<plugins-{type}s-{plugin}-add_hostname>> |<<boolean,boolean>>|No
|
67
94
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
68
95
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
69
96
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
@@ -76,6 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
76
103
|
| <<plugins-{type}s-{plugin}-ssl_key>> |a valid filesystem path|No
|
77
104
|
| <<plugins-{type}s-{plugin}-ssl_key_passphrase>> |<<password,password>>|No
|
78
105
|
| <<plugins-{type}s-{plugin}-ssl_verify_mode>> |<<string,string>>, one of `["none", "peer", "force_peer"]`|No
|
106
|
+
| <<plugins-{type}s-{plugin}-ssl_peer_metadata>> |<<boolean,boolean>>|No
|
79
107
|
| <<plugins-{type}s-{plugin}-tls_max_version>> |<<number,number>>|No
|
80
108
|
| <<plugins-{type}s-{plugin}-tls_min_version>> |<<number,number>>|No
|
81
109
|
|=======================================================================
|
@@ -85,8 +113,20 @@ input plugins.
|
|
85
113
|
|
86
114
|
|
87
115
|
|
116
|
+
[id="plugins-{type}s-{plugin}-add_hostname"]
|
117
|
+
===== `add_hostname`
|
118
|
+
|
119
|
+
added[5.1.4, Field was added to allow users to control whether or not the `host` field is automatically added to events.]
|
120
|
+
deprecated[5.1.4, In future versions of this plugin, this setting will be removed, and the 'hosts' field will not be added to events.]
|
121
|
+
|
122
|
+
* Value type is <<boolean,boolean>>
|
123
|
+
* Default value is `true`
|
124
|
+
|
125
|
+
Flag to determine whether to add `host` field to event using the value supplied by the beat in the `hostname` field.
|
126
|
+
|
127
|
+
|
88
128
|
[id="plugins-{type}s-{plugin}-cipher_suites"]
|
89
|
-
===== `cipher_suites`
|
129
|
+
===== `cipher_suites`
|
90
130
|
|
91
131
|
* Value type is <<array,array>>
|
92
132
|
* Default value is `java.lang.String[TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
|
@@ -94,25 +134,15 @@ input plugins.
|
|
94
134
|
The list of ciphers suite to use, listed by priorities.
|
95
135
|
|
96
136
|
[id="plugins-{type}s-{plugin}-client_inactivity_timeout"]
|
97
|
-
===== `client_inactivity_timeout`
|
137
|
+
===== `client_inactivity_timeout`
|
98
138
|
|
99
139
|
* Value type is <<number,number>>
|
100
140
|
* Default value is `60`
|
101
141
|
|
102
142
|
Close Idle clients after X seconds of inactivity.
|
103
143
|
|
104
|
-
[id="plugins-{type}s-{plugin}-congestion_threshold"]
|
105
|
-
===== `congestion_threshold` (DEPRECATED)
|
106
|
-
|
107
|
-
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
108
|
-
* Value type is <<number,number>>
|
109
|
-
* Default value is `5`
|
110
|
-
|
111
|
-
The number of seconds before we raise a timeout.
|
112
|
-
This option is useful to control how much time to wait if something is blocking the pipeline.
|
113
|
-
|
114
144
|
[id="plugins-{type}s-{plugin}-host"]
|
115
|
-
===== `host`
|
145
|
+
===== `host`
|
116
146
|
|
117
147
|
* Value type is <<string,string>>
|
118
148
|
* Default value is `"0.0.0.0"`
|
@@ -120,7 +150,7 @@ This option is useful to control how much time to wait if something is blocking
|
|
120
150
|
The IP address to listen on.
|
121
151
|
|
122
152
|
[id="plugins-{type}s-{plugin}-include_codec_tag"]
|
123
|
-
===== `include_codec_tag`
|
153
|
+
===== `include_codec_tag`
|
124
154
|
|
125
155
|
* Value type is <<boolean,boolean>>
|
126
156
|
* Default value is `true`
|
@@ -128,7 +158,7 @@ The IP address to listen on.
|
|
128
158
|
|
129
159
|
|
130
160
|
[id="plugins-{type}s-{plugin}-port"]
|
131
|
-
===== `port`
|
161
|
+
===== `port`
|
132
162
|
|
133
163
|
* This is a required setting.
|
134
164
|
* Value type is <<number,number>>
|
@@ -137,7 +167,7 @@ The IP address to listen on.
|
|
137
167
|
The port to listen on.
|
138
168
|
|
139
169
|
[id="plugins-{type}s-{plugin}-ssl"]
|
140
|
-
===== `ssl`
|
170
|
+
===== `ssl`
|
141
171
|
|
142
172
|
* Value type is <<boolean,boolean>>
|
143
173
|
* Default value is `false`
|
@@ -147,7 +177,7 @@ enable encryption by setting `ssl` to true and configuring
|
|
147
177
|
the `ssl_certificate` and `ssl_key` options.
|
148
178
|
|
149
179
|
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
150
|
-
===== `ssl_certificate`
|
180
|
+
===== `ssl_certificate`
|
151
181
|
|
152
182
|
* Value type is <<path,path>>
|
153
183
|
* There is no default value for this setting.
|
@@ -155,19 +185,19 @@ the `ssl_certificate` and `ssl_key` options.
|
|
155
185
|
SSL certificate to use.
|
156
186
|
|
157
187
|
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
158
|
-
===== `ssl_certificate_authorities`
|
188
|
+
===== `ssl_certificate_authorities`
|
159
189
|
|
160
190
|
* Value type is <<array,array>>
|
161
191
|
* Default value is `[]`
|
162
192
|
|
163
|
-
Validate client certificates against these authorities.
|
193
|
+
Validate client certificates against these authorities.
|
164
194
|
You can define multiple files or paths. All the certificates will
|
165
195
|
be read and added to the trust store. You need to configure the `ssl_verify_mode`
|
166
196
|
to `peer` or `force_peer` to enable the verification.
|
167
197
|
|
168
198
|
|
169
199
|
[id="plugins-{type}s-{plugin}-ssl_handshake_timeout"]
|
170
|
-
===== `ssl_handshake_timeout`
|
200
|
+
===== `ssl_handshake_timeout`
|
171
201
|
|
172
202
|
* Value type is <<number,number>>
|
173
203
|
* Default value is `10000`
|
@@ -175,7 +205,7 @@ to `peer` or `force_peer` to enable the verification.
|
|
175
205
|
Time in milliseconds for an incomplete ssl handshake to timeout
|
176
206
|
|
177
207
|
[id="plugins-{type}s-{plugin}-ssl_key"]
|
178
|
-
===== `ssl_key`
|
208
|
+
===== `ssl_key`
|
179
209
|
|
180
210
|
* Value type is <<path,path>>
|
181
211
|
* There is no default value for this setting.
|
@@ -185,7 +215,7 @@ NOTE: This key need to be in the PKCS8 format, you can convert it with https://w
|
|
185
215
|
for more information.
|
186
216
|
|
187
217
|
[id="plugins-{type}s-{plugin}-ssl_key_passphrase"]
|
188
|
-
===== `ssl_key_passphrase`
|
218
|
+
===== `ssl_key_passphrase`
|
189
219
|
|
190
220
|
* Value type is <<password,password>>
|
191
221
|
* There is no default value for this setting.
|
@@ -193,14 +223,14 @@ for more information.
|
|
193
223
|
SSL key passphrase to use.
|
194
224
|
|
195
225
|
[id="plugins-{type}s-{plugin}-ssl_verify_mode"]
|
196
|
-
===== `ssl_verify_mode`
|
226
|
+
===== `ssl_verify_mode`
|
197
227
|
|
198
228
|
* Value can be any of: `none`, `peer`, `force_peer`
|
199
229
|
* Default value is `"none"`
|
200
230
|
|
201
231
|
By default the server doesn't do any client verification.
|
202
232
|
|
203
|
-
`peer` will make the server ask the client to provide a certificate.
|
233
|
+
`peer` will make the server ask the client to provide a certificate.
|
204
234
|
If the client provides a certificate, it will be validated.
|
205
235
|
|
206
236
|
`force_peer` will make the server ask the client to provide a certificate.
|
@@ -208,17 +238,18 @@ If the client doesn't provide a certificate, the connection will be closed.
|
|
208
238
|
|
209
239
|
This option needs to be used with `ssl_certificate_authorities` and a defined list of CAs.
|
210
240
|
|
211
|
-
[id="plugins-{type}s-{plugin}-
|
212
|
-
===== `
|
241
|
+
[id="plugins-{type}s-{plugin}-ssl_peer_metadata"]
|
242
|
+
===== `ssl_peer_metadata`
|
213
243
|
|
214
|
-
*
|
215
|
-
*
|
216
|
-
* Default value is `"message"`
|
244
|
+
* Value type is <<boolean,boolean>>
|
245
|
+
* Default value is `false`
|
217
246
|
|
218
|
-
|
247
|
+
Enables storing client certificate information in event's metadata.
|
248
|
+
|
249
|
+
This option is only valid when `ssl_verify_mode` is set to `peer` or `force_peer`.
|
219
250
|
|
220
251
|
[id="plugins-{type}s-{plugin}-tls_max_version"]
|
221
|
-
===== `tls_max_version`
|
252
|
+
===== `tls_max_version`
|
222
253
|
|
223
254
|
* Value type is <<number,number>>
|
224
255
|
* Default value is `1.2`
|
@@ -227,7 +258,7 @@ The maximum TLS version allowed for the encrypted connections. The value must be
|
|
227
258
|
1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
|
228
259
|
|
229
260
|
[id="plugins-{type}s-{plugin}-tls_min_version"]
|
230
|
-
===== `tls_min_version`
|
261
|
+
===== `tls_min_version`
|
231
262
|
|
232
263
|
* Value type is <<number,number>>
|
233
264
|
* Default value is `1`
|
@@ -239,3 +270,5 @@ The minimum TLS version allowed for the encrypted connections. The value must be
|
|
239
270
|
|
240
271
|
[id="plugins-{type}s-{plugin}-common-options"]
|
241
272
|
include::{include_path}/{type}.asciidoc[]
|
273
|
+
|
274
|
+
:default_codec!:
|
@@ -12,6 +12,7 @@ module LogStash module Inputs class Beats
|
|
12
12
|
# Copies the beat.hostname field into the host field unless
|
13
13
|
# the host field is already defined
|
14
14
|
def copy_beat_hostname(event)
|
15
|
+
return unless @input.add_hostname
|
15
16
|
host = event.get("[beat][hostname]")
|
16
17
|
|
17
18
|
if host && event.get("host").nil?
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "thread_safe"
|
3
3
|
require "logstash-input-beats_jars"
|
4
|
+
import "javax.net.ssl.SSLPeerUnverifiedException"
|
4
5
|
import "org.logstash.beats.MessageListener"
|
5
6
|
|
6
7
|
module LogStash module Inputs class Beats
|
@@ -10,7 +11,7 @@ module LogStash module Inputs class Beats
|
|
10
11
|
FILEBEAT_LOG_LINE_FIELD = "message".freeze
|
11
12
|
LSF_LOG_LINE_FIELD = "line".freeze
|
12
13
|
|
13
|
-
ConnectionState = Struct.new(:ctx, :codec)
|
14
|
+
ConnectionState = Struct.new(:ctx, :codec, :ip_address)
|
14
15
|
|
15
16
|
attr_reader :logger, :input, :connections_list
|
16
17
|
|
@@ -27,10 +28,14 @@ module LogStash module Inputs class Beats
|
|
27
28
|
end
|
28
29
|
|
29
30
|
def onNewMessage(ctx, message)
|
30
|
-
hash = message.getData
|
31
|
+
hash = message.getData
|
32
|
+
ip_address = ip_address(ctx)
|
31
33
|
|
34
|
+
hash['@metadata']['ip_address'] = ip_address unless ip_address.nil? || hash['@metadata'].nil?
|
32
35
|
target_field = extract_target_field(hash)
|
33
36
|
|
37
|
+
extract_tls_peer(hash, ctx)
|
38
|
+
|
34
39
|
if target_field.nil?
|
35
40
|
event = LogStash::Event.new(hash)
|
36
41
|
@nocodec_transformer.transform(event)
|
@@ -58,9 +63,9 @@ module LogStash module Inputs class Beats
|
|
58
63
|
# This is mostly due to a bad certificate or keys, running Logstash in debug mode will show more information
|
59
64
|
if cause.is_a?(Java::JavaLang::IllegalArgumentException)
|
60
65
|
if input.logger.debug?
|
61
|
-
input.logger.error("Looks like you either have an invalid key or your private key was not in PKCS8 format.")
|
66
|
+
input.logger.error("Looks like you either have a bad certificate, an invalid key or your private key was not in PKCS8 format.", :exception => cause)
|
62
67
|
else
|
63
|
-
input.logger.error("Looks like you either have an invalid key or your private key was not in PKCS8 format."
|
68
|
+
input.logger.error("Looks like you either have a bad certificate, an invalid key or your private key was not in PKCS8 format.")
|
64
69
|
end
|
65
70
|
else
|
66
71
|
input.logger.warn("Error when creating a connection", :exception => cause.to_s)
|
@@ -77,8 +82,29 @@ module LogStash module Inputs class Beats
|
|
77
82
|
connections_list[ctx].codec
|
78
83
|
end
|
79
84
|
|
85
|
+
def ip_address(ctx)
|
86
|
+
return if connections_list[ctx].nil?
|
87
|
+
connections_list[ctx].ip_address
|
88
|
+
end
|
89
|
+
|
80
90
|
def register_connection(ctx)
|
81
|
-
connections_list[ctx] = ConnectionState.new(ctx, input.codec.
|
91
|
+
connections_list[ctx] = ConnectionState.new(ctx, input.codec.clone, ip_address_from_ctx(ctx))
|
92
|
+
end
|
93
|
+
|
94
|
+
def ip_address_from_ctx(ctx)
|
95
|
+
begin
|
96
|
+
remote_address = ctx.channel.remoteAddress
|
97
|
+
# Netty allows remoteAddress to be nil, which can cause a lot of log entries - see
|
98
|
+
# https://github.com/logstash-plugins/logstash-input-beats/issues/269
|
99
|
+
if remote_address.nil?
|
100
|
+
input.logger.debug("Cannot retrieve remote IP address for beats input - remoteAddress is nil")
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
remote_address.getAddress.getHostAddress
|
104
|
+
rescue => e # This should not happen, but should not block the beats input
|
105
|
+
input.logger.warn("Could not retrieve remote IP address for beats input.", :error => e)
|
106
|
+
nil
|
107
|
+
end
|
82
108
|
end
|
83
109
|
|
84
110
|
def unregister_connection(ctx)
|
@@ -96,6 +122,38 @@ module LogStash module Inputs class Beats
|
|
96
122
|
end
|
97
123
|
end
|
98
124
|
|
125
|
+
def extract_tls_peer(hash, ctx)
|
126
|
+
if @input.client_authentication_metadata?
|
127
|
+
tls_session = ctx.channel().pipeline().get("ssl-handler").engine().getSession()
|
128
|
+
tls_verified = true
|
129
|
+
|
130
|
+
if not @input.client_authentication_required?
|
131
|
+
# throws SSLPeerUnverifiedException if unverified
|
132
|
+
begin
|
133
|
+
tls_session.getPeerCertificates()
|
134
|
+
rescue SSLPeerUnverifiedException => e
|
135
|
+
tls_verified = false
|
136
|
+
if input.logger.debug?
|
137
|
+
input.logger.debug("SSL peer unverified. This is normal with 'peer' verification and client does not presents a certificate.", :exception => e)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
if tls_verified
|
143
|
+
hash['@metadata']['tls_peer'] = {
|
144
|
+
:status => "verified",
|
145
|
+
:protocol => tls_session.getProtocol(),
|
146
|
+
:subject => tls_session.getPeerPrincipal().getName(),
|
147
|
+
:cipher_suite => tls_session.getCipherSuite()
|
148
|
+
}
|
149
|
+
else
|
150
|
+
hash['@metadata']['tls_peer'] = {
|
151
|
+
:status => "unverified"
|
152
|
+
}
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
99
157
|
def extract_target_field(hash)
|
100
158
|
if from_filebeat?(hash)
|
101
159
|
hash.delete(FILEBEAT_LOG_LINE_FIELD).to_s
|