logstash-input-beats 6.2.0-java → 6.2.4-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -1
- data/VERSION +1 -1
- data/docs/index.asciidoc +20 -0
- data/lib/logstash/inputs/beats/message_listener.rb +8 -13
- data/lib/logstash/inputs/beats.rb +5 -4
- data/lib/logstash-input-beats_jars.rb +2 -2
- data/spec/inputs/beats_spec.rb +81 -8
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.17.0/log4j-api-2.17.0.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/{6.2.0/logstash-input-beats-6.2.0.jar → 6.2.4/logstash-input-beats-6.2.4.jar} +0 -0
- metadata +4 -6
- data/vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar +0 -0
- data/vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar +0 -0
- data/vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.1.3/logstash-input-beats-6.1.3.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e723378e0ae6fc82af7296c4e7be6482efdf2406bb2be40df42457a3377cc35c
|
4
|
+
data.tar.gz: 8919dc8acb0ebad123ec73b5e4184153f1da713d5ad6c5399607909d9ef0d407
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d5a2b1f9a71f97f344e6886d8e3fd86157fabc20e68f46f96a406413044b49be55c26aaa6abe17cf4f155165c92d0852e35b808abc0febce211753af3c8b84f7
|
7
|
+
data.tar.gz: c12543f02573b5bf2c386feb87dd183de18eeac96e9cead8a1a4a92a705f6b9fc8fa6dee02c448e8f966bc2df99a2341fcfe2d1994e07748278ed43a745897e5
|
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,22 @@
|
|
1
|
+
## 6.2.4
|
2
|
+
- Updated log4j dependency to 2.17.0
|
3
|
+
|
4
|
+
## 6.2.3
|
5
|
+
- Updated log4j dependency to 2.15.0
|
6
|
+
|
7
|
+
## 6.2.2
|
8
|
+
- Fix: update to Gradle 7 [#432](https://github.com/logstash-plugins/logstash-input-beats/pull/432)
|
9
|
+
- [DOC] Edit documentation for `executor_threads` [#435](https://github.com/logstash-plugins/logstash-input-beats/pull/435)
|
10
|
+
|
11
|
+
## 6.2.1
|
12
|
+
- Fix: LS failing with `ssl_peer_metadata => true` [#431](https://github.com/logstash-plugins/logstash-input-beats/pull/431)
|
13
|
+
- [DOC] described `executor_threads` configuration parameter [#421](https://github.com/logstash-plugins/logstash-input-beats/pull/421)
|
14
|
+
|
1
15
|
## 6.2.0
|
2
16
|
- ECS compatibility enablement: Adds alias to support upcoming ECS v8 with the existing ECS v1 implementation
|
3
17
|
|
4
18
|
## 6.1.7
|
5
|
-
- [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-
|
19
|
+
- [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-beats/pull/428)
|
6
20
|
|
7
21
|
## 6.1.6
|
8
22
|
- [DOC] Applied more attributes to manage plugin name in doc content, and implemented conditional text processing. [#423](https://github.com/logstash-plugins/logstash-input-http/pull/423)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
6.2.
|
1
|
+
6.2.4
|
data/docs/index.asciidoc
CHANGED
@@ -163,6 +163,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
163
163
|
| <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
|
164
164
|
| <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
|
165
165
|
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
166
|
+
| <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
|
166
167
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
167
168
|
| <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
|
168
169
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
|
@@ -224,6 +225,24 @@ Close Idle clients after X seconds of inactivity.
|
|
224
225
|
|
225
226
|
Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
|
226
227
|
|
228
|
+
[id="plugins-{type}s-{plugin}-executor_threads"]
|
229
|
+
===== `executor_threads`
|
230
|
+
|
231
|
+
* Value type is <<number,number>>
|
232
|
+
* Default value is 1 executor thread per CPU core
|
233
|
+
|
234
|
+
The number of threads to be used to process incoming beats requests.
|
235
|
+
By default the Beats input creates a number of threads equal to 2*CPU cores.
|
236
|
+
These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management.
|
237
|
+
Parsing the Lumberjack protocol is offloaded to a dedicated thread pool.
|
238
|
+
|
239
|
+
Generally you don't need to touch this setting.
|
240
|
+
In case you are sending very large events and observing "OutOfDirectMemory" exceptions,
|
241
|
+
you may want to reduce this number to half or 1/4 of the CPU cores.
|
242
|
+
This change reduces the number of threads decompressing batches of data into direct memory.
|
243
|
+
However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment,
|
244
|
+
either by increasing number of Logstash nodes or increasing the JVM's Direct Memory.
|
245
|
+
|
227
246
|
[id="plugins-{type}s-{plugin}-host"]
|
228
247
|
===== `host`
|
229
248
|
|
@@ -355,3 +374,4 @@ The minimum TLS version allowed for the encrypted connections. The value must be
|
|
355
374
|
include::{include_path}/{type}.asciidoc[]
|
356
375
|
|
357
376
|
:default_codec!:
|
377
|
+
|
@@ -132,7 +132,7 @@ module LogStash module Inputs class Beats
|
|
132
132
|
tls_session = ctx.channel().pipeline().get("ssl-handler").engine().getSession()
|
133
133
|
tls_verified = true
|
134
134
|
|
135
|
-
|
135
|
+
unless @input.client_authentication_required?
|
136
136
|
# throws SSLPeerUnverifiedException if unverified
|
137
137
|
begin
|
138
138
|
tls_session.getPeerCertificates()
|
@@ -144,18 +144,16 @@ module LogStash module Inputs class Beats
|
|
144
144
|
end
|
145
145
|
end
|
146
146
|
|
147
|
+
meta_data = hash['@metadata'] ||= {}
|
148
|
+
|
147
149
|
if tls_verified
|
148
|
-
|
149
|
-
set_nested(hash, @field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
|
150
|
-
set_nested(hash, @field_tls_cipher, tls_session.getCipherSuite())
|
150
|
+
meta_data['tls_peer'] = { :status => "verified" }
|
151
151
|
|
152
|
-
hash
|
153
|
-
|
154
|
-
|
152
|
+
set_nested(hash, input.field_tls_protocol_version, tls_session.getProtocol())
|
153
|
+
set_nested(hash, input.field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
|
154
|
+
set_nested(hash, input.field_tls_cipher, tls_session.getCipherSuite())
|
155
155
|
else
|
156
|
-
|
157
|
-
:status => "unverified"
|
158
|
-
}
|
156
|
+
meta_data['tls_peer'] = { :status => "unverified" }
|
159
157
|
end
|
160
158
|
end
|
161
159
|
end
|
@@ -166,9 +164,6 @@ module LogStash module Inputs class Beats
|
|
166
164
|
field_ref = Java::OrgLogstash::FieldReference.from(field_name)
|
167
165
|
# create @metadata sub-hash if needed
|
168
166
|
if field_ref.type == Java::OrgLogstash::FieldReference::META_CHILD
|
169
|
-
unless hash.key?("@metadata")
|
170
|
-
hash["@metadata"] = {}
|
171
|
-
end
|
172
167
|
nesting_hash = hash["@metadata"]
|
173
168
|
else
|
174
169
|
nesting_hash = hash
|
@@ -129,6 +129,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
129
129
|
config :executor_threads, :validate => :number, :default => LogStash::Config::CpuCoreStrategy.maximum
|
130
130
|
|
131
131
|
attr_reader :field_hostname, :field_hostip
|
132
|
+
attr_reader :field_tls_protocol_version, :field_tls_peer_subject, :field_tls_cipher
|
132
133
|
|
133
134
|
def register
|
134
135
|
# For Logstash 2.4 we need to make sure that the logger is correctly set for the
|
@@ -167,10 +168,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
|
|
167
168
|
|
168
169
|
# define ecs name mapping
|
169
170
|
@field_hostname = ecs_select[disabled: "host", v1: "[@metadata][input][beats][host][name]"]
|
170
|
-
@field_hostip
|
171
|
-
@field_tls_protocol_version
|
172
|
-
@field_tls_peer_subject
|
173
|
-
@field_tls_cipher
|
171
|
+
@field_hostip = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
|
172
|
+
@field_tls_protocol_version = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
|
173
|
+
@field_tls_peer_subject = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
|
174
|
+
@field_tls_cipher = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
|
174
175
|
|
175
176
|
@logger.info("Starting input listener", :address => "#{@host}:#{@port}")
|
176
177
|
|
@@ -7,5 +7,5 @@ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.9.10')
|
|
7
7
|
require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
|
8
8
|
require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.8')
|
9
9
|
require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
|
10
|
-
require_jar('org.apache.logging.log4j', 'log4j-api', '2.
|
11
|
-
require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.
|
10
|
+
require_jar('org.apache.logging.log4j', 'log4j-api', '2.17.0')
|
11
|
+
require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.4')
|
data/spec/inputs/beats_spec.rb
CHANGED
@@ -12,26 +12,28 @@ describe LogStash::Inputs::Beats do
|
|
12
12
|
let(:connection) { double("connection") }
|
13
13
|
let(:certificate) { BeatsInputTest.certificate }
|
14
14
|
let(:port) { BeatsInputTest.random_port }
|
15
|
+
let(:client_inactivity_timeout) { 400 }
|
16
|
+
let(:threads) { 1 + rand(9) }
|
15
17
|
let(:queue) { Queue.new }
|
16
18
|
let(:config) do
|
17
19
|
{
|
18
|
-
"port" =>
|
20
|
+
"port" => port,
|
19
21
|
"ssl_certificate" => certificate.ssl_cert,
|
20
22
|
"ssl_key" => certificate.ssl_key,
|
23
|
+
"client_inactivity_timeout" => client_inactivity_timeout,
|
24
|
+
"executor_threads" => threads,
|
21
25
|
"type" => "example",
|
22
26
|
"tags" => "beats"
|
23
27
|
}
|
24
28
|
end
|
25
29
|
|
30
|
+
subject(:plugin) { LogStash::Inputs::Beats.new(config) }
|
31
|
+
|
26
32
|
context "#register" do
|
27
33
|
context "host related configuration" do
|
28
|
-
let(:config) { super().merge("host" => host, "port" => port
|
34
|
+
let(:config) { super().merge("host" => host, "port" => port) }
|
29
35
|
let(:host) { "192.168.1.20" }
|
30
|
-
let(:port) {
|
31
|
-
let(:client_inactivity_timeout) { 400 }
|
32
|
-
let(:threads) { 10 }
|
33
|
-
|
34
|
-
subject(:plugin) { LogStash::Inputs::Beats.new(config) }
|
36
|
+
let(:port) { 9001 }
|
35
37
|
|
36
38
|
it "sends the required options to the server" do
|
37
39
|
expect(org.logstash.beats.Server).to receive(:new).with(host, port, client_inactivity_timeout, threads)
|
@@ -158,9 +160,80 @@ describe LogStash::Inputs::Beats do
|
|
158
160
|
|
159
161
|
it "raise a ConfigurationError when multiline codec is set" do
|
160
162
|
plugin = LogStash::Inputs::Beats.new(config)
|
161
|
-
expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
|
163
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
context "tls meta-data" do
|
169
|
+
let(:config) { super().merge("host" => host, "ssl_peer_metadata" => true, "ssl_certificate_authorities" => [ certificate.ssl_cert ]) }
|
170
|
+
let(:host) { "192.168.1.20" }
|
171
|
+
let(:port) { 9002 }
|
172
|
+
|
173
|
+
let(:queue) { Queue.new }
|
174
|
+
let(:event) { LogStash::Event.new }
|
175
|
+
|
176
|
+
subject(:plugin) { LogStash::Inputs::Beats.new(config) }
|
177
|
+
|
178
|
+
before do
|
179
|
+
@server = org.logstash.beats.Server.new(host, port, client_inactivity_timeout, threads)
|
180
|
+
expect( org.logstash.beats.Server ).to receive(:new).with(host, port, client_inactivity_timeout, threads).and_return @server
|
181
|
+
expect( @server ).to receive(:listen)
|
182
|
+
|
183
|
+
subject.register
|
184
|
+
subject.run(queue) # listen does nothing
|
185
|
+
@message_listener = @server.getMessageListener
|
186
|
+
|
187
|
+
allow( ssl_engine = double('ssl_engine') ).to receive(:getSession).and_return ssl_session
|
188
|
+
allow( ssl_handler = double('ssl-handler') ).to receive(:engine).and_return ssl_engine
|
189
|
+
allow( pipeline = double('pipeline') ).to receive(:get).and_return ssl_handler
|
190
|
+
allow( @channel = double('channel') ).to receive(:pipeline).and_return pipeline
|
191
|
+
end
|
192
|
+
|
193
|
+
let(:ctx) do
|
194
|
+
Java::io.netty.channel.ChannelHandlerContext.impl do |method, *args|
|
195
|
+
fail("unexpected #{method}( #{args} )") unless method.eql?(:channel)
|
196
|
+
@channel
|
162
197
|
end
|
163
198
|
end
|
199
|
+
|
200
|
+
let(:ssl_session) do
|
201
|
+
Java::javax.net.ssl.SSLSession.impl do |method, *args|
|
202
|
+
case method
|
203
|
+
when :getPeerCertificates
|
204
|
+
[].to_java(java.security.cert.Certificate)
|
205
|
+
when :getProtocol
|
206
|
+
'TLS-Mock'
|
207
|
+
when :getCipherSuite
|
208
|
+
'SSL_NULL_WITH_TEST_SPEC'
|
209
|
+
when :getPeerPrincipal
|
210
|
+
javax.security.auth.x500.X500Principal.new('CN=TEST, OU=RSpec, O=Logstash, C=NL', {})
|
211
|
+
else
|
212
|
+
fail("unexpected #{method}( #{args} )")
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
let(:ssl_session_peer_principal) do
|
218
|
+
javax.security.auth.x500.X500Principal
|
219
|
+
end
|
220
|
+
|
221
|
+
let(:message) do
|
222
|
+
org.logstash.beats.Message.new(0, java.util.HashMap.new('foo' => 'bar'))
|
223
|
+
end
|
224
|
+
|
225
|
+
it 'sets tls fields' do
|
226
|
+
@message_listener.onNewMessage(ctx, message)
|
227
|
+
|
228
|
+
expect( queue.size ).to be 1
|
229
|
+
expect( event = queue.pop ).to be_a LogStash::Event
|
230
|
+
|
231
|
+
expect( event.get('[@metadata][tls_peer][status]') ).to eql 'verified'
|
232
|
+
|
233
|
+
expect( event.get('[@metadata][tls_peer][protocol]') ).to eql 'TLS-Mock'
|
234
|
+
expect( event.get('[@metadata][tls_peer][cipher_suite]') ).to eql 'SSL_NULL_WITH_TEST_SPEC'
|
235
|
+
expect( event.get('[@metadata][tls_peer][subject]') ).to eql 'CN=TEST,OU=RSpec,O=Logstash,C=NL'
|
236
|
+
end
|
164
237
|
end
|
165
238
|
|
166
239
|
context "when interrupting the plugin" do
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-beats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.2.
|
4
|
+
version: 6.2.4
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -296,12 +296,10 @@ files:
|
|
296
296
|
- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.10/jackson-core-2.9.10.jar
|
297
297
|
- vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.10.8/jackson-databind-2.9.10.8.jar
|
298
298
|
- vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.10/jackson-module-afterburner-2.9.10.jar
|
299
|
-
- vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar
|
300
299
|
- vendor/jar-dependencies/io/netty/netty-all/4.1.65.Final/netty-all-4.1.65.Final.jar
|
301
|
-
- vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.
|
300
|
+
- vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.17.0/log4j-api-2.17.0.jar
|
302
301
|
- vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
|
303
|
-
- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.
|
304
|
-
- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.0/logstash-input-beats-6.2.0.jar
|
302
|
+
- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.4/logstash-input-beats-6.2.4.jar
|
305
303
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
306
304
|
licenses:
|
307
305
|
- Apache License (2.0)
|
Binary file
|
Binary file
|