logstash-input-beats 6.2.0-java → 6.2.4-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 16a55f0a53247ed30b7110d9f60dce425288fdd68c1bf936283698028bdfd421
4
- data.tar.gz: d8cda82e3d3a8fb3ceef48a96f52674eb04496e7a0957043b34d5c5e741484fa
3
+ metadata.gz: e723378e0ae6fc82af7296c4e7be6482efdf2406bb2be40df42457a3377cc35c
4
+ data.tar.gz: 8919dc8acb0ebad123ec73b5e4184153f1da713d5ad6c5399607909d9ef0d407
5
5
  SHA512:
6
- metadata.gz: e2869c6c573223e979f411c430ce1ddb6f65f6c967930b469c617a03b6a4fb0417a7eecfc23e2c8061599a72a07f656490bb76c9ceb5576f00613be0cdfde688
7
- data.tar.gz: 38dd3f7bfb548565ecc047bc0e044e89796f2fcbdf8187f2b3a37fc1bb822a00d73c759713996dd69471fbaa1e334fd3a4b1e730adcb0093ccf6956f14f4549e
6
+ metadata.gz: d5a2b1f9a71f97f344e6886d8e3fd86157fabc20e68f46f96a406413044b49be55c26aaa6abe17cf4f155165c92d0852e35b808abc0febce211753af3c8b84f7
7
+ data.tar.gz: c12543f02573b5bf2c386feb87dd183de18eeac96e9cead8a1a4a92a705f6b9fc8fa6dee02c448e8f966bc2df99a2341fcfe2d1994e07748278ed43a745897e5
data/CHANGELOG.md CHANGED
@@ -1,8 +1,22 @@
1
+ ## 6.2.4
2
+ - Updated log4j dependency to 2.17.0
3
+
4
+ ## 6.2.3
5
+ - Updated log4j dependency to 2.15.0
6
+
7
+ ## 6.2.2
8
+ - Fix: update to Gradle 7 [#432](https://github.com/logstash-plugins/logstash-input-beats/pull/432)
9
+ - [DOC] Edit documentation for `executor_threads` [#435](https://github.com/logstash-plugins/logstash-input-beats/pull/435)
10
+
11
+ ## 6.2.1
12
+ - Fix: LS failing with `ssl_peer_metadata => true` [#431](https://github.com/logstash-plugins/logstash-input-beats/pull/431)
13
+ - [DOC] described `executor_threads` configuration parameter [#421](https://github.com/logstash-plugins/logstash-input-beats/pull/421)
14
+
1
15
  ## 6.2.0
2
16
  - ECS compatibility enablement: Adds alias to support upcoming ECS v8 with the existing ECS v1 implementation
3
17
 
4
18
  ## 6.1.7
5
- - [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-http/pull/428)
19
+ - [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-beats/pull/428)
6
20
 
7
21
  ## 6.1.6
8
22
  - [DOC] Applied more attributes to manage plugin name in doc content, and implemented conditional text processing. [#423](https://github.com/logstash-plugins/logstash-input-http/pull/423)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 6.2.0
1
+ 6.2.4
data/docs/index.asciidoc CHANGED
@@ -163,6 +163,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
163
163
  | <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
164
164
  | <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
165
165
  | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
166
+ | <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
166
167
  | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
167
168
  | <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
168
169
  | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
@@ -224,6 +225,24 @@ Close Idle clients after X seconds of inactivity.
224
225
 
225
226
  Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
226
227
 
228
+ [id="plugins-{type}s-{plugin}-executor_threads"]
229
+ ===== `executor_threads`
230
+
231
+ * Value type is <<number,number>>
232
+ * Default value is 1 executor thread per CPU core
233
+
234
+ The number of threads to be used to process incoming beats requests.
235
+ By default the Beats input creates a number of threads equal to 2*CPU cores.
236
+ These threads handle incoming connections, reading from established sockets, and executing most of the tasks related to network connection management.
237
+ Parsing the Lumberjack protocol is offloaded to a dedicated thread pool.
238
+
239
+ Generally you don't need to touch this setting.
240
+ In case you are sending very large events and observing "OutOfDirectMemory" exceptions,
241
+ you may want to reduce this number to half or 1/4 of the CPU cores.
242
+ This change reduces the number of threads decompressing batches of data into direct memory.
243
+ However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment,
244
+ either by increasing number of Logstash nodes or increasing the JVM's Direct Memory.
245
+
227
246
  [id="plugins-{type}s-{plugin}-host"]
228
247
  ===== `host`
229
248
 
@@ -355,3 +374,4 @@ The minimum TLS version allowed for the encrypted connections. The value must be
355
374
  include::{include_path}/{type}.asciidoc[]
356
375
 
357
376
  :default_codec!:
377
+
@@ -132,7 +132,7 @@ module LogStash module Inputs class Beats
132
132
  tls_session = ctx.channel().pipeline().get("ssl-handler").engine().getSession()
133
133
  tls_verified = true
134
134
 
135
- if not @input.client_authentication_required?
135
+ unless @input.client_authentication_required?
136
136
  # throws SSLPeerUnverifiedException if unverified
137
137
  begin
138
138
  tls_session.getPeerCertificates()
@@ -144,18 +144,16 @@ module LogStash module Inputs class Beats
144
144
  end
145
145
  end
146
146
 
147
+ meta_data = hash['@metadata'] ||= {}
148
+
147
149
  if tls_verified
148
- set_nested(hash, @field_tls_protocol_version, tls_session.getProtocol())
149
- set_nested(hash, @field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
150
- set_nested(hash, @field_tls_cipher, tls_session.getCipherSuite())
150
+ meta_data['tls_peer'] = { :status => "verified" }
151
151
 
152
- hash['@metadata']['tls_peer'] = {
153
- :status => "verified"
154
- }
152
+ set_nested(hash, input.field_tls_protocol_version, tls_session.getProtocol())
153
+ set_nested(hash, input.field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
154
+ set_nested(hash, input.field_tls_cipher, tls_session.getCipherSuite())
155
155
  else
156
- hash['@metadata']['tls_peer'] = {
157
- :status => "unverified"
158
- }
156
+ meta_data['tls_peer'] = { :status => "unverified" }
159
157
  end
160
158
  end
161
159
  end
@@ -166,9 +164,6 @@ module LogStash module Inputs class Beats
166
164
  field_ref = Java::OrgLogstash::FieldReference.from(field_name)
167
165
  # create @metadata sub-hash if needed
168
166
  if field_ref.type == Java::OrgLogstash::FieldReference::META_CHILD
169
- unless hash.key?("@metadata")
170
- hash["@metadata"] = {}
171
- end
172
167
  nesting_hash = hash["@metadata"]
173
168
  else
174
169
  nesting_hash = hash
@@ -129,6 +129,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
129
129
  config :executor_threads, :validate => :number, :default => LogStash::Config::CpuCoreStrategy.maximum
130
130
 
131
131
  attr_reader :field_hostname, :field_hostip
132
+ attr_reader :field_tls_protocol_version, :field_tls_peer_subject, :field_tls_cipher
132
133
 
133
134
  def register
134
135
  # For Logstash 2.4 we need to make sure that the logger is correctly set for the
@@ -167,10 +168,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
167
168
 
168
169
  # define ecs name mapping
169
170
  @field_hostname = ecs_select[disabled: "host", v1: "[@metadata][input][beats][host][name]"]
170
- @field_hostip = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
171
- @field_tls_protocol_version = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
172
- @field_tls_peer_subject = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
173
- @field_tls_cipher = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
171
+ @field_hostip = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
172
+ @field_tls_protocol_version = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
173
+ @field_tls_peer_subject = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
174
+ @field_tls_cipher = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
174
175
 
175
176
  @logger.info("Starting input listener", :address => "#{@host}:#{@port}")
176
177
 
@@ -7,5 +7,5 @@ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.9.10')
7
7
  require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
8
8
  require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.8')
9
9
  require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
10
- require_jar('org.apache.logging.log4j', 'log4j-api', '2.11.1')
11
- require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.0')
10
+ require_jar('org.apache.logging.log4j', 'log4j-api', '2.17.0')
11
+ require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.4')
@@ -12,26 +12,28 @@ describe LogStash::Inputs::Beats do
12
12
  let(:connection) { double("connection") }
13
13
  let(:certificate) { BeatsInputTest.certificate }
14
14
  let(:port) { BeatsInputTest.random_port }
15
+ let(:client_inactivity_timeout) { 400 }
16
+ let(:threads) { 1 + rand(9) }
15
17
  let(:queue) { Queue.new }
16
18
  let(:config) do
17
19
  {
18
- "port" => 0,
20
+ "port" => port,
19
21
  "ssl_certificate" => certificate.ssl_cert,
20
22
  "ssl_key" => certificate.ssl_key,
23
+ "client_inactivity_timeout" => client_inactivity_timeout,
24
+ "executor_threads" => threads,
21
25
  "type" => "example",
22
26
  "tags" => "beats"
23
27
  }
24
28
  end
25
29
 
30
+ subject(:plugin) { LogStash::Inputs::Beats.new(config) }
31
+
26
32
  context "#register" do
27
33
  context "host related configuration" do
28
- let(:config) { super().merge("host" => host, "port" => port, "client_inactivity_timeout" => client_inactivity_timeout, "executor_threads" => threads) }
34
+ let(:config) { super().merge("host" => host, "port" => port) }
29
35
  let(:host) { "192.168.1.20" }
30
- let(:port) { 9000 }
31
- let(:client_inactivity_timeout) { 400 }
32
- let(:threads) { 10 }
33
-
34
- subject(:plugin) { LogStash::Inputs::Beats.new(config) }
36
+ let(:port) { 9001 }
35
37
 
36
38
  it "sends the required options to the server" do
37
39
  expect(org.logstash.beats.Server).to receive(:new).with(host, port, client_inactivity_timeout, threads)
@@ -158,9 +160,80 @@ describe LogStash::Inputs::Beats do
158
160
 
159
161
  it "raise a ConfigurationError when multiline codec is set" do
160
162
  plugin = LogStash::Inputs::Beats.new(config)
161
- expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
163
+ expect { plugin.register }.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
164
+ end
165
+ end
166
+ end
167
+
168
+ context "tls meta-data" do
169
+ let(:config) { super().merge("host" => host, "ssl_peer_metadata" => true, "ssl_certificate_authorities" => [ certificate.ssl_cert ]) }
170
+ let(:host) { "192.168.1.20" }
171
+ let(:port) { 9002 }
172
+
173
+ let(:queue) { Queue.new }
174
+ let(:event) { LogStash::Event.new }
175
+
176
+ subject(:plugin) { LogStash::Inputs::Beats.new(config) }
177
+
178
+ before do
179
+ @server = org.logstash.beats.Server.new(host, port, client_inactivity_timeout, threads)
180
+ expect( org.logstash.beats.Server ).to receive(:new).with(host, port, client_inactivity_timeout, threads).and_return @server
181
+ expect( @server ).to receive(:listen)
182
+
183
+ subject.register
184
+ subject.run(queue) # listen does nothing
185
+ @message_listener = @server.getMessageListener
186
+
187
+ allow( ssl_engine = double('ssl_engine') ).to receive(:getSession).and_return ssl_session
188
+ allow( ssl_handler = double('ssl-handler') ).to receive(:engine).and_return ssl_engine
189
+ allow( pipeline = double('pipeline') ).to receive(:get).and_return ssl_handler
190
+ allow( @channel = double('channel') ).to receive(:pipeline).and_return pipeline
191
+ end
192
+
193
+ let(:ctx) do
194
+ Java::io.netty.channel.ChannelHandlerContext.impl do |method, *args|
195
+ fail("unexpected #{method}( #{args} )") unless method.eql?(:channel)
196
+ @channel
162
197
  end
163
198
  end
199
+
200
+ let(:ssl_session) do
201
+ Java::javax.net.ssl.SSLSession.impl do |method, *args|
202
+ case method
203
+ when :getPeerCertificates
204
+ [].to_java(java.security.cert.Certificate)
205
+ when :getProtocol
206
+ 'TLS-Mock'
207
+ when :getCipherSuite
208
+ 'SSL_NULL_WITH_TEST_SPEC'
209
+ when :getPeerPrincipal
210
+ javax.security.auth.x500.X500Principal.new('CN=TEST, OU=RSpec, O=Logstash, C=NL', {})
211
+ else
212
+ fail("unexpected #{method}( #{args} )")
213
+ end
214
+ end
215
+ end
216
+
217
+ let(:ssl_session_peer_principal) do
218
+ javax.security.auth.x500.X500Principal
219
+ end
220
+
221
+ let(:message) do
222
+ org.logstash.beats.Message.new(0, java.util.HashMap.new('foo' => 'bar'))
223
+ end
224
+
225
+ it 'sets tls fields' do
226
+ @message_listener.onNewMessage(ctx, message)
227
+
228
+ expect( queue.size ).to be 1
229
+ expect( event = queue.pop ).to be_a LogStash::Event
230
+
231
+ expect( event.get('[@metadata][tls_peer][status]') ).to eql 'verified'
232
+
233
+ expect( event.get('[@metadata][tls_peer][protocol]') ).to eql 'TLS-Mock'
234
+ expect( event.get('[@metadata][tls_peer][cipher_suite]') ).to eql 'SSL_NULL_WITH_TEST_SPEC'
235
+ expect( event.get('[@metadata][tls_peer][subject]') ).to eql 'CN=TEST,OU=RSpec,O=Logstash,C=NL'
236
+ end
164
237
  end
165
238
 
166
239
  context "when interrupting the plugin" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-beats
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.2.0
4
+ version: 6.2.4
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-03 00:00:00.000000000 Z
11
+ date: 2021-12-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -296,12 +296,10 @@ files:
296
296
  - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.10/jackson-core-2.9.10.jar
297
297
  - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.10.8/jackson-databind-2.9.10.8.jar
298
298
  - vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.10/jackson-module-afterburner-2.9.10.jar
299
- - vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar
300
299
  - vendor/jar-dependencies/io/netty/netty-all/4.1.65.Final/netty-all-4.1.65.Final.jar
301
- - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar
300
+ - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.17.0/log4j-api-2.17.0.jar
302
301
  - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
303
- - vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.1.3/logstash-input-beats-6.1.3.jar
304
- - vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.0/logstash-input-beats-6.2.0.jar
302
+ - vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.4/logstash-input-beats-6.2.4.jar
305
303
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
306
304
  licenses:
307
305
  - Apache License (2.0)