logstash-input-beats 6.2.0-java → 6.2.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16a55f0a53247ed30b7110d9f60dce425288fdd68c1bf936283698028bdfd421
-  data.tar.gz: d8cda82e3d3a8fb3ceef48a96f52674eb04496e7a0957043b34d5c5e741484fa
+  metadata.gz: 53b66be9656d02212b1449645c83e128eb4fb32ceaa26ec76bc725a8cdd06a3a
+  data.tar.gz: 7ed11fe837aa9c6bf439ca0d7e16fdebec26e359f658410e18b6ec793a472ca5
 SHA512:
-  metadata.gz: e2869c6c573223e979f411c430ce1ddb6f65f6c967930b469c617a03b6a4fb0417a7eecfc23e2c8061599a72a07f656490bb76c9ceb5576f00613be0cdfde688
-  data.tar.gz: 38dd3f7bfb548565ecc047bc0e044e89796f2fcbdf8187f2b3a37fc1bb822a00d73c759713996dd69471fbaa1e334fd3a4b1e730adcb0093ccf6956f14f4549e
+  metadata.gz: 6c75faeb9ae359e700f2feffeab892a200c9e011102ee4d21211813647749630776a9f4e88336dddc7873ae223623cc84852c28e06e3f6e3788a3aa1ba57e4a7
+  data.tar.gz: af9985a8fd2fed373f455d8980721f05a2473603a578593476af13f009a18ea1527bb7164db20ae0c364f98e49703fee95c7e21706729c2e0c6fd2dad411e4cb

data/CHANGELOG.md

@@ -1,8 +1,12 @@
+## 6.2.1
+ - Fix: LS failing with `ssl_peer_metadata => true` [#431](https://github.com/logstash-plugins/logstash-input-beats/pull/431)
+ - [DOC] described `executor_threads` configuration parameter [#421](https://github.com/logstash-plugins/logstash-input-beats/pull/421)
+
 ## 6.2.0
  - ECS compatibility enablement: Adds alias to support upcoming ECS v8 with the existing ECS v1 implementation
 
 ## 6.1.7
- - [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-http/pull/428)
+ - [DOC] Remove limitations topic and link [#428](https://github.com/logstash-plugins/logstash-input-beats/pull/428)
 
 ## 6.1.6
  - [DOC] Applied more attributes to manage plugin name in doc content, and implemented conditional text processing. [#423](https://github.com/logstash-plugins/logstash-input-http/pull/423)

data/VERSION

@@ -1 +1 @@
-6.2.0
+6.2.1

data/docs/index.asciidoc

@@ -163,6 +163,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
+| <<plugins-{type}s-{plugin}-executor_threads>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-include_codec_tag>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-port>> |<<number,number>>|Yes
@@ -224,6 +225,24 @@ Close Idle clients after X seconds of inactivity.
 
 Refer to <<plugins-{type}s-{plugin}-ecs_metadata,ECS mapping>> for detailed information.
 
+[id="plugins-{type}s-{plugin}-executor_threads"]
+===== `executor_threads`
+
+  * Value type is <<number,number>>
+  * Default value is 1 executor thread per CPU core
+
+The number of threads to be used to process incoming beats requests.
+By default Beats input will create a number of threads equals to 2*CPU cores to handle incoming connections,
+reading from the established sockets and execute most of the tasks related to network connection managements, 
+except the parsing of Lumberjack protocol that's offloaded to a dedicated thread pool.
+
+Generally you don't need to touch this setting.
+In case you are sending very large events and observing "OutOfDirectMemory" exceptions,
+you may want to reduce this number to half or 1/4 of the CPU cores.
+This will reduce the number of threads decompressing batches of data into direct memory.
+However, this will only be a mitigating tweak, as the proper solution may require resizing your Logstash deployment,
+either by increasing number of Logstash nodes or increasing the JVM's Direct Memory.
+
 [id="plugins-{type}s-{plugin}-host"]
 ===== `host`
 
@@ -355,3 +374,4 @@ The minimum TLS version allowed for the encrypted connections. The value must be
 include::{include_path}/{type}.asciidoc[]
 
 :default_codec!:
+

data/lib/logstash/inputs/beats/message_listener.rb

@@ -132,7 +132,7 @@ module LogStash module Inputs class Beats
         tls_session = ctx.channel().pipeline().get("ssl-handler").engine().getSession()
         tls_verified = true
 
-        if not @input.client_authentication_required?
+        unless @input.client_authentication_required?
           # throws SSLPeerUnverifiedException if unverified
           begin
             tls_session.getPeerCertificates()
@@ -144,18 +144,16 @@ module LogStash module Inputs class Beats
           end
         end
 
+        meta_data = hash['@metadata'] ||= {}
+
         if tls_verified
-          set_nested(hash, @field_tls_protocol_version, tls_session.getProtocol())
-          set_nested(hash, @field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
-          set_nested(hash, @field_tls_cipher, tls_session.getCipherSuite())
+          meta_data['tls_peer'] = { :status => "verified" }
 
-          hash['@metadata']['tls_peer'] = {
-            :status       => "verified"
-          }
+          set_nested(hash, input.field_tls_protocol_version, tls_session.getProtocol())
+          set_nested(hash, input.field_tls_peer_subject, tls_session.getPeerPrincipal().getName())
+          set_nested(hash, input.field_tls_cipher, tls_session.getCipherSuite())
         else
-          hash['@metadata']['tls_peer'] = {
-            :status     => "unverified"
-          }
+          meta_data['tls_peer'] = { :status => "unverified" }
         end
       end
     end
@@ -166,9 +164,6 @@ module LogStash module Inputs class Beats
       field_ref = Java::OrgLogstash::FieldReference.from(field_name)
       # create @metadata sub-hash if needed
       if field_ref.type == Java::OrgLogstash::FieldReference::META_CHILD
-        unless hash.key?("@metadata")
-          hash["@metadata"] = {}
-        end
         nesting_hash = hash["@metadata"]
       else
         nesting_hash = hash

data/lib/logstash/inputs/beats.rb

@@ -129,6 +129,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   config :executor_threads, :validate => :number, :default => LogStash::Config::CpuCoreStrategy.maximum
 
   attr_reader :field_hostname, :field_hostip
+  attr_reader :field_tls_protocol_version, :field_tls_peer_subject, :field_tls_cipher
 
   def register
     # For Logstash 2.4 we need to make sure that the logger is correctly set for the
@@ -167,10 +168,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
 
     # define ecs name mapping
     @field_hostname = ecs_select[disabled: "host", v1: "[@metadata][input][beats][host][name]"]
-    @field_hostip   = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
-    @field_tls_protocol_version   = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
-    @field_tls_peer_subject   = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
-    @field_tls_cipher   = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
+    @field_hostip = ecs_select[disabled: "[@metadata][ip_address]", v1: "[@metadata][input][beats][host][ip]"]
+    @field_tls_protocol_version = ecs_select[disabled: "[@metadata][tls_peer][protocol]", v1: "[@metadata][input][beats][tls][version_protocol]"]
+    @field_tls_peer_subject = ecs_select[disabled: "[@metadata][tls_peer][subject]", v1: "[@metadata][input][beats][tls][client][subject]"]
+    @field_tls_cipher = ecs_select[disabled: "[@metadata][tls_peer][cipher_suite]", v1: "[@metadata][input][beats][tls][cipher]"]
 
     @logger.info("Starting input listener", :address => "#{@host}:#{@port}")
 

data/lib/logstash-input-beats_jars.rb

@@ -8,4 +8,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.8')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.11.1')
-require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.0')
+require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.1')

data/spec/inputs/beats_spec.rb

@@ -12,26 +12,28 @@ describe LogStash::Inputs::Beats do
   let(:connection) { double("connection") }
   let(:certificate) { BeatsInputTest.certificate }
   let(:port) { BeatsInputTest.random_port }
+  let(:client_inactivity_timeout) { 400 }
+  let(:threads) { 1 + rand(9) }
   let(:queue)  { Queue.new }
   let(:config)   do
     {
-        "port" => 0,
+        "port" => port,
         "ssl_certificate" => certificate.ssl_cert,
         "ssl_key" => certificate.ssl_key,
+        "client_inactivity_timeout" => client_inactivity_timeout,
+        "executor_threads" => threads,
         "type" => "example",
         "tags" => "beats"
     }
   end
 
+  subject(:plugin) { LogStash::Inputs::Beats.new(config) }
+
   context "#register" do
     context "host related configuration" do
-      let(:config) { super().merge("host" => host, "port" => port, "client_inactivity_timeout" => client_inactivity_timeout, "executor_threads" => threads) }
+      let(:config) { super().merge("host" => host, "port" => port) }
       let(:host) { "192.168.1.20" }
-      let(:port) { 9000 }
-      let(:client_inactivity_timeout) { 400 }
-      let(:threads) { 10 }
-
-      subject(:plugin) { LogStash::Inputs::Beats.new(config) }
+      let(:port) { 9001 }
 
       it "sends the required options to the server" do
         expect(org.logstash.beats.Server).to receive(:new).with(host, port, client_inactivity_timeout, threads)
@@ -158,9 +160,80 @@ describe LogStash::Inputs::Beats do
 
       it "raise a ConfigurationError when multiline codec is set" do
         plugin = LogStash::Inputs::Beats.new(config)
-        expect {plugin.register}.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, "Multiline codec with beats input is not supported. Please refer to the beats documentation for how to best manage multiline data. See https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html")
+      end
+    end
+  end
+
+  context "tls meta-data" do
+    let(:config) { super().merge("host" => host, "ssl_peer_metadata" => true, "ssl_certificate_authorities" => [ certificate.ssl_cert ]) }
+    let(:host) { "192.168.1.20" }
+    let(:port) { 9002 }
+
+    let(:queue) { Queue.new }
+    let(:event) { LogStash::Event.new }
+
+    subject(:plugin) { LogStash::Inputs::Beats.new(config) }
+
+    before do
+      @server = org.logstash.beats.Server.new(host, port, client_inactivity_timeout, threads)
+      expect( org.logstash.beats.Server ).to receive(:new).with(host, port, client_inactivity_timeout, threads).and_return @server
+      expect( @server ).to receive(:listen)
+
+      subject.register
+      subject.run(queue) # listen does nothing
+      @message_listener = @server.getMessageListener
+
+      allow( ssl_engine = double('ssl_engine') ).to receive(:getSession).and_return ssl_session
+      allow( ssl_handler = double('ssl-handler') ).to receive(:engine).and_return ssl_engine
+      allow( pipeline = double('pipeline') ).to receive(:get).and_return ssl_handler
+      allow( @channel = double('channel') ).to receive(:pipeline).and_return pipeline
+    end
+
+    let(:ctx) do
+      Java::io.netty.channel.ChannelHandlerContext.impl do |method, *args|
+        fail("unexpected #{method}( #{args} )") unless method.eql?(:channel)
+        @channel
       end
     end
+
+    let(:ssl_session) do
+      Java::javax.net.ssl.SSLSession.impl do |method, *args|
+        case method
+        when :getPeerCertificates
+          [].to_java(java.security.cert.Certificate)
+        when :getProtocol
+          'TLS-Mock'
+        when :getCipherSuite
+          'SSL_NULL_WITH_TEST_SPEC'
+        when :getPeerPrincipal
+          javax.security.auth.x500.X500Principal.new('CN=TEST, OU=RSpec, O=Logstash, C=NL', {})
+        else
+          fail("unexpected #{method}( #{args} )")
+        end
+      end
+    end
+
+    let(:ssl_session_peer_principal) do
+      javax.security.auth.x500.X500Principal
+    end
+
+    let(:message) do
+      org.logstash.beats.Message.new(0, java.util.HashMap.new('foo' => 'bar'))
+    end
+
+    it 'sets tls fields' do
+      @message_listener.onNewMessage(ctx, message)
+
+      expect( queue.size ).to be 1
+      expect( event = queue.pop ).to be_a LogStash::Event
+
+      expect( event.get('[@metadata][tls_peer][status]') ).to eql 'verified'
+
+      expect( event.get('[@metadata][tls_peer][protocol]') ).to eql 'TLS-Mock'
+      expect( event.get('[@metadata][tls_peer][cipher_suite]') ).to eql 'SSL_NULL_WITH_TEST_SPEC'
+      expect( event.get('[@metadata][tls_peer][subject]') ).to eql 'CN=TEST,OU=RSpec,O=Logstash,C=NL'
+    end
   end
 
   context "when interrupting the plugin" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 6.2.0
+  version: 6.2.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2021-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -296,12 +296,10 @@ files:
 - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-core/2.9.10/jackson-core-2.9.10.jar
 - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.10.8/jackson-databind-2.9.10.8.jar
 - vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.10/jackson-module-afterburner-2.9.10.jar
-- vendor/jar-dependencies/io/netty/netty-all/4.1.49.Final/netty-all-4.1.49.Final.jar
 - vendor/jar-dependencies/io/netty/netty-all/4.1.65.Final/netty-all-4.1.65.Final.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.11.1/log4j-api-2.11.1.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.1.3/logstash-input-beats-6.1.3.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.0/logstash-input-beats-6.2.0.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.1/logstash-input-beats-6.2.1.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)