logstash-input-beats 6.2.4-java → 6.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e723378e0ae6fc82af7296c4e7be6482efdf2406bb2be40df42457a3377cc35c
-  data.tar.gz: 8919dc8acb0ebad123ec73b5e4184153f1da713d5ad6c5399607909d9ef0d407
+  metadata.gz: e50f7332c2cc7d230b49310fd987ff4ed575b0d8eb4e62526899af20bfabe0d5
+  data.tar.gz: dfeb260b1a5c7866b624c769abc2c780efa194dbadd5c18d1f2cb282ff58537b
 SHA512:
-  metadata.gz: d5a2b1f9a71f97f344e6886d8e3fd86157fabc20e68f46f96a406413044b49be55c26aaa6abe17cf4f155165c92d0852e35b808abc0febce211753af3c8b84f7
-  data.tar.gz: c12543f02573b5bf2c386feb87dd183de18eeac96e9cead8a1a4a92a705f6b9fc8fa6dee02c448e8f966bc2df99a2341fcfe2d1994e07748278ed43a745897e5
+  metadata.gz: e4be1cf87358e1e02479c7692d8317382ffe96e497f9ea658ce2774f6583253ce9a05fb7ab322722eea0c59ca06cad09718f95ffce6b7b5a53dcfa9b8b4c24d7
+  data.tar.gz: dcb8dae4e25448aec7105bea439c7c04bfcdc49752c99624fc26722d2f056024cb8942f7af926b486f1c9eb00fb6b0b80ae636bdf3fb164084125b4636681b28

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 6.3.0
+ - Added support for TLSv1.3. [#447](https://github.com/logstash-plugins/logstash-input-beats/pull/447)
+
+## 6.2.6
+  - Update guidance regarding the private key format and encoding [#445](https://github.com/logstash-plugins/logstash-input-beats/pull/445)
+
+## 6.2.5
+ - Build: do not package log4j-api dependency [#441](https://github.com/logstash-plugins/logstash-input-beats/pull/441).
+   Logstash provides the log4j framework and the dependency is not needed except testing and compiling.
+
 ## 6.2.4
  - Updated log4j dependency to 2.17.0
 

data/README.md

@@ -45,6 +45,13 @@ bundle install
 bundle exec rspec
 ```
 
+- Run integration tests
+
+```sh
+bundle exec rake test:integration:setup
+bundle exec rspec spec --tag integration  -fd
+```
+
 ### 2. Running your unpublished Plugin in Logstash
 
 #### 2.1 Run in a local Logstash clone
@@ -95,4 +102,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/VERSION

@@ -1 +1 @@
-6.2.4
+6.3.0

data/docs/index.asciidoc

@@ -199,9 +199,10 @@ Flag to determine whether to add `host` field to event using the value supplied
 ===== `cipher_suites`
 
   * Value type is <<array,array>>
-  * Default value is `java.lang.String[TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
+  * Default value is `java.lang.String[TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256]@459cfcca`
 
 The list of ciphers suite to use, listed by priorities.
+The default values applies for OpenJDK 11.0.14 and higher, for older versions the list does not include suites not supported by the JDK, such as the ChaCha20 family of ciphers.
 
 [id="plugins-{type}s-{plugin}-client_inactivity_timeout"]
 ===== `client_inactivity_timeout`
@@ -313,8 +314,14 @@ Time in milliseconds for an incomplete ssl handshake to timeout
   * There is no default value for this setting.
 
 SSL key to use.
-NOTE: This key need to be in the PKCS8 format, you can convert it with https://www.openssl.org/docs/man1.1.0/apps/pkcs8.html[OpenSSL]
-for more information.
+This key must be in the PKCS8 format and PEM encoded. 
+You can use the https://www.openssl.org/docs/man1.1.1/man1/openssl-pkcs8.html[openssl pkcs8] command to complete the conversion.
+For example, the command to convert a PEM encoded PKCS1 private key to a PEM encoded, non-encrypted PKCS8 key is:
+
+[source,sh]
+-----
+openssl pkcs8 -inform PEM -in path/to/logstash.key -topk8 -nocrypt -outform PEM -out path/to/logstash.pkcs8.key
+-----
 
 [id="plugins-{type}s-{plugin}-ssl_key_passphrase"]
 ===== `ssl_key_passphrase`
@@ -354,10 +361,10 @@ This option is only valid when `ssl_verify_mode` is set to `peer` or `force_peer
 ===== `tls_max_version`
 
   * Value type is <<number,number>>
-  * Default value is `1.2`
+  * Default value is `1.3`
 
 The maximum TLS version allowed for the encrypted connections. The value must be the one of the following:
-1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
+1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3
 
 [id="plugins-{type}s-{plugin}-tls_min_version"]
 ===== `tls_min_version`
@@ -366,7 +373,7 @@ The maximum TLS version allowed for the encrypted connections. The value must be
   * Default value is `1`
 
 The minimum TLS version allowed for the encrypted connections. The value must be one of the following:
-1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2
+1.0 for TLS 1.0, 1.1 for TLS 1.1, 1.2 for TLS 1.2, 1.3 for TLS 1.3
 
 
 
@@ -374,4 +381,3 @@ The minimum TLS version allowed for the encrypted connections. The value must be
 include::{include_path}/{type}.asciidoc[]
 
 :default_codec!:
-

data/lib/logstash/inputs/beats/tls.rb

@@ -18,7 +18,8 @@ module LogStash module Inputs class Beats
     TLS_PROTOCOL_OPTIONS = [
       TLSOption.new("TLSv1", 1),
       TLSOption.new("TLSv1.1", 1.1),
-      TLSOption.new("TLSv1.2", 1.2)
+      TLSOption.new("TLSv1.2", 1.2),
+      TLSOption.new("TLSv1.3", 1.3)
     ]
 
     def self.min

data/lib/logstash-input-beats_jars.rb

@@ -7,5 +7,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.9.10')
 require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.8')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.10')
-require_jar('org.apache.logging.log4j', 'log4j-api', '2.17.0')
-require_jar('org.logstash.beats', 'logstash-input-beats', '6.2.4')
+require_jar('org.logstash.beats', 'logstash-input-beats', '6.3.0')

data/lib/tasks/test.rake

@@ -4,9 +4,9 @@ VENDOR_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "ve
 
 #TODO: Figure out better means to keep this version in sync
 if OS_PLATFORM == "linux"
-  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.5.4-linux-x86_64.tar.gz"
+  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-linux-x86_64.tar.gz"
 elsif OS_PLATFORM == "darwin"
-  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.5.4-darwin-x86_64.tar.gz"
+  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.6.0-darwin-x86_64.tar.gz"
 end
 
 LSF_URL = "https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_#{OS_PLATFORM}_amd64"

data/spec/inputs/beats/tls_spec.rb

@@ -10,8 +10,8 @@ describe LogStash::Inputs::Beats::TLS do
   end
 
   it "returns the maximum supported tls" do
-    expect(subject.max.version).to eq(1.2)
-    expect(subject.max.name).to eq("TLSv1.2")
+    expect(subject.max.version).to eq(1.3)
+    expect(subject.max.name).to eq("TLSv1.3")
   end
 
   describe ".get_supported" do

data/spec/inputs/beats_spec.rb

@@ -166,7 +166,14 @@ describe LogStash::Inputs::Beats do
   end
 
   context "tls meta-data" do
-    let(:config) { super().merge("host" => host, "ssl_peer_metadata" => true, "ssl_certificate_authorities" => [ certificate.ssl_cert ]) }
+    let(:config) do
+      super().merge(
+          "host" => host,
+          "ssl_peer_metadata" => true,
+          "ssl_certificate_authorities" => [ certificate.ssl_cert ],
+          "ecs_compatibility" => 'disabled'
+      )
+    end
     let(:host) { "192.168.1.20" }
     let(:port) { 9002 }
 

data/spec/integration/filebeat_spec.rb

@@ -37,7 +37,7 @@ describe "Filebeat", :integration => true do
   let(:filebeat_config) do
     {
       "filebeat" => {
-        "prospectors" => [{ "paths" => [log_file],  "type" => "log" }],
+        "inputs" => [{ "paths" => [log_file],  "type" => "log" }],
         "scan_frequency" => "1s"
       },
       "output" => {
@@ -174,30 +174,65 @@ describe "Filebeat", :integration => true do
           end
         end
 
+        context "with TLSv1.3 client" do
+          let(:filebeat_config) do
+            super().merge({
+              "output" => {
+                "logstash" => {
+                  "hosts" => ["#{host}:#{port}"],
+                  "ssl" => {
+                    "certificate_authorities" => certificate_authorities,
+                    "versions" => ["TLSv1.3"],
+                  }
+                }
+              },
+              "logging" => { "level" => "debug" }
+              })
+          end
+          include_examples "send events"
+
+          context "when TLSv1.3 enforced in plugin" do
+            let(:input_config) {
+              super().merge({
+                "tls_min_version" => "1.3"
+              })
+            }
+
+            include_examples "send events"
+          end
+        end
+
         # Refactor this to use Flores's PKI instead of openssl command line
         # see: https://github.com/jordansissel/ruby-flores/issues/7
         context "with a passphrase" do
-          let!(:temporary_directory) { Stud::Temporary.pathname }
-          let(:certificate_key_file) { ::File.join(temporary_directory, "certificate.key") }
-          let(:certificate_key_file_pkcs8) { ::File.join(temporary_directory, "certificate.pkcs8.key") }
-          let(:certificate_file) { ::File.join(temporary_directory, "certificate.crt") }
-          let(:passphrase) { "foobar" }
-          let(:beats) {
-            # Since we are using a shared context, this not obvious to make sure the openssl command
-            # is run before starting beats so we do it just before initializing it.
-            FileUtils.mkdir_p(temporary_directory)
-            openssl_cmd = "openssl req -x509  -batch -newkey rsa:2048 -keyout #{temporary_directory}/certificate.key -out #{temporary_directory}/certificate.crt -subj /CN=localhost -passout pass:#{passphrase}"
-            system(openssl_cmd)
-            convert_key_cmd = "openssl pkcs8 -topk8 -in #{temporary_directory}/certificate.key -out #{certificate_key_file_pkcs8} -passin pass:#{passphrase} -passout pass:#{passphrase}"
-            system(convert_key_cmd)
-
-            LogStash::Inputs::Beats.new(input_config)
-          }
-          let(:input_config) {
-            super().merge({
-            "ssl_key_passphrase" => passphrase,
-            "ssl_key" => certificate_key_file_pkcs8
-          })}
+
+          before(:all) do
+            @passphrase = "foobar".freeze
+
+            FileUtils.mkdir_p temporary_directory = Stud::Temporary.pathname
+
+            cert_key = ::File.join(temporary_directory, "certificate.key")
+            @cert_pub = ::File.join(temporary_directory, "certificate.crt")
+            @cert_key_pkcs8 = ::File.join(temporary_directory, "certificate.key.pkcs8")
+
+            cmd = "openssl req -x509  -batch -newkey rsa:2048 -keyout #{cert_key} -out #{@cert_pub} -passout pass:#{@passphrase} -subj \"/C=EU/O=Logstash/CN=localhost\""
+            unless system(cmd)
+              fail "failed to run openssl command: #{$?} \n#{cmd}"
+            end
+
+            # NOTE: CentOS 7 base image (LS < 7.17) uses OpenSSL 1.0 while later is using Ubuntu 20.04 with OpenSSL 1.1.1
+            # the default algorithm for `openssl pkcs8 -topk8` changed to -v2 which Java does not support (see GH-443)
+            cmd = "openssl pkcs8 -topk8 -in #{cert_key} -out #{@cert_key_pkcs8} -v1 PBE-SHA1-RC2-128 -passin pass:#{@passphrase} -passout pass:#{@passphrase}"
+            unless system(cmd)
+              fail "failed to run openssl command: #{$?} \n#{cmd}"
+            end
+          end
+
+          let(:certificate_authorities) { [ @cert_pub ] }
+
+          let(:input_config) do
+            super().merge("ssl_key_passphrase" => @passphrase, "ssl_key" => @cert_key_pkcs8, "ssl_certificate" => @cert_pub)
+          end
 
           include_examples "send events"
         end

data/spec/support/file_helpers.rb

@@ -19,7 +19,7 @@ module FileHelpers
   end
 
   def write_to_tmp_file(content)
-    file = Stud::Temporary.file
+    file = Stud::Temporary.file("test-logstash-input-beats", "w+", 0600)
     file.write(content.to_s)
     file.close
     file.path

data/spec/support/integration_shared_context.rb

@@ -50,6 +50,7 @@ shared_context "beats configuration" do
       begin
         beats.run(queue)
       rescue => e
+        warn e.inspect if $VERBOSE
         retry unless beats.stop?
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 6.2.4
+  version: 6.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-18 00:00:00.000000000 Z
+date: 2022-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -297,9 +297,8 @@ files:
 - vendor/jar-dependencies/com/fasterxml/jackson/core/jackson-databind/2.9.10.8/jackson-databind-2.9.10.8.jar
 - vendor/jar-dependencies/com/fasterxml/jackson/module/jackson-module-afterburner/2.9.10/jackson-module-afterburner-2.9.10.jar
 - vendor/jar-dependencies/io/netty/netty-all/4.1.65.Final/netty-all-4.1.65.Final.jar
-- vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.17.0/log4j-api-2.17.0.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.24.0-GA/javassist-3.24.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.2.4/logstash-input-beats-6.2.4.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/6.3.0/logstash-input-beats-6.3.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)