logstash-input-beats 3.1.0-java → 3.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f99fa6cabf279fddd07fb11aa025fbd6d57b303
-  data.tar.gz: ecf36cdb6ac73a282e6b2f3f7410e1c798efde10
+  metadata.gz: 768ccce03701a2fe20bcdefafe058fb754cfa00a
+  data.tar.gz: e28accad116b6b3e4feed499963e335cb7ce156c
 SHA512:
-  metadata.gz: 7c0bfd154e6db74356c9473cb6996744724539758fbd61baa326effd5435a364479c137c413daeceee5595d3f358290660293d5d34b2862c729e1905bbf071cb
-  data.tar.gz: 91436a6a28b8de8594aca1d263c879bc8e06682609fe3f0096a9be33bd598a891b147b5c10bbc18edc95249b13d4c17030387807873deb7eec2cf18a942b3c6f
+  metadata.gz: b3323df3ca381add52492e2162950cad4179f40b06d96d8eaceac395e0dfacb2011b50784e1d705241ad047f2d89086030065093623728fb4a365d364ea8da51
+  data.tar.gz: 79b695fa6f4f9c4316df81b7cbbdea25435358917e6eb32a53ed097a138dcd19bed249ae44b40738e226c4fdcb9a95b7886dc4463caabe87fa53833bbaaed2fa

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 3.1.1
+   - Remove the SSL Converter, Private Key must be in the PKCS8 format, which is the default of any newer OpenSSL library
+   - Replace FileInputStream with File reference to let netty handle correctly the certificates
+   - Tests now uses OpenSSL binary to convert PKCS7 Private generated from ruby to PKCS8
+   - Remove dependency on bouncycastle
+   - Fix an issue when the input could hang forever when stopping Logstash
+   - [Doc changes] Add Logstash config example and clarify use of the `type` config option
+
+
+
 ## 3.1.0
    - Fix a NullPointer Exception https://github.com/elastic/logstash/issues/5756
    - Log4j ERROR will now be propagated upstream, IE: InvalidCertificate OR InvalidFrameType.
@@ -62,7 +72,7 @@
 
 ## 2.2.4
 
-  - Fix bug where using `ssl_key_passphrase` wouldn't work 
+  - Fix bug where using `ssl_key_passphrase` wouldn't work
 
 ## 2.2.2
 
@@ -93,7 +103,7 @@
 
 ## 2.1.1
 
-  - Release a new version of the gem that doesn't included any other gems, 2.1.0 is yanked from rubygems 
+  - Release a new version of the gem that doesn't included any other gems, 2.1.0 is yanked from rubygems
 
 ## 2.1.0
 

data/VERSION

@@ -1 +1 @@
-3.1.0
+3.1.1

data/lib/logstash-input-beats_jars.rb

@@ -3,8 +3,6 @@
 require 'jar_dependencies'
 require_jar('org.apache.logging.log4j', 'log4j-1.2-api', '2.6.1')
 require_jar('org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.1')
-require_jar('org.bouncycastle', 'bcprov-jdk15on', '1.54')
-require_jar('org.bouncycastle', 'bcpkix-jdk15on', '1.54')
 require_jar('io.netty', 'netty-all', '4.1.3.Final')
 require_jar('io.netty', 'netty-tcnative-boringssl-static', '1.1.33.Fork17')
 require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.1')
@@ -14,4 +12,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.7.5')
 require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.7.5')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.5')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.5')
-require_jar('org.logstash.beats', 'logstash-input-beats', '3.1.0')
+require_jar('org.logstash.beats', 'logstash-input-beats', '3.1.1')

data/lib/logstash/inputs/beats.rb

@@ -10,12 +10,37 @@ require "logstash/logging" rescue nil # removed in logstash 5
 
 import "org.logstash.beats.Server"
 import "org.logstash.netty.SslSimpleBuilder"
-import "org.logstash.netty.PrivateKeyConverter"
 import "java.io.FileInputStream"
 
 # This input plugin enables Logstash to receive events from the
 # https://www.elastic.co/products/beats[Elastic Beats] framework.
 #
+# The following example shows how to configure Logstash to listen on port
+# 5044 for incoming Beats connections and to index into Elasticsearch:
+#
+# [source,ruby]
+# ------------------------------------------------------------------------------
+# input {
+#   beats {
+#     port => 5044
+#   }
+# }
+# 
+# output {
+#   elasticsearch {
+#     hosts => "localhost:9200"
+#     manage_template => false
+#     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+#     document_type => "%{[@metadata][type]}"
+#   }
+# }
+# ------------------------------------------------------------------------------
+#
+# NOTE: The Beats shipper automatically sets the `type` field on the event.
+# You cannot override this setting in the Logstash config. If you specify
+# a setting for the <<plugins-inputs-beats-type,`type`>> config option in
+# Logstash, it is ignored.
+#
 class LogStash::Codecs::Base
   # This monkey patch add callback based
   # flow to the codec until its shipped with core.
@@ -143,8 +168,7 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   def create_server
     server = org.logstash.beats.Server.new(@port)
     if @ssl
-      private_key_converter = org.logstash.netty.PrivateKeyConverter.new(ssl_key, ssl_key_passphrase)
-      ssl_builder = org.logstash.netty.SslSimpleBuilder.new(FileInputStream.new(ssl_certificate), private_key_converter.convert(), ssl_key_passphrase)
+      ssl_builder = org.logstash.netty.SslSimpleBuilder.new(ssl_certificate, ssl_key, ssl_key_passphrase)
         .setProtocols(convert_protocols)
         .setCipherSuites(normalized_ciphers)
 

data/spec/integration/filebeat_spec.rb

@@ -15,6 +15,7 @@ require_relative "../support/client_process_helpers"
 
 FILEBEAT_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "filebeat", "filebeat"))
 
+
 describe "Filebeat", :integration => true do
   include ClientProcessHelpers
   include FileHelpers
@@ -34,13 +35,13 @@ describe "Filebeat", :integration => true do
 
   let_empty_tmp_file(:registry_file)
   let(:filebeat_config) do
-    { 
-      "filebeat" => { 
+    {
+      "filebeat" => {
         "prospectors" => [{ "paths" => [log_file],  "input_type" => "log" }],
         "registry_file" => registry_file,
         "scan_frequency" => "1s"
       },
-      "output" => { 
+      "output" => {
         "logstash" => { "hosts" => ["#{host}:#{port}"] },
         "logging" => { "level" => "debug" }
       }
@@ -99,7 +100,8 @@ describe "Filebeat", :integration => true do
 
       let(:certificate_authorities) { [certificate_file] }
       let(:certificate_data) { Flores::PKI.generate }
-      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+
+      let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
       let_tmp_file(:certificate_file) { certificate_data.first }
 
       context "self signed certificate" do
@@ -155,10 +157,10 @@ describe "Filebeat", :integration => true do
         context "with a self signed certificate" do
           let(:certificate_authorities) { [certificate_file] }
           let(:certificate_data) { Flores::PKI.generate }
-          let_tmp_file(:certificate_key_file) { certificate_data.last } 
+          let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
           let_tmp_file(:certificate_file) { certificate_data.first }
           let_tmp_file(:server_certificate_file) { certificate_data.first }
-          let_tmp_file(:server_certificate_key_file) { certificate_data.last }
+          let_tmp_file(:server_certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
 
           include_examples "send events"
         end
@@ -167,7 +169,7 @@ describe "Filebeat", :integration => true do
           include_context "Root CA"
 
           let_tmp_file(:server_certificate_file) { server_certificate_data.first }
-          let_tmp_file(:server_certificate_key_file) { server_certificate_data.last }
+          let_tmp_file(:server_certificate_key_file) { convert_to_pkcs8(server_certificate_data.last) }
 
           context "directly signed client certificate" do
             let(:certificate_authorities) { [root_ca_certificate_file] }
@@ -197,7 +199,7 @@ describe "Filebeat", :integration => true do
 
               let(:secondary_client_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", secondary_ca_certificate, secondary_ca_key) }
               let_tmp_file(:secondary_client_certificate_file) { secondary_client_certificate_data.first }
-              let_tmp_file(:secondary_client_certificate_key_file) { secondary_client_certificate_data.last }
+              let_tmp_file(:secondary_client_certificate_key_file) { convert_to_pkcs8(secondary_client_certificate_data.last) }
               let(:certificate_authorities) { [root_ca_certificate_file, secondary_ca_certificate_file] }
               let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
 

data/spec/integration/logstash_forwarder_spec.rb

@@ -84,7 +84,7 @@ describe "Logstash-Forwarder", :integration => true do
 
       let(:certificate_data) { Flores::PKI.generate }
       let_tmp_file(:certificate_file) { certificate_data.first }
-      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+      let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
       let(:certificate_authorities) { certificate_file }
 
       context "self signed certificate" do

data/spec/spec_helper.rb

@@ -1,10 +1,11 @@
 # encoding: utf-8
-require 'rspec'
-require 'rspec/mocks'
-require 'rspec/wait'
+require "rspec"
+require "rspec/mocks"
+require "rspec/wait"
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/plain"
 require_relative "support/logstash_test"
+require_relative "support/helpers"
 
 $: << File.realpath(File.join(File.dirname(__FILE__), "..", "lib"))
 

data/spec/support/helpers.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "open3"
+OPEN_SSL_TOPK8 = "openssl pkcs8 -nocrypt -topk8 -inform PEM -outform PEM"
+
+# Netty only accepts PKC8 format for the private key, which in the real world is fine
+# because any newer version of OpenSSL with use that format by default.
+#
+# But in Ruby or Jruby-OpenSSL, the private key will be generates in PKCS7, which netty doesn't support.
+# Converting the format is a bit of hassle to do in code so In this case its just easier to use the `openssl` binary to do the work.
+#
+#
+def convert_to_pkcs8(key)
+  out, e, s = Open3.capture3(OPEN_SSL_TOPK8, :stdin_data => key.to_s)
+  raise e if e != ""
+  out
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.1.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-24 00:00:00.000000000 Z
+date: 2016-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -256,6 +256,7 @@ files:
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb
 - spec/support/flores_extensions.rb
+- spec/support/helpers.rb
 - spec/support/integration_shared_context.rb
 - spec/support/logstash_test.rb
 - spec/support/shared_examples.rb
@@ -269,10 +270,8 @@ files:
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-api/2.6.1/log4j-api-2.6.1.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-core/2.6.1/log4j-core-2.6.1.jar
 - vendor/jar-dependencies/org/apache/logging/log4j/log4j-slf4j-impl/2.6.1/log4j-slf4j-impl-2.6.1.jar
-- vendor/jar-dependencies/org/bouncycastle/bcpkix-jdk15on/1.54/bcpkix-jdk15on-1.54.jar
-- vendor/jar-dependencies/org/bouncycastle/bcprov-jdk15on/1.54/bcprov-jdk15on-1.54.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.20.0-GA/javassist-3.20.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/3.1.0/logstash-input-beats-3.1.0.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/3.1.1/logstash-input-beats-3.1.1.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -314,6 +313,7 @@ test_files:
 - spec/support/client_process_helpers.rb
 - spec/support/file_helpers.rb
 - spec/support/flores_extensions.rb
+- spec/support/helpers.rb
 - spec/support/integration_shared_context.rb
 - spec/support/logstash_test.rb
 - spec/support/shared_examples.rb