logstash-input-beats 4.0.2-java → 4.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 531fc3f9d3ae313fc42ee5624caeb51679696127
-  data.tar.gz: 3ca0d6794c140249ebdb912b0a3955f73a1740da
+  metadata.gz: d406faf577912c1b8e53353eb01a297b91ca4de5
+  data.tar.gz: ac879f92b4b6e6699ed9c56fd40db3248b6810ba
 SHA512:
-  metadata.gz: f0d7f04262228d660c39631ca10896ea038437c2be039b2aea933c5c3310391bd583f534d5783c9906389eec13e273fe6978ee0871280e5906083f6745a7e021
-  data.tar.gz: 67020e94ebb1a72d8f68fed727950ba980be9ebb9cd19b712632998615445adcd96c556671b8b6a046c119187ae12baf4c461ae501194ab55f544402369d2f7d
+  metadata.gz: 8cb70e0661a794b5be172bf87c7b5d186b0107d3e37cee6590592610ce00c06b2d05c494febbf9afb48b6a77a599199919f9c5f89063ad9dc766bc1419c4cd96
+  data.tar.gz: ba72af919b0d49bb3fb392e22a8d2475a0a11c92c2d321881c9f07cce7ffa5e4f25bc9ce74567d7d51c7e96dae196bd71d2430beb1219bb0ba309f029db05fcc

data/CHANGELOG.md

@@ -1,14 +1,29 @@
+## 4.0.3
+ - Include remote ip_address in metadata. #180
+ - Require Java 8 #221
+ - Fix ability to set SSL protocols  #228
+
 ## 4.0.2
- - Relax version of concurrent-ruby to `~> 1.0`
+  - Relax version of concurrent-ruby to `~> 1.0` #216
 
 ## 4.0.1
- - Breaking change: Logstash will no longer start when multiline codec is used with the Beats input plugin
+ - Breaking change: Logstash will no longer start when multiline codec is used with the Beats input plugin #201
 
 ## 4.0.0
- - Verion yanked from RubyGems for packaging issues
+ - Version yanked from RubyGems for packaging issues
+
+## 3.1.19 
+   - Fix ability to set SSL protocols  #228
+   
+## 3.1.18
+  - Relax version of concurrent-ruby to ~> 1.0 #216
+  
+## 3.1.17 
+ - Docs: Add note indicating that the multiline codec should not be used with the Beats input plugin
+ - Deprecate warning for multiline codec with the Beats input plugin
 
 ## 3.1.16
- - Docs: Add note indicating that the multiline codec should not be used with the Beats input plugin
+ - Version yanked from RubyGems for packaging issues
 
 ## 3.1.15
  - DEBUG: Add information about the remote when an exception is catched #192

data/VERSION

@@ -1 +1 @@
-4.0.2
+4.0.3

data/docs/index.asciidoc

@@ -14,7 +14,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 [id="plugins-{type}-{plugin}"]
 
-=== Beats
+=== Beats input plugin
 
 include::{include_path}/plugin_header.asciidoc[]
 

data/lib/logstash-input-beats_jars.rb

@@ -9,4 +9,4 @@ require_jar('com.fasterxml.jackson.core', 'jackson-annotations', '2.7.5')
 require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.5')
 require_jar('com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.7.5')
 require_jar('log4j', 'log4j', '1.2.17')
-require_jar('org.logstash.beats', 'logstash-input-beats', '4.0.2')
+require_jar('org.logstash.beats', 'logstash-input-beats', '4.0.3')

data/lib/logstash/inputs/beats.rb

@@ -39,8 +39,7 @@ require_relative "beats/patch"
 # IMPORTANT: If you are shipping events that span multiple lines, you need to
 # use the configuration options available in Filebeat to handle multiline events
 # before sending the event data to Logstash. You cannot use the
-# <<plugins-codecs-multiline>> codec to handle multiline events. Doing so may
-# result in the mixing of streams and corrupted event data. 
+# <<plugins-codecs-multiline>> codec to handle multiline events.
 #
 class LogStash::Inputs::Beats < LogStash::Inputs::Base
   require "logstash/inputs/beats/codec_callback_listener"

data/lib/logstash/inputs/beats/message_listener.rb

@@ -29,6 +29,12 @@ module LogStash module Inputs class Beats
     def onNewMessage(ctx, message)
       hash = message.getData()
 
+      begin
+        hash.get("@metadata").put("ip_address", ctx.channel().remoteAddress().getAddress().getHostAddress())
+      rescue #should never happen, but don't allow an error here to stop beats input
+        input.logger.warn("Could not retrieve remote IP address for beats input.")
+      end
+
       target_field = extract_target_field(hash)
 
       if target_field.nil?

data/lib/tasks/test.rake

@@ -2,10 +2,11 @@
 OS_PLATFORM = RbConfig::CONFIG["host_os"]
 VENDOR_PATH = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor"))
 
+#TODO: Figure out better means to keep this version in sync
 if OS_PLATFORM == "linux"
-  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.0.0-linux-x86_64.tar.gz"
+  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.0.0-alpha2-linux-x86_64.tar.gz"
 elsif OS_PLATFORM == "darwin"
-  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.0.0-darwin-x86_64.tar.gz"
+  FILEBEAT_URL = "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.0.0-alpha2-darwin-x86_64.tar.gz"
 end
 
 LSF_URL = "https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_#{OS_PLATFORM}_amd64"

data/spec/inputs/beats/message_listener_spec.rb

@@ -6,7 +6,10 @@ require "logstash/inputs/beats/message_listener"
 require "logstash/instrument/namespaced_null_metric"
 require "thread"
 
+java_import java.util.HashMap
+
 class MockMessage
+
   def initialize(identity_stream, data = {})
     @identity_stream = identity_stream
     @data = data
@@ -21,6 +24,22 @@ class MockMessage
   end
 end
 
+# General purpose single method mock class. Will keep generating mocks until requested method name (no args) is found.
+class OngoingMethodMock
+  def initialize(method_name, return_value)
+    @method_name = method_name
+    @return_value = return_value
+  end
+
+  def method_missing(method)
+    if(method.to_s.eql? @method_name)
+      return @return_value
+    else
+      return OngoingMethodMock.new(@method_name, @return_value)
+    end
+  end
+end
+
 class DummyCodec < LogStash::Codecs::Base
   DUMMY_EVENT = LogStash::Event.new
 
@@ -98,7 +117,14 @@ describe LogStash::Inputs::Beats::MessageListener do
     end
 
     context "when the message is from any libbeat" do
-      let(:message) { MockMessage.new("abc",  { "metric" => 1, "name" => "super-stats"} )}
+      #Requires data modeled as Java, not Ruby since the actual code pulls from Java backed (Netty) object
+      data = java.util.HashMap.new
+      data.put("@metadata", java.util.HashMap.new)
+      data.put("metric",  1)
+      data.put("name", "super-stats")
+
+      let(:message) { MockMessage.new("abc",  data)}
+      let(:ctx) {OngoingMethodMock.new("getHostAddress", "10.0.0.1")}
 
       it "extract the event" do
         subject.onNewMessage(ctx, message)
@@ -106,8 +132,10 @@ describe LogStash::Inputs::Beats::MessageListener do
         expect(event.get("message")).to be_nil
         expect(event.get("metric")).to eq(1)
         expect(event.get("name")).to eq("super-stats")
+        expect(event.get("[@metadata][ip_address]")).to eq("10.0.0.1")
       end
     end
+
   end
 
   context "onException" do

data/spec/integration/filebeat_spec.rb

@@ -52,6 +52,7 @@ describe "Filebeat", :integration => true do
   before :each do
     FileUtils.rm_rf(File.join(File.dirname(__FILE__), "..", "..", "vendor", "filebeat", "data"))
     start_client
+    raise 'Filebeat did not start in alloted time' unless is_alive
     sleep(20) # give some time to FB to send something
   end
 
@@ -76,7 +77,7 @@ describe "Filebeat", :integration => true do
   end
 
   ############################################################
-  # Actuals tests 
+  # Actuals tests
   context "Plain TCP" do
     include_examples "send events"
   end
@@ -190,6 +191,9 @@ describe "Filebeat", :integration => true do
       context "CA root" do
         include_context "Root CA"
 
+        let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
+        let_tmp_file(:certificate_file) { certificate_data.first }
+
         context "directly signed client certificate" do
           let(:certificate_authorities) { [root_ca_certificate_file] }
           let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
@@ -202,7 +206,7 @@ describe "Filebeat", :integration => true do
 
           let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
           let(:certificate_authorities) { [certificate_authorities_chain] }
-          
+
           include_examples "send events"
         end
       end
@@ -224,7 +228,7 @@ describe "Filebeat", :integration => true do
         end
 
         let(:input_config) do
-          super.merge({ 
+          super.merge({
             "ssl" => true,
             "ssl_certificate_authorities" => certificate_authorities,
             "ssl_certificate" => server_certificate_file,
@@ -249,6 +253,8 @@ describe "Filebeat", :integration => true do
 
           let_tmp_file(:server_certificate_file) { server_certificate_data.first }
           let_tmp_file(:server_certificate_key_file) { convert_to_pkcs8(server_certificate_data.last) }
+          let_tmp_file(:certificate_file) { certificate_data.first }
+          let_tmp_file(:certificate_key_file) { convert_to_pkcs8(certificate_data.last) }
 
           context "directly signed client certificate" do
             let(:certificate_authorities) { [root_ca_certificate_file] }
@@ -262,7 +268,7 @@ describe "Filebeat", :integration => true do
             include_context "Intermediate CA"
 
             let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
-            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) } 
+            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
             let(:certificate_authorities) { [intermediate_ca_certificate_file] }
 
             include_examples "send events"
@@ -298,7 +304,7 @@ describe "Filebeat", :integration => true do
 
               let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
 
-              context "client from primary CA" do 
+              context "client from primary CA" do
                 include_examples "send events"
               end
 

data/spec/support/client_process_helpers.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require "childprocess"
 module ClientProcessHelpers
-  def start_client(timeout = 1)
+  def start_client(timeout = 5)
     @client_out = Stud::Temporary.file
     @client_out.sync
 
@@ -11,13 +11,26 @@ module ClientProcessHelpers
     ChildProcess.posix_spawn = true
     @process.start
 
-    sleep(0.1)
+    sleep_interval = 0.1
+    max_iterations = (timeout / sleep_interval).to_i
+    max_iterations.times do
+      sleep(sleep_interval)
+      if @process.alive?
+        break
+      end
+    end
+    #Note - can not raise error here if process failed to start, since some tests expects for the process to not start due to invalid configuration
+
     @client_out.rewind
 
     # can be used to helper debugging when a test fails
     @execution_output = @client_out.read
   end
 
+  def is_alive
+    return @process.alive?
+  end
+
   def stop_client
     begin
       @process.poll_for_exit(5)

data/spec/support/flores_extensions.rb

@@ -1,49 +1,68 @@
 # encoding: utf-8
 require "flores/pki"
 require "flores/random"
+require "socket"
 
 module Flores
   module Random
     DEFAULT_PORT_RANGE = 1024..65535
-    DEFAULT_PORT_CHECK_TIMEOUT = 1
-    DEFAULT_MAXIMUM_PORT_FIND_TRY = 15
-
     class << self
       def port(range = DEFAULT_PORT_RANGE)
-        try = 0
-        while try < DEFAULT_MAXIMUM_PORT_FIND_TRY
-          candidate = integer(range)
-
-          if port_available?(candidate)
-            break
-          else
-            try += 1
-          end
+        integer(range)
+      end
+    end
+  end
+
+  module PKI
+
+    # Monkey patched the fix for https://github.com/jordansissel/ruby-flores/issues/9
+    # TODO: remove this once Flores is released with fix.
+    CertificateSigningRequest.class_eval do
+      def create
+        validate!
+        extensions = OpenSSL::X509::ExtensionFactory.new
+        extensions.subject_certificate = certificate
+        extensions.issuer_certificate = self_signed? ? certificate : signing_certificate
+
+        certificate.issuer = extensions.issuer_certificate.subject
+        certificate.add_extension(extensions.create_extension("subjectKeyIdentifier", "hash", false))
+
+        if want_signature_ability?
+          # Create a CA.
+          certificate.add_extension(extensions.create_extension("basicConstraints", "CA:TRUE", true))
+          # Rough googling seems to indicate at least keyCertSign is required for CA and intermediate certs.
+          certificate.add_extension(extensions.create_extension("keyUsage", "keyCertSign, cRLSign, digitalSignature", true))
+        else
+          # Create a client+server certificate
+          #
+          # It feels weird to create a certificate that's valid as both server and client, but a brief inspection of major
+          # web properties (apple.com, google.com, yahoo.com, github.com, fastly.com, mozilla.com, amazon.com) reveals that
+          # major web properties have certificates with both clientAuth and serverAuth extended key usages. Further,
+          # these major server certificates all have digitalSignature and keyEncipherment for key usage.
+          #
+          # Here's the command I used to check this:
+          #    echo mozilla.com apple.com github.com google.com yahoo.com fastly.com elastic.co amazon.com \
+          #    | xargs -n1 sh -c 'openssl s_client -connect $1:443 \
+          #    | sed -ne "/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p" \
+          #    | openssl x509 -text -noout | sed -ne "/X509v3 extensions/,/Signature Algorithm/p" | sed -e "s/^/$1 /"' - \
+          #    | grep -A2 'Key Usage'
+          certificate.add_extension(extensions.create_extension("keyUsage", "digitalSignature, keyEncipherment", true))
+          certificate.add_extension(extensions.create_extension("extendedKeyUsage", "clientAuth, serverAuth", false))
         end
-        
-        raise "Flores.random_port: Cannot find an available port, tried #{DEFAULT_MAXIMUM_PORT_FIND_TRY} times, range was: #{range}" if try == DEFAULT_MAXIMUM_PORT_FIND_TRY
 
-        candidate
-      end
-      
-      def port_available?(port)
-        begin
-          server = TCPServer.new(port)
-          available = true
-        rescue # Assume that any errors can do this
-          available = false
-        ensure
-          server.close if server
+        if @subject_alternates
+          certificate.add_extension(extensions.create_extension("subjectAltName", @subject_alternates.join(",")))
         end
 
-        return available
+        certificate.serial = OpenSSL::BN.new(serial)
+        certificate.sign(signing_key, digest_method)
+        certificate
       end
     end
-  end
 
-  module PKI
+
     DEFAULT_CERTIFICATE_OPTIONS = {
-      :duration => Flores::Random.number(100..2000),
+      :duration => 86400, #one day
       :key_size => GENERATE_DEFAULT_KEY_SIZE, 
       :exponent => GENERATE_DEFAULT_EXPONENT,
       :want_signature_ability => false
@@ -78,5 +97,9 @@ module Flores
 
       [csr.create, client_key]
     end
+
+
+
   end
 end
+

data/spec/support/integration_shared_context.rb

@@ -3,7 +3,7 @@ require "flores/random"
 
 shared_examples "send events" do
   it "successfully send the events" do
-    try(50) { expect(queue.size).to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" }
+    try(25) { expect(queue.size).to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" }
     expect(queue.collect { |e| e.get("message") }).to eq(events)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-beats
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.0.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-07 00:00:00.000000000 Z
+date: 2017-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -264,7 +264,7 @@ files:
 - vendor/jar-dependencies/io/netty/netty-tcnative-boringssl-static/1.1.33.Fork23/netty-tcnative-boringssl-static-1.1.33.Fork23.jar
 - vendor/jar-dependencies/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - vendor/jar-dependencies/org/javassist/javassist/3.20.0-GA/javassist-3.20.0-GA.jar
-- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/4.0.2/logstash-input-beats-4.0.2.jar
+- vendor/jar-dependencies/org/logstash/beats/logstash-input-beats/4.0.3/logstash-input-beats-4.0.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)