logstash-input-beats 2.1.4 → 2.2.0

data/spec/integration/logstash_forwarder_spec.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/json"
+require "fileutils"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+require "spec_helper"
+
+LSF_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "logstash-forwarder", "logstash-forwarder"))
+
+xdescribe "Logstash-Forwarder", :integration => true do
+  include ClientProcessHelpers
+
+  before :all do
+    unless File.exist?(LSF_BINARY)
+      raise "Cannot find `logstash-forwarder` binary in `vendor/logstash-forwarder` Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  let(:client_wait_time) { 5 }
+  include FileHelpers
+  include_context "beats configuration"
+
+  # Filebeat related variables
+  let(:cmd) { [lsf_exec, "-config", lsf_config_path] }
+
+  let(:lsf_exec) { LSF_BINARY }
+  let(:registry_file) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".logstash-forwarder")) }
+  let_tmp_file(:lsf_config_path) { lsf_config }
+  let(:log_file) { f = Stud::Temporary.file; f.close; f.path }
+  let(:lsf_config) do
+ <<-CONFIG
+{
+  "network": {
+    "servers": [ "#{host}:#{port}" ],
+    "ssl ca": "#{certificate_authorities}"
+  },
+  "files": [
+    { "paths": [ "#{log_file}" ] }
+  ]
+}
+ CONFIG
+  end
+  #
+
+  before :each do
+    FileUtils.rm_rf(registry_file) # ensure clean state between runs
+    start_client
+    sleep(1)
+    # let LSF start and than write the logs
+    File.open(log_file, "a") do |f|
+      f.write(events.join("\n") + "\n")
+    end
+  end
+
+  after :each do
+    stop_client
+  end
+
+  context "Plain TCP" do
+    include ClientProcessHelpers
+    let(:certificate_authorities) { "" }
+
+    it "should not send any events" do
+      expect(queue.size).to eq(0), @execution_output
+    end
+  end
+
+  context "TLS" do
+    context "Server Verification" do
+      let(:input_config) do
+        super.merge({ 
+          "ssl" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file,
+          "target_field_for_codec" => "line"
+        })
+      end
+
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_file) { certificate_data.first }
+      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+      let(:certificate_authorities) { certificate_file }
+
+      context "self signed certificate" do
+        include_examples "send events" do
+        end
+      end
+
+      context "invalid CA on the client" do
+        let(:invalid_data) { Flores::PKI.generate }
+        let(:certificate_authorities) { invalid_data.first }
+
+        it "should not send any events" do
+          expect(queue.size).to eq(0), @execution_output
+        end
+      end
+    end
+  end
+end

data/spec/integration_spec.rb

@@ -6,27 +6,47 @@ require "flores/pki"
 require "fileutils"
 require "thread"
 require "spec_helper"
+require_relative "./support/file_helpers"
+require_relative "./support/flores_extensions"
 
 Thread.abort_on_exception = true
 describe "A client" do
+  include FileHelpers
+
   let(:certificate) { Flores::PKI.generate }
-  let(:certificate_file_crt) { "certificate.crt" }
-  let(:certificate_file_key) { "certificate.key" }
+  let(:certificate_file_crt) do
+     p = Stud::Temporary.file
+     p.write(certificate.first.to_s)
+     p.close
+     p.path
+  end
+
+  let(:certificate_file_key) do
+    p = Stud::Temporary.file
+    p.write(certificate.last.to_s)
+    p.close
+    p.path
+  end
   let(:port) { Flores::Random.integer(1024..65335) }
   let(:tcp_port) { port + 1 }
   let(:host) { "127.0.0.1" }
   let(:queue) { [] }
+  let(:config_ssl) do
+    {
+      :port => port,
+      :address => host,
+      :ssl_certificate => certificate_file_crt,
+      :ssl_key => certificate_file_key
+    }
+  end
 
-  before :each do
-    expect(File).to receive(:read).at_least(1).with(certificate_file_crt) { certificate.first.to_s }
-    expect(File).to receive(:read).at_least(1).with(certificate_file_key) { certificate.last.to_s }
-
-    tcp_server = Lumberjack::Beats::Server.new(:port => tcp_port, :address => host, :ssl => false)
+  let(:config_tcp) do
+    { :port => tcp_port, :address => host, :ssl => false }
+  end
 
-    ssl_server = Lumberjack::Beats::Server.new(:port => port,
-                                        :address => host,
-                                        :ssl_certificate => certificate_file_crt,
-                                        :ssl_key => certificate_file_key)
+  before :each do
+    tcp_server = Lumberjack::Beats::Server.new(config_tcp)
+    ssl_server = Lumberjack::Beats::Server.new(config_ssl)
 
     @tcp_server = Thread.new do
       tcp_server.run { |data, identity_stream| queue << [data, identity_stream] }
@@ -42,7 +62,7 @@ describe "A client" do
               # parser for TCP based server trip.
               # Connection is closed by Server connection object
             end
-          end 
+          end
       end
     end
 
@@ -160,62 +180,334 @@ describe "A client" do
     end
   end
 
-  context "using ssl encrypted connection" do
-    context "with a valid certificate" do
-      it "successfully connect to the server" do
-        expect {
-          Lumberjack::Beats::Client.new(:port => port,
-                                 :host => host,
-                                 :addresses => host,
-                                 :ssl_certificate => certificate_file_crt)
-        }.not_to raise_error
+  context "using TLS/SSL encrypted connection" do
+    context "without a ca chain" do
+      context "with a valid certificate" do
+        it "successfully connect to the server" do
+          expect {
+            Lumberjack::Beats::Client.new(:port => port,
+                                   :host => host,
+                                   :addresses => host,
+                                   :ssl_certificate_authorities => certificate_file_crt)
+          }.not_to raise_error
+        end
+
+        it "should fail connecting to plain tcp server" do
+          expect {
+            Lumberjack::Beats::Client.new(:port => tcp_port,
+                                   :host => host,
+                                   :addresses => host,
+                                   :ssl_certificate_authorities => certificate_file_crt)
+          }.to raise_error(OpenSSL::SSL::SSLError)
+        end
       end
 
-      it "should fail connecting to plain tcp server" do
-        expect {
-          Lumberjack::Beats::Client.new(:port => tcp_port,
-                                 :host => host,
-                                 :addresses => host,
-                                 :ssl_certificate => certificate_file_crt)
-        }.to raise_error(OpenSSL::SSL::SSLError)
-      end
-    end
+      context "with an invalid certificate" do
+        let(:invalid_certificate) { Flores::PKI.generate }
+        let(:invalid_certificate_file) { Stud::Temporary.file.path }
 
-    context "with an invalid certificate" do
-      let(:invalid_certificate) { Flores::PKI.generate }
-      let(:invalid_certificate_file) { "invalid.crt" }
+        it "should refuse to connect" do
+          expect {
+            Lumberjack::Beats::Client.new(:port => port,
+                                          :host => host,
+                                          :addresses => host,
+                                          :ssl_certificate_authorities => invalid_certificate_file)
 
-      before do
-        expect(File).to receive(:read).with(invalid_certificate_file) { invalid_certificate.first.to_s }
+          }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
+        end
       end
 
-      it "should refuse to connect" do
-        expect {
+      context "When transmitting a payload" do
+        let(:client) do
           Lumberjack::Beats::Client.new(:port => port,
                                         :host => host,
                                         :addresses => host,
-                                        :ssl_certificate => invalid_certificate_file)
+                                        :ssl_certificate_authorities => certificate_file_crt)
+        end
+
+        context "json" do
+          let(:options) { super.merge({ :json => true }) }
+          include_examples "transmit payloads"
+        end
 
-        }.to raise_error(OpenSSL::SSL::SSLError, /certificate verify failed/)
+        context "v1 frame" do
+          include_examples "transmit payloads"
+        end
       end
     end
+  end
 
-    context "When transmitting a payload" do
-      let(:client) do
-        Lumberjack::Beats::Client.new(:port => port,
-                                      :host => host,
-                                      :addresses => host,
-                                      :ssl_certificate => certificate_file_crt)
+  def write_to_tmp_file(content)
+    file = Stud::Temporary.file
+    file.write(content.to_s)
+    file.close
+    file.path
+  end
+
+  context "Mutual validation" do
+    let(:host) { "localhost" }
+    let(:certificate_duration) { Flores::Random.number(1000..2000) }
+    let(:client) { Lumberjack::Beats::Client.new(options) }
+    let(:options) { { :ssl => true, :port => port, :addresses => host } }
+    let(:config_ssl) do
+      {
+        :port => port,
+        :address => host,
+        :ssl => true,
+        :ssl_verify_mode => "force_peer"
+      }
+    end
+
+    context "with a self signed certificate" do
+      let(:certificate) { Flores::PKI.generate }
+      let_tmp_file(:certificate_file) { certificate.first }
+      let_tmp_file(:certificate_key_file) { certificate.last }
+      let(:options) do
+        super.merge({ :ssl_certificate => certificate_file, 
+                      :ssl_certificate_key => certificate_key_file, 
+                      :ssl_certificate_authorities => certificate_file })
       end
 
-      context "json" do
-        let(:options) { super.merge({ :json => true }) }
+      let(:config_ssl) do
+        super.merge({ :ssl_certificate_authorities => certificate_file,
+                      :ssl_certificate => certificate_file,
+                      :ssl_key => certificate_key_file })
+      end
+
+      include_examples "transmit payloads"
+    end
+
+    context "with certificate authorities" do
+      let(:keysize) { 1024 }
+      let(:exponent) { 65537 }
+      let(:root_ca) { Flores::PKI.generate("CN=root.logstash") }
+      let(:root_ca_certificate) { root_ca.first }
+      let(:root_ca_key) { root_ca.last }
+      let_tmp_file(:root_ca_certificate_file) { root_ca_certificate }
+
+      context "directly signing a certificate" do
+        let(:client_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:client_certificate_key_file) { client_certificate_key }
+        let(:client_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = client_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = root_ca_key
+          csr.signing_certificate = root_ca_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:client_certificate_file) { client_certificate }
+
+        let(:server_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let(:server_certificate_key_file) { write_to_tmp_file(server_certificate_key) }
+        let(:server_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = server_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = root_ca_key
+          csr.signing_certificate = root_ca_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:server_certificate_file) { server_certificate }
+
+        let(:options) do
+          super.merge({ :ssl_certificate => client_certificate_file, 
+                        :ssl_certificate_key => client_certificate_key_file, 
+                        :ssl_certificate_authorities => root_ca_certificate_file })
+        end
+
+        let(:config_ssl) do
+          super.merge({ :ssl_certificate_authorities => root_ca_certificate_file,
+                        :ssl_certificate => server_certificate_file,
+                        :ssl_key => server_certificate_key_file, 
+                        :verify_mode => :force_peer })
+        end
+        
         include_examples "transmit payloads"
       end
 
-      context "v1 frame" do
+      # Doesnt work because of this issues in `jruby-openssl`
+      # https://github.com/jruby/jruby-openssl/issues/84
+      xcontext "CA given with client providing intermediate and client certificate" do
+        let(:intermediate_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:intermediate_certificate_key_file) { intermediate_certificate_key }
+        let(:intermediate_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = intermediate_certificate_key.public_key
+          csr.subject = "CN=intermediate 1"
+          csr.signing_key = root_ca_key
+          csr.signing_certificate = root_ca_certificate
+          csr.want_signature_ability = true
+          csr.create
+        end
+        let_tmp_file(:intermediate_certificate_file) { intermediate_certificate }
+        let(:client_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:client_certificate_key_file) { client_certificate_key }
+        let(:client_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = client_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = intermediate_certificate_key
+          csr.signing_certificate = intermediate_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:client_certificate_file) { client_certificate }
+
+        let(:server_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:server_certificate_key_file) { server_certificate_key }
+        let(:server_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = server_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = intermediate_certificate_key
+          csr.signing_certificate = intermediate_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:server_certificate_file) { server_certificate }
+        let_tmp_file(:certificate_chain_file) { Flores::PKI.chain_certificates(root_ca_certificate, intermediate_certificate) }
+
+        let(:options) do
+          super.merge({ :ssl_certificate => client_certificate_file, 
+                        :ssl_certificate_key => client_certificate_key_file, 
+                        :ssl_certificate_authorities => certificate_chain_file })
+        end
+
+        let(:config_ssl) do
+          super.merge({ :ssl_certificate_authorities => certificate_chain_file,
+                        :ssl_certificate => server_certificate_file,
+                        :ssl_key => server_certificate_key_file, 
+                        :verify_mode => :force_peer })
+
+
+        end
+
         include_examples "transmit payloads"
       end
+
+      context "Mutiple CA different clients" do
+        let(:secondary_root_ca) { Flores::PKI.generate("CN=secondary.root.logstash") }
+        let(:secondary_root_ca_certificate) { root_ca.first }
+        let(:secondary_root_ca_key) { root_ca.last }
+        let_tmp_file(:secondary_root_ca_certificate_file) { root_ca_certificate }
+
+        let(:secondary_client_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let(:secondary_client_certificate_key_file) { write_to_tmp_file(secondary_client_certificate_key) }
+        let(:secondary_client_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = secondary_client_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = secondary_root_ca_key
+          csr.signing_certificate = secondary_root_ca_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:secondary_client_certificate_file) { secondary_client_certificate }
+        let(:client_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:client_certificate_key_file) { client_certificate_key }
+        let(:client_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = client_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = root_ca_key
+          csr.signing_certificate = root_ca_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:client_certificate_file) { client_certificate }
+
+        let(:server_certificate_key) { OpenSSL::PKey::RSA.generate(keysize, exponent) } 
+        let_tmp_file(:server_certificate_key_file) { server_certificate_key }
+        let(:server_certificate) do
+          csr = Flores::PKI::CertificateSigningRequest.new
+          csr.start_time = Time.now
+          csr.expire_time = csr.start_time + certificate_duration
+          csr.public_key = server_certificate_key.public_key
+          csr.subject = "CN=localhost"
+          csr.signing_key = root_ca_key
+          csr.signing_certificate = root_ca_certificate
+          csr.want_signature_ability = false
+          csr.create
+        end
+        let_tmp_file(:server_certificate_file) { server_certificate }
+
+        shared_examples "multiples client" do
+          context "client from primary CA" do
+            let(:options) do
+              super.merge({ :ssl_certificate => client_certificate_file, 
+                            :ssl_certificate_key => client_certificate_key_file, 
+                            :ssl_certificate_authorities => [root_ca_certificate_file] })
+            end
+
+            include_examples "transmit payloads"
+          end
+
+          context "client from secondary CA" do
+            let(:options) do
+              super.merge({ :ssl_certificate => secondary_client_certificate_file, 
+                            :ssl_certificate_key => secondary_client_certificate_key_file, 
+                            :ssl_certificate_authorities => [root_ca_certificate_file] })
+            end
+
+            include_examples "transmit payloads"
+          end
+        end
+
+        context "when the CA are defined individually in the configuration" do
+          let(:config_ssl) do
+            super.merge({ :ssl_certificate_authorities => [secondary_root_ca_certificate_file, root_ca_certificate_file],
+                          :ssl_certificate => server_certificate_file,
+                          :ssl_key => server_certificate_key_file, 
+                          :verify_mode => "force_peer" })
+          end
+
+          include_examples "multiples client"
+        end
+
+        context "when the CA are defined by a path in the configuration" do
+          let(:ca_path) do
+            p = Stud::Temporary.directory
+            
+            # Doing this here make sure the path is populated when the server
+            # is started in the main before block
+            FileUtils.mkdir_p(p)
+            FileUtils.cp(secondary_root_ca_certificate_file, p)
+            FileUtils.cp(root_ca_certificate_file, p)
+
+            p
+          end
+
+          after :each do
+            FileUtils.rm_rf(ca_path)
+          end
+
+          let(:config_ssl) do
+            super.merge({ :ssl_certificate_authorities => ca_path,
+                          :ssl_certificate => server_certificate_file,
+                          :ssl_key => server_certificate_key_file, 
+                          :verify_mode => "force_peer" })
+          end
+
+          include_examples "multiples client"
+        end
+      end
     end
   end
 end