logstash-input-beats 2.1.4 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e842feb71aa235b190d3ecc62267f82250d66bd1
-  data.tar.gz: 82c27027f6f482b7959a8b2c8db6641a99830dab
+  metadata.gz: 2b44b745b034e4879863dafa18e2a47f6c5077cb
+  data.tar.gz: be9744dfb679a87d2c62d9c70d0556c5a4101e91
 SHA512:
-  metadata.gz: 5f1a5d58c37d7e4ea6adb24fbed54a85ce486257b3ab827b01ae9da11dfb644598d0e37301f8dd0c14aef4aaa81ed8229bab865bcd2b25aca012c9bbd8959768
-  data.tar.gz: b34f6a828c699c17aa4b7f00b6028c79104076369e5d48e3e1bd1a0afcb1ee38fd4232af34c3f2bc40b37830a1dfd90a5f14270cda482676a1b3adef5cd7561c
+  metadata.gz: c9a92bec0e4ae62083feb22c6736c474b31fe49b3853f6ae48c9dcd5d71259ac38da73fc340853f171f5fc3f4167bcf71a5e99071e5aee4d1fb8c8667ad94514
+  data.tar.gz: aadca9c26ab53386668cd0e4544178d0485cdd0d2e29546dc8bd261593a1711930a3ff598ab7c9c57f8a672cee34d82fab31a2ba25c4edad6e945349df55cc17

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.2.0
+  - The server can now do client side verification by providing a list of certificate authorities and configuring the `ssl_verify_mode`,
+    the server can use `peer`, if the client send a certificate it will be validated. Using `force_peer` will make sure the client provide a certificate
+    and it will be validated with the know CA.  #8
 # 2.1.4
   - Change the `logger#warn` for `logger.debug` when a peer get disconnected, keep alive check from proxy can generate a lot of logs  #46
 # 2.1.3

data/lib/logstash/inputs/beats.rb

@@ -68,6 +68,27 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   # SSL key passphrase to use.
   config :ssl_key_passphrase, :validate => :password
 
+  # Validate client certificates against theses authorities
+  # You can defined multiples files or path, all the certificates will
+  # be read and added to the trust store. You need to configure the `ssl_verify_mode`
+  # to `peer` or `force_peer` to enable the verification.
+  #
+  # This feature only support certificate directly signed by your root ca.
+  # Intermediate CA are currently not supported.
+  # 
+  config :ssl_certificate_authorities, :validate => :array, :default => []
+
+  # By default the server dont do any client verification,
+  # 
+  # `peer` will make the server ask the client to provide a certificate,
+  # if the client provide the certificate it will be validated.
+  #
+  # `force_peer` will make the server ask the client for their certificate, if the clients
+  # doesn't provide it the connection will be closed.
+  #
+  # This option need to be used with `ssl_certificate_authorities` and a defined list of CA.
+  config :ssl_verify_mode, :validate => ["none", "peer", "force_peer"], :default => "none"
+
   # The number of seconds before we raise a timeout,
   # this option is useful to control how much time to wait if something is blocking the pipeline.
   config :congestion_threshold, :validate => :number, :default => 5
@@ -87,9 +108,16 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
     end
 
     @logger.info("Beats inputs: Starting input listener", :address => "#{@host}:#{@port}")
-    @lumberjack = Lumberjack::Beats::Server.new(:address => @host, :port => @port,
-      :ssl => @ssl, :ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
-      :ssl_key_passphrase => @ssl_key_passphrase)
+
+
+    @lumberjack = Lumberjack::Beats::Server.new(:address => @host,
+      :port => @port,
+      :ssl => @ssl,
+      :ssl_certificate => @ssl_certificate,
+      :ssl_key => @ssl_key,
+      :ssl_key_passphrase => @ssl_key_passphrase,
+      :ssl_certificate_authorities => @ssl_certificate_authorities,
+      :ssl_verify_mode => @ssl_verify_mode)
 
     # in 1.5 the main SizeQueue doesnt have the concept of timeout
     # We are using a small plugin buffer to move events to the internal queue
@@ -103,8 +131,8 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
 
     # Keep a list of active connections so we can flush their codec on shutdown
-    
-    # Use threadsafe gem, since we have a strict dependency on concurrent-ruby 0.9.2 
+
+    # Use threadsafe gem, since we have a strict dependency on concurrent-ruby 0.9.2
     # in the core
     @connections_list = ThreadSafe::Hash.new
   end # def register
@@ -127,13 +155,13 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
         connection = @lumberjack.accept # call that creates a new connection
         # if the connection is nil the connection was closed upstream,
         # so we will try in another iteration to recover or stop.
-        next if connection.nil? 
+        next if connection.nil?
 
-        Thread.new do 
+        Thread.new do
           handle_new_connection(connection)
         end
       else
-        @logger.warn("Beats input: the pipeline is blocked, temporary refusing new connection.", 
+        @logger.warn("Beats input: the pipeline is blocked, temporary refusing new connection.",
                      :reconnect_backoff_sleep => RECONNECT_BACKOFF_SLEEP)
         sleep(RECONNECT_BACKOFF_SLEEP)
       end
@@ -186,9 +214,9 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
     @logger.warn("Beats input: The circuit breaker has detected a slowdown or stall in the pipeline, the input is closing the current connection and rejecting new connection until the pipeline recover.",
                 :exception => e.class)
   rescue Exception => e # If we have a malformed packet we should handle that so the input doesn't crash completely.
-    @logger.error("Beats input: unhandled exception", 
+    @logger.error("Beats input: unhandled exception",
                   :exception => e,
-                  :backtrace => e.backtrace) 
+                  :backtrace => e.backtrace)
   ensure
     transformer = LogStash::Inputs::BeatsSupport::EventTransformCommon.new(self)
 

data/lib/lumberjack/beats/client.rb

@@ -12,14 +12,21 @@ module Lumberjack module Beats
         :port => 0,
         :addresses => [],
         :ssl_certificate => nil,
+        :ssl_certificate_key => nil,
+        :ssl_certificate_authorities => nil,
         :ssl => true,
         :json => false,
       }.merge(opts)
 
-      @opts[:addresses] = [@opts[:addresses]] if @opts[:addresses].class == String
+      @opts[:addresses] = Array(@opts[:addresses])
       raise "Must set a port." if @opts[:port] == 0
       raise "Must set atleast one address" if @opts[:addresses].empty? == 0
-      raise "Must set a ssl certificate or path" if @opts[:ssl_certificate].nil? && @opts[:ssl]
+
+      if @opts[:ssl]
+        if @opts[:ssl_certificate_authorities].nil? && (@opts[:ssl_certificate].nil? || @opts[:ssl_certificate_key].nil?)
+          raise "Must set a ssl certificate or path"
+        end
+      end
 
       @socket = connect
     end
@@ -67,36 +74,78 @@ module Lumberjack module Beats
       @opts = {
         :port => 0,
         :address => "127.0.0.1",
+        :ssl_certificate_authorities => [], # use the same naming as beats' TLS options
         :ssl_certificate => nil,
+        :ssl_certificate_key => nil,
+        :ssl_certificate_password => nil,
         :ssl => true,
         :json => false,
       }.merge(opts)
       @host = @opts[:address]
 
-      connection_start(opts)
+      connection_start
     end
 
     private
-    def connection_start(opts)
-      tcp_socket = TCPSocket.new(opts[:address], opts[:port])
-      if !opts[:ssl]
+    def connection_start
+      tcp_socket = TCPSocket.new(@opts[:address], @opts[:port])
+
+      if !@opts[:ssl]
         @socket = tcp_socket
       else
-        certificate = OpenSSL::X509::Certificate.new(File.read(opts[:ssl_certificate]))
 
-        certificate_store = OpenSSL::X509::Store.new
-        certificate_store.add_cert(certificate)
+        @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, setup_ssl)
+        @socket.connect
+      end
+    end
+
+    private
+    def setup_ssl
+      ssl_context = OpenSSL::SSL::SSLContext.new
 
-        ssl_context = OpenSSL::SSL::SSLContext.new
-        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        ssl_context.cert_store = certificate_store
+      ssl_context.cert = certificate
+      ssl_context.key = private_key 
+      ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      ssl_context.cert_store = trust_store
+      ssl_context
+    end
 
-        @socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
-        @socket.connect
+    private
+    def certificate
+      if @opts[:ssl_certificate]
+        OpenSSL::X509::Certificate.new(File.open(@opts[:ssl_certificate]))
       end
     end
 
-    private 
+    private
+    def private_key
+      OpenSSL::PKey::RSA.new(File.read(@opts[:ssl_certificate_key]), @opts[:ssl_certificate_password]) if @opts[:ssl_certificate_key]
+    end
+
+    private
+    def trust_store
+      store = OpenSSL::X509::Store.new
+
+      Array(@opts[:ssl_certificate_authorities]).each do |certificate_authority|
+        if File.file?(certificate_authority)
+          store.add_file(certificate_authority)
+        else
+          # add_path is no implemented under jruby
+          # so recursively try to load all the certificate from this directory
+          # https://github.com/jruby/jruby-openssl/blob/master/src/main/java/org/jruby/ext/openssl/X509Store.java#L159
+          if !!(RUBY_PLATFORM == "java") 
+            Dir.glob(File.join(certificate_authority, "**", "*")).each { |f| store.add_file(f) }
+          else
+            store.add_path(certificate_authority)
+          end
+        end
+      end
+
+      store
+    end
+
+
+    private
     def inc
       @sequence = 0 if @sequence + 1 > Lumberjack::Beats::SEQUENCE_MAX
       @sequence = @sequence + 1
@@ -134,7 +183,7 @@ module Lumberjack module Beats
       ack(elements.size)
     end
 
-    private 
+    private
     def compress_payload(payload)
       compress = Zlib::Deflate.deflate(payload)
       ["1", "C", compress.bytesize, compress].pack("AANA*")

data/lib/lumberjack/beats/server.rb

@@ -29,35 +29,33 @@ module Lumberjack module Beats
         :ssl => true,
         :ssl_certificate => nil,
         :ssl_key => nil,
-        :ssl_key_passphrase => nil
+        :ssl_key_passphrase => nil,
+        :ssl_certificate_authorities => nil,
+        :ssl_verify_mode => :none # By default we dont verify client
       }.merge(options)
 
       if @options[:ssl]
-        [:ssl_certificate, :ssl_key].each do |k|
-          if @options[k].nil?
-            raise "You must specify #{k} in Lumberjack::Server.new(...)"
-          end
+        if verify_client?(@options[:ssl_verify_mode]) && certificate_authorities.empty?
+          raise "When `ssl_verify_mode` is set to `peer` OR `force_peer` you need to specify the `ssl_certificate_authorities`"
+        end
+
+        if !verify_client?(@options[:ssl_verify_mode]) && certificate_authorities.size > 0 
+          raise "When `ssl_certificate_authorities` is configured you need to set `ssl_verify_mode` to either `peer` or `force_peer`"
+        end
+
+        if @options[:ssl_certificate].nil? || @options[:ssl_key].nil?
+          raise "You must specify `ssl_certificate` AND `ssl_key`"
         end
       end
 
       @server = TCPServer.new(@options[:address], @options[:port])
-
       @close = Concurrent::AtomicBoolean.new
+      @port = retrieve_current_port
 
-      # Query the port in case the port number is '0'
-      # TCPServer#addr == [ address_family, port, address, address ]
-      @port = @server.addr[1]
-
-      if @options[:ssl]
-        # load SSL certificate
-        @ssl = OpenSSL::SSL::SSLContext.new
-        @ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
-        @ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]),
-          @options[:ssl_key_passphrase])
-      end
+      setup_ssl if ssl?
     end # def initialize
 
-    # Server#run method, allow the library to manage all the connection 
+    # Server#run method, allow the library to manage all the connection
     # threads, this handing is quite minimal and don't handler
     # all the possible cases deconnection/connection.
     #
@@ -76,8 +74,8 @@ module Lumberjack module Beats
         Thread.new(connection) do |connection|
           begin
             connection.run(&block)
-          rescue Lumberjack::Beats::Connection::ConnectionClosed 
-            # Connection will raise a wrapped exception upstream, 
+          rescue Lumberjack::Beats::Connection::ConnectionClosed
+            # Connection will raise a wrapped exception upstream,
             # but if the threads are managed by the library we can simply ignore it.
             #
             # Note: This follow the previous behavior of the perfect silence.
@@ -87,7 +85,7 @@ module Lumberjack module Beats
     end # def run
 
     def ssl?
-      @ssl
+      @options[:ssl]
     end
 
     def accept(&block)
@@ -139,13 +137,88 @@ module Lumberjack module Beats
       @close.make_true
       @server.close unless @server.closed?
     end
-  end # class Server
+
+    private
+    def verify_client?(mode)
+      mode = mode.to_sym
+      mode == :peer || mode == :force_peer
+    end
+
+    def retrieve_current_port
+      # Query the port in case the port number is '0'
+      # TCPServer#addr == [ address_family, port, address, address ]
+      @server.addr[1]
+    end
+
+    def certificate_authorities
+      Array(@options[:ssl_certificate_authorities])
+    end
+
+    def server_private_key
+      OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase])
+    end
+
+    def server_certificate
+      OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
+    end
+
+    def verify_mode
+      case @options[:ssl_verify_mode].to_sym
+      when :none
+        OpenSSL::SSL::VERIFY_NONE
+      when :peer
+        OpenSSL::SSL::VERIFY_PEER
+      when :force_peer
+        OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+      end
+    end
+
+    def jruby?
+      RUBY_PLATFORM == "java"
+    end
+
+    def trust_store
+      store = OpenSSL::X509::Store.new
+
+      if certificate_authorities.size > 0
+        certificate_authorities.each do |certificate_authority|
+          if File.file?(certificate_authority)
+            store.add_file(certificate_authority)
+          else
+            # `#add_path` is not implemented under jruby
+            # so recursively try to load all the certificate from this directory
+            # https://github.com/jruby/jruby-openssl/blob/master/src/main/java/org/jruby/ext/openssl/X509Store.java#L159
+            if jruby?
+              Dir.glob(File.join(certificate_authority, "**", "*")).each { |f| store.add_file(f) }
+            else
+              store.add_path(certificate_authority)
+            end
+          end
+        end
+      end
+
+      store
+    end
+
+    def setup_ssl
+      @ssl = OpenSSL::SSL::SSLContext.new
+
+      # @ssl.verify_callback = lambda do |preverify_ok, context|
+      #   require "pry"
+      #   binding.pry
+      # end
+      @ssl.cert_store = trust_store
+      @ssl.verify_mode = verify_mode
+      # @ssl.ca_file = certificate_authorities.first
+      @ssl.cert = server_certificate
+      @ssl.key = server_private_key
+    end
+  end
 
   class Parser
     PROTOCOL_VERSION_1 = "1".ord
     PROTOCOL_VERSION_2 = "2".ord
 
-    SUPPORTED_PROTOCOLS = [PROTOCOL_VERSION_1, PROTOCOL_VERSION_2]
     class UnsupportedProtocol < StandardError; end
 
     def initialize
@@ -242,7 +315,7 @@ module Lumberjack module Beats
     end
 
     def supported_protocol?(version)
-      SUPPORTED_PROTOCOLS.include?(version)
+      PROTOCOL_VERSION_2 == version || PROTOCOL_VERSION_1 == version
     end
 
     def window_size(&block)
@@ -313,7 +386,7 @@ module Lumberjack module Beats
   end # class Parser
 
   class Connection
-    # Wrap the original exception into a common one, 
+    # Wrap the original exception into a common one,
     # to make upstream managing and reporting easier
     # But lets make sure we keep the meaning of the original exception.
     class ConnectionClosed < StandardError
@@ -351,7 +424,7 @@ module Lumberjack module Beats
 
       @server = server
       @ack_handler = nil
-      
+
       # Fetch the details of the host before reading anything from the socket
       # se we can use that information when debugging connection issues with
       # remote hosts.

data/logstash-input-beats.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-input-beats"
-  s.version         = "2.1.4"
+  s.version         = "2.2.0"
   s.licenses        = ["Apache License (2.0)"]
   s.summary         = "Receive events using the lumberjack protocol."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -33,5 +33,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec-wait"
   s.add_development_dependency "logstash-devutils", "~> 0.0.18"
   s.add_development_dependency "logstash-codec-json"
+  s.add_development_dependency "childprocess" # To make filebeat/LSF integration test easier to write.
 end
 

data/spec/integration/filebeat_spec.rb

@@ -0,0 +1,235 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "stud/temporary"
+require "flores/pki"
+require "fileutils"
+require "thread"
+require "spec_helper"
+require "yaml"
+require "fileutils"
+require "flores/pki"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+
+FILEBEAT_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "filebeat", "filebeat"))
+
+describe "Filebeat", :integration => true do
+  include ClientProcessHelpers
+  include FileHelpers
+
+  before :all do
+    unless File.exist?(FILEBEAT_BINARY)
+      raise "Cannot find `Filebeat` binary in `vendor/filebeat`. Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  include_context "beats configuration"
+
+  # Filebeat related variables
+  let(:cmd) { [filebeat_exec, "-c", filebeat_config_path, "-e", "-v"] }
+
+  let(:filebeat_exec) { FILEBEAT_BINARY }
+
+  let_empty_tmp_file(:registry_file)
+  let(:filebeat_config) do
+    { 
+      "filebeat" => { 
+        "prospectors" => [{ "paths" => [log_file],  "input_type" => "log" }],
+        "registry_file" => registry_file,
+        "scan_frequency" => "1s"
+      },
+      "output" => { 
+        "logstash" => { "hosts" => ["#{host}:#{port}"] },
+        "logging" => { "level" => "debug" }
+      }
+    }
+  end
+
+  let_tmp_file(:filebeat_config_path) { YAML.dump(filebeat_config) }
+  before :each do
+    start_client
+    sleep(2) # give some time to FB to send somthing
+    stop_client
+  end
+
+  ###########################################################
+  shared_context "Root CA" do
+    let(:root_ca) { Flores::PKI.generate("CN=root.localhost") }
+    let(:root_ca_certificate) { root_ca.first }
+    let(:root_ca_key) { root_ca.last }
+    let_tmp_file(:root_ca_certificate_file) { root_ca_certificate }
+  end
+
+  shared_context "Intermediate CA" do
+    let(:intermediate_ca) { Flores::PKI.create_intermediate_certificate("CN=intermediate.localhost", root_ca_certificate, root_ca_key) }
+    let(:intermediate_ca_certificate) { intermediate_ca.first }
+    let(:intermediate_ca_key) { intermediate_ca.last }
+    let_tmp_file(:certificate_authorities_chain) { Flores::PKI.chain_certificates(root_ca_certificate, intermediate_ca_certificate) }
+  end
+
+  ############################################################
+  # Actuals tests 
+  context "Plain TCP" do
+    include_examples "send events"
+  end
+
+  context "TLS" do
+    context "Server verification" do
+      let(:filebeat_config) do
+        super.merge({
+          "output" => {
+            "logstash" => {
+              "hosts" => ["#{host}:#{port}"],
+              "tls" => { "certificate_authorities" => certificate_authorities }
+            },
+            "logging" => { "level" => "debug" }
+          }})
+      end
+
+      let(:input_config) do
+        super.merge({ 
+          "ssl" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file
+        })
+      end
+
+      let(:certificate_authorities) { [certificate_file] }
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+      let_tmp_file(:certificate_file) { certificate_data.first }
+
+      context "self signed certificate" do
+        include_examples "send events"
+      end
+
+      context "CA root" do
+        include_context "Root CA"
+
+        context "directly signed client certificate" do
+          let(:certificate_authorities) { [root_ca_certificate_file] }
+          let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+          include_examples "send events"
+        end
+
+        context "intermediate CA signs client certificate" do
+          include_context "Intermediate CA"
+
+          let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
+          let(:certificate_authorities) { [certificate_authorities_chain] }
+          
+          include_examples "send events"
+        end
+      end
+
+      context "Client verification / Mutual validation" do
+        let(:filebeat_config) do
+          super.merge({
+            "output" => {
+              "logstash" => {
+                "hosts" => ["#{host}:#{port}"],
+                "tls" => { 
+                  "certificate_authorities" => certificate_authorities,
+                  "certificate" => certificate_file,
+                  "certificate_key" => certificate_key_file
+                }
+              },
+              "logging" => { "level" => "debug" }
+            }})
+        end
+
+        let(:input_config) do
+          super.merge({ 
+            "ssl" => true,
+            "ssl_certificate_authorities" => certificate_authorities,
+            "ssl_certificate" => server_certificate_file,
+            "ssl_key" => server_certificate_key_file,
+            "ssl_verify_mode" => "force_peer"
+          })
+        end
+
+        context "with a self signed certificate" do
+          let(:certificate_authorities) { [certificate_file] }
+          let(:certificate_data) { Flores::PKI.generate }
+          let_tmp_file(:certificate_key_file) { certificate_data.last } 
+          let_tmp_file(:certificate_file) { certificate_data.first }
+          let_tmp_file(:server_certificate_file) { certificate_data.first }
+          let_tmp_file(:server_certificate_key_file) { certificate_data.last }
+
+          include_examples "send events"
+        end
+
+        context "CA root" do
+          include_context "Root CA"
+
+          let_tmp_file(:server_certificate_file) { server_certificate_data.first }
+          let_tmp_file(:server_certificate_key_file) { server_certificate_data.last }
+
+          context "directly signed client certificate" do
+            let(:certificate_authorities) { [root_ca_certificate_file] }
+            let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+            include_examples "send events"
+          end
+
+
+          # Doesnt work because of this issues in `jruby-openssl`
+          # https://github.com/jruby/jruby-openssl/issues/84
+          xcontext "intermediate create server and client certificate" do
+            include_context "Intermediate CA"
+
+            let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
+            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) } 
+            let(:certificate_authorities) { [certificate_authorities_chain] }
+
+            include_examples "send events"
+          end
+
+          context "and Secondary CA multiples clients" do
+            context "with CA in different files" do
+              let(:secondary_ca) { Flores::PKI.generate }
+              let(:secondary_ca_key) { secondary_ca.last }
+              let(:secondary_ca_certificate) { secondary_ca.first }
+              let_tmp_file(:secondary_ca_certificate_file) { secondary_ca.first }
+
+              let(:secondary_client_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", secondary_ca_certificate, secondary_ca_key) }
+              let_tmp_file(:secondary_client_certificate_file) { secondary_client_certificate_data.first }
+              let_tmp_file(:secondary_client_certificate_key_file) { secondary_client_certificate_data.last }
+              let(:certificate_authorities) { [root_ca_certificate_file, secondary_ca_certificate_file] }
+              let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+              let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+              context "client from primary CA" do 
+                include_examples "send events"
+              end
+
+              context "client from secondary CA" do
+                let(:filebeat_config) do
+                  super.merge({
+                    "output" => {
+                      "logstash" => {
+                        "hosts" => ["#{host}:#{port}"],
+                        "tls" => { 
+                          "certificate_authorities" => certificate_authorities,
+                          "certificate" => secondary_client_certificate_file,
+                          "certificate_key" => secondary_client_certificate_key_file
+                        }
+                      },
+                      "logging" => { "level" => "debug" }
+                    }})
+                end
+
+                include_examples "send events"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end