logstash-input-beats 3.1.0.beta1-java

data/spec/integration/logstash_forwarder_spec.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/json"
+require "fileutils"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+require "spec_helper"
+
+LSF_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "logstash-forwarder", "logstash-forwarder"))
+
+describe "Logstash-Forwarder", :integration => true do
+  include ClientProcessHelpers
+
+  before :all do
+    unless File.exist?(LSF_BINARY)
+      raise "Cannot find `logstash-forwarder` binary in `vendor/logstash-forwarder` Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  include FileHelpers
+  include_context "beats configuration"
+
+  let(:client_wait_time) { 5 }
+
+  # Filebeat related variables
+  let(:cmd) { [lsf_exec, "-config", lsf_config_path] }
+
+  let(:lsf_exec) { LSF_BINARY }
+  let(:registry_file) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".logstash-forwarder")) }
+  let_tmp_file(:lsf_config_path) { lsf_config }
+  let(:log_file) { f = Stud::Temporary.file; f.close; f.path }
+  let(:lsf_config) do
+ <<-CONFIG
+{
+  "network": {
+    "servers": [ "#{host}:#{port}" ],
+    "ssl ca": "#{certificate_authorities}"
+  },
+  "files": [
+    { "paths": [ "#{log_file}" ] }
+  ]
+}
+ CONFIG
+  end
+  #
+
+  before :each do
+    FileUtils.rm_rf(registry_file) # ensure clean state between runs
+    start_client
+    sleep(1)
+    # let LSF start and than write the logs
+    File.open(log_file, "a") do |f|
+      f.write(events.join("\n") + "\n")
+    end
+    sleep(1) # give some time to the clients to pick up the changes
+    stop_client
+  end
+
+  after :each do
+    stop_client
+  end
+
+  xcontext "Plain TCP" do
+    include ClientProcessHelpers
+
+    let(:certificate_authorities) { "" }
+
+    it "should not send any events" do
+      expect(queue.size).to eq(0), @execution_output
+    end
+  end
+
+  context "TLS" do
+    context "Server Verification" do
+      let(:input_config) do
+        super.merge({ 
+          "ssl" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file,
+        })
+      end
+
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_file) { certificate_data.first }
+      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+      let(:certificate_authorities) { certificate_file }
+
+      context "self signed certificate" do
+        include_examples "send events"
+      end
+
+      context "invalid CA on the client" do
+        let(:invalid_data) { Flores::PKI.generate }
+        let(:certificate_authorities) { f = Stud::Temporary.file; f.close; f.path }
+
+        it "should not send any events" do
+          expect(queue.size).to eq(0), @execution_output
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require 'rspec'
+require 'rspec/mocks'
+require 'rspec/wait'
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/plain"
+require_relative "support/logstash_test"
+
+$: << File.realpath(File.join(File.dirname(__FILE__), "..", "lib"))
+
+RSpec.configure do |config|
+  config.order = :rand
+end
+

data/spec/support/client_process_helpers.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require "childprocess"
+module ClientProcessHelpers
+  def start_client(timeout = 1)
+    @client_out = Stud::Temporary.file
+    @client_out.sync
+
+    @process = ChildProcess.build(*cmd)
+    @process.duplex = true
+    @process.io.stdout = @process.io.stderr = @client_out
+    ChildProcess.posix_spawn = true
+    @process.start
+
+    sleep(0.1)
+    @client_out.rewind
+
+    # can be used to helper debugging when a test fails
+    @execution_output = @client_out.read
+  end
+
+  def stop_client
+    begin
+      @process.poll_for_exit(5)
+    rescue ChildProcess::TimeoutError
+      Process.kill("KILL", @process.pid)
+    end
+  end
+end

data/spec/support/file_helpers.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+# Add an helper method named `let_tmp_file` to spec example, this
+# helper will create a temporary file with the content from the block,
+# it behave like a normal `let` statement. Also to make things easier temporary
+# debug it will create another `let` statement with the actual content of the block.
+#
+# Example:
+# ```
+# let_tmp_file(:hello_world_file) { "Hello world" } # return a path to a tmp file containing "Hello World"
+# and will create this debug `let`, the value of the file will be the same.
+# let(:hello_world_file_content) # return "Hello world"
+#
+module FileHelpers
+  AUTO_CLEAN = true
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  def write_to_tmp_file(content)
+    file = Stud::Temporary.file
+    file.write(content.to_s)
+    file.close
+    file.path
+  end
+
+  module ClassMethods
+    def let_empty_tmp_file(name, &block)
+      let(name) do
+        path = nil
+        f = Stud::Temporary.file
+        f.close
+        path = f.path
+        @__let_tmp_files = [] unless @__let_tmp_files
+        @__let_tmp_files << path
+        path
+      end
+    end
+
+    def let_tmp_file(name, &block)
+      after :each do
+        if @__let_tmp_files && FileHelpers::AUTO_CLEAN
+          @__let_tmp_files.each do |f|
+            FileUtils.rm_f(f)
+          end
+        end
+      end
+
+      name_content = "#{name}_content"
+      let(name_content, &block)
+      let(name) do
+        content = __send__(name_content)
+        path = write_to_tmp_file(content)
+        @__let_tmp_files = [] unless @__let_tmp_files
+        @__let_tmp_files << path
+        path
+      end
+    end
+  end
+end

data/spec/support/flores_extensions.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require "flores/pki"
+require "flores/random"
+
+module Flores
+  module Random
+    DEFAULT_PORT_RANGE = 1024..65535
+    DEFAULT_PORT_CHECK_TIMEOUT = 1
+    DEFAULT_MAXIMUM_PORT_FIND_TRY = 15
+
+    class << self
+      def port(range = DEFAULT_PORT_RANGE)
+        try = 0
+        while try < DEFAULT_MAXIMUM_PORT_FIND_TRY
+          candidate = integer(range)
+
+          if port_available?(candidate)
+            break
+          else
+            try += 1
+          end
+        end
+        
+        raise "Flores.random_port: Cannot find an available port, tried #{DEFAULT_MAXIMUM_PORT_FIND_TRY} times, range was: #{range}" if try == DEFAULT_MAXIMUM_PORT_FIND_TRY
+
+        candidate
+      end
+      
+      def port_available?(port)
+        begin
+          server = TCPServer.new(port)
+          available = true
+        rescue # Assume that any errors can do this
+          available = false
+        ensure
+          server.close if server
+        end
+
+        return available
+      end
+    end
+  end
+
+  module PKI
+    DEFAULT_CERTIFICATE_OPTIONS = {
+      :duration => Flores::Random.number(100..2000),
+      :key_size => GENERATE_DEFAULT_KEY_SIZE, 
+      :exponent => GENERATE_DEFAULT_EXPONENT,
+      :want_signature_ability => false
+    }
+
+    def self.chain_certificates(*certificates)
+      certificates.join("\n")
+    end
+
+    def self.create_intermediate_certificate(subject, signing_certificate, signing_private_key, options  = {})
+      create_a_signed_certificate(subject, signing_certificate, signing_private_key, options.merge({ :want_signature_ability => true }))
+    end
+
+    def self.create_client_certicate(subject, signing_certificate, signing_private_key, options = {})
+      create_a_signed_certificate(subject, signing_certificate, signing_private_key, options)
+    end
+
+    private
+    def self.create_a_signed_certificate(subject, signing_certificate, signing_private_key, options = {})
+      options = DEFAULT_CERTIFICATE_OPTIONS.merge(options)
+
+      client_key = OpenSSL::PKey::RSA.new(options[:key_size], options[:exponent])
+
+      csr = Flores::PKI::CertificateSigningRequest.new
+      csr.start_time = Time.now
+      csr.expire_time = csr.start_time + options[:duration]
+      csr.public_key = client_key.public_key
+      csr.subject = subject
+      csr.signing_key = signing_private_key
+      csr.signing_certificate = signing_certificate
+      csr.want_signature_ability = options[:want_signature_ability]
+
+      [csr.create, client_key]
+    end
+  end
+end

data/spec/support/integration_shared_context.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+require "flores/random"
+
+shared_examples "send events" do
+  it "successfully send the events" do
+    wait(20).for { queue.size }.to eq(number_of_events), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}" 
+    expect(queue.collect { |e| e.get("message") }).to eq(events)
+  end
+end
+
+shared_examples "doesn't send events" do
+  it "doesn't send any events" do
+    expect(queue.size).to eq(0), "Expected: #{number_of_events} got: #{queue.size}, execution output:\n #{@execution_output}"
+  end
+end
+
+shared_context "beats configuration" do
+  # common
+  let(:port) { Flores::Random.port }
+  let(:host) { "localhost" }
+
+  let(:queue) { [] }
+  let_tmp_file(:log_file) { events.join("\n") + "\n" } # make sure we end of line
+  let(:number_of_events) { 5 }
+  let(:event) { "Hello world" }
+  let(:events) do
+    events = []
+    number_of_events.times { |n| events << "#{event} #{n}" }
+    events
+  end
+
+  let(:input_config) do
+    {
+      "host" => host,
+      "port" => port
+    }
+  end
+
+  let(:beats) do
+    LogStash::Inputs::Beats.new(input_config)
+  end
+
+  before :each do
+    beats.register
+
+    @abort_on_exception = Thread.abort_on_exception
+    Thread.abort_on_exception = true
+
+    @server = Thread.new do
+      begin
+        # use to know what lumberjack is actually doing
+        if ENV["DEBUG"]
+          logger = Logger.new(STDOUT)
+          beats.logger = Cabin::Channel.new
+          beats.logger.subscribe(logger)
+          beats.logger.level = :debug
+        end
+
+        beats.run(queue)
+      rescue => e
+        retry unless beats.stop?
+      end
+    end
+    @server.abort_on_exception = true
+
+    sleep(1) while @server.status != "run"
+  end
+
+  after(:each) do
+    beats.stop
+    Thread.abort_on_exception = @abort_on_exception
+  end
+end

data/spec/support/logstash_test.rb

@@ -0,0 +1,66 @@
+require "stud/temporary"
+
+
+# namespace the Dummy* classes, they are reused names
+# use a more specific module name to prevent clashes
+module BeatsInputTest
+  class Certicate
+    attr_reader :ssl_key, :ssl_cert
+
+    def initialize
+      @ssl_cert = Stud::Temporary.pathname("ssl_certificate")
+      @ssl_key = Stud::Temporary.pathname("ssl_key")
+
+      system("openssl req -x509  -batch -nodes -newkey rsa:2048 -keyout #{ssl_key} -out #{ssl_cert} -subj /CN=localhost > /dev/null 2>&1")
+    end
+  end
+
+  class << self
+    def certificate
+      Certicate.new
+    end
+
+    def random_port
+      rand(2000..10000)
+    end
+  end
+
+  class DummyNeverBlockedQueue < Array
+    def offer(element, timeout = nil)
+      push(element)
+    end
+
+    alias_method :take, :shift
+  end
+
+  class DummyConnection
+    def initialize(events)
+      @events = events
+    end
+
+    def run
+      @events.each do |element|
+        yield element[:map], element[:identity_stream]
+      end
+    end
+
+    def peer
+      "localhost:5555"
+    end
+  end
+
+  class DummyCodec
+    def register() end
+    def decode(*) end
+    def clone() self; end
+    def base_codec
+      self
+    end
+    def self.config_name
+      "dummy"
+    end
+  end
+end
+
+
+

data/spec/support/shared_examples.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+shared_examples "Common Event Transformation" do
+  let(:tag) { "140-rpm-beats" }
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => tag
+    }
+  end
+
+  let(:input) do
+    LogStash::Inputs::Beats.new(config).tap do |i|
+      i.register
+    end
+  end
+  let(:event) { LogStash::Event.new(event_map) }
+  let(:event_map) do
+    {
+      "message" => "Hello world",
+    }
+  end
+
+  it "adds configured tags to the event" do
+    expect(subject.get("tags")).to include(tag)
+  end
+
+  context "when the `beast.hotname` doesnt exist on the event" do
+    let(:already_exist) { "already_exist" }
+    let(:event_map) { super.merge({ "host" => already_exist }) }
+
+    it "doesnt change the value" do
+      expect(subject.get("host")).to eq(already_exist)
+    end
+  end
+
+  context "when the `beat.hostname` exist in the event" do
+    let(:producer_host) { "newhost01" }
+    let(:event_map) { super.merge({ "beat" => { "hostname" => producer_host }}) }
+
+    context "when `host` key doesn't exist on the event" do
+      it "copy the `beat.hostname` to `host` or backward compatibility" do
+        expect(subject.get("host")).to eq(producer_host)
+      end
+    end
+
+    context "when `host` key exists on the event" do
+      let(:already_exist) { "already_exist" }
+      let(:event_map) { super.merge({ "host" => already_exist }) }
+
+      it "doesn't override it" do
+        expect(subject.get("host")).to eq(already_exist)
+      end
+    end
+  end
+end