logstash-input-beats 3.1.0.beta1-java

data/spec/inputs/beats/event_transform_common_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::Beats::EventTransformCommon do
+  subject { described_class.new(input).transform(event) }
+
+  include_examples "Common Event Transformation"
+end

data/spec/inputs/beats/message_listener_spec.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "logstash/event"
+require "logstash/inputs/beats/message_listener"
+require "thread"
+
+class MockMessage
+  def initialize(identity_stream, data = {})
+    @identity_stream = identity_stream
+    @data = data
+  end
+
+  def getData
+    @data
+  end
+
+  def getIdentityStream
+    @identity_stream
+  end
+end
+
+class DummyCodec < LogStash::Codecs::Base
+  DUMMY_EVENT = LogStash::Event.new
+
+  def decode(message, &block)
+    block.call(LogStash::Event.new({"message" => message}))
+  end
+
+  def flush(&block)
+    block.call(DUMMY_EVENT)
+  end
+end
+
+describe LogStash::Inputs::Beats::MessageListener do
+  let(:queue)  { Queue.new }
+  let(:codec) { DummyCodec.new }
+  let(:input) { LogStash::Inputs::Beats.new({ "port" => 5555, "codec" => codec }) }
+  let(:ctx) { double("ChannelHandlerContext") }
+  let(:message) { MockMessage.new("abc", { "message" => "hello world"}) }
+
+  subject { described_class.new(queue, input) }
+
+  before do
+    subject.onNewConnection(ctx)
+  end
+
+  context "onNewConnection" do
+    it "register the connection to the connection list" do
+      expect { subject.onNewConnection(double("ChannelHandlerContext")) }.to change { subject.connections_list.count }.by(1)
+    end
+  end
+
+  context "onNewMessage" do
+    context "when the message is from filebeat" do
+      let(:message) { MockMessage.new("abc", { "message" => "hello world" } )}
+
+      it "extract the event" do
+        subject.onNewMessage(ctx, message)
+        event = queue.pop
+        expect(event.get("message")).to eq("hello world")
+      end
+    end
+
+    context "when the message is from LSF" do
+      let(:message) { MockMessage.new("abc", { "line" => "hello world" } )}
+
+      it "extract the event" do
+        subject.onNewMessage(ctx, message)
+        event = queue.pop
+        expect(event.get("message")).to eq("hello world")
+      end
+    end
+
+    context "when the message is from any libbeat" do
+      let(:message) { MockMessage.new("abc",  { "metric" => 1, "name" => "super-stats"} )}
+
+      it "extract the event" do
+        subject.onNewMessage(ctx, message)
+        event = queue.pop
+        expect(event.get("message")).to be_nil
+        expect(event.get("metric")).to eq(1)
+        expect(event.get("name")).to eq("super-stats")
+      end
+    end
+  end
+
+  context "onException" do
+    it "remove the connection to the connection list" do
+      expect { subject.onException(ctx) }.to change { subject.connections_list.count }.by(-1)
+    end
+
+    it "calls flush on codec" do
+      subject.onConnectionClose(ctx)
+      expect(queue).not_to be_empty
+    end
+  end
+
+  context "onConnectionClose" do
+    it "remove the connection to the connection list" do
+      expect { subject.onConnectionClose(ctx) }.to change { subject.connections_list.count }.by(-1)
+    end
+
+    it "calls flush on codec" do
+      subject.onConnectionClose(ctx)
+      expect(queue).not_to be_empty
+    end
+  end
+end

data/spec/inputs/beats/raw_event_transform_spec.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require "logstash/event"
+require "logstash/inputs/beats"
+require_relative "../../support/shared_examples"
+require "spec_helper"
+
+describe LogStash::Inputs::Beats::RawEventTransform do
+  let(:config) do
+    {
+      "port" => 0,
+      "type" => "example",
+      "tags" => "beats"
+    }
+  end
+
+  let(:input) { LogStash::Inputs::Beats.new(config) }
+  let(:event) { LogStash::Event.new }
+
+  subject { described_class.new(input).transform(event) }
+
+  include_examples "Common Event Transformation"
+
+  it "tags the event" do
+    expect(subject.get("tags")).to include("beats_input_raw_event")
+  end
+end

data/spec/inputs/beats/tls_spec.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require "logstash/inputs/beats/tls"
+
+describe LogStash::Inputs::Beats::TLS do
+  subject { described_class }
+
+  it "returns the minimum supported tls" do
+    expect(subject.min.version).to eq(1)
+    expect(subject.min.name).to eq("TLSv1")
+  end
+
+  it "returns the maximum supported tls" do
+    expect(subject.max.version).to eq(1.2)
+    expect(subject.max.name).to eq("TLSv1.2")
+  end
+
+  describe ".get_supported" do
+    context "when a range is given" do
+      it "returns the list of compatible TLS from a range" do
+        expect(subject.get_supported((1.1)..(1.2)).map(&:version)).to match([1.1, 1.2])
+      end
+
+      it "it return an empty array when nothing match" do
+        expect(subject.get_supported((3.1)..(8.2))).to be_empty
+      end
+    end
+
+    context "when a scalar is given" do
+      it "when a scalar is given we return the compatible value" do
+        expect(subject.get_supported(1.1).map(&:version)).to match([1.1])
+      end
+
+
+      it "it return an empty array when nothing match" do
+        expect(subject.get_supported(9)).to be_empty
+      end
+    end
+  end
+end

data/spec/inputs/beats_spec.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "stud/temporary"
+require "logstash/inputs/beats"
+require "logstash/codecs/plain"
+require "logstash/codecs/json"
+require "logstash/codecs/multiline"
+require "logstash/event"
+
+describe LogStash::Inputs::Beats do
+  let(:connection) { double("connection") }
+  let(:certificate) { BeatsInputTest.certificate }
+  let(:port) { BeatsInputTest.random_port }
+  let(:queue)  { Queue.new }
+  let(:config)   { { "port" => 0, "ssl_certificate" => certificate.ssl_cert, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats"} }
+
+  context "#register" do
+    context "identity map" do
+      subject(:plugin) { LogStash::Inputs::Beats.new(config) }
+      before { plugin.register }
+
+      context "when using the multiline codec" do
+        let(:codec) { LogStash::Codecs::Multiline.new("pattern" => '^2015',
+                                                      "what" => "previous",
+                                                      "negate" => true) }
+        let(:config) { super.merge({ "codec" => codec }) }
+
+        it "wraps the codec with the identity_map" do
+          expect(plugin.codec).to be_kind_of(LogStash::Codecs::IdentityMapCodec)
+        end
+      end
+
+      context "when using non buffered codecs" do
+        let(:config) { super.merge({ "codec" => "json" }) }
+
+        it "doesnt wrap the codec with the identity map" do
+          expect(plugin.codec).to be_kind_of(LogStash::Codecs::JSON)
+        end
+      end
+    end
+
+    it "raise no exception" do
+      plugin = LogStash::Inputs::Beats.new(config)
+      expect { plugin.register }.not_to raise_error
+    end
+
+    context "with ssl enabled" do
+      context "without certificate configuration" do
+        let(:config) {{ "port" => 0, "ssl" => true, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats" }}
+
+        it "should fail to register the plugin with ConfigurationError" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "without key configuration" do
+        let(:config)   { { "port" => 0, "ssl" => true, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats"} }
+        it "should fail to register the plugin with ConfigurationError" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+    end
+
+    context "with ssl disabled" do
+      context "and certificate configuration" do
+        let(:config)   { { "port" => 0, "ssl" => false, "ssl_certificate" => certificate.ssl_cert, "type" => "example", "tags" => "Beats" } }
+
+        it "should not fail" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect {plugin.register}.not_to raise_error
+        end
+      end
+
+      context "and certificate key configuration" do
+        let(:config) {{ "port" => 0, "ssl" => false, "ssl_key" => certificate.ssl_key, "type" => "example", "tags" => "beats" }}
+
+        it "should not fail" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect {plugin.register}.not_to raise_error
+        end
+      end
+
+      context "and no certificate or key configured" do
+        let(:config) {{ "ssl" => false, "port" => 0, "type" => "example", "tags" => "beats" }}
+
+        it "should work just fine" do
+          plugin = LogStash::Inputs::Beats.new(config)
+          expect {plugin.register}.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "when interrupting the plugin" do
+    it_behaves_like "an interruptible input plugin"
+  end
+end

data/spec/integration/filebeat_spec.rb

@@ -0,0 +1,234 @@
+# encoding: utf-8
+require "logstash/inputs/beats"
+require "stud/temporary"
+require "flores/pki"
+require "fileutils"
+require "thread"
+require "spec_helper"
+require "yaml"
+require "fileutils"
+require "flores/pki"
+require_relative "../support/flores_extensions"
+require_relative "../support/file_helpers"
+require_relative "../support/integration_shared_context"
+require_relative "../support/client_process_helpers"
+
+FILEBEAT_BINARY = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "vendor", "filebeat", "filebeat"))
+
+describe "Filebeat", :integration => true do
+  include ClientProcessHelpers
+  include FileHelpers
+
+  before :all do
+    unless File.exist?(FILEBEAT_BINARY)
+      raise "Cannot find `Filebeat` binary in `vendor/filebeat`. Did you run `bundle exec rake test:integration:setup` before running the integration suite?"
+    end
+  end
+
+  include_context "beats configuration"
+
+  # Filebeat related variables
+  let(:cmd) { [filebeat_exec, "-c", filebeat_config_path, "-e", "-v"] }
+
+  let(:filebeat_exec) { FILEBEAT_BINARY }
+
+  let_empty_tmp_file(:registry_file)
+  let(:filebeat_config) do
+    { 
+      "filebeat" => { 
+        "prospectors" => [{ "paths" => [log_file],  "input_type" => "log" }],
+        "registry_file" => registry_file,
+        "scan_frequency" => "1s"
+      },
+      "output" => { 
+        "logstash" => { "hosts" => ["#{host}:#{port}"] },
+        "logging" => { "level" => "debug" }
+      }
+    }
+  end
+
+  let_tmp_file(:filebeat_config_path) { YAML.dump(filebeat_config) }
+  before :each do
+    start_client
+    sleep(2) # give some time to FB to send something
+    stop_client
+  end
+
+  ###########################################################
+  shared_context "Root CA" do
+    let(:root_ca) { Flores::PKI.generate("CN=root.localhost") }
+    let(:root_ca_certificate) { root_ca.first }
+    let(:root_ca_key) { root_ca.last }
+    let_tmp_file(:root_ca_certificate_file) { root_ca_certificate }
+  end
+
+  shared_context "Intermediate CA" do
+    let(:intermediate_ca) { Flores::PKI.create_intermediate_certificate("CN=intermediate.localhost", root_ca_certificate, root_ca_key) }
+    let(:intermediate_ca_certificate) { intermediate_ca.first }
+    let(:intermediate_ca_key) { intermediate_ca.last }
+    let_tmp_file(:intermediate_ca_certificate_file) { intermediate_ca_certificate }
+    let_tmp_file(:certificate_authorities_chain) { Flores::PKI.chain_certificates(root_ca_certificate, intermediate_ca_certificate) }
+  end
+
+  ############################################################
+  # Actuals tests 
+  context "Plain TCP" do
+    include_examples "send events"
+  end
+
+  context "TLS" do
+    context "Server verification" do
+      let(:filebeat_config) do
+        super.merge({
+          "output" => {
+            "logstash" => {
+              "hosts" => ["#{host}:#{port}"],
+              "tls" => { "certificate_authorities" => certificate_authorities }
+            },
+            "logging" => { "level" => "debug" }
+          }})
+      end
+
+      let(:input_config) do
+        super.merge({ 
+          "ssl" => true,
+          "ssl_certificate" => certificate_file,
+          "ssl_key" => certificate_key_file
+        })
+      end
+
+      let(:certificate_authorities) { [certificate_file] }
+      let(:certificate_data) { Flores::PKI.generate }
+      let_tmp_file(:certificate_key_file) { certificate_data.last } 
+      let_tmp_file(:certificate_file) { certificate_data.first }
+
+      context "self signed certificate" do
+        include_examples "send events"
+      end
+
+      context "CA root" do
+        include_context "Root CA"
+
+        context "directly signed client certificate" do
+          let(:certificate_authorities) { [root_ca_certificate_file] }
+          let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+          include_examples "send events"
+        end
+
+        context "intermediate CA signs client certificate" do
+          include_context "Intermediate CA"
+
+          let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
+          let(:certificate_authorities) { [certificate_authorities_chain] }
+          
+          include_examples "send events"
+        end
+      end
+
+      context "Client verification / Mutual validation" do
+        let(:filebeat_config) do
+          super.merge({
+            "output" => {
+              "logstash" => {
+                "hosts" => ["#{host}:#{port}"],
+                "tls" => { 
+                  "certificate_authorities" => certificate_authorities,
+                  "certificate" => certificate_file,
+                  "certificate_key" => certificate_key_file
+                }
+              },
+              "logging" => { "level" => "debug" }
+            }})
+        end
+
+        let(:input_config) do
+          super.merge({ 
+            "ssl" => true,
+            "ssl_certificate_authorities" => certificate_authorities,
+            "ssl_certificate" => server_certificate_file,
+            "ssl_key" => server_certificate_key_file,
+            "ssl_verify_mode" => "force_peer"
+          })
+        end
+
+        context "with a self signed certificate" do
+          let(:certificate_authorities) { [certificate_file] }
+          let(:certificate_data) { Flores::PKI.generate }
+          let_tmp_file(:certificate_key_file) { certificate_data.last } 
+          let_tmp_file(:certificate_file) { certificate_data.first }
+          let_tmp_file(:server_certificate_file) { certificate_data.first }
+          let_tmp_file(:server_certificate_key_file) { certificate_data.last }
+
+          include_examples "send events"
+        end
+
+        context "CA root" do
+          include_context "Root CA"
+
+          let_tmp_file(:server_certificate_file) { server_certificate_data.first }
+          let_tmp_file(:server_certificate_key_file) { server_certificate_data.last }
+
+          context "directly signed client certificate" do
+            let(:certificate_authorities) { [root_ca_certificate_file] }
+            let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+            include_examples "send events"
+          end
+
+          ### DOESNT WORK
+          context "intermediate create server and client certificate" do
+            include_context "Intermediate CA"
+
+            let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) }
+            let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", intermediate_ca_certificate, intermediate_ca_key) } 
+            let(:certificate_authorities) { [intermediate_ca_certificate_file] }
+
+            include_examples "send events"
+          end
+
+          context "and Secondary CA multiples clients" do
+            context "with CA in different files" do
+              let(:secondary_ca) { Flores::PKI.generate }
+              let(:secondary_ca_key) { secondary_ca.last }
+              let(:secondary_ca_certificate) { secondary_ca.first }
+              let_tmp_file(:secondary_ca_certificate_file) { secondary_ca.first }
+
+              let(:secondary_client_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", secondary_ca_certificate, secondary_ca_key) }
+              let_tmp_file(:secondary_client_certificate_file) { secondary_client_certificate_data.first }
+              let_tmp_file(:secondary_client_certificate_key_file) { secondary_client_certificate_data.last }
+              let(:certificate_authorities) { [root_ca_certificate_file, secondary_ca_certificate_file] }
+              let(:certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+              let(:server_certificate_data) { Flores::PKI.create_client_certicate("CN=localhost", root_ca_certificate, root_ca_key) }
+
+              context "client from primary CA" do 
+                include_examples "send events"
+              end
+
+              context "client from secondary CA" do
+                let(:filebeat_config) do
+                  super.merge({
+                    "output" => {
+                      "logstash" => {
+                        "hosts" => ["#{host}:#{port}"],
+                        "tls" => { 
+                          "certificate_authorities" => certificate_authorities,
+                          "certificate" => secondary_client_certificate_file,
+                          "certificate_key" => secondary_client_certificate_key_file
+                        }
+                      },
+                      "logging" => { "level" => "debug" }
+                    }})
+                end
+
+                include_examples "send events"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end