logstash-core 2.2.4.snapshot1

data/spec/logstash/json_spec.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/json"
+require "logstash/environment"
+require "logstash/util"
+
+describe "LogStash::Json" do
+
+  let(:hash)   {{"a" => 1}}
+  let(:json_hash)   {"{\"a\":1}"}
+
+  let(:string) {"foobar"}
+  let(:json_string) {"\"foobar\""}
+
+  let(:array)  {["foo", "bar"]}
+  let(:json_array)  {"[\"foo\",\"bar\"]"}
+
+  let(:multi) {
+    [
+      {:ruby => "foo bar baz", :json => "\"foo bar baz\""},
+      {:ruby => "1", :json => "\"1\""},
+      {:ruby => {"a" => true}, :json => "{\"a\":true}"},
+      {:ruby => {"a" => nil}, :json => "{\"a\":null}"},
+      {:ruby => ["a", "b"], :json => "[\"a\",\"b\"]"},
+      {:ruby => [1, 2], :json => "[1,2]"},
+      {:ruby => [1, nil], :json => "[1,null]"},
+      {:ruby => {"a" => [1, 2]}, :json => "{\"a\":[1,2]}"},
+      {:ruby => {"a" => {"b" => 2}}, :json => "{\"a\":{\"b\":2}}"},
+      # {:ruby => , :json => },
+    ]
+  }
+
+  if LogStash::Environment.jruby?
+
+    ### JRuby specific
+    # Former expectation in this code were removed because of https://github.com/rspec/rspec-mocks/issues/964
+    # as soon as is fix we can re introduce them if decired, however for now the completeness of the test
+    # is also not affected as the conversion would not work if the expectation where not meet.
+    ###
+    context "jruby deserialize" do
+      it "should respond to load and deserialize object" do
+        expect(LogStash::Json.load(json_hash)).to eql(hash)
+      end
+    end
+
+    context "jruby serialize" do
+      it "should respond to dump and serialize object" do
+        expect(LogStash::Json.dump(string)).to eql(json_string)
+      end
+
+      it "should call JrJackson::Raw.generate for Hash" do
+        expect(LogStash::Json.dump(hash)).to eql(json_hash)
+      end
+
+      it "should call JrJackson::Raw.generate for Array" do
+        expect(LogStash::Json.dump(array)).to eql(json_array)
+      end
+
+    end
+
+  else
+
+    ### MRI specific
+
+    it "should respond to load and deserialize object on mri" do
+      expect(Oj).to receive(:load).with(json).and_call_original
+      expect(LogStash::Json.load(json)).to eql(hash)
+    end
+
+    it "should respond to dump and serialize object on mri" do
+      expect(Oj).to receive(:dump).with(hash, anything).and_call_original
+      expect(LogStash::Json.dump(hash)).to eql(json)
+    end
+  end
+
+  ### non specific
+
+  it "should correctly deserialize" do
+    multi.each do |test|
+      # because JrJackson in :raw mode uses Java::JavaUtil::LinkedHashMap and
+      # Java::JavaUtil::ArrayList, we must cast to compare.
+      # other than that, they quack like their Ruby equivalent
+      expect(LogStash::Util.normalize(LogStash::Json.load(test[:json]))).to eql(test[:ruby])
+    end
+  end
+
+  it "should correctly serialize" do
+    multi.each do |test|
+      expect(LogStash::Json.dump(test[:ruby])).to eql(test[:json])
+    end
+  end
+
+  it "should raise Json::ParserError on invalid json" do
+    expect{LogStash::Json.load("abc")}.to raise_error LogStash::Json::ParserError
+  end
+end

data/spec/logstash/output_delegator_spec.rb

@@ -0,0 +1,144 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe LogStash::OutputDelegator do
+  let(:logger) { double("logger") }
+  let(:events) { 7.times.map { LogStash::Event.new }}
+  let(:default_worker_count) { 1 }
+
+  subject { described_class.new(logger, out_klass, default_worker_count) }
+
+  context "with a plain output plugin" do
+    let(:out_klass) { double("output klass") }
+    let(:out_inst) { double("output instance") }
+
+    before do
+      allow(out_klass).to receive(:new).with(any_args).and_return(out_inst)
+      allow(out_klass).to receive(:threadsafe?).and_return(false)
+      allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      allow(out_inst).to receive(:register)
+      allow(out_inst).to receive(:multi_receive)
+      allow(logger).to receive(:debug).with(any_args)
+    end
+
+    it "should initialize cleanly" do
+      expect { subject }.not_to raise_error
+    end
+
+    context "after having received a batch of events" do
+      before do
+        subject.register
+        subject.multi_receive(events)
+      end
+
+      it "should pass the events through" do
+        expect(out_inst).to have_received(:multi_receive).with(events)
+      end
+
+      it "should increment the number of events received" do
+        expect(subject.events_received).to eql(events.length)
+      end
+    end
+
+
+    describe "closing" do
+      before do
+        subject.register
+      end
+
+      it "should register all workers on register" do
+        expect(out_inst).to have_received(:register)
+      end
+
+      it "should close all workers when closing" do
+        expect(out_inst).to receive(:do_close)
+        subject.do_close
+      end
+    end
+
+    describe "concurrency and worker support" do
+      before do
+        allow(out_inst).to receive(:id).and_return("a-simple-plugin")
+        allow(out_inst).to receive(:metric=).with(any_args)
+        allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      end
+
+      describe "non-threadsafe outputs that allow workers" do
+        let(:default_worker_count) { 3 }
+
+        before do
+          allow(out_klass).to receive(:threadsafe?).and_return(false)
+          subject.register
+        end
+
+        it "should instantiate multiple workers" do
+          expect(subject.workers.length).to eql(default_worker_count)
+        end
+
+        it "should send received events to the worker" do
+          expect(out_inst).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+      end
+
+      describe "threadsafe outputs" do
+        before do
+          allow(out_klass).to receive(:threadsafe?).and_return(true)
+          subject.register
+        end
+
+        it "should return true when threadsafe? is invoked" do
+          expect(subject.threadsafe?).to eql(true)
+        end
+
+        it "should define a threadsafe_worker" do
+          expect(subject.send(:threadsafe_worker)).to eql(out_inst)
+        end
+
+        it "should utilize threadsafe_multi_receive" do
+          expect(subject.send(:threadsafe_worker)).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+
+        it "should not utilize the worker queue" do
+          expect(subject.send(:worker_queue)).to be_nil
+        end
+
+        it "should send received events to the worker" do
+          expect(out_inst).to receive(:multi_receive).with(events)
+          subject.multi_receive(events)
+        end
+
+        it "should close all workers when closing" do
+          expect(out_inst).to receive(:do_close)
+          subject.do_close
+        end
+      end
+    end
+  end
+
+  # This may seem suspiciously similar to the class in outputs/base_spec
+  # but, in fact, we need a whole new class because using this even once
+  # will immutably modify the base class
+  class LogStash::Outputs::NOOPDelLegacyNoWorkers < ::LogStash::Outputs::Base
+    LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
+
+    def register
+      workers_not_supported(LEGACY_WORKERS_NOT_SUPPORTED_REASON)
+    end
+  end
+
+  describe "legacy output workers_not_supported" do
+    let(:default_worker_count) { 2 }
+    let(:out_klass) { LogStash::Outputs::NOOPDelLegacyNoWorkers }
+
+    before(:each) do
+      allow(logger).to receive(:debug).with(any_args)
+    end
+
+    it "should only setup one worker" do
+      subject.register
+      expect(subject.worker_count).to eql(1)
+    end
+  end
+end

data/spec/logstash/outputs/base_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "spec_helper"
+
+# use a dummy NOOP output to test Outputs::Base
+class LogStash::Outputs::NOOP < LogStash::Outputs::Base
+  config_name "noop"
+  milestone 2
+
+  config :dummy_option, :validate => :string
+
+  def register; end
+
+  def receive(event)
+    return output?(event)
+  end
+end
+
+class LogStash::Outputs::NOOPLegacyNoWorkers < ::LogStash::Outputs::Base
+  LEGACY_WORKERS_NOT_SUPPORTED_REASON = "legacy reason"
+
+  def register
+    workers_not_supported(LEGACY_WORKERS_NOT_SUPPORTED_REASON)
+  end
+end
+
+describe "LogStash::Outputs::Base#new" do
+  it "should instantiate cleanly" do
+    params = { "dummy_option" => "potatoes", "codec" => "json", "workers" => 2 }
+    worker_params = params.dup; worker_params["workers"] = 1
+
+    expect do
+      LogStash::Outputs::NOOP.new(params.dup)
+    end.not_to raise_error
+  end
+
+  it "should move workers_not_supported declarations up to the class level" do
+    LogStash::Outputs::NOOPLegacyNoWorkers.new.register
+    expect(LogStash::Outputs::NOOPLegacyNoWorkers.workers_not_supported?).to eql(true)
+  end
+end

data/spec/logstash/patches_spec.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+require "socket"
+require "logstash/patches"
+require "flores/pki"
+
+describe "OpenSSL defaults" do
+  subject { OpenSSL::SSL::SSLContext.new }
+
+  # OpenSSL::SSL::SSLContext#ciphers returns an array of 
+  # [ [ ciphername, version, bits, alg_bits ], [ ... ], ... ]
+ 
+  # List of cipher names
+  let(:ciphers) { subject.ciphers.map(&:first) }
+
+  # List of cipher encryption bit strength. 
+  let(:encryption_bits) { subject.ciphers.map { |_, _, _, a| a } }
+
+  it "should not include any export ciphers" do
+    # SSLContext#ciphers returns an array of [ciphername, tlsversion, key_bits, alg_bits]
+    # Let's just check the cipher names
+    expect(ciphers).not_to be_any { |name| name =~ /EXPORT/ || name =~ /^EXP/ }
+  end
+
+  it "should not include any weak ciphers (w/ less than 128 bits in encryption algorithm)" do
+    # SSLContext#ciphers returns an array of [ciphername, tlsversion, key_bits, alg_bits]
+    expect(encryption_bits).not_to be_any { |bits| bits < 128 }
+  end
+
+  it "should not include a default `verify_mode`" do
+    expect(OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:verify_mode]).to eq(nil)
+  end
+
+  context "SSLSocket" do
+    # Code taken from the flores library by @jordansissels,
+    # https://github.com/jordansissel/ruby-flores/blob/master/spec/flores/pki_integration_spec.rb
+    # since these helpers were created to fix this particular issue
+    let(:csr) { Flores::PKI::CertificateSigningRequest.new }
+    # Here, I use a 1024-bit key for faster tests. 
+    # Please do not use such small keys in production.
+    let(:key_bits) { 1024 }
+    let(:key) { OpenSSL::PKey::RSA.generate(key_bits, 65537) }
+    let(:certificate_duration) { Flores::Random.number(1..86400) }
+
+    context "with self-signed client/server certificate" do
+      let(:certificate_subject) { "CN=server.example.com" }
+      let(:certificate) { csr.create }
+
+      # Returns [socket, address, port]
+      let(:listener) { Flores::Random.tcp_listener }
+      let(:server) { listener[0] }
+      let(:server_address) { listener[1] }
+      let(:server_port) { listener[2] }
+
+      let(:server_context) { OpenSSL::SSL::SSLContext.new }
+      let(:client_context) { OpenSSL::SSL::SSLContext.new }
+
+      before do
+        csr.subject = certificate_subject
+        csr.public_key = key.public_key
+        csr.start_time = Time.now
+        csr.expire_time = csr.start_time + certificate_duration
+        csr.signing_key = key
+        csr.want_signature_ability = true
+
+        server_context.cert = certificate
+        server_context.key = key
+
+        client_store = OpenSSL::X509::Store.new
+        client_store.add_cert(certificate)
+        client_context.cert_store = client_store
+        client_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+        ssl_server = OpenSSL::SSL::SSLServer.new(server, server_context)
+        Thread.new do
+          begin
+            ssl_server.accept
+          rescue => e
+            puts "Server accept failed: #{e}"
+          end
+        end
+      end
+
+      it "should successfully connect as a client" do
+        socket = TCPSocket.new(server_address, server_port)
+        ssl_client = OpenSSL::SSL::SSLSocket.new(socket, client_context)
+        expect { ssl_client.connect }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/logstash/pipeline_reporter_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/pipeline"
+require "logstash/pipeline_reporter"
+
+class DummyOutput < LogStash::Outputs::Base
+  config_name "dummyoutput"
+  milestone 2
+
+  attr_reader :num_closes, :events
+
+  def initialize(params={})
+    super
+    @num_closes = 0
+    @events = []
+  end
+
+  def register
+  end
+
+  def receive(event)
+    @events << event
+  end
+
+  def close
+    @num_closes += 1
+  end
+end
+
+#TODO: Figure out how to add more tests that actually cover inflight events
+#This will require some janky multithreading stuff
+describe LogStash::PipelineReporter do
+  let(:generator_count) { 5 }
+  let(:config) do
+    "input { generator { count => #{generator_count} } } output { dummyoutput {} } "
+  end
+  let(:pipeline) { LogStash::Pipeline.new(config)}
+  let(:reporter) { pipeline.reporter }
+
+  before do
+    allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutput").and_return(DummyOutput)
+    allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_call_original
+    allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
+
+    @pre_snapshot = reporter.snapshot
+    pipeline.run
+    @post_snapshot = reporter.snapshot
+  end
+
+  describe "events filtered" do
+    it "should start at zero" do
+      expect(@pre_snapshot.events_filtered).to eql(0)
+    end
+
+    it "should end at the number of generated events" do
+      expect(@post_snapshot.events_filtered).to eql(generator_count)
+    end
+  end
+
+  describe "events consumed" do
+    it "should start at zero" do
+      expect(@pre_snapshot.events_consumed).to eql(0)
+    end
+
+    it "should end at the number of generated events" do
+      expect(@post_snapshot.events_consumed).to eql(generator_count)
+    end
+  end
+
+  describe "inflight count" do
+    it "should be zero before running" do
+      expect(@pre_snapshot.inflight_count).to eql(0)
+    end
+
+    it "should be zero after running" do
+      expect(@post_snapshot.inflight_count).to eql(0)
+    end
+  end
+
+  describe "output states" do
+    it "should include the count of received events" do
+      expect(@post_snapshot.output_info.first[:events_received]).to eql(generator_count)
+    end
+  end
+end