logstash-core 2.2.4.snapshot1

data/spec/logstash/plugin_spec.rb

@@ -0,0 +1,169 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/plugin"
+
+describe LogStash::Plugin do
+  it "should fail lookup on inexisting type" do
+    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    expect { LogStash::Plugin.lookup("badbadtype", "badname") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail lookup on inexisting name" do
+    expect_any_instance_of(Cabin::Channel).to receive(:debug).once
+    expect { LogStash::Plugin.lookup("filter", "badname") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail on bad plugin class" do
+    LogStash::Filters::BadSuperClass = Class.new
+    expect { LogStash::Plugin.lookup("filter", "bad_super_class") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should fail on missing config_name method" do
+    LogStash::Filters::MissingConfigName = Class.new(LogStash::Filters::Base)
+    expect { LogStash::Plugin.lookup("filter", "missing_config_name") }.to raise_error(LogStash::PluginLoadingError)
+  end
+
+  it "should lookup an already defined plugin class" do
+    class LogStash::Filters::LadyGaga < LogStash::Filters::Base
+      config_name "lady_gaga"
+    end
+    expect(LogStash::Plugin.lookup("filter", "lady_gaga")).to eq(LogStash::Filters::LadyGaga)
+  end
+
+  describe "#inspect" do
+    class LogStash::Filters::MyTestFilter < LogStash::Filters::Base
+      config_name "param1"
+      config :num, :validate => :number, :default => 20
+      config :str, :validate => :string, :default => "test"
+    end
+    subject { LogStash::Filters::MyTestFilter.new("num" => 1, "str" => "hello") }
+
+    it "should print the class of the filter" do
+      expect(subject.inspect).to match(/^<LogStash::Filters::MyTestFilter/)
+    end
+    it "should list config options and values" do
+      expect(subject.inspect).to match(/num=>1, str=>"hello"/)
+    end
+  end
+
+  context "when validating the plugin version" do
+    let(:plugin_name) { 'logstash-filter-stromae' }
+    subject do
+      Class.new(LogStash::Filters::Base) do
+        config_name 'stromae'
+      end
+    end
+
+    it "doesn't warn the user if the version is superior or equal to 1.0.0" do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('1.0.0')))
+
+      expect_any_instance_of(Cabin::Channel).not_to receive(:info)
+      subject.validate({})
+    end
+
+    it 'warns the user if the plugin version is between 0.9.x and 1.0.0' do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.9.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .with(/Using version 0.9.x/)
+
+      subject.validate({})
+    end
+
+    it 'warns the user if the plugin version is inferior to 0.9.x' do
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.1.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .with(/Using version 0.1.x/)
+      subject.validate({})
+    end
+
+    it "doesnt show the version notice more than once" do
+      one_notice = Class.new(LogStash::Filters::Base) do
+        config_name "stromae"
+      end
+
+      allow(Gem::Specification).to receive(:find_by_name)
+        .with(plugin_name)
+        .and_return(double(:version => Gem::Version.new('0.1.1')))
+
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+        .once
+        .with(/Using version 0.1.x/)
+
+      one_notice.validate({})
+      one_notice.validate({})
+    end
+
+    it "warns the user if we can't find a defined version" do
+      expect_any_instance_of(Cabin::Channel).to receive(:warn)
+        .once
+        .with(/plugin doesn't have a version/)
+
+      subject.validate({})
+    end
+
+
+    it 'logs a warning if the plugin use the milestone option' do
+      expect_any_instance_of(Cabin::Channel).to receive(:warn)
+        .with(/stromae plugin is using the 'milestone' method/)
+
+      class LogStash::Filters::Stromae < LogStash::Filters::Base
+        config_name "stromae"
+        milestone 2
+      end
+    end
+  end
+
+  describe "subclass initialize" do
+    let(:args) { Hash.new }
+
+    [
+      StromaeCodec = Class.new(LogStash::Codecs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeFilter = Class.new(LogStash::Filters::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeInput = Class.new(LogStash::Inputs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end,
+      StromaeOutput = Class.new(LogStash::Outputs::Base) do
+        config_name "stromae"
+        config :foo_tag, :validate => :string, :default => "bar"
+      end
+    ].each do |klass|
+
+      it "subclass #{klass.name} does not modify params" do
+        klass.new(args)
+        expect(args).to be_empty
+      end
+    end
+
+    context "codec initialization" do
+
+      class LogStash::Codecs::Noop < LogStash::Codecs::Base
+        config_name "noop"
+
+        config :format, :validate => :string
+        def register; end
+      end
+
+      it "should only register once" do
+        args   = { "codec" => LogStash::Codecs::Noop.new("format" => ".") }
+        expect_any_instance_of(LogStash::Codecs::Noop).to receive(:register).once
+        LogStash::Plugin.new(args)
+      end
+
+    end
+  end
+end

data/spec/logstash/runner_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/runner"
+require "stud/task"
+require "stud/trap"
+
+class NullRunner
+  def run(args); end
+end
+
+describe LogStash::Runner do
+
+  context "argument parsing" do
+    it "should run agent" do
+      expect(Stud::Task).to receive(:new).once.and_return(nil)
+      args = ["agent", "-e", ""]
+      expect(subject.run(args)).to eq(nil)
+    end
+
+    it "should run agent help" do
+      expect(subject).to receive(:show_help).once.and_return(nil)
+      args = ["agent", "-h"]
+      expect(subject.run(args).wait).to eq(0)
+    end
+
+    it "should show help with no arguments" do
+      expect($stderr).to receive(:puts).once.and_return("No command given")
+      expect($stderr).to receive(:puts).once
+      args = []
+      expect(subject.run(args).wait).to eq(1)
+    end
+
+    it "should show help for unknown commands" do
+      expect($stderr).to receive(:puts).once.and_return("No such command welp")
+      expect($stderr).to receive(:puts).once
+      args = ["welp"]
+      expect(subject.run(args).wait).to eq(1)
+    end
+  end
+
+  describe "pipeline settings" do
+    let(:pipeline_string) { "input { stdin {} } output { stdout {} }" }
+    let(:base_pipeline_settings) { { :pipeline_id => "base", :debug_config => false } }
+    let(:pipeline) { double("pipeline") }
+
+    before(:each) do
+      task = Stud::Task.new { 1 }
+      allow(pipeline).to receive(:run).and_return(task)
+    end
+
+    context "when pipeline workers is not defined by the user" do
+      it "should not pass the value to the pipeline" do
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, base_pipeline_settings).and_return(pipeline)
+        args = ["agent", "-e", pipeline_string]
+        subject.run(args).wait
+      end
+    end
+
+    context "when pipeline workers is defined by the user" do
+      it "should pass the value to the pipeline" do
+        base_pipeline_settings[:pipeline_workers] = 2
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, base_pipeline_settings).and_return(pipeline)
+        args = ["agent", "-w", "2", "-e", pipeline_string]
+        subject.run(args).wait
+      end
+    end
+  end
+end

data/spec/logstash/shutdown_watcher_spec.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/shutdown_watcher"
+
+describe LogStash::ShutdownWatcher do
+
+  let(:check_every) { 0.01 }
+  let(:check_threshold) { 100 }
+  subject { LogStash::ShutdownWatcher.new(pipeline, check_every) }
+  let(:pipeline) { double("pipeline") }
+  let(:reporter) { double("reporter") }
+  let(:reporter_snapshot) { double("reporter snapshot") }
+  report_count = 0
+
+  before :each do
+    allow(pipeline).to receive(:reporter).and_return(reporter)
+    allow(reporter).to receive(:snapshot).and_return(reporter_snapshot)
+    allow(reporter_snapshot).to receive(:o_simple_hash).and_return({})
+
+    allow(subject).to receive(:pipeline_report_snapshot).and_wrap_original do |m, *args|
+      report_count += 1
+      m.call(*args)
+    end
+  end
+
+  after :each do
+    report_count = 0
+  end
+
+  context "when pipeline is stalled" do
+    let(:increasing_count) { (1..5000).to_a }
+    before :each do
+      allow(reporter_snapshot).to receive(:inflight_count).and_return(*increasing_count)
+      allow(reporter_snapshot).to receive(:stalling_threads) { { } }
+    end
+
+    describe ".unsafe_shutdown = true" do
+      let(:abort_threshold) { subject.abort_threshold }
+      let(:report_every) { subject.report_every }
+
+      before :each do
+        subject.class.unsafe_shutdown = true
+      end
+
+      it "should force the shutdown" do
+        expect(subject).to receive(:force_exit).once
+        subject.start
+      end
+
+      it "should do exactly \"abort_threshold\" stall checks" do
+        allow(subject).to receive(:force_exit)
+        expect(subject).to receive(:shutdown_stalled?).exactly(abort_threshold).times.and_call_original
+        subject.start
+      end
+
+      it "should do exactly \"abort_threshold\"*\"report_every\" stall checks" do
+        allow(subject).to receive(:force_exit)
+        expect(subject).to receive(:pipeline_report_snapshot).exactly(abort_threshold*report_every).times.and_call_original
+        subject.start
+      end
+    end
+
+    describe ".unsafe_shutdown = false" do
+
+      before :each do
+        subject.class.unsafe_shutdown = false
+      end
+
+      it "shouldn't force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+  end
+
+  context "when pipeline is not stalled" do
+    let(:decreasing_count) { (1..5000).to_a.reverse }
+    before :each do
+      allow(reporter_snapshot).to receive(:inflight_count).and_return(*decreasing_count)
+      allow(reporter_snapshot).to receive(:stalling_threads) { { } }
+    end
+
+    describe ".unsafe_shutdown = true" do
+
+      before :each do
+        subject.class.unsafe_shutdown = true
+      end
+
+      it "should force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+
+    describe ".unsafe_shutdown = false" do
+
+      before :each do
+        subject.class.unsafe_shutdown = false
+      end
+
+      it "shouldn't force the shutdown" do
+        expect(subject).to_not receive(:force_exit)
+        thread = Thread.new(subject) {|subject| subject.start }
+        sleep 0.1 until report_count > check_threshold
+        thread.kill
+      end
+    end
+  end
+end

data/spec/logstash/util/buftok_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/util/buftok"
+
+describe  FileWatch::BufferedTokenizer  do
+
+  subject { FileWatch::BufferedTokenizer.new }
+
+  it "should tokenize a single token" do
+    expect(subject.extract("foo\n")).to eq(["foo"])
+  end
+
+  it "should merge multiple token" do
+    expect(subject.extract("foo")).to eq([])
+    expect(subject.extract("bar\n")).to eq(["foobar"])
+  end
+
+  it "should tokenize multiple token" do
+    expect(subject.extract("foo\nbar\n")).to eq(["foo", "bar"])
+  end
+
+  it "should ignore empty payload" do
+    expect(subject.extract("")).to eq([])
+    expect(subject.extract("foo\nbar")).to eq(["foo"])
+  end
+
+  it "should tokenize empty payload with newline" do
+    expect(subject.extract("\n")).to eq([""])
+    expect(subject.extract("\n\n\n")).to eq(["", "", ""])
+  end
+end

data/spec/logstash/util/charset_spec.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/util/charset"
+
+describe LogStash::Util::Charset do
+  let(:logger) { double("logger") }
+
+  context "with valid UTF-8 source encoding" do
+    subject {LogStash::Util::Charset.new("UTF-8")}
+
+    it "should return untouched data" do
+      ["foobar", "κόσμε"].each do |data|
+        expect(data.encoding.name).to eq("UTF-8")
+        expect(subject.convert(data)).to eq(data)
+        expect(subject.convert(data).encoding.name).to eq("UTF-8")
+      end
+    end
+  end
+
+  context "with invalid UTF-8 source encoding" do
+    subject do
+      LogStash::Util::Charset.new("UTF-8").tap do |charset|
+        charset.logger = logger
+      end
+    end
+
+    it "should escape invalid sequences" do
+      ["foo \xED\xB9\x81\xC3", "bar \xAD"].each do |data|
+        expect(data.encoding.name).to eq("UTF-8")
+        expect(data.valid_encoding?).to eq(false)
+        expect(logger).to receive(:warn).exactly(2).times
+        #logger.should_receive(:warn).twice
+        expect(subject.convert(data)).to eq(data.inspect[1..-2])
+        expect(subject.convert(data).encoding.name).to eq("UTF-8")
+      end
+    end
+
+  end
+
+  context "with valid non UTF-8 source encoding" do
+    subject {LogStash::Util::Charset.new("ISO-8859-1")}
+
+    it "should encode to UTF-8" do
+      samples = [
+        ["foobar", "foobar"],
+        ["\xE0 Montr\xE9al", "à Montréal"],
+      ]
+      samples.map{|(a, b)| [a.force_encoding("ISO-8859-1"), b]}.each do |(a, b)|
+        expect(a.encoding.name).to eq("ISO-8859-1")
+        expect(b.encoding.name).to eq("UTF-8")
+        expect(a.valid_encoding?).to eq(true)
+        expect(subject.convert(a).encoding.name).to eq("UTF-8")
+        expect(subject.convert(a)).to eq(b)
+      end
+    end
+  end
+
+  context "with invalid non UTF-8 source encoding" do
+    subject {LogStash::Util::Charset.new("ASCII-8BIT")}
+
+    it "should encode to UTF-8 and replace invalid chars" do
+      samples = [
+        ["\xE0 Montr\xE9al", "� Montr�al"],
+        ["\xCE\xBA\xCF\x8C\xCF\x83\xCE\xBC\xCE\xB5", "����������"],
+      ]
+      samples.map{|(a, b)| [a.force_encoding("ASCII-8BIT"), b]}.each do |(a, b)|
+        expect(a.encoding.name).to eq("ASCII-8BIT")
+        expect(b.encoding.name).to eq("UTF-8")
+        expect(subject.convert(a).encoding.name).to eq("UTF-8")
+        expect(subject.convert(a)).to eq(b)
+      end
+    end
+  end
+end