logstash-core 2.2.4.snapshot2-java → 2.3.0-java

data/logstash-core.gemspec

@@ -17,9 +17,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event", "~> 2.2.4.snapshot2"
+  gem.add_runtime_dependency "logstash-core-event-java", "2.3.0"
 
-  gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
+  gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags

data/spec/logstash/agent_spec.rb

@@ -1,41 +1,246 @@
 # encoding: utf-8
 require 'spec_helper'
+require 'stud/temporary'
+require 'stud/task'
 
 describe LogStash::Agent do
-  subject { LogStash::Agent.new('') }
-  let(:dummy_config) { 'input {}' }
 
-  context "when loading the configuration" do
-    context "when local" do
-      before { expect(subject).to receive(:local_config).with(path) }
+  let(:logger) { double("logger") }
+  let(:agent_args) { [] }
+  subject { LogStash::Agent.new("", "") }
 
-      context "unix" do
-        let(:path) { './test.conf' }
-        it 'works with relative path' do
-          subject.load_config(path)
+  before :each do
+    [:log, :info, :warn, :error, :fatal, :debug].each do |level|
+      allow(logger).to receive(level)
+    end
+    [:info?, :warn?, :error?, :fatal?, :debug?].each do |level|
+      allow(logger).to receive(level)
+    end
+    allow(logger).to receive(:level=)
+    allow(logger).to receive(:subscribe)
+    subject.parse(agent_args)
+    subject.instance_variable_set("@reload_interval", 0.01)
+    subject.instance_variable_set("@logger", logger)
+  end
+
+  describe "register_pipeline" do
+    let(:pipeline_id) { "main" }
+    let(:config_string) { "input { } filter { } output { }" }
+    let(:settings) { {
+      :config_string => config_string,
+      :pipeline_workers => 4
+    } }
+
+    it "should delegate settings to new pipeline" do
+      expect(LogStash::Pipeline).to receive(:new).with(settings[:config_string], hash_including(settings))
+      subject.register_pipeline(pipeline_id, settings)
+    end
+  end
+
+  describe "#execute" do
+    let(:sample_config) { "input { generator { count => 100000 } } output { }" }
+    let(:config_file) { Stud::Temporary.pathname }
+
+    before :each do
+      File.open(config_file, "w") {|f| f.puts sample_config }
+    end
+
+    after :each do
+      File.unlink(config_file)
+    end
+
+    context "when auto_reload is false" do
+      let(:agent_args) { [ "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:pipeline_id) { "main" }
+      let(:pipeline_settings) { { :config_path => config_file } }
+
+      before(:each) do
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+      end
+
+      context "if state is clean" do
+        before :each do
+          allow(subject).to receive(:running_pipelines?).and_return(true)
+          allow(subject).to receive(:sleep)
+          allow(subject).to receive(:clean_state?).and_return(false)
+        end
+
+        it "should not reload_state!" do
+          expect(subject).to_not receive(:reload_state!)
+          t = Thread.new { subject.execute }
+          sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+          sleep 0.1
+          Stud.stop!(t)
+          t.join
         end
       end
 
-      context "windows" do
-        let(:path) { '.\test.conf' }
-        it 'work with relative windows path' do
-          subject.load_config(path)
+      context "when calling reload_state!" do
+        context "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to_not receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            subject.send(:reload_state!)
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+
+        context "with a config that does not contain reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            subject.send(:reload_state!)
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
         end
       end
     end
 
-    context "when remote" do
-      context 'supported scheme' do
-        let(:path) { "http://test.local/superconfig.conf" }
+    context "when auto_reload is true" do
+      let(:agent_args) { [ "--auto-reload", "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:pipeline_id) { "main" }
+      let(:pipeline_settings) { { :config_path => config_file } }
+
+      before(:each) do
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+      end
 
-        before { expect(Net::HTTP).to receive(:get) { dummy_config } }
-        it 'works with http' do
-          expect(subject.load_config(path)).to eq("#{dummy_config}\n")
+      context "if state is clean" do
+        it "should periodically reload_state" do
+          expect(subject).to receive(:reload_state!).at_least(3).times
+          t = Thread.new(subject) {|subject| subject.execute }
+          sleep 0.01 until (subject.running_pipelines? && subject.pipelines.values.first.ready?)
+          # now that the pipeline has started, give time for reload_state! to happen a few times
+          sleep 0.1
+          Stud.stop!(t)
+          t.join
+        end
+      end
+
+      context "when calling reload_state!" do
+        context "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to_not receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+
+        context "with a config that does not contain reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to receive(:upgrade_pipeline).at_least(2).times
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
         end
       end
     end
   end
 
+  describe "#reload_state!" do
+    let(:pipeline_id) { "main" }
+    let(:first_pipeline_config) { "input { } filter { } output { }" }
+    let(:second_pipeline_config) { "input { generator {} } filter { } output { }" }
+    let(:pipeline_settings) { {
+      :config_string => first_pipeline_config,
+      :pipeline_workers => 4
+    } }
+
+    before(:each) do
+      subject.register_pipeline(pipeline_id, pipeline_settings)
+    end
+
+    context "when fetching a new state" do
+      it "upgrades the state" do
+        expect(subject).to receive(:fetch_config).and_return(second_pipeline_config)
+        expect(subject).to receive(:upgrade_pipeline).with(pipeline_id, kind_of(LogStash::Pipeline))
+        subject.send(:reload_state!)
+      end
+    end
+    context "when fetching the same state" do
+      it "doesn't upgrade the state" do
+        expect(subject).to receive(:fetch_config).and_return(first_pipeline_config)
+        expect(subject).to_not receive(:upgrade_pipeline)
+        subject.send(:reload_state!)
+      end
+    end
+  end
+
+  describe "#upgrade_pipeline" do
+    let(:pipeline_id) { "main" }
+    let(:pipeline_config) { "input { } filter { } output { }" }
+    let(:pipeline_settings) { {
+      :config_string => pipeline_config,
+      :pipeline_workers => 4
+    } }
+    let(:new_pipeline_config) { "input { generator {} } output { }" }
+
+    before(:each) do
+      subject.register_pipeline(pipeline_id, pipeline_settings)
+    end
+
+    context "when the upgrade fails" do
+      before :each do
+        allow(subject).to receive(:fetch_config).and_return(new_pipeline_config)
+        allow(subject).to receive(:create_pipeline).and_return(nil)
+        allow(subject).to receive(:stop_pipeline)
+      end
+
+      it "leaves the state untouched" do
+        subject.send(:reload_state!)
+        expect(subject.pipelines[pipeline_id].config_str).to eq(pipeline_config)
+      end
+
+      context "and current state is empty" do
+        it "should not start a pipeline" do
+          expect(subject).to_not receive(:start_pipeline)
+          subject.send(:reload_state!)
+        end
+      end
+    end
+
+    context "when the upgrade succeeds" do
+      let(:new_config) { "input { generator { count => 1 } } output { }" }
+      before :each do
+        allow(subject).to receive(:fetch_config).and_return(new_config)
+        allow(subject).to receive(:stop_pipeline)
+      end
+      it "updates the state" do
+        subject.send(:reload_state!)
+        expect(subject.pipelines[pipeline_id].config_str).to eq(new_config)
+      end
+      it "starts the pipeline" do
+        expect(subject).to receive(:stop_pipeline)
+        expect(subject).to receive(:start_pipeline)
+        subject.send(:reload_state!)
+      end
+    end
+  end
+
   context "--pluginpath" do
     let(:single_path) { "/some/path" }
     let(:multiple_paths) { ["/some/path1", "/some/path2"] }
@@ -59,27 +264,66 @@ describe LogStash::Agent do
     end
   end
 
-  describe "debug_config" do
-    let(:pipeline_string) { "input {} output {}" }
+  describe "#fetch_config" do
+    let(:file_config) { "input { generator { count => 100 } } output { }" }
+    let(:cli_config) { "filter { drop { } } " }
+    let(:tmp_config_path) { Stud::Temporary.pathname }
+    let(:agent_args) { [ "-e", "filter { drop { } } ", "-f", tmp_config_path ] }
+
+    before :each do
+      IO.write(tmp_config_path, file_config)
+    end
+
+    after :each do
+      File.unlink(tmp_config_path)
+    end
+
+    it "should join the config string and config path content" do
+      settings = { :config_path => tmp_config_path, :config_string => cli_config }
+      fetched_config = subject.send(:fetch_config, settings)
+      expect(fetched_config.strip).to eq(cli_config + IO.read(tmp_config_path))
+    end
+  end
+
+  context "--pluginpath" do
+    let(:single_path) { "/some/path" }
+    let(:multiple_paths) { ["/some/path1", "/some/path2"] }
+
+    it "should fail with single invalid dir path" do
+      expect(File).to receive(:directory?).and_return(false)
+      expect(LogStash::Environment).not_to receive(:add_plugin_path)
+      expect{subject.configure_plugin_paths(single_path)}.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  describe "pipeline settings" do
+    let(:pipeline_string) { "input { stdin {} } output { stdout {} }" }
+    let(:main_pipeline_settings) { { :pipeline_id => "main" } }
     let(:pipeline) { double("pipeline") }
 
     before(:each) do
-      allow(pipeline).to receive(:run)
+      task = Stud::Task.new { 1 }
+      allow(pipeline).to receive(:run).and_return(task)
+      allow(pipeline).to receive(:shutdown)
     end
-    it "should set 'debug_config' to false by default" do
-      expect(LogStash::Pipeline).to receive(:new).
-        with(anything,hash_including(:debug_config => false)).
-        and_return(pipeline)
-      args = ["--debug", "-e", pipeline_string]
-      subject.run(args)
+
+    context "when :pipeline_workers is not defined by the user" do
+      it "should not pass the value to the pipeline" do
+        expect(LogStash::Pipeline).to receive(:new).once.with(pipeline_string, hash_excluding(:pipeline_workers)).and_return(pipeline)
+        args = ["-e", pipeline_string]
+        subject.run(args)
+      end
     end
 
-    it "should allow overriding debug_config" do
-      expect(LogStash::Pipeline).to receive(:new).
-        with(anything, hash_including(:debug_config => true))
-        .and_return(pipeline)
-      args = ["--debug", "--debug-config",  "-e", pipeline_string]
-      subject.run(args)
+    context "when :pipeline_workers is defined by the user" do
+      it "should pass the value to the pipeline" do
+        main_pipeline_settings[:pipeline_workers] = 2
+        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(main_pipeline_settings)).and_return(pipeline)
+        args = ["-w", "2", "-e", pipeline_string]
+        subject.run(args)
+      end
     end
   end
+
 end
+

data/spec/logstash/config/loader_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/config/loader"
+
+describe LogStash::Config::Loader do
+  subject { described_class.new(Cabin::Channel.get) }
+  context "when local" do
+    before { expect(subject).to receive(:local_config).with(path) }
+
+    context "unix" do
+      let(:path) { './test.conf' }
+      it 'works with relative path' do
+        subject.load_config(path)
+      end
+    end
+
+    context "windows" do
+      let(:path) { '.\test.conf' }
+      it 'work with relative windows path' do
+        subject.load_config(path)
+      end
+    end
+  end
+
+  context "when remote" do
+    context 'supported scheme' do
+      let(:path) { "http://test.local/superconfig.conf" }
+      let(:dummy_config) { 'input {}' }
+
+      before { expect(Net::HTTP).to receive(:get) { dummy_config } }
+      it 'works with http' do
+        expect(subject.load_config(path)).to eq("#{dummy_config}\n")
+      end
+    end
+  end
+end

data/spec/logstash/config/mixin_spec.rb

@@ -96,10 +96,6 @@ describe LogStash::Config::Mixin do
       clone = subject.class.new(subject.params)
       expect(clone.password.value).to(be == secret)
     end
-
-    it "should obfuscate original_params" do
-      expect(subject.original_params['password']).to(be_a(LogStash::Util::Password))
-    end
   end
 
   describe "obsolete settings" do
@@ -155,4 +151,80 @@ describe LogStash::Config::Mixin do
       expect(subject.params).to include("password")
     end
   end
+
+  context "environment variable evaluation" do
+    let(:plugin_class) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "one_plugin"
+        config :oneString, :validate => :string
+        config :oneBoolean, :validate => :boolean
+        config :oneNumber, :validate => :number
+        config :oneArray, :validate => :array
+        config :oneHash, :validate => :hash
+      end
+    end
+
+    context "when an environment variable is not set" do
+      context "and no default is given" do
+        before do
+          # Canary. Just in case somehow this is set.
+          expect(ENV["NoSuchVariable"]).to be_nil
+        end
+
+        it "should raise a configuration error" do
+          expect do
+            plugin_class.new("oneString" => "${NoSuchVariable}")
+          end.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "and a default is given" do
+        subject do
+          plugin_class.new(
+            "oneString" => "${notExistingVar:foo}",
+            "oneBoolean" => "${notExistingVar:true}",
+            "oneArray" => [ "first array value", "${notExistingVar:foo}", "${notExistingVar:}", "${notExistingVar: }", "${notExistingVar:foo bar}" ],
+            "oneHash" => { "key" => "${notExistingVar:foo}" }
+          )
+        end
+
+        it "should use the default" do
+          expect(subject.oneString).to(be == "foo")
+          expect(subject.oneBoolean).to be_truthy
+          expect(subject.oneArray).to(be == ["first array value", "foo", "", " ", "foo bar"])
+          expect(subject.oneHash).to(be == { "key" => "foo" })
+        end
+      end
+    end
+
+    context "when an environment variable is set" do
+      before do
+        ENV["FunString"] = "fancy"
+        ENV["FunBool"] = "true"
+      end
+
+      after do
+        ENV.delete("FunString")
+        ENV.delete("FunBool")
+      end
+
+      subject do
+        plugin_class.new(
+          "oneString" => "${FunString:foo}",
+          "oneBoolean" => "${FunBool:false}",
+          "oneArray" => [ "first array value", "${FunString:foo}" ],
+          "oneHash" => { "key1" => "${FunString:foo}", "key2" => "$FunString is ${FunBool}", "key3" => "${FunBool:false} or ${funbool:false}" }
+        )
+      end
+
+      it "should use the value in the variable" do
+        expect(subject.oneString).to(be == "fancy")
+        expect(subject.oneBoolean).to(be_truthy)
+        expect(subject.oneArray).to(be == [ "first array value", "fancy" ])
+        expect(subject.oneHash).to(be == { "key1" => "fancy", "key2" => "fancy is true", "key3" => "true or false" })
+      end
+
+    end
+  end
+
 end