logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/spec/logstash/config/config_ast_spec.rb

@@ -6,8 +6,6 @@ require "logstash/config/grammar"
 require "logstash/config/config_ast"
 
 describe LogStashConfigParser do
-  let(:settings) { mock_settings({}) }
-
   context '#parse' do
     context "valid configuration" do
       it "should permit single-quoted attribute names" do
@@ -79,7 +77,7 @@ describe LogStashConfigParser do
         }
       CONFIG
       subject { LogStashConfigParser.new }
-
+         
       it "should compile successfully" do
         result = subject.parse(config)
         expect(result).not_to(be_nil)
@@ -144,50 +142,12 @@ describe LogStashConfigParser do
         expect(config).to be_nil
       end
     end
-
-    context "when config.support_escapes" do
-      let(:parser) { LogStashConfigParser.new }
-
-      let(:processed_value)  { 'The computer says, "No"' }
-
-      let(:config) {
-        parser.parse(%q(
-          input {
-            foo {
-              bar => "The computer says, \"No\""
-            }
-          }
-        ))
-      }
-
-      let(:compiled_string) { eval(config.recursive_select(LogStash::Config::AST::String).first.compile) }
-
-      before do
-        config.process_escape_sequences = escapes
-      end
-
-      context "is enabled" do
-        let(:escapes) { true }
-
-        it "should process escape sequences" do
-          expect(compiled_string).to be == processed_value
-        end
-      end
-
-      context "is false" do
-        let(:escapes) { false }
-
-        it "should not process escape sequences" do
-          expect(compiled_string).not_to be == processed_value
-        end
-      end
-    end
   end
 
   context "when using two plugin sections of the same type" do
     let(:pipeline_klass) do
       Class.new do
-        def initialize(config, settings)
+        def initialize(config)
           grammar = LogStashConfigParser.new
           @config = grammar.parse(config)
           @code = @config.compile
@@ -206,7 +166,7 @@ describe LogStashConfigParser do
 
 
       it "should create a pipeline with both sections" do
-        generated_objects = pipeline_klass.new(config_string, settings).instance_variable_get("@generated_objects")
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
         filters = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^filter.+?_\d+$/) }
         expect(filters.size).to eq(2)
       end
@@ -221,13 +181,14 @@ describe LogStashConfigParser do
 
 
       it "should create a pipeline with both sections" do
-        generated_objects = pipeline_klass.new(config_string, settings).instance_variable_get("@generated_objects")
+        generated_objects = pipeline_klass.new(config_string).instance_variable_get("@generated_objects")
         outputs = generated_objects.keys.map(&:to_s).select {|obj_name| obj_name.match(/^output.+?_\d+$/) }
         expect(outputs.size).to eq(2)
       end
     end
   end
   context "when creating two instances of the same configuration" do
+
     let(:config_string) {
       "input { generator { } }
        filter {
@@ -240,7 +201,7 @@ describe LogStashConfigParser do
 
     let(:pipeline_klass) do
       Class.new do
-        def initialize(config, settings)
+        def initialize(config)
           grammar = LogStashConfigParser.new
           @config = grammar.parse(config)
           @code = @config.compile
@@ -252,8 +213,8 @@ describe LogStashConfigParser do
 
     describe "generated conditional functionals" do
       it "should be created per instance" do
-        instance_1 = pipeline_klass.new(config_string, settings)
-        instance_2 = pipeline_klass.new(config_string, settings)
+        instance_1 = pipeline_klass.new(config_string)
+        instance_2 = pipeline_klass.new(config_string)
         generated_method_1 = instance_1.instance_variable_get("@generated_objects")[:cond_func_1]
         generated_method_2 = instance_2.instance_variable_get("@generated_objects")[:cond_func_1]
         expect(generated_method_1).to_not be(generated_method_2)

data/spec/logstash/config/mixin_spec.rb

@@ -3,34 +3,6 @@ require "spec_helper"
 require "logstash/config/mixin"
 
 describe LogStash::Config::Mixin do
-  context "when encountering a deprecated option" do
-    let(:password) { "sekret" }
-    let(:double_logger) { double("logger").as_null_object }
-
-    subject do 
-      Class.new(LogStash::Filters::Base) do
-        include LogStash::Config::Mixin
-        config_name "test_deprecated"
-        milestone 1
-        config :old_opt, :validate => :string, :deprecated => "this is old school"
-        config :password, :validate => :password
-      end.new({
-        "old_opt" => "whut",
-        "password" => password
-      })
-    end
-
-    it "should not log the password" do
-      expect(LogStash::Logging::Logger).to receive(:new).with(anything).and_return(double_logger)
-      expect(double_logger).to receive(:warn) do |arg1,arg2|
-          message = 'You are using a deprecated config setting "old_opt" set in test_deprecated. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. this is old school If you have any questions about this, please visit the #logstash channel on freenode irc.'
-          expect(arg1).to eq(message)
-          expect(arg2[:plugin].to_s).to include('"password"=><password>')
-        end.once
-      subject
-    end
-  end
-
   context "when validating :bytes successfully" do
     subject do
       local_num_bytes = num_bytes # needs to be locally scoped :(
@@ -220,8 +192,8 @@ describe LogStash::Config::Mixin do
         expect(clone.uri.to_s).to eql(uri_hidden)
       end
 
-      it "should make the real java.net.URI object available under #uri" do
-        expect(subject.uri.uri).to be_a(java.net.URI)
+      it "should make the real URI object available under #uri" do
+        expect(subject.uri.uri).to be_a(::URI)
       end
 
       it "should obfuscate original_params" do
@@ -397,13 +369,11 @@ describe LogStash::Config::Mixin do
       before do
         ENV["FunString"] = "fancy"
         ENV["FunBool"] = "true"
-        ENV["SERVER_LS_TEST_ADDRESS"] = "some.host.address.tld"
       end
 
       after do
         ENV.delete("FunString")
         ENV.delete("FunBool")
-        ENV.delete("SERVER_LS_TEST_ADDRESS")
       end
 
       subject do
@@ -427,16 +397,6 @@ describe LogStash::Config::Mixin do
         expect(subject.nestedArray).to(be == { "level1" => [{ "key1" => "http://fancy:8080/blah.txt" }, { "key2" => "http://fancy:8080/foo.txt" }] })
         expect(subject.deepHash).to(be == { "level1" => { "level2" => { "level3" => { "key1" => "http://fancy:8080/blah.txt" } } } })
       end
-
-      it "should validate settings after interpolating ENV variables" do
-        expect {
-          Class.new(LogStash::Filters::Base) do
-            include LogStash::Config::Mixin
-            config_name "test"
-            config :server_address, :validate => :uri
-          end.new({"server_address" => "${SERVER_LS_TEST_ADDRESS}"})
-        }.not_to raise_error
-      end
     end
 
     context "should support $ in values" do

data/spec/logstash/config/pipeline_config_spec.rb

@@ -0,0 +1,75 @@
+# encoding: utf-8
+require "logstash/config/pipeline_config"
+require "logstash/config/source/local"
+
+describe LogStash::Config::PipelineConfig do
+  let(:source) { LogStash::Config::Source::Local }
+  let(:pipeline_id) { :main }
+  let(:ordered_config_parts) do
+    [
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/1", "input { generator1 }"),
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/2", "input { generator2 }"),
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/3", "input { generator3 }"),
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/4", "input { generator4 }"),
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/5", "input { generator5 }"),
+      org.logstash.common.SourceWithMetadata.new("file", "/tmp/6", "input { generator6 }"),
+      org.logstash.common.SourceWithMetadata.new("string", "config_string", "input { generator1 }"),
+    ]
+  end
+
+  let(:unordered_config_parts) { ordered_config_parts.shuffle }
+  let(:settings) { LogStash::SETTINGS }
+
+  subject { described_class.new(source, pipeline_id, unordered_config_parts, settings) }
+
+  it "returns the source" do
+    expect(subject.source).to eq(source)
+  end
+
+  it "returns the pipeline id" do
+    expect(subject.pipeline_id).to eq(pipeline_id)
+  end
+
+  it "returns the sorted config parts" do
+    expect(subject.config_parts).to eq(ordered_config_parts)
+  end
+
+  it "returns the config_hash" do
+    expect(subject.config_hash).not_to be_nil
+  end
+
+  it "returns the merged `ConfigPart#config_string`" do
+    expect(subject.config_string).to eq(ordered_config_parts.collect(&:text).join("\n"))
+  end
+
+  it "records when the config was read" do
+    expect(subject.read_at).to be <= Time.now
+  end
+
+  it "does object equality on config_hash and pipeline_id" do
+    another_exact_pipeline = described_class.new(source, pipeline_id, ordered_config_parts, settings)
+    expect(subject).to eq(another_exact_pipeline)
+
+    not_matching_pipeline = described_class.new(source, pipeline_id, [], settings)
+    expect(subject).not_to eq(not_matching_pipeline)
+
+    not_same_pipeline_id = described_class.new(source, :another_pipeline, unordered_config_parts, settings)
+    expect(subject).not_to eq(not_same_pipeline_id)
+  end
+
+  describe "#system?" do
+    context "when the pipeline is a system pipeline" do
+      let(:settings) { mock_settings({ "pipeline.system" => true })}
+
+      it "returns true if the pipeline is a system pipeline" do
+        expect(subject.system?).to be_truthy
+      end
+    end
+
+    context "when is not a system pipeline" do
+      it "returns false if the pipeline is not a system pipeline" do
+        expect(subject.system?).to be_falsey
+      end
+    end
+  end
+end

data/spec/logstash/config/source/local_spec.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+require "logstash/config/source/local"
+require "rspec/expectations"
+require "stud/temporary"
+require "fileutils"
+require "pathname"
+require_relative "../../../support/helpers"
+require_relative "../../../support/matchers"
+require "spec_helper"
+require "webmock/rspec"
+
+describe LogStash::Config::Source::Local::ConfigStringLoader do
+  subject { described_class }
+  let(:config_string) { "input { generator {} } output { stdout {} }"}
+
+  it "returns one config_parts" do
+    expect(subject.read(config_string).size).to eq(1)
+  end
+
+  it "returns a valid config part" do
+    config_part = subject.read(config_string).first
+    expect(config_part).to be_a_source_with_metadata("string", "config_string", config_string)
+  end
+end
+
+describe LogStash::Config::Source::Local::ConfigPathLoader do
+  subject { described_class }
+
+  context "no configs" do
+    context "in the directory" do
+      let(:directory) do
+        p =  Stud::Temporary.pathname
+        FileUtils.mkdir_p(p)
+        p
+      end
+
+      it "returns an empty array" do
+        expect(subject.read(directory)).to be_empty
+      end
+    end
+
+    context "target file doesn't exist" do
+      let(:directory) do
+        p =  Stud::Temporary.pathname
+        FileUtils.mkdir_p(p)
+        ::File.join(p, "ls.conf")
+      end
+
+      it "returns an empty array" do
+        expect(subject.read(directory)).to be_empty
+      end
+    end
+  end
+
+  context "when it exist" do
+    shared_examples "read config from files" do
+      let(:directory) { Stud::Temporary.pathname }
+
+      before do
+        files.each do |file, content|
+          temporary_file(content, file, directory)
+        end
+
+        expect(files.size).to be >= 1
+        expect(Dir.glob(::File.join(directory, "*")).size).to eq(files.size)
+      end
+
+      it "returns a `config_parts` per file" do
+        expect(subject.read(reader_config).size).to eq(files.size)
+      end
+
+      it "returns alphabetically sorted parts" do
+        parts = subject.read(reader_config)
+        expect(parts.collect { |part| ::File.basename(part.id) }).to eq(files.keys.sort)
+      end
+
+      it "returns valid `config_parts`" do
+        parts = subject.read(reader_config)
+
+        parts.each do |part|
+          basename = ::File.basename(part.id)
+          file_path = ::File.join(directory, basename)
+          content = files[basename]
+          expect(part).to be_a_source_with_metadata("file", file_path, content)
+        end
+      end
+    end
+
+    context "when the files have invalid encoding" do
+      let(:config_string) { "\x80" }
+      let(:file_path) { Stud::Temporary.pathname }
+      let(:file) { ::File.join(file_path, "wrong_encoding.conf") }
+
+      before do
+        FileUtils.mkdir_p(file_path)
+        f = File.open(file, "wb") do |file|
+          file.write(config_string)
+        end
+      end
+
+      it "raises an exception" do
+        expect { subject.read(file_path) }.to raise_error LogStash::ConfigLoadingError, /#{file_path}/
+      end
+    end
+
+    context "when we target one file" do
+      let(:reader_config) { ::File.join(directory, files.keys.first) }
+      let(:files) {
+        {
+          "config1.conf" => "input1",
+        }
+      }
+
+      include_examples "read config from files"
+    end
+
+    context "when we target a path with multiples files" do
+      let(:reader_config) { directory }
+
+      let(:files) {
+        {
+          "config1.conf" => "input1",
+          "config2.conf" => "input2",
+          "config3.conf" => "input3",
+          "config4.conf" => "input4"
+        }
+      }
+
+      include_examples "read config from files"
+    end
+
+    context "when there temporary files in the directory" do
+      let(:reader_config) { ::File.join(directory, "conf*.conf") }
+
+      let(:files) {
+        {
+          "config1.conf" => "input1",
+          "config2.conf" => "input2",
+          "config3.conf" => "input3",
+          "config4.conf" => "input4"
+        }
+      }
+
+      let(:other_files) do
+        {
+          "config1.conf~" => "input1",
+          "config2.conf~" => "input2",
+          "config3.conf~" => "input3",
+          "config4.conf~" => "input4"
+        }
+      end
+
+      include_examples "read config from files" do
+        before do
+          other_files.keys.shuffle.each do |file|
+            content = files[file]
+            temporary_file(content, file, directory)
+          end
+
+          # make sure we actually do some filtering
+          expect(Dir.glob(::File.join(directory, "*")).size).to eq(other_files.size + files.size)
+        end
+      end
+    end
+
+    context "when the path is a wildcard" do
+      let(:reader_config) { ::File.join(directory, "conf*.conf") }
+
+      let(:files) {
+        {
+          "config1.conf" => "input1",
+          "config2.conf" => "input2",
+          "config3.conf" => "input3",
+          "config4.conf" => "input4"
+        }
+      }
+
+      let(:other_files) do
+        {
+          "bad1.conf" => "input1",
+          "bad2.conf" => "input2",
+          "bad3.conf" => "input3",
+          "bad4.conf" => "input4"
+        }
+      end
+
+      include_examples "read config from files" do
+        before do
+          other_files.keys.shuffle.each do |file|
+            content = files[file]
+            temporary_file(content, file, directory)
+          end
+
+          # make sure we actually do some filtering
+          expect(Dir.glob(::File.join(directory, "*")).size).to eq(other_files.size + files.size)
+        end
+      end
+    end
+
+    context "URI defined path (file://..)" do
+      let(:reader_config) { "file://#{::File.join(directory, files.keys.first)}" }
+      let(:files) {
+        {
+          "config1.conf" => "input1",
+        }
+      }
+
+      include_examples "read config from files"
+    end
+
+    context "relative path" do
+      let(:reader_config) do
+        FileUtils.mkdir_p(::File.join(directory, "inside"))
+        ::File.join(directory, "inside", "../")
+      end
+
+      let(:files) {
+        {
+          "config2.conf" => "input1",
+          "config1.conf" => "input2",
+        }
+      }
+
+      include_examples "read config from files"
+    end
+  end
+end
+
+describe LogStash::Config::Source::Local::ConfigRemoteLoader do
+  before :all do
+    WebMock.disable_net_connect!
+  end
+
+  after :all do
+    WebMock.allow_net_connect!
+  end
+
+  subject { described_class }
+
+  let(:remote_url) { "http://test.dev/superconfig.conf" }
+
+  context "when the remote configuration exist" do
+    let(:config_string) { "input { generator {} } output { stdout {} }"}
+
+    before do
+      stub_request(:get, remote_url)
+        .to_return({
+        :body => config_string,
+        :status => 200
+      })
+    end
+
+    it "returns one config_parts" do
+      expect(subject.read(remote_url).size).to eq(1)
+    end
+
+    it "returns a valid config part" do
+      config_part = subject.read(remote_url).first
+      expect(config_part).to be_a_source_with_metadata("http", remote_url, config_string)
+    end
+  end
+
+  # I am aware that 656 http doesn't exist I am just testing the
+  # catch all block
+  [302, 404, 500, 403, 656].each do |code|
+    context "when the remote return an error code: #{code}" do
+      before do
+        stub_request(:get, remote_url)
+          .to_return({ :status => code })
+      end
+
+      it "raises the exception up" do
+        expect { subject.read(remote_url) }.to raise_error LogStash::ConfigLoadingError
+      end
+    end
+  end
+end
+
+describe LogStash::Config::Source::Local do
+  let(:input_block) { "input { generator {} }" }
+  let(:filter_block) { "filter { mutate {} } " }
+  let(:output_block) { "output { elasticsearch {}}" }
+  subject { described_class.new(settings) }
+
+  context "when `config.string` and `config.path` are set`" do
+    let(:config_file) { temporary_file(input_block) }
+
+    let(:settings) do
+      mock_settings(
+        "config.string" => "#{filter_block} #{output_block}",
+        "path.config" => config_file
+      )
+    end
+
+    it "returns a merged config" do
+      expect(subject.pipeline_configs.first.config_string).to include(input_block, output_block, filter_block)
+    end
+  end
+
+  context "when only the `config.string` is set" do
+    let(:settings) do
+      mock_settings( "config.string" => filter_block)
+    end
+
+    it "returns a config" do
+      expect(subject.pipeline_configs.first.config_string).to include(filter_block)
+    end
+  end
+
+  context "when only the `path.config` is set" do
+    let(:config_file) { temporary_file(input_block) }
+    let(:settings) do
+      mock_settings( "path.config" => config_file)
+    end
+
+    it "returns a config" do
+      expect(subject.pipeline_configs.first.config_string).to include(input_block)
+    end
+  end
+
+  context "when the `path.config` is an url" do
+    let(:remote_url) { "http://test.dev/superconfig.conf" }
+
+    before :all do
+      WebMock.disable_net_connect!
+    end
+
+    after :all do
+      WebMock.allow_net_connect!
+    end
+
+    before do
+      stub_request(:get, remote_url)
+        .to_return({
+        :body => input_block,
+        :status => 200
+      })
+    end
+
+    let(:settings) do
+      mock_settings( "path.config" => remote_url)
+    end
+
+    it "returns a config" do
+      expect(subject.pipeline_configs.first.config_string).to include(input_block)
+    end
+
+    context "when `config.string` is set" do
+      let(:settings) do
+        mock_settings(
+          "path.config" => remote_url,
+          "config.string" => filter_block
+        )
+      end
+
+      it "returns a merged config" do
+        expect(subject.pipeline_configs.first.config_string).to include(input_block, filter_block)
+      end
+    end
+  end
+
+  context "incomplete configuration" do
+    context "when the input block is missing" do
+      let(:settings) { mock_settings( "config.string" => "#{filter_block} #{output_block}") }
+
+      it "add stdin input" do
+        expect(subject.pipeline_configs.first.config_string).to include(LogStash::Config::Defaults.input)
+      end
+    end
+
+    context "when the output block is missing" do
+      let(:settings) { mock_settings( "config.string" => "#{input_block} #{filter_block}") }
+
+      it "add stdout output" do
+        expect(subject.pipeline_configs.first.config_string).to include(LogStash::Config::Defaults.output)
+      end
+    end
+
+    context "when both the output block and input block are missing" do
+      let(:settings) { mock_settings( "config.string" => "#{filter_block}") }
+
+      it "add stdin and output" do
+        expect(subject.pipeline_configs.first.config_string).to include(LogStash::Config::Defaults.output, LogStash::Config::Defaults.input)
+      end
+    end
+
+    context "when it has an input and an output" do
+      let(:settings) { mock_settings( "config.string" => "#{input_block} #{filter_block} #{output_block}") }
+
+      it "doesn't add anything" do
+        expect(subject.pipeline_configs.first.config_string).not_to include(LogStash::Config::Defaults.output, LogStash::Config::Defaults.input)
+      end
+    end
+  end
+end