logstash-core 2.0.0.beta1-java → 2.0.0.beta2-java

data/locales/en.yml

@@ -77,6 +77,10 @@ en:
         use to validate logstash's configuration before you choose
         to restart a running system.
       configuration:
+        obsolete: >-
+          The setting `%{name}` in plugin `%{plugin}` is obsolete and is no
+          longer available. %{extra} If you have any questions about this, you
+          are invited to visit https://discuss.elastic.co/c/logstash and ask.
         file-not-found: |-
           No config files found: %{path}
           Can you make sure this path is a logstash config file?

data/logstash-core.gemspec

@@ -23,6 +23,8 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
+  gem.add_runtime_dependency "concurrent-ruby", "~> 0.9.1"
+  gem.add_runtime_dependency "jruby-openssl", ">= 0.9.11" # Required to support TLSv1.2
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan
@@ -37,7 +39,7 @@ Gem::Specification.new do |gem|
 
   if RUBY_PLATFORM == 'java'
     gem.platform = RUBY_PLATFORM
-    gem.add_runtime_dependency "jrjackson", "~> 0.2.9" #(Apache 2.0 license)
+    gem.add_runtime_dependency "jrjackson", "~> 0.3.5" #(Apache 2.0 license)
   else
     gem.add_runtime_dependency "oj" #(MIT-style license)
   end

data/spec/core/config_cpu_core_strategy_spec.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/config/cpu_core_strategy"
+
+describe LogStash::Config::CpuCoreStrategy do
+
+  before do
+    allow(LogStash::Config::Defaults).to receive(:cpu_cores).and_return(cores)
+  end
+
+  context 'when the machine has 6 cores' do
+    let(:cores) { 6 }
+
+    it ".maximum should return 6" do
+      expect(described_class.maximum).to eq(6)
+    end
+
+    it ".fifty_percent should return 3" do
+      expect(described_class.fifty_percent).to eq(3)
+    end
+
+    it ".seventy_five_percent should return 4" do
+      expect(described_class.seventy_five_percent).to eq(4)
+    end
+
+    it ".twenty_five_percent should return 1" do
+      expect(described_class.twenty_five_percent).to eq(1)
+    end
+
+    it ".max_minus_one should return 5" do
+      expect(described_class.max_minus_one).to eq(5)
+    end
+
+    it ".max_minus_two should return 4" do
+      expect(described_class.max_minus_two).to eq(4)
+    end
+  end
+
+  context 'when the machine has 4 cores' do
+    let(:cores) { 4 }
+
+    it ".maximum should return 4" do
+      expect(described_class.maximum).to eq(4)
+    end
+
+    it ".fifty_percent should return 2" do
+      expect(described_class.fifty_percent).to eq(2)
+    end
+
+    it ".seventy_five_percent should return 3" do
+      expect(described_class.seventy_five_percent).to eq(3)
+    end
+
+    it ".twenty_five_percent should return 1" do
+      expect(described_class.twenty_five_percent).to eq(1)
+    end
+
+    it ".max_minus_one should return 3" do
+      expect(described_class.max_minus_one).to eq(3)
+    end
+
+    it ".max_minus_two should return 2" do
+      expect(described_class.max_minus_two).to eq(2)
+    end
+  end
+
+  context 'when the machine has 2 cores' do
+    let(:cores) { 2 }
+
+    it ".maximum should return 2" do
+      expect(described_class.maximum).to eq(2)
+    end
+
+    it ".fifty_percent should return 1" do
+      expect(described_class.fifty_percent).to eq(1)
+    end
+
+    it ".seventy_five_percent should return 1" do
+      expect(described_class.seventy_five_percent).to eq(1)
+    end
+
+    it ".twenty_five_percent should return 1" do
+      expect(described_class.twenty_five_percent).to eq(1)
+    end
+
+    it ".max_minus_one should return 1" do
+      expect(described_class.max_minus_one).to eq(1)
+    end
+
+    it ".max_minus_two should return 1" do
+      expect(described_class.max_minus_two).to eq(1)
+    end
+  end
+
+  context 'when the machine has 1 core' do
+    let(:cores) { 1 }
+
+    it ".maximum should return 1" do
+      expect(described_class.maximum).to eq(1)
+    end
+
+    it ".fifty_percent should return 1" do
+      expect(described_class.fifty_percent).to eq(1)
+    end
+
+    it ".seventy_five_percent should return 1" do
+      expect(described_class.seventy_five_percent).to eq(1)
+    end
+
+    it ".twenty_five_percent should return 1" do
+      expect(described_class.twenty_five_percent).to eq(1)
+    end
+
+    it ".max_minus_one should return 1" do
+      expect(described_class.max_minus_one).to eq(1)
+    end
+
+    it ".max_minus_two should return 1" do
+      expect(described_class.max_minus_two).to eq(1)
+    end
+  end
+
+end

data/spec/core/config_defaults_spec.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/config/defaults"
+
+describe LogStash::Config::Defaults do
+  it ".cpu_cores should return a positive integer" do
+    expect(described_class.cpu_cores.nil?).to be false
+    expect(described_class.cpu_cores.zero?).to be false
+  end
+end

data/spec/core/config_mixin_spec.rb

@@ -97,4 +97,58 @@ describe LogStash::Config::Mixin do
       expect(clone.password.value).to(be == secret)
     end
   end
+
+  describe "obsolete settings" do
+    let(:plugin_class) do
+      Class.new(LogStash::Inputs::Base) do
+        include LogStash::Config::Mixin
+        config_name "example"
+        config :foo, :validate => :string, :obsolete => "This feature was removed."
+      end
+    end
+
+    context "when using an obsolete setting" do
+      it "should cause a configuration error" do
+        expect {
+          plugin_class.new("foo" => "hello")
+        }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context "when using an obsolete settings from the parent class" do
+      it "should cause a configuration error" do
+        expect {
+          plugin_class.new("debug" => true)
+        }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context "when not using an obsolete setting" do
+      it "should not cause a configuration error" do
+        expect {
+          plugin_class.new({})
+        }.not_to raise_error
+      end
+    end
+  end
+
+  context "#params" do
+    let(:plugin_class) do
+      Class.new(LogStash::Filters::Base)  do
+        config_name "fake"
+        config :password, :validate => :password
+        config :bad, :validate => :string, :default => "my default", :obsolete => "not here"
+      end
+    end
+
+    subject { plugin_class.new({ "password" => "secret" }) }
+
+    it "should not return the obsolete options" do
+      expect(subject.params).not_to include("bad")
+    end
+
+    it "should include any other params" do
+      expect(subject.params).to include("password")
+    end
+  end
 end

data/spec/core/event_spec.rb

@@ -50,6 +50,10 @@ describe LogStash::Event do
         expect(subject.sprintf("%{+%s}")).to eq("1356998400")
       end
 
+      it "should work if there is no fieldref in the string" do
+        expect(subject.sprintf("bonjour")).to eq("bonjour")
+      end
+
       it "should raise error when formatting %{+%s} when @timestamp field is missing" do
         str = "hello-%{+%s}"
         subj = subject.clone
@@ -100,6 +104,10 @@ describe LogStash::Event do
         expect(subject.sprintf("%{[j][k3]}")).to eq("{\"4\":\"m\"}")
       end
 
+      it "should not strip last character" do
+        expect(subject.sprintf("%{type}%{message}|")).to eq("sprintfhello world|")
+      end
+
       context "#encoding" do
         it "should return known patterns as UTF-8" do
           expect(subject.sprintf("%{message}").encoding).to eq(Encoding::UTF_8)

data/spec/core/pipeline_spec.rb

@@ -11,7 +11,7 @@ class DummyInput < LogStash::Inputs::Base
   def run(queue)
   end
 
-  def teardown
+  def close
   end
 end
 
@@ -27,7 +27,7 @@ class DummyCodec < LogStash::Codecs::Base
     event
   end
 
-  def teardown
+  def close
   end
 end
 
@@ -35,11 +35,11 @@ class DummyOutput < LogStash::Outputs::Base
   config_name "dummyoutput"
   milestone 2
 
-  attr_reader :num_teardowns
+  attr_reader :num_closes
 
   def initialize(params={})
     super
-    @num_teardowns = 0
+    @num_closes = 0
   end
 
   def register
@@ -48,8 +48,8 @@ class DummyOutput < LogStash::Outputs::Base
   def receive(event)
   end
 
-  def teardown
-    @num_teardowns += 1
+  def close
+    @num_closes += 1
   end
 end
 
@@ -59,7 +59,7 @@ end
 
 describe LogStash::Pipeline do
 
-context "teardown" do
+context "close" do
 
   before(:each) do
     allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
@@ -93,24 +93,24 @@ context "teardown" do
       eos
     }
 
-    context "output teardown" do
-      it "should call teardown of output without output-workers" do
+    context "output close" do
+      it "should call close of output without output-workers" do
         pipeline = TestPipeline.new(test_config_without_output_workers)
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
         expect(pipeline.outputs.first.worker_plugins.size ).to eq(1)
-        expect(pipeline.outputs.first.worker_plugins.first.num_teardowns ).to eq(1)
+        expect(pipeline.outputs.first.worker_plugins.first.num_closes ).to eq(1)
       end
 
-      it "should call output teardown correctly with output workers" do
+      it "should call output close correctly with output workers" do
         pipeline = TestPipeline.new(test_config_with_output_workers)
         pipeline.run
 
         expect(pipeline.outputs.size ).to eq(1)
-        expect(pipeline.outputs.first.num_teardowns).to eq(0)
+        expect(pipeline.outputs.first.num_closes).to eq(0)
         pipeline.outputs.first.worker_plugins.each do |plugin|
-          expect(plugin.num_teardowns ).to eq(1)
+          expect(plugin.num_closes ).to eq(1)
         end
       end
     end

data/spec/filters/base_spec.rb

@@ -24,7 +24,7 @@ describe LogStash::Filters::Base do
   end
 
   it "should provide class public API" do
-    [:register, :filter, :multi_filter, :execute, :threadsafe?, :filter_matched, :filter?, :teardown].each do |method|
+    [:register, :filter, :multi_filter, :execute, :threadsafe?, :filter_matched, :filter?, :close].each do |method|
       expect(subject).to respond_to(method)
     end
   end
@@ -70,7 +70,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
         add_tag => ["test"]
       }
     }
@@ -79,25 +78,19 @@ describe LogStash::Filters::NOOP do
     sample("type" => "noop") do
       insist { subject["tags"] } == ["test"]
     end
-
-    sample("type" => "not_noop") do
-      insist { subject["tags"] }.nil?
-    end
   end
 
   describe "tags parsing with one tag" do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
-        tags => ["t1"]
         add_tag => ["test"]
       }
     }
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] }.nil?
+      insist { subject["tags"] } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
@@ -109,19 +102,17 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
-        tags => ["t1", "t2"]
         add_tag => ["test"]
       }
     }
     CONFIG
 
     sample("type" => "noop") do
-      insist { subject["tags"] }.nil?
+      insist { subject["tags"] } == ["test"]
     end
 
     sample("type" => "noop", "tags" => ["t1"]) do
-      insist { subject["tags"] } == ["t1"]
+      insist { subject["tags"] } == ["t1", "test"]
     end
 
     sample("type" => "noop", "tags" => ["t1", "t2"]) do
@@ -133,62 +124,10 @@ describe LogStash::Filters::NOOP do
     end
   end
 
-  describe "exclude_tags with 1 tag" do
-    config <<-CONFIG
-    filter {
-      noop {
-        type => "noop"
-        tags => ["t1"]
-        add_tag => ["test"]
-        exclude_tags => ["t2"]
-      }
-    }
-    CONFIG
-
-    sample("type" => "noop") do
-      insist { subject["tags"] }.nil?
-    end
-
-    sample("type" => "noop", "tags" => ["t1"]) do
-      insist { subject["tags"] } == ["t1", "test"]
-    end
-
-    sample("type" => "noop", "tags" => ["t1", "t2"]) do
-      insist { subject["tags"] } == ["t1", "t2"]
-    end
-  end
-
-  describe "exclude_tags with >1 tags" do
-    config <<-CONFIG
-    filter {
-      noop {
-        type => "noop"
-        tags => ["t1"]
-        add_tag => ["test"]
-        exclude_tags => ["t2", "t3"]
-      }
-    }
-    CONFIG
-
-    sample("type" => "noop", "tags" => ["t1", "t2", "t4"]) do
-      insist { subject["tags"] } == ["t1", "t2", "t4"]
-    end
-
-    sample("type" => "noop", "tags" => ["t1", "t3", "t4"]) do
-      insist { subject["tags"] } == ["t1", "t3", "t4"]
-    end
-
-    sample("type" => "noop", "tags" => ["t1", "t4", "t5"]) do
-      insist { subject["tags"] } == ["t1", "t4", "t5", "test"]
-    end
-  end
-
   describe "remove_tag" do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
-        tags => ["t1"]
         remove_tag => ["t2", "t3"]
       }
     }
@@ -223,8 +162,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
-        tags => ["t1"]
         remove_tag => ["%{blackhole}"]
       }
     }
@@ -245,7 +182,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
         remove_field => ["t2", "t3"]
       }
     }
@@ -271,7 +207,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
         remove_field => ["[t1][t2]"]
       }
     }
@@ -288,7 +223,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
         remove_field => ["[t1][0]"]
       }
     }
@@ -304,7 +238,6 @@ describe LogStash::Filters::NOOP do
     config <<-CONFIG
     filter {
       noop {
-        type => "noop"
         remove_field => ["%{blackhole}"]
       }
     }