logstash-core 2.3.0.snapshot3-java → 2.3.0.snapshot4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75d090462d8c05728226be1cff308e149ae54e27
-  data.tar.gz: 322a66722d833de679b086e946bb01a7d9168966
+  metadata.gz: dacf1faefe3817fb4158d35c81709879f3753206
+  data.tar.gz: 92ab29d4716a1569922a30bb11e88444de71472a
 SHA512:
-  metadata.gz: b28f06d3063faa64d8526298319c0c5f12f2ceaf0aef7071360d3011718c4c62acbc8bde2d80a6a5e8418e0c7b23138c52223f691c8fcf8e9d091632ec79dd31
-  data.tar.gz: 443aa26815d0ca0e0d9199b622934dcba1e98bb0fb24e55dfbd79746d40024c913a0be424028ad203f639075cd68a609c726a112489302928512d10f11154aad
+  metadata.gz: 701b4d572012650152f1af359d8a91a3837344dde79a8c8afabec771105550fc01f7fb9959e6c4516573587f662c8aeca5bec8e70c6b0e2066e4721e2cf2c83d
+  data.tar.gz: 1e1957c427e027b467c47cd1fabcbecee272118b5d7f7bffe10b7b7fb7ee07d5427d7f64ffd8d55879de5856e5e05a960afb55281e31fe9c4783d0a59c8c1dff

data/lib/logstash/agent.rb

@@ -83,10 +83,6 @@ class LogStash::Agent < Clamp::Command
     I18n.t("logstash.agent.flag.reload_interval"),
     :attribute_name => :reload_interval, :default => 3, &:to_i
 
-  option ["-n", "--node-name"], "NAME",
-    I18n.t("logstash.runner.flag.node_name"),
-    :attribute_name => :node_name, :default => Socket.gethostname
-
   def initialize(*params)
     super(*params)
     @logger = Cabin::Channel.get(LogStash)
@@ -342,6 +338,12 @@ class LogStash::Agent < Clamp::Command
   def register_pipeline(pipeline_id, settings)
     pipeline = create_pipeline(settings.merge(:pipeline_id => pipeline_id))
     return unless pipeline.is_a?(LogStash::Pipeline)
+    if @auto_reload && pipeline.non_reloadable_plugins.any?
+      @logger.error(I18n.t("logstash.agent.non_reloadable_config_register"),
+                    :pipeline_id => pipeline_id,
+                    :plugins => pipeline.non_reloadable_plugins.map(&:class))
+      return
+    end
     @pipelines[pipeline_id] = pipeline
   end
 
@@ -424,6 +426,10 @@ class LogStash::Agent < Clamp::Command
     if old_pipeline.config_str == new_pipeline.config_str
       @logger.debug("no configuration change for pipeline",
                     :pipeline => id, :config => old_pipeline.config_str)
+    elsif new_pipeline.non_reloadable_plugins.any?
+      @logger.error(I18n.t("logstash.agent.non_reloadable_config_reload"),
+                    :pipeline_id => id,
+                    :plugins => new_pipeline.non_reloadable_plugins.map(&:class))
     else
       @logger.log("fetched new config for pipeline. upgrading..",
                    :pipeline => id, :config => new_pipeline.config_str)

data/lib/logstash/output_delegator.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "concurrent/atomic/atomic_fixnum"
+java_import "java.util.concurrent.CopyOnWriteArrayList"
 
 # This class goes hand in hand with the pipeline to provide a pool of
 # free workers to be used by pipeline worker threads. The pool is
@@ -8,63 +9,19 @@ require "concurrent/atomic/atomic_fixnum"
 #
 # This plugin also records some basic statistics
 module LogStash class OutputDelegator
-  attr_reader :workers, :config, :worker_count, :threadsafe
+  attr_reader :workers, :config, :threadsafe
 
   # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
   # Internally these just get merged together into a single hash
-  def initialize(logger, klass, default_worker_count, *args)
+  def initialize(logger, klass, default_worker_count, *plugin_args)
     @logger = logger
     @threadsafe = klass.threadsafe?
-    @config = args.reduce({}, :merge)
+    @config = plugin_args.reduce({}, :merge)
     @klass = klass
-
-    # We define this as an array regardless of threadsafety
-    # to make reporting simpler, even though a threadsafe plugin will just have
-    # a single instance
-    #
-    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
-    # To detect these we need an instance to be created first :()
-    # TODO: In the next major version after 2.x remove support for this
-    @workers = [@klass.new(*args)]
-    @workers.first.register # Needed in case register calls `workers_not_supported`
-
-    # DO NOT move this statement before the instantiation of the first single instance
-    # Read the note above to understand why
-    @worker_count = calculate_worker_count(default_worker_count)
-    @logger.debug("Will start workers for output", :worker_count => @worker_count, :class => klass)
-
-    warn_on_worker_override!
-    # This queue is used to manage sharing across threads
-    @worker_queue = SizedQueue.new(@worker_count)
-
-    @workers += (@worker_count - 1).times.map do
-      inst = @klass.new(*args)
-      inst.register
-      inst
-    end
-
-    @workers.each { |w| @worker_queue << w }
-
+    @workers = java.util.concurrent.CopyOnWriteArrayList.new
+    @default_worker_count = default_worker_count
+    @registered = false
     @events_received = Concurrent::AtomicFixnum.new(0)
-
-
-    # One might wonder why we don't use something like
-    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
-    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
-    # The other 2% you get weird errors about rebinding to the same object
-    # Until we switch to Jruby 9.x keep the define_singleton_method parts
-    # the way they are, with a block
-    # See https://github.com/jruby/jruby/issues/3582
-    if threadsafe?
-      @threadsafe_worker = @workers.first
-      define_singleton_method(:multi_receive) do |events|
-        threadsafe_multi_receive(events)
-      end
-    else
-      define_singleton_method(:multi_receive) do |events|
-        worker_multi_receive(events)
-      end
-    end
   end
 
   def threadsafe?
@@ -89,11 +46,14 @@ module LogStash class OutputDelegator
     @config["workers"] && @config["workers"] > 1 && @klass.workers_not_supported?
   end
 
-  def calculate_worker_count(default_worker_count)
+  def target_worker_count
+    # Remove in 5.0 after all plugins upgraded to use class level declarations
+    raise ArgumentError, "Attempted to detect target worker count before instantiating a worker to test for legacy workers_not_supported!" if @workers.size == 0
+
     if @threadsafe || @klass.workers_not_supported?
       1
     else
-      @config["workers"] || default_worker_count
+      @config["workers"] || @default_worker_count
     end
   end
 
@@ -102,7 +62,58 @@ module LogStash class OutputDelegator
   end
 
   def register
-    @workers.each {|w| w.register}
+    raise ArgumentError, "Attempted to register #{self} twice!" if @registered
+    @registered = true
+    # We define this as an array regardless of threadsafety
+    # to make reporting simpler, even though a threadsafe plugin will just have
+    # a single instance
+    #
+    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
+    # To detect these we need an instance to be created first :()
+    # TODO: In the next major version after 2.x remove support for this
+    @workers << @klass.new(@config)
+    @workers.first.register # Needed in case register calls `workers_not_supported`
+
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+
+    # Threadsafe versions don't need additional workers
+    setup_additional_workers!(target_worker_count) unless @threadsafe
+    # We skip the first worker because that's pre-registered to deal with legacy workers_not_supported
+    @workers.subList(1,@workers.size).each(&:register)
+    setup_multi_receive!
+  end
+
+  def setup_additional_workers!(target_worker_count)
+    warn_on_worker_override!
+
+    (target_worker_count - 1).times do
+      inst = @klass.new(@config)
+      @workers << inst
+    end
+
+    # This queue is used to manage sharing across threads
+    @worker_queue = SizedQueue.new(target_worker_count)
+    @workers.each {|w| @worker_queue << w }
+  end
+
+  def setup_multi_receive!
+    # One might wonder why we don't use something like
+    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
+    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
+    # The other 2% you get weird errors about rebinding to the same object
+    # Until we switch to Jruby 9.x keep the define_singleton_method parts
+    # the way they are, with a block
+    # See https://github.com/jruby/jruby/issues/3582
+    if threadsafe?
+      @threadsafe_worker = @workers.first
+      define_singleton_method(:multi_receive) do |events|
+        threadsafe_multi_receive(events)
+      end
+    else
+      define_singleton_method(:multi_receive) do |events|
+        worker_multi_receive(events)
+      end
+    end
   end
 
   def threadsafe_multi_receive(events)
@@ -125,9 +136,13 @@ module LogStash class OutputDelegator
   def do_close
     @logger.debug("closing output delegator", :klass => self)
 
-    @worker_count.times do
-      worker = @worker_queue.pop
-      worker.do_close
+    if @threadsafe
+      @workers.each(&:do_close)
+    else
+      worker_count.times do
+        worker = @worker_queue.pop
+        worker.do_close
+      end
     end
   end
 
@@ -140,10 +155,17 @@ module LogStash class OutputDelegator
     if @threadsafe
       0
     else
+      # The pipeline reporter can run before the outputs are registered trying to pull a value here
+      # In that case @worker_queue is empty, we just return 0
+      return 0 unless @worker_queue
       @workers.size - @worker_queue.size
     end
   end
 
+  def worker_count
+    @workers.size
+  end
+
   private
   # Needed for testing, so private
   attr_reader :threadsafe_worker, :worker_queue

data/lib/logstash/pipeline.rb

@@ -28,6 +28,10 @@ module LogStash; class Pipeline
   }
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
+  RELOAD_INCOMPATIBLE_PLUGINS = [
+    "LogStash::Inputs::Stdin"
+  ]
+
   def self.validate_config(config_str, settings = {})
     begin
       # There should be a better way to test this
@@ -488,4 +492,11 @@ module LogStash; class Pipeline
       .each {|t| t.delete("blocked_on") }
       .each {|t| t.delete("status") }
   end
+
+  def non_reloadable_plugins
+    (inputs + filters + outputs).select do |plugin|
+      RELOAD_INCOMPATIBLE_PLUGINS.include?(plugin.class.name)
+    end
+  end
+
 end end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.3.0.snapshot3"
+LOGSTASH_VERSION = "2.3.0.snapshot4"

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.3.0.snapshot3"
+LOGSTASH_CORE_VERSION = "2.3.0.snapshot4"

data/locales/en.yml

@@ -73,6 +73,11 @@ en:
         data loss.
       forced_sigint: >-
         SIGINT received. Terminating immediately..
+      non_reloadable_config_reload: >-
+        Unable to reload configuration because it does not support dynamic reloading
+      non_reloadable_config_register: |-
+        Logstash was not able to load configuration since it does not support
+        dynamic reloading and -r or --auto-reload flag was enabled
       configtest-flag-information: |-
         You may be interested in the '--configtest' flag which you can
         use to validate logstash's configuration before you choose

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "2.3.0.snapshot3"
+  gem.add_runtime_dependency "logstash-core-event-java", "2.3.0.snapshot4"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/logstash/agent_spec.rb

@@ -25,8 +25,9 @@ describe LogStash::Agent do
 
   describe "register_pipeline" do
     let(:pipeline_id) { "main" }
+    let(:config_string) { "input { } filter { } output { }" }
     let(:settings) { {
-      :config_string => "input { } filter { } output { }",
+      :config_string => config_string,
       :pipeline_workers => 4
     } }
 
@@ -50,14 +51,20 @@ describe LogStash::Agent do
 
     context "when auto_reload is false" do
       let(:agent_args) { [ "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:pipeline_id) { "main" }
+      let(:pipeline_settings) { { :config_path => config_file } }
 
-      before :each do
-        allow(subject).to receive(:sleep)
-        allow(subject).to receive(:clean_state?).and_return(false)
-        allow(subject).to receive(:running_pipelines?).and_return(true)
+      before(:each) do
+        subject.register_pipeline(pipeline_id, pipeline_settings)
       end
 
       context "if state is clean" do
+        before :each do
+          allow(subject).to receive(:running_pipelines?).and_return(true)
+          allow(subject).to receive(:sleep)
+          allow(subject).to receive(:clean_state?).and_return(false)
+        end
+
         it "should not reload_state!" do
           expect(subject).to_not receive(:reload_state!)
           t = Thread.new { subject.execute }
@@ -66,10 +73,49 @@ describe LogStash::Agent do
           t.join
         end
       end
+
+      context "when calling reload_state!" do
+        context "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to_not receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            subject.send(:reload_state!)
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+
+        context "with a config that does not contain reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            subject.send(:reload_state!)
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+      end
     end
 
     context "when auto_reload is true" do
       let(:agent_args) { [ "--auto-reload", "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:pipeline_id) { "main" }
+      let(:pipeline_settings) { { :config_path => config_file } }
+
+      before(:each) do
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+      end
+
       context "if state is clean" do
         it "should periodically reload_state" do
           expect(subject).to receive(:reload_state!).at_least(3).times
@@ -81,6 +127,36 @@ describe LogStash::Agent do
           t.join
         end
       end
+
+      context "when calling reload_state!" do
+        context "with a config that contains reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { stdin {} } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to_not receive(:upgrade_pipeline)
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+
+        context "with a config that does not contain reload incompatible plugins" do
+          let(:second_pipeline_config) { "input { generator { } } filter { } output { }" }
+
+          it "does not reload if new config contains reload incompatible plugins" do
+            t = Thread.new { subject.execute }
+            sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
+            expect(subject).to receive(:upgrade_pipeline).at_least(2).times
+            File.open(config_file, "w") { |f| f.puts second_pipeline_config }
+            sleep 0.1
+            Stud.stop!(t)
+            t.join
+          end
+        end
+      end
     end
   end
 

data/spec/logstash/output_delegator_spec.rb

@@ -27,6 +27,7 @@ describe LogStash::OutputDelegator do
 
     context "after having received a batch of events" do
       before do
+        subject.register
         subject.multi_receive(events)
       end
 
@@ -39,23 +40,35 @@ describe LogStash::OutputDelegator do
       end
     end
 
-    it "should register all workers on register" do
-      expect(out_inst).to receive(:register)
-      subject.register
-    end
 
-    it "should close all workers when closing" do
-      expect(out_inst).to receive(:do_close)
-      subject.do_close
+    describe "closing" do
+      before do
+        subject.register
+      end
+
+      it "should register all workers on register" do
+        expect(out_inst).to have_received(:register)
+      end
+
+      it "should close all workers when closing" do
+        expect(out_inst).to receive(:do_close)
+        subject.do_close
+      end
     end
 
     describe "concurrency and worker support" do
+      before do
+        allow(out_inst).to receive(:id).and_return("a-simple-plugin")
+        allow(out_inst).to receive(:metric=).with(any_args)
+        allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      end
+
       describe "non-threadsafe outputs that allow workers" do
         let(:default_worker_count) { 3 }
 
         before do
           allow(out_klass).to receive(:threadsafe?).and_return(false)
-          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+          subject.register
         end
 
         it "should instantiate multiple workers" do
@@ -71,7 +84,7 @@ describe LogStash::OutputDelegator do
       describe "threadsafe outputs" do
         before do
           allow(out_klass).to receive(:threadsafe?).and_return(true)
-          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+          subject.register
         end
 
         it "should return true when threadsafe? is invoked" do
@@ -88,14 +101,18 @@ describe LogStash::OutputDelegator do
         end
 
         it "should not utilize the worker queue" do
-          expect(subject.send(:worker_queue)).not_to receive(:pop)
-          subject.multi_receive(events)
+          expect(subject.send(:worker_queue)).to be_nil
         end
 
         it "should send received events to the worker" do
           expect(out_inst).to receive(:multi_receive).with(events)
           subject.multi_receive(events)
         end
+
+        it "should close all workers when closing" do
+          expect(out_inst).to receive(:do_close)
+          subject.do_close
+        end
       end
     end
   end
@@ -115,11 +132,12 @@ describe LogStash::OutputDelegator do
     let(:default_worker_count) { 2 }
     let(:out_klass) { LogStash::Outputs::NOOPDelLegacyNoWorkers }
 
-    before do
+    before(:each) do
       allow(logger).to receive(:debug).with(any_args)
     end
 
     it "should only setup one worker" do
+      subject.register
       expect(subject.worker_count).to eql(1)
     end
   end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 2.3.0.snapshot3
+  version: 2.3.0.snapshot4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-21 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0.snapshot3
+        version: 2.3.0.snapshot4
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0.snapshot3
+        version: 2.3.0.snapshot4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: