logstash-core 2.2.2-java → 2.2.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1368f724e3cb3fe9e131d4b4c958fb02534ab8f
-  data.tar.gz: 2a5766cd24dbe8461e5d6c77ec1df8bd5af23441
+  metadata.gz: 39c92820b7e391a83095308508aea54dc84e1ea5
+  data.tar.gz: 324d6bd33d6caa20684de9c51430bae2db728c6d
 SHA512:
-  metadata.gz: c66d1d39b2e770296fb2901b1853e235dce5239733845fe779b79f2ba23f3be3b794d79db9dc5c20140c4d4e9a03e53b2963b5496e85af9e4d1e50b146fd7d20
-  data.tar.gz: 51a37e8aceafd7e3ad11d7a19d537eb2959d0919085ab9ca4d6763e420249f4a58631f04006ce8905e49906aa9a33ca1659c5b6633c7a8b43ce449d816203ec8
+  metadata.gz: 7db2d7f379a4d8b842a46cc3be888cab6867dbe2fa444f843d3e55480d2b1c59cbf35572d349cd4795b3e357f5eec5f18d83d33e38591befb7b395a6a168896a
+  data.tar.gz: 7ae5726ef2510ca14b70984c101cbc679f4a8b91753c9a90b0297d182100f9188191266223045286a5a19c19dcffb65548cbae5bb8c087ee98688ce3141e6a2c

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "2.2.2"
+LOGSTASH_CORE_VERSION = "2.2.3"

data/lib/logstash/agent.rb

@@ -107,12 +107,6 @@ class LogStash::Agent < Clamp::Command
     raise LogStash::ConfigurationError, message
   end # def fail
 
-  def report(message)
-    # Print to stdout just in case we're logging to a file
-    puts message
-    @logger.log(message) if log_file
-  end
-
   # Run the agent. This method is invoked after clamp parses the
   # flags given to this program.
   def execute
@@ -201,7 +195,7 @@ class LogStash::Agent < Clamp::Command
 
     # Stop now if we are only asking for a config test.
     if config_test?
-      report "Configuration OK"
+      @logger.terminal "Configuration OK"
       return
     end
 
@@ -212,15 +206,14 @@ class LogStash::Agent < Clamp::Command
     return 0
   rescue LogStash::ConfigurationError => e
     @logger.unsubscribe(stdout_logs) if show_startup_errors
-    report I18n.t("logstash.agent.error", :error => e)
+    @logger.error I18n.t("logstash.agent.error", :error => e)
     if !config_test?
-      report I18n.t("logstash.agent.configtest-flag-information")
+      @logger.info I18n.t("logstash.agent.configtest-flag-information")
     end
     return 1
   rescue => e
     @logger.unsubscribe(stdout_logs) if show_startup_errors
-    report I18n.t("oops", :error => e)
-    report e.backtrace if @logger.debug? || $DEBUGLIST.include?("stacktrace")
+    @logger.warn(I18n.t("oops"), :error => e, :class => e.class.name, :backtrace => e.backtrace)
     return 1
   ensure
     @log_fd.close if @log_fd

data/lib/logstash/output_delegator.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "concurrent/atomic/atomic_fixnum"
+java_import "java.util.concurrent.CopyOnWriteArrayList"
 
 # This class goes hand in hand with the pipeline to provide a pool of
 # free workers to be used by pipeline worker threads. The pool is
@@ -8,63 +9,19 @@ require "concurrent/atomic/atomic_fixnum"
 #
 # This plugin also records some basic statistics
 module LogStash class OutputDelegator
-  attr_reader :workers, :config, :worker_count, :threadsafe
+  attr_reader :workers, :config, :threadsafe
 
   # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
   # Internally these just get merged together into a single hash
-  def initialize(logger, klass, default_worker_count, *args)
+  def initialize(logger, klass, default_worker_count, *plugin_args)
     @logger = logger
     @threadsafe = klass.threadsafe?
-    @config = args.reduce({}, :merge)
+    @config = plugin_args.reduce({}, :merge)
     @klass = klass
-
-    # We define this as an array regardless of threadsafety
-    # to make reporting simpler, even though a threadsafe plugin will just have
-    # a single instance
-    #
-    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
-    # To detect these we need an instance to be created first :()
-    # TODO: In the next major version after 2.x remove support for this
-    @workers = [@klass.new(*args)]
-    @workers.first.register # Needed in case register calls `workers_not_supported`
-
-    # DO NOT move this statement before the instantiation of the first single instance
-    # Read the note above to understand why
-    @worker_count = calculate_worker_count(default_worker_count)
-    @logger.debug("Will start workers for output", :worker_count => @worker_count, :class => klass)
-
-    warn_on_worker_override!
-    # This queue is used to manage sharing across threads
-    @worker_queue = SizedQueue.new(@worker_count)
-
-    @workers += (@worker_count - 1).times.map do
-      inst = @klass.new(*args)
-      inst.register
-      inst
-    end
-
-    @workers.each { |w| @worker_queue << w }
-
+    @workers = java.util.concurrent.CopyOnWriteArrayList.new
+    @default_worker_count = default_worker_count
+    @registered = false
     @events_received = Concurrent::AtomicFixnum.new(0)
-
-
-    # One might wonder why we don't use something like
-    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
-    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
-    # The other 2% you get weird errors about rebinding to the same object
-    # Until we switch to Jruby 9.x keep the define_singleton_method parts
-    # the way they are, with a block
-    # See https://github.com/jruby/jruby/issues/3582
-    if threadsafe?
-      @threadsafe_worker = @workers.first
-      define_singleton_method(:multi_receive) do |events|
-        threadsafe_multi_receive(events)
-      end
-    else
-      define_singleton_method(:multi_receive) do |events|
-        worker_multi_receive(events)
-      end
-    end
   end
 
   def threadsafe?
@@ -89,11 +46,14 @@ module LogStash class OutputDelegator
     @config["workers"] && @config["workers"] > 1 && @klass.workers_not_supported?
   end
 
-  def calculate_worker_count(default_worker_count)
+  def target_worker_count
+    # Remove in 5.0 after all plugins upgraded to use class level declarations
+    raise ArgumentError, "Attempted to detect target worker count before instantiating a worker to test for legacy workers_not_supported!" if @workers.size == 0
+
     if @threadsafe || @klass.workers_not_supported?
       1
     else
-      @config["workers"] || default_worker_count
+      @config["workers"] || @default_worker_count
     end
   end
 
@@ -102,7 +62,58 @@ module LogStash class OutputDelegator
   end
 
   def register
-    @workers.each {|w| w.register}
+    raise ArgumentError, "Attempted to register #{self} twice!" if @registered
+    @registered = true
+    # We define this as an array regardless of threadsafety
+    # to make reporting simpler, even though a threadsafe plugin will just have
+    # a single instance
+    #
+    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
+    # To detect these we need an instance to be created first :()
+    # TODO: In the next major version after 2.x remove support for this
+    @workers << @klass.new(@config)
+    @workers.first.register # Needed in case register calls `workers_not_supported`
+
+    @logger.debug("Will start workers for output", :worker_count => target_worker_count, :class => @klass)
+
+    # Threadsafe versions don't need additional workers
+    setup_additional_workers!(target_worker_count) unless @threadsafe
+    # We skip the first worker because that's pre-registered to deal with legacy workers_not_supported
+    @workers.subList(1,@workers.size).each(&:register)
+    setup_multi_receive!
+  end
+
+  def setup_additional_workers!(target_worker_count)
+    warn_on_worker_override!
+
+    (target_worker_count - 1).times do
+      inst = @klass.new(@config)
+      @workers << inst
+    end
+
+    # This queue is used to manage sharing across threads
+    @worker_queue = SizedQueue.new(target_worker_count)
+    @workers.each {|w| @worker_queue << w }
+  end
+
+  def setup_multi_receive!
+    # One might wonder why we don't use something like
+    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
+    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
+    # The other 2% you get weird errors about rebinding to the same object
+    # Until we switch to Jruby 9.x keep the define_singleton_method parts
+    # the way they are, with a block
+    # See https://github.com/jruby/jruby/issues/3582
+    if threadsafe?
+      @threadsafe_worker = @workers.first
+      define_singleton_method(:multi_receive) do |events|
+        threadsafe_multi_receive(events)
+      end
+    else
+      define_singleton_method(:multi_receive) do |events|
+        worker_multi_receive(events)
+      end
+    end
   end
 
   def threadsafe_multi_receive(events)
@@ -125,9 +136,13 @@ module LogStash class OutputDelegator
   def do_close
     @logger.debug("closing output delegator", :klass => self)
 
-    @worker_count.times do
-      worker = @worker_queue.pop
-      worker.do_close
+    if @threadsafe
+      @workers.each(&:do_close)
+    else
+      worker_count.times do
+        worker = @worker_queue.pop
+        worker.do_close
+      end
     end
   end
 
@@ -140,10 +155,17 @@ module LogStash class OutputDelegator
     if @threadsafe
       0
     else
+      # The pipeline reporter can run before the outputs are registered trying to pull a value here
+      # In that case @worker_queue is empty, we just return 0
+      return 0 unless @worker_queue
       @workers.size - @worker_queue.size
     end
   end
 
+  def worker_count
+    @workers.size
+  end
+
   private
   # Needed for testing, so private
   attr_reader :threadsafe_worker, :worker_queue

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.2.2"
+LOGSTASH_VERSION = "2.2.3"

data/locales/en.yml

@@ -4,8 +4,7 @@
 #     for unformatted text.
 en:
   oops: |-
-    The error reported is: 
-      %{error}
+    An unexpected error occurred!
   logstash:
     environment:
       jruby-required:  >-

data/logstash-core.gemspec

@@ -4,8 +4,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'logstash-core/version'
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"]
-  gem.email         = ["jls@semicomplete.com", "petef@databits.net", "info@elasticsearch.com"]
+  gem.authors       = ["Elastic"]
+  gem.email         = ["info@elastic.co"]
   gem.description   = %q{The core components of logstash, the scalable log and event management tool}
   gem.summary       = %q{logstash-core - The core components of logstash}
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event", "~> 2.2.2"
+  gem.add_runtime_dependency "logstash-core-event", "~> 2.2.3"
 
   gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/logstash/output_delegator_spec.rb

@@ -27,6 +27,7 @@ describe LogStash::OutputDelegator do
 
     context "after having received a batch of events" do
       before do
+        subject.register
         subject.multi_receive(events)
       end
 
@@ -39,23 +40,35 @@ describe LogStash::OutputDelegator do
       end
     end
 
-    it "should register all workers on register" do
-      expect(out_inst).to receive(:register)
-      subject.register
-    end
 
-    it "should close all workers when closing" do
-      expect(out_inst).to receive(:do_close)
-      subject.do_close
+    describe "closing" do
+      before do
+        subject.register
+      end
+
+      it "should register all workers on register" do
+        expect(out_inst).to have_received(:register)
+      end
+
+      it "should close all workers when closing" do
+        expect(out_inst).to receive(:do_close)
+        subject.do_close
+      end
     end
 
     describe "concurrency and worker support" do
+      before do
+        allow(out_inst).to receive(:id).and_return("a-simple-plugin")
+        allow(out_inst).to receive(:metric=).with(any_args)
+        allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+      end
+
       describe "non-threadsafe outputs that allow workers" do
         let(:default_worker_count) { 3 }
 
         before do
           allow(out_klass).to receive(:threadsafe?).and_return(false)
-          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+          subject.register
         end
 
         it "should instantiate multiple workers" do
@@ -71,7 +84,7 @@ describe LogStash::OutputDelegator do
       describe "threadsafe outputs" do
         before do
           allow(out_klass).to receive(:threadsafe?).and_return(true)
-          allow(out_klass).to receive(:workers_not_supported?).and_return(false)
+          subject.register
         end
 
         it "should return true when threadsafe? is invoked" do
@@ -88,14 +101,18 @@ describe LogStash::OutputDelegator do
         end
 
         it "should not utilize the worker queue" do
-          expect(subject.send(:worker_queue)).not_to receive(:pop)
-          subject.multi_receive(events)
+          expect(subject.send(:worker_queue)).to be_nil
         end
 
         it "should send received events to the worker" do
           expect(out_inst).to receive(:multi_receive).with(events)
           subject.multi_receive(events)
         end
+
+        it "should close all workers when closing" do
+          expect(out_inst).to receive(:do_close)
+          subject.do_close
+        end
       end
     end
   end
@@ -115,11 +132,12 @@ describe LogStash::OutputDelegator do
     let(:default_worker_count) { 2 }
     let(:out_klass) { LogStash::Outputs::NOOPDelLegacyNoWorkers }
 
-    before do
+    before(:each) do
       allow(logger).to receive(:debug).with(any_args)
     end
 
     it "should only setup one worker" do
+      subject.register
       expect(subject.worker_count).to eql(1)
     end
   end

metadata

@@ -1,23 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.3
 platform: java
 authors:
-- Jordan Sissel
-- Pete Fritchman
-- Elasticsearch
+- Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2016-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.3
   name: logstash-core-event
   prerelease: false
   type: :runtime
@@ -25,7 +23,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.3
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -224,9 +222,7 @@ dependencies:
         version: 0.3.7
 description: The core components of logstash, the scalable log and event management tool
 email:
-- jls@semicomplete.com
-- petef@databits.net
-- info@elasticsearch.com
+- info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -332,7 +328,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: logstash-core - The core components of logstash