logstash-core 2.1.3-java → 2.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30d5b09c260219583243c4a9b051ba90f711f8f8
-  data.tar.gz: f1f41b55b7cc31ddcb5719af686d254521303198
+  metadata.gz: 7797b3d45a9c4f51bb59252dba06af6076ade965
+  data.tar.gz: faf5ef78e9b530598bddb9778d99244e73cb0125
 SHA512:
-  metadata.gz: d1a4112b84784a6203cfd81cfa39604dd6541f50cb903796c06922b017e0c1da4c87632cb1757e2e0b922636bbe0095fb374ad704027ec9efeff3298f9692650
-  data.tar.gz: 7c62b5a5f63b8b702af1b7e2dd111e4acd6c31a70f9313ef517adc5215995795bde732d695012dfdfc58d4dd8b3d806c470dde9cc944fd2880581a044bd58c5a
+  metadata.gz: 8021fa77ce2b27961dfa76fd42a65993f14b9273df0ecf013b77098feaf68779909469c6bd69535d52f800619f43d2d31984320de626e7d725e218a5cdced39b
+  data.tar.gz: 97992197bce414ee23b8f62c703334296274b943a414a5b2abf318eb9843c15964e029eb626f6d4458f7209053d1958fd3f25f2439518bb74530cc1866ef2a0d

data/lib/logstash-core.rb

@@ -1,3 +1 @@
-# encoding: utf-8
-module LogStash
-end
+require "logstash-core/logstash-core"

data/lib/logstash-core/logstash-core.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+module LogStash
+end

data/lib/logstash-core/version.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+# The version of logstash core gem.
+#
+# Note to authors: this should not include dashes because 'gem' barfs if
+# you include a dash in the version string.
+
+LOGSTASH_CORE_VERSION = "2.2.0"

data/lib/logstash/agent.rb

@@ -5,6 +5,7 @@ require "logstash/errors"
 require "logstash/config/cpu_core_strategy"
 require "uri"
 require "net/http"
+require "logstash/pipeline"
 LogStash::Environment.load_locale!
 
 class LogStash::Agent < Clamp::Command
@@ -20,10 +21,21 @@ class LogStash::Agent < Clamp::Command
            :default_input => DEFAULT_INPUT, :default_output => DEFAULT_OUTPUT),
     :default => "", :attribute_name => :config_string
 
-  option ["-w", "--filterworkers"], "COUNT",
-    I18n.t("logstash.agent.flag.filterworkers"),
-    :attribute_name => :filter_workers,
-    :default => 0, &:to_i
+  option ["-w", "--pipeline-workers"], "COUNT",
+         I18n.t("logstash.runner.flag.pipeline-workers"),
+         :attribute_name => :pipeline_workers,
+         :default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
+
+
+  option ["-b", "--pipeline-batch-size"], "SIZE",
+         I18n.t("logstash.runner.flag.pipeline-batch-size"),
+         :attribute_name => :pipeline_batch_size,
+         :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_size]
+
+  option ["-u", "--pipeline-batch-delay"], "DELAY_IN_MS",
+         I18n.t("logstash.runner.flag.pipeline-batch-delay"),
+         :attribute_name => :pipeline_batch_delay,
+         :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_delay]
 
   option ["-l", "--log"], "FILE",
     I18n.t("logstash.agent.flag.log"),
@@ -55,6 +67,32 @@ class LogStash::Agent < Clamp::Command
     :attribute_name => :unsafe_shutdown,
     :default => false
 
+  def initialize(*args)
+    super(*args)
+    @pipeline_settings ||= { :pipeline_id => "base" }
+  end
+
+  def pipeline_workers=(pipeline_workers_value)
+    @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
+  end
+
+  def pipeline_batch_size=(pipeline_batch_size_value)
+    @pipeline_settings[:pipeline_batch_size] = validate_positive_integer(pipeline_batch_size_value)
+  end
+
+  def pipeline_batch_delay=(pipeline_batch_delay_value)
+    @pipeline_settings[:pipeline_batch_delay] = validate_positive_integer(pipeline_batch_delay_value)
+  end
+
+  def validate_positive_integer(str_arg)
+    int_arg = str_arg.to_i
+    if str_arg !~ /^\d+$/ || int_arg < 1
+      raise ArgumentError, "Expected a positive integer, got '#{str_arg}'"
+    end
+
+    int_arg
+  end
+
   # Emit a warning message.
   def warn(message)
     # For now, all warnings are fatal.
@@ -80,8 +118,8 @@ class LogStash::Agent < Clamp::Command
     require "logstash/plugin"
     @logger = Cabin::Channel.get(LogStash)
 
-    LogStash::ShutdownController.unsafe_shutdown = unsafe_shutdown?
-    LogStash::ShutdownController.logger = @logger
+    LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
+    LogStash::ShutdownWatcher.logger = @logger
 
     if version?
       show_version
@@ -121,7 +159,7 @@ class LogStash::Agent < Clamp::Command
     end
 
     begin
-      pipeline = LogStash::Pipeline.new(@config_string)
+      pipeline = LogStash::Pipeline.new(@config_string, @pipeline_settings)
     rescue LoadError => e
       fail("Configuration problem.")
     end
@@ -151,8 +189,6 @@ class LogStash::Agent < Clamp::Command
       configure_logging(log_file)
     end
 
-    pipeline.configure("filter-workers", filter_workers) if filter_workers > 0
-
     # Stop now if we are only asking for a config test.
     if config_test?
       report "Configuration OK"
@@ -184,7 +220,7 @@ class LogStash::Agent < Clamp::Command
 
   def shutdown(pipeline)
     pipeline.shutdown do
-      ::LogStash::ShutdownController.start(pipeline)
+      ::LogStash::ShutdownWatcher.start(pipeline)
     end
   end
 
@@ -319,27 +355,19 @@ class LogStash::Agent < Clamp::Command
     Dir.glob(path).sort.each do |file|
       next unless File.file?(file)
       if file.match(/~$/)
-        @logger.debug("NOT reading config file because it is a temp file", :config_file => file)
+        @logger.debug("NOT reading config file because it is a temp file", :file => file)
         next
       end
-      @logger.debug("Reading config file", :config_file => file)
+      @logger.debug("Reading config file", :file => file)
       cfg = File.read(file)
       if !cfg.ascii_only? && !cfg.valid_encoding?
         encoding_issue_files << file
       end
       config << cfg + "\n"
-      if config_test?
-        @logger.debug? && @logger.debug("\nThe following is the content of a file", :config_file => file.to_s)
-        @logger.debug? && @logger.debug("\n" + cfg + "\n\n")
-      end
     end
     if (encoding_issue_files.any?)
       fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
     end
-    if config_test?
-      @logger.debug? && @logger.debug("\nThe following is the merged configuration")
-      @logger.debug? && @logger.debug("\n" + config + "\n\n")
-    end
     return config
   end # def load_config
 

data/lib/logstash/codecs/base.rb

@@ -11,7 +11,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
 
   def initialize(params={})
     super
-    config_init(params)
+    config_init(@params)
     register if respond_to?(:register)
   end
 
@@ -27,7 +27,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
     raise "#{self.class}#encode must be overidden"
   end # def encode
 
-  public 
+  public
   def close; end;
 
   # @param block [Proc(event, data)] the callback proc passing the original event and the encoded event

data/lib/logstash/config/config_ast.rb

@@ -107,7 +107,11 @@ module LogStash; module Config; module AST
       ["filter", "output"].each do |type|
         # defines @filter_func and @output_func
 
-        definitions << "def #{type}_func(event)"
+        # This need to be defined as a singleton method
+        # so each instance of the pipeline has his own implementation
+        # of the output/filter function
+        definitions << "define_singleton_method :#{type}_func do |event|"
+        definitions << "  targeted_outputs = []" if type == "output"
         definitions << "  events = [event]" if type == "filter"
         definitions << "  @logger.debug? && @logger.debug(\"#{type} received\", :event => event.to_hash)"
 
@@ -116,6 +120,7 @@ module LogStash; module Config; module AST
         end
 
         definitions << "  events" if type == "filter"
+        definitions << "  targeted_outputs" if type == "output"
         definitions << "end"
       end
 
@@ -237,7 +242,7 @@ module LogStash; module Config; module AST
           events = #{variable_name}.multi_filter(events)
         CODE
       when "output"
-        return "#{variable_name}.handle(event)\n"
+        return "targeted_outputs << #{variable_name}\n"
       when "codec"
         settings = attributes.recursive_select(Attribute).collect(&:compile).reject(&:empty?)
         attributes_code = "LogStash::Util.hash_merge_many(#{settings.map { |c| "{ #{c} }" }.join(", ")})"
@@ -402,7 +407,7 @@ module LogStash; module Config; module AST
         <<-CODE
           events = cond_func_#{i}(events)
         CODE
-      else
+      else # Output
         <<-CODE
           #{super}
           end

data/lib/logstash/environment.rb

@@ -1,18 +1,10 @@
 # encoding: utf-8
 require "logstash/errors"
-require "logstash/version"
 
 module LogStash
   module Environment
     extend self
 
-    # rehydrate the bootstrap environment if the startup was not done by executing bootstrap.rb
-    # and we are in the context of the logstash package
-    if !LogStash::Environment.const_defined?("LOGSTASH_HOME") &&  !ENV["LOGSTASH_HOME"].to_s.empty?
-      $LOAD_PATH << ::File.join(ENV["LOGSTASH_HOME"], "lib")
-      require "bootstrap/environment"
-    end
-
     LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
     LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
 
@@ -81,14 +73,6 @@ module LogStash
       ::Gem.win_platform?
     end
 
-    def vendor_path(path)
-      return ::File.join(LOGSTASH_HOME, "vendor", path)
-    end
-
-    def pattern_path(path)
-      return ::File.join(LOGSTASH_HOME, "patterns", path)
-    end
-
     def locales_path(path)
       return ::File.join(LOGSTASH_CORE, "locales", path)
     end

data/lib/logstash/filters/base.rb

@@ -120,7 +120,7 @@ class LogStash::Filters::Base < LogStash::Plugin
   public
   def initialize(params)
     super
-    config_init(params)
+    config_init(@params)
     @threadsafe = true
   end # def initialize
 
@@ -179,12 +179,16 @@ class LogStash::Filters::Base < LogStash::Plugin
 
     LogStash::Util::Decorators.add_tags(@add_tag,event,"filters/#{self.class.name}")
 
+    # note below that the tags array field needs to be updated then reassigned to the event.
+    # this is important because a construct like event["tags"].delete(tag) will not work
+    # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
     @remove_tag.each do |tag|
-      break if event["tags"].nil?
+      tags = event["tags"]
+      break if tags.nil? || tags.empty?
       tag = event.sprintf(tag)
-      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag",
-                                       :tag => tag)
-      event["tags"].delete(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
+      tags.delete(tag)
+      event["tags"] = tags
     end
   end # def filter_matched
 

data/lib/logstash/inputs/base.rb

@@ -53,7 +53,7 @@ class LogStash::Inputs::Base < LogStash::Plugin
     super
     @threadable = false
     @stop_called = Concurrent::AtomicBoolean.new(false)
-    config_init(params)
+    config_init(@params)
     @tags ||= []
   end # def initialize
 

data/lib/logstash/output_delegator.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+require "concurrent/atomic/atomic_fixnum"
+
+# This class goes hand in hand with the pipeline to provide a pool of
+# free workers to be used by pipeline worker threads. The pool is
+# internally represented with a SizedQueue set the the size of the number
+# of 'workers' the output plugin is configured with.
+#
+# This plugin also records some basic statistics
+module LogStash class OutputDelegator
+  attr_reader :workers, :config, :worker_count, :threadsafe
+
+  # The *args this takes are the same format that a Outputs::Base takes. A list of hashes with parameters in them
+  # Internally these just get merged together into a single hash
+  def initialize(logger, klass, default_worker_count, *args)
+    @logger = logger
+    @threadsafe = klass.threadsafe?
+    @config = args.reduce({}, :merge)
+    @klass = klass
+
+    # We define this as an array regardless of threadsafety
+    # to make reporting simpler, even though a threadsafe plugin will just have
+    # a single instance
+    #
+    # Older plugins invoke the instance method Outputs::Base#workers_not_supported
+    # To detect these we need an instance to be created first :()
+    # TODO: In the next major version after 2.x remove support for this
+    @workers = [@klass.new(*args)]
+    @workers.first.register # Needed in case register calls `workers_not_supported`
+
+    # DO NOT move this statement before the instantiation of the first single instance
+    # Read the note above to understand why
+    @worker_count = calculate_worker_count(default_worker_count)
+    @logger.debug("Will start workers for output", :worker_count => @worker_count, :class => klass)
+
+    warn_on_worker_override!
+    # This queue is used to manage sharing across threads
+    @worker_queue = SizedQueue.new(@worker_count)
+
+    @workers += (@worker_count - 1).times.map do
+      inst = @klass.new(*args)
+      inst.register
+      inst
+    end
+
+    @workers.each { |w| @worker_queue << w }
+
+    @events_received = Concurrent::AtomicFixnum.new(0)
+
+
+    # One might wonder why we don't use something like
+    # define_singleton_method(:multi_receive, method(:threadsafe_multi_receive)
+    # and the answer is this is buggy on Jruby 1.7.x . It works 98% of the time!
+    # The other 2% you get weird errors about rebinding to the same object
+    # Until we switch to Jruby 9.x keep the define_singleton_method parts
+    # the way they are, with a block
+    # See https://github.com/jruby/jruby/issues/3582
+    if threadsafe?
+      @threadsafe_worker = @workers.first
+      define_singleton_method(:multi_receive) do |events|
+        threadsafe_multi_receive(events)
+      end
+    else
+      define_singleton_method(:multi_receive) do |events|
+        worker_multi_receive(events)
+      end
+    end
+  end
+
+  def threadsafe?
+    !!@threadsafe
+  end
+
+  def warn_on_worker_override!
+    # The user has configured extra workers, but this plugin doesn't support it :(
+    if worker_limits_overriden?
+      message = @klass.workers_not_supported_message
+      warning_meta = {:plugin => @klass.config_name, :worker_count => @config["workers"]}
+      if message
+        warning_meta[:message] = message
+        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", warning_meta))
+      else
+        @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", warning_meta))
+      end
+    end
+  end
+
+  def worker_limits_overriden?
+    @config["workers"] && @config["workers"] > 1 && @klass.workers_not_supported?
+  end
+
+  def calculate_worker_count(default_worker_count)
+    if @threadsafe || @klass.workers_not_supported?
+      1
+    else
+      @config["workers"] || default_worker_count
+    end
+  end
+
+  def config_name
+    @klass.config_name
+  end
+
+  def register
+    @workers.each {|w| w.register}
+  end
+
+  def threadsafe_multi_receive(events)
+    @events_received.increment(events.length)
+
+    @threadsafe_worker.multi_receive(events)
+  end
+
+  def worker_multi_receive(events)
+    @events_received.increment(events.length)
+
+    worker = @worker_queue.pop
+    begin
+      worker.multi_receive(events)
+    ensure
+      @worker_queue.push(worker)
+    end
+  end
+
+  def do_close
+    @logger.debug("closing output delegator", :klass => self)
+
+    @worker_count.times do
+      worker = @worker_queue.pop
+      worker.do_close
+    end
+  end
+
+  def events_received
+    @events_received.value
+  end
+
+  # There's no concept of 'busy' workers for a threadsafe plugin!
+  def busy_workers
+    if @threadsafe
+      0
+    else
+      @workers.size - @worker_queue.size
+    end
+  end
+
+  private
+  # Needed for testing, so private
+  attr_reader :threadsafe_worker, :worker_queue
+end end

data/lib/logstash/outputs/base.rb

@@ -4,6 +4,8 @@ require "logstash/logging"
 require "logstash/plugin"
 require "logstash/namespace"
 require "logstash/config/mixin"
+require "logstash/util/wrapped_synchronous_queue"
+require "concurrent/atomic/atomic_fixnum"
 
 class LogStash::Outputs::Base < LogStash::Plugin
   include LogStash::Config::Mixin
@@ -23,23 +25,46 @@ class LogStash::Outputs::Base < LogStash::Plugin
   # Note that this setting may not be useful for all outputs.
   config :workers, :validate => :number, :default => 1
 
-  attr_reader :worker_plugins, :worker_queue, :worker_threads
+  attr_reader :worker_plugins, :available_workers, :workers, :worker_plugins, :workers_not_supported
+
+  def self.declare_threadsafe!
+    declare_workers_not_supported!
+    @threadsafe = true
+  end
+
+  def self.threadsafe?
+    @threadsafe == true
+  end
+
+  def self.declare_workers_not_supported!(message=nil)
+    @workers_not_supported_message = message
+    @workers_not_supported = true
+  end
+
+  def self.workers_not_supported_message
+    @workers_not_supported_message
+  end
+
+  def self.workers_not_supported?
+    !!@workers_not_supported
+  end
 
   public
+  # TODO: Remove this in the next major version after Logstash 2.x
+  # Post 2.x it should raise an error and tell people to use the class level
+  # declaration
   def workers_not_supported(message=nil)
-    return if @workers == 1
-    if message
-      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported-with-message", :plugin => self.class.config_name, :worker_count => @workers, :message => message))
-    else
-      @logger.warn(I18n.t("logstash.pipeline.output-worker-unsupported", :plugin => self.class.config_name, :worker_count => @workers))
-    end
-    @workers = 1
+    self.class.declare_workers_not_supported!(message)
   end
 
   public
   def initialize(params={})
     super
-    config_init(params)
+    config_init(@params)
+
+    # If we're running with a single thread we must enforce single-threaded concurrency by default
+    # Maybe in a future version we'll assume output plugins are threadsafe
+    @single_worker_mutex = Mutex.new
   end
 
   public
@@ -53,37 +78,9 @@ class LogStash::Outputs::Base < LogStash::Plugin
   end # def receive
 
   public
-  def worker_setup
-    if @workers == 1
-      @worker_plugins = [self]
-      @worker_threads = []
-    else
-      define_singleton_method(:handle, method(:handle_worker))
-      @worker_queue = SizedQueue.new(20)
-      @worker_plugins = @workers.times.map { self.class.new(@original_params.merge("workers" => 1)) }
-      @worker_threads = @worker_plugins.map.with_index do |plugin, i|
-        Thread.new(original_params, @worker_queue) do |params, queue|
-          LogStash::Util.set_thread_name(">#{self.class.config_name}.#{i}")
-          LogStash::Util.set_thread_plugin(self)
-          plugin.register
-          while true
-            event = queue.pop
-            plugin.handle(event)
-          end
-        end
-      end
-    end
-  end
-
-  public
-  def handle(event)
-    LogStash::Util.set_thread_plugin(self)
-    receive(event)
-  end # def handle
-
-  def handle_worker(event)
-    LogStash::Util.set_thread_plugin(self)
-    @worker_queue.push(event)
+  # To be overriden in implementations
+  def multi_receive(events)
+    events.each {|event| receive(event) }
   end
 
   private