logstash-core 2.0.0.beta1-java → 2.0.0.beta2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8280b889d663e0310425acfb0f7d75f4e1a024e0
-  data.tar.gz: a164ff373d9abbd7189adf99af24bacb3bc91238
+  metadata.gz: 6026efcfb1f8b27ca3aead70d7055af4b89acc91
+  data.tar.gz: 04df43c7a2703f14b7511906d0ab74c60b4e6954
 SHA512:
-  metadata.gz: c8b4f816cac02bf178e9b408a28a8178d4b608adbca9d58e0528fe9f9b52b1c0ab37edddd88f83b8f8b9b66de4fed6f9aafe8b4867b7ab5aca48588df8edeccb
-  data.tar.gz: 4dd5b531749f62087e2816b5855e4093f8039ad07a5c9753b1a8d2a58f9d3a311b7459f129ed4eaf38c311132c16537c766c6f85556bf438fe5809d924b75173
+  metadata.gz: 1d3cbf47dd791f55bb26a8c39ded8f13f7c44c44af5b6975e0f9133153d0896d5eb2807ba93564634a17ae25d6cd09ef961c687e1375cef8647ce6f458525602
+  data.tar.gz: 4f5f02dc8a9f35ffa97f4a6789712d0e542e70e73a637055da1a143268f5e5882476549ee87918018c3ea8b689b1bac008f83a47bb1d28115b89daff2abd178e

data/lib/logstash/agent.rb

@@ -2,6 +2,7 @@
 require "clamp" # gem 'clamp'
 require "logstash/environment"
 require "logstash/errors"
+require "logstash/config/cpu_core_strategy"
 require "uri"
 require "net/http"
 LogStash::Environment.load_locale!
@@ -21,7 +22,8 @@ class LogStash::Agent < Clamp::Command
 
   option ["-w", "--filterworkers"], "COUNT",
     I18n.t("logstash.agent.flag.filterworkers"),
-    :attribute_name => :filter_workers, :default => 1, &:to_i
+    :attribute_name => :filter_workers,
+    :default => LogStash::Config::CpuCoreStrategy.fifty_percent, &:to_i
 
   option ["-l", "--log"], "FILE",
     I18n.t("logstash.agent.flag.log"),
@@ -126,14 +128,14 @@ class LogStash::Agent < Clamp::Command
         @logger.warn(I18n.t("logstash.agent.sigint"))
         Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
         @interrupted_once = true
-        pipeline.shutdown
+        shutdown(pipeline)
       end
     end
 
     # Make SIGTERM shutdown the pipeline.
     sigterm_id = Stud::trap("TERM") do
       @logger.warn(I18n.t("logstash.agent.sigterm"))
-      pipeline.shutdown
+      shutdown(pipeline)
     end
 
     Stud::trap("HUP") do
@@ -172,6 +174,13 @@ class LogStash::Agent < Clamp::Command
     Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
   end # def execute
 
+  def shutdown(pipeline)
+    pipeline.shutdown do
+      InflightEventsReporter.logger = @logger
+      InflightEventsReporter.start(pipeline.input_to_filter, pipeline.filter_to_output, pipeline.outputs)
+    end
+  end
+
   def show_version
     show_version_logstash
 

data/lib/logstash/codecs/base.rb

@@ -28,7 +28,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
   end # def encode
 
   public 
-  def teardown; end;
+  def close; end;
 
   # @param block [Proc(event, data)] the callback proc passing the original event and the encoded event
   public

data/lib/logstash/config/cpu_core_strategy.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/config/defaults"
+
+module LogStash module Config module CpuCoreStrategy
+
+  extend self
+
+  def maximum
+    LogStash::Config::Defaults.cpu_cores
+  end
+
+  def fifty_percent
+    [1, (maximum * 0.5)].max.floor
+  end
+
+  def seventy_five_percent
+    [1, (maximum * 0.75)].max.floor
+  end
+
+  def twenty_five_percent
+    [1, (maximum * 0.25)].max.floor
+  end
+
+  def max_minus_one
+    [1, (maximum - 1)].max.floor
+  end
+
+  def max_minus_two
+    [1, (maximum - 2)].max.floor
+  end
+end end end

data/lib/logstash/config/defaults.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "concurrent"
+
+module LogStash module Config module Defaults
+
+  extend self
+
+  def cpu_cores
+    Concurrent.processor_count
+  end
+end end end

data/lib/logstash/config/mixin.rb

@@ -76,6 +76,13 @@ module LogStash::Config::Mixin
                      "about this, please visit the #logstash channel " +
                      "on freenode irc.", :name => name, :plugin => self)
       end
+      if opts && opts[:obsolete]
+        extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
+        extra.gsub!("%PLUGIN%", self.class.config_name)
+        raise LogStash::ConfigurationError,
+          I18n.t("logstash.agent.configuration.obsolete", :name => name,
+                 :plugin => self.class.config_name, :extra => extra)
+      end
     end
 
     # Set defaults from 'config :foo, :default => somevalue'
@@ -103,6 +110,16 @@ module LogStash::Config::Mixin
         I18n.t("logstash.agent.configuration.invalid_plugin_settings")
     end
 
+    # We remove any config options marked as obsolete,
+    # no code should be associated to them and their values should not bleed
+    # to the plugin context.
+    #
+    # This need to be done after fetching the options from the parents classed
+    params.reject! do |name, value|
+      opts = self.class.get_config[name]
+      opts.include?(:obsolete)
+    end
+
     # set instance variables like '@foo'  for each config value given.
     params.each do |key, value|
       next if key[0, 1] == "@"

data/lib/logstash/filters/base.rb

@@ -11,22 +11,11 @@ class LogStash::Filters::Base < LogStash::Plugin
 
   config_name "filter"
 
-  # Note that all of the specified routing options (`type`,`tags`,`exclude_tags`,`include_fields`,
-  # `exclude_fields`) must be met in order for the event to be handled by the filter.
+  config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
 
-  # The type to act on. If a type is given, then this filter will only
-  # act on messages with the same type. See any input plugin's `type`
-  # attribute for more.
-  # Optional.
-  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
-
-  # Only handle events with all of these tags.
-  # Optional.
-  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+  config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
 
-  # Only handle events without any of these tags.
-  # Optional.
-  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+  config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
 
   # If this filter is successful, add arbitrary tags to the event.
   # Tags can be dynamic and include parts of the event using the `%{field}`
@@ -202,40 +191,12 @@ class LogStash::Filters::Base < LogStash::Plugin
 
   protected
   def filter?(event)
-    if !@type.empty?
-      if event["type"] != @type
-        @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because type doesn't match",
-                                         :type=> @type, :event => event)
-        return false
-      end
-    end
-
-    if !@tags.empty?
-      # this filter has only works on events with certain tags,
-      # and this event has no tags.
-      return false if !event["tags"]
-
-      # Is @tags a subset of the event's tags? If not, skip it.
-      if (event["tags"] & @tags).size != @tags.size
-        @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because tags don't match",
-                                         :tags => tags, :event => event)
-        return false
-      end
-    end
-
-    if !@exclude_tags.empty? && event["tags"]
-      if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
-        @logger.debug("filters/#{self.class.name}: Skipping event because tags contains excluded tags:",
-                      :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
-        return false
-      end
-    end
-
-    return true
-  end
+    # TODO: noop for now, remove this once we delete this call from all plugins
+    true
+  end # def filter?
 
   public
-  def teardown
+  def close
     # Nothing to do by default.
   end
 end # class LogStash::Filters::Base

data/lib/logstash/inputs/base.rb

@@ -26,31 +26,16 @@ class LogStash::Inputs::Base < LogStash::Plugin
   # when sent to another Logstash server.
   config :type, :validate => :string
 
-  config :debug, :validate => :boolean, :default => false, :deprecated => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
+  config :debug, :validate => :boolean, :default => false, :obsolete => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
 
-  # The format of input data (plain, json, json_event)
-  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :deprecated => "You should use the newer 'codec' setting instead."
+  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :obsolete => "You should use the newer 'codec' setting instead."
 
-  # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
-  config :codec, :validate => :codec, :default => "plain"
+  config :charset, :obsolete => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
 
-  # The character encoding used in this input. Examples include `UTF-8`
-  # and `cp1252`
-  #
-  # This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
-  # or in another character set other than `UTF-8`.
-  #
-  # This only affects `plain` format logs since json is `UTF-8` already.
-  config :charset, :deprecated => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
+  config :message_format, :validate => :string, :obsolete => "Setting is no longer valid."
 
-  # If format is `json`, an event `sprintf` string to build what
-  # the display `@message` should be given (defaults to the raw JSON).
-  # `sprintf` format strings look like `%{fieldname}`
-  #
-  # If format is `json_event`, ALL fields except for `@type`
-  # are expected to be present. Not receiving all fields
-  # will cause unexpected results.
-  config :message_format, :validate => :string, :deprecated => true
+  # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
 
   # Add any number of arbitrary tags to your event.
   #
@@ -67,27 +52,9 @@ class LogStash::Inputs::Base < LogStash::Plugin
   def initialize(params={})
     super
     @threadable = false
+    @stop_called = Concurrent::AtomicBoolean.new(false)
     config_init(params)
     @tags ||= []
-
-    if @charset && @codec.class.get_config.include?("charset")
-      # charset is deprecated on inputs, but provide backwards compatibility
-      # by copying the charset setting into the codec.
-
-      @logger.info("Copying input's charset setting into codec", :input => self, :codec => @codec)
-      charset = @charset
-      @codec.instance_eval { @charset = charset }
-    end
-
-    # Backwards compat for the 'format' setting
-    case @format
-      when "plain"; # do nothing
-      when "json"
-        @codec = LogStash::Plugin.lookup("codec", "json").new
-      when "json_event"
-        @codec = LogStash::Plugin.lookup("codec", "oldlogstashjson").new
-    end
-
   end # def initialize
 
   public
@@ -100,6 +67,28 @@ class LogStash::Inputs::Base < LogStash::Plugin
     @tags << newtag
   end # def tag
 
+  public
+  # override stop if you need to do more than do_stop to
+  # enforce the input plugin to return from `run`.
+  # e.g. a tcp plugin might need to close the tcp socket
+  # so blocking read operation aborts
+  def stop
+    # override if necessary
+  end
+
+  public
+  def do_stop
+    @logger.debug("stopping", :plugin => self)
+    @stop_called.make_true
+    stop
+  end
+
+  # stop? should never be overriden
+  public
+  def stop?
+    @stop_called.value
+  end
+
   protected
   def to_event(raw, source)
     raise LogStash::ThisMethodWasRemoved("LogStash::Inputs::Base#to_event - you should use codecs now instead of to_event. Not sure what this means? Get help on https://discuss.elastic.co/c/logstash")

data/lib/logstash/json.rb

@@ -32,15 +32,23 @@ module LogStash
     ### JRuby
 
     def jruby_load(data, options = {})
-      options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
+      # TODO [guyboertje] remove these comments in 5.0
+      # options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
+
+      JrJackson::Ruby.parse(data, options)
+
     rescue JrJackson::ParseError => e
       raise LogStash::Json::ParserError.new(e.message)
     end
 
     def jruby_dump(o)
+      # TODO [guyboertje] remove these comments in 5.0
       # test for enumerable here to work around an omission in JrJackson::Json.dump to
       # also look for Java::JavaUtil::ArrayList, see TODO submit issue
-      o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
+      # o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
+
+      JrJackson::Base.generate(o, {})
+
     rescue => e
       raise LogStash::Json::GeneratorError.new(e.message)
     end

data/lib/logstash/outputs/base.rb

@@ -12,19 +12,11 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   config_name "output"
 
-  # The type to act on. If a type is given, then this output will only
-  # act on messages with the same type. See any input plugin's `type`
-  # attribute for more.
-  # Optional.
-  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+  config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
 
-  # Only handle events with all of these tags.
-  # Optional.
-  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+  config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
 
-  # Only handle events without any of these tags.
-  # Optional.
-  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+  config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
 
   # The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output, without needing a separate filter in your Logstash pipeline.
   config :codec, :validate => :codec, :default => "plain"
@@ -94,31 +86,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
 
   private
   def output?(event)
-    if !@type.empty?
-      if event["type"] != @type
-        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because type doesn't match",
-                                         :type => @type, :event => event)
-        return false
-      end
-    end
-
-    if !@tags.empty?
-      return false if !event["tags"]
-      if (event["tags"] & @tags).size != @tags.size
-        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags don't match",
-                                         :tags => @tags, :event => event)
-        return false
-      end
-    end
-
-    if !@exclude_tags.empty? && event["tags"]
-      if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
-        @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags contains excluded tags",
-                                         :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
-        return false
-      end
-    end
-
-    return true
-  end
+    # TODO: noop for now, remove this once we delete this call from all plugins
+    true
+  end # def output?
 end # class LogStash::Outputs::Base

data/lib/logstash/patches.rb

@@ -3,3 +3,4 @@ require "logstash/patches/bugfix_jruby_2558"
 require "logstash/patches/cabin"
 require "logstash/patches/profile_require_calls"
 require "logstash/patches/stronger_openssl_defaults"
+require "logstash/patches/silence_concurrent_ruby_warning"

data/lib/logstash/patches/silence_concurrent_ruby_warning.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "concurrent/concern/logging"
+require "concurrent/concern/deprecation"
+require "concurrent/version"
+require "cabin"
+
+# Concurrent-ruby is throwing warning when the code is run under jdk7, and they
+# will provide best effort support, logstash has to support JDK7 for a few months.
+#
+# By default all deprecation warnings of the concurrent ruby
+# library use the `WARN` level which is show everytime we boot logstash,
+# This monkeypatch change the log level of the deprecation warning to be `debug`
+# instead. This monkey patch might be a bit over kill but there is no
+# easy way to override the java version check.
+#
+# ref: https://github.com/ruby-concurrency/concurrent-ruby/blob/v0.9.1/lib/concurrent/configuration.rb#L284-L295
+#
+# This patch is only valid for 0.9.1
+if Concurrent::VERSION == "0.9.1"
+  module Concurrent
+    module Concern
+      module Deprecation
+        include Concern::Logging
+
+        def deprecated(message, strip = 2)
+          caller_line = caller(strip).first if strip > 0
+          klass       = if Module === self
+                          self
+                        else
+                          self.class
+                        end
+          message     = if strip > 0
+                          format("[DEPRECATED] %s\ncalled on: %s", message, caller_line)
+                        else
+                          format('[DEPRECATED] %s', message)
+                        end
+
+          # lets use our logger
+          logger = Cabin::Channel.get(LogStash)
+          logger.debug(message, :class => klass.to_s)
+        end
+
+        extend self
+      end
+    end
+  end
+else
+  # This is added a guard to check if we need to update this code or not.
+  # Keep in mind, the latest releases of concurrent-ruby brokes a few stuff.
+  #
+  # Even the latest master version changed how they handle deprecation.
+  raise "Logstash expects concurrent-ruby version 0.9.1 and version #{Concurrent::VERSION} is installed, please verify this patch: #{__FILE__}"
+end