logstash-core 2.2.4.snapshot1

data/lib/logstash/config/registry.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+# Global config registry.
+module LogStash::Config::Registry
+  @registry = Hash.new
+  class << self
+    attr_accessor :registry
+
+    # TODO(sissel): Add some helper methods here.
+  end
+end # module LogStash::Config::Registry
+  

data/lib/logstash/environment.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+require "logstash/errors"
+
+module LogStash
+  module Environment
+    extend self
+
+    LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
+    LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
+
+    def env
+      LOGSTASH_ENV
+    end
+
+    def production?
+      env.downcase == "production"
+    end
+
+    def development?
+      env.downcase == "development"
+    end
+
+    def test?
+      env.downcase == "test"
+    end
+
+    def runtime_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "runtime-jars")
+    end
+
+    def test_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "test-jars")
+    end
+
+    def load_runtime_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(runtime_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_test_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(test_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_jars!(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.jruby-required")) unless LogStash::Environment.jruby?
+
+      jar_files = find_jars(pattern)
+      require_jars! jar_files
+    end
+
+    def find_jars(pattern)
+      require 'java'
+      jar_files = Dir.glob(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.missing-jars", :pattern => pattern)) if jar_files.empty?
+      jar_files
+    end
+
+    def require_jars!(files)
+      files.each do |jar_file|
+        loaded = require jar_file
+        puts("Loaded #{jar_file}") if $DEBUG && loaded
+      end
+    end
+
+    def ruby_bin
+      ENV["USE_RUBY"] == "1" ? "ruby" : File.join("vendor", "jruby", "bin", "jruby")
+    end
+
+    def jruby?
+      @jruby ||= !!(RUBY_PLATFORM == "java")
+    end
+
+    def windows?
+      ::Gem.win_platform?
+    end
+
+    def locales_path(path)
+      return ::File.join(LOGSTASH_CORE, "locales", path)
+    end
+
+    def load_locale!
+      require "i18n"
+      I18n.enforce_available_locales = true
+      I18n.load_path << LogStash::Environment.locales_path("en.yml")
+      I18n.reload!
+      fail "No locale? This is a bug." if I18n.available_locales.empty?
+    end
+
+    # add path for bare/ungemified plugins lookups. the path must be the base path that will include
+    # the dir structure 'logstash/TYPE/NAME.rb' where TYPE is 'inputs' 'filters', 'outputs' or 'codecs'
+    # and NAME is the name of the plugin
+    # @param path [String] plugins path to add
+    def add_plugin_path(path)
+      $LOAD_PATH << path
+    end
+  end
+end
+
+require "logstash/patches"

data/lib/logstash/errors.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+module LogStash
+  class Error < ::StandardError; end
+  class EnvironmentError < Error; end
+  class ConfigurationError < Error; end
+  class PluginLoadingError < Error; end
+  class ShutdownSignal < StandardError; end
+  class PluginNoVersionError < Error; end
+
+  class Bug < Error; end
+  class ThisMethodWasRemoved < Bug; end
+end

data/lib/logstash/filters/base.rb

@@ -0,0 +1,205 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/config/mixin"
+require "logstash/util/decorators"
+
+class LogStash::Filters::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "filter"
+
+  config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
+
+  # If this filter is successful, add arbitrary tags to the event.
+  # Tags can be dynamic and include parts of the event using the `%{field}`
+  # syntax.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also add multiple tags at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "foo_%{somefield}", "taggedy_tag"]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would add a tag `foo_hello` (and the second example would of course add a `taggedy_tag` tag).
+  config :add_tag, :validate => :array, :default => []
+
+  # If this filter is successful, remove arbitrary tags from the event.
+  # Tags can be dynamic and include parts of the event using the `%{field}`
+  # syntax.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also remove multiple tags at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_tag => [ "foo_%{somefield}", "sad_unwanted_tag"]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would remove the tag `foo_hello` if it is present. The second example
+  # would remove a sad, unwanted tag as well.
+  config :remove_tag, :validate => :array, :default => []
+
+  # If this filter is successful, add any arbitrary fields to this event.
+  # Field names can be dynamic and include parts of the event using the `%{field}`.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         add_field => { "foo_%{somefield}" => "Hello world, from %{host}" }
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also add multiple fields at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         add_field => {
+  #           "foo_%{somefield}" => "Hello world, from %{host}"
+  #           "new_field" => "new_static_value"
+  #         }
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would add field `foo_hello` if it is present, with the
+  # value above and the `%{host}` piece replaced with that value from the
+  # event. The second example would also add a hardcoded field.
+  config :add_field, :validate => :hash, :default => {}
+
+  # If this filter is successful, remove arbitrary fields from this event.
+  # Fields names can be dynamic and include parts of the event using the %{field}
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_field => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also remove multiple fields at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_field => [ "foo_%{somefield}", "my_extraneous_field" ]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would remove the field with name `foo_hello` if it is present. The second
+  # example would remove an additional, non-dynamic field.
+  config :remove_field, :validate => :array, :default => []
+
+  # Call the filter flush method at regular interval.
+  # Optional.
+  config :periodic_flush, :validate => :boolean, :default => false
+
+  public
+  def initialize(params)
+    super
+    config_init(@params)
+    @threadsafe = true
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def filter(event)
+    raise "#{self.class}#filter must be overidden"
+  end # def filter
+
+  # in 1.5.0 multi_filter is meant to be used in the generated filter function in LogStash::Config::AST::Plugin only
+  # and is temporary until we refactor the filter method interface to accept events list and return events list,
+  # just list in multi_filter see https://github.com/elastic/logstash/issues/2872.
+  # refactoring the filter method will mean updating all plugins which we want to avoid doing for 1.5.0.
+  #
+  # @param events [Array<LogStash::Event] list of events to filter
+  # @return [Array<LogStash::Event] filtered events and any new events generated by the filter
+  public
+  def multi_filter(events)
+    LogStash::Util.set_thread_plugin(self)
+    result = []
+    events.each do |event|
+      unless event.cancelled?
+        result << event
+        filter(event){|new_event| result << new_event}
+      end
+    end
+    result
+  end
+
+  public
+  def execute(event, &block)
+    filter(event, &block)
+  end # def execute
+
+  public
+  def threadsafe?
+    @threadsafe
+  end
+
+  # a filter instance should call filter_matched from filter if the event
+  # matches the filter's conditions (right type, etc)
+  protected
+  def filter_matched(event)
+    LogStash::Util::Decorators.add_fields(@add_field,event,"filters/#{self.class.name}")
+
+    @remove_field.each do |field|
+      field = event.sprintf(field)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing field",
+                                       :field => field)
+      event.remove(field)
+    end
+
+    LogStash::Util::Decorators.add_tags(@add_tag,event,"filters/#{self.class.name}")
+
+    # note below that the tags array field needs to be updated then reassigned to the event.
+    # this is important because a construct like event["tags"].delete(tag) will not work
+    # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
+    @remove_tag.each do |tag|
+      tags = event["tags"]
+      break if tags.nil? || tags.empty?
+      tag = event.sprintf(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
+      tags.delete(tag)
+      event["tags"] = tags
+    end
+  end # def filter_matched
+
+  protected
+  def filter?(event)
+    # TODO: noop for now, remove this once we delete this call from all plugins
+    true
+  end # def filter?
+
+  public
+  def close
+    # Nothing to do by default.
+  end
+end # class LogStash::Filters::Base

data/lib/logstash/inputs/base.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/plugin"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "logstash/codecs/base"
+require "logstash/util/decorators"
+
+# This is the base class for Logstash inputs.
+class LogStash::Inputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+  config_name "input"
+
+  # Add a `type` field to all events handled by this input.
+  #
+  # Types are used mainly for filter activation.
+  #
+  # The type is stored as part of the event itself, so you can
+  # also use the type to search for it in Kibana.
+  #
+  # If you try to set a type on an event that already has one (for
+  # example when you send an event from a shipper to an indexer) then
+  # a new input will not override the existing type. A type set at
+  # the shipper stays with that event for its life even
+  # when sent to another Logstash server.
+  config :type, :validate => :string
+
+  config :debug, :validate => :boolean, :default => false, :obsolete => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
+
+  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :obsolete => "You should use the newer 'codec' setting instead."
+
+  config :charset, :obsolete => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
+
+  config :message_format, :validate => :string, :obsolete => "Setting is no longer valid."
+
+  # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
+
+  # Add any number of arbitrary tags to your event.
+  #
+  # This can help with processing later.
+  config :tags, :validate => :array
+
+  # Add a field to an event
+  config :add_field, :validate => :hash, :default => {}
+
+  attr_accessor :params
+  attr_accessor :threadable
+
+  public
+  def initialize(params={})
+    super
+    @threadable = false
+    @stop_called = Concurrent::AtomicBoolean.new(false)
+    config_init(@params)
+    @tags ||= []
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def tag(newtag)
+    @tags << newtag
+  end # def tag
+
+  public
+  # override stop if you need to do more than do_stop to
+  # enforce the input plugin to return from `run`.
+  # e.g. a tcp plugin might need to close the tcp socket
+  # so blocking read operation aborts
+  def stop
+    # override if necessary
+  end
+
+  public
+  def do_stop
+    @logger.debug("stopping", :plugin => self)
+    @stop_called.make_true
+    stop
+  end
+
+  # stop? should never be overriden
+  public
+  def stop?
+    @stop_called.value
+  end
+
+  protected
+  def decorate(event)
+    # Only set 'type' if not already set. This is backwards-compatible behavior
+    event["type"] = @type if @type && !event.include?("type")
+
+    LogStash::Util::Decorators.add_fields(@add_field,event,"inputs/#{self.class.name}")
+    LogStash::Util::Decorators.add_tags(@tags,event,"inputs/#{self.class.name}")
+  end
+
+  protected
+  def fix_streaming_codecs
+    require "logstash/codecs/plain"
+    require "logstash/codecs/line"
+    require "logstash/codecs/json"
+    require "logstash/codecs/json_lines"
+    case @codec
+      when LogStash::Codecs::Plain
+        @logger.info("Automatically switching from #{@codec.class.config_name} to line codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::Line.new("charset" => @codec.charset)
+      when LogStash::Codecs::JSON
+        @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
+    end
+  end
+end # class LogStash::Inputs::Base

data/lib/logstash/inputs/threadable.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/inputs/base"
+
+# This is the threadable class for logstash inputs. 
+# Use this class in your inputs if it can support multiple threads
+class LogStash::Inputs::Threadable < LogStash::Inputs::Base
+
+  # Set this to the number of threads you want this input to spawn.
+  # This is the same as declaring the input multiple times
+  config :threads, :validate => :number, :default => 1
+ 
+  def initialize(params)
+    super
+    @threadable = true
+  end
+
+end # class LogStash::Inputs::Threadable