logstash-core 1.5.0.beta2-java

data/lib/logstash/filters/base.rb

@@ -0,0 +1,239 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/config/mixin"
+
+class LogStash::Filters::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "filter"
+
+  # Note that all of the specified routing options (`type`,`tags`,`exclude_tags`,`include_fields`,
+  # `exclude_fields`) must be met in order for the event to be handled by the filter.
+
+  # The type to act on. If a type is given, then this filter will only
+  # act on messages with the same type. See any input plugin's `type`
+  # attribute for more.
+  # Optional.
+  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  # Only handle events with all of these tags.
+  # Optional.
+  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  # Only handle events without any of these tags.
+  # Optional.
+  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+
+  # If this filter is successful, add arbitrary tags to the event.
+  # Tags can be dynamic and include parts of the event using the `%{field}`
+  # syntax.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also add multiple tags at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "foo_%{somefield}", "taggedy_tag"]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would add a tag `foo_hello` (and the second example would of course add a `taggedy_tag` tag).
+  config :add_tag, :validate => :array, :default => []
+
+  # If this filter is successful, remove arbitrary tags from the event.
+  # Tags can be dynamic and include parts of the event using the `%{field}`
+  # syntax.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also remove multiple tags at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_tag => [ "foo_%{somefield}", "sad_unwanted_tag"]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would remove the tag `foo_hello` if it is present. The second example
+  # would remove a sad, unwanted tag as well.
+  config :remove_tag, :validate => :array, :default => []
+
+  # If this filter is successful, add any arbitrary fields to this event.
+  # Field names can be dynamic and include parts of the event using the `%{field}`.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         add_field => { "foo_%{somefield}" => "Hello world, from %{host}" }
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also add multiple fields at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         add_field => {
+  #           "foo_%{somefield}" => "Hello world, from %{host}"
+  #           "new_field" => "new_static_value"
+  #         }
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would add field `foo_hello` if it is present, with the
+  # value above and the `%{host}` piece replaced with that value from the
+  # event. The second example would also add a hardcoded field.
+  config :add_field, :validate => :hash, :default => {}
+
+  # If this filter is successful, remove arbitrary fields from this event.
+  # Fields names can be dynamic and include parts of the event using the %{field}
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_field => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  # [source,ruby]
+  #     # You can also remove multiple fields at once:
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_field => [ "foo_%{somefield}", "my_extraneous_field" ]
+  #       }
+  #     }
+  #
+  # If the event has field `"somefield" == "hello"` this filter, on success,
+  # would remove the field with name `foo_hello` if it is present. The second
+  # example would remove an additional, non-dynamic field.
+  config :remove_field, :validate => :array, :default => []
+
+  # Call the filter flush method at regular interval.
+  # Optional.
+  config :periodic_flush, :validate => :boolean, :default => false
+
+  RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field", "include_any", "exclude_any"]
+
+  public
+  def initialize(params)
+    super
+    config_init(params)
+    @threadsafe = true
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def filter(event)
+    raise "#{self.class}#filter must be overidden"
+  end # def filter
+
+  public
+  def execute(event, &block)
+    filter(event, &block)
+  end # def execute
+
+  public
+  def threadsafe?
+    @threadsafe
+  end
+
+  # a filter instance should call filter_matched from filter if the event
+  # matches the filter's conditions (right type, etc)
+  protected
+  def filter_matched(event)
+    @add_field.each do |field, value|
+      field = event.sprintf(field)
+      value = [value] if !value.is_a?(Array)
+      value.each do |v|
+        v = event.sprintf(v)
+        if event.include?(field)
+          event[field] = [event[field]] if !event[field].is_a?(Array)
+          event[field] << v
+        else
+          event[field] = v
+        end
+        @logger.debug? and @logger.debug("filters/#{self.class.name}: adding value to field",
+                                       :field => field, :value => value)
+      end
+    end
+
+    @remove_field.each do |field|
+      field = event.sprintf(field)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing field",
+                                       :field => field)
+      event.remove(field)
+    end
+
+    @add_tag.each do |tag|
+      tag = event.sprintf(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: adding tag",
+                                       :tag => tag)
+      (event["tags"] ||= []) << tag
+    end
+
+    @remove_tag.each do |tag|
+      break if event["tags"].nil?
+      tag = event.sprintf(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag",
+                                       :tag => tag)
+      event["tags"].delete(tag)
+    end
+  end # def filter_matched
+
+  protected
+  def filter?(event)
+    if !@type.empty?
+      if event["type"] != @type
+        @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because type doesn't match",
+                                         :type=> @type, :event => event)
+        return false
+      end
+    end
+
+    if !@tags.empty?
+      # this filter has only works on events with certain tags,
+      # and this event has no tags.
+      return false if !event["tags"]
+
+      # Is @tags a subset of the event's tags? If not, skip it.
+      if (event["tags"] & @tags).size != @tags.size
+        @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because tags don't match",
+                                         :tags => tags, :event => event)
+        return false
+      end
+    end
+
+    if !@exclude_tags.empty? && event["tags"]
+      if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
+        @logger.debug("filters/#{self.class.name}: Skipping event because tags contains excluded tags:",
+                      :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
+        return false
+      end
+    end
+
+    return true
+  end
+
+  public
+  def teardown
+    # Nothing to do by default.
+  end
+end # class LogStash::Filters::Base

data/lib/logstash/gemfile.rb

@@ -0,0 +1,175 @@
+module LogStash
+
+  class GemfileError < StandardError; end
+
+  class Gemfile
+    attr_accessor :gemset
+
+    HEADER = \
+      "# This is a Logstash generated Gemfile.\n" + \
+      "# If you modify this file manually all comments and formatting will be lost.\n\n"
+
+    # @params io [IO] any IO object that supports read, write, truncate, rewind
+    def initialize(io)
+      @io = io
+      @gemset = nil
+    end
+
+    def load
+      @gemset ||= DSL.parse(@io.read)
+      self
+    end
+
+    def save
+      raise(GemfileError, "a Gemfile must first be loaded") unless @gemset
+      @io.truncate(0)
+      @io.rewind
+      @io.write(HEADER)
+      @io.write(@gemset.to_s)
+      @io.flush
+    end
+
+    def find(name)
+      @gemset.find_gem(name)
+    end
+
+    # @param name [String] gem name
+    # @param *requirements params following name use the same notation as the Gemfile gem DSL statement
+    # @raise GemfileError if gem already exists in Gemfile
+    def add(name, *requirements)
+      @gemset.add_gem(Gem.parse(name, *requirements))
+    end
+
+    # update existing or add new
+    # @param name [String] gem name
+    # @param *requirements params following name use the same notation as the Gemfile gem DSL statement
+    def update(name, *requirements)
+      @gemset.update_gem(Gem.parse(name, *requirements))
+    end
+
+    # @return [Gem] removed gem or nil if not found
+    def remove(name)
+      @gemset.remove_gem(name)
+    end
+  end
+
+  class Gemset
+    attr_accessor :sources, :gems, :gemspec
+
+    def initialize
+      @sources = []      # list of urls
+      @gems = []         # list of Gem class
+      @gems_by_name = {} # hash of name => Gem
+      @gemspec = {}      # gemspec is a options hash
+    end
+
+    def to_s
+      [sources_to_s, gemspec_to_s, gems_to_s].select{|s| !s.empty?}.join("\n") + "\n"
+    end
+
+    # @return [Gem] found gem or nil if not found
+    def find_gem(name)
+      @gems_by_name[name.downcase]
+    end
+
+    # @raise GemfileError if gem already exists
+    def add_gem(_gem)
+      raise(GemfileError, "duplicate gem #{_gem.name}") if find_gem(_gem.name)
+      @gems << _gem
+      @gems_by_name[_gem.name.downcase] = _gem
+    end
+
+    # update existing or add new
+    def update_gem(_gem)
+      if old = find_gem(_gem.name)
+        @gems[@gems.index(old)] = _gem
+      else
+        @gems << _gem
+      end
+      @gems_by_name[_gem.name.downcase] = _gem
+    end
+
+    # @return [Gem] removed gem or nil if not found
+    def remove_gem(name)
+      if _gem = @gems_by_name.delete(name.downcase)
+        @gems.delete_at(@gems.index(_gem))
+      end
+      _gem
+    end
+
+    # deep clone self
+    def copy
+      Marshal.load(Marshal.dump(self))
+    end
+
+    private
+
+    def sources_to_s
+      return "" if @sources.empty?
+      @sources.map{|source| "source #{source.inspect}"}.join("\n")
+    end
+
+    def gems_to_s
+      return "" if @gems.empty?
+      @gems.map do |gem|
+        requirements = gem.requirements.empty? ? nil : gem.requirements.map{|r| r.inspect}.join(", ")
+        options = gem.options.empty? ? nil : gem.options.map{|k, v| "#{k.inspect} => #{v.inspect}"}.join(", ")
+        "gem " + [gem.name.inspect, requirements, options].compact.join(", ")
+      end.join("\n")
+    end
+
+    def gemspec_to_s
+      return "" if @gemspec.empty?
+      options = @gemspec.map{|k, v| "#{k.inspect} => #{v.inspect}"}.join(", ")
+      "gemspec #{options}"
+    end
+  end
+
+  # DSL is a minimal, incomplete Gemfile DSL subset parser, only what is currently required is implemented.
+  class DSL
+    attr_reader :gemset
+
+    def initialize
+      @gemset = Gemset.new
+    end
+
+    # @param gemfile_content [String] the Gemfile string content
+    # @return [Gemset] parsed Gemfile content as a Gemset
+    def self.parse(gemfile_content)
+      dsl = self.new
+      dsl.instance_eval(gemfile_content)
+      dsl.gemset
+    end
+
+    # DSL methods
+
+    def source(url)
+      @gemset.sources << url
+    end
+
+    def gem(name, *requirements)
+      parsed = Gem.parse(name, *requirements)
+      @gemset.add_gem(parsed)
+    end
+
+    def gemspec(options = {})
+      raise(GemfileError, "cannot declare multiple gemspec directives") unless @gemset.gemspec.empty?
+      @gemset.gemspec = options
+    end
+  end
+
+  class Gem
+    attr_accessor :name, :requirements, :options
+
+    def initialize(name, requirements = [], options = {})
+      @name = name
+      @requirements = requirements.map{|r| r.to_s.strip}.select{|r| !r.empty?}
+      @options = options
+    end
+
+    def self.parse(name, *requirements)
+      options = requirements.last.is_a?(Hash) ? requirements.pop : {}
+      self.new(name, requirements, options)
+    end
+  end
+end

data/lib/logstash/inputs/base.rb

@@ -0,0 +1,137 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/plugin"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "logstash/codecs/base"
+
+# This is the base class for Logstash inputs.
+class LogStash::Inputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+  config_name "input"
+
+  # Add a `type` field to all events handled by this input.
+  #
+  # Types are used mainly for filter activation.
+  #
+  # The type is stored as part of the event itself, so you can
+  # also use the type to search for it in Kibana.
+  #
+  # If you try to set a type on an event that already has one (for
+  # example when you send an event from a shipper to an indexer) then
+  # a new input will not override the existing type. A type set at
+  # the shipper stays with that event for its life even
+  # when sent to another Logstash server.
+  config :type, :validate => :string
+
+  config :debug, :validate => :boolean, :default => false, :deprecated => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
+
+  # The format of input data (plain, json, json_event)
+  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :deprecated => "You should use the newer 'codec' setting instead."
+
+  # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
+
+  # The character encoding used in this input. Examples include `UTF-8`
+  # and `cp1252`
+  #
+  # This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
+  # or in another character set other than `UTF-8`.
+  #
+  # This only affects `plain` format logs since json is `UTF-8` already.
+  config :charset, :validate => ::Encoding.name_list, :deprecated => true
+
+  # If format is `json`, an event `sprintf` string to build what
+  # the display `@message` should be given (defaults to the raw JSON).
+  # `sprintf` format strings look like `%{fieldname}`
+  #
+  # If format is `json_event`, ALL fields except for `@type`
+  # are expected to be present. Not receiving all fields
+  # will cause unexpected results.
+  config :message_format, :validate => :string, :deprecated => true
+
+  # Add any number of arbitrary tags to your event.
+  #
+  # This can help with processing later.
+  config :tags, :validate => :array
+
+  # Add a field to an event
+  config :add_field, :validate => :hash, :default => {}
+
+  attr_accessor :params
+  attr_accessor :threadable
+
+  public
+  def initialize(params={})
+    super
+    @threadable = false
+    config_init(params)
+    @tags ||= []
+
+    if @charset && @codec.class.get_config.include?("charset")
+      # charset is deprecated on inputs, but provide backwards compatibility
+      # by copying the charset setting into the codec.
+
+      @logger.info("Copying input's charset setting into codec", :input => self, :codec => @codec)
+      charset = @charset
+      @codec.instance_eval { @charset = charset }
+    end
+
+    # Backwards compat for the 'format' setting
+    case @format
+      when "plain"; # do nothing
+      when "json"
+        @codec = LogStash::Plugin.lookup("codec", "json").new
+      when "json_event"
+        @codec = LogStash::Plugin.lookup("codec", "oldlogstashjson").new
+    end
+
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def tag(newtag)
+    @tags << newtag
+  end # def tag
+
+  protected
+  def to_event(raw, source)
+    raise LogStash::ThisMethodWasRemoved("LogStash::Inputs::Base#to_event - you should use codecs now instead of to_event. Not sure what this means? Get help on logstash-users@googlegroups.com!")
+  end # def to_event
+
+  protected
+  def decorate(event)
+    # Only set 'type' if not already set. This is backwards-compatible behavior
+    event["type"] = @type if @type && !event.include?("type")
+
+    if @tags.any?
+      event["tags"] ||= []
+      event["tags"] += @tags
+    end
+
+    @add_field.each do |field, value|
+      event[field] = value
+    end
+  end
+
+  protected
+  def fix_streaming_codecs
+    require "logstash/codecs/plain"
+    require "logstash/codecs/line"
+    require "logstash/codecs/json"
+    require "logstash/codecs/json_lines"
+    case @codec
+      when LogStash::Codecs::Plain
+        @logger.info("Automatically switching from #{@codec.class.config_name} to line codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::Line.new("charset" => @codec.charset)
+      when LogStash::Codecs::JSON
+        @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
+    end
+  end
+end # class LogStash::Inputs::Base