logstash-lib 1.3.2

data/lib/logstash/filters/uuid.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "securerandom"
+
+# The uuid filter allows you to add a UUID field to messages.
+# This is useful to be able to control the _id messages are indexed into Elasticsearch
+# with, so that you can insert duplicate messages (i.e. the same message multiple times
+# without creating duplicates) - for log pipeline reliability
+#
+class LogStash::Filters::Uuid < LogStash::Filters::Base
+  config_name "uuid"
+  milestone 2
+
+  # Add a UUID to a field.
+  #
+  # Example:
+  #
+  #     filter {
+  #       uuid {
+  #         target => "@uuid"
+  #       }
+  #     }
+  config :target, :validate => :string, :required => true
+
+  # If the value in the field currently (if any) should be overridden
+  # by the generated UUID. Defaults to false (i.e. if the field is
+  # present, with ANY value, it won't be overridden)
+  #
+  # Example:
+  #
+  #    filter {
+  #       uuid {
+  #         target    => "@uuid"
+  #         overwrite => true
+  #       }
+  #    }
+  config :overwrite, :validate => :boolean, :default => false
+
+  public
+  def register
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    if overwrite
+      event[target] = SecureRandom.uuid
+    else
+      event[target] ||= SecureRandom.uuid
+    end
+
+    filter_matched(event)
+  end # def filter
+
+end # class LogStash::Filters::Uuid
+

data/lib/logstash/filters/xml.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# XML filter. Takes a field that contains XML and expands it into
+# an actual datastructure.
+class LogStash::Filters::Xml < LogStash::Filters::Base
+
+  config_name "xml"
+  milestone 1
+
+  # Config for xml to hash is:
+  #
+  #     source => source_field
+  #
+  # For example, if you have the whole xml document in your @message field:
+  #
+  #     filter {
+  #       xml {
+  #         source => "message"
+  #       }
+  #     }
+  #
+  # The above would parse the xml from the @message field
+  config :source, :validate => :string
+
+  # Define target for placing the data
+  #
+  # for example if you want the data to be put in the 'doc' field:
+  #
+  #     filter {
+  #       xml {
+  #         target => "doc"
+  #       }
+  #     }
+  #
+  # XML in the value of the source field will be expanded into a
+  # datastructure in the "target" field.
+  # Note: if the "target" field already exists, it will be overridden
+  # Required
+  config :target, :validate => :string
+
+  # xpath will additionally select string values (.to_s on whatever is selected)
+  # from parsed XML (using each source field defined using the method above)
+  # and place those values in the destination fields. Configuration:
+  #
+  # xpath => [ "xpath-syntax", "destination-field" ]
+  #
+  # Values returned by XPath parsring from xpath-synatx will be put in the
+  # destination field. Multiple values returned will be pushed onto the
+  # destination field as an array. As such, multiple matches across
+  # multiple source fields will produce duplicate entries in the field
+  #
+  # More on xpath: http://www.w3schools.com/xpath/
+  #
+  # The xpath functions are particularly powerful:
+  # http://www.w3schools.com/xpath/xpath_functions.asp
+  #
+  config :xpath, :validate => :hash, :default => {}
+
+  # By default the filter will store the whole parsed xml in the destination
+  # field as described above. Setting this to false will prevent that.
+  config :store_xml, :validate => :boolean, :default => true
+
+  public
+  def register
+    require "nokogiri"
+    require "xmlsimple"
+
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    matched = false
+
+    @logger.debug("Running xml filter", :event => event)
+
+    return unless event.include?(@source)
+
+    value = event[@source]
+
+    if value.is_a?(Array) && value.length > 1
+      @logger.warn("XML filter only works on fields of length 1",
+                   :source => @source, :value => value)
+      return
+    end
+
+    # Do nothing with an empty string.
+    return if value.strip.length == 0
+
+    if @xpath
+      begin
+        doc = Nokogiri::XML(value)
+      rescue => e
+        event.tag("_xmlparsefailure")
+        @logger.warn("Trouble parsing xml", :source => @source, :value => value,
+                     :exception => e, :backtrace => e.backtrace)
+        return
+      end
+
+      @xpath.each do |xpath_src, xpath_dest|
+        nodeset = doc.xpath(xpath_src)
+
+        # If asking xpath for a String, like "name(/*)", we get back a
+        # String instead of a NodeSet.  We normalize that here.
+        normalized_nodeset = nodeset.kind_of?(Nokogiri::XML::NodeSet) ? nodeset : [nodeset]
+
+        normalized_nodeset.each do |value|
+          # some XPath functions return empty arrays as string
+          if value.is_a?(Array)
+            return if value.length == 0
+          end
+
+          unless value.nil?
+            matched = true
+            event[xpath_dest] ||= []
+            event[xpath_dest] << value.to_s
+          end
+        end # XPath.each
+      end # @xpath.each
+    end # if @xpath
+
+    if @store_xml
+      begin
+        event[@target] = XmlSimple.xml_in(value)
+        matched = true
+      rescue => e
+        event.tag("_xmlparsefailure")
+        @logger.warn("Trouble parsing xml with XmlSimple", :source => @source,
+                     :value => value, :exception => e, :backtrace => e.backtrace)
+        return
+      end
+    end # if @store_xml
+
+    filter_matched(event) if matched
+    @logger.debug("Event after xml filter", :event => event)
+  end # def filter
+end # class LogStash::Filters::Xml

data/lib/logstash/filters/zeromq.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# ZeroMQ filter. This is the best way to send an event externally for filtering
+# It works much like an exec filter would by sending the event "offsite"
+# for processing and waiting for a response
+#
+# The protocol here is:
+#   * REQ sent with JSON-serialized logstash event
+#   * REP read expected to be the full JSON 'filtered' event
+#   * - if reply read is an empty string, it will cancel the event.
+#
+# Note that this is a limited subset of the zeromq functionality in
+# inputs and outputs. The only topology that makes sense here is:
+# REQ/REP. 
+class LogStash::Filters::ZeroMQ < LogStash::Filters::Base
+
+  config_name "zeromq"
+  milestone 1
+
+  # 0mq socket address to connect or bind
+  # Please note that inproc:// will not work with logstash
+  # as we use a context per thread
+  # By default, filters connect
+  config :address, :validate => :string, :default => "tcp://127.0.0.1:2121"
+
+  # The field to send off-site for processing
+  # If this is unset, the whole event will be sent
+  # TODO (lusis)
+  # Allow filtering multiple fields
+  config :field, :validate => :string
+
+  # 0mq mode
+  # server mode binds/listens
+  # client mode connects
+  config :mode, :validate => ["server", "client"], :default => "client"
+
+  
+  # 0mq socket options
+  # This exposes zmq_setsockopt
+  # for advanced tuning
+  # see http://api.zeromq.org/2-1:zmq-setsockopt for details
+  #
+  # This is where you would set values like:
+  # ZMQ::HWM - high water mark
+  # ZMQ::IDENTITY - named queues
+  # ZMQ::SWAP_SIZE - space for disk overflow
+  # ZMQ::SUBSCRIBE - topic filters for pubsub
+  #
+  # example: sockopt => ["ZMQ::HWM", 50, "ZMQ::IDENTITY", "my_named_queue"]
+  config :sockopt, :validate => :hash
+
+  public
+  def initialize(params)
+    super(params)
+
+    @threadsafe = false
+  end
+
+  public
+  def register
+    require "ffi-rzmq"
+    require "logstash/util/zeromq"
+    self.class.send(:include, LogStash::Util::ZeroMQ)
+
+    @zsocket = context.socket(ZMQ::REQ)
+
+    error_check(@zsocket.setsockopt(ZMQ::LINGER, 1),
+                "while setting ZMQ::LINGER == 1)")
+
+    if @sockopt
+      setopts(@zsocket, @sockopt)
+    end
+
+    setup(@zsocket, @address)
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    # TODO (lusis)
+    # Need to set a timeout on the socket
+    # If it never gets a reply, filtering stops cold
+    begin
+      if @field
+        @logger.debug("0mq: sending", :request => event[@field])
+        error_check(@zsocket.send_string(event[@field]), "in send_string")
+      else
+        @logger.debug("0mq: sending", :request => event.to_json)
+        error_check(@zsocket.send_string(event.to_json), "in send_string")
+      end
+      reply = ''
+      rc = @zsocket.recv_string(reply)
+      error_check(rc, "in recv_string")
+
+      # If we receive an empty reply, this is an indication that the filter
+      # wishes to cancel this event.
+      if reply.empty?
+        event.cancel
+        return
+      end
+      @logger.debug("0mq: receiving", :reply => reply)
+      if @field
+        event[@field] = event.sprintf(reply)
+        filter_matched(event)
+      else
+        reply = JSON.parse(reply)
+        event.overwrite(reply)
+      end
+      filter_matched(event)
+    rescue => e
+      @logger.warn("0mq filter exception", :address => @address, :exception => e, :backtrace => e.backtrace)
+    end
+  end # def filter
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+end # class LogStash::Filters::ZeroMQ

data/lib/logstash/filterworker.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/config/mixin"
+require "stud/interval"
+
+# TODO(sissel): Should this really be a 'plugin' ?
+class LogStash::FilterWorker < LogStash::Plugin
+  include Stud
+  attr_accessor :logger
+  attr_accessor :filters
+  attr_reader	:after_filter
+
+  Exceptions = [Exception]
+  Exceptions << java.lang.Exception if RUBY_ENGINE == "jruby"
+
+  def initialize(filters, input_queue, output_queue)
+    @filters = filters
+    @input_queue = input_queue
+    @output_queue = output_queue
+    @shutdown_requested = false
+  end # def initialize
+
+  #This block is called after each filter is done on an event. 
+  #The filtered event and filter class name is passed to the block.
+  #This could be used to add metrics in the future?
+  def after_filter(&block)
+    @after_filter = block
+  end
+
+  def run
+    # TODO(sissel): Run a flusher thread for each plugin requesting flushes
+    # > It seems reasonable that you could want a multiline filter to flush
+    #   after 5 seconds, but want a metrics filter to flush every 10 or 60.
+
+    # Set up the periodic flusher thread.
+    @flusher = Thread.new { interval(5) { flusher } }
+
+    while !@shutdown_requested && event = @input_queue.pop
+      if event == LogStash::SHUTDOWN
+        finished
+        @input_queue << LogStash::SHUTDOWN # for the next filter thread
+        return
+      end
+
+      filter(event)
+    end # while @input_queue.pop
+    finished
+  end # def run
+
+  def flusher
+    events = []
+    @filters.each do |filter|
+
+      # Filter any events generated so far in this flush.
+      events.each do |event|
+        # TODO(sissel): watchdog on flush filtration?
+        unless event.cancelled?
+          filter.filter(event)
+          @after_filter.call(event,filter) unless @after_filter.nil?
+        end
+      end
+
+      # TODO(sissel): watchdog on flushes?
+      if filter.respond_to?(:flush)
+        flushed = filter.flush 
+        events += flushed if !flushed.nil? && flushed.any?
+      end
+    end
+
+    events.each do |event|
+      @logger.debug? and @logger.debug("Pushing flushed events", :event => event)
+      @output_queue.push(event) unless event.cancelled?
+    end
+  end # def flusher
+
+  def teardown
+    @shutdown_requested = true
+  end
+
+  def filter(original_event)
+    # Make an 'events' array that filters can push onto if they
+    # need to generate additional events based on the current event.
+    # The 'split' filter does this, for example.
+    events = [original_event]
+
+    events.each do |event|
+      @filters.each do |filter|
+        # Filter can emit multiple events, like the 'split' event, so
+        # give the input queue to dump generated events into.
+
+        # TODO(sissel): This may require some refactoring later, I am not sure
+        # this is the best approach. The goal is to allow filters to modify
+        # the current event, but if necessary, create new events based on
+        # this event.
+        begin
+          update_watchdog(:event => event, :filter => filter)
+          filter.execute(event) do |newevent|
+            events << newevent
+          end
+        rescue *Exceptions => e
+          @logger.warn("Exception during filter", :event => event,
+                       :exception => $!, :backtrace => e.backtrace,
+                       :filter => filter)
+        ensure
+          clear_watchdog
+        end
+        if event.cancelled?
+          @logger.debug? and @logger.debug("Event cancelled", :event => event,
+                                           :filter => filter.class)
+          break
+        end
+        @after_filter.call(event,filter) unless @after_filter.nil?
+      end # @filters.each
+
+      @logger.debug? and @logger.debug("Event finished filtering", :event => event,
+                                       :thread => Thread.current[:name])
+      @output_queue.push(event) unless event.cancelled?
+    end # events.each
+  end # def filter
+end # class LogStash::FilterWorker

data/lib/logstash/inputs/base.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/plugin"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "logstash/codecs/base"
+
+# This is the base class for logstash inputs.
+class LogStash::Inputs::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+  config_name "input"
+
+  # Add a 'type' field to all events handled by this input.
+  #
+  # Types are used mainly for filter activation.
+  #
+  # If you create an input with type "foobar", then only filters
+  # which also have type "foobar" will act on them.
+  #
+  # The type is also stored as part of the event itself, so you
+  # can also use the type to search for in the web interface.
+  #
+  # If you try to set a type on an event that already has one (for
+  # example when you send an event from a shipper to an indexer) then
+  # a new input will not override the existing type. A type set at 
+  # the shipper stays with that event for its life even
+  # when sent to another LogStash server.
+  config :type, :validate => :string
+
+  # Set this to true to enable debugging on an input.
+  config :debug, :validate => :boolean, :default => false
+
+  # The format of input data (plain, json, json_event)
+  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :deprecated => "You should use the newer 'codec' setting instead."
+
+  # The codec used for input data
+  config :codec, :validate => :codec, :default => "plain"
+
+  # The character encoding used in this input. Examples include "UTF-8"
+  # and "cp1252"
+  #
+  # This setting is useful if your log files are in Latin-1 (aka cp1252)
+  # or in another character set other than UTF-8.
+  #
+  # This only affects "plain" format logs since json is UTF-8 already.
+  config :charset, :validate => ::Encoding.name_list, :deprecated => true
+
+  # If format is "json", an event sprintf string to build what
+  # the display @message should be given (defaults to the raw JSON).
+  # sprintf format strings look like %{fieldname}
+  #
+  # If format is "json_event", ALL fields except for @type
+  # are expected to be present. Not receiving all fields
+  # will cause unexpected results.
+  config :message_format, :validate => :string, :deprecated => true
+
+  # Add any number of arbitrary tags to your event.
+  #
+  # This can help with processing later.
+  config :tags, :validate => :array
+
+  # Add a field to an event
+  config :add_field, :validate => :hash, :default => {}
+
+  attr_accessor :params
+  attr_accessor :threadable
+
+  public
+  def initialize(params={})
+    super
+    @threadable = false
+    config_init(params)
+    @tags ||= []
+
+    if @charset && @codec.class.get_config.include?("charset")
+      # charset is deprecated on inputs, but provide backwards compatibility
+      # by copying the charset setting into the codec.
+
+      @logger.info("Copying input's charset setting into codec", :input => self, :codec => @codec)
+      charset = @charset
+      @codec.instance_eval { @charset = charset }
+    end
+
+    # Backwards compat for the 'format' setting
+    case @format
+      when "plain"; # do nothing
+      when "json"
+        @codec = LogStash::Plugin.lookup("codec", "json").new
+      when "json_event"
+        @codec = LogStash::Plugin.lookup("codec", "oldlogstashjson").new
+    end
+
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def tag(newtag)
+    @tags << newtag
+  end # def tag
+
+  protected
+  def to_event(raw, source) 
+    raise LogStash::ThisMethodWasRemoved("LogStash::Inputs::Base#to_event - you should use codecs now instead of to_event. Not sure what this means? Get help on logstash-users@googlegroups.com!")
+  end # def to_event
+
+  protected
+  def decorate(event)
+    # Only set 'type' if not already set. This is backwards-compatible behavior
+    event["type"] = @type if @type && !event.include?("type")
+
+    if @tags.any?
+      event["tags"] ||= []
+      event["tags"] += @tags
+    end
+
+    @add_field.each do |field, value|
+      event[field] = value
+    end
+  end
+end # class LogStash::Inputs::Base