logstash-lib 1.3.2

data/lib/logstash/inputs/gelf.rb

@@ -0,0 +1,138 @@
+# encoding: utf-8
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Read gelf messages as events over the network.
+#
+# This input is a good choice if you already use graylog2 today.
+#
+# The main reasoning for this input is to leverage existing GELF
+# logging libraries such as the gelf log4j appender
+#
+class LogStash::Inputs::Gelf < LogStash::Inputs::Base
+  config_name "gelf"
+  milestone 2
+
+  default :codec, "plain"
+
+  # The address to listen on
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on. Remember that ports less than 1024 (privileged
+  # ports) may require root to use.
+  config :port, :validate => :number, :default => 12201
+
+  # Whether or not to remap the gelf message fields to logstash event fields or
+  # leave them intact.
+  #
+  # Default is true
+  #
+  # Remapping converts the following gelf fields to logstash equivalents:
+  #
+  # * event["message"] becomes full_message
+  #   if no full_message, use event["message"] becomes short_message
+  #   if no short_message, event["message"] is the raw json input
+  config :remap, :validate => :boolean, :default => true
+
+  # Whether or not to remove the leading '_' in GELF fields or leave them
+  # in place. (Logstash < 1.2 did not remove them by default.)
+  #
+  # _foo becomes foo
+  #
+  # Default is true
+  config :strip_leading_underscore, :validate => :boolean, :default => true
+
+  public
+  def initialize(params)
+    super
+    BasicSocket.do_not_reverse_lookup = true
+  end # def initialize
+
+  public
+  def register
+    require 'gelfd'
+    @udp = nil
+  end # def register
+
+  public
+  def run(output_queue)
+    begin
+      # udp server
+      udp_listener(output_queue)
+    rescue => e
+      @logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
+      sleep(5)
+      retry
+    end # begin
+  end # def run
+
+  private
+  def udp_listener(output_queue)
+    @logger.info("Starting gelf listener", :address => "#{@host}:#{@port}")
+
+    if @udp 
+      @udp.close_read rescue nil
+      @udp.close_write rescue nil
+    end
+
+    @udp = UDPSocket.new(Socket::AF_INET)
+    @udp.bind(@host, @port)
+
+    while true
+      line, client = @udp.recvfrom(8192)
+      begin
+        data = Gelfd::Parser.parse(line)
+      rescue => ex
+        @logger.warn("Gelfd failed to parse a message skipping", :exception => ex, :backtrace => ex.backtrace)
+        next
+      end
+      
+      # Gelfd parser outputs null if it received and cached a non-final chunk
+      next if data.nil?    
+ 
+      event = LogStash::Event.new(JSON.parse(data))
+      event["source_host"] = client[3]
+      if event["timestamp"].is_a?(Numeric)
+        event["@timestamp"] = Time.at(event["timestamp"]).gmtime
+        event.remove("timestamp")
+      end
+      remap_gelf(event) if @remap
+      strip_leading_underscore(event) if @strip_leading_underscore
+      decorate(event)
+      output_queue << event
+    end
+  rescue LogStash::ShutdownSignal
+    # Do nothing, shutdown.
+  ensure
+    if @udp
+      @udp.close_read rescue nil
+      @udp.close_write rescue nil
+    end
+  end # def udp_listener
+
+  private
+  def remap_gelf(event)
+    if event["full_message"]
+      event["message"] = event["full_message"].dup
+      event.remove("full_message")
+      if event["short_message"] == event["message"]
+        event.remove("short_message")
+      end
+    elsif event["short_message"]
+      event["message"] = event["short_message"].dup
+      event.remove("short_message")
+    end
+  end # def remap_gelf
+
+  private
+  def strip_leading_underscore(event)
+     # Map all '_foo' fields to simply 'foo'
+     event.to_hash.keys.each do |key|
+       next unless key[0,1] == "_"
+       event[key[1..-1]] = event[key]
+       event.remove(key)
+     end
+  end # deef removing_leading_underscores
+end # class LogStash::Inputs::Gelf

data/lib/logstash/inputs/gemfire.rb

@@ -0,0 +1,222 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+
+# Push events to a GemFire region.
+#
+# GemFire is an object database.
+#
+# To use this plugin you need to add gemfire.jar to your CLASSPATH.
+# Using format=json requires jackson.jar too; use of continuous
+# queries requires antlr.jar.
+#
+# Note: this plugin has only been tested with GemFire 7.0.
+#
+class LogStash::Inputs::Gemfire < LogStash::Inputs::Threadable
+
+  config_name "gemfire"
+  milestone 1
+
+  default :codec, "plain"
+
+  # Your client cache name
+  config :cache_name, :validate => :string, :default => "logstash"
+
+  # The path to a GemFire client cache XML file.
+  #
+  # Example:
+  #
+  #      <client-cache>
+  #        <pool name="client-pool" subscription-enabled="true" subscription-redundancy="1">
+  #            <locator host="localhost" port="31331"/>
+  #        </pool>
+  #        <region name="Logstash">
+  #            <region-attributes refid="CACHING_PROXY" pool-name="client-pool" >
+  #            </region-attributes>
+  #        </region>
+  #      </client-cache>
+  #
+  config :cache_xml_file, :validate => :string, :default => nil
+
+  # The region name
+  config :region_name, :validate => :string, :default => "Logstash"
+
+  # A regexp to use when registering interest for cache events.
+  # Ignored if a :query is specified.
+  config :interest_regexp, :validate => :string, :default => ".*"
+
+  # A query to run as a GemFire "continuous query"; if specified it takes
+  # precedence over :interest_regexp which will be ignore.
+  #
+  # Important: use of continuous queries requires subscriptions to be enabled on the client pool.
+  config :query, :validate => :string, :default => nil
+
+  # How the message is serialized in the cache. Can be one of "json" or "plain"; default is plain
+  config :serialization, :validate => :string, :default => nil
+
+  public
+  def register
+    import com.gemstone.gemfire.cache.AttributesMutator
+    import com.gemstone.gemfire.cache.InterestResultPolicy
+    import com.gemstone.gemfire.cache.client.ClientCacheFactory
+    import com.gemstone.gemfire.cache.client.ClientRegionShortcut
+    import com.gemstone.gemfire.cache.query.CqQuery
+    import com.gemstone.gemfire.cache.query.CqAttributes
+    import com.gemstone.gemfire.cache.query.CqAttributesFactory
+    import com.gemstone.gemfire.cache.query.QueryService
+    import com.gemstone.gemfire.cache.query.SelectResults
+    import com.gemstone.gemfire.pdx.JSONFormatter
+
+    @logger.info("Registering input", :plugin => self)
+  end # def register
+
+  def run(queue)
+    return if terminating?
+    connect
+
+    @logstash_queue = queue
+
+    if @query
+      continuous_query(@query)
+    else
+      register_interest(@interest_regexp)
+    end
+  end # def run
+
+  def teardown
+    @cache.close if @cache
+    @cache = nil
+    finished
+  end # def teardown
+
+  protected
+  def connect
+    begin
+      @logger.debug("Connecting to GemFire #{@cache_name}")
+
+      @cache = ClientCacheFactory.new.
+        set("name", @cache_name).
+        set("cache-xml-file", @cache_xml_file).create
+      @logger.debug("Created cache #{@cache.inspect}")
+
+    rescue => e
+      if terminating?
+        return
+      else
+        @logger.error("Gemfire connection error (during connect), will reconnect",
+                      :exception => e, :backtrace => e.backtrace)
+        sleep(1)
+        retry
+      end
+    end
+
+    @region = @cache.getRegion(@region_name);
+    @logger.debug("Created region #{@region.inspect}")
+  end # def connect
+
+  protected
+  def continuous_query(query)
+    qs = @cache.getQueryService
+
+    cqAf = CqAttributesFactory.new
+    cqAf.addCqListener(self)
+    cqa = cqAf.create
+
+    @logger.debug("Running continuous query #{query}")
+    cq = qs.newCq("logstashCQ" + self.object_id.to_s, query, cqa)
+
+    cq.executeWithInitialResults
+  end
+
+  def register_interest(interest)
+    @region.getAttributesMutator.addCacheListener(self)
+    @region.registerInterestRegex(interest, InterestResultPolicy::NONE, false, true)
+  end
+
+  def deserialize_message(message)
+    if @serialization == "json"
+      message ? JSONFormatter.toJSON(message) : "{}"
+    else
+      message
+    end
+  end
+
+  def process_event(event, event_name)
+    message = deserialize_message(event)
+    @codec.decode(message) do |event|
+      decorate(event)
+      @logstash_queue << event
+    end
+  end
+
+  # multiple interfaces
+  def close
+  end
+
+  #
+  # CqListener interface
+  #
+  def onEvent(event)
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("onEvent #{event.getQueryOperation} #{key} #{newValue}")
+
+    process_event(event.getNewValue, "onEvent", "gemfire://query/#{key}/#{event.getQueryOperation}")
+  end
+
+  def onError(event)
+    @logger.debug("onError #{event}")
+  end
+
+  #
+  # CacheListener interface
+  #
+  protected
+  def afterCreate(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("afterCreate #{regionName} #{key} #{newValue}")
+
+    process_event(event.getNewValue, "afterCreate", "gemfire://#{regionName}/#{key}/afterCreate")
+  end
+
+  def afterDestroy(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    newValue = event.getNewValue
+    @logger.debug("afterDestroy #{regionName} #{key} #{newValue}")
+
+    process_event(nil, "afterDestroy", "gemfire://#{regionName}/#{key}/afterDestroy")
+  end
+
+  def afterUpdate(event)
+    regionName = event.getRegion.getName
+    key = event.getKey
+    oldValue = event.getOldValue
+    newValue = event.getNewValue
+    @logger.debug("afterUpdate #{regionName} #{key} #{oldValue} -> #{newValue}")
+
+    process_event(event.getNewValue, "afterUpdate", "gemfire://#{regionName}/#{key}/afterUpdate")
+  end
+
+  def afterRegionLive(event)
+    @logger.debug("afterRegionLive #{event}")
+  end
+
+  def afterRegionCreate(event)
+    @logger.debug("afterRegionCreate #{event}")
+  end
+
+  def afterRegionClear(event)
+    @logger.debug("afterRegionClear #{event}")
+  end
+
+  def afterRegionDestroy(event)
+    @logger.debug("afterRegionDestroy #{event}")
+  end
+
+  def afterRegionInvalidate(event)
+    @logger.debug("afterRegionInvalidate #{event}")
+  end
+end # class LogStash::Inputs::Gemfire

data/lib/logstash/inputs/generator.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+# Generate random log events.
+#
+# The general intention of this is to test performance of plugins.
+#
+# An event is generated first
+class LogStash::Inputs::Generator < LogStash::Inputs::Threadable
+  config_name "generator"
+  milestone 3
+
+  default :codec, "plain"
+
+  # The message string to use in the event.
+  #
+  # If you set this to 'stdin' then this plugin will read a single line from
+  # stdin and use that as the message string for every event.
+  #
+  # Otherwise, this value will be used verbatim as the event message.
+  config :message, :validate => :string, :default => "Hello world!"
+
+  # The lines to emit, in order. This option cannot be used with the 'message'
+  # setting.
+  #
+  # Example:
+  #
+  #     input {
+  #       generator {
+  #         lines => [
+  #           "line 1",
+  #           "line 2",
+  #           "line 3"
+  #         ]
+  #       }
+  #
+  #       # Emit all lines 3 times.
+  #       count => 3
+  #     }
+  #
+  # The above will emit "line 1" then "line 2" then "line", then "line 1", etc... 
+  config :lines, :validate => :array
+
+  # Set how many messages should be generated.
+  #
+  # The default, 0, means generate an unlimited number of events.
+  config :count, :validate => :number, :default => 0
+
+  public
+  def register
+    @host = Socket.gethostname
+    @count = @count.first if @count.is_a?(Array)
+    @lines = [@message] if @lines.nil?
+  end # def register
+
+  def run(queue)
+    number = 0
+
+    if @message == "stdin"
+      @logger.info("Generator plugin reading a line from stdin")
+      @message = $stdin.readline
+      @logger.debug("Generator line read complete", :message => @message)
+    end
+
+    while !finished? && (@count <= 0 || number < @count)
+      @lines.each do |line|
+        @codec.decode(line.clone) do |event|
+          decorate(event)
+          event["host"] = @host
+          event["sequence"] = number
+          queue << event
+        end
+      end
+      number += 1
+    end # loop
+
+    if @codec.respond_to?(:flush)
+      @codec.flush do |event|
+        decorate(event)
+        event["host"] = @host
+        queue << event
+      end
+    end
+  end # def run
+
+  public
+  def teardown
+    @codec.flush do |event|
+      decorate(event)
+      event["host"] = @host
+      queue << event
+    end
+    finished
+  end # def teardown
+end # class LogStash::Inputs::Generator