logstash-lib 1.3.2

data/lib/logstash/config/registry.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "logstash/namespace"
+
+# Global config registry.
+module LogStash::Config::Registry
+  @registry = Hash.new
+  class << self
+    attr_accessor :registry
+
+    # TODO(sissel): Add some helper methods here.
+  end
+end # module LogStash::Config::Registry
+  

data/lib/logstash/config/test.conf

@@ -0,0 +1,18 @@
+input {
+  rabbitmq {
+    port => 12345 
+    tag => [ a, b, c ]
+  }
+
+  stomp {
+    port => 12345
+    tag => [ stomp ]
+  }
+}
+
+filter {
+  date { 
+    hello => world
+    hello => "Hello"
+  }
+}

data/lib/logstash/errors.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+module LogStash
+  class Error < ::StandardError; end
+  class ConfigurationError < Error; end
+  class PluginLoadingError < Error; end
+  class ShutdownSignal < StandardError; end
+
+  class Bug < Error; end
+  class ThisMethodWasRemoved < Bug; end
+end

data/lib/logstash/event.rb

@@ -0,0 +1,262 @@
+# encoding: utf-8
+require "json"
+require "time"
+require "date"
+require "logstash/namespace"
+require "logstash/util/fieldreference"
+require "logstash/time_addon"
+
+# Use a custom serialization for jsonifying Time objects.
+# TODO(sissel): Put this in a separate file.
+class Time
+  def to_json(*args)
+    return iso8601(3).to_json(*args)
+  end
+
+  def inspect
+    return to_json
+  end
+end
+
+# the logstash event object.
+#
+# An event is simply a tuple of (timestamp, data).
+# The 'timestamp' is an ISO8601 timestamp. Data is anything - any message,
+# context, references, etc that are relevant to this event.
+#
+# Internally, this is represented as a hash with only two guaranteed fields.
+#
+# * "@timestamp" - an ISO8601 timestamp representing the time the event
+#   occurred at.
+# * "@version" - the version of the schema. Currently "1"
+#
+# They are prefixed with an "@" symbol to avoid clashing with your
+# own custom fields. 
+#
+# When serialized, this is represented in JSON. For example:
+#
+#     {
+#       "@timestamp": "2013-02-09T20:39:26.234Z",
+#       "@version": "1",
+#       message: "hello world"
+#     }
+class LogStash::Event
+  class DeprecatedMethod < StandardError; end
+
+  CHAR_PLUS = "+"
+  TIMESTAMP = "@timestamp"
+  VERSION = "@version"
+  VERSION_ONE = "1"
+
+  public
+  def initialize(data={})
+    @cancelled = false
+
+    @data = data
+    data[VERSION] = VERSION_ONE if !@data.include?(VERSION)
+    if data.include?(TIMESTAMP) 
+      t = data[TIMESTAMP]
+      if t.is_a?(String)
+        data[TIMESTAMP] = LogStash::Time.parse_iso8601(t)
+      end
+    else
+      data[TIMESTAMP] = ::Time.now.utc
+    end
+  end # def initialize
+
+  public
+  def cancel
+    @cancelled = true
+  end # def cancel
+
+  public
+  def uncancel
+    @cancelled = false
+  end # def uncancel
+
+  public
+  def cancelled?
+    return @cancelled
+  end # def cancelled?
+
+  # Create a deep-ish copy of this event.
+  public
+  def clone
+    copy = {}
+    @data.each do |k,v|
+      # TODO(sissel): Recurse if this is a hash/array?
+      copy[k] = v.clone
+    end
+    return self.class.new(copy)
+  end # def clone
+
+  if RUBY_ENGINE == "jruby"
+    public
+    def to_s
+      return self.sprintf("%{+yyyy-MM-dd'T'HH:mm:ss.SSSZ} %{host} %{message}")
+    end # def to_s
+  else
+    public
+    def to_s
+      return self.sprintf("#{self["@timestamp"].iso8601} %{host} %{message}")
+    end # def to_s
+  end
+
+  public
+  def timestamp; return @data[TIMESTAMP]; end # def timestamp
+  def timestamp=(val); return @data[TIMESTAMP] = val; end # def timestamp=
+
+  def unix_timestamp
+    raise DeprecatedMethod
+  end # def unix_timestamp
+
+  def ruby_timestamp
+    raise DeprecatedMethod
+  end # def unix_timestamp
+  
+  # field-related access
+  public
+  def [](str)
+    if str[0,1] == CHAR_PLUS
+      # nothing?
+    else
+      return LogStash::Util::FieldReference.exec(str, @data)
+    end
+  end # def []
+  
+  public
+  def []=(str, value)
+    if str == TIMESTAMP && !value.is_a?(Time)
+      raise TypeError, "The field '@timestamp' must be a Time, not a #{value.class} (#{value})"
+    end
+
+    r = LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
+      obj[key] = value
+    end
+
+    # The assignment can fail if the given field reference (str) does not exist
+    # In this case, we'll want to set the value manually.
+    if r.nil?
+      # TODO(sissel): Implement this in LogStash::Util::FieldReference
+      if str[0,1] != "["
+        return @data[str] = value
+      end
+
+      # No existing element was found, so let's set one.
+      *parents, key = str.scan(/(?<=\[)[^\]]+(?=\])/)
+      obj = @data
+      parents.each do |p|
+        if obj.include?(p)
+          obj = obj[p]
+        else
+          obj[p] = {}
+          obj = obj[p]
+        end
+      end
+      obj[key] = value
+    end
+    return value
+  end # def []=
+
+  public
+  def fields
+    raise DeprecatedMethod
+  end
+  
+  public
+  def to_json(*args)
+    return @data.to_json(*args) 
+  end # def to_json
+
+  def to_hash
+    return @data
+  end # def to_hash
+
+  public
+  def overwrite(event)
+    @data = event.to_hash
+  end
+
+  public
+  def include?(key)
+    return !self[key].nil?
+  end # def include?
+
+  # Append an event to this one.
+  public
+  def append(event)
+    # non-destructively merge that event with ourselves.
+    LogStash::Util.hash_merge(@data, event.to_hash)
+  end # append
+
+  # Remove a field or field reference. Returns the value of that field when
+  # deleted
+  public
+  def remove(str)
+    return LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
+      next obj.delete(key)
+    end
+  end # def remove
+
+  # sprintf. This could use a better method name.
+  # The idea is to take an event and convert it to a string based on 
+  # any format values, delimited by %{foo} where 'foo' is a field or
+  # metadata member.
+  #
+  # For example, if the event has type == "foo" and source == "bar"
+  # then this string:
+  #   "type is %{type} and source is %{host}"
+  # will return
+  #   "type is foo and source is bar"
+  #
+  # If a %{name} value is an array, then we will join by ','
+  # If a %{name} value does not exist, then no substitution occurs.
+  #
+  # TODO(sissel): It is not clear what the value of a field that 
+  # is an array (or hash?) should be. Join by comma? Something else?
+  public
+  def sprintf(format)
+    format = format.to_s
+    if format.index("%").nil?
+      return format
+    end
+
+    return format.gsub(/%\{[^}]+\}/) do |tok|
+      # Take the inside of the %{ ... }
+      key = tok[2 ... -1]
+
+      if key == "+%s"
+        # Got %{+%s}, support for unix epoch time
+        next @data["@timestamp"].to_i
+      elsif key[0,1] == "+"
+        t = @data["@timestamp"]
+        formatter = org.joda.time.format.DateTimeFormat.forPattern(key[1 .. -1])\
+          .withZone(org.joda.time.DateTimeZone::UTC)
+        #next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).toDateTime.toString(formatter)
+        # Invoke a specific Instant constructor to avoid this warning in JRuby
+        #  > ambiguous Java methods found, using org.joda.time.Instant(long)
+        org.joda.time.Instant.java_class.constructor(Java::long).new_instance(
+          t.tv_sec * 1000 + t.tv_usec / 1000
+        ).to_java.toDateTime.toString(formatter)
+      else
+        value = self[key]
+        case value
+          when nil
+            tok # leave the %{foo} if this field does not exist in this event.
+          when Array
+            value.join(",") # Join by ',' if value is an array
+          when Hash
+            value.to_json # Convert hashes to json
+          else
+            value # otherwise return the value
+        end # case value
+      end # 'key' checking
+    end # format.gsub...
+  end # def sprintf
+
+  def tag(value)
+    # Generalize this method for more usability
+    self["tags"] ||= []
+    self["tags"] << value unless self["tags"].include?(value)
+  end
+end # class LogStash::Event

data/lib/logstash/filters/advisor.rb

@@ -0,0 +1,178 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# INFORMATION:
+# The filter Advisor is designed for capture and confrontation the events. 
+# The events must be grep by a filter first, then it can pull out a copy of it, like clone, whit tags "advisor_first",
+# this copy is the first occurrence of this event verified in time_adv.
+# After time_adv Advisor will pull out an event tagged "advisor_info" who will tell you the number of same events verified in time_adv.
+
+# INFORMATION ABOUT CLASS:
+ 
+# For do this job, i used a thread that will sleep time adv. I assume that events coming on advisor are tagged, then i use an array for storing different events.
+# If an events is not present on array, then is the first and if the option is activate then  advisor push out a copy of event.
+# Else if the event is present on array, then is another same event and not the first, let's count it.  
+
+# USAGE:
+
+# This is an example of logstash config:
+
+# filter{
+#  advisor {
+#     time_adv => 1                     #(optional)
+#     send_first => true                #(optional)
+#  }
+# }
+
+# We analize this:
+
+# time_adv => 1
+# Means the time when the events matched and collected are pushed on outputs with tag "advisor_info".
+
+# send_first => true
+# Means you can push out the first events different who came in advisor like clone copy and tagged with "advisor_first"
+
+class LogStash::Filters::Advisor < LogStash::Filters::Base
+
+ config_name "advisor"
+ milestone 1
+
+ # If you do not set time_adv the plugin does nothing.
+ config :time_adv, :validate => :number, :default => 0
+ 
+ # If you want the first different event will be pushed out like a copy
+ config :send_first, :validate => :boolean, :default => true
+ 
+ public
+ def register
+
+  # Control the correct config
+  if (!(@time_adv == 0))
+    
+    @flag = false
+    @first = false
+    # Is used for store the different events.
+    @sarray = Array.new
+    # Is used for count the number of equals events.
+    @carray = Array.new
+
+    @thread = time_alert(@time_adv.to_i*60) do
+     # if collected any events then pushed out a new event after time_adv
+     if (@sarray.size !=0) 
+        @flag = true
+     end
+    end
+  
+  else
+   @logger.warn("Advisor: you have not specified Time_adv. This filter will do nothing!")
+  end
+
+ end
+ 
+ # This method is used to manage sleep and awaken threads (thanks StackOverflow for the support)
+  def time_alert(interval)
+     Thread.new do
+      loop do
+       start_time = Time.now
+       yield
+       elapsed = Time.now - start_time
+       sleep([interval - elapsed, 0].max)
+     end
+   end
+  end
+
+ public
+ def filter(event)
+  return unless filter?(event)
+  
+  # Control the correct config
+  if(!(@time_adv == 0))
+
+    new_event = true
+    @message = event["message"]
+    
+    # control if the events are new or they are came before
+    for i in (0..@sarray.size-1)
+      if (@message == @sarray[i].to_s)
+        @logger.debug("Avisor: Event match")
+        # if came before then count it
+        new_event = false
+        @carray[i] = @carray[i].to_i+1
+        @logger.debug("Advisor: "+@carray[i].to_s+" Events matched")
+        break
+      end
+    end
+     
+    if (new_event == true)
+       # else is a new event
+
+       @sarray << @message
+       @carray << 1
+       if (send_first == true)
+           @logger.debug("Advisor: is the first to send out")
+           @first = true
+       end
+    end
+     
+  else
+   @logger.warn("Advisor: you have not specified Time_adv. This filter will do nothing!")
+  end
+ end
+
+
+  # This method is used for generate events every 5 seconds (Thanks Jordan Sissel for explanation).
+  # In this case we generate an event when advisor thread trigger the flag or is the first different event. 
+
+  def flush
+      
+        if (@first == true)
+          event = LogStash::Event.new
+          event["host"] = Socket.gethostname
+          event["message"] = @message
+          event.tag "advisor_first"
+          filter_matched(event)
+         
+          @first = false
+          return [event]
+        end
+   
+         if (@flag == true)
+ 
+          if (@tags.size != 0)
+            @tag_path = ""
+            for i in (0..@tags.size-1)
+              @tag_path += @tags[i].to_s+"."
+            end
+          end
+            
+          # Prepare message 
+          message = "Advisor: Found events who match: "+@tag_path.to_s+"\n\n"
+
+          # See on messagge partial part of different events
+          for i in (0..@sarray.size-1)
+            message = message+@carray[i].to_s+" events like: "+(@sarray[i].to_s).slice(0, 300)+"\n\n"
+          end
+         
+          event = LogStash::Event.new
+          event["host"] = Socket.gethostname 
+          event["message"] = message  
+          event.tag << "advisor_info"
+          filter_matched(event)
+   
+          # reset flag and counter 
+          @flag = false
+          @carray = nil
+          @sarray = nil
+          @carray = Array.new
+          @sarray = Array.new
+
+          # push the event
+          return [event]
+         end
+    return
+ 
+  end
+
+end
+# By Bistic:)