logstash-lib 1.3.2

data/lib/logstash/filters/punct.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Strip everything but punctuation from a field and store the remainder in the
+# a separate field. This is often used for fingerprinting log events.
+class LogStash::Filters::Punct < LogStash::Filters::Base
+  config_name "punct"
+  milestone 1
+
+  # The field reference to use for punctuation stripping
+  config :source, :validate => :string, :default => "message"
+
+  # The field to store the result.
+  config :target, :validate => :string, :default => "punct"
+
+  public
+  def register
+    # Nothing to do
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    original_value = event[@source]
+
+    # If for some reason the field is an array of values, take the first only.
+    original_value = original_value.first if original_value.is_a?(Array)
+    event[@target] = original_value.tr('A-Za-z0-9 \t','')
+  end # def filter
+end # class LogStash::Filters::Punct

data/lib/logstash/filters/railsparallelrequest.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+# parallel request filter
+#
+# This filter will separate out the parallel requests into separate events.
+#
+
+require "logstash/filters/base"
+require "logstash/namespace"
+require "set"
+
+class LogStash::Filters::Railsparallelrequest < LogStash::Filters::Base
+
+  config_name "railsparallelrequest"
+  milestone 1
+
+  public
+  def initialize(config = {})
+    super
+    @threadsafe = false
+    @pending = Hash.new
+    @last_event = nil
+    @recently_error = nil
+    @last_uuid = nil
+  end
+
+  def register ;end
+
+  def filter(event)
+    return unless filter?(event)
+    return if event["tags"].include? self.class.config_name
+
+    event["tags"] << self.class.config_name
+
+    line = event["message"]
+
+    if line =~ /^\[(.*?)\]/
+      uuid = $1
+      event["uuid"] = uuid
+      if @recently_error
+        if @last_uuid == uuid
+          merge_events(@recently_error, event, uuid)
+          event.cancel
+          return
+        else
+          @recently_error.uncancel
+          yield @recently_error
+          @recently_error = nil
+        end
+      end
+
+      @last_uuid = uuid
+      if @pending[uuid]
+        merge_events(@pending[uuid], event, uuid)
+      else
+        @pending[uuid] = event
+      end
+      @last_event = @pending[uuid]
+
+      if line =~ /Error/
+        event.overwrite(@pending[uuid].to_hash)
+        @pending.delete uuid
+        @recently_error = event
+      elsif line =~ /Completed/
+        event.overwrite(@pending[uuid])
+        @pending.delete uuid
+        return
+      end
+      event.cancel
+    elsif @last_event
+      @last_event.append(event)
+      event.cancel
+    end
+  end
+
+  def flush
+    events = @pending.values.each { |event| event.uncancel }
+    @pending.clear
+    events
+  end
+
+  private
+  def merge_events(dest, source, uuid)
+    source["message"].gsub!("[#{uuid}]", "")
+    dest.append(source)
+  end
+end

data/lib/logstash/filters/range.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This filter is used to check that certain fields are within expected size/length ranges.
+# Supported types are numbers and strings.
+# Numbers are checked to be within numeric value range.
+# Strings are checked to be within string length range.
+# More than one range can be specified for same fieldname, actions will be applied incrementally.
+# When field value is within a specified range an action will be taken.
+# Supported actions are drop event, add tag, or add field with specified value.
+#
+# Example use cases are for histogram-like tagging of events
+# or for finding anomaly values in fields or too big events that should be dropped.
+
+class LogStash::Filters::Range < LogStash::Filters::Base
+  config_name "range"
+  milestone 1
+
+  # An array of field, min, max, action tuples.
+  # Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         ranges => [ "message", 0, 10, "tag:short",
+  #                     "message", 11, 100, "tag:medium",
+  #                     "message", 101, 1000, "tag:long",
+  #                     "message", 1001, 1e1000, "drop",
+  #                     "duration", 0, 100, "field:latency:fast",
+  #                     "duration", 101, 200, "field:latency:normal",
+  #                     "duration", 201, 1000, "field:latency:slow",
+  #                     "duration", 1001, 1e1000, "field:latency:outlier",
+  #                     "requests", 0, 10, "tag:too_few_%{host}_requests" ]
+  #       }
+  #     }
+  #
+  # Supported actions are drop tag or field with specified value.
+  # Added tag names and field names and field values can have %{dynamic} values.
+  #
+  # TODO(piavlo): The action syntax is ugly at the moment due to logstash grammar limitations - arrays grammar should support
+  # TODO(piavlo): simple not nested hashses as values in addition to numaric and string values to prettify the syntax.
+  config :ranges, :validate => :array, :default => []
+
+  # Negate the range match logic, events should be outsize of the specified range to match.
+  config :negate, :validate => :boolean, :default => false
+
+  public
+  def register
+    if @ranges.length % 4 != 0
+      raise "#{self.class.name}: ranges array should consist of 4 field tuples (field,min,max,action)"
+    end
+  
+    @range_tuples = {}
+
+    while !@ranges.empty?
+      fieldname, min, max, action = @ranges.shift(4)
+      
+      raise "#{self.class.name}: range field name value should be a string" if !fieldname.is_a?(String)
+      raise "#{self.class.name}: range min value should be a number" if !min.is_a?(Integer) and !min.is_a?(Float)
+      raise "#{self.class.name}: range max value should be a number" if !max.is_a?(Integer) and !max.is_a?(Float)
+      raise "#{self.class.name}: range action value should be a string" if !action.is_a?(String)
+      
+      action = action.split(':')
+      
+      case action.first
+      when "drop"
+        raise "#{self.class.name}: drop action does not accept any parameters" unless action.length == 1
+        action = { :name => :drop }
+      when "tag"
+        raise "#{self.class.name}: tag action accepts exactly one arg which is a tag name" unless action.length == 2
+        action = { :name => :add_tag, :tag => action.last }
+      when "field"
+        raise "#{self.class.name}: field action accepts exactly 2 args which are a field name and field value" unless action.length == 3
+        if action.last == action.last.to_i.to_s
+          value = action.last.to_i
+        elsif action.last == action.last.to_f.to_s
+          value = action.last.to_f
+        else
+          value = action.last
+        end
+        action = { :name => :add_field, :field => action[1], :value => value }
+      else
+        raise "#{self.class.name}: unsupported action #{action}"
+      end
+      
+      @range_tuples[fieldname] ||= []
+      @range_tuples[fieldname] << { :min => min, :max => max, :action => action }
+    end
+  end # def register
+
+ 
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @range_tuples.each_key do |fieldname|
+      if event.include?(fieldname)
+        @range_tuples[fieldname].each do |range|
+          matched = false
+        
+          field = event[fieldname]
+          case field
+          when Integer
+            matched = field.between?(range[:min], range[:max])
+          when Float
+            matched = field.between?(range[:min], range[:max])
+          when String
+            matched = field.length.between?(range[:min], range[:max])
+          else
+            @logger.warn("#{self.class.name}: action field value has unsupported type")
+          end
+
+          matched = !matched if @negate
+          next unless matched
+        
+          case range[:action][:name]
+          when :drop
+            @logger.debug? and @logger.debug("#{self.class.name}: dropping event due to range match", :event => event)
+            event.cancel
+            return
+          when :add_tag
+            @logger.debug? and @logger.debug("#{self.class.name}: adding tag due to range match",
+                                             :event => event, :tag => range[:action][:tag] )
+            event.tag(event.sprintf(range[:action][:tag]))
+          when :add_field
+            @logger.debug? and @logger.debug("#{self.class.name}: adding field due to range match",
+                                              :event => event, :field => range[:action][:field], :value => range[:action][:value])
+            new_field = event.sprintf(range[:action][:field])
+            if event[new_field]
+              event[new_field] = [event[new_field]] if !event[new_field].is_a?(Array)
+              event[new_field] << event.sprintf(range[:action][:value])
+            else
+              event[new_field] = range[:action][:value].is_a?(String) ? event.sprintf(range[:action][:value]) : range[:action][:value]
+            end
+          end
+        end
+      end
+    end
+    
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Range

data/lib/logstash/filters/ruby.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Execute ruby code.
+#
+# For example, to cancel 90% of events, you can do this:
+#
+#     filter {
+#       ruby {
+#         # Cancel 90% of events
+#         code => "event.cancel if rand <= 0.90"
+#       } 
+#     } 
+#
+class LogStash::Filters::Ruby < LogStash::Filters::Base
+  config_name "ruby"
+  milestone 1
+
+  # Any code to execute at logstash startup-time
+  config :init, :validate => :string
+
+  # The code to execute for every event.
+  # You will have an 'event' variable available that is the event itself.
+  config :code, :validate => :string, :required => true
+
+  public
+  def register
+    # TODO(sissel): Compile the ruby code
+    eval(@init, binding, "(ruby filter init)") if @init
+    eval("@codeblock = lambda { |event| #{@code} }", binding, "(ruby filter code)")
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @codeblock.call(event)
+
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Ruby

data/lib/logstash/filters/sleep.rb

@@ -0,0 +1,111 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Sleep a given amount of time. This will cause logstash
+# to stall for the given amount of time. This is useful
+# for rate limiting, etc.
+#
+class LogStash::Filters::Sleep < LogStash::Filters::Base
+  config_name "sleep"
+  milestone 1
+
+  # The length of time to sleep, in seconds, for every event.
+  #
+  # This can be a number (eg, 0.5), or a string (eg, "%{foo}") 
+  # The second form (string with a field value) is useful if
+  # you have an attribute of your event that you want to use
+  # to indicate the amount of time to sleep.
+  #
+  # Example:
+  #
+  #     filter {
+  #       sleep {
+  #         # Sleep 1 second for every event.
+  #         time => "1"
+  #       }
+  #     }
+  config :time, :validate => :string
+
+  # Sleep on every N'th. This option is ignored in replay mode.
+  #
+  # Example:
+  #
+  #     filter {
+  #       sleep {
+  #         time => "1"   # Sleep 1 second 
+  #         every => 10   # on every 10th event
+  #       }
+  #     }
+  config :every, :validate => :string, :default => 1
+
+  # Enable replay mode.
+  #
+  # Replay mode tries to sleep based on timestamps in each event.
+  #
+  # The amount of time to sleep is computed by subtracting the
+  # previous event's timestamp from the current event's timestamp.
+  # This helps you replay events in the same timeline as original.
+  #
+  # If you specify a `time` setting as well, this filter will
+  # use the `time` value as a speed modifier. For example,
+  # a `time` value of 2 will replay at double speed, while a
+  # value of 0.25 will replay at 1/4th speed.
+  #
+  # For example:
+  #
+  #     filter {
+  #       sleep {
+  #         time => 2
+  #         replay => true
+  #       }
+  #     }
+  #
+  # The above will sleep in such a way that it will perform
+  # replay 2-times faster than the original time speed.
+  config :replay, :validate => :boolean, :default => false
+
+  public
+  def register
+    if @replay && @time.nil?
+      # Default time multiplier is 1 when replay is set.
+      @time = 1
+    end
+    if @time.nil?
+      raise ArgumentError, "Missing required parameter 'time' for input/eventlog"
+    end
+    @count = 0
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    @count += 1
+
+    case @time
+      when Fixnum, Float; time = @time
+      when nil; # nothing
+      else; time = event.sprintf(@time).to_f
+    end
+
+    if @replay
+      clock = event["@timestamp"].to_f
+      if @last_clock
+        delay = clock - @last_clock
+        time = delay/time
+        if time > 0
+          @logger.debug? && @logger.debug("Sleeping", :delay => time)
+          sleep(time)
+        end
+      end
+      @last_clock = clock
+    else
+      if @count >= @every
+        @count = 0
+        @logger.debug? && @logger.debug("Sleeping", :delay => time)
+        sleep(time)
+      end
+    end
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Sleep

data/lib/logstash/filters/split.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The split filter is for splitting multiline messages into separate events.
+#
+# An example use case of this filter is for taking output from the 'exec' input
+# which emits one event for the whole output of a command and splitting that
+# output by newline - making each line an event.
+#
+# The end result of each split is a complete copy of the event 
+# with only the current split section of the given field changed.
+class LogStash::Filters::Split < LogStash::Filters::Base
+
+  config_name "split"
+  milestone 2
+
+  # The string to split on. This is usually a line terminator, but can be any
+  # string.
+  config :terminator, :validate => :string, :default => "\n"
+
+  # The field which value is split by the terminator
+  config :field, :validate => :string, :default => "message"
+
+  public
+  def register
+    # Nothing to do
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    events = []
+
+    original_value = event[@field]
+
+    # If for some reason the field is an array of values, take the first only.
+    original_value = original_value.first if original_value.is_a?(Array)
+
+    # Using -1 for 'limit' on String#split makes ruby not drop trailing empty
+    # splits.
+    splits = original_value.split(@terminator, -1)
+
+    # Skip filtering if splitting this event resulted in only one thing found.
+    return if splits.length == 1
+    #or splits[1].empty?
+
+    splits.each do |value|
+      next if value.empty?
+
+      event_split = event.clone
+      @logger.debug("Split event", :value => value, :field => @field)
+      event_split[@field] = value
+      filter_matched(event_split)
+
+      # Push this new event onto the stack at the LogStash::FilterWorker
+      yield event_split
+    end
+
+    # Cancel this event, we'll use the newly generated ones above.
+    event.cancel
+  end # def filter
+end # class LogStash::Filters::Split