logstash-lib 1.3.2

data/lib/logstash/filters/mutate.rb

@@ -0,0 +1,399 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The mutate filter allows you to do general mutations to fields. You
+# can rename, remove, replace, and modify fields in your events.
+#
+# TODO(sissel): Support regexp replacements like String#gsub ?
+class LogStash::Filters::Mutate < LogStash::Filters::Base
+  config_name "mutate"
+  milestone 3
+
+  # Rename one or more fields.
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate {
+  #         # Renames the 'HOSTORIP' field to 'client_ip'
+  #         rename => [ "HOSTORIP", "client_ip" ]
+  #       }
+  #     }
+  config :rename, :validate => :hash
+
+  # Remove one or more fields.
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate {
+  #         remove => [ "client" ]  # Removes the 'client' field
+  #       }
+  #     }
+  #
+  # This option is deprecated, instead use remove_field option available in all
+  # filters.
+  config :remove, :validate => :array, :deprecated => true
+
+  # Replace a field with a new value. The new value can include %{foo} strings
+  # to help you build a new value from other parts of the event.
+  #
+  # Example:
+  # 
+  #     filter {
+  #       mutate {
+  #         replace => [ "message", "%{source_host}: My new message" ]
+  #       }
+  #     }
+  config :replace, :validate => :hash
+
+  # Update an existing field with a new value. If the field does not exist,
+  # then no action will be taken.
+  #
+  # Example:
+  # 
+  #     filter {
+  #       mutate {
+  #         update => [ "sample", "My new message" ]
+  #       }
+  #     }
+  config :update, :validate => :hash
+
+  # Convert a field's value to a different type, like turning a string to an
+  # integer. If the field value is an array, all members will be converted.
+  # If the field is a hash, no action will be taken.
+  #
+  # Valid conversion targets are: integer, float, string
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate {
+  #         convert => [ "fieldname", "integer" ]
+  #       }
+  #     }
+  config :convert, :validate => :hash
+
+  # Convert a string field by applying a regular expression and a replacement
+  # if the field is not a string, no action will be taken.
+  # 
+  # This configuration takes an array consisting of 3 elements per
+  # field/substitution.
+  #
+  # Be aware of escaping any backslash in the config file.
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate {
+  #         gsub => [
+  #           # replace all forward slashes with underscore
+  #           "fieldname", "/", "_",
+  #
+  #           # replace backslashes, question marks, hashes, and minuses with
+  #           # dot
+  #           "fieldname2", "[\\?#-]", "."
+  #         ]
+  #       }
+  #     }
+  #
+  config :gsub, :validate => :array
+
+  # Convert a string to its uppercase equivalent
+  # 
+  # Example:
+  # 
+  #     filter {
+  #       mutate {
+  #         uppercase => [ "fieldname" ]
+  #       }
+  #     }
+  config :uppercase, :validate => :array
+  
+  # Convert a string to its lowercase equivalent
+  # 
+  # Example:
+  # 
+  #     filter {
+  #       mutate {
+  #         lowercase => [ "fieldname" ]
+  #       }
+  #     }
+  config :lowercase, :validate => :array
+
+  # Split a field to an array using a separator character. Only works on string
+  # fields.
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate { 
+  #          split => ["fieldname", ","]
+  #       }
+  #     }
+  config :split, :validate => :hash
+
+  # Join an array with a separator character, does nothing on non-array fields
+  #
+  # Example:
+  #
+  #    filter {
+  #      mutate { 
+  #        join => ["fieldname", ","]
+  #      }
+  #    }
+  config :join, :validate => :hash
+
+  # Strip whitespaces
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate { 
+  #          strip => ["field1", "field2"]
+  #       }
+  #     }
+  config :strip, :validate => :array
+
+  # merge two fields or arrays or hashes
+  # String fields will be converted in array, so 
+  #  array + string will work
+  #  string + string will result in an 2 entry array in dest_field
+  #  array and hash will not work
+  #
+  # Example:
+  #
+  #     filter {
+  #       mutate { 
+  #          merge => ["dest_field", "added_field"]
+  #       }
+  #     }
+  config :merge, :validate => :hash
+
+  public
+  def register
+    valid_conversions = %w(string integer float)
+    # TODO(sissel): Validate conversion requests if provided.
+    @convert.nil? or @convert.each do |field, type|
+      if !valid_conversions.include?(type)
+        @logger.error("Invalid conversion type",
+                      "type" => type, "expected one of" => valid_types)
+        # TODO(sissel): It's 2011, man, let's actually make like.. a proper
+        # 'configuration broken' exception
+        raise "Bad configuration, aborting."
+      end
+    end # @convert.each
+    
+    @gsub_parsed = []
+    @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement|
+      if [field, needle, replacement].any? {|n| n.nil?}
+        @logger.error("Invalid gsub configuration. gsub has to define 3 elements per config entry", :field => field, :needle => needle, :replacement => replacement)
+        raise "Bad configuration, aborting."
+      end
+      @gsub_parsed << {
+        :field        => field,
+        :needle       => Regexp.new(needle),
+        :replacement  => replacement
+      }
+    end
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    rename(event) if @rename
+    update(event) if @update
+    replace(event) if @replace
+    convert(event) if @convert
+    gsub(event) if @gsub
+    uppercase(event) if @uppercase
+    lowercase(event) if @lowercase
+    strip(event) if @strip
+    remove(event) if @remove
+    split(event) if @split
+    join(event) if @join
+    merge(event) if @merge
+
+    filter_matched(event)
+  end # def filter
+
+  private
+  def remove(event)
+    # TODO(sissel): use event.sprintf on the field names?
+    @remove.each do |field|
+      event.remove(field)
+    end
+  end # def remove
+
+  private
+  def rename(event)
+    # TODO(sissel): use event.sprintf on the field names?
+    @rename.each do |old, new|
+      next unless event.include?(old)
+      event[new] = event.remove(old)
+    end
+  end # def rename
+
+  private
+  def update(event)
+    @update.each do |field, newvalue|
+      next unless event.include?(field)
+      event[field] = event.sprintf(newvalue)
+    end
+  end # def update
+
+  private
+  def replace(event)
+    @replace.each do |field, newvalue|
+      event[field] = event.sprintf(newvalue)
+    end
+  end # def replace
+
+  def convert(event)
+    @convert.each do |field, type|
+      next unless event.include?(field)
+      original = event[field]
+
+      # calls convert_{string,integer,float} depending on type requested.
+      converter = method("convert_" + type)
+      if original.nil?
+        next
+      elsif original.is_a?(Hash)
+        @logger.debug("I don't know how to type convert a hash, skipping",
+                      :field => field, :value => original)
+        next
+      elsif original.is_a?(Array)
+        value = original.map { |v| converter.call(v) }
+      else
+        value = converter.call(original)
+      end
+      event[field] = value
+    end
+  end # def convert
+
+  def convert_string(value)
+    return value.to_s
+  end # def convert_string
+
+  def convert_integer(value)
+    return value.to_i
+  end # def convert_integer
+
+  def convert_float(value)
+    return value.to_f
+  end # def convert_float
+
+  private
+  def gsub(event)
+    @gsub_parsed.each do |config|
+      field = config[:field]
+      needle = config[:needle]
+      replacement = config[:replacement]
+
+      if event[field].is_a?(Array)
+        event[field] = event[field].map do |v|
+          if not v.is_a?(String)
+            @logger.warn("gsub mutation is only applicable for Strings, " +
+                          "skipping", :field => field, :value => v)
+            v
+          else
+            v.gsub(needle, replacement)
+          end
+        end
+      else
+        if not event[field].is_a?(String)
+          @logger.debug("gsub mutation is only applicable for Strings, " +
+                        "skipping", :field => field, :value => event[field])
+          next
+        end
+        event[field] = event[field].gsub(needle, replacement)
+      end
+    end # @gsub_parsed.each
+  end # def gsub
+  
+  private
+  def uppercase(event)
+    @uppercase.each do |field|
+      if event[field].is_a?(Array)
+        event[field].each { |v| v.upcase! }
+      elsif event[field].is_a?(String)
+        event[field].upcase!
+      else
+        @logger.debug("Can't uppercase something that isn't a string",
+                      :field => field, :value => event[field])
+      end
+    end
+  end # def uppercase
+
+  private
+  def lowercase(event)
+    @lowercase.each do |field|
+      if event[field].is_a?(Array)
+        event[field].each { |v| v.downcase! }
+      elsif event[field].is_a?(String)
+        event[field].downcase!
+      else
+        @logger.debug("Can't lowercase something that isn't a string",
+                      :field => field, :value => event[field])
+      end
+    end
+  end # def lowercase
+
+  private
+  def split(event)
+    @split.each do |field, separator|
+      if event[field].is_a?(String)
+        event[field] = event[field].split(separator)
+      else 
+        @logger.debug("Can't split something that isn't a string",
+                      :field => field, :value => event[field])
+      end
+    end
+  end
+
+  private
+  def join(event)
+    @join.each do |field, separator|
+      if event[field].is_a?(Array)
+        event[field] = event[field].join(separator)
+      end
+    end
+  end
+
+  private
+  def strip(event)
+    @strip.each do |field|
+      if event[field].is_a?(Array)
+        event[field] = event[field].map{|s| s.strip }
+      elsif event[field].is_a?(String)
+        event[field] = event[field].strip
+      end
+    end
+  end
+
+  private
+  def merge(event)
+    @merge.each do |dest_field, added_fields|
+      #When multiple calls, added_field is an array
+      added_fields = [ added_fields ] if ! added_fields.is_a?(Array)
+      added_fields.each do |added_field|
+        if event[dest_field].is_a?(Hash) ^ event[added_field].is_a?(Hash)
+          @logger.error("Not possible to merge an array and a hash: ",
+                        :dest_field => dest_field,
+                        :added_field => added_field )
+          next
+        end
+        if event[dest_field].is_a?(Hash) #No need to test the other
+          event[dest_field].update(event[added_field])
+        else
+          event[dest_field] = [event[dest_field]] if ! event[dest_field].is_a?(Array)
+          event[added_field] = [event[added_field]] if ! event[added_field].is_a?(Array)
+         event[dest_field].concat(event[added_field])
+        end
+      end
+    end
+  end
+
+end # class LogStash::Filters::Mutate

data/lib/logstash/filters/noop.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# No-op filter. This is used generally for internal/dev testing.
+class LogStash::Filters::NOOP < LogStash::Filters::Base
+  config_name "noop"
+  milestone 2
+
+  public
+  def register
+    # Nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    # Nothing to do
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::NOOP

data/lib/logstash/filters/prune.rb

@@ -0,0 +1,149 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The prune filter is for pruning event data from @fileds based on whitelist/blacklist
+# of field names or their values (names and values can also be regular expressions).
+
+class LogStash::Filters::Prune < LogStash::Filters::Base
+  config_name "prune"
+  milestone 1
+
+  # Trigger whether configation fields and values should be interpolated for
+  # dynamic values.
+  # Probably adds some performance overhead. Defaults to false.
+  config :interpolate, :validate => :boolean, :default => false
+
+  # Include only fields only if their names match specified regexps, default to empty list which means include everything.
+  # 
+  #     filter { 
+  #       %PLUGIN% { 
+  #         tags            => [ "apache-accesslog" ]
+  #         whitelist_names => [ "method", "(referrer|status)", "${some}_field" ]
+  #       }
+  #     }
+  config :whitelist_names, :validate => :array, :default => []
+
+  # Exclude fields which names match specified regexps, by default exclude unresolved %{field} strings.
+  #
+  #     filter { 
+  #       %PLUGIN% { 
+  #         tags            => [ "apache-accesslog" ]
+  #         blacklist_names => [ "method", "(referrer|status)", "${some}_field" ]
+  #       }
+  #     }
+  config :blacklist_names, :validate => :array, :default => [ "%\{[^}]+\}" ]
+
+  # Include specified fields only if their values match regexps.
+  # In case field values are arrays, the fields are pruned on per array item
+  # thus only matching array items will be included.
+  # 
+  #     filter { 
+  #       %PLUGIN% { 
+  #         tags             => [ "apache-accesslog" ]
+  #         whitelist_values => [ "uripath", "/index.php",
+  #                               "method", "(GET|POST)",
+  #                               "status", "^[^2]" ]
+  #       }
+  #     }
+  config :whitelist_values, :validate => :hash, :default => {}
+
+  # Exclude specified fields if their values match regexps.
+  # In case field values are arrays, the fields are pruned on per array item
+  # in case all array items are matched whole field will be deleted.
+  #
+  #     filter { 
+  #       %PLUGIN% { 
+  #         tags             => [ "apache-accesslog" ]
+  #         blacklist_values => [ "uripath", "/index.php",
+  #                               "method", "(HEAD|OPTIONS)",
+  #                               "status", "^[^2]" ]
+  #       }
+  #     }
+  config :blacklist_values, :validate => :hash, :default => {}
+
+  public
+  def register
+    unless @interpolate
+      @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(x)})
+      @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(x)})
+      @whitelist_values.each do |key, value|
+        @whitelist_values[key] = Regexp.new(value)
+      end
+      @blacklist_values.each do |key, value|
+        @blacklist_values[key] = Regexp.new(value)
+      end
+    end
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    hash = event.to_hash
+
+    # We need to collect fields which needs to be remove ,and only in the end
+    # actually remove it since then interpolation mode you can get unexpected
+    # results as fields with dynamic values will not match since the fields to
+    # which they refer have already been removed.
+    fields_to_remove = []
+
+    unless @whitelist_names.empty?
+      @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate
+      hash.each_key do |field|
+        fields_to_remove << field unless field.match(@whitelist_names_regexp)
+      end
+    end
+
+    unless @blacklist_names.empty?
+      @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate
+      hash.each_key do |field|
+        fields_to_remove << field if field.match(@blacklist_names_regexp)
+      end
+    end
+
+    @whitelist_values.each do |key, value|
+      if @interpolate
+        key = event.sprintf(key)
+        value = Regexp.new(event.sprintf(value))
+      end
+      if hash[key]
+        if hash[key].is_a?(Array)
+          subvalues_to_remove = hash[key].find_all{|x| not x.match(value)}
+          unless subvalues_to_remove.empty?
+            fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove })
+          end
+        else
+          fields_to_remove << key if not hash[key].match(value)
+        end
+      end
+    end
+
+    @blacklist_values.each do |key, value|
+      if @interpolate
+        key = event.sprintf(key)
+        value = Regexp.new(event.sprintf(value))
+      end
+      if hash[key]
+        if hash[key].is_a?(Array)
+          subvalues_to_remove = hash[key].find_all{|x| x.match(value)}
+          unless subvalues_to_remove.empty?
+            fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove })
+          end
+        else
+          fields_to_remove << key if hash[key].match(value)
+        end
+      end
+    end
+
+    fields_to_remove.each do |field|
+      if field.is_a?(Hash)
+        hash[field[:key]] = hash[field[:key]] - field[:values]
+      else
+        hash.delete(field)
+      end
+    end
+
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Prune