logstash-lib 1.3.2

data/lib/logstash/filters/alter.rb

@@ -0,0 +1,173 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The alter filter allows you to do general alterations to fields 
+# that are not included in the normal mutate filter. 
+#
+#
+# NOTE: The functionality provided by this plugin is likely to
+# be merged into the 'mutate' filter in future versions.
+class LogStash::Filters::Alter < LogStash::Filters::Base
+  config_name "alter"
+  milestone 1
+  
+  # Change the content of the field to the specified value
+  # if the actual content is equal to the expected one.
+  #
+  # Example:
+  #
+  #     filter {
+  #       alter {
+  #         condrewrite => [ 
+  #              "field_name", "expected_value", "new_value" 
+  #              "field_name2", "expected_value2, "new_value2"
+  #              ....
+  #            ]
+  #       }
+  #     }
+  config :condrewrite, :validate => :array
+  
+  # Change the content of the field to the specified value
+  # if the content of another field is equal to the expected one.
+  #
+  # Example:
+  #
+  #     filter {
+  #       alter {
+  #         condrewriteother => [ 
+  #              "field_name", "expected_value", "field_name_to_change", "value",
+  #              "field_name2", "expected_value2, "field_name_to_change2", "value2",
+  #              ....
+  #         ]
+  #       }
+  #     }
+  config :condrewriteother, :validate => :array
+  
+  # Sets the value of field_name to the first nonnull expression among its arguments.
+  #
+  # Example:
+  #
+  #     filter {
+  #       alter {
+  #         coalesce => [
+  #              "field_name", "value1", "value2", "value3", ...
+  #         ]
+  #       }
+  #     }
+  config :coalesce, :validate => :array
+  
+  public
+  def register 
+    @condrewrite_parsed = []
+    @condrewrite.nil? or @condrewrite.each_slice(3) do |field, expected, replacement|
+      if [field, expected, replacement].any? {|n| n.nil?}
+        @logger.error("Invalid condrewrte configuration. condrewrite has to define 3 elements per config entry", :field => field, :expected => expected, :replacement => replacement)
+        raise "Bad configuration, aborting."
+      end
+      @condrewrite_parsed << {
+        :field        => field,
+        :expected       => expected,
+        :replacement  => replacement
+      }
+    end # condrewrite
+    
+    @condrewriteother_parsed = []
+    @condrewriteother.nil? or @condrewriteother.each_slice(4) do |field, expected, replacement_field, replacement_value|
+      if [field, expected, replacement_field, replacement_value].any? {|n| n.nil?}
+        @logger.error("Invalid condrewrteother configuration. condrewriteother has to define 4 elements per config entry", :field => field, :expected => expected, :replacement_field => replacement_field, :replacement_value => replacement_value)
+        raise "Bad configuration, aborting."
+      end
+      @condrewriteother_parsed << {
+        :field        => field,
+        :expected       => expected,
+        :replacement_field  => replacement_field,
+        :replacement_value => replacement_value
+      }
+    end # condrewriteother
+    
+    @coalesce_parsed = []
+    @coalesce.nil? or if not @coalesce.is_a?(Array) or @coalesce.length < 2
+      @logger.error("Invalid coalesce configuration. coalesce has to define one Array of at least 2 elements")
+      raise "Bad configuration, aborting."
+    else
+      @coalesce_parsed << {
+        :field  => @coalesce.slice!(0),
+        :subst_array => @coalesce
+      }
+    end
+    
+       
+  end # def register
+  
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    condrewrite(event) if @condrewrite
+    condrewriteother(event) if @condrewriteother
+    coalesce(event) if @coalesce
+
+    filter_matched(event)
+  end # def filter
+  
+  private
+  def condrewrite(event)
+    @condrewrite_parsed.each do |config|
+      field = config[:field]
+      expected = config[:expected]
+      replacement = config[:replacement]
+
+      if event[field].is_a?(Array)
+        event[field] = event[field].map do |v|
+          if v == event.sprintf(expected)
+            v = event.sprintf(replacement)
+          else
+            v
+          end
+        end
+      else
+        if event[field] == event.sprintf(expected)
+          event[field] = event.sprintf(replacement)
+        end
+      end
+    end # @condrewrite_parsed.each
+  end # def condrewrite
+  
+  private
+  def condrewriteother(event)
+    @condrewriteother_parsed.each do |config|
+      field = config[:field]
+      expected = config[:expected]
+      replacement_field = config[:replacement_field]
+      replacement_value = config[:replacement_value]
+
+      if event[field].is_a?(Array)
+        event[field].each do |v|
+          if v == event.sprintf(expected)
+            event[replacement_field] = event.sprintf(replacement_value)
+          end
+        end
+      else
+        if event[field] == event.sprintf(expected)
+          event[replacement_field] = event.sprintf(replacement_value)
+        end
+      end
+    end # @condrewriteother_parsed.each
+  end # def condrewriteother
+  
+  private
+  def coalesce(event)
+    @coalesce_parsed.each do |config|
+      field = config[:field]
+      subst_array = config[:subst_array]
+      
+      substitution_parsed = subst_array.map { |x| event.sprintf(x) }
+      not_nul_index = substitution_parsed.find_index { |x| not x.nil? and not x.eql?("nil") and not (not x.index("%").nil? && x.match(/%\{[^}]\}/).nil?) }
+      if not not_nul_index.nil?
+        event[field] = substitution_parsed[not_nul_index]
+      end
+    end # @coalesce_parsed.each
+  end # def coalesce
+  
+end # class LogStash::Filters::Alter

data/lib/logstash/filters/anonymize.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Anonymize fields using by replacing values with a consistent hash.
+class LogStash::Filters::Anonymize < LogStash::Filters::Base
+  config_name "anonymize"
+  milestone 1
+
+  # The fields to be anonymized
+  config :fields, :validate => :array, :required => true
+
+  # Hashing key
+  # When using MURMUR3 the key is ignored but must still be set.
+  # When using IPV4_NETWORK key is the subnet prefix lentgh
+  config :key, :validate => :string, :required => true
+
+  # digest/hash type
+  config :algorithm, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK"], :required => true, :default => 'SHA1'
+
+  public
+  def register
+    # require any library and set the anonymize function
+    case @algorithm
+    when "IPV4_NETWORK"
+      require 'ipaddr'
+      class << self; alias_method :anonymize, :anonymize_ipv4_network; end
+    when "MURMUR3"
+      require "murmurhash3"
+      class << self; alias_method :anonymize, :anonymize_murmur3; end
+    else
+      require 'openssl'
+      class << self; alias_method :anonymize, :anonymize_openssl; end
+    end
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    @fields.each do |field|
+      next unless event.include?(field)
+      if event[field].is_a?(Array)
+        event[field] = event[field].collect { |v| anonymize(v) }
+      else
+        event[field] = anonymize(event[field])
+      end
+    end
+  end # def filter
+
+  private
+  def anonymize_ipv4_network(ip_string)
+    IPAddr.new(ip_string).mask(@key.to_i).to_s
+  end  
+
+  def anonymize_openssl(data)
+    digest = algorithm()
+    OpenSSL::HMAC.hexdigest(digest, @key, data)
+  end
+
+  def anonymize_murmur3(value)
+    case value
+    when Fixnum
+      MurmurHash3::V32.int_hash(value)
+    when String
+      MurmurHash3::V32.str_hash(value)
+    end
+  end
+
+  def algorithm
+ 
+   case @algorithm
+      #when 'SHA'
+        #return OpenSSL::Digest::SHA.new
+      when 'SHA1'
+        return OpenSSL::Digest::SHA1.new
+      #when 'SHA224'
+        #return OpenSSL::Digest::SHA224.new
+      when 'SHA256'
+        return OpenSSL::Digest::SHA256.new
+      when 'SHA384'
+        return OpenSSL::Digest::SHA384.new
+      when 'SHA512'
+        return OpenSSL::Digest::SHA512.new
+      #when 'MD4'
+        #return OpenSSL::Digest::MD4.new
+      when 'MD5'
+        return OpenSSL::Digest::MD5.new
+      else
+        @logger.error("Unknown algorithm")
+    end
+  end
+      
+end # class LogStash::Filters::Anonymize

data/lib/logstash/filters/base.rb

@@ -0,0 +1,190 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/plugin"
+require "logstash/config/mixin"
+
+class LogStash::Filters::Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+
+  config_name "filter"
+
+  # Note that all of the specified routing options (type,tags.exclude\_tags,include\_fields,exclude\_fields)
+  # must be met in order for the event to be handled by the filter.
+
+  # The type to act on. If a type is given, then this filter will only
+  # act on messages with the same type. See any input plugin's "type"
+  # attribute for more.
+  # Optional.
+  config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
+
+  # Only handle events with all/any (controlled by include\_any config option) of these tags.
+  # Optional.
+  config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
+
+  # Only handle events without all/any (controlled by exclude\_any config
+  # option) of these tags.
+  # Optional.
+  config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
+
+  # If this filter is successful, add arbitrary tags to the event.
+  # Tags can be dynamic and include parts of the event using the %{field}
+  # syntax. Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  #
+  # If the event has field "somefield" == "hello" this filter, on success,
+  # would add a tag "foo_hello"
+  config :add_tag, :validate => :array, :default => []
+
+  # If this filter is successful, remove arbitrary tags from the event.
+  # Tags can be dynamic and include parts of the event using the %{field}
+  # syntax. Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_tag => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  #
+  # If the event has field "somefield" == "hello" this filter, on success,
+  # would remove the tag "foo_hello" if it is present
+  config :remove_tag, :validate => :array, :default => []
+
+  # If this filter is successful, add any arbitrary fields to this event.
+  # Tags can be dynamic and include parts of the event using the %{field}
+  # Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         add_field => [ "foo_%{somefield}", "Hello world, from %{host}" ]
+  #       }
+  #     }
+  #
+  # If the event has field "somefield" == "hello" this filter, on success,
+  # would add field "foo_hello" if it is present, with the
+  # value above and the %{host} piece replaced with that value from the
+  # event.
+  config :add_field, :validate => :hash, :default => {}
+
+  # If this filter is successful, remove arbitrary fields from this event.
+  # Fields names can be dynamic and include parts of the event using the %{field}
+  # Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         remove_field => [ "foo_%{somefield}" ]
+  #       }
+  #     }
+  #
+  # If the event has field "somefield" == "hello" this filter, on success,
+  # would remove the field with name "foo_hello" if it is present
+  config :remove_field, :validate => :array, :default => []
+
+  RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field", "include_any", "exclude_any"]
+
+  public
+  def initialize(params)
+    super
+    config_init(params)
+    @threadsafe = true
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def filter(event)
+    raise "#{self.class}#filter must be overidden"
+  end # def filter
+
+  public
+  def execute(event, &block)
+    filter(event, &block)
+  end # def execute
+
+  public
+  def threadsafe?
+    @threadsafe
+  end
+
+  # a filter instance should call filter_matched from filter if the event
+  # matches the filter's conditions (right type, etc)
+  protected
+  def filter_matched(event)
+    @add_field.each do |field, value|
+      field = event.sprintf(field)
+      value = [value] if !value.is_a?(Array)
+      value.each do |v|
+        v = event.sprintf(v)
+        if event.include?(field)
+          event[field] = [event[field]] if !event[field].is_a?(Array)
+          event[field] << v
+        else
+          event[field] = v
+        end
+        @logger.debug? and @logger.debug("filters/#{self.class.name}: adding value to field",
+                                       :field => field, :value => value)
+      end
+    end
+    
+    @remove_field.each do |field|
+      field = event.sprintf(field)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing field",
+                                       :field => field) 
+      event.remove(field)
+    end
+
+    @add_tag.each do |tag|
+      tag = event.sprintf(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: adding tag",
+                                       :tag => tag)
+      (event["tags"] ||= []) << tag
+    end
+
+    @remove_tag.each do |tag|
+      break if event["tags"].nil?
+      tag = event.sprintf(tag)
+      @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag",
+                                       :tag => tag)
+      event["tags"].delete(tag)
+    end
+  end # def filter_matched
+
+  protected
+  def filter?(event)
+    if !@type.empty?
+      if event["type"] != @type
+        @logger.debug? and @logger.debug(["filters/#{self.class.name}: Skipping event because type doesn't match #{@type}", event])
+        return false
+      end
+    end
+
+    if !@tags.empty?
+      # this filter has only works on events with certain tags,
+      # and this event has no tags.
+      return false if !event["tags"]
+
+      # Is @tags a subset of the event's tags? If not, skip it.
+      if (event["tags"] & @tags).size != @tags.size
+        @logger.debug(["filters/#{self.class.name}: Skipping event because tags don't match #{@tags.inspect}", event])
+        return false
+      end
+    end
+
+    if !@exclude_tags.empty? && event["tags"]
+      if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
+        @logger.debug(["filters/#{self.class.name}: Skipping event because tags contains excluded tags: #{diff_tags.inspect}", event])
+        return false
+      end
+    end
+
+    return true
+  end
+end # class LogStash::Filters::Base