logstash-lib 1.3.2

data/lib/logstash/filters/grokdiscovery.rb

@@ -0,0 +1,75 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# TODO(sissel): This is not supported yet. There is a bug in grok discovery
+# that causes segfaults in libgrok.
+class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
+
+  config_name "grokdiscovery"
+  milestone 1
+
+  public
+  def initialize(config = {})
+    super
+
+    @discover_fields = {}
+  end # def initialize
+
+  public
+  def register
+    gem "jls-grok", ">=0.4.3"
+    require "grok" # rubygem 'jls-grok'
+
+    # TODO(sissel): Make patterns files come from the config
+    @config.each do |type, typeconfig|
+      @logger.debug("Registering type with grok: #{type}")
+      @grok = Grok.new
+      Dir.glob("patterns/*").each do |path|
+        @grok.add_patterns_from_file(path)
+      end
+      @discover_fields[type] = typeconfig
+      @logger.debug(["Enabling discovery", { :type => type, :fields => typeconfig }])
+      @logger.warn(@discover_fields)
+    end # @config.each
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    # parse it with grok
+    message = event["message"]
+    match = false
+
+    if event.type and @discover_fields.include?(event.type)
+      discover = @discover_fields[event.type] & event.to_hash.keys
+      discover.each do |field|
+        value = event[field]
+        value = [value] if value.is_a?(String)
+
+        value.each do |v| 
+          pattern = @grok.discover(v)
+          @logger.warn("Trying #{v} => #{pattern}")
+          @grok.compile(pattern)
+          match = @grok.match(v)
+          if match
+            @logger.warn(["Match", match.captures])
+            event.to_hash.merge!(match.captures) do |key, oldval, newval|
+              @logger.warn(["Merging #{key}", oldval, newval])
+              oldval + newval # should both be arrays...
+            end
+          else
+            @logger.warn(["Discovery produced something not matchable?", { :input => v }])
+          end
+        end # value.each
+      end # discover.each
+    else
+      @logger.info("Unknown type for #{event.source} (type: #{event.type})")
+      @logger.debug(event.to_hash)
+    end
+    @logger.debug(["Event now: ", event.to_hash])
+
+    filter_matched(event) if !event.cancelled?
+  end # def filter
+end # class LogStash::Filters::Grokdiscovery

data/lib/logstash/filters/i18n.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "i18n"
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The i18n filter allows you to remove special characters from
+# from a field
+class LogStash::Filters::I18n < LogStash::Filters::Base
+  config_name "i18n"
+  milestone 0
+
+  # Replaces non-ASCII characters with an ASCII approximation, or
+  # if none exists, a replacement character which defaults to “?”
+  #
+  # Example:
+  #
+  #     filter {
+  #       i18n {
+  #          transliterate => ["field1", "field2"]
+  #       }
+  #     }
+  config :transliterate, :validate => :array
+
+  public
+  def register
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    transliterate(event) if @transliterate
+
+    filter_matched(event)
+  end # def filter
+
+  private
+  def transliterate(event)
+    @transliterate.each do |field|
+      if event[field].is_a?(Array)
+        event[field].map! { |v| I18n.transliterate(v).encode('UTF-8') }
+      elsif event[field].is_a?(String)
+        event[field] = I18n.transliterate(event[field].encode('UTF-8'))
+      else
+        @logger.debug("Can't transliterate something that isn't a string",
+                      :field => field, :value => event[field])
+      end
+    end
+  end # def transliterate
+
+end # class LogStash::Filters::I18n

data/lib/logstash/filters/json.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# JSON filter. Takes a field that contains JSON and expands it into
+# an actual datastructure.
+class LogStash::Filters::Json < LogStash::Filters::Base
+
+  config_name "json"
+  milestone 2
+
+  # Config for json is:
+  #
+  #     source => source_field
+  #
+  # For example, if you have json data in the @message field:
+  #
+  #     filter {
+  #       json {
+  #         source => "message"
+  #       }
+  #     }
+  #
+  # The above would parse the json from the @message field
+  config :source, :validate => :string, :required => true
+
+  # Define target for placing the data. If this setting is omitted,
+  # the json data will be stored at the root of the event.
+  #
+  # For example if you want the data to be put in the 'doc' field:
+  #
+  #     filter {
+  #       json {
+  #         target => "doc"
+  #       }
+  #     }
+  #
+  # json in the value of the source field will be expanded into a
+  # datastructure in the "target" field.
+  #
+  # Note: if the "target" field already exists, it will be overwritten.
+  config :target, :validate => :string
+
+  public
+  def register
+    # Nothing to do here
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @logger.debug("Running json filter", :event => event)
+
+    return unless event.include?(@source)
+
+    if @target.nil?
+      # Default is to write to the root of the event.
+      dest = event.to_hash
+    else
+      dest = event[@target] ||= {}
+    end
+
+    begin
+      # TODO(sissel): Note, this will not successfully handle json lists
+      # like your text is '[ 1,2,3 ]' JSON.parse gives you an array (correctly)
+      # which won't merge into a hash. If someone needs this, we can fix it
+      # later.
+      dest.merge!(JSON.parse(event[@source]))
+
+      # This is a hack to help folks who are mucking with @timestamp during
+      # their json filter. You aren't supposed to do anything with "@timestamp"
+      # outside of the date filter, but nobody listens... ;)
+      if event["@timestamp"].is_a?(String)
+        event["@timestamp"] = Time.parse(event["@timestamp"]).gmtime
+      end
+
+      filter_matched(event)
+    rescue => e
+      event.tag("_jsonparsefailure")
+      @logger.warn("Trouble parsing json", :source => @source,
+                   :raw => event[@source], :exception => e)
+      return
+    end
+
+    @logger.debug("Event after json filter", :event => event)
+
+  end # def filter
+
+end # class LogStash::Filters::Json

data/lib/logstash/filters/json_encode.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# JSON encode filter. Takes a field and serializes it into JSON
+#
+# If no target is specified, the source field is overwritten with the JSON
+# text.
+#
+# For example, if you have a field named 'foo', and you want to store the
+# JSON encoded string in 'bar', do this:
+#
+#     filter {
+#       json_encode {
+#         source => "foo"
+#         target => "bar"
+#       }
+#     }
+class LogStash::Filters::JSONEncode < LogStash::Filters::Base
+
+  config_name "json_encode"
+  milestone 2
+
+  # The field to convert to JSON.
+  config :source, :validate => :string, :required => true
+
+  # The field to write the JSON into. If not specified, the source
+  # field will be overwritten.
+  config :target, :validate => :string
+
+  public
+  def register
+    @target = @source if @target.nil?
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @logger.debug("Running JSON encoder", :event => event)
+
+    begin
+      event[@target] = JSON.pretty_generate(event[@source])
+      filter_matched(event)
+    rescue => e
+      event.tag "_jsongeneratefailure"
+      @logger.warn("Trouble encoding JSON", :source => @source, :raw => event[@source].inspect, :exception => e)
+    end
+
+    @logger.debug? && @logger.debug("Event after JSON encoder", :event => event)
+  end # def filter
+end # class LogStash::Filters::JSONEncode

data/lib/logstash/filters/kv.rb

@@ -0,0 +1,232 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This filter helps automatically parse messages which are of the 'foo=bar'
+# variety.
+#
+# For example, if you have a log message which contains 'ip=1.2.3.4
+# error=REFUSED', you can parse those automatically by doing:
+#
+#     filter {
+#       kv { }
+#     }
+#
+# The above will result in a message of "ip=1.2.3.4 error=REFUSED" having
+# the fields:
+#
+# * ip: 1.2.3.4
+# * error: REFUSED
+#
+# This is great for postfix, iptables, and other types of logs that
+# tend towards 'key=value' syntax.
+#
+# Further, this can often be used to parse query parameters like
+# 'foo=bar&baz=fizz' by setting the field_split to "&"
+class LogStash::Filters::KV < LogStash::Filters::Base
+  config_name "kv"
+  milestone 2
+
+  # A string of characters to trim from the value. This is useful if your
+  # values are wrapped in brackets or are terminated by comma (like postfix
+  # logs)
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
+  # Example, to strip '<' '>' '[' ']' and ',' characters from values:
+  #
+  #     filter {
+  #       kv {
+  #         trim => "<>\[\],"
+  #       }
+  #     }
+  config :trim, :validate => :string
+
+  # A string of characters to trim from the key. This is useful if your
+  # key are wrapped in brackets or starts with space
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
+  # Example, to strip '<' '>' '[' ']' and ',' characters from keys:
+  #
+  #     filter {
+  #       kv {
+  #         trimkey => "<>\[\],"
+  #       }
+  #     }
+  config :trimkey, :validate => :string
+
+  # A string of characters to use as delimiters for parsing out key-value pairs.
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
+  # #### Example with URL Query Strings
+  #
+  # Example, to split out the args from a url query string such as
+  # '?pin=12345~0&d=123&e=foo@bar.com&oq=bobo&ss=12345':
+  #
+  #     filter {
+  #       kv {
+  #         field_split => "&?"
+  #       }
+  #     }
+  #
+  # The above splits on both "&" and "?" characters, giving you the following
+  # fields:
+  #
+  # * pin: 12345~0
+  # * d: 123
+  # * e: foo@bar.com
+  # * oq: bobo
+  # * ss: 12345
+  config :field_split, :validate => :string, :default => ' '
+
+
+  # A string of characters to use as delimiters for identifying key-value relations.
+  #
+  # These characters form a regex character class and thus you must escape special regex
+  # characters like [ or ] using \.
+  #
+  # Example, to identify key-values such as
+  # 'key1:value1 key2:value2':
+  #
+  #     filter { kv { value_split => ":" } }
+  config :value_split, :validate => :string, :default => '='
+
+  # A string to prepend to all of the extracted keys
+  #
+  # Example, to prepend arg_ to all keys:
+  #
+  #     filter { kv { prefix => "arg_" } }
+  config :prefix, :validate => :string, :default => ''
+
+  # The fields to perform 'key=value' searching on
+  #
+  # Example, to use the message field:
+  #
+  #     filter { kv { source => "message" } }
+  config :source, :validate => :string, :default => "message"
+
+  # The name of the container to put all of the key-value pairs into
+  #
+  # If this setting is omitted, fields will be written to the root of the
+  # event.
+  #
+  # Example, to place all keys into field kv:
+  #
+  #     filter { kv { target => "kv" } }
+  config :target, :validate => :string
+
+  # An array that specifies the parsed keys which should be added to event.
+  # By default all keys will be added.
+  #
+  # Example, to include only "from" and "to" from a source like "Hey, from=<abc>, to=def foo=bar"
+  # while "foo" key will not be added to event.
+  #
+  #     filter {
+  #       kv {
+  #         include_keys = [ "from", "to" ]
+  #       }
+  #     }
+  config :include_keys, :validate => :array, :default => []
+
+  # An array that specifies the parsed keys which should not be added to event.
+  # By default no keys will be excluded.
+  #
+  # Example, to exclude "from" and "to" from a source like "Hey, from=<abc>, to=def foo=bar"
+  # while "foo" key will be added to event.
+  #
+  #     filter {
+  #       kv {
+  #         exclude_keys = [ "from", "to" ]
+  #       }
+  #     }
+  config :exclude_keys, :validate => :array, :default => []
+
+  # A hash that specifies the default keys and their values that should be added to event
+  # in case these keys do no exist in the source field being parsed.
+  #
+  #     filter {
+  #       kv {
+  #         default_keys = [ "from", "logstash@example.com",
+  #                          "to", "default@dev.null" ]
+  #       }
+  #     }
+  config :default_keys, :validate => :hash, :default => {}
+
+  def register
+    @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
+    @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
+    @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)["+@value_split+"](?:\"([^\"]+)\"|'([^']+)'|((?:\\\\ |[^"+@field_split+"])+))")
+  end # def register
+
+  def filter(event)
+    return unless filter?(event)
+
+    kv = Hash.new
+
+    value = event[@source]
+
+    case value
+      when nil; # Nothing to do
+      when String; kv = parse(value, event, kv)
+      when Array; value.each { |v| kv = parse(v, event, kv) }
+      else
+        @logger.warn("kv filter has no support for this type of data",
+                     :type => value.class, :value => value)
+    end # case value
+
+    # Add default key-values for missing keys
+    kv = @default_keys.merge(kv)
+
+    # If we have any keys, create/append the hash
+    if kv.length > 0
+      if @target.nil?
+        # Default is to write to the root of the event.
+        dest = event.to_hash
+      else
+        if !event[@target].is_a?(Hash)
+          @logger.debug("Overwriting existing target field", :target => @target)
+          dest = event[@target] = {}
+        else
+          dest = event[@target]
+        end
+      end
+
+      dest.merge!(kv)
+      filter_matched(event)
+    end
+  end # def filter
+
+  private
+  def parse(text, event, kv_keys)
+    if !event =~ /[@field_split]/
+      return kv_keys
+    end
+    text.scan(@scan_re) do |key, v1, v2, v3|
+      value = v1 || v2 || v3
+      key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "")
+
+      # Bail out as per the values of @include_keys and @exclude_keys
+      next if not @include_keys.empty? and not @include_keys.include?(key)
+      next if @exclude_keys.include?(key)
+
+      key = event.sprintf(@prefix) + key
+
+      value = @trim.nil? ? value : value.gsub(@trim_re, "")
+      if kv_keys.has_key?(key)
+        if kv_keys[key].is_a? Array
+          kv_keys[key].push(value)
+        else
+          kv_keys[key] = [kv_keys[key], value]
+        end
+      else
+        kv_keys[key] = value
+      end
+    end
+    return kv_keys
+  end
+end # class LogStash::Filters::KV