logstash-lib 1.3.2

data/lib/logstash/filters/sumnumbers.rb

@@ -0,0 +1,73 @@
+require 'logstash/namespace'
+require 'logstash/filters/base'
+
+
+# This filter automatically sum all numbers found inside a string
+#
+# The sum is returned in a new field, "sumTotal".
+# The total numbers summed will be in a new field, "sumNums"
+#
+# The fields produced by this filter are extra useful used in combination
+# with kibana number plotting features.
+#
+# If the field is an array, all of the numbers in it will be summed.
+# If the field is a hash, all of the values of the top-level keys will be summed.
+# If the field is a string, it will be split, numbers extracted, and summed.
+class LogStash::Filters::SumNumbers < LogStash::Filters::Base
+  config_name 'sumnumbers'
+  milestone 1
+
+  # The source field for the data. By default is message.
+  config :source, :validate => :string, :default => 'message'
+
+  public
+  def register
+  end
+
+  public
+  def filter(event)
+    msg = event[@source]
+    sumnums = 0
+    sumtotal = 0
+
+    if not msg
+      return
+    end
+
+    # If for some reason the field is an array of values, take the first only.
+    if msg.is_a?(Array)
+      fields = msg.first.split
+      # If msg is json, get an array from the values
+    elsif msg.is_a?(Hash)
+      fields = msg.values
+      # Else, we have a string. Split it.
+    else
+      fields = msg.split
+    end
+
+    for elem in fields
+      int = str_as_integer(elem)
+      if int != nil
+        sumtotal += int
+        sumnums += 1
+        next
+      end
+      f = str_as_float(elem)
+      if f != nil
+        sumtotal += f
+        sumnums += 1
+      end
+    end
+
+    event["sumNums"] = sumnums
+    event["sumTotal"] = sumtotal
+  end
+
+  def str_as_integer(str)
+    Integer(str) rescue nil
+  end
+
+  def str_as_float(str)
+    Float(str) rescue nil
+  end
+end # class LogStash::Filters::SumNumbers

data/lib/logstash/filters/syslog_pri.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Filter plugin for logstash to parse the PRI field from the front
+# of a Syslog (RFC3164) message.  If no priority is set, it will
+# default to 13 (per RFC).
+#
+# This filter is based on the original syslog.rb code shipped
+# with logstash.
+class LogStash::Filters::Syslog_pri < LogStash::Filters::Base
+  config_name "syslog_pri"
+
+  # set the status to experimental/beta/stable
+  milestone 1
+
+  # Add human-readable names after parsing severity and facility from PRI
+  config :use_labels, :validate => :boolean, :default => true
+
+  # Name of field which passes in the extracted PRI part of the syslog message
+  config :syslog_pri_field_name, :validate => :string, :default => "syslog_pri"
+
+  # Labels for facility levels. This comes from RFC3164.
+  config :facility_labels, :validate => :array, :default => [
+    "kernel",
+    "user-level",
+    "mail",
+    "daemon",
+    "security/authorization",
+    "syslogd",
+    "line printer",
+    "network news",
+    "uucp",
+    "clock",
+    "security/authorization",
+    "ftp",
+    "ntp",
+    "log audit",
+    "log alert",
+    "clock",
+    "local0",
+    "local1",
+    "local2",
+    "local3",
+    "local4",
+    "local5",
+    "local6",
+    "local7",
+  ]
+
+  # Labels for severity levels. This comes from RFC3164.
+  config :severity_labels, :validate => :array, :default => [
+    "emergency",
+    "alert",
+    "critical",
+    "error",
+    "warning",
+    "notice",
+    "informational",
+    "debug",
+  ]
+
+  public
+  def register
+    # Nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    parse_pri(event)
+    filter_matched(event)
+  end # def filter
+
+  private
+  def parse_pri(event)
+    # Per RFC3164, priority = (facility * 8) + severity
+    # = (facility << 3) & (severity)
+    if event[@syslog_pri_field_name]
+      if event[@syslog_pri_field_name].is_a?(Array)
+        priority = event[@syslog_pri_field_name].first.to_i
+      else
+        priority = event[@syslog_pri_field_name].to_i
+      end
+    else
+      priority = 13  # default
+    end
+    severity = priority & 7 # 7 is 111 (3 bits)
+    facility = priority >> 3
+    event["syslog_severity_code"] = severity
+    event["syslog_facility_code"] = facility
+
+    # Add human-readable names after parsing severity and facility from PRI
+    if @use_labels
+      facility_number = event["syslog_facility_code"]
+      severity_number = event["syslog_severity_code"]
+
+      if @facility_labels[facility_number]
+        event["syslog_facility"] = @facility_labels[facility_number]
+      end
+
+      if @severity_labels[severity_number]
+        event["syslog_severity"] = @severity_labels[severity_number]
+      end
+    end
+  end # def parse_pri
+end # class LogStash::Filters::SyslogPRI

data/lib/logstash/filters/translate.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Originally written to translate HTTP response codes 
+# but turned into a general translation tool which uses
+# configured has or/and .yaml files as a dictionary.
+# response codes in default dictionary were scraped from 
+# 'gem install cheat; cheat status_codes'
+#
+# Alternatively for simple string search and replacements for just a few values
+# use the gsub function of the mutate filter.
+
+class LogStash::Filters::Translate < LogStash::Filters::Base
+  config_name "translate"
+  milestone 1
+
+  # The field containing a response code If this field is an
+  # array, only the first value will be used.
+  config :field, :validate => :string, :required => true
+
+  # In case dstination field already exists should we skip translation(default) or override it with new translation
+  config :override, :validate => :boolean, :default => false
+
+  # Dictionary to use for translation.
+  # Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         dictionary => [ "100", "Continue",
+  #                         "101", "Switching Protocols",
+  #                         "200", "OK",
+  #                         "201", "Created",
+  #                         "202", "Accepted" ]
+  #       }
+  #     }
+  config :dictionary, :validate => :hash,  :default => {}
+
+  # name with full path of external dictionary file.    
+  # format of the table should be a YAML file which will be merged with the @dictionary.
+  # make sure you encase any integer based keys in quotes.
+  # The YAML file should look something like this:
+  #
+  #     100: Continue
+  #     101: Switching Protocols
+  config :dictionary_path, :validate => :path
+
+  # The destination field you wish to populate with the translation code.
+  # default is "translation".
+  # Set to the same value as source if you want to do a substitution, in this case filter will allways succeed.
+  config :destination, :validate => :string, :default => "translation"
+
+  # set to false if you want to match multiple terms
+  # a large dictionary could get expensive if set to false.
+  config :exact, :validate => :boolean, :default => true
+
+  # treat dictionary keys as regular expressions to match against, used only then @exact enabled.
+  config :regex, :validate => :boolean, :default => false
+
+  # Incase no translation was made add default translation string
+  config :fallback, :validate => :string
+
+  public
+  def register
+    if @dictionary_path
+      raise "#{self.class.name}: dictionary file #{@dictionary_path} does not exists" unless File.exists?(@dictionary_path)
+      begin
+        @dictionary.merge!(YAML.load_file(@dictionary_path))
+      rescue Exception => e
+        raise "#{self.class.name}: Bad Syntax in dictionary file #{@dictionary_path}"
+      end
+    end
+    
+    @logger.debug? and @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @dictionary)
+    if @exact
+      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
+    else
+      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
+    end
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    return unless event.include?(@field) # Skip translation in case event does not have @event field.
+    return if event.include?(@destination) and not @override # Skip translation in case @destination field already exists and @override is disabled.
+
+    begin
+      #If source field is array use first value and make sure source value is string
+      source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
+      matched = false
+      if @exact
+        if @regex
+          key = @dictionary.keys.detect{|k| source.match(Regexp.new(k))}
+          if key
+            event[@destination] = @dictionary[key]
+            matched = true
+          end
+        elsif @dictionary.include?(source)
+          event[@destination] = @dictionary[source]
+          matched = true
+        end
+      else 
+        translation = source.gsub(Regexp.union(@dictionary.keys), @dictionary)
+        if source != translation
+          event[@destination] = translation
+          matched = true
+        end
+      end
+
+      if not matched and @fallback
+        event[@destination] = @fallback
+        matched = true
+      end
+      filter_matched(event) if matched or @field == @destination
+    rescue Exception => e
+      @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
+    end
+  end # def filter
+end # class LogStash::Filters::Translate

data/lib/logstash/filters/unique.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+class LogStash::Filters::Unique < LogStash::Filters::Base
+
+  config_name "unique"
+  milestone 1
+
+  # The fields on which to run the unique filter.
+  config :fields, :validate => :array, :required => true
+
+  public
+  def register
+    # Nothing to do
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @fields.each do |field|
+      next unless event[field].class == Array
+
+      event[field] = event[field].uniq
+    end
+  end # def filter
+
+end # class Logstash::Filters::Unique

data/lib/logstash/filters/urldecode.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "uri"
+
+# The urldecode filter is for decoding fields that are urlencoded.
+class LogStash::Filters::Urldecode < LogStash::Filters::Base
+  config_name "urldecode"
+  milestone 2
+
+  # The field which value is urldecoded
+  config :field, :validate => :string, :default => "message"
+
+  # Urldecode all fields
+  config :all_fields, :validate => :boolean, :default => false
+
+  public
+  def register
+    # Nothing to do
+  end #def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    # If all_fields is true then try to decode them all
+    if @all_fields
+      event.to_hash.each do |name, value|
+        event[name] = urldecode(value)
+      end
+    # Else decode the specified field
+    else
+      event[@field] = urldecode(event[@field])
+    end
+    filter_matched(event)
+  end # def filter
+
+  # Attempt to handle string, array, and hash values for fields.
+  # For all other datatypes, just return, URI.unescape doesn't support them.
+  private
+  def urldecode(value)
+    case value
+    when String
+      return URI.unescape(value)
+    when Array
+      ret_values = []
+      value.each { |v| ret_values << urldecode(v) }
+      return ret_values
+    when Hash
+      ret_values = {}
+      value.each { |k,v| ret_values[k] = urldecode(v) }
+      return ret_values
+    else
+      return value
+    end
+  end
+end # class LogStash::Filters::Urldecode

data/lib/logstash/filters/useragent.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "tempfile"
+
+# Parse user agent strings into structured data based on BrowserScope data
+#
+# UserAgent filter, adds information about user agent like family, operating
+# system, version, and device
+#
+# Logstash releases ship with the regexes.yaml database made available from
+# ua-parser with an Apache 2.0 license. For more details on ua-parser, see
+# <https://github.com/tobie/ua-parser/>.
+class LogStash::Filters::UserAgent < LogStash::Filters::Base
+  config_name "useragent"
+  milestone 1
+
+  # The field containing the user agent string. If this field is an
+  # array, only the first value will be used.
+  config :source, :validate => :string, :required => true
+
+  # The name of the field to assign user agent data into.
+  #
+  # If not specified user agent data will be stored in the root of the event.
+  config :target, :validate => :string
+
+  # regexes.yaml file to use
+  #
+  # If not specified, this will default to the regexes.yaml that ships
+  # with logstash.
+  #
+  # You can find the latest version of this here:
+  # <https://github.com/tobie/ua-parser/blob/master/regexes.yaml>
+  config :regexes, :validate => :string
+
+  # A string to prepend to all of the extracted keys
+  config :prefix, :validate => :string, :default => ''
+
+  public
+  def register
+    require 'user_agent_parser'
+    if @regexes.nil?
+      begin
+        @parser = UserAgentParser::Parser.new()
+      rescue Exception => e
+        begin
+          if __FILE__ =~ /file:\/.*\.jar!/
+            # Running from a flatjar which has a different layout
+            regexes_file = [__FILE__.split("!").first, "/vendor/ua-parser/regexes.yaml"].join("!")
+            @parser = UserAgentParser::Parser.new(:patterns_path => regexes_file)
+          else
+            # assume operating from the git checkout
+            @parser = UserAgentParser::Parser.new(:patterns_path => "vendor/ua-parser/regexes.yaml")
+          end
+        rescue => ex
+          raise "Failed to cache, due to: #{ex}\n"
+        end
+      end
+    else
+      @logger.info("Using user agent regexes", :regexes => @regexes)
+      @parser = UserAgentParser::Parser.new(:patterns_path => @regexes)
+    end
+  end #def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    ua_data = nil
+
+    useragent = event[@source]
+    useragent = useragent.first if useragent.is_a? Array
+
+    begin
+      ua_data = @parser.parse(useragent)
+    rescue Exception => e
+      @logger.error("Uknown error while parsing user agent data", :exception => e, :field => @source, :event => event)
+    end
+
+    if !ua_data.nil?
+      if @target.nil?
+        # default write to the root of the event
+        target = event
+      else
+        target = event[@target] ||= {}
+      end
+
+      target[@prefix + "name"] = ua_data.name
+
+      #OSX, Andriod and maybe iOS parse correctly, ua-agent parsing for Windows does not provide this level of detail
+      unless ua_data.os.nil?
+        target[@prefix + "os"] = ua_data.os.to_s
+        target[@prefix + "os_name"] = ua_data.os.name.to_s
+        target[@prefix + "os_major"] = ua_data.os.version.major.to_s unless ua_data.os.version.nil?
+        target[@prefix + "os_minor"] = ua_data.os.version.minor.to_s unless ua_data.os.version.nil?
+      end
+
+      target[@prefix + "device"] = ua_data.device.to_s if not ua_data.device.nil?
+
+      if not ua_data.version.nil?
+        ua_version = ua_data.version
+        target[@prefix + "major"] = ua_version.major
+        target[@prefix + "minor"] = ua_version.minor
+        target[@prefix + "patch"] = ua_version.patch if ua_version.patch
+        target[@prefix + "build"] = ua_version.patch_minor if ua_version.patch_minor 
+      end
+
+      filter_matched(event)
+    end
+
+  end # def filter
+end # class LogStash::Filters::UserAgent
+