logstash-lib 1.3.2

data/lib/logstash/filters/date.rb

@@ -0,0 +1,244 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The date filter is used for parsing dates from fields and using that
+# date or timestamp as the timestamp for the event.
+#
+# For example, syslog events usually have timestamps like this:
+#
+#     "Apr 17 09:32:01"
+#
+# You would use the date format "MMM dd HH:mm:ss" to parse this.
+#
+# The date filter is especially important for sorting events and for
+# backfilling old data. If you don't get the date correct in your
+# event, then searching for them later will likely sort out of order.
+#
+# In the absence of this filter, logstash will choose a timestamp based on the
+# first time it sees the event (at input time), if the timestamp is not already
+# set in the event. For example, with file input, the timestamp is set to the
+# time of each read.
+class LogStash::Filters::Date < LogStash::Filters::Base
+  if RUBY_ENGINE == "jruby"
+    JavaException = java.lang.Exception
+    UTC = org.joda.time.DateTimeZone.forID("UTC")
+  end
+
+  config_name "date"
+  milestone 3
+
+  # Specify a timezone canonical ID to be used for date parsing.
+  # The valid ID are listed on http://joda-time.sourceforge.net/timezones.html
+  # Useful in case the timezone cannot be extracted from the value,
+  # and is not the platform default.
+  # If this is not specified the platform default will be used.
+  # Canonical ID is good as it takes care of daylight saving time for you
+  # For example, America/Los_Angeles or Europe/France are valid IDs.
+  config :timezone, :validate => :string
+
+  # specify a locale to be used for date parsing. If this is not specified the
+  # platform default will be used
+  #
+  # The locale is mostly necessary to be set for parsing month names and
+  # weekday names
+  #
+  config :locale, :validate => :string
+
+  # The date formats allowed are anything allowed by Joda-Time (java time
+  # library): You can see the docs for this format here:
+  #
+  # [joda.time.format.DateTimeFormat](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html)
+  #
+  # An array with field name first, and format patterns following, `[ field,
+  # formats... ]`
+  #
+  # If your time field has multiple possible formats, you can do this:
+  #
+  #     match => [ "logdate", "MMM dd YYY HH:mm:ss",
+  #               "MMM  d YYY HH:mm:ss", "ISO8601" ]
+  #
+  # The above will match a syslog (rfc3164) or iso8601 timestamp.
+  #
+  # There are a few special exceptions, the following format literals exist
+  # to help you save time and ensure correctness of date parsing.
+  #
+  # * "ISO8601" - should parse any valid ISO8601 timestamp, such as
+  #   2011-04-19T03:44:01.103Z
+  # * "UNIX" - will parse unix time in seconds since epoch
+  # * "UNIX_MS" - will parse unix time in milliseconds since epoch
+  # * "TAI64N" - will parse tai64n time values
+  #
+  # For example, if you have a field 'logdate' and with a value that looks like
+  # 'Aug 13 2010 00:03:44', you would use this configuration:
+  #
+  #     filter {
+  #       date {
+  #         match => [ "logdate", "MMM dd YYYY HH:mm:ss" ]
+  #       }
+  #     }
+  #
+  # If your field is nested in your structure, you can use the nested
+  # syntax [foo][bar] to match its value. For more information, please refer to
+  # http://logstash.net/docs/latest/configuration#fieldreferences
+  config :match, :validate => :array, :default => []
+
+  # Store the matching timestamp into the given target field.  If not provided,
+  # default to updating the @timestamp field of the event.
+  config :target, :validate => :string, :default => "@timestamp"
+
+  # LOGSTASH-34
+  DATEPATTERNS = %w{ y d H m s S } 
+
+  # The 'date' filter will take a value from your event and use it as the
+  # event timestamp. This is useful for parsing logs generated on remote
+  # servers or for importing old logs.
+  #
+  # The config looks like this:
+  #
+  #     filter {
+  #       date {
+  #         type => "typename"
+  #         filename => fieldformat
+  #         # Example:
+  #         timestamp => "mmm DD HH:mm:ss"
+  #       }
+  #     }
+  #
+  # The format is whatever is supported by Joda; generally:
+  # http://download.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html
+  #
+  # TODO(sissel): Support 'seconds since epoch' parsing (nagios uses this)
+  public
+  def initialize(config = {})
+    super
+
+    @parsers = Hash.new { |h,k| h[k] = [] }
+  end # def initialize
+
+  private
+  def parseLocale(localeString)
+    return nil if localeString == nil
+    matches = localeString.match(/(?<lang>.+?)(?:_(?<country>.+?))?(?:_(?<variant>.+))?/)
+    lang = matches['lang'] == nil ? "" : matches['lang'].strip()
+    country = matches['country'] == nil ? "" : matches['country'].strip()
+    variant = matches['variant'] == nil ? "" : matches['variant'].strip()
+    return lang.length > 0 ? java.util.Locale.new(lang, country, variant) : nil
+  end
+
+  public
+  def register
+    require "java"
+    if @match.length < 2
+      raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register", 
+        :plugin => "filter", :type => "date",
+        :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
+    end
+    # TODO(sissel): Need a way of capturing regexp configs better.
+    locale = parseLocale(@config["locale"][0]) if @config["locale"] != nil and @config["locale"][0] != nil
+    setupMatcher(@config["match"].shift, locale, @config["match"] )
+  end
+
+  def setupMatcher(field, locale, value)
+    value.each do |format|
+      case format
+        when "ISO8601"
+          joda_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
+          if @timezone
+            joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
+          else
+            joda_parser = joda_parser.withOffsetParsed
+          end
+          parser = lambda { |date| joda_parser.parseMillis(date) }
+        when "UNIX" # unix epoch
+          joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
+          #parser = lambda { |date| joda_instant.call((date.to_f * 1000).to_i).to_java.toDateTime }
+          parser = lambda { |date| (date.to_f * 1000).to_i }
+        when "UNIX_MS" # unix epoch in ms
+          joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
+          parser = lambda do |date| 
+            #return joda_instant.call(date.to_i).to_java.toDateTime
+            return date.to_i
+          end
+        when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
+          joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
+          parser = lambda do |date| 
+            # Skip leading "@" if it is present (common in tai64n times)
+            date = date[1..-1] if date[0, 1] == "@"
+            #return joda_instant.call((date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)).to_java.toDateTime 
+            return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
+          end
+        else
+          joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
+          if @timezone
+            joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
+          else
+            joda_parser = joda_parser.withOffsetParsed
+          end
+          if (locale != nil)
+            joda_parser = joda_parser.withLocale(locale)
+          end
+          parser = lambda { |date| joda_parser.parseMillis(date) }
+      end
+
+      @logger.debug("Adding type with date config", :type => @type,
+                    :field => field, :format => format)
+      @parsers[field] << {
+        :parser => parser,
+        :format => format
+      }
+    end
+  end
+
+  # def register
+
+  public
+  def filter(event)
+    @logger.debug? && @logger.debug("Date filter: received event", :type => event["type"])
+    return unless filter?(event)
+    @parsers.each do |field, fieldparsers|
+      @logger.debug? && @logger.debug("Date filter looking for field",
+                                      :type => event["type"], :field => field)
+      next unless event.include?(field)
+
+      fieldvalues = event[field]
+      fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
+      fieldvalues.each do |value|
+        next if value.nil?
+        begin
+          epochmillis = nil
+          success = false
+          last_exception = RuntimeError.new "Unknown"
+          fieldparsers.each do |parserconfig|
+            parser = parserconfig[:parser]
+            begin
+              epochmillis = parser.call(value)
+              success = true
+              break # success
+            rescue StandardError, JavaException => e
+              last_exception = e
+            end
+          end # fieldparsers.each
+
+          raise last_exception unless success
+
+          # Convert joda DateTime to a ruby Time
+          event[@target] = Time.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
+          #event[@target] = Time.at(epochmillis / 1000.0).utc
+
+          @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event[@target])
+        rescue StandardError, JavaException => e
+          @logger.warn("Failed parsing date from field", :field => field,
+                       :value => value, :exception => e)
+          # Raising here will bubble all the way up and cause an exit.
+          # TODO(sissel): Maybe we shouldn't raise?
+          # TODO(sissel): What do we do on a failure? Tag it like grok does?
+          #raise e
+        end # begin
+      end # fieldvalue.each 
+    end # @parsers.each
+
+    filter_matched(event) if !event.cancelled?
+    return event
+  end # def filter
+end # class LogStash::Filters::Date

data/lib/logstash/filters/dns.rb

@@ -0,0 +1,201 @@
+# encoding: utf-8
+# DNS Filter
+#
+# This filter will resolve any IP addresses from a field of your choosing.
+#
+
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The DNS filter performs a lookup (either an A record/CNAME record lookup
+# or a reverse lookup at the PTR record) on records specified under the
+# "reverse" and "resolve" arrays.
+#
+# The config should look like this:
+#
+#     filter {
+#       dns {
+#         type => 'type'
+#         reverse => [ "source_host", "field_with_address" ]
+#         resolve => [ "field_with_fqdn" ]
+#         action => "replace"
+#       }
+#     }
+#
+# Caveats: at the moment, there's no way to tune the timeout with the 'resolv'
+# core library.  It does seem to be fixed in here:
+#
+#   http://redmine.ruby-lang.org/issues/5100
+#
+# but isn't currently in JRuby.
+class LogStash::Filters::DNS < LogStash::Filters::Base
+
+  config_name "dns"
+  milestone 2
+
+  # Reverse resolve one or more fields.
+  config :reverse, :validate => :array
+
+  # Forward resolve one or more fields.
+  config :resolve, :validate => :array
+
+  # Determine what action to do: append or replace the values in the fields
+  # specified under "reverse" and "resolve."
+  config :action, :validate => [ "append", "replace" ], :default => "append"
+
+  # Use custom nameserver.
+  config :nameserver, :validate => :string
+
+  # TODO(sissel): make 'action' required? This was always the intent, but it
+  # due to a typo it was never enforced. Thus the default behavior in past
+  # versions was 'append' by accident.
+
+  # resolv calls will be wrapped in a timeout instance
+  config :timeout, :validate => :number, :default => 2
+
+  public
+  def register
+    require "resolv"
+    require "timeout"
+    if @nameserver.nil?
+      @resolv = Resolv.new
+    else
+      @resolv = Resolv.new(resolvers=[::Resolv::Hosts.new, ::Resolv::DNS.new(:nameserver => [@nameserver], :search => [], :ndots => 1)])
+    end
+
+    @ip_validator = Resolv::AddressRegex
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    if @resolve
+      begin
+        status = Timeout::timeout(@timeout) { 
+          resolve(event)
+        }
+      rescue Timeout::Error
+        @logger.debug("DNS: resolve action timed out")
+        return
+      end
+    end
+
+    if @reverse
+      begin
+        status = Timeout::timeout(@timeout) { 
+          reverse(event)
+        }
+      rescue Timeout::Error
+        @logger.debug("DNS: reverse action timed out")
+        return
+      end
+    end
+
+    filter_matched(event)
+  end
+
+  private
+  def resolve(event)
+    @resolve.each do |field|
+      is_array = false
+      raw = event[field]
+      if raw.is_a?(Array)
+        is_array = true
+        if raw.length > 1
+          @logger.warn("DNS: skipping resolve, can't deal with multiple values", :field => field, :value => raw)
+          return
+        end
+        raw = raw.first
+      end
+
+      begin
+        address = @resolv.getaddress(raw)
+      rescue Resolv::ResolvError
+        @logger.debug("DNS: couldn't resolve the hostname.",
+                      :field => field, :value => raw)
+        return
+      rescue Resolv::ResolvTimeout
+        @logger.debug("DNS: timeout on resolving the hostname.",
+                      :field => field, :value => raw)
+        return
+      rescue SocketError => e
+        @logger.debug("DNS: Encountered SocketError.",
+                      :field => field, :value => raw)
+        return
+      rescue NoMethodError => e
+        # see JRUBY-5647
+        @logger.debug("DNS: couldn't resolve the hostname.",
+                      :field => field, :value => raw,
+                      :extra => "NameError instead of ResolvError")
+        return
+      end
+
+      if @action == "replace"
+        if is_array
+          event[field] = [address]
+        else
+          event[field] = address
+        end
+      else
+        if !is_array
+          event[field] = [event[field], address]
+        else
+          event[field] << address
+        end
+      end
+
+    end
+  end
+
+  private
+  def reverse(event)
+    @reverse.each do |field|
+      raw = event[field]
+      is_array = false
+      if raw.is_a?(Array)
+          is_array = true
+          if raw.length > 1
+            @logger.warn("DNS: skipping reverse, can't deal with multiple values", :field => field, :value => raw)
+            return
+          end
+          raw = raw.first
+      end
+
+      if ! @ip_validator.match(raw)
+        @logger.debug("DNS: not an address",
+                      :field => field, :value => event[field])
+        return
+      end
+      begin
+        hostname = @resolv.getname(raw)
+      rescue Resolv::ResolvError
+        @logger.debug("DNS: couldn't resolve the address.",
+                      :field => field, :value => raw)
+        return
+      rescue Resolv::ResolvTimeout
+        @logger.debug("DNS: timeout on resolving address.",
+                      :field => field, :value => raw)
+        return
+      rescue SocketError => e
+        @logger.debug("DNS: Encountered SocketError.",
+                      :field => field, :value => raw)
+        return
+      end
+
+      if @action == "replace"
+        if is_array
+          event[field] = [hostname]
+        else
+          event[field] = hostname
+        end
+      else
+        if !is_array
+          event[field] = [event[field], hostname]
+        else
+          event[field] << hostname
+        end
+      end
+    end
+  end
+end # class LogStash::Filters::DNS

data/lib/logstash/filters/drop.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Drop filter.
+#
+# Drops everything that gets to this filter.
+#
+# This is best used in combination with conditionals, for example:
+#
+#     filter {
+#       if [loglevel] == "debug" { 
+#         drop { } 
+#       }
+#     }
+#
+# The above will only pass events to the drop filter if the loglevel field is
+# "debug". This will cause all events matching to be dropped.
+class LogStash::Filters::Drop < LogStash::Filters::Base
+  config_name "drop"
+  milestone 3
+
+  public
+  def register
+    # nothing to do.
+  end
+
+  public
+  def filter(event)
+    event.cancel
+  end # def filter
+end # class LogStash::Filters::Drop