fluent-plugin-grok-parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cb4cafd272f76ee654cf6d3b71dcf8f9090d58f1
+  data.tar.gz: 4b314f74bac26038137a979ce8396bc299f745f1
+SHA512:
+  metadata.gz: 97235a3241c2b2f536fed27660a2e13ae6f9f01a46fbf05c5cc5db87093d0e60af3ccbef2d73b51ffb007099d9f8299530d45633ac4103bbeb225d268c5cf475
+  data.tar.gz: 86b2052f11b2a00a0a63bdbc53aa43b0d41d16fde1e30fa1424f4b120c5af79e8b601a40aafe2b5824ffb41d89fce30ba014d5ff10b4026a5560719659c6080e

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-grok-parser (0.0.1)
+      fluentd
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.2.4)
+    fluentd (0.10.49)
+      cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      sigdump (~> 0.2.2)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    json (1.8.1)
+    msgpack (0.5.8)
+    rake (10.1.1)
+    sigdump (0.2.2)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  fluent-plugin-grok-parser!
+  rake

data/README.md

@@ -0,0 +1,63 @@
+# Grok Parser for Fluentd
+
+This is a Fluentd plugin to enable Logstash's Grok-like parsing logic.
+
+## What's Grok?
+
+Grok is a macro to simplify and reuse regexes, originally developed by [Jordan Sissel](http://github.com/semicomplete).
+
+This is a partial implementation of Grok's grammer that should meet most of the needs.
+
+## How It Works
+
+You can use it wherever you used the `format` parameter to parse texts. In the following example, it
+extracts the first IP address that matches in the log.
+
+```
+<source>
+  type tail
+  path /path/to/log
+  format grok
+  grok_pattern %{IP:ip_address}
+</source>
+```
+
+## How to write Grok patterns
+
+Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
+becomes a named capture. So, for example, if you have the grok pattern
+
+```
+%{IP} %{HOST:host}
+```
+
+it matches
+
+```
+127.0.0.1 foo.example
+```
+
+but only extracts "foo.example" as {"host": "foo.example"}
+
+Please see `patterns/*` for the patterns that are supported out of the box.
+
+## How to add your own Grok pattern
+
+You can add your own Grok patterns by creating your own Grok file and telling the plugin to read it.
+This is what the `custom_pattern_path` parameter is for.
+
+```
+<source>
+  type tail
+  path /path/to/log
+  format grok
+  grok_pattern %{MY_SUPER_PATTERN}
+  custom_pattern_path /path/to/my_pattern
+</source>
+```
+
+`custom_pattern_path` can be either a directory or file. If it's a directory, it reads all the files in it.
+
+## License
+
+Apache 2.0 License

data/Rakefile

@@ -0,0 +1,16 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'rake/clean'
+
+task :test => [:base_test]
+
+desc 'Run test_unit based test'
+Rake::TestTask.new(:base_test) do |t|
+  t.libs << "test"
+  t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
+  t.verbose = true
+  #t.warning = true
+end
+
+task :default => [:test, :build]

data/fluent-plugin-grok-parser.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-grok-parser"
+  spec.version       = "0.0.1"
+  spec.authors       = ["kiyoto"]
+  spec.email         = ["kiyoto@treasure-data.com"]
+  spec.summary       = %q{Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs}
+  spec.homepage      = "https://github.com/kiyoto/fluent-plugin-grok-parser"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/parser_grok.rb

@@ -0,0 +1,87 @@
+module Fluent
+  class TextParser
+    class GrokPatternNotFoundError < Exception; end
+
+    class GrokParser
+      include Configurable
+      config_param :time_format, :string, :default => nil
+      config_param :grok_pattern, :string
+      config_param :custom_pattern_path, :string, :default => nil
+
+      PATTERN_RE = \
+          /%\{    # match '%{' not prefixed with '\'
+             (?<name>     # match the pattern name
+               (?<pattern>[A-z0-9]+)
+               (?::(?<subname>[@\[\]A-z0-9_:.-]+))?
+             )
+           \}/x
+
+      def initialize
+        super
+        @pattern_map = {}
+        default_pattern_dir = File.expand_path('../../../../patterns/*', __FILE__)
+        Dir.glob(default_pattern_dir) do |pattern_file_path|
+          add_patterns_from_file(pattern_file_path)
+        end
+      end
+
+      def configure(conf={})
+        super
+
+        if @custom_pattern_path
+          if Dir.exists? @custom_pattern_path
+            Dir.glob(@custom_pattern_path + '/*') do |pattern_file_path|
+              add_patterns_from_file(pattern_file_path)
+            end
+          elsif File.exists? @custom_pattern_path
+            add_patterns_from_file(@custom_pattern_path)
+          end
+        end
+
+        begin
+          regexp = expand_pattern(conf['grok_pattern'])
+          $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
+          @parser = RegexpParser.new(Regexp.new(regexp), conf)
+        rescue => e
+          $log.error e.backtrace
+        end
+      end
+
+      def add_patterns_from_file(path)
+        File.new(path).each_line do |line|
+          next if line[0] == '#' || /^$/ =~ line
+          name, pat = line.chomp.split(/\s+/, 2)
+          @pattern_map[name] = pat
+        end
+      end
+
+      def expand_pattern(pattern)
+        # It's okay to modify in place. no need to expand it more than once.
+        while true
+          m = PATTERN_RE.match(pattern)
+          break if not m
+          curr_pattern = @pattern_map[m["pattern"]]
+          raise GrokPatternNotFoundError if not curr_pattern
+          replacement_pattern = if m["subname"]
+                                  "(?<#{m["subname"]}>#{curr_pattern})"
+                                else
+                                  curr_pattern
+                                end
+          pattern.sub!(m[0]) do |s| replacement_pattern end
+        end
+      
+        pattern 
+      end
+
+      def call(text, &block)
+        if block
+          @parser.call(text, &block)
+        else
+          @parser.call(text)
+        end
+      end
+    end
+
+    TextParser.register_template('grok', Proc.new { GrokParser.new })
+  end
+end

data/patterns/grok-patterns

@@ -0,0 +1,94 @@
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+IP (?:%{IPV6}|%{IPV4})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME}
+IPORHOST (?:%{HOSTNAME}|%{IP})
+HOSTPORT %{IPORHOST}:%{POSINT}
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:time} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:time}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+
+# Log Levels
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

data/patterns/nagios

@@ -0,0 +1,108 @@
+##################################################################################
+##################################################################################
+# Chop Nagios log files to smithereens!
+#
+# A set of GROK filters to process logfiles generated by Nagios.
+# While it does not, this set intends to cover all possible Nagios logs.
+#
+# Some more work needs to be done to cover all External Commands:
+#	http://old.nagios.org/developerinfo/externalcommands/commandlist.php
+#
+# If you need some support on these rules please contact:
+#	Jelle Smet http://smetj.net
+#
+#################################################################################
+#################################################################################
+
+NAGIOSTIME \[%{NUMBER:time}\]
+
+###############################################
+######## Begin nagios log types
+###############################################
+NAGIOS_TYPE_CURRENT_SERVICE_STATE CURRENT SERVICE STATE
+NAGIOS_TYPE_CURRENT_HOST_STATE CURRENT HOST STATE
+
+NAGIOS_TYPE_SERVICE_NOTIFICATION SERVICE NOTIFICATION
+NAGIOS_TYPE_HOST_NOTIFICATION HOST NOTIFICATION
+
+NAGIOS_TYPE_SERVICE_ALERT SERVICE ALERT
+NAGIOS_TYPE_HOST_ALERT HOST ALERT
+
+NAGIOS_TYPE_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT
+NAGIOS_TYPE_HOST_FLAPPING_ALERT HOST FLAPPING ALERT
+
+NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT SERVICE DOWNTIME ALERT
+NAGIOS_TYPE_HOST_DOWNTIME_ALERT HOST DOWNTIME ALERT
+
+NAGIOS_TYPE_PASSIVE_SERVICE_CHECK PASSIVE SERVICE CHECK
+NAGIOS_TYPE_PASSIVE_HOST_CHECK PASSIVE HOST CHECK
+
+NAGIOS_TYPE_SERVICE_EVENT_HANDLER SERVICE EVENT HANDLER
+NAGIOS_TYPE_HOST_EVENT_HANDLER HOST EVENT HANDLER
+
+NAGIOS_TYPE_EXTERNAL_COMMAND EXTERNAL COMMAND
+NAGIOS_TYPE_TIMEPERIOD_TRANSITION TIMEPERIOD TRANSITION
+###############################################
+######## End nagios log types
+###############################################
+
+###############################################
+######## Begin external check types
+###############################################
+NAGIOS_EC_DISABLE_SVC_CHECK DISABLE_SVC_CHECK
+NAGIOS_EC_ENABLE_SVC_CHECK ENABLE_SVC_CHECK
+NAGIOS_EC_DISABLE_HOST_CHECK DISABLE_HOST_CHECK
+NAGIOS_EC_ENABLE_HOST_CHECK ENABLE_HOST_CHECK
+NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT PROCESS_SERVICE_CHECK_RESULT
+NAGIOS_EC_PROCESS_HOST_CHECK_RESULT PROCESS_HOST_CHECK_RESULT
+NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME SCHEDULE_SERVICE_DOWNTIME
+NAGIOS_EC_SCHEDULE_HOST_DOWNTIME SCHEDULE_HOST_DOWNTIME
+###############################################
+######## End external check types
+###############################################
+NAGIOS_WARNING Warning:%{SPACE}%{GREEDYDATA:nagios_message}
+
+NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}
+NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}
+
+NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
+NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
+
+NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
+NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
+
+NAGIOS_SERVICE_FLAPPING_ALERT %{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}
+NAGIOS_HOST_FLAPPING_ALERT %{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}
+
+NAGIOS_SERVICE_DOWNTIME_ALERT %{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
+NAGIOS_HOST_DOWNTIME_ALERT %{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
+
+NAGIOS_PASSIVE_SERVICE_CHECK %{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
+NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
+
+NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
+NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
+
+NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2};
+
+####################
+#### External checks
+####################
+
+#Disable host & service check
+NAGIOS_EC_LINE_DISABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}
+NAGIOS_EC_LINE_DISABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}
+
+#Enable host & service check
+NAGIOS_EC_LINE_ENABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}
+NAGIOS_EC_LINE_ENABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}
+
+#Process host & service check
+NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}
+NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}
+
+#Schedule host & service downtime
+NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment}
+
+#End matching line
+NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME})

data/test/test_grok_parser.rb

@@ -0,0 +1,97 @@
+require 'fluent/test'
+require 'fluent/parser'
+require 'fluent/plugin/parser_grok'
+require 'tempfile'
+
+
+include Fluent
+
+def str2time(str_time, format = nil)
+  if format
+    Time.strptime(str_time, format).to_i
+  else
+    Time.parse(str_time).to_i
+  end
+end
+
+class GrokParserTest < ::Test::Unit::TestCase
+  def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
+    parser = TextParser::GrokParser.new
+    parser.configure({"grok_pattern" => grok_pattern}.merge(options))
+    [parser.call(text), parser.call(text) { |time, record| return time, record}].each { |time, record|
+      assert_equal(expected_time, time) if expected_time
+      assert_equal(expected_record, record)
+    }
+  end
+
+  def test_call_for_timestamp
+    internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
+                               str2time('2014-01-01T00:00:00+0900'), {})
+    internal_test_grok_pattern('%{DATESTAMP_RFC822:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
+                               str2time('Mon Aug 15 2005 15:52:01 UTC'), {})
+    internal_test_grok_pattern('%{DATESTAMP_RFC2822:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
+                               str2time('Mon, 15 Aug 2005 15:52:01 +0000'), {})
+    internal_test_grok_pattern('%{SYSLOGTIMESTAMP:time}', 'Some stuff at Aug 01 00:00:00',
+                               str2time('Aug 01 00:00:00'), {})
+  end
+
+  def test_call_for_grok_pattern_not_found
+    assert_raise TextParser::GrokPatternNotFoundError do
+      internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})
+    end
+  end
+
+  def test_call_for_multiple_fields
+    internal_test_grok_pattern('%{MAC:mac_address} %{IP:ip_address}', 'this.wont.match DEAD.BEEF.1234 127.0.0.1', nil,
+                               {"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
+  end
+
+  def test_call_for_apache_pattern
+    internal_test_grok_pattern('%{COMBINEDAPACHELOG}', '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
+                                str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
+                                {
+                                  "clientip"    => "127.0.0.1",
+                                  "ident"       => "192.168.0.1",
+                                  "auth"        => "-",
+                                  "verb"        => "GET",
+                                  "request"     => "/",
+                                  "httpversion" => "1.1",
+                                  "response"    => "200",
+                                  "bytes"       => "777",
+                                  "referrer"    => "\"-\"",
+                                  "agent"       => "\"Opera/12.0\""
+                                },
+                                "time_format" => "%d/%b/%Y:%H:%M:%S %z"
+                              )
+  end
+
+  def test_call_for_nagios_pattern
+    internal_test_grok_pattern('%{NAGIOSLOGLINE}', '[1404239939] SERVICE ALERT: servername;PING;OK;SOFT;2;PING OK - Packet loss = 16%, RTA = 72.18 ms',
+                               str2time('1404239939', '%s'),
+                               {
+                                 "nagios_message" => "PING OK - Packet loss = 16%, RTA = 72.18 ms",
+                                 "nagios_type" => "SERVICE ALERT",
+                                 "nagios_hostname" => "servername",
+                                 "nagios_service" => "PING",
+                                 "nagios_state" => "OK",
+                                 "nagios_statelevel" => "SOFT",
+                                 "nagios_attempt" => "2"
+                               },
+                               "time_format" => "%s"
+                              )
+  end
+
+  def test_call_for_custom_pattern
+    pattern_file = File.new(File.expand_path("../my_pattern", __FILE__), "w")
+    pattern_file.write("MY_AWESOME_PATTERN %{GREEDYDATA:message}\n")
+    pattern_file.close
+    begin
+      internal_test_grok_pattern('%{MY_AWESOME_PATTERN:message}', 'this is awesome',
+                                 nil, {"message" => "this is awesome"},
+                                 "custom_pattern_path" => pattern_file.path
+                                )
+    ensure
+      File.delete(pattern_file.path)
+    end
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-grok-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kiyoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- kiyoto@treasure-data.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- fluent-plugin-grok-parser.gemspec
+- lib/fluent/plugin/parser_grok.rb
+- patterns/grok-patterns
+- patterns/nagios
+- test/test_grok_parser.rb
+homepage: https://github.com/kiyoto/fluent-plugin-grok-parser
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs
+test_files:
+- test/test_grok_parser.rb