fluent-plugin-grok-parser 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +31 -0
- data/README.md +63 -0
- data/Rakefile +16 -0
- data/fluent-plugin-grok-parser.gemspec +22 -0
- data/lib/fluent/plugin/parser_grok.rb +87 -0
- data/patterns/grok-patterns +94 -0
- data/patterns/nagios +108 -0
- data/test/test_grok_parser.rb +97 -0
- metadata +96 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: cb4cafd272f76ee654cf6d3b71dcf8f9090d58f1
|
4
|
+
data.tar.gz: 4b314f74bac26038137a979ce8396bc299f745f1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 97235a3241c2b2f536fed27660a2e13ae6f9f01a46fbf05c5cc5db87093d0e60af3ccbef2d73b51ffb007099d9f8299530d45633ac4103bbeb225d268c5cf475
|
7
|
+
data.tar.gz: 86b2052f11b2a00a0a63bdbc53aa43b0d41d16fde1e30fa1424f4b120c5af79e8b601a40aafe2b5824ffb41d89fce30ba014d5ff10b4026a5560719659c6080e
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-grok-parser (0.0.1)
|
5
|
+
fluentd
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
cool.io (1.2.4)
|
11
|
+
fluentd (0.10.49)
|
12
|
+
cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
|
13
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
14
|
+
json (>= 1.4.3)
|
15
|
+
msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
|
16
|
+
sigdump (~> 0.2.2)
|
17
|
+
yajl-ruby (~> 1.0)
|
18
|
+
http_parser.rb (0.6.0)
|
19
|
+
json (1.8.1)
|
20
|
+
msgpack (0.5.8)
|
21
|
+
rake (10.1.1)
|
22
|
+
sigdump (0.2.2)
|
23
|
+
yajl-ruby (1.2.1)
|
24
|
+
|
25
|
+
PLATFORMS
|
26
|
+
ruby
|
27
|
+
|
28
|
+
DEPENDENCIES
|
29
|
+
bundler
|
30
|
+
fluent-plugin-grok-parser!
|
31
|
+
rake
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# Grok Parser for Fluentd
|
2
|
+
|
3
|
+
This is a Fluentd plugin to enable Logstash's Grok-like parsing logic.
|
4
|
+
|
5
|
+
## What's Grok?
|
6
|
+
|
7
|
+
Grok is a macro to simplify and reuse regexes, originally developed by [Jordan Sissel](http://github.com/semicomplete).
|
8
|
+
|
9
|
+
This is a partial implementation of Grok's grammer that should meet most of the needs.
|
10
|
+
|
11
|
+
## How It Works
|
12
|
+
|
13
|
+
You can use it wherever you used the `format` parameter to parse texts. In the following example, it
|
14
|
+
extracts the first IP address that matches in the log.
|
15
|
+
|
16
|
+
```
|
17
|
+
<source>
|
18
|
+
type tail
|
19
|
+
path /path/to/log
|
20
|
+
format grok
|
21
|
+
grok_pattern %{IP:ip_address}
|
22
|
+
</source>
|
23
|
+
```
|
24
|
+
|
25
|
+
## How to write Grok patterns
|
26
|
+
|
27
|
+
Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
|
28
|
+
becomes a named capture. So, for example, if you have the grok pattern
|
29
|
+
|
30
|
+
```
|
31
|
+
%{IP} %{HOST:host}
|
32
|
+
```
|
33
|
+
|
34
|
+
it matches
|
35
|
+
|
36
|
+
```
|
37
|
+
127.0.0.1 foo.example
|
38
|
+
```
|
39
|
+
|
40
|
+
but only extracts "foo.example" as {"host": "foo.example"}
|
41
|
+
|
42
|
+
Please see `patterns/*` for the patterns that are supported out of the box.
|
43
|
+
|
44
|
+
## How to add your own Grok pattern
|
45
|
+
|
46
|
+
You can add your own Grok patterns by creating your own Grok file and telling the plugin to read it.
|
47
|
+
This is what the `custom_pattern_path` parameter is for.
|
48
|
+
|
49
|
+
```
|
50
|
+
<source>
|
51
|
+
type tail
|
52
|
+
path /path/to/log
|
53
|
+
format grok
|
54
|
+
grok_pattern %{MY_SUPER_PATTERN}
|
55
|
+
custom_pattern_path /path/to/my_pattern
|
56
|
+
</source>
|
57
|
+
```
|
58
|
+
|
59
|
+
`custom_pattern_path` can be either a directory or file. If it's a directory, it reads all the files in it.
|
60
|
+
|
61
|
+
## License
|
62
|
+
|
63
|
+
Apache 2.0 License
|
data/Rakefile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rake/clean'
|
5
|
+
|
6
|
+
task :test => [:base_test]
|
7
|
+
|
8
|
+
desc 'Run test_unit based test'
|
9
|
+
Rake::TestTask.new(:base_test) do |t|
|
10
|
+
t.libs << "test"
|
11
|
+
t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
|
12
|
+
t.verbose = true
|
13
|
+
#t.warning = true
|
14
|
+
end
|
15
|
+
|
16
|
+
task :default => [:test, :build]
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "fluent-plugin-grok-parser"
|
7
|
+
spec.version = "0.0.1"
|
8
|
+
spec.authors = ["kiyoto"]
|
9
|
+
spec.email = ["kiyoto@treasure-data.com"]
|
10
|
+
spec.summary = %q{Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs}
|
11
|
+
spec.homepage = "https://github.com/kiyoto/fluent-plugin-grok-parser"
|
12
|
+
spec.license = "Apache License, Version 2.0"
|
13
|
+
|
14
|
+
spec.files = `git ls-files`.split($/)
|
15
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
16
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
|
+
spec.require_paths = ["lib"]
|
18
|
+
|
19
|
+
spec.add_development_dependency "bundler"
|
20
|
+
spec.add_development_dependency "rake"
|
21
|
+
spec.add_runtime_dependency "fluentd"
|
22
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Fluent
|
2
|
+
class TextParser
|
3
|
+
class GrokPatternNotFoundError < Exception; end
|
4
|
+
|
5
|
+
class GrokParser
|
6
|
+
include Configurable
|
7
|
+
config_param :time_format, :string, :default => nil
|
8
|
+
config_param :grok_pattern, :string
|
9
|
+
config_param :custom_pattern_path, :string, :default => nil
|
10
|
+
|
11
|
+
PATTERN_RE = \
|
12
|
+
/%\{ # match '%{' not prefixed with '\'
|
13
|
+
(?<name> # match the pattern name
|
14
|
+
(?<pattern>[A-z0-9]+)
|
15
|
+
(?::(?<subname>[@\[\]A-z0-9_:.-]+))?
|
16
|
+
)
|
17
|
+
\}/x
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
super
|
21
|
+
@pattern_map = {}
|
22
|
+
default_pattern_dir = File.expand_path('../../../../patterns/*', __FILE__)
|
23
|
+
Dir.glob(default_pattern_dir) do |pattern_file_path|
|
24
|
+
add_patterns_from_file(pattern_file_path)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def configure(conf={})
|
29
|
+
super
|
30
|
+
|
31
|
+
if @custom_pattern_path
|
32
|
+
if Dir.exists? @custom_pattern_path
|
33
|
+
Dir.glob(@custom_pattern_path + '/*') do |pattern_file_path|
|
34
|
+
add_patterns_from_file(pattern_file_path)
|
35
|
+
end
|
36
|
+
elsif File.exists? @custom_pattern_path
|
37
|
+
add_patterns_from_file(@custom_pattern_path)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
begin
|
42
|
+
regexp = expand_pattern(conf['grok_pattern'])
|
43
|
+
$log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
|
44
|
+
@parser = RegexpParser.new(Regexp.new(regexp), conf)
|
45
|
+
rescue => e
|
46
|
+
$log.error e.backtrace
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_patterns_from_file(path)
|
51
|
+
File.new(path).each_line do |line|
|
52
|
+
next if line[0] == '#' || /^$/ =~ line
|
53
|
+
name, pat = line.chomp.split(/\s+/, 2)
|
54
|
+
@pattern_map[name] = pat
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def expand_pattern(pattern)
|
59
|
+
# It's okay to modify in place. no need to expand it more than once.
|
60
|
+
while true
|
61
|
+
m = PATTERN_RE.match(pattern)
|
62
|
+
break if not m
|
63
|
+
curr_pattern = @pattern_map[m["pattern"]]
|
64
|
+
raise GrokPatternNotFoundError if not curr_pattern
|
65
|
+
replacement_pattern = if m["subname"]
|
66
|
+
"(?<#{m["subname"]}>#{curr_pattern})"
|
67
|
+
else
|
68
|
+
curr_pattern
|
69
|
+
end
|
70
|
+
pattern.sub!(m[0]) do |s| replacement_pattern end
|
71
|
+
end
|
72
|
+
|
73
|
+
pattern
|
74
|
+
end
|
75
|
+
|
76
|
+
def call(text, &block)
|
77
|
+
if block
|
78
|
+
@parser.call(text, &block)
|
79
|
+
else
|
80
|
+
@parser.call(text)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
TextParser.register_template('grok', Proc.new { GrokParser.new })
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
USERNAME [a-zA-Z0-9._-]+
|
2
|
+
USER %{USERNAME}
|
3
|
+
INT (?:[+-]?(?:[0-9]+))
|
4
|
+
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
|
5
|
+
NUMBER (?:%{BASE10NUM})
|
6
|
+
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
|
7
|
+
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
|
8
|
+
|
9
|
+
POSINT \b(?:[1-9][0-9]*)\b
|
10
|
+
NONNEGINT \b(?:[0-9]+)\b
|
11
|
+
WORD \b\w+\b
|
12
|
+
NOTSPACE \S+
|
13
|
+
SPACE \s*
|
14
|
+
DATA .*?
|
15
|
+
GREEDYDATA .*
|
16
|
+
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
|
17
|
+
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
|
18
|
+
|
19
|
+
# Networking
|
20
|
+
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
|
21
|
+
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
|
22
|
+
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
|
23
|
+
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
|
24
|
+
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
|
25
|
+
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
|
26
|
+
IP (?:%{IPV6}|%{IPV4})
|
27
|
+
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
|
28
|
+
HOST %{HOSTNAME}
|
29
|
+
IPORHOST (?:%{HOSTNAME}|%{IP})
|
30
|
+
HOSTPORT %{IPORHOST}:%{POSINT}
|
31
|
+
|
32
|
+
# paths
|
33
|
+
PATH (?:%{UNIXPATH}|%{WINPATH})
|
34
|
+
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
|
35
|
+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
|
36
|
+
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
|
37
|
+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
|
38
|
+
URIHOST %{IPORHOST}(?::%{POSINT:port})?
|
39
|
+
# uripath comes loosely from RFC1738, but mostly from what Firefox
|
40
|
+
# doesn't turn into %XX
|
41
|
+
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
|
42
|
+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
|
43
|
+
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
|
44
|
+
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
|
45
|
+
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
|
46
|
+
|
47
|
+
# Months: January, Feb, 3, 03, 12, December
|
48
|
+
MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
|
49
|
+
MONTHNUM (?:0?[1-9]|1[0-2])
|
50
|
+
MONTHNUM2 (?:0[1-9]|1[0-2])
|
51
|
+
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
|
52
|
+
|
53
|
+
# Days: Monday, Tue, Thu, etc...
|
54
|
+
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
|
55
|
+
|
56
|
+
# Years?
|
57
|
+
YEAR (?>\d\d){1,2}
|
58
|
+
HOUR (?:2[0123]|[01]?[0-9])
|
59
|
+
MINUTE (?:[0-5][0-9])
|
60
|
+
# '60' is a leap second in most time standards and thus is valid.
|
61
|
+
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
|
62
|
+
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
|
63
|
+
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
|
64
|
+
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
|
65
|
+
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
|
66
|
+
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
|
67
|
+
ISO8601_SECOND (?:%{SECOND}|60)
|
68
|
+
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
|
69
|
+
DATE %{DATE_US}|%{DATE_EU}
|
70
|
+
DATESTAMP %{DATE}[- ]%{TIME}
|
71
|
+
TZ (?:[PMCE][SD]T|UTC)
|
72
|
+
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
|
73
|
+
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
|
74
|
+
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
|
75
|
+
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
|
76
|
+
|
77
|
+
# Syslog Dates: Month Day HH:MM:SS
|
78
|
+
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
|
79
|
+
PROG (?:[\w._/%-]+)
|
80
|
+
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
|
81
|
+
SYSLOGHOST %{IPORHOST}
|
82
|
+
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
|
83
|
+
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
|
84
|
+
|
85
|
+
# Shortcuts
|
86
|
+
QS %{QUOTEDSTRING}
|
87
|
+
|
88
|
+
# Log formats
|
89
|
+
SYSLOGBASE %{SYSLOGTIMESTAMP:time} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
|
90
|
+
COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:time}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
|
91
|
+
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
|
92
|
+
|
93
|
+
# Log Levels
|
94
|
+
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
|
data/patterns/nagios
ADDED
@@ -0,0 +1,108 @@
|
|
1
|
+
##################################################################################
|
2
|
+
##################################################################################
|
3
|
+
# Chop Nagios log files to smithereens!
|
4
|
+
#
|
5
|
+
# A set of GROK filters to process logfiles generated by Nagios.
|
6
|
+
# While it does not, this set intends to cover all possible Nagios logs.
|
7
|
+
#
|
8
|
+
# Some more work needs to be done to cover all External Commands:
|
9
|
+
# http://old.nagios.org/developerinfo/externalcommands/commandlist.php
|
10
|
+
#
|
11
|
+
# If you need some support on these rules please contact:
|
12
|
+
# Jelle Smet http://smetj.net
|
13
|
+
#
|
14
|
+
#################################################################################
|
15
|
+
#################################################################################
|
16
|
+
|
17
|
+
NAGIOSTIME \[%{NUMBER:time}\]
|
18
|
+
|
19
|
+
###############################################
|
20
|
+
######## Begin nagios log types
|
21
|
+
###############################################
|
22
|
+
NAGIOS_TYPE_CURRENT_SERVICE_STATE CURRENT SERVICE STATE
|
23
|
+
NAGIOS_TYPE_CURRENT_HOST_STATE CURRENT HOST STATE
|
24
|
+
|
25
|
+
NAGIOS_TYPE_SERVICE_NOTIFICATION SERVICE NOTIFICATION
|
26
|
+
NAGIOS_TYPE_HOST_NOTIFICATION HOST NOTIFICATION
|
27
|
+
|
28
|
+
NAGIOS_TYPE_SERVICE_ALERT SERVICE ALERT
|
29
|
+
NAGIOS_TYPE_HOST_ALERT HOST ALERT
|
30
|
+
|
31
|
+
NAGIOS_TYPE_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT
|
32
|
+
NAGIOS_TYPE_HOST_FLAPPING_ALERT HOST FLAPPING ALERT
|
33
|
+
|
34
|
+
NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT SERVICE DOWNTIME ALERT
|
35
|
+
NAGIOS_TYPE_HOST_DOWNTIME_ALERT HOST DOWNTIME ALERT
|
36
|
+
|
37
|
+
NAGIOS_TYPE_PASSIVE_SERVICE_CHECK PASSIVE SERVICE CHECK
|
38
|
+
NAGIOS_TYPE_PASSIVE_HOST_CHECK PASSIVE HOST CHECK
|
39
|
+
|
40
|
+
NAGIOS_TYPE_SERVICE_EVENT_HANDLER SERVICE EVENT HANDLER
|
41
|
+
NAGIOS_TYPE_HOST_EVENT_HANDLER HOST EVENT HANDLER
|
42
|
+
|
43
|
+
NAGIOS_TYPE_EXTERNAL_COMMAND EXTERNAL COMMAND
|
44
|
+
NAGIOS_TYPE_TIMEPERIOD_TRANSITION TIMEPERIOD TRANSITION
|
45
|
+
###############################################
|
46
|
+
######## End nagios log types
|
47
|
+
###############################################
|
48
|
+
|
49
|
+
###############################################
|
50
|
+
######## Begin external check types
|
51
|
+
###############################################
|
52
|
+
NAGIOS_EC_DISABLE_SVC_CHECK DISABLE_SVC_CHECK
|
53
|
+
NAGIOS_EC_ENABLE_SVC_CHECK ENABLE_SVC_CHECK
|
54
|
+
NAGIOS_EC_DISABLE_HOST_CHECK DISABLE_HOST_CHECK
|
55
|
+
NAGIOS_EC_ENABLE_HOST_CHECK ENABLE_HOST_CHECK
|
56
|
+
NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT PROCESS_SERVICE_CHECK_RESULT
|
57
|
+
NAGIOS_EC_PROCESS_HOST_CHECK_RESULT PROCESS_HOST_CHECK_RESULT
|
58
|
+
NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME SCHEDULE_SERVICE_DOWNTIME
|
59
|
+
NAGIOS_EC_SCHEDULE_HOST_DOWNTIME SCHEDULE_HOST_DOWNTIME
|
60
|
+
###############################################
|
61
|
+
######## End external check types
|
62
|
+
###############################################
|
63
|
+
NAGIOS_WARNING Warning:%{SPACE}%{GREEDYDATA:nagios_message}
|
64
|
+
|
65
|
+
NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}
|
66
|
+
NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}
|
67
|
+
|
68
|
+
NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
|
69
|
+
NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}
|
70
|
+
|
71
|
+
NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
|
72
|
+
NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}
|
73
|
+
|
74
|
+
NAGIOS_SERVICE_FLAPPING_ALERT %{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}
|
75
|
+
NAGIOS_HOST_FLAPPING_ALERT %{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}
|
76
|
+
|
77
|
+
NAGIOS_SERVICE_DOWNTIME_ALERT %{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
|
78
|
+
NAGIOS_HOST_DOWNTIME_ALERT %{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
|
79
|
+
|
80
|
+
NAGIOS_PASSIVE_SERVICE_CHECK %{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
|
81
|
+
NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}
|
82
|
+
|
83
|
+
NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
|
84
|
+
NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
|
85
|
+
|
86
|
+
NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2};
|
87
|
+
|
88
|
+
####################
|
89
|
+
#### External checks
|
90
|
+
####################
|
91
|
+
|
92
|
+
#Disable host & service check
|
93
|
+
NAGIOS_EC_LINE_DISABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}
|
94
|
+
NAGIOS_EC_LINE_DISABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}
|
95
|
+
|
96
|
+
#Enable host & service check
|
97
|
+
NAGIOS_EC_LINE_ENABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}
|
98
|
+
NAGIOS_EC_LINE_ENABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}
|
99
|
+
|
100
|
+
#Process host & service check
|
101
|
+
NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}
|
102
|
+
NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}
|
103
|
+
|
104
|
+
#Schedule host & service downtime
|
105
|
+
NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment}
|
106
|
+
|
107
|
+
#End matching line
|
108
|
+
NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME})
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/parser'
|
3
|
+
require 'fluent/plugin/parser_grok'
|
4
|
+
require 'tempfile'
|
5
|
+
|
6
|
+
|
7
|
+
include Fluent
|
8
|
+
|
9
|
+
def str2time(str_time, format = nil)
|
10
|
+
if format
|
11
|
+
Time.strptime(str_time, format).to_i
|
12
|
+
else
|
13
|
+
Time.parse(str_time).to_i
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class GrokParserTest < ::Test::Unit::TestCase
|
18
|
+
def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
|
19
|
+
parser = TextParser::GrokParser.new
|
20
|
+
parser.configure({"grok_pattern" => grok_pattern}.merge(options))
|
21
|
+
[parser.call(text), parser.call(text) { |time, record| return time, record}].each { |time, record|
|
22
|
+
assert_equal(expected_time, time) if expected_time
|
23
|
+
assert_equal(expected_record, record)
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_call_for_timestamp
|
28
|
+
internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
|
29
|
+
str2time('2014-01-01T00:00:00+0900'), {})
|
30
|
+
internal_test_grok_pattern('%{DATESTAMP_RFC822:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
|
31
|
+
str2time('Mon Aug 15 2005 15:52:01 UTC'), {})
|
32
|
+
internal_test_grok_pattern('%{DATESTAMP_RFC2822:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
|
33
|
+
str2time('Mon, 15 Aug 2005 15:52:01 +0000'), {})
|
34
|
+
internal_test_grok_pattern('%{SYSLOGTIMESTAMP:time}', 'Some stuff at Aug 01 00:00:00',
|
35
|
+
str2time('Aug 01 00:00:00'), {})
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_call_for_grok_pattern_not_found
|
39
|
+
assert_raise TextParser::GrokPatternNotFoundError do
|
40
|
+
internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_call_for_multiple_fields
|
45
|
+
internal_test_grok_pattern('%{MAC:mac_address} %{IP:ip_address}', 'this.wont.match DEAD.BEEF.1234 127.0.0.1', nil,
|
46
|
+
{"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_call_for_apache_pattern
|
50
|
+
internal_test_grok_pattern('%{COMBINEDAPACHELOG}', '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
|
51
|
+
str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
|
52
|
+
{
|
53
|
+
"clientip" => "127.0.0.1",
|
54
|
+
"ident" => "192.168.0.1",
|
55
|
+
"auth" => "-",
|
56
|
+
"verb" => "GET",
|
57
|
+
"request" => "/",
|
58
|
+
"httpversion" => "1.1",
|
59
|
+
"response" => "200",
|
60
|
+
"bytes" => "777",
|
61
|
+
"referrer" => "\"-\"",
|
62
|
+
"agent" => "\"Opera/12.0\""
|
63
|
+
},
|
64
|
+
"time_format" => "%d/%b/%Y:%H:%M:%S %z"
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_call_for_nagios_pattern
|
69
|
+
internal_test_grok_pattern('%{NAGIOSLOGLINE}', '[1404239939] SERVICE ALERT: servername;PING;OK;SOFT;2;PING OK - Packet loss = 16%, RTA = 72.18 ms',
|
70
|
+
str2time('1404239939', '%s'),
|
71
|
+
{
|
72
|
+
"nagios_message" => "PING OK - Packet loss = 16%, RTA = 72.18 ms",
|
73
|
+
"nagios_type" => "SERVICE ALERT",
|
74
|
+
"nagios_hostname" => "servername",
|
75
|
+
"nagios_service" => "PING",
|
76
|
+
"nagios_state" => "OK",
|
77
|
+
"nagios_statelevel" => "SOFT",
|
78
|
+
"nagios_attempt" => "2"
|
79
|
+
},
|
80
|
+
"time_format" => "%s"
|
81
|
+
)
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_call_for_custom_pattern
|
85
|
+
pattern_file = File.new(File.expand_path("../my_pattern", __FILE__), "w")
|
86
|
+
pattern_file.write("MY_AWESOME_PATTERN %{GREEDYDATA:message}\n")
|
87
|
+
pattern_file.close
|
88
|
+
begin
|
89
|
+
internal_test_grok_pattern('%{MY_AWESOME_PATTERN:message}', 'this is awesome',
|
90
|
+
nil, {"message" => "this is awesome"},
|
91
|
+
"custom_pattern_path" => pattern_file.path
|
92
|
+
)
|
93
|
+
ensure
|
94
|
+
File.delete(pattern_file.path)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
metadata
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-grok-parser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- kiyoto
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-07-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
- kiyoto@treasure-data.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- Gemfile
|
63
|
+
- Gemfile.lock
|
64
|
+
- README.md
|
65
|
+
- Rakefile
|
66
|
+
- fluent-plugin-grok-parser.gemspec
|
67
|
+
- lib/fluent/plugin/parser_grok.rb
|
68
|
+
- patterns/grok-patterns
|
69
|
+
- patterns/nagios
|
70
|
+
- test/test_grok_parser.rb
|
71
|
+
homepage: https://github.com/kiyoto/fluent-plugin-grok-parser
|
72
|
+
licenses:
|
73
|
+
- Apache License, Version 2.0
|
74
|
+
metadata: {}
|
75
|
+
post_install_message:
|
76
|
+
rdoc_options: []
|
77
|
+
require_paths:
|
78
|
+
- lib
|
79
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
requirements: []
|
90
|
+
rubyforge_project:
|
91
|
+
rubygems_version: 2.2.2
|
92
|
+
signing_key:
|
93
|
+
specification_version: 4
|
94
|
+
summary: Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs
|
95
|
+
test_files:
|
96
|
+
- test/test_grok_parser.rb
|