fluent-plugin-grok-parser 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/appveyor.yml +21 -2
- data/fluent-plugin-grok-parser.gemspec +2 -2
- data/lib/fluent/plugin/grok.rb +37 -3
- data/lib/fluent/plugin/parser_grok.rb +3 -0
- data/test/test_grok_parser.rb +28 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ee0a491eb3c582a83f228c34648869c693cf028aba0beed0404f0ffa3dff182
|
4
|
+
data.tar.gz: b421584afadea006497e075a5e198bbd2497654cb446148e1223320172a85aac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0db5d145e6d33b285393c75a8383b93120ba623f99859356831e2b08622ca8d1c12ac37d9597ea5fc123d2997c7d42d6b75f4015e9502eaccc830fe41a1919db
|
7
|
+
data.tar.gz: 55d52fd681a761529947a779ebec12de1b7468e1d95546b086fd34d1dcaba6a4122726a954ddbc2f26cd872ca401ec0d6a4ce1294118b8db4bc48a47df79fade
|
data/appveyor.yml
CHANGED
@@ -1,10 +1,15 @@
|
|
1
1
|
version: '{build}'
|
2
|
+
|
3
|
+
# init:
|
4
|
+
# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
|
5
|
+
|
2
6
|
install:
|
3
7
|
- SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
|
4
|
-
- "%devkit%\\devkitvars.bat"
|
8
|
+
- IF %ridk%==0 "%devkit%\\devkitvars.bat"
|
5
9
|
- ruby --version
|
6
10
|
- gem --version
|
7
|
-
- bundle install
|
11
|
+
- IF %ridk%==0 bundle install
|
12
|
+
- IF %ridk%==1 ridk.cmd exec bundle install
|
8
13
|
build: off
|
9
14
|
test_script:
|
10
15
|
- bundle exec rake test
|
@@ -12,20 +17,34 @@ test_script:
|
|
12
17
|
# https://www.appveyor.com/docs/installed-software/#ruby
|
13
18
|
environment:
|
14
19
|
matrix:
|
20
|
+
- ruby_version: "25-x64"
|
21
|
+
ridk: 1
|
22
|
+
- ruby_version: "25"
|
23
|
+
ridk: 1
|
24
|
+
- ruby_version: "24-x64"
|
25
|
+
ridk: 1
|
26
|
+
- ruby_version: "24"
|
27
|
+
ridk: 1
|
15
28
|
- ruby_version: "23-x64"
|
16
29
|
devkit: C:\Ruby23-x64\DevKit
|
30
|
+
ridk: 0
|
17
31
|
- ruby_version: "23"
|
18
32
|
devkit: C:\Ruby23\DevKit
|
33
|
+
ridk: 0
|
19
34
|
- ruby_version: "22-x64"
|
20
35
|
devkit: C:\Ruby23-x64\DevKit
|
36
|
+
ridk: 0
|
21
37
|
- ruby_version: "21-x64"
|
22
38
|
devkit: C:\Ruby23-x64\DevKit
|
39
|
+
ridk: 0
|
23
40
|
- ruby_version: "22"
|
24
41
|
devkit: C:\Ruby23\DevKit
|
25
42
|
WIN_RAPID: true
|
43
|
+
ridk: 0
|
26
44
|
- ruby_version: "21"
|
27
45
|
devkit: C:\Ruby23\DevKit
|
28
46
|
WIN_RAPID: true
|
47
|
+
ridk: 0
|
29
48
|
matrix:
|
30
49
|
allow_failures:
|
31
50
|
- ruby_version: "21"
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-grok-parser"
|
7
|
-
spec.version = "2.
|
7
|
+
spec.version = "2.4.0"
|
8
8
|
spec.authors = ["kiyoto", "Kenji Okimoto"]
|
9
9
|
spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
|
10
10
|
spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
|
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.add_development_dependency "bundler"
|
20
20
|
spec.add_development_dependency "rake"
|
21
21
|
spec.add_development_dependency "test-unit", ">=3.1.5"
|
22
|
-
spec.add_runtime_dependency "fluentd", ">=0.14.6"
|
22
|
+
spec.add_runtime_dependency "fluentd", ">=0.14.6", "< 2"
|
23
23
|
end
|
data/lib/fluent/plugin/grok.rb
CHANGED
@@ -36,6 +36,9 @@ module Fluent
|
|
36
36
|
if @plugin.respond_to?(:keep_time_key)
|
37
37
|
@keep_time_key = @plugin.keep_time_key
|
38
38
|
end
|
39
|
+
if @plugin.respond_to?(:time_format)
|
40
|
+
@time_format = @plugin.time_format
|
41
|
+
end
|
39
42
|
end
|
40
43
|
|
41
44
|
def add_patterns_from_file(path)
|
@@ -48,10 +51,10 @@ module Fluent
|
|
48
51
|
|
49
52
|
def setup
|
50
53
|
if @plugin.grok_pattern
|
51
|
-
@parsers[:grok_pattern] =
|
54
|
+
@parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf)
|
52
55
|
else
|
53
56
|
@plugin.grok_confs.each.with_index do |grok_conf, index|
|
54
|
-
@parsers[grok_conf.name || index] =
|
57
|
+
@parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf)
|
55
58
|
end
|
56
59
|
end
|
57
60
|
@parsers.reject! do |key, parser|
|
@@ -64,7 +67,7 @@ module Fluent
|
|
64
67
|
|
65
68
|
private
|
66
69
|
|
67
|
-
def
|
70
|
+
def expand_pattern_expression_grok_pattern(grok_pattern, conf)
|
68
71
|
regexp, types = expand_pattern(grok_pattern)
|
69
72
|
$log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
|
70
73
|
_conf = conf.to_h
|
@@ -83,6 +86,37 @@ module Fluent
|
|
83
86
|
nil
|
84
87
|
end
|
85
88
|
|
89
|
+
def expand_pattern_expression_grok_section(conf)
|
90
|
+
regexp, types = expand_pattern(conf.pattern)
|
91
|
+
$log.info "Expanded the pattern #{conf.pattern} into #{regexp}"
|
92
|
+
_conf = conf.to_h
|
93
|
+
unless types.empty?
|
94
|
+
_conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
|
95
|
+
end
|
96
|
+
if conf["multiline"] || @multiline_mode
|
97
|
+
_conf["multiline"] = conf["multiline"] || @multiline_mode
|
98
|
+
end
|
99
|
+
if conf["keep_time_key"] || @keep_time_key
|
100
|
+
_conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key
|
101
|
+
end
|
102
|
+
if conf["time_key"]
|
103
|
+
_conf["time_key"] = conf["time_key"]
|
104
|
+
end
|
105
|
+
if conf["time_format"] || @time_format
|
106
|
+
_conf["time_format"] = conf["time_format"] || @time_format
|
107
|
+
end
|
108
|
+
_conf["expression"] = regexp
|
109
|
+
config = Fluent::Config::Element.new("parse", "", _conf, [])
|
110
|
+
parser = Fluent::Plugin::RegexpParser.new
|
111
|
+
parser.configure(config)
|
112
|
+
parser
|
113
|
+
rescue GrokPatternNotFoundError => e
|
114
|
+
raise e
|
115
|
+
rescue => e
|
116
|
+
$log.error(error: e)
|
117
|
+
nil
|
118
|
+
end
|
119
|
+
|
86
120
|
def expand_pattern(pattern)
|
87
121
|
# It's okay to modify in place. no need to expand it more than once.
|
88
122
|
type_map = {}
|
@@ -22,6 +22,9 @@ module Fluent
|
|
22
22
|
config_param :name, :string, default: nil
|
23
23
|
desc "The pattern of grok"
|
24
24
|
config_param :pattern, :string
|
25
|
+
config_param :keep_time_key, :bool, default: false
|
26
|
+
config_param :time_key, :string, default: "time"
|
27
|
+
config_param :time_format, :string, default: nil
|
25
28
|
end
|
26
29
|
|
27
30
|
def initialize
|
data/test/test_grok_parser.rb
CHANGED
@@ -321,6 +321,34 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
321
321
|
end
|
322
322
|
end
|
323
323
|
|
324
|
+
sub_test_case "grok section" do
|
325
|
+
test "complex pattern" do
|
326
|
+
d = create_driver(%[
|
327
|
+
<grok>
|
328
|
+
pattern %{COMBINEDAPACHELOG}
|
329
|
+
time_key timestamp
|
330
|
+
time_format %d/%b/%Y:%H:%M:%S %z
|
331
|
+
</grok>
|
332
|
+
])
|
333
|
+
expected_record = {
|
334
|
+
"clientip" => "127.0.0.1",
|
335
|
+
"ident" => "192.168.0.1",
|
336
|
+
"auth" => "-",
|
337
|
+
"verb" => "GET",
|
338
|
+
"request" => "/",
|
339
|
+
"httpversion" => "1.1",
|
340
|
+
"response" => "200",
|
341
|
+
"bytes" => "777",
|
342
|
+
"referrer" => "\"-\"",
|
343
|
+
"agent" => "\"Opera/12.0\""
|
344
|
+
}
|
345
|
+
d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
|
346
|
+
assert_equal(expected_record, record)
|
347
|
+
assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
324
352
|
private
|
325
353
|
|
326
354
|
def create_driver(conf)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grok-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kiyoto
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-11-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -60,6 +60,9 @@ dependencies:
|
|
60
60
|
- - ">="
|
61
61
|
- !ruby/object:Gem::Version
|
62
62
|
version: 0.14.6
|
63
|
+
- - "<"
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '2'
|
63
66
|
type: :runtime
|
64
67
|
prerelease: false
|
65
68
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -67,6 +70,9 @@ dependencies:
|
|
67
70
|
- - ">="
|
68
71
|
- !ruby/object:Gem::Version
|
69
72
|
version: 0.14.6
|
73
|
+
- - "<"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2'
|
70
76
|
description:
|
71
77
|
email:
|
72
78
|
- kiyoto@treasure-data.com
|