fluent-plugin-grok-parser 2.1.6 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -4
- data/README.md +44 -85
- data/fluent-plugin-grok-parser.gemspec +1 -1
- data/lib/fluent/plugin/grok.rb +16 -13
- data/lib/fluent/plugin/parser_grok.rb +11 -1
- data/lib/fluent/plugin/parser_multiline_grok.rb +2 -1
- data/test/test_grok_parser.rb +136 -26
- data/test/test_grok_parser_in_tcp.rb +6 -7
- data/test/test_multiline_grok_parser.rb +5 -5
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7652c7a0665b1f6d46aa18bd2e1b373d45facfebe5d0343ab83e34fda23b74f
|
4
|
+
data.tar.gz: 9a124bd9fedd164691d1eedc434d3f29dbfcddc8fe0dba94cdc5babe05185194
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60b36ca6f266b804b95f2e325c16a3e2ff003d75f5b49826b5bb12fc4160b32dccf7b540b9741b5a9a58a8996afbb7dcdad3d66679369ba254a27a8e4e73a6bc
|
7
|
+
data.tar.gz: c45fa99fcb9cd319b6d9d7956d7db94a0c2f40ed8e0919d7807cec51139c7fe845fdbf0c852778735ff45ed2f74bc5b78c2d59c81eef6951f4718fa38c6afc38
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -6,8 +6,8 @@ This is a Fluentd plugin to enable Logstash's Grok-like parsing logic.
|
|
6
6
|
|
7
7
|
| fluent-plugin-grok-parser | fluentd | ruby |
|
8
8
|
|---------------------------|------------|--------|
|
9
|
-
| >=
|
10
|
-
| <
|
9
|
+
| >= 2.0.0 | >= v0.14.0 | >= 2.1 |
|
10
|
+
| < 2.0.0 | >= v0.12.0 | >= 1.9 |
|
11
11
|
|
12
12
|
|
13
13
|
## What's Grok?
|
@@ -33,18 +33,6 @@ extracts the first IP address that matches in the log.
|
|
33
33
|
</source>
|
34
34
|
```
|
35
35
|
|
36
|
-
For Fluentd v0.12, use following style:
|
37
|
-
|
38
|
-
```aconf
|
39
|
-
<source>
|
40
|
-
@type tail
|
41
|
-
path /path/to/log
|
42
|
-
tag grokked_log
|
43
|
-
format grok
|
44
|
-
grok_pattern %{IP:ip_address}
|
45
|
-
</source>
|
46
|
-
```
|
47
|
-
|
48
36
|
**If you want to try multiple grok patterns and use the first matched one**, you can use the following syntax:
|
49
37
|
|
50
38
|
```aconf
|
@@ -68,27 +56,6 @@ For Fluentd v0.12, use following style:
|
|
68
56
|
</source>
|
69
57
|
```
|
70
58
|
|
71
|
-
For Fluentd v0.12, use following style:
|
72
|
-
|
73
|
-
```aconf
|
74
|
-
<source>
|
75
|
-
@type tail
|
76
|
-
path /path/to/log
|
77
|
-
tag grokked_log
|
78
|
-
format grok
|
79
|
-
<grok>
|
80
|
-
pattern %{COMBINEDAPACHELOG}
|
81
|
-
time_format "%d/%b/%Y:%H:%M:%S %z"
|
82
|
-
</grok>
|
83
|
-
<grok>
|
84
|
-
pattern %{IP:ip_address}
|
85
|
-
</grok>
|
86
|
-
<grok>
|
87
|
-
pattern %{GREEDYDATA:message}
|
88
|
-
</grok>
|
89
|
-
</source>
|
90
|
-
```
|
91
|
-
|
92
59
|
### Multiline support
|
93
60
|
|
94
61
|
You can parse multiple line text.
|
@@ -106,19 +73,6 @@ You can parse multiple line text.
|
|
106
73
|
</source>
|
107
74
|
```
|
108
75
|
|
109
|
-
For Fluentd v0.12, use following style:
|
110
|
-
|
111
|
-
```aconf
|
112
|
-
<source>
|
113
|
-
@type tail
|
114
|
-
path /path/to/log
|
115
|
-
format multiline_grok
|
116
|
-
grok_pattern %{IP:ip_address}%{GREEDYDATA:message}
|
117
|
-
multiline_start_regexp /^[^\s]/
|
118
|
-
tag grokked_log
|
119
|
-
</source>
|
120
|
-
```
|
121
|
-
|
122
76
|
You can use multiple grok patterns to parse your data.
|
123
77
|
|
124
78
|
```aconf
|
@@ -135,41 +89,25 @@ You can use multiple grok patterns to parse your data.
|
|
135
89
|
</source>
|
136
90
|
```
|
137
91
|
|
138
|
-
For Fluentd v0.12, use following style:
|
139
|
-
|
140
|
-
```aconf
|
141
|
-
<source>
|
142
|
-
@type tail
|
143
|
-
path /path/to/log
|
144
|
-
format multiline_grok
|
145
|
-
<grok>
|
146
|
-
pattern Started %{WORD:verb} "%{URIPATH:pathinfo}" for %{IP:ip} at %{TIMESTAMP_ISO8601:timestamp}\nProcessing by %{WORD:controller}#%{WORD:action} as %{WORD:format}%{DATA:message}Completed %{NUMBER:response} %{WORD} in %{NUMBER:elapsed} (%{DATA:elapsed_details})
|
147
|
-
</grok>
|
148
|
-
tag grokked_log
|
149
|
-
</source>
|
150
|
-
```
|
151
|
-
|
152
92
|
Fluentd accumulates data in the buffer forever to parse complete data when no pattern matches.
|
153
93
|
|
154
94
|
You can use this parser without `multiline_start_regexp` when you know your data structure perfectly.
|
155
95
|
|
156
96
|
## Configurations
|
157
97
|
|
158
|
-
|
159
|
-
|
160
|
-
The format of the time field.
|
161
|
-
|
162
|
-
**grok_pattern**
|
98
|
+
* See also: [TimeParameters Plugin Overview](https://docs.fluentd.org/v1.0/articles/timeparameters-plugin-overview)
|
99
|
+
* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
|
163
100
|
|
164
|
-
|
101
|
+
* **time_format** (string) (optional): The format of the time field.
|
102
|
+
* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
|
103
|
+
* **custom_pattern_path** (string) (optional): Path to the file that includes custom grok patterns
|
104
|
+
* **grok_failure_key** (string) (optional): The key has grok failure reason.
|
105
|
+
* **grok_name_key** (string) (optional): The key name to store grok section's name
|
106
|
+
* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
|
165
107
|
|
166
|
-
|
108
|
+
## Examples
|
167
109
|
|
168
|
-
|
169
|
-
|
170
|
-
**grok_failure_key**
|
171
|
-
|
172
|
-
The key has grok failure reason. Default is `nil`.
|
110
|
+
### Using grok\_failure\_key
|
173
111
|
|
174
112
|
```aconf
|
175
113
|
<source>
|
@@ -209,21 +147,42 @@ This generates following events:
|
|
209
147
|
2016-11-28 13:07:09.010400923 +0900 dummy.log: {"message1":"/","prog":"bar","path":"/"}
|
210
148
|
```
|
211
149
|
|
212
|
-
|
213
|
-
**grok/pattern**
|
214
|
-
|
215
|
-
Section for grok patterns. You can use multiple grok patterns with
|
216
|
-
multiple `<grok>` sections.
|
150
|
+
### Using grok\_name\_key
|
217
151
|
|
218
152
|
```aconf
|
219
|
-
<
|
220
|
-
|
221
|
-
|
153
|
+
<source>
|
154
|
+
@type tail
|
155
|
+
path /path/to/log
|
156
|
+
tag grokked_log
|
157
|
+
grok_name_key grok_name
|
158
|
+
grok_failure_key grokfailure
|
159
|
+
<parse>
|
160
|
+
@type grok
|
161
|
+
<grok>
|
162
|
+
name apache_log
|
163
|
+
pattern %{COMBINEDAPACHELOG}
|
164
|
+
time_format "%d/%b/%Y:%H:%M:%S %z"
|
165
|
+
</grok>
|
166
|
+
<grok>
|
167
|
+
name ip_address
|
168
|
+
pattern %{IP:ip_address}
|
169
|
+
</grok>
|
170
|
+
<grok>
|
171
|
+
name rest_message
|
172
|
+
pattern %{GREEDYDATA:message}
|
173
|
+
</grok>
|
174
|
+
</parse>
|
175
|
+
</source>
|
222
176
|
```
|
223
177
|
|
224
|
-
|
178
|
+
This will add keys like following:
|
179
|
+
|
180
|
+
* Add `grok_name: "apache_log"` if the record matches `COMBINEDAPACHELOG`
|
181
|
+
* Add `grok_name: "ip_address"` if the record matches `IP`
|
182
|
+
* Add `grok_name: "rest_message"` if the record matches `GREEDYDATA`
|
225
183
|
|
226
|
-
|
184
|
+
Add `grokfailure` key to the record if the record does not match any grok pattern.
|
185
|
+
See also test code for more details.
|
227
186
|
|
228
187
|
## How to write Grok patterns
|
229
188
|
|
@@ -310,7 +269,7 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
|
|
310
269
|
|
311
270
|
## Notice
|
312
271
|
|
313
|
-
If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.
|
272
|
+
If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.x.
|
314
273
|
|
315
274
|
See also: [Plugin Management | Fluentd](http://docs.fluentd.org/articles/plugin-management#plugin-version-management)
|
316
275
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-grok-parser"
|
7
|
-
spec.version = "2.
|
7
|
+
spec.version = "2.2.0"
|
8
8
|
spec.authors = ["kiyoto", "Kenji Okimoto"]
|
9
9
|
spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
|
10
10
|
spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
|
data/lib/fluent/plugin/grok.rb
CHANGED
@@ -23,14 +23,15 @@ module Fluent
|
|
23
23
|
|
24
24
|
def initialize(plugin, conf)
|
25
25
|
@pattern_map = {}
|
26
|
-
@parsers =
|
26
|
+
@parsers = {}
|
27
27
|
@multiline_mode = false
|
28
28
|
@conf = conf
|
29
|
-
|
29
|
+
@plugin = plugin
|
30
|
+
if @plugin.respond_to?(:firstline?)
|
30
31
|
@multiline_mode = true
|
31
32
|
end
|
32
|
-
if @
|
33
|
-
@multiline_start_regexp = Regexp.compile(@
|
33
|
+
if @plugin.respond_to?(:multiline_start_regexp) && @plugin.multiline_start_regexp
|
34
|
+
@multiline_start_regexp = Regexp.compile(@plugin.multiline_start_regexp[1..-2])
|
34
35
|
end
|
35
36
|
end
|
36
37
|
|
@@ -43,15 +44,16 @@ module Fluent
|
|
43
44
|
end
|
44
45
|
|
45
46
|
def setup
|
46
|
-
if @
|
47
|
-
@parsers
|
47
|
+
if @plugin.grok_pattern
|
48
|
+
@parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
|
48
49
|
else
|
49
|
-
|
50
|
-
|
51
|
-
@parsers << expand_pattern_expression(grok_conf["pattern"], grok_conf)
|
50
|
+
@plugin.grok_confs.each.with_index do |grok_conf, index|
|
51
|
+
@parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
|
52
52
|
end
|
53
53
|
end
|
54
|
-
@parsers.
|
54
|
+
@parsers.reject! do |key, parser|
|
55
|
+
parser.nil?
|
56
|
+
end
|
55
57
|
if @parsers.empty?
|
56
58
|
raise Fluent::ConfigError, 'no grok patterns. Check configuration, e.g. typo, configuration syntax, etc'
|
57
59
|
end
|
@@ -61,11 +63,12 @@ module Fluent
|
|
61
63
|
|
62
64
|
def expand_pattern_expression(grok_pattern, conf)
|
63
65
|
regexp, types = expand_pattern(grok_pattern)
|
64
|
-
$log.info "Expanded the pattern #{
|
66
|
+
$log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
|
67
|
+
_conf = conf.to_h
|
65
68
|
unless types.empty?
|
66
|
-
|
69
|
+
_conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
|
67
70
|
end
|
68
|
-
_conf =
|
71
|
+
_conf = _conf.merge("expression" => regexp, "multiline" => @multiline_mode)
|
69
72
|
config = Fluent::Config::Element.new("parse", nil, _conf, [])
|
70
73
|
parser = Fluent::Plugin::RegexpParser.new
|
71
74
|
parser.configure(config)
|
@@ -14,6 +14,15 @@ module Fluent
|
|
14
14
|
config_param :custom_pattern_path, :string, default: nil
|
15
15
|
desc "The key has grok failure reason"
|
16
16
|
config_param :grok_failure_key, :string, default: nil
|
17
|
+
desc "The key name to store grok section's name"
|
18
|
+
config_param :grok_name_key, :string, default: nil
|
19
|
+
|
20
|
+
config_section :grok, param_name: "grok_confs", multi: true do
|
21
|
+
desc "The name of this grok section"
|
22
|
+
config_param :name, :string, default: nil
|
23
|
+
desc "The pattern of grok"
|
24
|
+
config_param :pattern, :string
|
25
|
+
end
|
17
26
|
|
18
27
|
def initialize
|
19
28
|
super
|
@@ -44,9 +53,10 @@ module Fluent
|
|
44
53
|
end
|
45
54
|
|
46
55
|
def parse(text)
|
47
|
-
@grok.parsers.each do |parser|
|
56
|
+
@grok.parsers.each do |name_or_index, parser|
|
48
57
|
parser.parse(text) do |time, record|
|
49
58
|
if time and record
|
59
|
+
record[@grok_name_key] = name_or_index if @grok_name_key
|
50
60
|
yield time, record
|
51
61
|
return
|
52
62
|
end
|
@@ -17,9 +17,10 @@ module Fluent
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def parse(text)
|
20
|
-
@grok.parsers.each do |parser|
|
20
|
+
@grok.parsers.each do |name_or_index, parser|
|
21
21
|
parser.parse(text) do |time, record|
|
22
22
|
if time and record
|
23
|
+
record[@grok_name_key] = name_or_index if @grok_name_key
|
23
24
|
yield time, record
|
24
25
|
return
|
25
26
|
end
|
data/test/test_grok_parser.rb
CHANGED
@@ -11,49 +11,49 @@ def str2time(str_time, format = nil)
|
|
11
11
|
end
|
12
12
|
|
13
13
|
class GrokParserTest < ::Test::Unit::TestCase
|
14
|
-
|
14
|
+
setup do
|
15
15
|
Fluent::Test.setup
|
16
16
|
end
|
17
17
|
|
18
|
-
|
19
|
-
|
18
|
+
sub_test_case "timestamp" do
|
19
|
+
test "timestamp iso8601" do
|
20
20
|
internal_test_grok_pattern("%{TIMESTAMP_ISO8601:time}", "Some stuff at 2014-01-01T00:00:00+0900",
|
21
21
|
event_time("2014-01-01T00:00:00+0900"), {})
|
22
22
|
end
|
23
23
|
|
24
|
-
|
24
|
+
test "datestamp rfc822 with zone" do
|
25
25
|
internal_test_grok_pattern("%{DATESTAMP_RFC822:time}", "Some stuff at Mon Aug 15 2005 15:52:01 UTC",
|
26
26
|
event_time("Mon Aug 15 2005 15:52:01 UTC"), {})
|
27
27
|
end
|
28
28
|
|
29
|
-
|
29
|
+
test "datestamp rfc822 with numeric zone" do
|
30
30
|
internal_test_grok_pattern("%{DATESTAMP_RFC2822:time}", "Some stuff at Mon, 15 Aug 2005 15:52:01 +0000",
|
31
31
|
event_time("Mon, 15 Aug 2005 15:52:01 +0000"), {})
|
32
32
|
end
|
33
33
|
|
34
|
-
|
34
|
+
test "syslogtimestamp" do
|
35
35
|
internal_test_grok_pattern("%{SYSLOGTIMESTAMP:time}", "Some stuff at Aug 01 00:00:00",
|
36
36
|
event_time("Aug 01 00:00:00"), {})
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
|
40
|
+
test "date" do
|
41
41
|
internal_test_grok_pattern("\\[(?<date>%{DATE} %{TIME} (?:AM|PM))\\]", "[2/16/2018 10:19:34 AM]",
|
42
42
|
nil, { "date" => "2/16/2018 10:19:34 AM" })
|
43
43
|
end
|
44
44
|
|
45
|
-
|
45
|
+
test "grok pattern not found" do
|
46
46
|
assert_raise Fluent::Grok::GrokPatternNotFoundError do
|
47
47
|
internal_test_grok_pattern("%{THIS_PATTERN_DOESNT_EXIST}", "Some stuff at somewhere", nil, {})
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
-
|
51
|
+
test "multiple fields" do
|
52
52
|
internal_test_grok_pattern("%{MAC:mac_address} %{IP:ip_address}", "this.wont.match DEAD.BEEF.1234 127.0.0.1", nil,
|
53
53
|
{"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
|
54
54
|
end
|
55
55
|
|
56
|
-
|
56
|
+
test "complex pattern" do
|
57
57
|
internal_test_grok_pattern("%{COMBINEDAPACHELOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
|
58
58
|
str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
|
59
59
|
{
|
@@ -73,59 +73,59 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
73
73
|
)
|
74
74
|
end
|
75
75
|
|
76
|
-
|
76
|
+
test "custom pattern" do
|
77
77
|
internal_test_grok_pattern("%{MY_AWESOME_PATTERN:message}", "this is awesome",
|
78
78
|
nil, {"message" => "this is awesome"},
|
79
79
|
"custom_pattern_path" => fixtures("my_pattern").to_s)
|
80
80
|
end
|
81
81
|
|
82
|
-
|
83
|
-
|
82
|
+
sub_test_case "OptionalType" do
|
83
|
+
test "simple" do
|
84
84
|
internal_test_grok_pattern("%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}",
|
85
85
|
"12345 paid 6789.10", nil,
|
86
86
|
{"user_id" => 12345, "paid_amount" => 6789.1 })
|
87
87
|
end
|
88
88
|
|
89
|
-
|
89
|
+
test "array" do
|
90
90
|
internal_test_grok_pattern("%{GREEDYDATA:message:array}",
|
91
91
|
"a,b,c,d", nil,
|
92
92
|
{"message" => %w(a b c d)})
|
93
93
|
end
|
94
94
|
|
95
|
-
|
95
|
+
test "array with delimiter" do
|
96
96
|
internal_test_grok_pattern("%{GREEDYDATA:message:array:|}",
|
97
97
|
"a|b|c|d", nil,
|
98
98
|
{"message" => %w(a b c d)})
|
99
99
|
end
|
100
100
|
|
101
|
-
|
101
|
+
test "timestamp iso8601" do
|
102
102
|
internal_test_grok_pattern("%{TIMESTAMP_ISO8601:stamp:time}", "Some stuff at 2014-01-01T00:00:00+0900",
|
103
103
|
nil, {"stamp" => event_time("2014-01-01T00:00:00+0900")})
|
104
104
|
end
|
105
105
|
|
106
|
-
|
106
|
+
test "datestamp rfc822 with zone" do
|
107
107
|
internal_test_grok_pattern("%{DATESTAMP_RFC822:stamp:time}", "Some stuff at Mon Aug 15 2005 15:52:01 UTC",
|
108
108
|
nil, {"stamp" => event_time("Mon Aug 15 2005 15:52:01 UTC")})
|
109
109
|
end
|
110
110
|
|
111
|
-
|
111
|
+
test "datestamp rfc822 with numeric zone" do
|
112
112
|
internal_test_grok_pattern("%{DATESTAMP_RFC2822:stamp:time}", "Some stuff at Mon, 15 Aug 2005 15:52:01 +0000",
|
113
113
|
nil, {"stamp" => event_time("Mon, 15 Aug 2005 15:52:01 +0000")})
|
114
114
|
end
|
115
115
|
|
116
|
-
|
116
|
+
test "syslogtimestamp" do
|
117
117
|
internal_test_grok_pattern("%{SYSLOGTIMESTAMP:stamp:time}", "Some stuff at Aug 01 00:00:00",
|
118
118
|
nil, {"stamp" => event_time("Aug 01 00:00:00")})
|
119
119
|
end
|
120
120
|
|
121
|
-
|
121
|
+
test "timestamp with format" do
|
122
122
|
internal_test_grok_pattern("%{TIMESTAMP_ISO8601:stamp:time:%Y-%m-%d %H%M}", "Some stuff at 2014-01-01 1000",
|
123
123
|
nil, {"stamp" => event_time("2014-01-01 10:00")})
|
124
124
|
end
|
125
125
|
end
|
126
126
|
|
127
|
-
|
128
|
-
|
127
|
+
sub_test_case "NoGrokPatternMatched" do
|
128
|
+
test "with grok_failure_key" do
|
129
129
|
config = %[
|
130
130
|
grok_failure_key grok_failure
|
131
131
|
<grok>
|
@@ -142,7 +142,7 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
|
145
|
+
test "without grok_failure_key" do
|
146
146
|
config = %[
|
147
147
|
<grok>
|
148
148
|
pattern %{PATH:path}
|
@@ -158,13 +158,13 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
158
158
|
end
|
159
159
|
end
|
160
160
|
|
161
|
-
|
161
|
+
test "no grok patterns" do
|
162
162
|
assert_raise Fluent::ConfigError do
|
163
163
|
create_driver('')
|
164
164
|
end
|
165
165
|
end
|
166
166
|
|
167
|
-
|
167
|
+
test "invalid config value type" do
|
168
168
|
assert_raise Fluent::ConfigError do
|
169
169
|
create_driver(%[
|
170
170
|
<grok>
|
@@ -174,7 +174,7 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
174
174
|
end
|
175
175
|
end
|
176
176
|
|
177
|
-
|
177
|
+
test "invalid config value type and normal grok pattern" do
|
178
178
|
d = create_driver(%[
|
179
179
|
<grok>
|
180
180
|
pattern %{PATH:path:foo}
|
@@ -191,6 +191,116 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
191
191
|
assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
|
192
192
|
end
|
193
193
|
|
194
|
+
sub_test_case "grok_name_key" do
|
195
|
+
test "one grok section with name" do
|
196
|
+
d = create_driver(%[
|
197
|
+
grok_name_key grok_name
|
198
|
+
<grok>
|
199
|
+
name path
|
200
|
+
pattern %{PATH:path}
|
201
|
+
</grok>
|
202
|
+
])
|
203
|
+
expected = {
|
204
|
+
"path" => "/",
|
205
|
+
"grok_name" => "path"
|
206
|
+
}
|
207
|
+
d.instance.parse("/") do |time, record|
|
208
|
+
assert_equal(expected, record)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
test "one grok section without name" do
|
213
|
+
d = create_driver(%[
|
214
|
+
grok_name_key grok_name
|
215
|
+
<grok>
|
216
|
+
pattern %{PATH:path}
|
217
|
+
</grok>
|
218
|
+
])
|
219
|
+
expected = {
|
220
|
+
"path" => "/",
|
221
|
+
"grok_name" => 0
|
222
|
+
}
|
223
|
+
d.instance.parse("/") do |time, record|
|
224
|
+
assert_equal(expected, record)
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
test "multiple grok sections with name" do
|
229
|
+
d = create_driver(%[
|
230
|
+
grok_name_key grok_name
|
231
|
+
<grok>
|
232
|
+
name path
|
233
|
+
pattern %{PATH:path}
|
234
|
+
</grok>
|
235
|
+
<grok>
|
236
|
+
name ip
|
237
|
+
pattern %{IP:ip_address}
|
238
|
+
</grok>
|
239
|
+
])
|
240
|
+
expected = [
|
241
|
+
{ "path" => "/", "grok_name" => "path" },
|
242
|
+
{ "ip_address" => "127.0.0.1", "grok_name" => "ip" },
|
243
|
+
]
|
244
|
+
records = []
|
245
|
+
d.instance.parse("/") do |time, record|
|
246
|
+
records << record
|
247
|
+
end
|
248
|
+
d.instance.parse("127.0.0.1") do |time, record|
|
249
|
+
records << record
|
250
|
+
end
|
251
|
+
assert_equal(expected, records)
|
252
|
+
end
|
253
|
+
|
254
|
+
test "multiple grok sections without name" do
|
255
|
+
d = create_driver(%[
|
256
|
+
grok_name_key grok_name
|
257
|
+
<grok>
|
258
|
+
pattern %{PATH:path}
|
259
|
+
</grok>
|
260
|
+
<grok>
|
261
|
+
pattern %{IP:ip_address}
|
262
|
+
</grok>
|
263
|
+
])
|
264
|
+
expected = [
|
265
|
+
{ "path" => "/", "grok_name" => 0 },
|
266
|
+
{ "ip_address" => "127.0.0.1", "grok_name" => 1 },
|
267
|
+
]
|
268
|
+
records = []
|
269
|
+
d.instance.parse("/") do |time, record|
|
270
|
+
records << record
|
271
|
+
end
|
272
|
+
d.instance.parse("127.0.0.1") do |time, record|
|
273
|
+
records << record
|
274
|
+
end
|
275
|
+
assert_equal(expected, records)
|
276
|
+
end
|
277
|
+
|
278
|
+
test "multiple grok sections with both name and index" do
|
279
|
+
d = create_driver(%[
|
280
|
+
grok_name_key grok_name
|
281
|
+
<grok>
|
282
|
+
name path
|
283
|
+
pattern %{PATH:path}
|
284
|
+
</grok>
|
285
|
+
<grok>
|
286
|
+
pattern %{IP:ip_address}
|
287
|
+
</grok>
|
288
|
+
])
|
289
|
+
expected = [
|
290
|
+
{ "path" => "/", "grok_name" => "path" },
|
291
|
+
{ "ip_address" => "127.0.0.1", "grok_name" => 1 },
|
292
|
+
]
|
293
|
+
records = []
|
294
|
+
d.instance.parse("/") do |time, record|
|
295
|
+
records << record
|
296
|
+
end
|
297
|
+
d.instance.parse("127.0.0.1") do |time, record|
|
298
|
+
records << record
|
299
|
+
end
|
300
|
+
assert_equal(expected, records)
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
194
304
|
private
|
195
305
|
|
196
306
|
def create_driver(conf)
|
@@ -16,7 +16,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
|
19
|
+
setup do
|
20
20
|
Fluent::Test.setup
|
21
21
|
end
|
22
22
|
|
@@ -54,15 +54,14 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
|
|
54
54
|
configs[:ipv6] = ["::1", IPv6_CONFIG] if ipv6_enabled?
|
55
55
|
configs
|
56
56
|
end
|
57
|
-
|
58
|
-
k, config = data
|
57
|
+
test "configure" do |(ip, config)|
|
59
58
|
d = create_driver(config)
|
60
59
|
assert_equal PORT, d.instance.port
|
61
|
-
assert_equal
|
60
|
+
assert_equal ip, d.instance.bind
|
62
61
|
assert_equal "\n", d.instance.delimiter
|
63
62
|
end
|
64
63
|
|
65
|
-
|
64
|
+
test "grok_pattern" do
|
66
65
|
tests = [
|
67
66
|
{"msg" => "tcptest1\n", "expected" => "tcptest1"},
|
68
67
|
{"msg" => "tcptest2\n", "expected" => "tcptest2"},
|
@@ -77,7 +76,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
|
|
77
76
|
internal_test_grok(config, tests)
|
78
77
|
end
|
79
78
|
|
80
|
-
|
79
|
+
test "grok_pattern_block_config" do
|
81
80
|
tests = [
|
82
81
|
{"msg" => "tcptest1\n", "expected" => "tcptest1"},
|
83
82
|
{"msg" => "tcptest2\n", "expected" => "tcptest2"},
|
@@ -94,7 +93,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
|
|
94
93
|
internal_test_grok(block_config, tests)
|
95
94
|
end
|
96
95
|
|
97
|
-
|
96
|
+
test "grok_multi_patterns" do
|
98
97
|
tests = [
|
99
98
|
{"msg" => "Current time is 2014-01-01T00:00:00+0900\n", "expected" => "2014-01-01T00:00:00+0900"},
|
100
99
|
{"msg" => "The first word matches\n", "expected" => "The"}
|
@@ -50,7 +50,7 @@ TEXT
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
-
|
53
|
+
test "empty_range_text_in_text" do
|
54
54
|
text = " [b-a]"
|
55
55
|
conf = %[
|
56
56
|
grok_pattern %{HOSTNAME:hostname} %{GREEDYDATA:message}
|
@@ -61,8 +61,8 @@ TEXT
|
|
61
61
|
assert(d.instance.firstline?(text))
|
62
62
|
end
|
63
63
|
|
64
|
-
|
65
|
-
|
64
|
+
sub_test_case "NoGrokPatternMatched" do
|
65
|
+
test "with grok_failure_key" do
|
66
66
|
config = %[
|
67
67
|
grok_failure_key grok_failure
|
68
68
|
<grok>
|
@@ -79,7 +79,7 @@ TEXT
|
|
79
79
|
end
|
80
80
|
end
|
81
81
|
|
82
|
-
|
82
|
+
test "without grok_failure" do
|
83
83
|
config = %[
|
84
84
|
<grok>
|
85
85
|
pattern %{PATH:path}
|
@@ -95,7 +95,7 @@ TEXT
|
|
95
95
|
end
|
96
96
|
end
|
97
97
|
|
98
|
-
|
98
|
+
test "no_grok_patterns" do
|
99
99
|
assert_raise Fluent::ConfigError do
|
100
100
|
create_driver('')
|
101
101
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grok-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kiyoto
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-08-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
133
133
|
version: '0'
|
134
134
|
requirements: []
|
135
135
|
rubyforge_project:
|
136
|
-
rubygems_version: 2.7.
|
136
|
+
rubygems_version: 2.7.6
|
137
137
|
signing_key:
|
138
138
|
specification_version: 4
|
139
139
|
summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs
|