fluentd 0.12.0.pre.3 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/fluent.conf +26 -16
- data/lib/fluent/agent.rb +7 -5
- data/lib/fluent/config.rb +1 -8
- data/lib/fluent/config/v1_parser.rb +2 -1
- data/lib/fluent/event_router.rb +1 -8
- data/lib/fluent/filter.rb +16 -0
- data/lib/fluent/formatter.rb +19 -18
- data/lib/fluent/label.rb +16 -0
- data/lib/fluent/output.rb +2 -2
- data/lib/fluent/parser.rb +57 -65
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/filter_grep.rb +16 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +16 -0
- data/lib/fluent/plugin/in_http.rb +17 -5
- data/lib/fluent/plugin/in_monitor_agent.rb +52 -19
- data/lib/fluent/plugin/in_syslog.rb +2 -2
- data/lib/fluent/plugin/in_tail.rb +3 -3
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_roundrobin.rb +1 -1
- data/lib/fluent/plugin/socket_util.rb +1 -1
- data/lib/fluent/root_agent.rb +7 -12
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/test_in_http.rb +28 -8
- data/test/test_formatter.rb +12 -0
- data/test/test_output.rb +16 -0
- data/test/test_parser.rb +119 -75
- data/test/test_plugin_classes.rb +38 -0
- data/test/test_root_agent.rb +135 -0
- metadata +6 -5
- data/Gemfile.cool.io.1.1.1 +0 -4
data/test/test_formatter.rb
CHANGED
@@ -36,6 +36,18 @@ module FormatterTest
|
|
36
36
|
ENV['TZ'] = oldtz
|
37
37
|
end
|
38
38
|
|
39
|
+
class BaseFormatterTest < ::Test::Unit::TestCase
|
40
|
+
include FormatterTest
|
41
|
+
|
42
|
+
def test_call
|
43
|
+
formatter = Formatter.new
|
44
|
+
formatter.configure({})
|
45
|
+
assert_raise NotImplementedError do
|
46
|
+
formatter.format('tag', Engine.now, {})
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
39
51
|
class OutFileFormatterTest < ::Test::Unit::TestCase
|
40
52
|
include FormatterTest
|
41
53
|
|
data/test/test_output.rb
CHANGED
@@ -62,6 +62,22 @@ module FluentOutputTest
|
|
62
62
|
assert_equal 4, d.instance.calc_retry_wait
|
63
63
|
end
|
64
64
|
|
65
|
+
def test_large_num_retries
|
66
|
+
# Test that everything works properly after a very large number of
|
67
|
+
# retries and we hit the expected max_retry_wait.
|
68
|
+
exp_max_retry_wait = 300
|
69
|
+
d = create_driver(CONFIG + %[
|
70
|
+
disable_retry_limit true
|
71
|
+
max_retry_wait #{exp_max_retry_wait}
|
72
|
+
])
|
73
|
+
d.instance.instance_eval { @num_errors += 1000 }
|
74
|
+
assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
|
75
|
+
d.instance.instance_eval { @num_errors += 1000 }
|
76
|
+
assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
|
77
|
+
d.instance.instance_eval { @num_errors += 1000 }
|
78
|
+
assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
|
79
|
+
end
|
80
|
+
|
65
81
|
def create_mock_driver(conf=CONFIG)
|
66
82
|
Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
|
67
83
|
attr_accessor :submit_flush_threads
|
data/test/test_parser.rb
CHANGED
@@ -13,6 +13,32 @@ module ParserTest
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
+
class BaseParserTest < ::Test::Unit::TestCase
|
17
|
+
include ParserTest
|
18
|
+
|
19
|
+
def create_parser
|
20
|
+
parser = Parser.new
|
21
|
+
parser.configure({})
|
22
|
+
parser
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_init
|
26
|
+
assert_true create_parser.estimate_current_event
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_parse
|
30
|
+
assert_raise NotImplementedError do
|
31
|
+
create_parser.parse('')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_call
|
36
|
+
assert_raise NotImplementedError do
|
37
|
+
create_parser.call('')
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
16
42
|
class TimeParserTest < ::Test::Unit::TestCase
|
17
43
|
include ParserTest
|
18
44
|
|
@@ -23,18 +49,18 @@ module ParserTest
|
|
23
49
|
assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))
|
24
50
|
end
|
25
51
|
|
26
|
-
def
|
52
|
+
def test_parse_with_strptime
|
27
53
|
parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
|
28
54
|
|
29
55
|
time = str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z')
|
30
56
|
assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
|
31
57
|
end
|
32
58
|
|
33
|
-
def
|
59
|
+
def test_parse_with_invalid_argument
|
34
60
|
parser = TextParser::TimeParser.new(nil)
|
35
61
|
|
36
62
|
[[], {}, nil, true, 10000].each { |v|
|
37
|
-
assert_raise Fluent::
|
63
|
+
assert_raise Fluent::ParserError do
|
38
64
|
parser.parse(v)
|
39
65
|
end
|
40
66
|
}
|
@@ -46,7 +72,7 @@ module ParserTest
|
|
46
72
|
|
47
73
|
def internal_test_case(parser)
|
48
74
|
text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
|
49
|
-
[parser.
|
75
|
+
[parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
|
50
76
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
51
77
|
assert_equal({
|
52
78
|
'user' => '-',
|
@@ -60,12 +86,12 @@ module ParserTest
|
|
60
86
|
}
|
61
87
|
end
|
62
88
|
|
63
|
-
def
|
89
|
+
def test_parse_with_typed
|
64
90
|
# Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
|
65
91
|
internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
|
66
92
|
end
|
67
93
|
|
68
|
-
def
|
94
|
+
def test_parse_with_configure
|
69
95
|
# Specify conf by configure method instaed of intializer
|
70
96
|
regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
|
71
97
|
parser = TextParser::RegexpParser.new(regexp)
|
@@ -75,18 +101,18 @@ module ParserTest
|
|
75
101
|
assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.patterns['time_format'])
|
76
102
|
end
|
77
103
|
|
78
|
-
def
|
104
|
+
def test_parse_with_typed_and_name_separator
|
79
105
|
internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
|
80
106
|
end
|
81
107
|
|
82
|
-
def
|
108
|
+
def test_parse_without_time
|
83
109
|
time_at_start = Time.now.to_i
|
84
110
|
text = "tagomori_satoshi tagomoris 34\n"
|
85
111
|
|
86
112
|
parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
|
87
113
|
parser.configure('types'=>'name:string,user:string,age:bool')
|
88
114
|
|
89
|
-
[parser.
|
115
|
+
[parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
|
90
116
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
91
117
|
assert_equal "tagomori_satoshi", record["name"]
|
92
118
|
assert_equal "tagomoris", record["user"]
|
@@ -97,14 +123,13 @@ module ParserTest
|
|
97
123
|
parser2.configure('types'=>'name:string,user:string,age:bool')
|
98
124
|
parser2.time_default_current = false
|
99
125
|
|
100
|
-
[parser2.
|
126
|
+
[parser2.parse(text), parser2.parse(text) { |time, record| return time, record}].each { |time, record|
|
101
127
|
assert_equal "tagomori_satoshi", record["name"]
|
102
128
|
assert_equal "tagomoris", record["user"]
|
103
129
|
assert_equal 34, record["age"]
|
104
130
|
|
105
131
|
assert_nil time, "parser returns nil if configured so"
|
106
132
|
}
|
107
|
-
|
108
133
|
end
|
109
134
|
end
|
110
135
|
|
@@ -115,8 +140,10 @@ module ParserTest
|
|
115
140
|
@parser = TextParser::TEMPLATE_REGISTRY.lookup('apache').call
|
116
141
|
end
|
117
142
|
|
118
|
-
|
119
|
-
|
143
|
+
data('parse' => :parse, 'call' => :call)
|
144
|
+
def test_call(method_name)
|
145
|
+
m = @parser.method(method_name)
|
146
|
+
m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
|
120
147
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
121
148
|
assert_equal({
|
122
149
|
'user' => '-',
|
@@ -142,22 +169,22 @@ module ParserTest
|
|
142
169
|
}
|
143
170
|
end
|
144
171
|
|
145
|
-
def
|
146
|
-
@parser.
|
172
|
+
def test_parse
|
173
|
+
@parser.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
|
147
174
|
assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
|
148
175
|
assert_equal(@expected, record)
|
149
176
|
}
|
150
177
|
end
|
151
178
|
|
152
|
-
def
|
153
|
-
@parser.
|
179
|
+
def test_parse_with_pid
|
180
|
+
@parser.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
|
154
181
|
assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
|
155
182
|
assert_equal(@expected.merge('pid' => '1000'), record)
|
156
183
|
}
|
157
184
|
end
|
158
185
|
|
159
|
-
def
|
160
|
-
@parser.
|
186
|
+
def test_parse_without_client
|
187
|
+
@parser.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
|
161
188
|
assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
|
162
189
|
assert_equal({
|
163
190
|
'level' => 'notice',
|
@@ -174,8 +201,8 @@ module ParserTest
|
|
174
201
|
@parser = TextParser::ApacheParser.new
|
175
202
|
end
|
176
203
|
|
177
|
-
def
|
178
|
-
@parser.
|
204
|
+
def test_parse
|
205
|
+
@parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
|
179
206
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
180
207
|
assert_equal({
|
181
208
|
'user' => nil,
|
@@ -206,9 +233,9 @@ module ParserTest
|
|
206
233
|
}
|
207
234
|
end
|
208
235
|
|
209
|
-
def
|
236
|
+
def test_parse
|
210
237
|
@parser.configure({})
|
211
|
-
@parser.
|
238
|
+
@parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
212
239
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
213
240
|
assert_equal(@expected, record)
|
214
241
|
}
|
@@ -216,18 +243,18 @@ module ParserTest
|
|
216
243
|
assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
|
217
244
|
end
|
218
245
|
|
219
|
-
def
|
246
|
+
def test_parse_with_time_format
|
220
247
|
@parser.configure('time_format' => '%b %d %M:%S:%H')
|
221
|
-
@parser.
|
248
|
+
@parser.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
222
249
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
223
250
|
assert_equal(@expected, record)
|
224
251
|
}
|
225
252
|
assert_equal('%b %d %M:%S:%H', @parser.patterns['time_format'])
|
226
253
|
end
|
227
254
|
|
228
|
-
def
|
255
|
+
def test_parse_with_priority
|
229
256
|
@parser.configure('with_priority' => true)
|
230
|
-
@parser.
|
257
|
+
@parser.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
231
258
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
232
259
|
assert_equal(@expected.merge('pri' => 6), record)
|
233
260
|
}
|
@@ -235,9 +262,9 @@ module ParserTest
|
|
235
262
|
assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
|
236
263
|
end
|
237
264
|
|
238
|
-
def
|
265
|
+
def test_parse_without_colon
|
239
266
|
@parser.configure({})
|
240
|
-
@parser.
|
267
|
+
@parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
|
241
268
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
242
269
|
assert_equal(@expected, record)
|
243
270
|
}
|
@@ -253,8 +280,8 @@ module ParserTest
|
|
253
280
|
@parser = TextParser::JSONParser.new
|
254
281
|
end
|
255
282
|
|
256
|
-
def
|
257
|
-
@parser.
|
283
|
+
def test_parse
|
284
|
+
@parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
|
258
285
|
assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
|
259
286
|
assert_equal({
|
260
287
|
'host' => '192.168.0.1',
|
@@ -264,10 +291,10 @@ module ParserTest
|
|
264
291
|
}
|
265
292
|
end
|
266
293
|
|
267
|
-
def
|
294
|
+
def test_parse_without_time
|
268
295
|
time_at_start = Time.now.to_i
|
269
296
|
|
270
|
-
@parser.
|
297
|
+
@parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
|
271
298
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
272
299
|
assert_equal({
|
273
300
|
'host' => '192.168.0.1',
|
@@ -279,7 +306,7 @@ module ParserTest
|
|
279
306
|
parser = TextParser::JSONParser.new
|
280
307
|
parser.estimate_current_event = false
|
281
308
|
parser.configure({})
|
282
|
-
parser.
|
309
|
+
parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
|
283
310
|
assert_equal({
|
284
311
|
'host' => '192.168.0.1',
|
285
312
|
'size' => 777,
|
@@ -289,9 +316,9 @@ module ParserTest
|
|
289
316
|
}
|
290
317
|
end
|
291
318
|
|
292
|
-
def
|
293
|
-
assert_raise Fluent::
|
294
|
-
@parser.
|
319
|
+
def test_parse_with_invalid_time
|
320
|
+
assert_raise Fluent::ParserError do
|
321
|
+
@parser.parse('{"time":[],"k":"v"}') { |time, record| }
|
295
322
|
end
|
296
323
|
end
|
297
324
|
end
|
@@ -314,15 +341,15 @@ module ParserTest
|
|
314
341
|
}
|
315
342
|
end
|
316
343
|
|
317
|
-
def
|
318
|
-
@parser.
|
344
|
+
def test_parse
|
345
|
+
@parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
|
319
346
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
320
347
|
assert_equal(@expected, record)
|
321
348
|
}
|
322
349
|
end
|
323
350
|
|
324
|
-
def
|
325
|
-
@parser.
|
351
|
+
def test_parse_with_empty_included_path
|
352
|
+
@parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
|
326
353
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
327
354
|
assert_equal(@expected.merge('path' => '/a[ ]b'), record)
|
328
355
|
}
|
@@ -345,10 +372,10 @@ module ParserTest
|
|
345
372
|
assert_equal ",", parser.delimiter
|
346
373
|
end
|
347
374
|
|
348
|
-
def
|
375
|
+
def test_parse
|
349
376
|
parser = TextParser::TSVParser.new
|
350
377
|
parser.configure('keys' => 'time,a,b', 'time_key' => 'time')
|
351
|
-
parser.
|
378
|
+
parser.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
|
352
379
|
assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
|
353
380
|
assert_equal({
|
354
381
|
'a' => '192.168.0.1',
|
@@ -357,12 +384,12 @@ module ParserTest
|
|
357
384
|
}
|
358
385
|
end
|
359
386
|
|
360
|
-
def
|
387
|
+
def test_parse_with_time
|
361
388
|
time_at_start = Time.now.to_i
|
362
389
|
|
363
390
|
parser = TextParser::TSVParser.new
|
364
391
|
parser.configure('keys' => 'a,b')
|
365
|
-
parser.
|
392
|
+
parser.parse("192.168.0.1\t111") { |time, record|
|
366
393
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
367
394
|
assert_equal({
|
368
395
|
'a' => '192.168.0.1',
|
@@ -373,7 +400,7 @@ module ParserTest
|
|
373
400
|
parser = TextParser::TSVParser.new
|
374
401
|
parser.estimate_current_event = false
|
375
402
|
parser.configure('keys' => 'a,b', 'time_key' => 'time')
|
376
|
-
parser.
|
403
|
+
parser.parse("192.168.0.1\t111") { |time, record|
|
377
404
|
assert_equal({
|
378
405
|
'a' => '192.168.0.1',
|
379
406
|
'b' => '111',
|
@@ -381,15 +408,32 @@ module ParserTest
|
|
381
408
|
assert_nil time, "parser returns nil w/o time and if configured so"
|
382
409
|
}
|
383
410
|
end
|
411
|
+
|
412
|
+
data(
|
413
|
+
'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
|
414
|
+
'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
|
415
|
+
'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
|
416
|
+
'2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
|
417
|
+
'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
|
418
|
+
'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
|
419
|
+
def test_black_column(data)
|
420
|
+
line, expected = data
|
421
|
+
|
422
|
+
parser = TextParser::TSVParser.new
|
423
|
+
parser.configure('keys' => '1,2,3')
|
424
|
+
parser.parse(line) { |time, record|
|
425
|
+
assert_equal(expected, record)
|
426
|
+
}
|
427
|
+
end
|
384
428
|
end
|
385
429
|
|
386
430
|
class CSVParserTest < ::Test::Unit::TestCase
|
387
431
|
include ParserTest
|
388
432
|
|
389
|
-
def
|
433
|
+
def test_parse
|
390
434
|
parser = TextParser::CSVParser.new
|
391
435
|
parser.configure('keys' => 'time,c,d', 'time_key' => 'time')
|
392
|
-
parser.
|
436
|
+
parser.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
|
393
437
|
assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
|
394
438
|
assert_equal({
|
395
439
|
'c' => '192.168.0.1',
|
@@ -398,12 +442,12 @@ module ParserTest
|
|
398
442
|
}
|
399
443
|
end
|
400
444
|
|
401
|
-
def
|
445
|
+
def test_parse_without_time
|
402
446
|
time_at_start = Time.now.to_i
|
403
447
|
|
404
448
|
parser = TextParser::CSVParser.new
|
405
449
|
parser.configure('keys' => 'c,d')
|
406
|
-
parser.
|
450
|
+
parser.parse("192.168.0.1,111") { |time, record|
|
407
451
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
408
452
|
assert_equal({
|
409
453
|
'c' => '192.168.0.1',
|
@@ -414,7 +458,7 @@ module ParserTest
|
|
414
458
|
parser = TextParser::CSVParser.new
|
415
459
|
parser.estimate_current_event = false
|
416
460
|
parser.configure('keys' => 'c,d', 'time_key' => 'time')
|
417
|
-
parser.
|
461
|
+
parser.parse("192.168.0.1,111") { |time, record|
|
418
462
|
assert_equal({
|
419
463
|
'c' => '192.168.0.1',
|
420
464
|
'd' => '111',
|
@@ -442,10 +486,10 @@ module ParserTest
|
|
442
486
|
assert_equal "=", parser.label_delimiter
|
443
487
|
end
|
444
488
|
|
445
|
-
def
|
489
|
+
def test_parse
|
446
490
|
parser = TextParser::LabeledTSVParser.new
|
447
491
|
parser.configure({})
|
448
|
-
parser.
|
492
|
+
parser.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
|
449
493
|
assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
|
450
494
|
assert_equal({
|
451
495
|
'host' => '192.168.0.1',
|
@@ -454,13 +498,13 @@ module ParserTest
|
|
454
498
|
}
|
455
499
|
end
|
456
500
|
|
457
|
-
def
|
501
|
+
def test_parse_with_customized_delimiter
|
458
502
|
parser = TextParser::LabeledTSVParser.new
|
459
503
|
parser.configure(
|
460
504
|
'delimiter' => ',',
|
461
505
|
'label_delimiter' => '=',
|
462
506
|
)
|
463
|
-
parser.
|
507
|
+
parser.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
|
464
508
|
assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
|
465
509
|
assert_equal({
|
466
510
|
'host' => '192.168.0.1',
|
@@ -469,13 +513,13 @@ module ParserTest
|
|
469
513
|
}
|
470
514
|
end
|
471
515
|
|
472
|
-
def
|
516
|
+
def test_parse_with_customized_time_format
|
473
517
|
parser = TextParser::LabeledTSVParser.new
|
474
518
|
parser.configure(
|
475
519
|
'time_key' => 'mytime',
|
476
520
|
'time_format' => '%d/%b/%Y:%H:%M:%S %z',
|
477
521
|
)
|
478
|
-
parser.
|
522
|
+
parser.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
|
479
523
|
assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
|
480
524
|
assert_equal({
|
481
525
|
'host' => '192.168.0.1',
|
@@ -484,12 +528,12 @@ module ParserTest
|
|
484
528
|
}
|
485
529
|
end
|
486
530
|
|
487
|
-
def
|
531
|
+
def test_parse_without_time
|
488
532
|
time_at_start = Time.now.to_i
|
489
533
|
|
490
534
|
parser = TextParser::LabeledTSVParser.new
|
491
535
|
parser.configure({})
|
492
|
-
parser.
|
536
|
+
parser.parse("host:192.168.0.1\treq_id:111") { |time, record|
|
493
537
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
494
538
|
assert_equal({
|
495
539
|
'host' => '192.168.0.1',
|
@@ -500,7 +544,7 @@ module ParserTest
|
|
500
544
|
parser = TextParser::LabeledTSVParser.new
|
501
545
|
parser.estimate_current_event = false
|
502
546
|
parser.configure({})
|
503
|
-
parser.
|
547
|
+
parser.parse("host:192.168.0.1\treq_id:111") { |time, record|
|
504
548
|
assert_equal({
|
505
549
|
'host' => '192.168.0.1',
|
506
550
|
'req_id' => '111',
|
@@ -522,28 +566,28 @@ module ParserTest
|
|
522
566
|
assert_equal "foobar", parser.message_key
|
523
567
|
end
|
524
568
|
|
525
|
-
def
|
569
|
+
def test_parse
|
526
570
|
parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
|
527
571
|
parser.configure({})
|
528
|
-
parser.
|
572
|
+
parser.parse('log message!') { |time, record|
|
529
573
|
assert_equal({'message' => 'log message!'}, record)
|
530
574
|
}
|
531
575
|
end
|
532
576
|
|
533
|
-
def
|
577
|
+
def test_parse_with_message_key
|
534
578
|
parser = TextParser::NoneParser.new
|
535
579
|
parser.configure('message_key' => 'foobar')
|
536
|
-
parser.
|
580
|
+
parser.parse('log message!') { |time, record|
|
537
581
|
assert_equal({'foobar' => 'log message!'}, record)
|
538
582
|
}
|
539
583
|
end
|
540
584
|
|
541
|
-
def
|
585
|
+
def test_parse_without_default_time
|
542
586
|
time_at_start = Time.now.to_i
|
543
587
|
|
544
588
|
parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
|
545
589
|
parser.configure({})
|
546
|
-
parser.
|
590
|
+
parser.parse('log message!') { |time, record|
|
547
591
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
548
592
|
assert_equal({'message' => 'log message!'}, record)
|
549
593
|
}
|
@@ -551,7 +595,7 @@ module ParserTest
|
|
551
595
|
parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
|
552
596
|
parser.estimate_current_event = false
|
553
597
|
parser.configure({})
|
554
|
-
parser.
|
598
|
+
parser.parse('log message!') { |time, record|
|
555
599
|
assert_equal({'message' => 'log message!'}, record)
|
556
600
|
assert_nil time, "parser returns nil w/o time if configured so"
|
557
601
|
}
|
@@ -575,9 +619,9 @@ module ParserTest
|
|
575
619
|
}
|
576
620
|
end
|
577
621
|
|
578
|
-
def
|
622
|
+
def test_parse
|
579
623
|
parser = create_parser('format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/')
|
580
|
-
parser.
|
624
|
+
parser.parse(<<EOS.chomp) { |time, record|
|
581
625
|
2013-3-03 14:27:33 [main] ERROR Main - Exception
|
582
626
|
javax.management.RuntimeErrorException: null
|
583
627
|
\tat Main.main(Main.java:16) ~[bin/:na]
|
@@ -592,9 +636,9 @@ EOS
|
|
592
636
|
}
|
593
637
|
end
|
594
638
|
|
595
|
-
def
|
639
|
+
def test_parse_with_firstline
|
596
640
|
parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/')
|
597
|
-
parser.
|
641
|
+
parser.parse(<<EOS.chomp) { |time, record|
|
598
642
|
----
|
599
643
|
time=2013-3-03 14:27:33
|
600
644
|
message=test1
|
@@ -606,7 +650,7 @@ EOS
|
|
606
650
|
}
|
607
651
|
end
|
608
652
|
|
609
|
-
def
|
653
|
+
def test_parse_with_multiple_formats
|
610
654
|
parser = create_parser('format_firstline' => '/^Started/',
|
611
655
|
'format1' => '/Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/',
|
612
656
|
'format2' => '/Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/',
|
@@ -614,7 +658,7 @@ EOS
|
|
614
658
|
'format4' => '/ Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/',
|
615
659
|
'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/'
|
616
660
|
)
|
617
|
-
parser.
|
661
|
+
parser.parse(<<EOS.chomp) { |time, record|
|
618
662
|
Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
|
619
663
|
Processing by UsersController#show as HTML
|
620
664
|
Parameters: {"user_id"=>"123"}
|
@@ -646,10 +690,10 @@ EOS
|
|
646
690
|
class TextParserTest < ::Test::Unit::TestCase
|
647
691
|
include ParserTest
|
648
692
|
|
649
|
-
class MultiEventTestParser
|
693
|
+
class MultiEventTestParser < ::Fluent::Parser
|
650
694
|
include Fluent::Configurable
|
651
695
|
|
652
|
-
def
|
696
|
+
def parse(text)
|
653
697
|
2.times { |i|
|
654
698
|
record = {}
|
655
699
|
record['message'] = text
|