fluentd 1.6.3 → 1.7.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
- data/CHANGELOG.md +83 -0
- data/README.md +5 -1
- data/fluentd.gemspec +3 -2
- data/lib/fluent/clock.rb +4 -0
- data/lib/fluent/compat/output.rb +3 -3
- data/lib/fluent/compat/socket_util.rb +1 -1
- data/lib/fluent/config/element.rb +3 -3
- data/lib/fluent/config/literal_parser.rb +1 -1
- data/lib/fluent/config/section.rb +4 -1
- data/lib/fluent/error.rb +4 -0
- data/lib/fluent/event.rb +28 -24
- data/lib/fluent/event_router.rb +2 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +8 -0
- data/lib/fluent/plugin/bare_output.rb +4 -4
- data/lib/fluent/plugin/buf_file.rb +10 -1
- data/lib/fluent/plugin/buf_file_single.rb +219 -0
- data/lib/fluent/plugin/buffer.rb +62 -63
- data/lib/fluent/plugin/buffer/chunk.rb +21 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +44 -12
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
- data/lib/fluent/plugin/compressable.rb +10 -6
- data/lib/fluent/plugin/filter_grep.rb +2 -2
- data/lib/fluent/plugin/formatter_csv.rb +10 -6
- data/lib/fluent/plugin/in_syslog.rb +10 -3
- data/lib/fluent/plugin/in_tail.rb +7 -2
- data/lib/fluent/plugin/in_tcp.rb +34 -7
- data/lib/fluent/plugin/multi_output.rb +4 -4
- data/lib/fluent/plugin/out_exec_filter.rb +1 -0
- data/lib/fluent/plugin/out_file.rb +13 -3
- data/lib/fluent/plugin/out_forward.rb +144 -588
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
- data/lib/fluent/plugin/out_http.rb +231 -0
- data/lib/fluent/plugin/output.rb +29 -35
- data/lib/fluent/plugin/parser.rb +77 -0
- data/lib/fluent/plugin/parser_csv.rb +75 -0
- data/lib/fluent/plugin/parser_syslog.rb +106 -3
- data/lib/fluent/plugin_helper/server.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +14 -1
- data/lib/fluent/plugin_helper/thread.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/time.rb +4 -2
- data/lib/fluent/timezone.rb +21 -7
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +1 -1
- data/test/command/test_plugin_generator.rb +18 -2
- data/test/config/test_configurable.rb +78 -40
- data/test/counter/test_store.rb +1 -1
- data/test/helper.rb +1 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
- data/test/plugin/out_forward/test_load_balancer.rb +60 -0
- data/test/plugin/out_forward/test_socket_cache.rb +139 -0
- data/test/plugin/test_buf_file.rb +172 -2
- data/test/plugin/test_buf_file_single.rb +801 -0
- data/test/plugin/test_buffer.rb +4 -48
- data/test/plugin/test_buffer_file_chunk.rb +38 -1
- data/test/plugin/test_buffer_file_single_chunk.rb +621 -0
- data/test/plugin/test_buffer_memory_chunk.rb +1 -0
- data/test/plugin/test_formatter_csv.rb +16 -0
- data/test/plugin/test_in_syslog.rb +56 -6
- data/test/plugin/test_in_tail.rb +1 -1
- data/test/plugin/test_in_tcp.rb +25 -0
- data/test/plugin/test_out_forward.rb +150 -201
- data/test/plugin/test_out_http.rb +352 -0
- data/test/plugin/test_output_as_buffered.rb +27 -24
- data/test/plugin/test_parser.rb +40 -0
- data/test/plugin/test_parser_csv.rb +83 -0
- data/test/plugin/test_parser_syslog.rb +118 -19
- data/test/plugin_helper/test_record_accessor.rb +1 -1
- data/test/test_time_formatter.rb +140 -121
- metadata +35 -6
@@ -100,4 +100,87 @@ class CSVParserTest < ::Test::Unit::TestCase
|
|
100
100
|
assert_equal record['c'], '789'
|
101
101
|
end
|
102
102
|
end
|
103
|
+
|
104
|
+
sub_test_case 'parser' do
|
105
|
+
data('normal' => 'normal',
|
106
|
+
'fast' => 'fast')
|
107
|
+
def test_compatibility_between_normal_and_fast_parser(param)
|
108
|
+
d = create_driver(
|
109
|
+
'keys' => 'time,key1,key2,key3,key4,key5',
|
110
|
+
'time_key' => 'time',
|
111
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
112
|
+
'keep_time_key' => 'false',
|
113
|
+
'parser_type' => param
|
114
|
+
)
|
115
|
+
|
116
|
+
# non quoted
|
117
|
+
text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'
|
118
|
+
expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => "value3",
|
119
|
+
'key4' => 'value4', 'key5' => "value5"}
|
120
|
+
d.instance.parse(text) do |time, record|
|
121
|
+
assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
122
|
+
assert_equal expected, record
|
123
|
+
end
|
124
|
+
|
125
|
+
# quoted
|
126
|
+
text = '28/Feb/2013:12:00:00 +0900,"value1","val,ue2","va,lu,e3","val ue4",""'
|
127
|
+
expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => "va,lu,e3",
|
128
|
+
'key4' => 'val ue4', 'key5' => ""}
|
129
|
+
d.instance.parse(text) do |time, record|
|
130
|
+
assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
131
|
+
assert_equal expected, record
|
132
|
+
end
|
133
|
+
|
134
|
+
# mixed
|
135
|
+
text = '28/Feb/2013:12:00:00 +0900,message,"mes,sage","me,ssa,ge",mess age,""'
|
136
|
+
expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => "me,ssa,ge",
|
137
|
+
'key4' => 'mess age', 'key5' => ""}
|
138
|
+
d.instance.parse(text) do |time, record|
|
139
|
+
assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
140
|
+
assert_equal expected, record
|
141
|
+
end
|
142
|
+
|
143
|
+
# escaped
|
144
|
+
text = '28/Feb/2013:12:00:00 +0900,"message","mes""sage","""message""",,""""""'
|
145
|
+
expected = {'key1' => 'message', 'key2' => 'mes"sage', 'key3' => '"message"',
|
146
|
+
'key4' => nil, 'key5' => '""'}
|
147
|
+
d.instance.parse(text) do |time, record|
|
148
|
+
assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
149
|
+
assert_equal expected, record
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_incompatibility_between_normal_and_fast_parser
|
154
|
+
normal = create_driver(
|
155
|
+
'keys' => 'key1,key2',
|
156
|
+
'parser_type' => 'normal'
|
157
|
+
)
|
158
|
+
fast = create_driver(
|
159
|
+
'keys' => 'key1,key2',
|
160
|
+
'parser_type' => 'fast'
|
161
|
+
)
|
162
|
+
|
163
|
+
# unexpected quote position
|
164
|
+
text = 'a"b,"a"""c"'
|
165
|
+
assert_raise(CSV::MalformedCSVError) {
|
166
|
+
normal.instance.parse(text) { |t, r| }
|
167
|
+
}
|
168
|
+
assert_nothing_raised {
|
169
|
+
# generate broken record
|
170
|
+
fast.instance.parse(text) { |t, r| }
|
171
|
+
}
|
172
|
+
|
173
|
+
# incorrect the number of column
|
174
|
+
text = 'a,b,c'
|
175
|
+
expected = {"key1" => 'a', "key2" => 'b'}
|
176
|
+
normal.instance.parse(text) { |t, r|
|
177
|
+
assert_equal expected, r
|
178
|
+
}
|
179
|
+
fast.instance.parse(text) { |t, r|
|
180
|
+
assert_not_equal expected, r
|
181
|
+
}
|
182
|
+
|
183
|
+
# And more...
|
184
|
+
end
|
185
|
+
end
|
103
186
|
end
|
@@ -14,8 +14,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
14
14
|
}
|
15
15
|
end
|
16
16
|
|
17
|
-
|
18
|
-
|
17
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
18
|
+
def test_parse(param)
|
19
|
+
@parser.configure({'parser_type' => param})
|
19
20
|
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
20
21
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
21
22
|
assert_equal(@expected, record)
|
@@ -24,8 +25,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
24
25
|
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
25
26
|
end
|
26
27
|
|
27
|
-
|
28
|
-
|
28
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
29
|
+
def test_parse_with_time_format(param)
|
30
|
+
@parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)
|
29
31
|
@parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
30
32
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
31
33
|
assert_equal(@expected, record)
|
@@ -33,8 +35,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
33
35
|
assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
|
34
36
|
end
|
35
37
|
|
36
|
-
|
37
|
-
|
38
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
39
|
+
def test_parse_with_subsecond_time(param)
|
40
|
+
@parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)
|
41
|
+
@parser.instance.parse('Feb 28 12:00:00.456 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
42
|
+
assert_equal(event_time('Feb 28 12:00:00.456', format: '%b %d %H:%M:%S.%N'), time)
|
43
|
+
assert_equal(@expected, record)
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
48
|
+
def test_parse_with_priority(param)
|
49
|
+
@parser.configure('with_priority' => true, 'parser_type' => param)
|
38
50
|
@parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
39
51
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
40
52
|
assert_equal(@expected.merge('pri' => 6), record)
|
@@ -43,8 +55,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
43
55
|
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
44
56
|
end
|
45
57
|
|
46
|
-
|
47
|
-
|
58
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
59
|
+
def test_parse_with_empty_priority(param)
|
60
|
+
@parser.configure('with_priority' => true, 'parser_type' => param)
|
61
|
+
@parser.instance.parse('<>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
62
|
+
assert_nil time
|
63
|
+
assert_nil record
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
68
|
+
def test_parse_without_colon(param)
|
69
|
+
@parser.configure({'parser_type' => param})
|
48
70
|
@parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
|
49
71
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
50
72
|
assert_equal(@expected, record)
|
@@ -53,10 +75,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
53
75
|
assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
|
54
76
|
end
|
55
77
|
|
56
|
-
|
78
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
79
|
+
def test_parse_with_keep_time_key(param)
|
57
80
|
@parser.configure(
|
58
81
|
'time_format' => '%b %d %M:%S:%H',
|
59
82
|
'keep_time_key'=>'true',
|
83
|
+
'parser_type' => param
|
60
84
|
)
|
61
85
|
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
62
86
|
@parser.instance.parse(text) do |time, record|
|
@@ -64,24 +88,87 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
64
88
|
end
|
65
89
|
end
|
66
90
|
|
67
|
-
|
91
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
92
|
+
def test_parse_various_characters_for_tag(param)
|
68
93
|
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
|
69
|
-
@parser.configure({})
|
94
|
+
@parser.configure({'parser_type' => param})
|
70
95
|
@parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
|
71
96
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
72
97
|
assert_equal(@expected.merge('ident' => ident), record)
|
73
98
|
}
|
74
99
|
end
|
75
100
|
|
76
|
-
|
101
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
102
|
+
def test_parse_various_characters_for_tag_with_priority(param)
|
77
103
|
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
|
78
|
-
@parser.configure('with_priority' => true)
|
104
|
+
@parser.configure('with_priority' => true, 'parser_type' => param)
|
79
105
|
@parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
|
80
106
|
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
81
107
|
assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
|
82
108
|
}
|
83
109
|
end
|
84
110
|
|
111
|
+
sub_test_case 'Check the difference of regexp and string parser' do
|
112
|
+
# examples from rfc3164
|
113
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
114
|
+
test 'wrong result with no ident message by default' do |param|
|
115
|
+
@parser.configure('parser_type' => param)
|
116
|
+
@parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
|
117
|
+
assert_equal({'host' => '10.0.0.99', 'ident' => 'Use', 'message' => 'the BFG!'}, record)
|
118
|
+
}
|
119
|
+
end
|
120
|
+
|
121
|
+
test "proper result with no ident message by 'support_colonless_ident false'" do
|
122
|
+
@parser.configure('parser_type' => 'string', 'support_colonless_ident' => false)
|
123
|
+
@parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
|
124
|
+
assert_equal({'host' => '10.0.0.99', 'message' => 'Use the BFG!'}, record)
|
125
|
+
}
|
126
|
+
end
|
127
|
+
|
128
|
+
test "string parsers can't parse broken syslog message and generate wrong record" do
|
129
|
+
@parser.configure('parser_type' => 'string')
|
130
|
+
@parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record|
|
131
|
+
expected = {'host' => 'scapegoat.dmz.example.org', 'ident' => 'sched', 'pid' => '0', 'message' => "That's All Folks!"}
|
132
|
+
assert_not_equal(expected, record)
|
133
|
+
}
|
134
|
+
end
|
135
|
+
|
136
|
+
test "regexp parsers can't parse broken syslog message and raises an error" do
|
137
|
+
@parser.configure('parser_type' => 'regexp')
|
138
|
+
assert_raise(Fluent::TimeParser::TimeParseError) {
|
139
|
+
@parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record| }
|
140
|
+
}
|
141
|
+
end
|
142
|
+
|
143
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
144
|
+
test "':' included message breaks regexp parser" do |param|
|
145
|
+
@parser.configure('parser_type' => param)
|
146
|
+
@parser.instance.parse('Aug 10 12:00:00 127.0.0.1 test foo:bar') { |time, record|
|
147
|
+
expected = {'host' => '127.0.0.1', 'ident' => 'test', 'message' => 'foo:bar'}
|
148
|
+
if param == 'string'
|
149
|
+
assert_equal(expected, record)
|
150
|
+
else
|
151
|
+
assert_not_equal(expected, record)
|
152
|
+
end
|
153
|
+
}
|
154
|
+
end
|
155
|
+
|
156
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
157
|
+
test "Only no whitespace content in MSG causes different result" do |param|
|
158
|
+
@parser.configure('parser_type' => param)
|
159
|
+
@parser.instance.parse('Aug 10 12:00:00 127.0.0.1 value1,value2,value3,value4') { |time, record|
|
160
|
+
# 'message' is correct but regexp set it as 'ident'
|
161
|
+
if param == 'string'
|
162
|
+
expected = {'host' => '127.0.0.1', 'message' => 'value1,value2,value3,value4'}
|
163
|
+
assert_equal(expected, record)
|
164
|
+
else
|
165
|
+
expected = {'host' => '127.0.0.1', 'ident' => 'value1,value2,value3,value4', 'message' => ''}
|
166
|
+
assert_equal(expected, record)
|
167
|
+
end
|
168
|
+
}
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
85
172
|
class TestRFC5424Regexp < self
|
86
173
|
def test_parse_with_rfc5424_message
|
87
174
|
@parser.configure(
|
@@ -273,10 +360,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
273
360
|
end
|
274
361
|
|
275
362
|
class TestAutoRegexp < self
|
276
|
-
|
363
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
364
|
+
def test_auto_with_legacy_syslog_message(param)
|
277
365
|
@parser.configure(
|
278
366
|
'time_format' => '%b %d %M:%S:%H',
|
279
367
|
'message_format' => 'auto',
|
368
|
+
'parser_type' => param
|
280
369
|
)
|
281
370
|
text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
282
371
|
@parser.instance.parse(text) do |time, record|
|
@@ -286,11 +375,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
286
375
|
assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
|
287
376
|
end
|
288
377
|
|
289
|
-
|
378
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
379
|
+
def test_auto_with_legacy_syslog_priority_message(param)
|
290
380
|
@parser.configure(
|
291
381
|
'time_format' => '%b %d %M:%S:%H',
|
292
382
|
'with_priority' => true,
|
293
383
|
'message_format' => 'auto',
|
384
|
+
'parser_type' => param
|
294
385
|
)
|
295
386
|
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
296
387
|
@parser.instance.parse(text) do |time, record|
|
@@ -300,11 +391,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
300
391
|
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
301
392
|
end
|
302
393
|
|
303
|
-
|
394
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
395
|
+
def test_parse_with_rfc5424_message(param)
|
304
396
|
@parser.configure(
|
305
397
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
306
398
|
'message_format' => 'auto',
|
307
399
|
'with_priority' => true,
|
400
|
+
'parser_type' => param
|
308
401
|
)
|
309
402
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
310
403
|
@parser.instance.parse(text) do |time, record|
|
@@ -318,11 +411,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
318
411
|
@parser.instance.patterns['format'])
|
319
412
|
end
|
320
413
|
|
321
|
-
|
414
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
415
|
+
def test_parse_with_rfc5424_structured_message(param)
|
322
416
|
@parser.configure(
|
323
417
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
324
418
|
'message_format' => 'auto',
|
325
419
|
'with_priority' => true,
|
420
|
+
'parser_type' => param
|
326
421
|
)
|
327
422
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
328
423
|
@parser.instance.parse(text) do |time, record|
|
@@ -337,12 +432,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
337
432
|
@parser.instance.patterns['format'])
|
338
433
|
end
|
339
434
|
|
340
|
-
|
435
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
436
|
+
def test_parse_with_both_message_type(param)
|
341
437
|
@parser.configure(
|
342
438
|
'time_format' => '%b %d %M:%S:%H',
|
343
439
|
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
344
440
|
'message_format' => 'auto',
|
345
441
|
'with_priority' => true,
|
442
|
+
'parser_type' => param
|
346
443
|
)
|
347
444
|
text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
348
445
|
@parser.instance.parse(text) do |time, record|
|
@@ -382,12 +479,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
382
479
|
@parser.instance.patterns['format'])
|
383
480
|
end
|
384
481
|
|
385
|
-
|
482
|
+
data('regexp' => 'regexp', 'string' => 'string')
|
483
|
+
def test_parse_with_both_message_type_and_priority(param)
|
386
484
|
@parser.configure(
|
387
485
|
'time_format' => '%b %d %M:%S:%H',
|
388
486
|
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
389
487
|
'with_priority' => true,
|
390
488
|
'message_format' => 'auto',
|
489
|
+
'parser_type' => param
|
391
490
|
)
|
392
491
|
text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
393
492
|
@parser.instance.parse(text) do |time, record|
|
@@ -141,7 +141,7 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
|
|
141
141
|
|
142
142
|
data("missing ']'" => "$['key1'",
|
143
143
|
"missing array index with dot" => "$.hello[]",
|
144
|
-
"missing array index with
|
144
|
+
"missing array index with bracket" => "$['hello'][]",
|
145
145
|
"whitespace char included key in dot notation" => "$.key[0].ke y",
|
146
146
|
"more chars" => "$.key1[0]foo",
|
147
147
|
"empty keys with dot" => "$.",
|
data/test/test_time_formatter.rb
CHANGED
@@ -3,9 +3,8 @@ require 'fluent/test'
|
|
3
3
|
require 'fluent/time'
|
4
4
|
|
5
5
|
class TimeFormatterTest < ::Test::Unit::TestCase
|
6
|
-
|
7
|
-
@
|
8
|
-
@fmt = "%Y%m%d %H%M%z" # YYYYMMDD HHMM[+-]HHMM
|
6
|
+
setup do
|
7
|
+
@fmt ="%Y%m%d %H%M%z" # YYYYMMDD HHMM[+-]HHMM
|
9
8
|
end
|
10
9
|
|
11
10
|
def format(format, localtime, timezone)
|
@@ -13,166 +12,186 @@ class TimeFormatterTest < ::Test::Unit::TestCase
|
|
13
12
|
formatter.format(@time)
|
14
13
|
end
|
15
14
|
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
module TestLists
|
16
|
+
def test_default_utc_nil
|
17
|
+
assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil))
|
18
|
+
end
|
19
19
|
|
20
|
-
|
21
|
-
|
22
|
-
|
20
|
+
def test_default_utc_pHH_MM
|
21
|
+
assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30"))
|
22
|
+
end
|
23
23
|
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
def test_default_utc_nHH_MM
|
25
|
+
assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30"))
|
26
|
+
end
|
27
27
|
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
def test_default_utc_pHHMM
|
29
|
+
assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230"))
|
30
|
+
end
|
31
31
|
|
32
|
-
|
33
|
-
|
34
|
-
|
32
|
+
def test_default_utc_nHHMM
|
33
|
+
assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230"))
|
34
|
+
end
|
35
35
|
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
def test_default_utc_pHH
|
37
|
+
assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03"))
|
38
|
+
end
|
39
39
|
|
40
|
-
|
41
|
-
|
42
|
-
|
40
|
+
def test_default_utc_nHH
|
41
|
+
assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03"))
|
42
|
+
end
|
43
43
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
def test_default_utc_timezone_1
|
45
|
+
# Asia/Tokyo (+09:00) does not have daylight saving time.
|
46
|
+
assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo"))
|
47
|
+
end
|
48
48
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
49
|
+
def test_default_utc_timezone_2
|
50
|
+
# Pacific/Honolulu (-10:00) does not have daylight saving time.
|
51
|
+
assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu"))
|
52
|
+
end
|
53
53
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
54
|
+
def test_default_utc_timezone_3
|
55
|
+
# America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
|
56
|
+
assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires"))
|
57
|
+
end
|
58
58
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
59
|
+
def test_default_utc_timezone_4
|
60
|
+
# Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
|
61
|
+
# UTC offset in DST is +02:00. In September, Europe/Paris is in DST.
|
62
|
+
assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris"))
|
63
|
+
end
|
64
64
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
65
|
+
def test_default_utc_timezone_5
|
66
|
+
# Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
|
67
|
+
# UTC offset in DST is +02:00. In January, Europe/Paris is not in DST.
|
68
|
+
@time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
|
69
|
+
assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris"))
|
70
|
+
end
|
71
71
|
|
72
|
-
|
73
|
-
|
74
|
-
|
72
|
+
def test_default_utc_invalid
|
73
|
+
assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid"))
|
74
|
+
end
|
75
75
|
|
76
|
-
|
77
|
-
|
78
|
-
|
76
|
+
def test_default_localtime_nil_1
|
77
|
+
with_timezone("UTC-04") do
|
78
|
+
assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil))
|
79
|
+
end
|
79
80
|
end
|
80
|
-
end
|
81
81
|
|
82
|
-
|
83
|
-
|
84
|
-
|
82
|
+
def test_default_localtime_nil_2
|
83
|
+
with_timezone("UTC+05") do
|
84
|
+
assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil))
|
85
|
+
end
|
85
86
|
end
|
86
|
-
end
|
87
87
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
88
|
+
def test_default_localtime_timezone
|
89
|
+
# 'timezone' takes precedence over 'localtime'.
|
90
|
+
with_timezone("UTC-06") do
|
91
|
+
assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07"))
|
92
|
+
end
|
92
93
|
end
|
93
|
-
end
|
94
94
|
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
def test_specific_utc_nil
|
96
|
+
assert_equal("20140927 0000+0000", format(@fmt, false, nil))
|
97
|
+
end
|
98
98
|
|
99
|
-
|
100
|
-
|
101
|
-
|
99
|
+
def test_specific_utc_pHH_MM
|
100
|
+
assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30"))
|
101
|
+
end
|
102
102
|
|
103
|
-
|
104
|
-
|
105
|
-
|
103
|
+
def test_specific_utc_nHH_MM
|
104
|
+
assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30"))
|
105
|
+
end
|
106
106
|
|
107
|
-
|
108
|
-
|
109
|
-
|
107
|
+
def test_specific_utc_pHHMM
|
108
|
+
assert_equal("20140927 1030+1030", format(@fmt, false, "+1030"))
|
109
|
+
end
|
110
110
|
|
111
|
-
|
112
|
-
|
113
|
-
|
111
|
+
def test_specific_utc_nHHMM
|
112
|
+
assert_equal("20140926 1230-1130", format(@fmt, false, "-1130"))
|
113
|
+
end
|
114
114
|
|
115
|
-
|
116
|
-
|
117
|
-
|
115
|
+
def test_specific_utc_pHH
|
116
|
+
assert_equal("20140927 1200+1200", format(@fmt, false, "+12"))
|
117
|
+
end
|
118
118
|
|
119
|
-
|
120
|
-
|
121
|
-
|
119
|
+
def test_specific_utc_nHH
|
120
|
+
assert_equal("20140926 1100-1300", format(@fmt, false, "-13"))
|
121
|
+
end
|
122
122
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
123
|
+
def test_specific_utc_timezone_1
|
124
|
+
# Europe/Moscow (+04:00) does not have daylight saving time.
|
125
|
+
assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow"))
|
126
|
+
end
|
127
127
|
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
128
|
+
def test_specific_utc_timezone_2
|
129
|
+
# Pacific/Galapagos (-06:00) does not have daylight saving time.
|
130
|
+
assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos"))
|
131
|
+
end
|
132
132
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
133
|
+
def test_specific_utc_timezone_3
|
134
|
+
# America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
|
135
|
+
assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires"))
|
136
|
+
end
|
137
137
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
138
|
+
def test_specific_utc_timezone_4
|
139
|
+
# America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
|
140
|
+
# UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST.
|
141
|
+
assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles"))
|
142
|
+
end
|
143
143
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
144
|
+
def test_specific_utc_timezone_5
|
145
|
+
# America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
|
146
|
+
# UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST.
|
147
|
+
@time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
|
148
|
+
assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles"))
|
149
|
+
end
|
150
150
|
|
151
|
-
|
152
|
-
|
153
|
-
|
151
|
+
def test_specific_utc_invalid
|
152
|
+
assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid"))
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_specific_localtime_nil_1
|
156
|
+
with_timezone("UTC-07") do
|
157
|
+
assert_equal("20140927 0700+0700", format(@fmt, true, nil))
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def test_specific_localtime_nil_2
|
162
|
+
with_timezone("UTC+08") do
|
163
|
+
assert_equal("20140926 1600-0800", format(@fmt, true, nil))
|
164
|
+
end
|
165
|
+
end
|
154
166
|
|
155
|
-
|
156
|
-
|
157
|
-
|
167
|
+
def test_specific_localtime_timezone
|
168
|
+
# 'timezone' takes precedence over 'localtime'.
|
169
|
+
with_timezone("UTC-09") do
|
170
|
+
assert_equal("20140926 1400-1000", format(@fmt, true, "-10"))
|
171
|
+
end
|
158
172
|
end
|
159
173
|
end
|
160
174
|
|
161
|
-
|
162
|
-
|
163
|
-
|
175
|
+
sub_test_case 'Fluent::EventTime time' do
|
176
|
+
setup do
|
177
|
+
@time = Fluent::EventTime.from_time(Time.new(2014, 9, 27, 0, 0, 0, 0))
|
164
178
|
end
|
179
|
+
|
180
|
+
include TestLists
|
165
181
|
end
|
166
182
|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
183
|
+
# for v0.12 compatibility
|
184
|
+
sub_test_case 'Integer time' do
|
185
|
+
setup do
|
186
|
+
@time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
|
171
187
|
end
|
188
|
+
|
189
|
+
include TestLists
|
172
190
|
end
|
173
191
|
|
174
192
|
def test_format_with_subsec
|
175
|
-
time =
|
193
|
+
time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
|
194
|
+
time = Fluent::EventTime.new(time)
|
176
195
|
formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil)
|
177
196
|
assert_equal("20140927 0000.000000000", formatter.format(time))
|
178
197
|
end
|