fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +16 -5
- data/ADOPTERS.md +5 -0
- data/{ChangeLog → CHANGELOG.md} +495 -6
- data/CONTRIBUTING.md +5 -2
- data/GOVERNANCE.md +55 -0
- data/LICENSE +202 -0
- data/MAINTAINERS.md +7 -5
- data/README.md +17 -10
- data/bin/fluent-ca-generate +6 -0
- data/example/counter.conf +18 -0
- data/example/secondary_file.conf +3 -2
- data/fluentd.gemspec +3 -3
- data/lib/fluent/agent.rb +1 -1
- data/lib/fluent/command/binlog_reader.rb +11 -2
- data/lib/fluent/command/ca_generate.rb +181 -0
- data/lib/fluent/command/cat.rb +28 -15
- data/lib/fluent/command/debug.rb +4 -4
- data/lib/fluent/command/fluentd.rb +2 -2
- data/lib/fluent/command/plugin_config_formatter.rb +24 -2
- data/lib/fluent/command/plugin_generator.rb +26 -8
- data/lib/fluent/config/configure_proxy.rb +7 -1
- data/lib/fluent/config/dsl.rb +8 -5
- data/lib/fluent/config/element.rb +5 -0
- data/lib/fluent/config/literal_parser.rb +7 -1
- data/lib/fluent/config/types.rb +28 -2
- data/lib/fluent/config/v1_parser.rb +1 -2
- data/lib/fluent/configurable.rb +1 -0
- data/lib/fluent/counter.rb +23 -0
- data/lib/fluent/counter/base_socket.rb +46 -0
- data/lib/fluent/counter/client.rb +297 -0
- data/lib/fluent/counter/error.rb +86 -0
- data/lib/fluent/counter/mutex_hash.rb +163 -0
- data/lib/fluent/counter/server.rb +273 -0
- data/lib/fluent/counter/store.rb +205 -0
- data/lib/fluent/counter/validator.rb +145 -0
- data/lib/fluent/env.rb +1 -0
- data/lib/fluent/event_router.rb +1 -1
- data/lib/fluent/log.rb +119 -29
- data/lib/fluent/plugin/base.rb +12 -0
- data/lib/fluent/plugin/buf_file.rb +20 -16
- data/lib/fluent/plugin/buffer.rb +130 -32
- data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
- data/lib/fluent/plugin/compressable.rb +1 -1
- data/lib/fluent/plugin/filter_grep.rb +135 -21
- data/lib/fluent/plugin/filter_parser.rb +13 -2
- data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +5 -1
- data/lib/fluent/plugin/in_debug_agent.rb +8 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +84 -3
- data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
- data/lib/fluent/plugin/in_syslog.rb +31 -10
- data/lib/fluent/plugin/in_tail.rb +142 -53
- data/lib/fluent/plugin/in_tcp.rb +5 -6
- data/lib/fluent/plugin/in_udp.rb +6 -2
- data/lib/fluent/plugin/in_unix.rb +1 -1
- data/lib/fluent/plugin/multi_output.rb +1 -0
- data/lib/fluent/plugin/out_copy.rb +25 -2
- data/lib/fluent/plugin/out_file.rb +26 -7
- data/lib/fluent/plugin/out_forward.rb +81 -42
- data/lib/fluent/plugin/out_secondary_file.rb +2 -2
- data/lib/fluent/plugin/out_stdout.rb +0 -1
- data/lib/fluent/plugin/out_stream.rb +1 -1
- data/lib/fluent/plugin/output.rb +221 -57
- data/lib/fluent/plugin/parser_apache.rb +1 -1
- data/lib/fluent/plugin/parser_apache2.rb +5 -1
- data/lib/fluent/plugin/parser_apache_error.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +10 -3
- data/lib/fluent/plugin/parser_ltsv.rb +7 -0
- data/lib/fluent/plugin/parser_multiline.rb +2 -1
- data/lib/fluent/plugin/parser_nginx.rb +1 -1
- data/lib/fluent/plugin/parser_none.rb +1 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -14
- data/lib/fluent/plugin/parser_syslog.rb +9 -5
- data/lib/fluent/plugin_helper.rb +2 -0
- data/lib/fluent/plugin_helper/cert_option.rb +28 -9
- data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
- data/lib/fluent/plugin_helper/counter.rb +51 -0
- data/lib/fluent/plugin_helper/event_loop.rb +9 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
- data/lib/fluent/plugin_helper/retry_state.rb +15 -7
- data/lib/fluent/plugin_helper/server.rb +87 -25
- data/lib/fluent/plugin_helper/socket_option.rb +5 -2
- data/lib/fluent/plugin_helper/timer.rb +8 -7
- data/lib/fluent/root_agent.rb +18 -9
- data/lib/fluent/supervisor.rb +63 -23
- data/lib/fluent/system_config.rb +30 -2
- data/lib/fluent/test/helpers.rb +1 -1
- data/lib/fluent/time.rb +15 -7
- data/lib/fluent/timezone.rb +26 -2
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/README.md.erb +2 -2
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
- data/test/command/test_ca_generate.rb +70 -0
- data/test/command/test_fluentd.rb +2 -2
- data/test/command/test_plugin_config_formatter.rb +8 -7
- data/test/command/test_plugin_generator.rb +65 -39
- data/test/config/test_config_parser.rb +7 -2
- data/test/config/test_configurable.rb +7 -2
- data/test/config/test_configure_proxy.rb +41 -3
- data/test/config/test_dsl.rb +10 -10
- data/test/config/test_element.rb +10 -0
- data/test/config/test_literal_parser.rb +8 -0
- data/test/config/test_plugin_configuration.rb +56 -0
- data/test/config/test_system_config.rb +19 -1
- data/test/config/test_types.rb +37 -0
- data/test/counter/test_client.rb +559 -0
- data/test/counter/test_error.rb +44 -0
- data/test/counter/test_mutex_hash.rb +179 -0
- data/test/counter/test_server.rb +589 -0
- data/test/counter/test_store.rb +258 -0
- data/test/counter/test_validator.rb +137 -0
- data/test/plugin/test_buf_file.rb +124 -0
- data/test/plugin/test_buffer.rb +3 -2
- data/test/plugin/test_filter_grep.rb +580 -2
- data/test/plugin/test_filter_parser.rb +33 -2
- data/test/plugin/test_filter_record_transformer.rb +22 -1
- data/test/plugin/test_formatter_ltsv.rb +3 -0
- data/test/plugin/test_formatter_tsv.rb +68 -0
- data/test/plugin/test_in_debug_agent.rb +21 -0
- data/test/plugin/test_in_exec.rb +3 -5
- data/test/plugin/test_in_http.rb +178 -0
- data/test/plugin/test_in_monitor_agent.rb +1 -1
- data/test/plugin/test_in_syslog.rb +64 -0
- data/test/plugin/test_in_tail.rb +116 -6
- data/test/plugin/test_in_tcp.rb +21 -0
- data/test/plugin/test_in_udp.rb +78 -0
- data/test/plugin/test_metadata.rb +89 -0
- data/test/plugin/test_out_copy.rb +31 -0
- data/test/plugin/test_out_file.rb +108 -2
- data/test/plugin/test_out_forward.rb +195 -2
- data/test/plugin/test_out_secondary_file.rb +14 -0
- data/test/plugin/test_output.rb +159 -45
- data/test/plugin/test_output_as_buffered.rb +19 -0
- data/test/plugin/test_output_as_buffered_backup.rb +307 -0
- data/test/plugin/test_output_as_buffered_retries.rb +70 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin/test_parser_apache2.rb +1 -0
- data/test/plugin/test_parser_labeled_tsv.rb +17 -0
- data/test/plugin/test_parser_nginx.rb +40 -0
- data/test/plugin/test_parser_regexp.rb +6 -7
- data/test/plugin/test_parser_syslog.rb +155 -5
- data/test/plugin_helper/test_child_process.rb +4 -4
- data/test/plugin_helper/test_compat_parameters.rb +22 -0
- data/test/plugin_helper/test_record_accessor.rb +197 -0
- data/test/plugin_helper/test_retry_state.rb +20 -0
- data/test/plugin_helper/test_server.rb +30 -2
- data/test/test_config.rb +3 -3
- data/test/test_configdsl.rb +2 -2
- data/test/test_log.rb +51 -1
- data/test/test_root_agent.rb +33 -0
- data/test/test_supervisor.rb +105 -0
- metadata +68 -8
- data/COPYING +0 -14
@@ -544,7 +544,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
544
544
|
test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
|
545
545
|
written = []
|
546
546
|
chunks = []
|
547
|
-
priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false})
|
547
|
+
priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})
|
548
548
|
secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
|
549
549
|
@i.configure(config_element('ROOT','',{},[priconf,secconf]))
|
550
550
|
@i.register(:prefer_buffered_processing){ true }
|
@@ -125,4 +125,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
|
|
125
125
|
assert_equal record['b'], ' '
|
126
126
|
end
|
127
127
|
end
|
128
|
+
|
129
|
+
data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
|
130
|
+
"multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
|
131
|
+
"reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
|
132
|
+
"tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
|
133
|
+
"tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
|
134
|
+
def test_parse_with_delimiter_pattern(data)
|
135
|
+
text, expected = data
|
136
|
+
parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
|
137
|
+
parser.configure(
|
138
|
+
'delimiter_pattern' => '/\s+/',
|
139
|
+
'label_delimiter' => '='
|
140
|
+
)
|
141
|
+
parser.instance.parse(text) do |_time, record|
|
142
|
+
assert_equal(expected, record)
|
143
|
+
end
|
144
|
+
end
|
128
145
|
end
|
@@ -16,6 +16,30 @@ class NginxParserTest < ::Test::Unit::TestCase
|
|
16
16
|
'referer' => '-',
|
17
17
|
'agent' => 'Opera/12.0'
|
18
18
|
}
|
19
|
+
@expected_extended = {
|
20
|
+
'remote' => '127.0.0.1',
|
21
|
+
'host' => '192.168.0.1',
|
22
|
+
'user' => '-',
|
23
|
+
'method' => 'GET',
|
24
|
+
'path' => '/',
|
25
|
+
'code' => '200',
|
26
|
+
'size' => '777',
|
27
|
+
'referer' => '-',
|
28
|
+
'agent' => 'Opera/12.0',
|
29
|
+
'http_x_forwarded_for' => '-'
|
30
|
+
}
|
31
|
+
@expected_extended_multiple_ip = {
|
32
|
+
'remote' => '127.0.0.1',
|
33
|
+
'host' => '192.168.0.1',
|
34
|
+
'user' => '-',
|
35
|
+
'method' => 'GET',
|
36
|
+
'path' => '/',
|
37
|
+
'code' => '200',
|
38
|
+
'size' => '777',
|
39
|
+
'referer' => '-',
|
40
|
+
'agent' => 'Opera/12.0',
|
41
|
+
'http_x_forwarded_for' => '127.0.0.1, 192.168.0.1'
|
42
|
+
}
|
19
43
|
end
|
20
44
|
|
21
45
|
def create_driver
|
@@ -45,4 +69,20 @@ class NginxParserTest < ::Test::Unit::TestCase
|
|
45
69
|
assert_equal(@expected, record)
|
46
70
|
}
|
47
71
|
end
|
72
|
+
|
73
|
+
def test_parse_with_http_x_forwarded_for
|
74
|
+
d = create_driver
|
75
|
+
d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
|
76
|
+
assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
77
|
+
assert_equal(@expected_extended, record)
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_parse_with_http_x_forwarded_for_multiple_ip
|
82
|
+
d = create_driver
|
83
|
+
d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" "127.0.0.1, 192.168.0.1"') { |time, record|
|
84
|
+
assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
|
85
|
+
assert_equal(@expected_extended_multiple_ip, record)
|
86
|
+
}
|
87
|
+
end
|
48
88
|
end
|
@@ -160,25 +160,24 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
160
160
|
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
161
161
|
}
|
162
162
|
d = create_driver(conf)
|
163
|
-
regexp = d.instance.
|
163
|
+
regexp = d.instance.expression
|
164
164
|
assert_equal(0, regexp.options)
|
165
165
|
end
|
166
166
|
|
167
167
|
data(
|
168
|
-
ignorecase: [
|
169
|
-
multiline: [
|
170
|
-
ignorecase_multiline: [
|
168
|
+
ignorecase: ["i", Regexp::IGNORECASE],
|
169
|
+
multiline: ["m", Regexp::MULTILINE],
|
170
|
+
ignorecase_multiline: ["im", Regexp::IGNORECASE | Regexp::MULTILINE],
|
171
171
|
)
|
172
172
|
def test_options(data)
|
173
173
|
regexp_option, expected = data
|
174
174
|
conf = {
|
175
|
-
'expression' => %
|
175
|
+
'expression' => %Q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/#{regexp_option}!,
|
176
176
|
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
177
177
|
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
178
178
|
}
|
179
|
-
conf = conf.merge(regexp_option)
|
180
179
|
d = create_driver(conf)
|
181
|
-
regexp = d.instance.
|
180
|
+
regexp = d.instance.expression
|
182
181
|
assert_equal(expected, regexp.options)
|
183
182
|
end
|
184
183
|
end
|
@@ -64,11 +64,30 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
+
def test_parse_various_characters_for_tag
|
68
|
+
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
|
69
|
+
@parser.configure({})
|
70
|
+
@parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
|
71
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
72
|
+
assert_equal(@expected.merge('ident' => ident), record)
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_parse_various_characters_for_tag_with_priority
|
77
|
+
ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
|
78
|
+
@parser.configure('with_priority' => true)
|
79
|
+
@parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
|
80
|
+
assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
|
81
|
+
assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
67
85
|
class TestRFC5424Regexp < self
|
68
86
|
def test_parse_with_rfc5424_message
|
69
87
|
@parser.configure(
|
70
88
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
71
89
|
'message_format' => 'rfc5424',
|
90
|
+
'with_priority' => true,
|
72
91
|
)
|
73
92
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
74
93
|
@parser.instance.parse(text) do |time, record|
|
@@ -78,11 +97,48 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
78
97
|
assert_equal "-", record["extradata"]
|
79
98
|
assert_equal "Hi, from Fluentd!", record["message"]
|
80
99
|
end
|
100
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
101
|
+
@parser.instance.patterns['format'])
|
102
|
+
end
|
103
|
+
|
104
|
+
def test_parse_with_rfc5424_message_and_without_priority
|
105
|
+
@parser.configure(
|
106
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
107
|
+
'message_format' => 'rfc5424',
|
108
|
+
)
|
109
|
+
text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
110
|
+
@parser.instance.parse(text) do |time, record|
|
111
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
112
|
+
assert_equal "-", record["pid"]
|
113
|
+
assert_equal "-", record["msgid"]
|
114
|
+
assert_equal "-", record["extradata"]
|
115
|
+
assert_equal "Hi, from Fluentd!", record["message"]
|
116
|
+
end
|
117
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
|
118
|
+
@parser.instance.patterns['format'])
|
119
|
+
end
|
120
|
+
|
121
|
+
def test_parse_with_rfc5424_empty_message_and_without_priority
|
122
|
+
@parser.configure(
|
123
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
124
|
+
'message_format' => 'rfc5424',
|
125
|
+
)
|
126
|
+
text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
|
127
|
+
@parser.instance.parse(text) do |time, record|
|
128
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
129
|
+
assert_equal "-", record["pid"]
|
130
|
+
assert_equal "-", record["msgid"]
|
131
|
+
assert_equal "-", record["extradata"]
|
132
|
+
assert_nil record["message"]
|
133
|
+
end
|
134
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
|
135
|
+
@parser.instance.patterns['format'])
|
81
136
|
end
|
82
137
|
|
83
138
|
def test_parse_with_rfc5424_message_without_time_format
|
84
139
|
@parser.configure(
|
85
140
|
'message_format' => 'rfc5424',
|
141
|
+
'with_priority' => true,
|
86
142
|
)
|
87
143
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
88
144
|
@parser.instance.parse(text) do |time, record|
|
@@ -94,21 +150,88 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
94
150
|
end
|
95
151
|
end
|
96
152
|
|
153
|
+
def test_parse_with_rfc5424_message_with_priority_and_pid
|
154
|
+
@parser.configure(
|
155
|
+
'message_format' => 'rfc5424',
|
156
|
+
'with_priority' => true,
|
157
|
+
)
|
158
|
+
text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - - peer 192.168.0.5:50123 is not from a LAN'
|
159
|
+
@parser.instance.parse(text) do |time, record|
|
160
|
+
assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
161
|
+
assert_equal "1298", record["pid"]
|
162
|
+
assert_equal "-", record["msgid"]
|
163
|
+
assert_equal "-", record["extradata"]
|
164
|
+
assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
97
168
|
def test_parse_with_rfc5424_structured_message
|
98
169
|
@parser.configure(
|
99
170
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
100
171
|
'message_format' => 'rfc5424',
|
172
|
+
'with_priority' => true,
|
101
173
|
)
|
102
174
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
103
175
|
@parser.instance.parse(text) do |time, record|
|
104
176
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
105
|
-
assert_equal "11111", record["pid"]
|
177
|
+
assert_equal "11111", record["pid"]
|
106
178
|
assert_equal "ID24224", record["msgid"]
|
107
179
|
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
|
108
180
|
record["extradata"]
|
109
181
|
assert_equal "Hi, from Fluentd!", record["message"]
|
110
182
|
end
|
111
183
|
end
|
184
|
+
|
185
|
+
def test_parse_with_rfc5424_multiple_structured_message
|
186
|
+
@parser.configure(
|
187
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
188
|
+
'message_format' => 'rfc5424',
|
189
|
+
'with_priority' => true,
|
190
|
+
)
|
191
|
+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
|
192
|
+
@parser.instance.parse(text) do |time, record|
|
193
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
194
|
+
assert_equal "11111", record["pid"]
|
195
|
+
assert_equal "ID24224", record["msgid"]
|
196
|
+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
|
197
|
+
record["extradata"]
|
198
|
+
assert_equal "Hi, from Fluentd!", record["message"]
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def test_parse_with_rfc5424_message_includes_right_bracket
|
203
|
+
@parser.configure(
|
204
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
205
|
+
'message_format' => 'rfc5424',
|
206
|
+
'with_priority' => true,
|
207
|
+
)
|
208
|
+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
|
209
|
+
@parser.instance.parse(text) do |time, record|
|
210
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
211
|
+
assert_equal "11111", record["pid"]
|
212
|
+
assert_equal "ID24224", record["msgid"]
|
213
|
+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
|
214
|
+
record["extradata"]
|
215
|
+
assert_equal "Hi, from Fluentd]!", record["message"]
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
def test_parse_with_rfc5424_empty_message
|
220
|
+
@parser.configure(
|
221
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
222
|
+
'message_format' => 'rfc5424',
|
223
|
+
'with_priority' => true,
|
224
|
+
)
|
225
|
+
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
|
226
|
+
@parser.instance.parse(text) do |time, record|
|
227
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
228
|
+
assert_equal "11111", record["pid"]
|
229
|
+
assert_equal "ID24224", record["msgid"]
|
230
|
+
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
|
231
|
+
record["extradata"]
|
232
|
+
assert_nil record["message"]
|
233
|
+
end
|
234
|
+
end
|
112
235
|
end
|
113
236
|
|
114
237
|
class TestAutoRegexp < self
|
@@ -122,6 +245,7 @@ assert_equal "11111", record["pid"]
|
|
122
245
|
assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
|
123
246
|
assert_equal(@expected, record)
|
124
247
|
end
|
248
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
|
125
249
|
end
|
126
250
|
|
127
251
|
def test_auto_with_legacy_syslog_priority_message
|
@@ -135,12 +259,14 @@ assert_equal "11111", record["pid"]
|
|
135
259
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
136
260
|
assert_equal(@expected.merge('pri' => 6), record)
|
137
261
|
end
|
262
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
138
263
|
end
|
139
264
|
|
140
265
|
def test_parse_with_rfc5424_message
|
141
266
|
@parser.configure(
|
142
267
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
143
268
|
'message_format' => 'auto',
|
269
|
+
'with_priority' => true,
|
144
270
|
)
|
145
271
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
146
272
|
@parser.instance.parse(text) do |time, record|
|
@@ -150,12 +276,15 @@ assert_equal "11111", record["pid"]
|
|
150
276
|
assert_equal "-", record["extradata"]
|
151
277
|
assert_equal "Hi, from Fluentd!", record["message"]
|
152
278
|
end
|
279
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
280
|
+
@parser.instance.patterns['format'])
|
153
281
|
end
|
154
282
|
|
155
283
|
def test_parse_with_rfc5424_structured_message
|
156
284
|
@parser.configure(
|
157
285
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
158
286
|
'message_format' => 'auto',
|
287
|
+
'with_priority' => true,
|
159
288
|
)
|
160
289
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
161
290
|
@parser.instance.parse(text) do |time, record|
|
@@ -166,6 +295,8 @@ assert_equal "11111", record["pid"]
|
|
166
295
|
record["extradata"]
|
167
296
|
assert_equal "Hi, from Fluentd!", record["message"]
|
168
297
|
end
|
298
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
299
|
+
@parser.instance.patterns['format'])
|
169
300
|
end
|
170
301
|
|
171
302
|
def test_parse_with_both_message_type
|
@@ -173,12 +304,15 @@ assert_equal "11111", record["pid"]
|
|
173
304
|
'time_format' => '%b %d %M:%S:%H',
|
174
305
|
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
175
306
|
'message_format' => 'auto',
|
307
|
+
'with_priority' => true,
|
176
308
|
)
|
177
|
-
text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
309
|
+
text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
178
310
|
@parser.instance.parse(text) do |time, record|
|
179
311
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
180
|
-
assert_equal(@expected, record)
|
312
|
+
assert_equal(@expected.merge('pri' => 1), record)
|
181
313
|
end
|
314
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
315
|
+
|
182
316
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
183
317
|
@parser.instance.parse(text) do |time, record|
|
184
318
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -188,11 +322,16 @@ assert_equal "11111", record["pid"]
|
|
188
322
|
record["extradata"]
|
189
323
|
assert_equal "Hi, from Fluentd!", record["message"]
|
190
324
|
end
|
191
|
-
|
325
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
326
|
+
@parser.instance.patterns['format'])
|
327
|
+
|
328
|
+
text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
192
329
|
@parser.instance.parse(text) do |time, record|
|
193
330
|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
|
194
|
-
assert_equal(@expected, record)
|
331
|
+
assert_equal(@expected.merge('pri' => 1), record)
|
195
332
|
end
|
333
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
334
|
+
|
196
335
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
197
336
|
@parser.instance.parse(text) do |time, record|
|
198
337
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -201,6 +340,8 @@ assert_equal "11111", record["pid"]
|
|
201
340
|
assert_equal "-", record["extradata"]
|
202
341
|
assert_equal "Hi, from Fluentd!", record["message"]
|
203
342
|
end
|
343
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
344
|
+
@parser.instance.patterns['format'])
|
204
345
|
end
|
205
346
|
|
206
347
|
def test_parse_with_both_message_type_and_priority
|
@@ -215,6 +356,8 @@ assert_equal "11111", record["pid"]
|
|
215
356
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
216
357
|
assert_equal(@expected.merge('pri' => 6), record)
|
217
358
|
end
|
359
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
360
|
+
|
218
361
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
219
362
|
@parser.instance.parse(text) do |time, record|
|
220
363
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -224,11 +367,16 @@ assert_equal "11111", record["pid"]
|
|
224
367
|
record["extradata"]
|
225
368
|
assert_equal "Hi, from Fluentd!", record["message"]
|
226
369
|
end
|
370
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
371
|
+
@parser.instance.patterns['format'])
|
372
|
+
|
227
373
|
text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
228
374
|
@parser.instance.parse(text) do |time, record|
|
229
375
|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
|
230
376
|
assert_equal(@expected.merge('pri' => 16), record)
|
231
377
|
end
|
378
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
|
379
|
+
|
232
380
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
233
381
|
@parser.instance.parse(text) do |time, record|
|
234
382
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -237,6 +385,8 @@ assert_equal "11111", record["pid"]
|
|
237
385
|
assert_equal "-", record["extradata"]
|
238
386
|
assert_equal "Hi, from Fluentd!", record["message"]
|
239
387
|
end
|
388
|
+
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
389
|
+
@parser.instance.patterns['format'])
|
240
390
|
end
|
241
391
|
end
|
242
392
|
end
|
@@ -510,7 +510,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
510
510
|
unless Fluent.windows?
|
511
511
|
test 'can specify subprocess name' do
|
512
512
|
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
513
|
-
process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
|
513
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
|
514
514
|
Process.kill(:TERM, io.pid) rescue nil
|
515
515
|
io.close rescue nil
|
516
516
|
|
@@ -527,15 +527,15 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
527
527
|
m.lock
|
528
528
|
ran = true
|
529
529
|
pids << @d.child_process_id
|
530
|
-
proc_lines += open("|ps"){|_io| _io.readlines }
|
530
|
+
proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
|
531
531
|
m.unlock
|
532
532
|
readio.read
|
533
533
|
end
|
534
534
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
535
535
|
m.lock
|
536
536
|
pid = pids.first
|
537
|
-
#
|
538
|
-
assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[
|
537
|
+
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
538
|
+
assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
539
539
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
540
540
|
end
|
541
541
|
end
|
@@ -328,4 +328,26 @@ class CompatParameterTest < Test::Unit::TestCase
|
|
328
328
|
# TODO:
|
329
329
|
end
|
330
330
|
end
|
331
|
+
|
332
|
+
sub_test_case 'parser plugins' do
|
333
|
+
test 'syslog parser parameters' do
|
334
|
+
hash = {
|
335
|
+
'format' => 'syslog',
|
336
|
+
'message_format' => 'rfc5424',
|
337
|
+
'with_priority' => 'true',
|
338
|
+
'rfc5424_time_format' => '%Y'
|
339
|
+
}
|
340
|
+
conf = config_element('ROOT', '', hash)
|
341
|
+
@i = DummyI0.new
|
342
|
+
@i.configure(conf)
|
343
|
+
@i.start
|
344
|
+
@i.after_start
|
345
|
+
|
346
|
+
parser = @i.parser
|
347
|
+
assert_kind_of(Fluent::Plugin::SyslogParser, parser)
|
348
|
+
assert_equal :rfc5424, parser.message_format
|
349
|
+
assert_equal true, parser.with_priority
|
350
|
+
assert_equal '%Y', parser.rfc5424_time_format
|
351
|
+
end
|
352
|
+
end
|
331
353
|
end
|