fluentd 1.9.3 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/CHANGELOG.md +182 -0
- data/CONTRIBUTING.md +1 -1
- data/README.md +4 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/lib/fluent/command/debug.rb +1 -0
- data/lib/fluent/command/fluentd.rb +25 -1
- data/lib/fluent/config.rb +1 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/log.rb +45 -6
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +13 -6
- data/lib/fluent/plugin/buffer.rb +2 -2
- data/lib/fluent/plugin/in_dummy.rb +3 -3
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_gc_stat.rb +16 -0
- data/lib/fluent/plugin/in_http.rb +146 -75
- data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +4 -4
- data/lib/fluent/plugin/in_tail.rb +40 -31
- data/lib/fluent/plugin/in_tail/position_file.rb +23 -6
- data/lib/fluent/plugin/in_unix.rb +77 -77
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +25 -22
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +4 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
- data/lib/fluent/plugin/out_http.rb +15 -2
- data/lib/fluent/plugin/parser_multiline.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +303 -62
- data/lib/fluent/plugin/sd_file.rb +1 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin_helper/cert_option.rb +15 -2
- data/lib/fluent/plugin_helper/child_process.rb +3 -2
- data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
- data/lib/fluent/plugin_helper/server.rb +3 -1
- data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
- data/lib/fluent/plugin_helper/socket.rb +20 -2
- data/lib/fluent/plugin_helper/socket_option.rb +21 -3
- data/lib/fluent/supervisor.rb +21 -9
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/test/filter_test.rb +2 -2
- data/lib/fluent/test/output_test.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +71 -12
- data/test/config/test_system_config.rb +2 -0
- data/test/helper.rb +2 -2
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +132 -0
- data/test/plugin/in_tail/test_position_file.rb +25 -1
- data/test/plugin/out_forward/test_handshake_protocol.rb +10 -1
- data/test/plugin/out_forward/test_load_balancer.rb +46 -0
- data/test/plugin/test_buf_file.rb +3 -1
- data/test/plugin/test_buffer.rb +20 -0
- data/test/plugin/test_compressable.rb +7 -4
- data/test/plugin/test_in_dummy.rb +12 -14
- data/test/plugin/test_in_forward.rb +2 -2
- data/test/plugin/test_in_gc_stat.rb +24 -1
- data/test/plugin/test_in_http.rb +57 -0
- data/test/plugin/test_in_syslog.rb +16 -1
- data/test/plugin/test_in_tail.rb +43 -20
- data/test/plugin/test_in_unix.rb +128 -73
- data/test/plugin/test_out_forward.rb +39 -3
- data/test/plugin/test_out_http.rb +38 -0
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_output_as_buffered_retries.rb +12 -4
- data/test/plugin/test_output_as_buffered_secondary.rb +9 -1
- data/test/plugin/test_parser_syslog.rb +106 -46
- data/test/plugin/test_sd_file.rb +17 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
- data/test/plugin_helper/http_server/test_app.rb +1 -1
- data/test/plugin_helper/http_server/test_route.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +2 -0
- data/test/plugin_helper/test_child_process.rb +20 -3
- data/test/plugin_helper/test_http_server_helper.rb +2 -2
- data/test/plugin_helper/test_record_accessor.rb +41 -0
- data/test/plugin_helper/test_server.rb +1 -1
- data/test/plugin_helper/test_service_discovery.rb +37 -4
- data/test/plugin_helper/test_socket.rb +131 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_log.rb +44 -0
- data/test/test_msgpack_factory.rb +18 -0
- metadata +28 -2
|
@@ -6,7 +6,7 @@ require 'tempfile'
|
|
|
6
6
|
|
|
7
7
|
class IntailPositionFileTest < Test::Unit::TestCase
|
|
8
8
|
setup do
|
|
9
|
-
@file = Tempfile.new('intail_position_file_test')
|
|
9
|
+
@file = Tempfile.new('intail_position_file_test').binmode
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
teardown do
|
|
@@ -77,6 +77,30 @@ class IntailPositionFileTest < Test::Unit::TestCase
|
|
|
77
77
|
lines = @file.readlines
|
|
78
78
|
assert_equal 5, lines.size
|
|
79
79
|
end
|
|
80
|
+
|
|
81
|
+
test 'update seek postion of remained position entry' do
|
|
82
|
+
pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
|
|
83
|
+
pf['path1']
|
|
84
|
+
pf['path2']
|
|
85
|
+
pf['path3']
|
|
86
|
+
pf.unwatch('path1')
|
|
87
|
+
|
|
88
|
+
pf.try_compact
|
|
89
|
+
|
|
90
|
+
@file.seek(0)
|
|
91
|
+
lines = @file.readlines
|
|
92
|
+
assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0]
|
|
93
|
+
assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
|
|
94
|
+
assert_equal 2, lines.size
|
|
95
|
+
|
|
96
|
+
pf.unwatch('path2')
|
|
97
|
+
pf.unwatch('path3')
|
|
98
|
+
@file.seek(0)
|
|
99
|
+
lines = @file.readlines
|
|
100
|
+
assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
|
|
101
|
+
assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
|
|
102
|
+
assert_equal 2, lines.size
|
|
103
|
+
end
|
|
80
104
|
end
|
|
81
105
|
|
|
82
106
|
sub_test_case '#load' do
|
|
@@ -81,6 +81,15 @@ class HandshakeProtocolTest < Test::Unit::TestCase
|
|
|
81
81
|
assert_equal(ri.state, :established)
|
|
82
82
|
end
|
|
83
83
|
|
|
84
|
+
test 'raises an error when password and username are nil if auth exists' do
|
|
85
|
+
handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
|
|
86
|
+
ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
|
|
87
|
+
|
|
88
|
+
assert_raise(Fluent::Plugin::ForwardOutput::PingpongError.new('username and password are required')) do
|
|
89
|
+
handshake.invoke('', ri, ['HELO', { 'auth' => 'auth' }])
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
84
93
|
data(
|
|
85
94
|
lack_of_elem: ['PONG', true, '', 'client_hostname'],
|
|
86
95
|
wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
|
|
@@ -89,7 +98,7 @@ class HandshakeProtocolTest < Test::Unit::TestCase
|
|
|
89
98
|
wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],
|
|
90
99
|
)
|
|
91
100
|
test 'raises an error when message is' do |msg|
|
|
92
|
-
handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password:
|
|
101
|
+
handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: '', username: '')
|
|
93
102
|
handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
|
|
94
103
|
|
|
95
104
|
ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
|
|
@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
|
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
+
test 'call like round robin without weight=0 node' do
|
|
53
|
+
lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
|
|
54
|
+
n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
|
|
55
|
+
n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
|
|
56
|
+
n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
|
|
57
|
+
|
|
58
|
+
lb.rebuild_weight_array([n1, n2, n3])
|
|
59
|
+
|
|
60
|
+
lb.select_healthy_node do |node|
|
|
61
|
+
# to handle random choice
|
|
62
|
+
if node == n1
|
|
63
|
+
lb.select_healthy_node do |node|
|
|
64
|
+
assert_equal(node, n2)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
lb.select_healthy_node do |node|
|
|
68
|
+
assert_equal(node, n1)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
lb.select_healthy_node do |node|
|
|
72
|
+
assert_equal(node, n2)
|
|
73
|
+
end
|
|
74
|
+
else
|
|
75
|
+
lb.select_healthy_node do |node|
|
|
76
|
+
assert_equal(node, n1)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
lb.select_healthy_node do |node|
|
|
80
|
+
assert_equal(node, n2)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
lb.select_healthy_node do |node|
|
|
84
|
+
assert_equal(node, n1)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
52
90
|
test 'raise an error if all node are unavialble' do
|
|
53
91
|
lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
|
|
54
92
|
lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
|
|
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
|
|
|
56
94
|
lb.select_healthy_node
|
|
57
95
|
end
|
|
58
96
|
end
|
|
97
|
+
|
|
98
|
+
test 'it regards weight=0 node as unavialble' do
|
|
99
|
+
lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
|
|
100
|
+
lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
|
|
101
|
+
assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
|
|
102
|
+
lb.select_healthy_node
|
|
103
|
+
end
|
|
104
|
+
end
|
|
59
105
|
end
|
|
60
106
|
end
|
|
@@ -1062,7 +1062,9 @@ class FileBufferTest < Test::Unit::TestCase
|
|
|
1062
1062
|
if @bufdir
|
|
1063
1063
|
Dir.glob(File.join(@bufdir, '*')).each do |path|
|
|
1064
1064
|
next if ['.', '..'].include?(File.basename(path))
|
|
1065
|
-
|
|
1065
|
+
# Windows does not permit to delete files which are used in another process.
|
|
1066
|
+
# Just ignore for removing failure.
|
|
1067
|
+
File.delete(path) rescue nil
|
|
1066
1068
|
end
|
|
1067
1069
|
end
|
|
1068
1070
|
end
|
data/test/plugin/test_buffer.rb
CHANGED
|
@@ -1202,4 +1202,24 @@ class BufferTest < Test::Unit::TestCase
|
|
|
1202
1202
|
assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
|
|
1203
1203
|
end
|
|
1204
1204
|
end
|
|
1205
|
+
|
|
1206
|
+
sub_test_case '#statistics' do
|
|
1207
|
+
setup do
|
|
1208
|
+
@p = create_buffer({ "total_limit_size" => 1024 })
|
|
1209
|
+
dm = create_metadata(Time.parse('2020-03-13 16:00:00 +0000').to_i, nil, nil)
|
|
1210
|
+
|
|
1211
|
+
(class << @p; self; end).module_eval do
|
|
1212
|
+
define_method(:resume) {
|
|
1213
|
+
queued = [create_chunk(dm, ["a" * (1024 - 102)]).enqueued!]
|
|
1214
|
+
return {}, queued
|
|
1215
|
+
}
|
|
1216
|
+
end
|
|
1217
|
+
|
|
1218
|
+
@p.start
|
|
1219
|
+
end
|
|
1220
|
+
|
|
1221
|
+
test 'returns available_buffer_space_ratios' do
|
|
1222
|
+
assert_equal 10.0, @p.statistics['buffer']['available_buffer_space_ratios']
|
|
1223
|
+
end
|
|
1224
|
+
end
|
|
1205
1225
|
end
|
|
@@ -4,6 +4,12 @@ require 'fluent/plugin/compressable'
|
|
|
4
4
|
class CompressableTest < Test::Unit::TestCase
|
|
5
5
|
include Fluent::Plugin::Compressable
|
|
6
6
|
|
|
7
|
+
def compress_assert_equal(expected, actual)
|
|
8
|
+
e = Zlib::GzipReader.new(StringIO.new(expected)).read
|
|
9
|
+
a = Zlib::GzipReader.new(StringIO.new(actual)).read
|
|
10
|
+
assert_equal(e, a)
|
|
11
|
+
end
|
|
12
|
+
|
|
7
13
|
sub_test_case '#compress' do
|
|
8
14
|
setup do
|
|
9
15
|
@src = 'text data for compressing' * 5
|
|
@@ -18,8 +24,7 @@ class CompressableTest < Test::Unit::TestCase
|
|
|
18
24
|
test 'write compressed data to IO with output_io option' do
|
|
19
25
|
io = StringIO.new
|
|
20
26
|
compress(@src, output_io: io)
|
|
21
|
-
|
|
22
|
-
assert_equal @gzipped_src, io.string
|
|
27
|
+
compress_assert_equal @gzipped_src, io.string
|
|
23
28
|
end
|
|
24
29
|
end
|
|
25
30
|
|
|
@@ -36,7 +41,6 @@ class CompressableTest < Test::Unit::TestCase
|
|
|
36
41
|
test 'write decompressed data to IO with output_io option' do
|
|
37
42
|
io = StringIO.new
|
|
38
43
|
decompress(@gzipped_src, output_io: io)
|
|
39
|
-
waiting(10){ sleep 0.1 until @src == io.string }
|
|
40
44
|
assert_equal @src, io.string
|
|
41
45
|
end
|
|
42
46
|
|
|
@@ -58,7 +62,6 @@ class CompressableTest < Test::Unit::TestCase
|
|
|
58
62
|
output_io = StringIO.new
|
|
59
63
|
|
|
60
64
|
decompress(input_io: input_io, output_io: output_io)
|
|
61
|
-
waiting(10){ sleep 0.1 until @src == output_io.string }
|
|
62
65
|
assert_equal @src, output_io.string
|
|
63
66
|
end
|
|
64
67
|
|
|
@@ -95,21 +95,20 @@ class DummyTest < Test::Unit::TestCase
|
|
|
95
95
|
TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
|
|
96
96
|
FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
|
|
97
97
|
|
|
98
|
-
sub_test_case
|
|
98
|
+
sub_test_case 'when dummy plugin has storage which is not specified the path' do
|
|
99
99
|
config1 = {
|
|
100
100
|
'tag' => 'dummy',
|
|
101
|
-
'rate' => '
|
|
101
|
+
'rate' => '0',
|
|
102
102
|
'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
|
|
103
103
|
'auto_increment_key' => 'id',
|
|
104
|
-
'suspend' => false,
|
|
105
104
|
}
|
|
106
105
|
conf1 = config_element('ROOT', '', config1, [])
|
|
107
|
-
test "value of auto increment key is not
|
|
106
|
+
test "value of auto increment key is not kept after stop-and-start" do
|
|
108
107
|
assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
|
|
109
108
|
|
|
110
109
|
d1 = create_driver(conf1)
|
|
111
110
|
d1.run(timeout: 0.5) do
|
|
112
|
-
d1.instance.emit(
|
|
111
|
+
d1.instance.emit(2)
|
|
113
112
|
end
|
|
114
113
|
|
|
115
114
|
events = d1.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
|
|
@@ -124,7 +123,7 @@ class DummyTest < Test::Unit::TestCase
|
|
|
124
123
|
|
|
125
124
|
d2 = create_driver(conf1)
|
|
126
125
|
d2.run(timeout: 0.5) do
|
|
127
|
-
d2.instance.emit(
|
|
126
|
+
d2.instance.emit(2)
|
|
128
127
|
end
|
|
129
128
|
|
|
130
129
|
events = d2.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
|
|
@@ -136,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
|
|
|
136
135
|
end
|
|
137
136
|
end
|
|
138
137
|
|
|
139
|
-
sub_test_case
|
|
138
|
+
sub_test_case 'when dummy plugin has storage which is specified the path' do
|
|
140
139
|
setup do
|
|
141
140
|
FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
|
|
142
141
|
FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
|
|
@@ -146,10 +145,9 @@ class DummyTest < Test::Unit::TestCase
|
|
|
146
145
|
config2 = {
|
|
147
146
|
'@id' => 'test-02',
|
|
148
147
|
'tag' => 'dummy',
|
|
149
|
-
'rate' => '
|
|
148
|
+
'rate' => '0',
|
|
150
149
|
'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
|
|
151
150
|
'auto_increment_key' => 'id',
|
|
152
|
-
'suspend' => true,
|
|
153
151
|
}
|
|
154
152
|
conf2 = config_element('ROOT', '', config2, [
|
|
155
153
|
config_element(
|
|
@@ -161,12 +159,12 @@ class DummyTest < Test::Unit::TestCase
|
|
|
161
159
|
'persistent' => true,
|
|
162
160
|
})
|
|
163
161
|
])
|
|
164
|
-
test "value of auto increment key is
|
|
162
|
+
test "value of auto increment key is kept after stop-and-start" do
|
|
165
163
|
assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
|
|
166
164
|
|
|
167
165
|
d1 = create_driver(conf2)
|
|
168
|
-
d1.run(timeout:
|
|
169
|
-
d1.instance.emit(
|
|
166
|
+
d1.run(timeout: 1) do
|
|
167
|
+
d1.instance.emit(2)
|
|
170
168
|
end
|
|
171
169
|
|
|
172
170
|
first_id1 = d1.events.first[2]['id']
|
|
@@ -178,8 +176,8 @@ class DummyTest < Test::Unit::TestCase
|
|
|
178
176
|
assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
|
|
179
177
|
|
|
180
178
|
d2 = create_driver(conf2)
|
|
181
|
-
d2.run(timeout:
|
|
182
|
-
d2.instance.emit(
|
|
179
|
+
d2.run(timeout: 1) do
|
|
180
|
+
d2.instance.emit(2)
|
|
183
181
|
end
|
|
184
182
|
d2.events
|
|
185
183
|
|
|
@@ -218,7 +218,7 @@ class ForwardInputTest < Test::Unit::TestCase
|
|
|
218
218
|
}
|
|
219
219
|
end
|
|
220
220
|
|
|
221
|
-
assert_equal(records, d.events.sort_by {|a| a[
|
|
221
|
+
assert_equal(records, d.events.sort_by {|a| a[0] })
|
|
222
222
|
end
|
|
223
223
|
|
|
224
224
|
test 'json_with_newline' do
|
|
@@ -237,7 +237,7 @@ class ForwardInputTest < Test::Unit::TestCase
|
|
|
237
237
|
}
|
|
238
238
|
end
|
|
239
239
|
|
|
240
|
-
assert_equal(records, d.events.sort_by {|a| a[
|
|
240
|
+
assert_equal(records, d.events.sort_by {|a| a[0] })
|
|
241
241
|
end
|
|
242
242
|
end
|
|
243
243
|
|
|
@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
|
|
|
22
22
|
assert_equal("t1", d.instance.tag)
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
def
|
|
25
|
+
def setup_gc_stat
|
|
26
26
|
stat = GC.stat
|
|
27
27
|
stub(GC).stat { stat }
|
|
28
|
+
stat
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def test_emit
|
|
32
|
+
stat = setup_gc_stat
|
|
28
33
|
|
|
29
34
|
d = create_driver
|
|
30
35
|
d.run(expect_emits: 2)
|
|
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
|
|
|
36
41
|
assert(events[i][1].is_a?(Fluent::EventTime))
|
|
37
42
|
}
|
|
38
43
|
end
|
|
44
|
+
|
|
45
|
+
def test_emit_with_use_symbol_keys_false
|
|
46
|
+
stat = setup_gc_stat
|
|
47
|
+
result = {}
|
|
48
|
+
stat.each_pair { |k, v|
|
|
49
|
+
result[k.to_s] = v
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
d = create_driver(CONFIG + "use_symbol_keys false")
|
|
53
|
+
d.run(expect_emits: 2)
|
|
54
|
+
|
|
55
|
+
events = d.events
|
|
56
|
+
assert(events.length > 0)
|
|
57
|
+
events.each_index {|i|
|
|
58
|
+
assert_equal(result, events[i][2])
|
|
59
|
+
assert(events[i][1].is_a?(Fluent::EventTime))
|
|
60
|
+
}
|
|
61
|
+
end
|
|
39
62
|
end
|
data/test/plugin/test_in_http.rb
CHANGED
|
@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
|
|
|
116
116
|
assert_equal_event_time time, d.events[1][1]
|
|
117
117
|
end
|
|
118
118
|
|
|
119
|
+
data('json' => ['json', :to_json],
|
|
120
|
+
'msgpack' => ['msgpack', :to_msgpack])
|
|
121
|
+
def test_default_with_time_format(data)
|
|
122
|
+
param, method_name = data
|
|
123
|
+
d = create_driver(CONFIG + %[
|
|
124
|
+
<parse>
|
|
125
|
+
keep_time_key
|
|
126
|
+
time_format %iso8601
|
|
127
|
+
</parse>
|
|
128
|
+
])
|
|
129
|
+
|
|
130
|
+
time = event_time("2020-06-10T01:14:27+00:00")
|
|
131
|
+
events = [
|
|
132
|
+
["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
|
|
133
|
+
["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
|
|
134
|
+
]
|
|
135
|
+
res_codes = []
|
|
136
|
+
|
|
137
|
+
d.run(expect_records: 2) do
|
|
138
|
+
events.each do |tag, t, record|
|
|
139
|
+
res = post("/#{tag}", {param => record.__send__(method_name)})
|
|
140
|
+
res_codes << res.code
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
assert_equal ["200", "200"], res_codes
|
|
144
|
+
assert_equal events, d.events
|
|
145
|
+
assert_equal_event_time time, d.events[0][1]
|
|
146
|
+
assert_equal_event_time time, d.events[1][1]
|
|
147
|
+
end
|
|
148
|
+
|
|
119
149
|
def test_multi_json
|
|
120
150
|
d = create_driver
|
|
121
151
|
time = event_time("2011-01-02 13:14:15 UTC")
|
|
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
|
|
|
163
193
|
assert_equal_event_time time, d.events[1][1]
|
|
164
194
|
end
|
|
165
195
|
|
|
196
|
+
data('json' => ['json', :to_json],
|
|
197
|
+
'msgpack' => ['msgpack', :to_msgpack])
|
|
198
|
+
def test_default_multi_with_time_format(data)
|
|
199
|
+
param, method_name = data
|
|
200
|
+
d = create_driver(CONFIG + %[
|
|
201
|
+
<parse>
|
|
202
|
+
keep_time_key
|
|
203
|
+
time_format %iso8601
|
|
204
|
+
</parse>
|
|
205
|
+
])
|
|
206
|
+
time = event_time("2020-06-10T01:14:27+00:00")
|
|
207
|
+
events = [
|
|
208
|
+
["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
|
|
209
|
+
["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
|
|
210
|
+
]
|
|
211
|
+
tag = "tag1"
|
|
212
|
+
res_codes = []
|
|
213
|
+
d.run(expect_records: 2, timeout: 5) do
|
|
214
|
+
res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
|
|
215
|
+
res_codes << res.code
|
|
216
|
+
end
|
|
217
|
+
assert_equal ["200"], res_codes
|
|
218
|
+
assert_equal events, d.events
|
|
219
|
+
assert_equal_event_time time, d.events[0][1]
|
|
220
|
+
assert_equal_event_time time, d.events[1][1]
|
|
221
|
+
end
|
|
222
|
+
|
|
166
223
|
def test_multi_json_with_nonexistent_time_key
|
|
167
224
|
d = create_driver(CONFIG + %[
|
|
168
225
|
<parse>
|
|
@@ -163,6 +163,21 @@ EOS
|
|
|
163
163
|
compare_test_result(d.events, tests)
|
|
164
164
|
end
|
|
165
165
|
|
|
166
|
+
def test_emit_rfc5452
|
|
167
|
+
d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
|
|
168
|
+
msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
|
|
169
|
+
|
|
170
|
+
d.run(expect_emits: 1, timeout: 2) do
|
|
171
|
+
u = UDPSocket.new
|
|
172
|
+
u.connect('127.0.0.1', PORT)
|
|
173
|
+
u.send(msg, 0)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
tag, _, event = d.events[0]
|
|
177
|
+
assert_equal('syslog.kern.alert', tag)
|
|
178
|
+
assert_equal('kern', event['pri'])
|
|
179
|
+
end
|
|
180
|
+
|
|
166
181
|
def test_msg_size_with_same_tcp_connection
|
|
167
182
|
d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
|
|
168
183
|
tests = create_test_case
|
|
@@ -360,7 +375,7 @@ EOS
|
|
|
360
375
|
]
|
|
361
376
|
msgs.each { |msg|
|
|
362
377
|
m = msg['msg']
|
|
363
|
-
msg['msg'] = "#{m.size
|
|
378
|
+
msg['msg'] = "#{m.size} #{m}"
|
|
364
379
|
}
|
|
365
380
|
msgs
|
|
366
381
|
end
|
data/test/plugin/test_in_tail.rb
CHANGED
|
@@ -23,9 +23,6 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
23
23
|
def cleanup_directory(path)
|
|
24
24
|
FileUtils.rm_rf(path, secure: true)
|
|
25
25
|
if File.exist?(path)
|
|
26
|
-
# ensure files are closed for Windows, on which deleted files
|
|
27
|
-
# are still visible from filesystem
|
|
28
|
-
GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
|
|
29
26
|
FileUtils.remove_entry_secure(path, true)
|
|
30
27
|
end
|
|
31
28
|
FileUtils.mkdir_p(path)
|
|
@@ -105,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
105
102
|
assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
|
|
106
103
|
end
|
|
107
104
|
|
|
105
|
+
test "multi paths with same path configured twice" do
|
|
106
|
+
c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
|
|
107
|
+
d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
|
|
108
|
+
assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
|
|
109
|
+
end
|
|
110
|
+
|
|
108
111
|
test "multi paths with invaid path_delimiter" do
|
|
109
112
|
c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
|
|
110
113
|
assert_raise(Fluent::ConfigError) do
|
|
@@ -226,7 +229,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
226
229
|
d = create_driver(config)
|
|
227
230
|
msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
|
|
228
231
|
|
|
229
|
-
d.run(expect_emits: 1) do
|
|
232
|
+
d.run(expect_emits: num_events, timeout: 1) do
|
|
230
233
|
File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
|
|
231
234
|
f.puts msg
|
|
232
235
|
f.puts msg
|
|
@@ -237,7 +240,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
237
240
|
assert_equal(true, events.length > 0)
|
|
238
241
|
assert_equal({"message" => msg}, events[0][2])
|
|
239
242
|
assert_equal({"message" => msg}, events[1][2])
|
|
240
|
-
|
|
243
|
+
assert num_events <= d.emit_count
|
|
241
244
|
end
|
|
242
245
|
|
|
243
246
|
data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
|
|
@@ -1025,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1025
1028
|
assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
|
|
1026
1029
|
end
|
|
1027
1030
|
|
|
1031
|
+
def test_expand_paths_with_duplicate_configuration
|
|
1032
|
+
expanded_paths = [
|
|
1033
|
+
'test/plugin/data/log/foo/bar.log',
|
|
1034
|
+
'test/plugin/data/log/test.log'
|
|
1035
|
+
]
|
|
1036
|
+
duplicate_config = EX_CONFIG.dup
|
|
1037
|
+
duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
|
|
1038
|
+
plugin = create_driver(EX_CONFIG, false).instance
|
|
1039
|
+
assert_equal expanded_paths, plugin.expand_paths.sort
|
|
1040
|
+
end
|
|
1041
|
+
|
|
1028
1042
|
def test_expand_paths_with_timezone
|
|
1029
1043
|
['Asia/Taipei', '+08'].each do |tz_type|
|
|
1030
1044
|
taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
|
|
@@ -1065,9 +1079,6 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1065
1079
|
assert_equal expected_files, plugin.expand_paths.sort
|
|
1066
1080
|
end
|
|
1067
1081
|
|
|
1068
|
-
# For https://github.com/fluent/fluentd/issues/1455
|
|
1069
|
-
# This test is fragile because test content depends on internal implementation.
|
|
1070
|
-
# So if you modify in_tail internal, this test may break.
|
|
1071
1082
|
def test_unwatched_files_should_be_removed
|
|
1072
1083
|
config = config_element("", "", {
|
|
1073
1084
|
"tag" => "tail",
|
|
@@ -1078,23 +1089,16 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1078
1089
|
"refresh_interval" => 1,
|
|
1079
1090
|
})
|
|
1080
1091
|
d = create_driver(config, false)
|
|
1092
|
+
d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
|
|
1081
1093
|
d.run(expect_emits: 1, shutdown: false) do
|
|
1082
1094
|
File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
|
|
1083
1095
|
end
|
|
1084
1096
|
|
|
1085
|
-
assert_equal 1, d.instance.instance_variable_get(:@tails).keys.size
|
|
1086
1097
|
cleanup_directory(TMP_DIR)
|
|
1087
1098
|
waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
|
|
1088
|
-
waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size
|
|
1099
|
+
waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
|
|
1089
1100
|
|
|
1090
|
-
|
|
1091
|
-
# Following code checks such unexpected bug by counting actual object allocation.
|
|
1092
|
-
base_num = count_timer_object
|
|
1093
|
-
2.times {
|
|
1094
|
-
sleep 1
|
|
1095
|
-
num = count_timer_object
|
|
1096
|
-
assert_equal base_num, num
|
|
1097
|
-
}
|
|
1101
|
+
assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
|
|
1098
1102
|
|
|
1099
1103
|
d.instance_shutdown
|
|
1100
1104
|
end
|
|
@@ -1164,7 +1168,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1164
1168
|
|
|
1165
1169
|
Timecop.freeze(2010, 1, 2, 3, 4, 5) do
|
|
1166
1170
|
EX_PATHS.each do |path|
|
|
1167
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true,
|
|
1171
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
|
|
1168
1172
|
end
|
|
1169
1173
|
|
|
1170
1174
|
plugin.refresh_watchers
|
|
@@ -1173,7 +1177,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1173
1177
|
mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
|
|
1174
1178
|
|
|
1175
1179
|
Timecop.freeze(2010, 1, 2, 3, 4, 6) do
|
|
1176
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true,
|
|
1180
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
|
|
1177
1181
|
plugin.refresh_watchers
|
|
1178
1182
|
|
|
1179
1183
|
flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
|
|
@@ -1183,6 +1187,25 @@ class TailInputTest < Test::Unit::TestCase
|
|
|
1183
1187
|
end
|
|
1184
1188
|
end
|
|
1185
1189
|
|
|
1190
|
+
sub_test_case "refresh of pos file" do
|
|
1191
|
+
test 'type of pos_file_compaction_interval is time' do
|
|
1192
|
+
tail = {
|
|
1193
|
+
"tag" => "tail",
|
|
1194
|
+
"path" => "#{TMP_DIR}/*.txt",
|
|
1195
|
+
"format" => "none",
|
|
1196
|
+
"pos_file" => "#{TMP_DIR}/pos/tail.pos",
|
|
1197
|
+
"refresh_interval" => 1,
|
|
1198
|
+
"read_from_head" => true,
|
|
1199
|
+
'pos_file_compaction_interval' => '24h',
|
|
1200
|
+
}
|
|
1201
|
+
config = config_element("", "", tail)
|
|
1202
|
+
d = create_driver(config, false)
|
|
1203
|
+
mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
|
|
1204
|
+
mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
|
|
1205
|
+
d.run # call start
|
|
1206
|
+
end
|
|
1207
|
+
end
|
|
1208
|
+
|
|
1186
1209
|
sub_test_case "receive_lines" do
|
|
1187
1210
|
DummyWatcher = Struct.new("DummyWatcher", :tag)
|
|
1188
1211
|
|