fluentd 1.11.1-x86-mingw32 → 1.12.0-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.travis.yml +22 -2
- data/CHANGELOG.md +111 -0
- data/README.md +1 -1
- data/appveyor.yml +3 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-ctl +7 -0
- data/example/copy_roundrobin.conf +3 -3
- data/example/counter.conf +1 -1
- data/example/filter_stdout.conf +2 -2
- data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
- data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
- data/example/logevents.conf +5 -5
- data/example/multi_filters.conf +1 -1
- data/example/out_exec_filter.conf +2 -2
- data/example/out_forward.conf +1 -1
- data/example/out_forward_buf_file.conf +1 -1
- data/example/out_forward_client.conf +5 -5
- data/example/out_forward_heartbeat_none.conf +1 -1
- data/example/out_forward_sd.conf +1 -1
- data/example/out_forward_shared_key.conf +2 -2
- data/example/out_forward_tls.conf +1 -1
- data/example/out_forward_users.conf +3 -3
- data/example/out_null.conf +4 -4
- data/example/secondary_file.conf +1 -1
- data/fluentd.gemspec +7 -6
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/plugin_config_formatter.rb +2 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/log.rb +33 -3
- data/lib/fluent/match.rb +9 -0
- data/lib/fluent/plugin.rb +5 -0
- data/lib/fluent/plugin/buffer.rb +32 -42
- data/lib/fluent/plugin/buffer/chunk.rb +2 -1
- data/lib/fluent/plugin/formatter.rb +24 -0
- data/lib/fluent/plugin/formatter_csv.rb +1 -1
- data/lib/fluent/plugin/formatter_hash.rb +3 -1
- data/lib/fluent/plugin/formatter_json.rb +3 -1
- data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
- data/lib/fluent/plugin/formatter_out_file.rb +6 -4
- data/lib/fluent/plugin/formatter_single_value.rb +4 -2
- data/lib/fluent/plugin/formatter_tsv.rb +4 -2
- data/lib/fluent/plugin/in_dummy.rb +2 -123
- data/lib/fluent/plugin/in_exec.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +25 -4
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_tail.rb +109 -41
- data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
- data/lib/fluent/plugin/in_tcp.rb +1 -0
- data/lib/fluent/plugin/out_http.rb +20 -2
- data/lib/fluent/plugin/output.rb +15 -6
- data/lib/fluent/plugin/parser_json.rb +5 -2
- data/lib/fluent/plugin_helper/cert_option.rb +5 -8
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
- data/lib/fluent/plugin_helper/inject.rb +4 -1
- data/lib/fluent/plugin_helper/retry_state.rb +4 -0
- data/lib/fluent/plugin_helper/socket.rb +1 -1
- data/lib/fluent/supervisor.rb +151 -48
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/time.rb +1 -0
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +22 -4
- data/test/command/test_binlog_reader.rb +22 -6
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_plugin_config_formatter.rb +57 -2
- data/test/plugin/in_tail/test_position_file.rb +45 -25
- data/test/plugin/test_buffer.rb +4 -0
- data/test/plugin/test_filter_stdout.rb +6 -1
- data/test/plugin/test_formatter_hash.rb +6 -3
- data/test/plugin/test_formatter_json.rb +14 -4
- data/test/plugin/test_formatter_ltsv.rb +13 -5
- data/test/plugin/test_formatter_out_file.rb +35 -14
- data/test/plugin/test_formatter_single_value.rb +12 -6
- data/test/plugin/test_formatter_tsv.rb +12 -4
- data/test/plugin/test_in_exec.rb +18 -0
- data/test/plugin/test_in_http.rb +25 -0
- data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
- data/test/plugin/test_in_tail.rb +433 -30
- data/test/plugin/test_out_file.rb +23 -18
- data/test/plugin/test_output.rb +12 -0
- data/test/plugin/test_parser_syslog.rb +2 -2
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/test_cert_option.rb +7 -0
- data/test/plugin_helper/test_compat_parameters.rb +7 -2
- data/test/plugin_helper/test_http_server_helper.rb +5 -0
- data/test/plugin_helper/test_inject.rb +42 -0
- data/test/plugin_helper/test_server.rb +34 -0
- data/test/plugin_helper/test_socket.rb +8 -0
- data/test/test_capability.rb +74 -0
- data/test/test_formatter.rb +34 -10
- data/test/test_log.rb +44 -0
- data/test/test_match.rb +11 -0
- data/test/test_output.rb +6 -1
- data/test/test_static_config_analysis.rb +2 -2
- data/test/test_supervisor.rb +119 -1
- metadata +50 -18
data/test/plugin/test_in_tail.rb
CHANGED
@@ -17,29 +17,62 @@ class TailInputTest < Test::Unit::TestCase
|
|
17
17
|
|
18
18
|
def teardown
|
19
19
|
super
|
20
|
+
cleanup_directory(TMP_DIR)
|
20
21
|
Fluent::Engine.stop
|
21
22
|
end
|
22
23
|
|
23
24
|
def cleanup_directory(path)
|
24
|
-
|
25
|
+
begin
|
26
|
+
FileUtils.rm_f(path, secure: true)
|
27
|
+
rescue ArgumentError
|
28
|
+
FileUtils.rm_f(path) # For Ruby 2.6 or before.
|
29
|
+
end
|
25
30
|
if File.exist?(path)
|
26
31
|
FileUtils.remove_entry_secure(path, true)
|
27
32
|
end
|
28
33
|
FileUtils.mkdir_p(path)
|
29
34
|
end
|
30
35
|
|
36
|
+
def cleanup_file(path)
|
37
|
+
begin
|
38
|
+
FileUtils.rm_f(path, secure: true)
|
39
|
+
rescue ArgumentError
|
40
|
+
FileUtils.rm_f(path) # For Ruby 2.6 or before.
|
41
|
+
end
|
42
|
+
if File.exist?(path)
|
43
|
+
# ensure files are closed for Windows, on which deleted files
|
44
|
+
# are still visible from filesystem
|
45
|
+
GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
|
46
|
+
FileUtils.remove_entry_secure(path, true)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def create_target_info(path)
|
51
|
+
Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
|
52
|
+
end
|
53
|
+
|
31
54
|
TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
|
32
55
|
|
33
56
|
CONFIG = config_element("ROOT", "", {
|
34
57
|
"path" => "#{TMP_DIR}/tail.txt",
|
35
58
|
"tag" => "t1",
|
36
|
-
"rotate_wait" => "2s"
|
59
|
+
"rotate_wait" => "2s",
|
60
|
+
"refresh_interval" => "1s"
|
37
61
|
})
|
38
62
|
COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
|
39
63
|
CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
|
40
64
|
CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
|
41
65
|
CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
|
42
66
|
CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
|
67
|
+
COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
|
68
|
+
"path" => "#{TMP_DIR}/tail.txt*",
|
69
|
+
"pos_file" => "#{TMP_DIR}/tail.pos",
|
70
|
+
"tag" => "t1",
|
71
|
+
"refresh_interval" => "1s",
|
72
|
+
"read_from_head" => "true",
|
73
|
+
"format" => "none",
|
74
|
+
"follow_inodes" => "true"
|
75
|
+
})
|
43
76
|
SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
|
44
77
|
PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
|
45
78
|
MULTILINE_CONFIG = config_element(
|
@@ -86,6 +119,9 @@ class TailInputTest < Test::Unit::TestCase
|
|
86
119
|
assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
|
87
120
|
assert_equal 1000, d.instance.read_lines_limit
|
88
121
|
assert_equal false, d.instance.ignore_repeated_permission_error
|
122
|
+
assert_nothing_raised do
|
123
|
+
d.instance.have_read_capability?
|
124
|
+
end
|
89
125
|
end
|
90
126
|
|
91
127
|
data("empty" => config_element,
|
@@ -115,6 +151,12 @@ class TailInputTest < Test::Unit::TestCase
|
|
115
151
|
end
|
116
152
|
end
|
117
153
|
|
154
|
+
test "follow_inodes w/o pos file" do
|
155
|
+
assert_raise(Fluent::ConfigError) do
|
156
|
+
create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
118
160
|
test "both enable_watch_timer and enable_stat_watcher are false" do
|
119
161
|
assert_raise(Fluent::ConfigError) do
|
120
162
|
create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
|
@@ -229,7 +271,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
229
271
|
d = create_driver(config)
|
230
272
|
msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
|
231
273
|
|
232
|
-
d.run(expect_emits: num_events, timeout:
|
274
|
+
d.run(expect_emits: num_events, timeout: 2) do
|
233
275
|
File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
|
234
276
|
f.puts msg
|
235
277
|
f.puts msg
|
@@ -999,6 +1041,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
999
1041
|
# * path test
|
1000
1042
|
# TODO: Clean up tests
|
1001
1043
|
EX_ROTATE_WAIT = 0
|
1044
|
+
EX_FOLLOW_INODES = false
|
1002
1045
|
|
1003
1046
|
EX_CONFIG = config_element("", "", {
|
1004
1047
|
"tag" => "tail",
|
@@ -1008,38 +1051,43 @@ class TailInputTest < Test::Unit::TestCase
|
|
1008
1051
|
"read_from_head" => true,
|
1009
1052
|
"refresh_interval" => 30,
|
1010
1053
|
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1054
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1011
1055
|
})
|
1012
|
-
EX_PATHS = [
|
1013
|
-
'test/plugin/data/2010/01/20100102-030405.log',
|
1014
|
-
'test/plugin/data/log/foo/bar.log',
|
1015
|
-
'test/plugin/data/log/test.log'
|
1016
|
-
]
|
1017
|
-
|
1018
1056
|
def test_expand_paths
|
1057
|
+
ex_paths = [
|
1058
|
+
create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
|
1059
|
+
create_target_info('test/plugin/data/log/foo/bar.log'),
|
1060
|
+
create_target_info('test/plugin/data/log/test.log')
|
1061
|
+
]
|
1019
1062
|
plugin = create_driver(EX_CONFIG, false).instance
|
1020
1063
|
flexstub(Time) do |timeclass|
|
1021
1064
|
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
|
1022
|
-
assert_equal
|
1065
|
+
assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1023
1066
|
end
|
1024
1067
|
|
1025
1068
|
# Test exclusion
|
1026
|
-
exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{
|
1069
|
+
exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
|
1027
1070
|
plugin = create_driver(exclude_config, false).instance
|
1028
|
-
assert_equal
|
1071
|
+
assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1029
1072
|
end
|
1030
1073
|
|
1031
1074
|
def test_expand_paths_with_duplicate_configuration
|
1032
1075
|
expanded_paths = [
|
1033
|
-
'test/plugin/data/log/foo/bar.log',
|
1034
|
-
'test/plugin/data/log/test.log'
|
1076
|
+
create_target_info('test/plugin/data/log/foo/bar.log'),
|
1077
|
+
create_target_info('test/plugin/data/log/test.log')
|
1035
1078
|
]
|
1036
1079
|
duplicate_config = EX_CONFIG.dup
|
1037
1080
|
duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
|
1038
1081
|
plugin = create_driver(EX_CONFIG, false).instance
|
1039
|
-
assert_equal expanded_paths, plugin.expand_paths.
|
1082
|
+
assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1040
1083
|
end
|
1041
1084
|
|
1042
1085
|
def test_expand_paths_with_timezone
|
1086
|
+
ex_paths = [
|
1087
|
+
create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
|
1088
|
+
create_target_info('test/plugin/data/log/foo/bar.log'),
|
1089
|
+
create_target_info('test/plugin/data/log/test.log')
|
1090
|
+
]
|
1043
1091
|
['Asia/Taipei', '+08'].each do |tz_type|
|
1044
1092
|
taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
|
1045
1093
|
plugin = create_driver(taipei_config, false).instance
|
@@ -1053,8 +1101,8 @@ class TailInputTest < Test::Unit::TestCase
|
|
1053
1101
|
# env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
|
1054
1102
|
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
|
1055
1103
|
|
1056
|
-
assert_equal
|
1057
|
-
assert_equal
|
1104
|
+
assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1105
|
+
assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1058
1106
|
end
|
1059
1107
|
end
|
1060
1108
|
end
|
@@ -1062,10 +1110,10 @@ class TailInputTest < Test::Unit::TestCase
|
|
1062
1110
|
|
1063
1111
|
def test_log_file_without_extension
|
1064
1112
|
expected_files = [
|
1065
|
-
'test/plugin/data/log/bar',
|
1066
|
-
'test/plugin/data/log/foo/bar.log',
|
1067
|
-
'test/plugin/data/log/foo/bar2',
|
1068
|
-
'test/plugin/data/log/test.log'
|
1113
|
+
create_target_info('test/plugin/data/log/bar'),
|
1114
|
+
create_target_info('test/plugin/data/log/foo/bar.log'),
|
1115
|
+
create_target_info('test/plugin/data/log/foo/bar2'),
|
1116
|
+
create_target_info('test/plugin/data/log/test.log')
|
1069
1117
|
]
|
1070
1118
|
|
1071
1119
|
config = config_element("", "", {
|
@@ -1076,7 +1124,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1076
1124
|
})
|
1077
1125
|
|
1078
1126
|
plugin = create_driver(config, false).instance
|
1079
|
-
assert_equal expected_files, plugin.expand_paths.
|
1127
|
+
assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1080
1128
|
end
|
1081
1129
|
|
1082
1130
|
def test_unwatched_files_should_be_removed
|
@@ -1112,6 +1160,49 @@ class TailInputTest < Test::Unit::TestCase
|
|
1112
1160
|
end
|
1113
1161
|
end
|
1114
1162
|
|
1163
|
+
sub_test_case "path w/ Linux capability" do
|
1164
|
+
def capability_enabled?
|
1165
|
+
if Fluent.linux?
|
1166
|
+
begin
|
1167
|
+
require 'capng'
|
1168
|
+
true
|
1169
|
+
rescue LoadError
|
1170
|
+
false
|
1171
|
+
end
|
1172
|
+
else
|
1173
|
+
false
|
1174
|
+
end
|
1175
|
+
end
|
1176
|
+
|
1177
|
+
setup do
|
1178
|
+
omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
|
1179
|
+
|
1180
|
+
@capng = CapNG.new(:current_process)
|
1181
|
+
flexstub(Fluent::Capability) do |klass|
|
1182
|
+
klass.should_receive(:new).with(:current_process).and_return(@capng)
|
1183
|
+
end
|
1184
|
+
end
|
1185
|
+
|
1186
|
+
data("dac_read_search" => [:dac_read_search, true, 1],
|
1187
|
+
"dac_override" => [:dac_override, true, 1],
|
1188
|
+
"chown" => [:chown, false, 0],
|
1189
|
+
)
|
1190
|
+
test "with partially elevated privileges" do |data|
|
1191
|
+
cap, result, readable_paths = data
|
1192
|
+
@capng.update(:add, :effective, cap)
|
1193
|
+
|
1194
|
+
d = create_driver(
|
1195
|
+
config_element("ROOT", "", {
|
1196
|
+
"path" => "/var/log/ker*.log", # Use /var/log/kern.log
|
1197
|
+
"tag" => "t1",
|
1198
|
+
"rotate_wait" => "2s"
|
1199
|
+
}) + PARSE_SINGLE_LINE_CONFIG, false)
|
1200
|
+
|
1201
|
+
assert_equal readable_paths, d.instance.expand_paths.length
|
1202
|
+
assert_equal result, d.instance.have_read_capability?
|
1203
|
+
end
|
1204
|
+
end
|
1205
|
+
|
1115
1206
|
def test_pos_file_dir_creation
|
1116
1207
|
config = config_element("", "", {
|
1117
1208
|
"tag" => "tail",
|
@@ -1159,25 +1250,35 @@ class TailInputTest < Test::Unit::TestCase
|
|
1159
1250
|
end
|
1160
1251
|
|
1161
1252
|
def test_z_refresh_watchers
|
1253
|
+
ex_paths = [
|
1254
|
+
create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
|
1255
|
+
create_target_info('test/plugin/data/log/foo/bar.log'),
|
1256
|
+
create_target_info('test/plugin/data/log/test.log'),
|
1257
|
+
]
|
1162
1258
|
plugin = create_driver(EX_CONFIG, false).instance
|
1163
1259
|
sio = StringIO.new
|
1164
1260
|
plugin.instance_eval do
|
1165
|
-
@pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
|
1261
|
+
@pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
|
1166
1262
|
@loop = Coolio::Loop.new
|
1167
1263
|
end
|
1168
1264
|
|
1169
1265
|
Timecop.freeze(2010, 1, 2, 3, 4, 5) do
|
1170
|
-
|
1171
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(
|
1266
|
+
ex_paths.each do |target_info|
|
1267
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
|
1172
1268
|
end
|
1173
1269
|
|
1174
1270
|
plugin.refresh_watchers
|
1175
1271
|
end
|
1176
1272
|
|
1177
|
-
|
1273
|
+
path = 'test/plugin/data/2010/01/20100102-030405.log'
|
1274
|
+
target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
|
1275
|
+
mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
|
1178
1276
|
|
1179
1277
|
Timecop.freeze(2010, 1, 2, 3, 4, 6) do
|
1180
|
-
|
1278
|
+
path = "test/plugin/data/2010/01/20100102-030406.log"
|
1279
|
+
inode = Fluent::FileWrapper.stat(path).ino
|
1280
|
+
target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
|
1281
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
|
1181
1282
|
plugin.refresh_watchers
|
1182
1283
|
|
1183
1284
|
flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
|
@@ -1323,6 +1424,305 @@ class TailInputTest < Test::Unit::TestCase
|
|
1323
1424
|
end
|
1324
1425
|
end
|
1325
1426
|
|
1427
|
+
sub_test_case 'inode_processing' do
|
1428
|
+
def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
|
1429
|
+
config = COMMON_FOLLOW_INODE_CONFIG
|
1430
|
+
|
1431
|
+
path = "#{TMP_DIR}/tail.txt"
|
1432
|
+
ino = 1
|
1433
|
+
pos = 1234
|
1434
|
+
File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
|
1435
|
+
f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
|
1436
|
+
}
|
1437
|
+
|
1438
|
+
d = create_driver(config, false)
|
1439
|
+
d.run
|
1440
|
+
|
1441
|
+
pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
|
1442
|
+
pos_file.pos = 0
|
1443
|
+
|
1444
|
+
assert_raise(EOFError) do
|
1445
|
+
pos_file.readline
|
1446
|
+
end
|
1447
|
+
end
|
1448
|
+
|
1449
|
+
def test_should_write_latest_offset_after_rotate_wait
|
1450
|
+
config = COMMON_FOLLOW_INODE_CONFIG
|
1451
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1452
|
+
f.puts "test1"
|
1453
|
+
f.puts "test2"
|
1454
|
+
}
|
1455
|
+
|
1456
|
+
d = create_driver(config, false)
|
1457
|
+
d.run(expect_emits: 2, shutdown: false) do
|
1458
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1459
|
+
FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
|
1460
|
+
sleep 1
|
1461
|
+
File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
|
1462
|
+
end
|
1463
|
+
|
1464
|
+
pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
|
1465
|
+
pos_file.pos = 0
|
1466
|
+
line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1467
|
+
waiting(5) {
|
1468
|
+
while line_parts[2].to_i(16) != 24
|
1469
|
+
sleep(0.1)
|
1470
|
+
pos_file.pos = 0
|
1471
|
+
line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1472
|
+
end
|
1473
|
+
}
|
1474
|
+
assert_equal(24, line_parts[2].to_i(16))
|
1475
|
+
d.instance_shutdown
|
1476
|
+
end
|
1477
|
+
|
1478
|
+
def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
|
1479
|
+
config = config_element("", "", {"format" => "none"})
|
1480
|
+
|
1481
|
+
path = "#{TMP_DIR}/tail.txt"
|
1482
|
+
ino = 1
|
1483
|
+
pos = 1234
|
1484
|
+
File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
|
1485
|
+
f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
|
1486
|
+
}
|
1487
|
+
|
1488
|
+
d = create_driver(config)
|
1489
|
+
d.run(shutdown: false)
|
1490
|
+
|
1491
|
+
pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
|
1492
|
+
pos_file.pos = 0
|
1493
|
+
|
1494
|
+
path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1495
|
+
assert_equal(path, path_pos_ino[1])
|
1496
|
+
assert_equal(pos, path_pos_ino[2].to_i(16))
|
1497
|
+
assert_equal(ino, path_pos_ino[3].to_i(16))
|
1498
|
+
|
1499
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1500
|
+
f.puts "test1"
|
1501
|
+
f.puts "test2"
|
1502
|
+
}
|
1503
|
+
Timecop.travel(Time.now + 10) do
|
1504
|
+
sleep 5
|
1505
|
+
pos_file.pos = 0
|
1506
|
+
tuple = create_target_info("#{TMP_DIR}/tail.txt")
|
1507
|
+
path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1508
|
+
assert_equal(tuple.path, path_pos_ino[1])
|
1509
|
+
assert_equal(12, path_pos_ino[2].to_i(16))
|
1510
|
+
assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
|
1511
|
+
end
|
1512
|
+
d.instance_shutdown
|
1513
|
+
end
|
1514
|
+
|
1515
|
+
def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
|
1516
|
+
config = config_element("ROOT", "", {
|
1517
|
+
"path" => "#{TMP_DIR}/tail.txt*",
|
1518
|
+
"pos_file" => "#{TMP_DIR}/tail.pos",
|
1519
|
+
"tag" => "t1",
|
1520
|
+
"rotate_wait" => "1s",
|
1521
|
+
"refresh_interval" => "1s",
|
1522
|
+
"limit_recently_modified" => "1s",
|
1523
|
+
"read_from_head" => "true",
|
1524
|
+
"format" => "none",
|
1525
|
+
"follow_inodes" => "true",
|
1526
|
+
})
|
1527
|
+
|
1528
|
+
d = create_driver(config, false)
|
1529
|
+
|
1530
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1531
|
+
f.puts "test1"
|
1532
|
+
f.puts "test2"
|
1533
|
+
}
|
1534
|
+
target_info = create_target_info("#{TMP_DIR}/tail.txt")
|
1535
|
+
|
1536
|
+
d.run(expect_emits: 1, shutdown: false) do
|
1537
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1538
|
+
end
|
1539
|
+
|
1540
|
+
|
1541
|
+
Timecop.travel(Time.now + 10) do
|
1542
|
+
waiting(5) {
|
1543
|
+
# @pos will be reset as 0 when UNWATCHED_POSITION is specified.
|
1544
|
+
sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
|
1545
|
+
}
|
1546
|
+
end
|
1547
|
+
|
1548
|
+
assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
|
1549
|
+
|
1550
|
+
d.instance_shutdown
|
1551
|
+
end
|
1552
|
+
|
1553
|
+
def test_should_read_from_head_on_file_renaming_with_star_in_pattern
|
1554
|
+
config = config_element("ROOT", "", {
|
1555
|
+
"path" => "#{TMP_DIR}/tail.txt*",
|
1556
|
+
"pos_file" => "#{TMP_DIR}/tail.pos",
|
1557
|
+
"tag" => "t1",
|
1558
|
+
"rotate_wait" => "10s",
|
1559
|
+
"refresh_interval" => "1s",
|
1560
|
+
"limit_recently_modified" => "60s",
|
1561
|
+
"read_from_head" => "true",
|
1562
|
+
"format" => "none",
|
1563
|
+
"follow_inodes" => "true"
|
1564
|
+
})
|
1565
|
+
|
1566
|
+
d = create_driver(config, false)
|
1567
|
+
|
1568
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1569
|
+
f.puts "test1"
|
1570
|
+
f.puts "test2"
|
1571
|
+
}
|
1572
|
+
|
1573
|
+
d.run(expect_emits: 2, shutdown: false) do
|
1574
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1575
|
+
FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
|
1576
|
+
end
|
1577
|
+
|
1578
|
+
events = d.events
|
1579
|
+
assert_equal(3, events.length)
|
1580
|
+
d.instance_shutdown
|
1581
|
+
end
|
1582
|
+
|
1583
|
+
def test_should_not_read_from_head_on_rotation_when_watching_inodes
|
1584
|
+
config = COMMON_FOLLOW_INODE_CONFIG
|
1585
|
+
|
1586
|
+
d = create_driver(config, false)
|
1587
|
+
|
1588
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1589
|
+
f.puts "test1"
|
1590
|
+
f.puts "test2"
|
1591
|
+
}
|
1592
|
+
|
1593
|
+
d.run(expect_emits: 1, shutdown: false) do
|
1594
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1595
|
+
end
|
1596
|
+
|
1597
|
+
FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
|
1598
|
+
Timecop.travel(Time.now + 10) do
|
1599
|
+
sleep 2
|
1600
|
+
events = d.events
|
1601
|
+
assert_equal(3, events.length)
|
1602
|
+
end
|
1603
|
+
|
1604
|
+
d.instance_shutdown
|
1605
|
+
end
|
1606
|
+
|
1607
|
+
def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
|
1608
|
+
config = COMMON_FOLLOW_INODE_CONFIG
|
1609
|
+
|
1610
|
+
d = create_driver(config, false)
|
1611
|
+
|
1612
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1613
|
+
f.puts "test1"
|
1614
|
+
f.puts "test2"
|
1615
|
+
}
|
1616
|
+
target_info = create_target_info("#{TMP_DIR}/tail.txt")
|
1617
|
+
|
1618
|
+
d.run(expect_emits: 2, shutdown: false) do
|
1619
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1620
|
+
cleanup_file("#{TMP_DIR}/tail.txt")
|
1621
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
|
1622
|
+
end
|
1623
|
+
|
1624
|
+
new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
|
1625
|
+
|
1626
|
+
pos_file = d.instance.instance_variable_get(:@pf)
|
1627
|
+
|
1628
|
+
waiting(10) {
|
1629
|
+
# @pos will be reset as 0 when UNWATCHED_POSITION is specified.
|
1630
|
+
sleep 0.1 until pos_file[target_info].read_pos == 0
|
1631
|
+
}
|
1632
|
+
new_position = pos_file[new_target_info].read_pos
|
1633
|
+
assert_equal(6, new_position)
|
1634
|
+
|
1635
|
+
d.instance_shutdown
|
1636
|
+
end
|
1637
|
+
|
1638
|
+
def test_should_close_watcher_after_rotate_wait
|
1639
|
+
now = Time.now
|
1640
|
+
config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
|
1641
|
+
|
1642
|
+
d = create_driver(config, false)
|
1643
|
+
|
1644
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1645
|
+
f.puts "test1"
|
1646
|
+
f.puts "test2"
|
1647
|
+
}
|
1648
|
+
target_info = create_target_info("#{TMP_DIR}/tail.txt")
|
1649
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
|
1650
|
+
d.run(shutdown: false)
|
1651
|
+
assert d.instance.instance_variable_get(:@tails)[target_info]
|
1652
|
+
|
1653
|
+
Timecop.travel(now + 10) do
|
1654
|
+
d.instance.instance_eval do
|
1655
|
+
sleep 0.1 until @tails[target_info] == nil
|
1656
|
+
end
|
1657
|
+
assert_nil d.instance.instance_variable_get(:@tails)[target_info]
|
1658
|
+
end
|
1659
|
+
d.instance_shutdown
|
1660
|
+
end
|
1661
|
+
|
1662
|
+
def test_should_create_new_watcher_for_new_file_with_same_name
|
1663
|
+
now = Time.now
|
1664
|
+
config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
|
1665
|
+
|
1666
|
+
d = create_driver(config, false)
|
1667
|
+
|
1668
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1669
|
+
f.puts "test1"
|
1670
|
+
f.puts "test2"
|
1671
|
+
}
|
1672
|
+
path_ino = create_target_info("#{TMP_DIR}/tail.txt")
|
1673
|
+
|
1674
|
+
d.run(expect_emits: 1, shutdown: false) do
|
1675
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1676
|
+
end
|
1677
|
+
|
1678
|
+
cleanup_file("#{TMP_DIR}/tail.txt")
|
1679
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1680
|
+
f.puts "test3"
|
1681
|
+
f.puts "test4"
|
1682
|
+
}
|
1683
|
+
new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
|
1684
|
+
|
1685
|
+
Timecop.travel(now + 10) do
|
1686
|
+
sleep 3
|
1687
|
+
d.instance.instance_eval do
|
1688
|
+
@tails[path_ino] == nil
|
1689
|
+
@tails[new_path_ino] != nil
|
1690
|
+
end
|
1691
|
+
end
|
1692
|
+
|
1693
|
+
events = d.events
|
1694
|
+
|
1695
|
+
assert_equal(5, events.length)
|
1696
|
+
|
1697
|
+
d.instance_shutdown
|
1698
|
+
end
|
1699
|
+
|
1700
|
+
def test_truncate_file_with_follow_inodes
|
1701
|
+
config = COMMON_FOLLOW_INODE_CONFIG
|
1702
|
+
|
1703
|
+
d = create_driver(config, false)
|
1704
|
+
|
1705
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1706
|
+
f.puts "test1"
|
1707
|
+
f.puts "test2"
|
1708
|
+
}
|
1709
|
+
|
1710
|
+
d.run(expect_emits: 3, shutdown: false) do
|
1711
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
|
1712
|
+
sleep 2
|
1713
|
+
File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
|
1714
|
+
end
|
1715
|
+
|
1716
|
+
events = d.events
|
1717
|
+
assert_equal(4, events.length)
|
1718
|
+
assert_equal({"message" => "test1"}, events[0][2])
|
1719
|
+
assert_equal({"message" => "test2"}, events[1][2])
|
1720
|
+
assert_equal({"message" => "test3"}, events[2][2])
|
1721
|
+
assert_equal({"message" => "test4"}, events[3][2])
|
1722
|
+
d.instance_shutdown
|
1723
|
+
end
|
1724
|
+
end
|
1725
|
+
|
1326
1726
|
sub_test_case "tail_path" do
|
1327
1727
|
def test_tail_path_with_singleline
|
1328
1728
|
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
@@ -1403,6 +1803,9 @@ class TailInputTest < Test::Unit::TestCase
|
|
1403
1803
|
end
|
1404
1804
|
|
1405
1805
|
def test_tail_path_with_multiline_with_multiple_paths
|
1806
|
+
if ENV["APPVEYOR"] && Fluent.windows?
|
1807
|
+
omit "This testcase is unstable on AppVeyor."
|
1808
|
+
end
|
1406
1809
|
files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
|
1407
1810
|
files.each { |file| File.open(file, "wb") { |f| } }
|
1408
1811
|
|
@@ -1449,13 +1852,13 @@ class TailInputTest < Test::Unit::TestCase
|
|
1449
1852
|
})
|
1450
1853
|
|
1451
1854
|
expected_files = [
|
1452
|
-
"#{TMP_DIR}/tail_watch1.txt",
|
1453
|
-
"#{TMP_DIR}/tail_watch2.txt"
|
1855
|
+
create_target_info("#{TMP_DIR}/tail_watch1.txt"),
|
1856
|
+
create_target_info("#{TMP_DIR}/tail_watch2.txt")
|
1454
1857
|
]
|
1455
1858
|
|
1456
1859
|
Timecop.freeze(now) do
|
1457
1860
|
plugin = create_driver(config, false).instance
|
1458
|
-
assert_equal expected_files, plugin.expand_paths.
|
1861
|
+
assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
|
1459
1862
|
end
|
1460
1863
|
end
|
1461
1864
|
|