fluentd 1.11.1-x64-mingw32 → 1.12.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
  3. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. data/.github/workflows/stale-actions.yml +22 -0
  5. data/.travis.yml +22 -2
  6. data/CHANGELOG.md +111 -0
  7. data/README.md +1 -1
  8. data/appveyor.yml +3 -0
  9. data/bin/fluent-cap-ctl +7 -0
  10. data/bin/fluent-ctl +7 -0
  11. data/example/copy_roundrobin.conf +3 -3
  12. data/example/counter.conf +1 -1
  13. data/example/filter_stdout.conf +2 -2
  14. data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
  15. data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
  16. data/example/logevents.conf +5 -5
  17. data/example/multi_filters.conf +1 -1
  18. data/example/out_exec_filter.conf +2 -2
  19. data/example/out_forward.conf +1 -1
  20. data/example/out_forward_buf_file.conf +1 -1
  21. data/example/out_forward_client.conf +5 -5
  22. data/example/out_forward_heartbeat_none.conf +1 -1
  23. data/example/out_forward_sd.conf +1 -1
  24. data/example/out_forward_shared_key.conf +2 -2
  25. data/example/out_forward_tls.conf +1 -1
  26. data/example/out_forward_users.conf +3 -3
  27. data/example/out_null.conf +4 -4
  28. data/example/secondary_file.conf +1 -1
  29. data/fluentd.gemspec +7 -6
  30. data/lib/fluent/capability.rb +87 -0
  31. data/lib/fluent/command/cap_ctl.rb +174 -0
  32. data/lib/fluent/command/ctl.rb +177 -0
  33. data/lib/fluent/command/plugin_config_formatter.rb +2 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/log.rb +33 -3
  36. data/lib/fluent/match.rb +9 -0
  37. data/lib/fluent/plugin.rb +5 -0
  38. data/lib/fluent/plugin/buffer.rb +32 -42
  39. data/lib/fluent/plugin/buffer/chunk.rb +2 -1
  40. data/lib/fluent/plugin/formatter.rb +24 -0
  41. data/lib/fluent/plugin/formatter_csv.rb +1 -1
  42. data/lib/fluent/plugin/formatter_hash.rb +3 -1
  43. data/lib/fluent/plugin/formatter_json.rb +3 -1
  44. data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
  45. data/lib/fluent/plugin/formatter_out_file.rb +6 -4
  46. data/lib/fluent/plugin/formatter_single_value.rb +4 -2
  47. data/lib/fluent/plugin/formatter_tsv.rb +4 -2
  48. data/lib/fluent/plugin/in_dummy.rb +2 -123
  49. data/lib/fluent/plugin/in_exec.rb +4 -2
  50. data/lib/fluent/plugin/in_http.rb +25 -4
  51. data/lib/fluent/plugin/in_sample.rb +141 -0
  52. data/lib/fluent/plugin/in_tail.rb +109 -41
  53. data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
  54. data/lib/fluent/plugin/in_tcp.rb +1 -0
  55. data/lib/fluent/plugin/out_http.rb +20 -2
  56. data/lib/fluent/plugin/output.rb +15 -6
  57. data/lib/fluent/plugin/parser_json.rb +5 -2
  58. data/lib/fluent/plugin_helper/cert_option.rb +5 -8
  59. data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
  60. data/lib/fluent/plugin_helper/inject.rb +4 -1
  61. data/lib/fluent/plugin_helper/retry_state.rb +4 -0
  62. data/lib/fluent/plugin_helper/socket.rb +1 -1
  63. data/lib/fluent/supervisor.rb +151 -48
  64. data/lib/fluent/system_config.rb +2 -1
  65. data/lib/fluent/time.rb +1 -0
  66. data/lib/fluent/version.rb +1 -1
  67. data/lib/fluent/winsvc.rb +22 -4
  68. data/test/command/test_binlog_reader.rb +22 -6
  69. data/test/command/test_cap_ctl.rb +100 -0
  70. data/test/command/test_ctl.rb +57 -0
  71. data/test/command/test_plugin_config_formatter.rb +57 -2
  72. data/test/plugin/in_tail/test_position_file.rb +45 -25
  73. data/test/plugin/test_buffer.rb +4 -0
  74. data/test/plugin/test_filter_stdout.rb +6 -1
  75. data/test/plugin/test_formatter_hash.rb +6 -3
  76. data/test/plugin/test_formatter_json.rb +14 -4
  77. data/test/plugin/test_formatter_ltsv.rb +13 -5
  78. data/test/plugin/test_formatter_out_file.rb +35 -14
  79. data/test/plugin/test_formatter_single_value.rb +12 -6
  80. data/test/plugin/test_formatter_tsv.rb +12 -4
  81. data/test/plugin/test_in_exec.rb +18 -0
  82. data/test/plugin/test_in_http.rb +25 -0
  83. data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
  84. data/test/plugin/test_in_tail.rb +433 -30
  85. data/test/plugin/test_out_file.rb +23 -18
  86. data/test/plugin/test_output.rb +12 -0
  87. data/test/plugin/test_parser_syslog.rb +2 -2
  88. data/test/plugin_helper/data/cert/empty.pem +0 -0
  89. data/test/plugin_helper/test_cert_option.rb +7 -0
  90. data/test/plugin_helper/test_compat_parameters.rb +7 -2
  91. data/test/plugin_helper/test_http_server_helper.rb +5 -0
  92. data/test/plugin_helper/test_inject.rb +42 -0
  93. data/test/plugin_helper/test_server.rb +34 -0
  94. data/test/plugin_helper/test_socket.rb +8 -0
  95. data/test/test_capability.rb +74 -0
  96. data/test/test_formatter.rb +34 -10
  97. data/test/test_log.rb +44 -0
  98. data/test/test_match.rb +11 -0
  99. data/test/test_output.rb +6 -1
  100. data/test/test_static_config_analysis.rb +2 -2
  101. data/test/test_supervisor.rb +119 -1
  102. metadata +50 -18
@@ -17,29 +17,62 @@ class TailInputTest < Test::Unit::TestCase
17
17
 
18
18
  def teardown
19
19
  super
20
+ cleanup_directory(TMP_DIR)
20
21
  Fluent::Engine.stop
21
22
  end
22
23
 
23
24
  def cleanup_directory(path)
24
- FileUtils.rm_rf(path, secure: true)
25
+ begin
26
+ FileUtils.rm_f(path, secure: true)
27
+ rescue ArgumentError
28
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
29
+ end
25
30
  if File.exist?(path)
26
31
  FileUtils.remove_entry_secure(path, true)
27
32
  end
28
33
  FileUtils.mkdir_p(path)
29
34
  end
30
35
 
36
+ def cleanup_file(path)
37
+ begin
38
+ FileUtils.rm_f(path, secure: true)
39
+ rescue ArgumentError
40
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
41
+ end
42
+ if File.exist?(path)
43
+ # ensure files are closed for Windows, on which deleted files
44
+ # are still visible from filesystem
45
+ GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
46
+ FileUtils.remove_entry_secure(path, true)
47
+ end
48
+ end
49
+
50
+ def create_target_info(path)
51
+ Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
52
+ end
53
+
31
54
  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
32
55
 
33
56
  CONFIG = config_element("ROOT", "", {
34
57
  "path" => "#{TMP_DIR}/tail.txt",
35
58
  "tag" => "t1",
36
- "rotate_wait" => "2s"
59
+ "rotate_wait" => "2s",
60
+ "refresh_interval" => "1s"
37
61
  })
38
62
  COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
39
63
  CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
40
64
  CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
41
65
  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
42
66
  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
67
+ COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
68
+ "path" => "#{TMP_DIR}/tail.txt*",
69
+ "pos_file" => "#{TMP_DIR}/tail.pos",
70
+ "tag" => "t1",
71
+ "refresh_interval" => "1s",
72
+ "read_from_head" => "true",
73
+ "format" => "none",
74
+ "follow_inodes" => "true"
75
+ })
43
76
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
44
77
  PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
45
78
  MULTILINE_CONFIG = config_element(
@@ -86,6 +119,9 @@ class TailInputTest < Test::Unit::TestCase
86
119
  assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
87
120
  assert_equal 1000, d.instance.read_lines_limit
88
121
  assert_equal false, d.instance.ignore_repeated_permission_error
122
+ assert_nothing_raised do
123
+ d.instance.have_read_capability?
124
+ end
89
125
  end
90
126
 
91
127
  data("empty" => config_element,
@@ -115,6 +151,12 @@ class TailInputTest < Test::Unit::TestCase
115
151
  end
116
152
  end
117
153
 
154
+ test "follow_inodes w/o pos file" do
155
+ assert_raise(Fluent::ConfigError) do
156
+ create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
157
+ end
158
+ end
159
+
118
160
  test "both enable_watch_timer and enable_stat_watcher are false" do
119
161
  assert_raise(Fluent::ConfigError) do
120
162
  create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -229,7 +271,7 @@ class TailInputTest < Test::Unit::TestCase
229
271
  d = create_driver(config)
230
272
  msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
231
273
 
232
- d.run(expect_emits: num_events, timeout: 1) do
274
+ d.run(expect_emits: num_events, timeout: 2) do
233
275
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
234
276
  f.puts msg
235
277
  f.puts msg
@@ -999,6 +1041,7 @@ class TailInputTest < Test::Unit::TestCase
999
1041
  # * path test
1000
1042
  # TODO: Clean up tests
1001
1043
  EX_ROTATE_WAIT = 0
1044
+ EX_FOLLOW_INODES = false
1002
1045
 
1003
1046
  EX_CONFIG = config_element("", "", {
1004
1047
  "tag" => "tail",
@@ -1008,38 +1051,43 @@ class TailInputTest < Test::Unit::TestCase
1008
1051
  "read_from_head" => true,
1009
1052
  "refresh_interval" => 30,
1010
1053
  "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1054
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1011
1055
  })
1012
- EX_PATHS = [
1013
- 'test/plugin/data/2010/01/20100102-030405.log',
1014
- 'test/plugin/data/log/foo/bar.log',
1015
- 'test/plugin/data/log/test.log'
1016
- ]
1017
-
1018
1056
  def test_expand_paths
1057
+ ex_paths = [
1058
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1059
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1060
+ create_target_info('test/plugin/data/log/test.log')
1061
+ ]
1019
1062
  plugin = create_driver(EX_CONFIG, false).instance
1020
1063
  flexstub(Time) do |timeclass|
1021
1064
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1022
- assert_equal EX_PATHS, plugin.expand_paths.sort
1065
+ assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1023
1066
  end
1024
1067
 
1025
1068
  # Test exclusion
1026
- exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
1069
+ exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1027
1070
  plugin = create_driver(exclude_config, false).instance
1028
- assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
1071
+ assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1029
1072
  end
1030
1073
 
1031
1074
  def test_expand_paths_with_duplicate_configuration
1032
1075
  expanded_paths = [
1033
- 'test/plugin/data/log/foo/bar.log',
1034
- 'test/plugin/data/log/test.log'
1076
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1077
+ create_target_info('test/plugin/data/log/test.log')
1035
1078
  ]
1036
1079
  duplicate_config = EX_CONFIG.dup
1037
1080
  duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1038
1081
  plugin = create_driver(EX_CONFIG, false).instance
1039
- assert_equal expanded_paths, plugin.expand_paths.sort
1082
+ assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1040
1083
  end
1041
1084
 
1042
1085
  def test_expand_paths_with_timezone
1086
+ ex_paths = [
1087
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1088
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1089
+ create_target_info('test/plugin/data/log/test.log')
1090
+ ]
1043
1091
  ['Asia/Taipei', '+08'].each do |tz_type|
1044
1092
  taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
1045
1093
  plugin = create_driver(taipei_config, false).instance
@@ -1053,8 +1101,8 @@ class TailInputTest < Test::Unit::TestCase
1053
1101
  # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1054
1102
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1055
1103
 
1056
- assert_equal EX_PATHS, plugin.expand_paths.sort
1057
- assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
1104
+ assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1105
+ assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1058
1106
  end
1059
1107
  end
1060
1108
  end
@@ -1062,10 +1110,10 @@ class TailInputTest < Test::Unit::TestCase
1062
1110
 
1063
1111
  def test_log_file_without_extension
1064
1112
  expected_files = [
1065
- 'test/plugin/data/log/bar',
1066
- 'test/plugin/data/log/foo/bar.log',
1067
- 'test/plugin/data/log/foo/bar2',
1068
- 'test/plugin/data/log/test.log'
1113
+ create_target_info('test/plugin/data/log/bar'),
1114
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1115
+ create_target_info('test/plugin/data/log/foo/bar2'),
1116
+ create_target_info('test/plugin/data/log/test.log')
1069
1117
  ]
1070
1118
 
1071
1119
  config = config_element("", "", {
@@ -1076,7 +1124,7 @@ class TailInputTest < Test::Unit::TestCase
1076
1124
  })
1077
1125
 
1078
1126
  plugin = create_driver(config, false).instance
1079
- assert_equal expected_files, plugin.expand_paths.sort
1127
+ assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1080
1128
  end
1081
1129
 
1082
1130
  def test_unwatched_files_should_be_removed
@@ -1112,6 +1160,49 @@ class TailInputTest < Test::Unit::TestCase
1112
1160
  end
1113
1161
  end
1114
1162
 
1163
+ sub_test_case "path w/ Linux capability" do
1164
+ def capability_enabled?
1165
+ if Fluent.linux?
1166
+ begin
1167
+ require 'capng'
1168
+ true
1169
+ rescue LoadError
1170
+ false
1171
+ end
1172
+ else
1173
+ false
1174
+ end
1175
+ end
1176
+
1177
+ setup do
1178
+ omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
1179
+
1180
+ @capng = CapNG.new(:current_process)
1181
+ flexstub(Fluent::Capability) do |klass|
1182
+ klass.should_receive(:new).with(:current_process).and_return(@capng)
1183
+ end
1184
+ end
1185
+
1186
+ data("dac_read_search" => [:dac_read_search, true, 1],
1187
+ "dac_override" => [:dac_override, true, 1],
1188
+ "chown" => [:chown, false, 0],
1189
+ )
1190
+ test "with partially elevated privileges" do |data|
1191
+ cap, result, readable_paths = data
1192
+ @capng.update(:add, :effective, cap)
1193
+
1194
+ d = create_driver(
1195
+ config_element("ROOT", "", {
1196
+ "path" => "/var/log/ker*.log", # Use /var/log/kern.log
1197
+ "tag" => "t1",
1198
+ "rotate_wait" => "2s"
1199
+ }) + PARSE_SINGLE_LINE_CONFIG, false)
1200
+
1201
+ assert_equal readable_paths, d.instance.expand_paths.length
1202
+ assert_equal result, d.instance.have_read_capability?
1203
+ end
1204
+ end
1205
+
1115
1206
  def test_pos_file_dir_creation
1116
1207
  config = config_element("", "", {
1117
1208
  "tag" => "tail",
@@ -1159,25 +1250,35 @@ class TailInputTest < Test::Unit::TestCase
1159
1250
  end
1160
1251
 
1161
1252
  def test_z_refresh_watchers
1253
+ ex_paths = [
1254
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1255
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1256
+ create_target_info('test/plugin/data/log/test.log'),
1257
+ ]
1162
1258
  plugin = create_driver(EX_CONFIG, false).instance
1163
1259
  sio = StringIO.new
1164
1260
  plugin.instance_eval do
1165
- @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
1261
+ @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1166
1262
  @loop = Coolio::Loop.new
1167
1263
  end
1168
1264
 
1169
1265
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1170
- EX_PATHS.each do |path|
1171
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
1266
+ ex_paths.each do |target_info|
1267
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1172
1268
  end
1173
1269
 
1174
1270
  plugin.refresh_watchers
1175
1271
  end
1176
1272
 
1177
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
1273
+ path = 'test/plugin/data/2010/01/20100102-030405.log'
1274
+ target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
1275
+ mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
1178
1276
 
1179
1277
  Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1180
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
1278
+ path = "test/plugin/data/2010/01/20100102-030406.log"
1279
+ inode = Fluent::FileWrapper.stat(path).ino
1280
+ target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1281
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1181
1282
  plugin.refresh_watchers
1182
1283
 
1183
1284
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1323,6 +1424,305 @@ class TailInputTest < Test::Unit::TestCase
1323
1424
  end
1324
1425
  end
1325
1426
 
1427
+ sub_test_case 'inode_processing' do
1428
+ def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
1429
+ config = COMMON_FOLLOW_INODE_CONFIG
1430
+
1431
+ path = "#{TMP_DIR}/tail.txt"
1432
+ ino = 1
1433
+ pos = 1234
1434
+ File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
1435
+ f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1436
+ }
1437
+
1438
+ d = create_driver(config, false)
1439
+ d.run
1440
+
1441
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1442
+ pos_file.pos = 0
1443
+
1444
+ assert_raise(EOFError) do
1445
+ pos_file.readline
1446
+ end
1447
+ end
1448
+
1449
+ def test_should_write_latest_offset_after_rotate_wait
1450
+ config = COMMON_FOLLOW_INODE_CONFIG
1451
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1452
+ f.puts "test1"
1453
+ f.puts "test2"
1454
+ }
1455
+
1456
+ d = create_driver(config, false)
1457
+ d.run(expect_emits: 2, shutdown: false) do
1458
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1459
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
1460
+ sleep 1
1461
+ File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
1462
+ end
1463
+
1464
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1465
+ pos_file.pos = 0
1466
+ line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1467
+ waiting(5) {
1468
+ while line_parts[2].to_i(16) != 24
1469
+ sleep(0.1)
1470
+ pos_file.pos = 0
1471
+ line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1472
+ end
1473
+ }
1474
+ assert_equal(24, line_parts[2].to_i(16))
1475
+ d.instance_shutdown
1476
+ end
1477
+
1478
+ def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1479
+ config = config_element("", "", {"format" => "none"})
1480
+
1481
+ path = "#{TMP_DIR}/tail.txt"
1482
+ ino = 1
1483
+ pos = 1234
1484
+ File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
1485
+ f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1486
+ }
1487
+
1488
+ d = create_driver(config)
1489
+ d.run(shutdown: false)
1490
+
1491
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1492
+ pos_file.pos = 0
1493
+
1494
+ path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1495
+ assert_equal(path, path_pos_ino[1])
1496
+ assert_equal(pos, path_pos_ino[2].to_i(16))
1497
+ assert_equal(ino, path_pos_ino[3].to_i(16))
1498
+
1499
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1500
+ f.puts "test1"
1501
+ f.puts "test2"
1502
+ }
1503
+ Timecop.travel(Time.now + 10) do
1504
+ sleep 5
1505
+ pos_file.pos = 0
1506
+ tuple = create_target_info("#{TMP_DIR}/tail.txt")
1507
+ path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1508
+ assert_equal(tuple.path, path_pos_ino[1])
1509
+ assert_equal(12, path_pos_ino[2].to_i(16))
1510
+ assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1511
+ end
1512
+ d.instance_shutdown
1513
+ end
1514
+
1515
+ def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
1516
+ config = config_element("ROOT", "", {
1517
+ "path" => "#{TMP_DIR}/tail.txt*",
1518
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1519
+ "tag" => "t1",
1520
+ "rotate_wait" => "1s",
1521
+ "refresh_interval" => "1s",
1522
+ "limit_recently_modified" => "1s",
1523
+ "read_from_head" => "true",
1524
+ "format" => "none",
1525
+ "follow_inodes" => "true",
1526
+ })
1527
+
1528
+ d = create_driver(config, false)
1529
+
1530
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1531
+ f.puts "test1"
1532
+ f.puts "test2"
1533
+ }
1534
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1535
+
1536
+ d.run(expect_emits: 1, shutdown: false) do
1537
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1538
+ end
1539
+
1540
+
1541
+ Timecop.travel(Time.now + 10) do
1542
+ waiting(5) {
1543
+ # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
1544
+ sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
1545
+ }
1546
+ end
1547
+
1548
+ assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
1549
+
1550
+ d.instance_shutdown
1551
+ end
1552
+
1553
+ def test_should_read_from_head_on_file_renaming_with_star_in_pattern
1554
+ config = config_element("ROOT", "", {
1555
+ "path" => "#{TMP_DIR}/tail.txt*",
1556
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1557
+ "tag" => "t1",
1558
+ "rotate_wait" => "10s",
1559
+ "refresh_interval" => "1s",
1560
+ "limit_recently_modified" => "60s",
1561
+ "read_from_head" => "true",
1562
+ "format" => "none",
1563
+ "follow_inodes" => "true"
1564
+ })
1565
+
1566
+ d = create_driver(config, false)
1567
+
1568
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1569
+ f.puts "test1"
1570
+ f.puts "test2"
1571
+ }
1572
+
1573
+ d.run(expect_emits: 2, shutdown: false) do
1574
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1575
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
1576
+ end
1577
+
1578
+ events = d.events
1579
+ assert_equal(3, events.length)
1580
+ d.instance_shutdown
1581
+ end
1582
+
1583
+ def test_should_not_read_from_head_on_rotation_when_watching_inodes
1584
+ config = COMMON_FOLLOW_INODE_CONFIG
1585
+
1586
+ d = create_driver(config, false)
1587
+
1588
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1589
+ f.puts "test1"
1590
+ f.puts "test2"
1591
+ }
1592
+
1593
+ d.run(expect_emits: 1, shutdown: false) do
1594
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1595
+ end
1596
+
1597
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
1598
+ Timecop.travel(Time.now + 10) do
1599
+ sleep 2
1600
+ events = d.events
1601
+ assert_equal(3, events.length)
1602
+ end
1603
+
1604
+ d.instance_shutdown
1605
+ end
1606
+
1607
+ def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
1608
+ config = COMMON_FOLLOW_INODE_CONFIG
1609
+
1610
+ d = create_driver(config, false)
1611
+
1612
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1613
+ f.puts "test1"
1614
+ f.puts "test2"
1615
+ }
1616
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1617
+
1618
+ d.run(expect_emits: 2, shutdown: false) do
1619
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1620
+ cleanup_file("#{TMP_DIR}/tail.txt")
1621
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
1622
+ end
1623
+
1624
+ new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
1625
+
1626
+ pos_file = d.instance.instance_variable_get(:@pf)
1627
+
1628
+ waiting(10) {
1629
+ # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
1630
+ sleep 0.1 until pos_file[target_info].read_pos == 0
1631
+ }
1632
+ new_position = pos_file[new_target_info].read_pos
1633
+ assert_equal(6, new_position)
1634
+
1635
+ d.instance_shutdown
1636
+ end
1637
+
1638
+ def test_should_close_watcher_after_rotate_wait
1639
+ now = Time.now
1640
+ config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1641
+
1642
+ d = create_driver(config, false)
1643
+
1644
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1645
+ f.puts "test1"
1646
+ f.puts "test2"
1647
+ }
1648
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1649
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
1650
+ d.run(shutdown: false)
1651
+ assert d.instance.instance_variable_get(:@tails)[target_info]
1652
+
1653
+ Timecop.travel(now + 10) do
1654
+ d.instance.instance_eval do
1655
+ sleep 0.1 until @tails[target_info] == nil
1656
+ end
1657
+ assert_nil d.instance.instance_variable_get(:@tails)[target_info]
1658
+ end
1659
+ d.instance_shutdown
1660
+ end
1661
+
1662
+ def test_should_create_new_watcher_for_new_file_with_same_name
1663
+ now = Time.now
1664
+ config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
1665
+
1666
+ d = create_driver(config, false)
1667
+
1668
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1669
+ f.puts "test1"
1670
+ f.puts "test2"
1671
+ }
1672
+ path_ino = create_target_info("#{TMP_DIR}/tail.txt")
1673
+
1674
+ d.run(expect_emits: 1, shutdown: false) do
1675
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1676
+ end
1677
+
1678
+ cleanup_file("#{TMP_DIR}/tail.txt")
1679
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1680
+ f.puts "test3"
1681
+ f.puts "test4"
1682
+ }
1683
+ new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
1684
+
1685
+ Timecop.travel(now + 10) do
1686
+ sleep 3
1687
+ d.instance.instance_eval do
1688
+ @tails[path_ino] == nil
1689
+ @tails[new_path_ino] != nil
1690
+ end
1691
+ end
1692
+
1693
+ events = d.events
1694
+
1695
+ assert_equal(5, events.length)
1696
+
1697
+ d.instance_shutdown
1698
+ end
1699
+
1700
+ def test_truncate_file_with_follow_inodes
1701
+ config = COMMON_FOLLOW_INODE_CONFIG
1702
+
1703
+ d = create_driver(config, false)
1704
+
1705
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1706
+ f.puts "test1"
1707
+ f.puts "test2"
1708
+ }
1709
+
1710
+ d.run(expect_emits: 3, shutdown: false) do
1711
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1712
+ sleep 2
1713
+ File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
1714
+ end
1715
+
1716
+ events = d.events
1717
+ assert_equal(4, events.length)
1718
+ assert_equal({"message" => "test1"}, events[0][2])
1719
+ assert_equal({"message" => "test2"}, events[1][2])
1720
+ assert_equal({"message" => "test3"}, events[2][2])
1721
+ assert_equal({"message" => "test4"}, events[3][2])
1722
+ d.instance_shutdown
1723
+ end
1724
+ end
1725
+
1326
1726
  sub_test_case "tail_path" do
1327
1727
  def test_tail_path_with_singleline
1328
1728
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
@@ -1403,6 +1803,9 @@ class TailInputTest < Test::Unit::TestCase
1403
1803
  end
1404
1804
 
1405
1805
  def test_tail_path_with_multiline_with_multiple_paths
1806
+ if ENV["APPVEYOR"] && Fluent.windows?
1807
+ omit "This testcase is unstable on AppVeyor."
1808
+ end
1406
1809
  files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
1407
1810
  files.each { |file| File.open(file, "wb") { |f| } }
1408
1811
 
@@ -1449,13 +1852,13 @@ class TailInputTest < Test::Unit::TestCase
1449
1852
  })
1450
1853
 
1451
1854
  expected_files = [
1452
- "#{TMP_DIR}/tail_watch1.txt",
1453
- "#{TMP_DIR}/tail_watch2.txt"
1855
+ create_target_info("#{TMP_DIR}/tail_watch1.txt"),
1856
+ create_target_info("#{TMP_DIR}/tail_watch2.txt")
1454
1857
  ]
1455
1858
 
1456
1859
  Timecop.freeze(now) do
1457
1860
  plugin = create_driver(config, false).instance
1458
- assert_equal expected_files, plugin.expand_paths.sort
1861
+ assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1459
1862
  end
1460
1863
  end
1461
1864