fluentd 1.11.5-x86-mingw32 → 1.12.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
  4. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  5. data/.github/workflows/linux-test.yaml +36 -0
  6. data/.github/workflows/macos-test.yaml +30 -0
  7. data/.github/workflows/stale-actions.yml +22 -0
  8. data/.github/workflows/windows-test.yaml +35 -0
  9. data/.gitlab-ci.yml +41 -19
  10. data/CHANGELOG.md +158 -0
  11. data/MAINTAINERS.md +5 -2
  12. data/README.md +7 -4
  13. data/bin/fluent-cap-ctl +7 -0
  14. data/bin/fluent-ctl +7 -0
  15. data/fluentd.gemspec +6 -4
  16. data/lib/fluent/capability.rb +87 -0
  17. data/lib/fluent/command/bundler_injection.rb +1 -1
  18. data/lib/fluent/command/ca_generate.rb +6 -3
  19. data/lib/fluent/command/cap_ctl.rb +174 -0
  20. data/lib/fluent/command/cat.rb +0 -1
  21. data/lib/fluent/command/ctl.rb +177 -0
  22. data/lib/fluent/command/fluentd.rb +4 -0
  23. data/lib/fluent/command/plugin_config_formatter.rb +18 -2
  24. data/lib/fluent/command/plugin_generator.rb +31 -1
  25. data/lib/fluent/compat/parser.rb +2 -2
  26. data/lib/fluent/config/section.rb +2 -2
  27. data/lib/fluent/config/types.rb +2 -2
  28. data/lib/fluent/env.rb +4 -0
  29. data/lib/fluent/event.rb +3 -13
  30. data/lib/fluent/load.rb +0 -1
  31. data/lib/fluent/plugin.rb +5 -0
  32. data/lib/fluent/plugin/buffer.rb +2 -21
  33. data/lib/fluent/plugin/file_wrapper.rb +39 -3
  34. data/lib/fluent/plugin/formatter.rb +2 -2
  35. data/lib/fluent/plugin/formatter_csv.rb +1 -1
  36. data/lib/fluent/plugin/formatter_hash.rb +1 -1
  37. data/lib/fluent/plugin/formatter_ltsv.rb +5 -5
  38. data/lib/fluent/plugin/formatter_out_file.rb +3 -3
  39. data/lib/fluent/plugin/formatter_single_value.rb +2 -2
  40. data/lib/fluent/plugin/formatter_tsv.rb +2 -2
  41. data/lib/fluent/plugin/in_http.rb +24 -3
  42. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  43. data/lib/fluent/plugin/in_tail.rb +129 -41
  44. data/lib/fluent/plugin/in_tail/position_file.rb +53 -14
  45. data/lib/fluent/plugin/in_tcp.rb +1 -0
  46. data/lib/fluent/plugin/out_copy.rb +18 -5
  47. data/lib/fluent/plugin/out_exec_filter.rb +3 -3
  48. data/lib/fluent/plugin/out_forward.rb +61 -28
  49. data/lib/fluent/plugin/out_http.rb +9 -2
  50. data/lib/fluent/plugin/output.rb +18 -10
  51. data/lib/fluent/plugin/parser_csv.rb +2 -2
  52. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  53. data/lib/fluent/plugin/storage_local.rb +4 -4
  54. data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
  55. data/lib/fluent/plugin_helper/inject.rb +4 -2
  56. data/lib/fluent/plugin_helper/retry_state.rb +4 -0
  57. data/lib/fluent/plugin_helper/server.rb +4 -2
  58. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  59. data/lib/fluent/supervisor.rb +153 -47
  60. data/lib/fluent/system_config.rb +2 -1
  61. data/lib/fluent/time.rb +58 -1
  62. data/lib/fluent/version.rb +1 -1
  63. data/lib/fluent/winsvc.rb +22 -4
  64. data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
  65. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  66. data/test/command/test_cap_ctl.rb +100 -0
  67. data/test/command/test_ctl.rb +57 -0
  68. data/test/command/test_fluentd.rb +38 -0
  69. data/test/command/test_plugin_config_formatter.rb +124 -2
  70. data/test/config/test_configurable.rb +1 -1
  71. data/test/plugin/in_tail/test_position_file.rb +100 -26
  72. data/test/plugin/test_file_wrapper.rb +105 -0
  73. data/test/plugin/test_in_exec.rb +1 -1
  74. data/test/plugin/test_in_http.rb +25 -0
  75. data/test/plugin/test_in_tail.rb +503 -42
  76. data/test/plugin/test_out_copy.rb +87 -0
  77. data/test/plugin/test_out_forward.rb +94 -6
  78. data/test/plugin/test_out_http.rb +20 -1
  79. data/test/plugin/test_output.rb +15 -3
  80. data/test/plugin/test_output_as_buffered_backup.rb +2 -0
  81. data/test/plugin/test_parser_csv.rb +14 -0
  82. data/test/plugin/test_parser_syslog.rb +16 -2
  83. data/test/plugin/test_sd_file.rb +1 -1
  84. data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
  85. data/test/plugin_helper/test_child_process.rb +5 -2
  86. data/test/plugin_helper/test_http_server_helper.rb +4 -2
  87. data/test/plugin_helper/test_inject.rb +29 -0
  88. data/test/plugin_helper/test_server.rb +26 -7
  89. data/test/test_capability.rb +74 -0
  90. data/test/test_event.rb +16 -0
  91. data/test/test_formatter.rb +30 -0
  92. data/test/test_output.rb +2 -2
  93. data/test/test_supervisor.rb +133 -10
  94. data/test/test_time_parser.rb +109 -0
  95. metadata +85 -31
  96. data/.travis.yml +0 -57
  97. data/appveyor.yml +0 -28
@@ -1453,7 +1453,7 @@ module Fluent::Config
1453
1453
  @example = ConfigurableSpec::ExampleWithSkipAccessor.new
1454
1454
  @example.configure(config_element('ROOT'))
1455
1455
  assert_equal 'example7', @example.instance_variable_get(:@name)
1456
- assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
1456
+ assert_raise NoMethodError do
1457
1457
  @example.name
1458
1458
  end
1459
1459
  end
@@ -1,5 +1,6 @@
1
1
  require_relative '../../helper'
2
2
  require 'fluent/plugin/in_tail/position_file'
3
+ require 'fluent/plugin/in_tail'
3
4
 
4
5
  require 'fileutils'
5
6
  require 'tempfile'
@@ -27,9 +28,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
27
28
  f.seek(0)
28
29
  end
29
30
 
31
+ def follow_inodes_block
32
+ [true, false].each do |follow_inodes|
33
+ yield follow_inodes
34
+ end
35
+ end
36
+
30
37
  test '.load' do
31
38
  write_data(@file, TEST_CONTENT)
32
- Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
39
+ Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
33
40
 
34
41
  @file.seek(0)
35
42
  lines = @file.readlines
@@ -41,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
41
48
  sub_test_case '#try_compact' do
42
49
  test 'compact invalid and convert 32 bit inode value' do
43
50
  write_data(@file, TEST_CONTENT)
44
- Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
51
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
45
52
 
46
53
  @file.seek(0)
47
54
  lines = @file.readlines
@@ -55,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
55
62
  valid_path\t0000000000000002\t0000000000000001
56
63
  valid_path\t0000000000000003\t0000000000000004
57
64
  EOF
58
- Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
65
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
59
66
 
60
67
  @file.seek(0)
61
68
  lines = @file.readlines
@@ -64,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
64
71
 
65
72
  test 'does not change when the file is changed' do
66
73
  write_data(@file, TEST_CONTENT)
67
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
74
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
68
75
 
69
76
  mock.proxy(pf).fetch_compacted_entries do |r|
70
77
  @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -78,12 +85,17 @@ class IntailPositionFileTest < Test::Unit::TestCase
78
85
  assert_equal 5, lines.size
79
86
  end
80
87
 
81
- test 'update seek postion of remained position entry' do
82
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
83
- pf['path1']
84
- pf['path2']
85
- pf['path3']
86
- pf.unwatch('path1')
88
+ test 'update seek position of remained position entry' do
89
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
90
+ target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
91
+ target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
92
+ target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
93
+ pf[target_info1]
94
+ pf[target_info2]
95
+ pf[target_info3]
96
+
97
+ target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
98
+ pf.unwatch(target_info1_2)
87
99
 
88
100
  pf.try_compact
89
101
 
@@ -93,8 +105,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
93
105
  assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
94
106
  assert_equal 2, lines.size
95
107
 
96
- pf.unwatch('path2')
97
- pf.unwatch('path3')
108
+ target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
109
+ target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
110
+ pf.unwatch(target_info2_2)
111
+ pf.unwatch(target_info3_2)
98
112
  @file.seek(0)
99
113
  lines = @file.readlines
100
114
  assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
@@ -106,7 +120,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
106
120
  sub_test_case '#load' do
107
121
  test 'compact invalid and convert 32 bit inode value' do
108
122
  write_data(@file, TEST_CONTENT)
109
- Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
123
+ Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
110
124
 
111
125
  @file.seek(0)
112
126
  lines = @file.readlines
@@ -120,7 +134,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
120
134
  valid_path\t0000000000000002\t0000000000000001
121
135
  valid_path\t0000000000000003\t0000000000000004
122
136
  EOF
123
- Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
137
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
124
138
 
125
139
  @file.seek(0)
126
140
  lines = @file.readlines
@@ -131,9 +145,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
131
145
  sub_test_case '#[]' do
132
146
  test 'return entry' do
133
147
  write_data(@file, TEST_CONTENT)
134
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
148
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
135
149
 
136
- f = pf['valid_path']
150
+ valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
151
+ f = pf[valid_target_info]
137
152
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
138
153
  assert_equal 2, f.read_pos
139
154
  assert_equal 1, f.read_inode
@@ -142,7 +157,8 @@ class IntailPositionFileTest < Test::Unit::TestCase
142
157
  lines = @file.readlines
143
158
  assert_equal 2, lines.size
144
159
 
145
- f = pf['nonexist_path']
160
+ nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
161
+ f = pf[nonexistent_target_info]
146
162
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
147
163
  assert_equal 0, f.read_pos
148
164
  assert_equal 0, f.read_inode
@@ -155,19 +171,19 @@ class IntailPositionFileTest < Test::Unit::TestCase
155
171
 
156
172
  test 'does not change other value position if other entry try to write' do
157
173
  write_data(@file, TEST_CONTENT)
158
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
174
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
159
175
 
160
- f = pf['nonexist_path']
176
+ f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
161
177
  assert_equal 0, f.read_inode
162
178
  assert_equal 0, f.read_pos
163
179
 
164
- pf['valid_path'].update(1, 2)
180
+ pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
165
181
 
166
- f = pf['nonexist_path']
182
+ f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
167
183
  assert_equal 0, f.read_inode
168
184
  assert_equal 0, f.read_pos
169
185
 
170
- pf['nonexist_path'].update(1, 2)
186
+ pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
171
187
  assert_equal 1, f.read_inode
172
188
  assert_equal 2, f.read_pos
173
189
  end
@@ -176,14 +192,18 @@ class IntailPositionFileTest < Test::Unit::TestCase
176
192
  sub_test_case '#unwatch' do
177
193
  test 'deletes entry by path' do
178
194
  write_data(@file, TEST_CONTENT)
179
- pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
180
- p1 = pf['valid_path']
195
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
196
+ inode1 = File.stat(@file).ino
197
+ target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
198
+ p1 = pf[target_info1]
181
199
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
182
200
 
183
- pf.unwatch('valid_path')
201
+ pf.unwatch(target_info1)
184
202
  assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
185
203
 
186
- p2 = pf['valid_path']
204
+ inode2 = File.stat(@file).ino
205
+ target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
206
+ p2 = pf[target_info2]
187
207
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
188
208
 
189
209
  assert_not_equal p1, p2
@@ -262,4 +282,58 @@ class IntailPositionFileTest < Test::Unit::TestCase
262
282
  assert_equal 2, f.read_inode
263
283
  end
264
284
  end
285
+
286
+ sub_test_case "TargetInfo equality rules" do
287
+ sub_test_case "== operator" do
288
+ def test_equal
289
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
290
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
291
+
292
+ assert_equal t1, t2
293
+ end
294
+
295
+ def test_not_equal
296
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
297
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
298
+
299
+ assert_not_equal t1, t2
300
+ end
301
+ end
302
+
303
+ sub_test_case "eql? method" do
304
+ def test_eql?
305
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
306
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
307
+
308
+ assert do
309
+ t1.eql? t2
310
+ end
311
+ end
312
+
313
+ def test_not_eql?
314
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
315
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
316
+
317
+ assert do
318
+ !t1.eql? t2
319
+ end
320
+ end
321
+ end
322
+
323
+ sub_test_case "hash" do
324
+ def test_equal
325
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
326
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
327
+
328
+ assert_equal t1.hash, t2.hash
329
+ end
330
+
331
+ def test_not_equal
332
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
333
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
334
+
335
+ assert_not_equal t1.hash, t2.hash
336
+ end
337
+ end
338
+ end
265
339
  end
@@ -0,0 +1,105 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/file_wrapper'
3
+
4
+ class FileWrapperTest < Test::Unit::TestCase
5
+ require 'windows/file'
6
+ require 'windows/error'
7
+ include Windows::File
8
+ include Windows::Error
9
+
10
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
11
+
12
+ def setup
13
+ FileUtils.mkdir_p(TMP_DIR)
14
+ end
15
+
16
+ def teardown
17
+ FileUtils.rm_rf(TMP_DIR)
18
+ end
19
+
20
+ sub_test_case 'Win32Error' do
21
+ test 'equal' do
22
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
23
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
24
+ end
25
+
26
+ test 'different error code' do
27
+ assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
28
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
29
+ end
30
+
31
+ test 'different error message' do
32
+ assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
33
+ Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
34
+ end
35
+
36
+ test 'different class' do
37
+ assert_not_equal(Errno::EPIPE,
38
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
39
+ end
40
+
41
+ test 'ERROR_SHARING_VIOLATION message' do
42
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
43
+ "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
44
+ end
45
+
46
+ test 'ERROR_SHARING_VIOLATION with a message' do
47
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
48
+ "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
49
+ " - cannot open the file")
50
+ end
51
+
52
+ test 'to_s' do
53
+ assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
54
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
55
+ end
56
+
57
+ test 'inspect' do
58
+ assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
59
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
60
+ end
61
+ end
62
+
63
+ sub_test_case 'WindowsFile exceptions' do
64
+ test 'nothing raised' do
65
+ begin
66
+ path = "#{TMP_DIR}/test_windows_file.txt"
67
+ file1 = file2 = nil
68
+ file1 = File.open(path, "wb") do |f|
69
+ end
70
+ assert_nothing_raised do
71
+ file2 = Fluent::WindowsFile.new(path)
72
+ ensure
73
+ file2.close
74
+ end
75
+ ensure
76
+ file1.close if file1
77
+ end
78
+ end
79
+
80
+ test 'Errno::ENOENT raised' do
81
+ path = "#{TMP_DIR}/nofile.txt"
82
+ file = nil
83
+ assert_raise(Errno::ENOENT) do
84
+ file = Fluent::WindowsFile.new(path)
85
+ ensure
86
+ file.close if file
87
+ end
88
+ end
89
+
90
+ test 'ERROR_SHARING_VIOLATION raised' do
91
+ begin
92
+ path = "#{TMP_DIR}/test_windows_file.txt"
93
+ file1 = file2 = nil
94
+ file1 = File.open(path, "wb")
95
+ assert_raise(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)) do
96
+ file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
97
+ ensure
98
+ file2.close if file2
99
+ end
100
+ ensure
101
+ file1.close if file1
102
+ end
103
+ end
104
+ end
105
+ end if Fluent.windows?
@@ -255,7 +255,7 @@ EOC
255
255
  assert{ d.events.length > 0 }
256
256
  d.events.each do |event|
257
257
  assert_equal 'test', event[0]
258
- assert_match /LoadError/, event[2]['message']
258
+ assert_match(/LoadError/, event[2]['message'])
259
259
  end
260
260
  end
261
261
  end
@@ -46,6 +46,7 @@ class HttpInputTest < Test::Unit::TestCase
46
46
  assert_equal 10*1024*1024, d.instance.body_size_limit
47
47
  assert_equal 5, d.instance.keepalive_timeout
48
48
  assert_equal false, d.instance.add_http_headers
49
+ assert_equal false, d.instance.add_query_params
49
50
  end
50
51
 
51
52
  def test_time
@@ -907,6 +908,30 @@ class HttpInputTest < Test::Unit::TestCase
907
908
  assert_equal ["403", "403"], res_codes
908
909
  end
909
910
 
911
+ def test_add_query_params
912
+ d = create_driver(CONFIG + "add_query_params true")
913
+ assert_equal true, d.instance.add_query_params
914
+
915
+ time = event_time("2011-01-02 13:14:15 UTC")
916
+ time_i = time.to_i
917
+ events = [
918
+ ["tag1", time, {"a"=>1, "QUERY_A"=>"b"}],
919
+ ["tag2", time, {"a"=>2, "QUERY_A"=>"b"}],
920
+ ]
921
+ res_codes = []
922
+ res_bodies = []
923
+
924
+ d.run do
925
+ events.each do |tag, _t, record|
926
+ res = post("/#{tag}?a=b", {"json"=>record.to_json, "time"=>time_i.to_s})
927
+ res_codes << res.code
928
+ end
929
+ end
930
+ assert_equal ["200", "200"], res_codes
931
+ assert_equal [], res_bodies
932
+ assert_equal events, d.events
933
+ end
934
+
910
935
  $test_in_http_connection_object_ids = []
911
936
  $test_in_http_content_types = []
912
937
  $test_in_http_content_types_flag = false
@@ -6,6 +6,8 @@ require 'fluent/system_config'
6
6
  require 'net/http'
7
7
  require 'flexmock/test_unit'
8
8
  require 'timecop'
9
+ require 'tmpdir'
10
+ require 'securerandom'
9
11
 
10
12
  class TailInputTest < Test::Unit::TestCase
11
13
  include FlexMock::TestCase
@@ -17,29 +19,98 @@ class TailInputTest < Test::Unit::TestCase
17
19
 
18
20
  def teardown
19
21
  super
22
+ cleanup_directory(TMP_DIR)
20
23
  Fluent::Engine.stop
21
24
  end
22
25
 
23
26
  def cleanup_directory(path)
24
- FileUtils.rm_rf(path, secure: true)
25
- if File.exist?(path)
26
- FileUtils.remove_entry_secure(path, true)
27
+ unless Dir.exist?(path)
28
+ FileUtils.mkdir_p(path)
29
+ return
30
+ end
31
+
32
+ if Fluent.windows?
33
+ Dir.glob("*", base: path).each do |name|
34
+ begin
35
+ cleanup_file(File.join(path, name))
36
+ rescue
37
+ # expect test driver block release already owned file handle.
38
+ end
39
+ end
40
+ else
41
+ begin
42
+ FileUtils.rm_f(path, secure:true)
43
+ rescue ArgumentError
44
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
45
+ end
46
+ if File.exist?(path)
47
+ FileUtils.remove_entry_secure(path, true)
48
+ end
27
49
  end
28
50
  FileUtils.mkdir_p(path)
29
51
  end
30
52
 
53
+ def cleanup_file(path)
54
+ if Fluent.windows?
55
+ # On Windows, when the file or directory is removed and created
56
+ # frequently, there is a case that creating file or directory will
57
+ # fail. This situation is caused by pending file or directory
58
+ # deletion which is mentioned on win32 API document [1]
59
+ # As a workaround, execute rename and remove method.
60
+ #
61
+ # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
62
+ #
63
+ file = File.join(Dir.tmpdir, SecureRandom.hex(10))
64
+ begin
65
+ FileUtils.mv(path, file)
66
+ FileUtils.rm_rf(file, secure: true)
67
+ rescue ArgumentError
68
+ FileUtils.rm_rf(file) # For Ruby 2.6 or before.
69
+ end
70
+ if File.exist?(file)
71
+ # ensure files are closed for Windows, on which deleted files
72
+ # are still visible from filesystem
73
+ GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
74
+ FileUtils.remove_entry_secure(file, true)
75
+ end
76
+ else
77
+ begin
78
+ FileUtils.rm_f(path, secure: true)
79
+ rescue ArgumentError
80
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
81
+ end
82
+ if File.exist?(path)
83
+ FileUtils.remove_entry_secure(path, true)
84
+ end
85
+ end
86
+ end
87
+
88
+ def create_target_info(path)
89
+ Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
90
+ end
91
+
31
92
  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
32
93
 
33
94
  CONFIG = config_element("ROOT", "", {
34
95
  "path" => "#{TMP_DIR}/tail.txt",
35
96
  "tag" => "t1",
36
- "rotate_wait" => "2s"
97
+ "rotate_wait" => "2s",
98
+ "refresh_interval" => "1s"
37
99
  })
38
100
  COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
39
101
  CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
40
102
  CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
41
103
  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
42
104
  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
105
+ COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
106
+ "path" => "#{TMP_DIR}/tail.txt*",
107
+ "pos_file" => "#{TMP_DIR}/tail.pos",
108
+ "tag" => "t1",
109
+ "refresh_interval" => "1s",
110
+ "read_from_head" => "true",
111
+ "format" => "none",
112
+ "follow_inodes" => "true"
113
+ })
43
114
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
44
115
  PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
45
116
  MULTILINE_CONFIG = config_element(
@@ -86,6 +157,9 @@ class TailInputTest < Test::Unit::TestCase
86
157
  assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
87
158
  assert_equal 1000, d.instance.read_lines_limit
88
159
  assert_equal false, d.instance.ignore_repeated_permission_error
160
+ assert_nothing_raised do
161
+ d.instance.have_read_capability?
162
+ end
89
163
  end
90
164
 
91
165
  data("empty" => config_element,
@@ -115,6 +189,12 @@ class TailInputTest < Test::Unit::TestCase
115
189
  end
116
190
  end
117
191
 
192
+ test "follow_inodes w/o pos file" do
193
+ assert_raise(Fluent::ConfigError) do
194
+ create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
195
+ end
196
+ end
197
+
118
198
  test "both enable_watch_timer and enable_stat_watcher are false" do
119
199
  assert_raise(Fluent::ConfigError) do
120
200
  create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -229,7 +309,7 @@ class TailInputTest < Test::Unit::TestCase
229
309
  d = create_driver(config)
230
310
  msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
231
311
 
232
- d.run(expect_emits: num_events, timeout: 1) do
312
+ d.run(expect_emits: num_events, timeout: 2) do
233
313
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
234
314
  f.puts msg
235
315
  f.puts msg
@@ -509,6 +589,7 @@ class TailInputTest < Test::Unit::TestCase
509
589
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
510
590
  f.puts "test1"
511
591
  f.puts "test2"
592
+ f.flush
512
593
  }
513
594
 
514
595
  d = create_driver(config)
@@ -518,19 +599,23 @@ class TailInputTest < Test::Unit::TestCase
518
599
  f.puts "test3\ntest4"
519
600
  f.flush
520
601
  }
521
- sleep 1
602
+ waiting(2) { sleep 0.1 until d.events.length == 2 }
522
603
  File.truncate("#{TMP_DIR}/tail.txt", 6)
523
604
  end
524
605
 
525
- events = d.events
526
- assert_equal(3, events.length)
527
- assert_equal({"message" => "test3"}, events[0][2])
528
- assert_equal({"message" => "test4"}, events[1][2])
529
- assert_equal({"message" => "test1"}, events[2][2])
530
- assert(events[0][1].is_a?(Fluent::EventTime))
531
- assert(events[1][1].is_a?(Fluent::EventTime))
532
- assert(events[2][1].is_a?(Fluent::EventTime))
533
- assert_equal(2, d.emit_count)
606
+ expected = {
607
+ emit_count: 2,
608
+ events: [
609
+ [Fluent::EventTime, {"message" => "test3"}],
610
+ [Fluent::EventTime, {"message" => "test4"}],
611
+ [Fluent::EventTime, {"message" => "test1"}],
612
+ ]
613
+ }
614
+ actual = {
615
+ emit_count: d.emit_count,
616
+ events: d.events.collect{|event| [event[1].class, event[2]]}
617
+ }
618
+ assert_equal(expected, actual)
534
619
  end
535
620
 
536
621
  def test_move_truncate_move_back
@@ -999,6 +1084,7 @@ class TailInputTest < Test::Unit::TestCase
999
1084
  # * path test
1000
1085
  # TODO: Clean up tests
1001
1086
  EX_ROTATE_WAIT = 0
1087
+ EX_FOLLOW_INODES = false
1002
1088
 
1003
1089
  EX_CONFIG = config_element("", "", {
1004
1090
  "tag" => "tail",
@@ -1008,38 +1094,43 @@ class TailInputTest < Test::Unit::TestCase
1008
1094
  "read_from_head" => true,
1009
1095
  "refresh_interval" => 30,
1010
1096
  "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1097
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1011
1098
  })
1012
- EX_PATHS = [
1013
- 'test/plugin/data/2010/01/20100102-030405.log',
1014
- 'test/plugin/data/log/foo/bar.log',
1015
- 'test/plugin/data/log/test.log'
1016
- ]
1017
-
1018
1099
  def test_expand_paths
1100
+ ex_paths = [
1101
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1102
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1103
+ create_target_info('test/plugin/data/log/test.log')
1104
+ ]
1019
1105
  plugin = create_driver(EX_CONFIG, false).instance
1020
1106
  flexstub(Time) do |timeclass|
1021
1107
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1022
- assert_equal EX_PATHS, plugin.expand_paths.sort
1108
+ assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1023
1109
  end
1024
1110
 
1025
1111
  # Test exclusion
1026
- exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
1112
+ exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1027
1113
  plugin = create_driver(exclude_config, false).instance
1028
- assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
1114
+ assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1029
1115
  end
1030
1116
 
1031
1117
  def test_expand_paths_with_duplicate_configuration
1032
1118
  expanded_paths = [
1033
- 'test/plugin/data/log/foo/bar.log',
1034
- 'test/plugin/data/log/test.log'
1119
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1120
+ create_target_info('test/plugin/data/log/test.log')
1035
1121
  ]
1036
1122
  duplicate_config = EX_CONFIG.dup
1037
1123
  duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1038
1124
  plugin = create_driver(EX_CONFIG, false).instance
1039
- assert_equal expanded_paths, plugin.expand_paths.sort
1125
+ assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1040
1126
  end
1041
1127
 
1042
1128
  def test_expand_paths_with_timezone
1129
+ ex_paths = [
1130
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1131
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1132
+ create_target_info('test/plugin/data/log/test.log')
1133
+ ]
1043
1134
  ['Asia/Taipei', '+08'].each do |tz_type|
1044
1135
  taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
1045
1136
  plugin = create_driver(taipei_config, false).instance
@@ -1053,8 +1144,8 @@ class TailInputTest < Test::Unit::TestCase
1053
1144
  # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1054
1145
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1055
1146
 
1056
- assert_equal EX_PATHS, plugin.expand_paths.sort
1057
- assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
1147
+ assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1148
+ assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1058
1149
  end
1059
1150
  end
1060
1151
  end
@@ -1062,10 +1153,10 @@ class TailInputTest < Test::Unit::TestCase
1062
1153
 
1063
1154
  def test_log_file_without_extension
1064
1155
  expected_files = [
1065
- 'test/plugin/data/log/bar',
1066
- 'test/plugin/data/log/foo/bar.log',
1067
- 'test/plugin/data/log/foo/bar2',
1068
- 'test/plugin/data/log/test.log'
1156
+ create_target_info('test/plugin/data/log/bar'),
1157
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1158
+ create_target_info('test/plugin/data/log/foo/bar2'),
1159
+ create_target_info('test/plugin/data/log/test.log')
1069
1160
  ]
1070
1161
 
1071
1162
  config = config_element("", "", {
@@ -1076,7 +1167,7 @@ class TailInputTest < Test::Unit::TestCase
1076
1167
  })
1077
1168
 
1078
1169
  plugin = create_driver(config, false).instance
1079
- assert_equal expected_files, plugin.expand_paths.sort
1170
+ assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1080
1171
  end
1081
1172
 
1082
1173
  def test_unwatched_files_should_be_removed
@@ -1112,6 +1203,49 @@ class TailInputTest < Test::Unit::TestCase
1112
1203
  end
1113
1204
  end
1114
1205
 
1206
+ sub_test_case "path w/ Linux capability" do
1207
+ def capability_enabled?
1208
+ if Fluent.linux?
1209
+ begin
1210
+ require 'capng'
1211
+ true
1212
+ rescue LoadError
1213
+ false
1214
+ end
1215
+ else
1216
+ false
1217
+ end
1218
+ end
1219
+
1220
+ setup do
1221
+ omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
1222
+
1223
+ @capng = CapNG.new(:current_process)
1224
+ flexstub(Fluent::Capability) do |klass|
1225
+ klass.should_receive(:new).with(:current_process).and_return(@capng)
1226
+ end
1227
+ end
1228
+
1229
+ data("dac_read_search" => [:dac_read_search, true, 1],
1230
+ "dac_override" => [:dac_override, true, 1],
1231
+ "chown" => [:chown, false, 0],
1232
+ )
1233
+ test "with partially elevated privileges" do |data|
1234
+ cap, result, readable_paths = data
1235
+ @capng.update(:add, :effective, cap)
1236
+
1237
+ d = create_driver(
1238
+ config_element("ROOT", "", {
1239
+ "path" => "/var/log/ker*.log", # Use /var/log/kern.log
1240
+ "tag" => "t1",
1241
+ "rotate_wait" => "2s"
1242
+ }) + PARSE_SINGLE_LINE_CONFIG, false)
1243
+
1244
+ assert_equal readable_paths, d.instance.expand_paths.length
1245
+ assert_equal result, d.instance.have_read_capability?
1246
+ end
1247
+ end
1248
+
1115
1249
  def test_pos_file_dir_creation
1116
1250
  config = config_element("", "", {
1117
1251
  "tag" => "tail",
@@ -1159,25 +1293,35 @@ class TailInputTest < Test::Unit::TestCase
1159
1293
  end
1160
1294
 
1161
1295
  def test_z_refresh_watchers
1296
+ ex_paths = [
1297
+ create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1298
+ create_target_info('test/plugin/data/log/foo/bar.log'),
1299
+ create_target_info('test/plugin/data/log/test.log'),
1300
+ ]
1162
1301
  plugin = create_driver(EX_CONFIG, false).instance
1163
1302
  sio = StringIO.new
1164
1303
  plugin.instance_eval do
1165
- @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
1304
+ @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1166
1305
  @loop = Coolio::Loop.new
1167
1306
  end
1168
1307
 
1169
1308
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1170
- EX_PATHS.each do |path|
1171
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
1309
+ ex_paths.each do |target_info|
1310
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1172
1311
  end
1173
1312
 
1174
1313
  plugin.refresh_watchers
1175
1314
  end
1176
1315
 
1177
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
1316
+ path = 'test/plugin/data/2010/01/20100102-030405.log'
1317
+ target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
1318
+ mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
1178
1319
 
1179
1320
  Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1180
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
1321
+ path = "test/plugin/data/2010/01/20100102-030406.log"
1322
+ inode = Fluent::FileWrapper.stat(path).ino
1323
+ target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1324
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1181
1325
  plugin.refresh_watchers
1182
1326
 
1183
1327
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1323,6 +1467,305 @@ class TailInputTest < Test::Unit::TestCase
1323
1467
  end
1324
1468
  end
1325
1469
 
1470
+ sub_test_case 'inode_processing' do
1471
+ def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
1472
+ config = COMMON_FOLLOW_INODE_CONFIG
1473
+
1474
+ path = "#{TMP_DIR}/tail.txt"
1475
+ ino = 1
1476
+ pos = 1234
1477
+ File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
1478
+ f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1479
+ }
1480
+
1481
+ d = create_driver(config, false)
1482
+ d.run
1483
+
1484
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1485
+ pos_file.pos = 0
1486
+
1487
+ assert_raise(EOFError) do
1488
+ pos_file.readline
1489
+ end
1490
+ end
1491
+
1492
+ def test_should_write_latest_offset_after_rotate_wait
1493
+ config = COMMON_FOLLOW_INODE_CONFIG
1494
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1495
+ f.puts "test1"
1496
+ f.puts "test2"
1497
+ }
1498
+
1499
+ d = create_driver(config, false)
1500
+ d.run(expect_emits: 2, shutdown: false) do
1501
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1502
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
1503
+ sleep 1
1504
+ File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
1505
+ end
1506
+
1507
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1508
+ pos_file.pos = 0
1509
+ line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1510
+ waiting(5) {
1511
+ while line_parts[2].to_i(16) != 24
1512
+ sleep(0.1)
1513
+ pos_file.pos = 0
1514
+ line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1515
+ end
1516
+ }
1517
+ assert_equal(24, line_parts[2].to_i(16))
1518
+ d.instance_shutdown
1519
+ end
1520
+
1521
+ def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1522
+ config = config_element("", "", {"format" => "none"})
1523
+
1524
+ path = "#{TMP_DIR}/tail.txt"
1525
+ ino = 1
1526
+ pos = 1234
1527
+ File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
1528
+ f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1529
+ }
1530
+
1531
+ d = create_driver(config)
1532
+ d.run(shutdown: false)
1533
+
1534
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1535
+ pos_file.pos = 0
1536
+
1537
+ path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1538
+ assert_equal(path, path_pos_ino[1])
1539
+ assert_equal(pos, path_pos_ino[2].to_i(16))
1540
+ assert_equal(ino, path_pos_ino[3].to_i(16))
1541
+
1542
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1543
+ f.puts "test1"
1544
+ f.puts "test2"
1545
+ }
1546
+ Timecop.travel(Time.now + 10) do
1547
+ sleep 5
1548
+ pos_file.pos = 0
1549
+ tuple = create_target_info("#{TMP_DIR}/tail.txt")
1550
+ path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1551
+ assert_equal(tuple.path, path_pos_ino[1])
1552
+ assert_equal(12, path_pos_ino[2].to_i(16))
1553
+ assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1554
+ end
1555
+ d.instance_shutdown
1556
+ end
1557
+
1558
+ def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
1559
+ config = config_element("ROOT", "", {
1560
+ "path" => "#{TMP_DIR}/tail.txt*",
1561
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1562
+ "tag" => "t1",
1563
+ "rotate_wait" => "1s",
1564
+ "refresh_interval" => "1s",
1565
+ "limit_recently_modified" => "1s",
1566
+ "read_from_head" => "true",
1567
+ "format" => "none",
1568
+ "follow_inodes" => "true",
1569
+ })
1570
+
1571
+ d = create_driver(config, false)
1572
+
1573
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1574
+ f.puts "test1"
1575
+ f.puts "test2"
1576
+ }
1577
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1578
+
1579
+ d.run(expect_emits: 1, shutdown: false) do
1580
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1581
+ end
1582
+
1583
+
1584
+ Timecop.travel(Time.now + 10) do
1585
+ waiting(5) {
1586
+ # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
1587
+ sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
1588
+ }
1589
+ end
1590
+
1591
+ assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
1592
+
1593
+ d.instance_shutdown
1594
+ end
1595
+
1596
+ def test_should_read_from_head_on_file_renaming_with_star_in_pattern
1597
+ config = config_element("ROOT", "", {
1598
+ "path" => "#{TMP_DIR}/tail.txt*",
1599
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1600
+ "tag" => "t1",
1601
+ "rotate_wait" => "10s",
1602
+ "refresh_interval" => "1s",
1603
+ "limit_recently_modified" => "60s",
1604
+ "read_from_head" => "true",
1605
+ "format" => "none",
1606
+ "follow_inodes" => "true"
1607
+ })
1608
+
1609
+ d = create_driver(config, false)
1610
+
1611
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1612
+ f.puts "test1"
1613
+ f.puts "test2"
1614
+ }
1615
+
1616
+ d.run(expect_emits: 2, shutdown: false) do
1617
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1618
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
1619
+ end
1620
+
1621
+ events = d.events
1622
+ assert_equal(3, events.length)
1623
+ d.instance_shutdown
1624
+ end
1625
+
1626
+ def test_should_not_read_from_head_on_rotation_when_watching_inodes
1627
+ config = COMMON_FOLLOW_INODE_CONFIG
1628
+
1629
+ d = create_driver(config, false)
1630
+
1631
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1632
+ f.puts "test1"
1633
+ f.puts "test2"
1634
+ }
1635
+
1636
+ d.run(expect_emits: 1, shutdown: false) do
1637
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1638
+ end
1639
+
1640
+ FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
1641
+ Timecop.travel(Time.now + 10) do
1642
+ sleep 2
1643
+ events = d.events
1644
+ assert_equal(3, events.length)
1645
+ end
1646
+
1647
+ d.instance_shutdown
1648
+ end
1649
+
1650
+ def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
1651
+ config = COMMON_FOLLOW_INODE_CONFIG
1652
+
1653
+ d = create_driver(config, false)
1654
+
1655
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1656
+ f.puts "test1"
1657
+ f.puts "test2"
1658
+ }
1659
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1660
+
1661
+ d.run(expect_emits: 2, shutdown: false) do
1662
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1663
+ cleanup_file("#{TMP_DIR}/tail.txt")
1664
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
1665
+ end
1666
+
1667
+ new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
1668
+
1669
+ pos_file = d.instance.instance_variable_get(:@pf)
1670
+
1671
+ waiting(10) {
1672
+ # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
1673
+ sleep 0.1 until pos_file[target_info].read_pos == 0
1674
+ }
1675
+ new_position = pos_file[new_target_info].read_pos
1676
+ assert_equal(6, new_position)
1677
+
1678
+ d.instance_shutdown
1679
+ end
1680
+
1681
+ def test_should_close_watcher_after_rotate_wait
1682
+ now = Time.now
1683
+ config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1684
+
1685
+ d = create_driver(config, false)
1686
+
1687
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1688
+ f.puts "test1"
1689
+ f.puts "test2"
1690
+ }
1691
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1692
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
1693
+ d.run(shutdown: false)
1694
+ assert d.instance.instance_variable_get(:@tails)[target_info]
1695
+
1696
+ Timecop.travel(now + 10) do
1697
+ d.instance.instance_eval do
1698
+ sleep 0.1 until @tails[target_info] == nil
1699
+ end
1700
+ assert_nil d.instance.instance_variable_get(:@tails)[target_info]
1701
+ end
1702
+ d.instance_shutdown
1703
+ end
1704
+
1705
+ def test_should_create_new_watcher_for_new_file_with_same_name
1706
+ now = Time.now
1707
+ config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
1708
+
1709
+ d = create_driver(config, false)
1710
+
1711
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1712
+ f.puts "test1"
1713
+ f.puts "test2"
1714
+ }
1715
+ path_ino = create_target_info("#{TMP_DIR}/tail.txt")
1716
+
1717
+ d.run(expect_emits: 1, shutdown: false) do
1718
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1719
+ end
1720
+
1721
+ cleanup_file("#{TMP_DIR}/tail.txt")
1722
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1723
+ f.puts "test3"
1724
+ f.puts "test4"
1725
+ }
1726
+ new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
1727
+
1728
+ Timecop.travel(now + 10) do
1729
+ sleep 3
1730
+ d.instance.instance_eval do
1731
+ @tails[path_ino] == nil
1732
+ @tails[new_path_ino] != nil
1733
+ end
1734
+ end
1735
+
1736
+ events = d.events
1737
+
1738
+ assert_equal(5, events.length)
1739
+
1740
+ d.instance_shutdown
1741
+ end
1742
+
1743
+ def test_truncate_file_with_follow_inodes
1744
+ config = COMMON_FOLLOW_INODE_CONFIG
1745
+
1746
+ d = create_driver(config, false)
1747
+
1748
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1749
+ f.puts "test1"
1750
+ f.puts "test2"
1751
+ }
1752
+
1753
+ d.run(expect_emits: 3, shutdown: false) do
1754
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1755
+ sleep 2
1756
+ File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
1757
+ end
1758
+
1759
+ events = d.events
1760
+ assert_equal(4, events.length)
1761
+ assert_equal({"message" => "test1"}, events[0][2])
1762
+ assert_equal({"message" => "test2"}, events[1][2])
1763
+ assert_equal({"message" => "test3"}, events[2][2])
1764
+ assert_equal({"message" => "test4"}, events[3][2])
1765
+ d.instance_shutdown
1766
+ end
1767
+ end
1768
+
1326
1769
  sub_test_case "tail_path" do
1327
1770
  def test_tail_path_with_singleline
1328
1771
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
@@ -1452,13 +1895,13 @@ class TailInputTest < Test::Unit::TestCase
1452
1895
  })
1453
1896
 
1454
1897
  expected_files = [
1455
- "#{TMP_DIR}/tail_watch1.txt",
1456
- "#{TMP_DIR}/tail_watch2.txt"
1898
+ create_target_info("#{TMP_DIR}/tail_watch1.txt"),
1899
+ create_target_info("#{TMP_DIR}/tail_watch2.txt")
1457
1900
  ]
1458
1901
 
1459
1902
  Timecop.freeze(now) do
1460
1903
  plugin = create_driver(config, false).instance
1461
- assert_equal expected_files, plugin.expand_paths.sort
1904
+ assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1462
1905
  end
1463
1906
  end
1464
1907
 
@@ -1476,4 +1919,22 @@ class TailInputTest < Test::Unit::TestCase
1476
1919
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
1477
1920
  d.instance_shutdown
1478
1921
  end
1922
+
1923
+ def test_ENOENT_error_after_setup_watcher
1924
+ path = "#{TMP_DIR}/tail.txt"
1925
+ FileUtils.touch(path)
1926
+ config = config_element('', '', {
1927
+ 'format' => 'none',
1928
+ })
1929
+ d = create_driver(config)
1930
+ mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
1931
+ cleanup_file(path)
1932
+ tw
1933
+ end
1934
+ assert_nothing_raised do
1935
+ d.run(shutdown: false) {}
1936
+ end
1937
+ d.instance_shutdown
1938
+ assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with ENOENT. Drop tail watcher for now.\n") })
1939
+ end
1479
1940
  end