fluentd 1.14.5-x86-mingw32 → 1.15.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +105 -21
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +1 -1
  10. data/fluentd.gemspec +2 -1
  11. data/lib/fluent/command/ctl.rb +4 -1
  12. data/lib/fluent/command/fluentd.rb +14 -0
  13. data/lib/fluent/config/literal_parser.rb +2 -2
  14. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  15. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  16. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  17. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  18. data/lib/fluent/config/yaml_parser.rb +56 -0
  19. data/lib/fluent/config.rb +14 -1
  20. data/lib/fluent/error.rb +3 -0
  21. data/lib/fluent/event_router.rb +19 -1
  22. data/lib/fluent/plugin/bare_output.rb +1 -1
  23. data/lib/fluent/plugin/base.rb +19 -0
  24. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  25. data/lib/fluent/plugin/in_forward.rb +1 -1
  26. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  27. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  28. data/lib/fluent/plugin/in_tail.rb +68 -48
  29. data/lib/fluent/plugin/out_file.rb +11 -1
  30. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  31. data/lib/fluent/plugin/output.rb +43 -37
  32. data/lib/fluent/plugin/parser.rb +3 -4
  33. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  34. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  35. data/lib/fluent/plugin_helper/server.rb +23 -4
  36. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  37. data/lib/fluent/rpc.rb +4 -3
  38. data/lib/fluent/supervisor.rb +119 -28
  39. data/lib/fluent/system_config.rb +2 -1
  40. data/lib/fluent/version.rb +1 -1
  41. data/lib/fluent/winsvc.rb +2 -0
  42. data/test/command/test_ctl.rb +0 -1
  43. data/test/command/test_fluentd.rb +33 -0
  44. data/test/config/test_system_config.rb +3 -1
  45. data/test/config/test_types.rb +1 -1
  46. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  47. data/test/plugin/in_tail/test_position_file.rb +0 -63
  48. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  49. data/test/plugin/test_base.rb +34 -0
  50. data/test/plugin/test_file_wrapper.rb +0 -68
  51. data/test/plugin/test_in_forward.rb +0 -2
  52. data/test/plugin/test_in_object_space.rb +9 -3
  53. data/test/plugin/test_in_syslog.rb +1 -1
  54. data/test/plugin/test_in_tail.rb +629 -353
  55. data/test/plugin/test_out_forward.rb +30 -20
  56. data/test/plugin/test_output_as_buffered_retries.rb +7 -7
  57. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  58. data/test/plugin/test_parser_syslog.rb +1 -1
  59. data/test/plugin_helper/test_cert_option.rb +1 -1
  60. data/test/plugin_helper/test_child_process.rb +16 -4
  61. data/test/plugin_helper/test_retry_state.rb +602 -38
  62. data/test/plugin_helper/test_server.rb +18 -0
  63. data/test/test_config.rb +135 -4
  64. data/test/test_event_router.rb +17 -0
  65. data/test/test_supervisor.rb +196 -6
  66. metadata +25 -5
@@ -12,15 +12,21 @@ require 'securerandom'
12
12
  class TailInputTest < Test::Unit::TestCase
13
13
  include FlexMock::TestCase
14
14
 
15
+ def tmp_dir
16
+ File.join(File.dirname(__FILE__), "..", "tmp", "tail#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
17
+ end
18
+
15
19
  def setup
16
20
  Fluent::Test.setup
17
- cleanup_directory(TMP_DIR)
21
+ @tmp_dir = tmp_dir
22
+ cleanup_directory(@tmp_dir)
18
23
  end
19
24
 
20
25
  def teardown
21
26
  super
22
- cleanup_directory(TMP_DIR)
27
+ cleanup_directory(@tmp_dir)
23
28
  Fluent::Engine.stop
29
+ Timecop.return
24
30
  end
25
31
 
26
32
  def cleanup_directory(path)
@@ -29,89 +35,48 @@ class TailInputTest < Test::Unit::TestCase
29
35
  return
30
36
  end
31
37
 
32
- if Fluent.windows?
33
- Dir.glob("*", base: path).each do |name|
34
- begin
35
- cleanup_file(File.join(path, name))
36
- rescue
37
- # expect test driver block release already owned file handle.
38
- end
39
- end
40
- else
41
- begin
42
- FileUtils.rm_f(path, secure:true)
43
- rescue ArgumentError
44
- FileUtils.rm_f(path) # For Ruby 2.6 or before.
45
- end
46
- if File.exist?(path)
47
- FileUtils.remove_entry_secure(path, true)
48
- end
49
- end
50
- FileUtils.mkdir_p(path)
38
+ FileUtils.remove_entry_secure(path, true)
51
39
  end
52
40
 
53
41
  def cleanup_file(path)
54
- if Fluent.windows?
55
- # On Windows, when the file or directory is removed and created
56
- # frequently, there is a case that creating file or directory will
57
- # fail. This situation is caused by pending file or directory
58
- # deletion which is mentioned on win32 API document [1]
59
- # As a workaround, execute rename and remove method.
60
- #
61
- # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
62
- #
63
- file = File.join(Dir.tmpdir, SecureRandom.hex(10))
64
- begin
65
- FileUtils.mv(path, file)
66
- FileUtils.rm_rf(file, secure: true)
67
- rescue ArgumentError
68
- FileUtils.rm_rf(file) # For Ruby 2.6 or before.
69
- end
70
- if File.exist?(file)
71
- # ensure files are closed for Windows, on which deleted files
72
- # are still visible from filesystem
73
- GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
74
- FileUtils.remove_entry_secure(file, true)
75
- end
76
- else
77
- begin
78
- FileUtils.rm_f(path, secure: true)
79
- rescue ArgumentError
80
- FileUtils.rm_f(path) # For Ruby 2.6 or before.
81
- end
82
- if File.exist?(path)
83
- FileUtils.remove_entry_secure(path, true)
84
- end
85
- end
42
+ FileUtils.remove_entry_secure(path, true)
86
43
  end
87
44
 
88
45
  def create_target_info(path)
89
46
  Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
90
47
  end
91
48
 
92
- TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
93
-
94
- CONFIG = config_element("ROOT", "", {
95
- "path" => "#{TMP_DIR}/tail.txt",
49
+ ROOT_CONFIG = config_element("ROOT", "", {
96
50
  "tag" => "t1",
97
51
  "rotate_wait" => "2s",
98
52
  "refresh_interval" => "1s"
99
- })
100
- COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
53
+ })
54
+
55
+ def base_config
56
+ ROOT_CONFIG + config_element("", "", { "path" => "#{@tmp_dir}/tail.txt" })
57
+ end
58
+
59
+ def common_config
60
+ base_config + config_element("", "", { "pos_file" => "#{@tmp_dir}/tail.pos" })
61
+ end
62
+
63
+ def common_follow_inode_config
64
+ config_element("ROOT", "", {
65
+ "path" => "#{@tmp_dir}/tail.txt*",
66
+ "pos_file" => "#{@tmp_dir}/tail.pos",
67
+ "tag" => "t1",
68
+ "refresh_interval" => "1s",
69
+ "read_from_head" => "true",
70
+ "format" => "none",
71
+ "rotate_wait" => "1s",
72
+ "follow_inodes" => "true"
73
+ })
74
+ end
75
+
101
76
  CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
102
77
  CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
103
78
  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
104
79
  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
105
- COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
106
- "path" => "#{TMP_DIR}/tail.txt*",
107
- "pos_file" => "#{TMP_DIR}/tail.pos",
108
- "tag" => "t1",
109
- "refresh_interval" => "1s",
110
- "read_from_head" => "true",
111
- "format" => "none",
112
- "rotate_wait" => "1s",
113
- "follow_inodes" => "true"
114
- })
115
80
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
116
81
  PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
117
82
  MULTILINE_CONFIG = config_element(
@@ -144,21 +109,67 @@ class TailInputTest < Test::Unit::TestCase
144
109
  })
145
110
  ])
146
111
 
112
+ EX_ROTATE_WAIT = 0
113
+ EX_FOLLOW_INODES = false
114
+
115
+ def ex_config
116
+ config_element("", "", {
117
+ "tag" => "tail",
118
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
119
+ "format" => "none",
120
+ "pos_file" => "#{@tmp_dir}/tail.pos",
121
+ "read_from_head" => true,
122
+ "refresh_interval" => 30,
123
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
124
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
125
+ })
126
+ end
127
+
128
+ def tailing_group_pattern
129
+ "/#{@tmp_dir}\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{6})\.log$/"
130
+ end
131
+
132
+ DEBUG_LOG_LEVEL = config_element("", "", {
133
+ "@log_level" => "debug"
134
+ })
135
+
136
+ def create_group_directive(pattern, rate_period, *rules)
137
+ config_element("", "", {}, [
138
+ config_element("group", "", {
139
+ "pattern" => pattern,
140
+ "rate_period" => rate_period
141
+ }, rules)
142
+ ])
143
+ end
144
+
145
+ def create_rule_directive(match_named_captures, limit)
146
+ params = {
147
+ "limit" => limit,
148
+ "match" => match_named_captures,
149
+ }
150
+
151
+ config_element("rule", "", params)
152
+ end
153
+
154
+ def create_path_element(path)
155
+ config_element("source", "", { "path" => "#{@tmp_dir}/#{path}" })
156
+ end
157
+
147
158
  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
148
- config = use_common_conf ? COMMON_CONFIG + conf : conf
159
+ config = use_common_conf ? common_config + conf : conf
149
160
  Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
150
161
  end
151
162
 
152
163
  sub_test_case "configure" do
153
164
  test "plain single line" do
154
165
  d = create_driver
155
- assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
156
- assert_equal "t1", d.instance.tag
157
- assert_equal 2, d.instance.rotate_wait
158
- assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
159
- assert_equal 1000, d.instance.read_lines_limit
160
- assert_equal -1, d.instance.read_bytes_limit_per_second
161
- assert_equal false, d.instance.ignore_repeated_permission_error
166
+ assert_equal(["#{@tmp_dir}/tail.txt"], d.instance.paths)
167
+ assert_equal("t1", d.instance.tag)
168
+ assert_equal(2, d.instance.rotate_wait)
169
+ assert_equal("#{@tmp_dir}/tail.pos", d.instance.pos_file)
170
+ assert_equal(1000, d.instance.read_lines_limit)
171
+ assert_equal(-1, d.instance.read_bytes_limit_per_second)
172
+ assert_equal(false, d.instance.ignore_repeated_permission_error)
162
173
  assert_nothing_raised do
163
174
  d.instance.have_read_capability?
164
175
  end
@@ -175,13 +186,13 @@ class TailInputTest < Test::Unit::TestCase
175
186
  test "multi paths with path_delimiter" do
176
187
  c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "|" })
177
188
  d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
178
- assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
189
+ assert_equal(["tail.txt", "test2", "tmp,dev"], d.instance.paths)
179
190
  end
180
191
 
181
192
  test "multi paths with same path configured twice" do
182
193
  c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
183
194
  d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
184
- assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
195
+ assert_equal(["test2.txt","test1.txt"].sort, d.instance.paths.sort)
185
196
  end
186
197
 
187
198
  test "multi paths with invaid path_delimiter" do
@@ -193,7 +204,7 @@ class TailInputTest < Test::Unit::TestCase
193
204
 
194
205
  test "follow_inodes w/o pos file" do
195
206
  assert_raise(Fluent::ConfigError) do
196
- create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
207
+ create_driver(base_config + config_element('', '', {'follow_inodes' => 'true'}))
197
208
  end
198
209
  end
199
210
 
@@ -223,7 +234,7 @@ class TailInputTest < Test::Unit::TestCase
223
234
  test "valid" do
224
235
  conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
225
236
  d = create_driver(conf)
226
- assert_equal Encoding::UTF_8, d.instance.encoding
237
+ assert_equal(Encoding::UTF_8, d.instance.encoding)
227
238
  end
228
239
 
229
240
  test "invalid" do
@@ -263,12 +274,171 @@ class TailInputTest < Test::Unit::TestCase
263
274
  end
264
275
  end
265
276
 
277
+ sub_test_case "configure group" do
278
+ test "<rule> required" do
279
+ conf = create_group_directive('.', '1m') + SINGLE_LINE_CONFIG
280
+ assert_raise(Fluent::ConfigError) do
281
+ create_driver(conf)
282
+ end
283
+ end
284
+
285
+ test "valid configuration" do
286
+ rule1 = create_rule_directive({
287
+ "namespace"=> "/namespace-a/",
288
+ "podname"=> "/podname-[b|c]/"
289
+ }, 100)
290
+ rule2 = create_rule_directive({
291
+ "namespace"=> "/namespace-[d|e]/",
292
+ "podname"=> "/podname-f/",
293
+ }, 50)
294
+ rule3 = create_rule_directive({
295
+ "podname"=> "/podname-g/",
296
+ }, -1)
297
+ rule4 = create_rule_directive({
298
+ "namespace"=> "/namespace-h/",
299
+ }, 0)
300
+
301
+ conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2, rule3, rule4) + SINGLE_LINE_CONFIG
302
+ assert_nothing_raised do
303
+ create_driver(conf)
304
+ end
305
+ end
306
+
307
+ test "limit should be greater than DEFAULT_LIMIT (-1)" do
308
+ rule1 = create_rule_directive({
309
+ "namespace"=> "/namespace-a/",
310
+ "podname"=> "/podname-[b|c]/",
311
+ }, -100)
312
+ rule2 = create_rule_directive({
313
+ "namespace"=> "/namespace-[d|e]/",
314
+ "podname"=> "/podname-f/",
315
+ }, 50)
316
+ conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2) + SINGLE_LINE_CONFIG
317
+ assert_raise(RuntimeError) do
318
+ create_driver(conf)
319
+ end
320
+ end
321
+ end
322
+
323
+ sub_test_case "group rules line limit resolution" do
324
+ test "valid" do
325
+ rule1 = create_rule_directive({
326
+ "namespace"=> "/namespace-a/",
327
+ "podname"=> "/podname-[b|c]/",
328
+ }, 50)
329
+ rule2 = create_rule_directive({
330
+ "podname"=> "/podname-[b|c]/",
331
+ }, 400)
332
+ rule3 = create_rule_directive({
333
+ "namespace"=> "/namespace-a/",
334
+ }, 100)
335
+
336
+ conf = create_group_directive(tailing_group_pattern, '1m', rule3, rule1, rule2) + SINGLE_LINE_CONFIG
337
+ assert_nothing_raised do
338
+ d = create_driver(conf)
339
+ instance = d.instance
340
+
341
+ metadata = {
342
+ "namespace"=> "namespace-a",
343
+ "podname"=> "podname-b",
344
+ }
345
+ assert_equal(50, instance.find_group(metadata).limit)
346
+
347
+ metadata = {
348
+ "namespace" => "namespace-a",
349
+ "podname" => "podname-c",
350
+ }
351
+ assert_equal(50, instance.find_group(metadata).limit)
352
+
353
+ metadata = {
354
+ "namespace" => "namespace-a",
355
+ "podname" => "podname-d",
356
+ }
357
+ assert_equal(100, instance.find_group(metadata).limit)
358
+
359
+ metadata = {
360
+ "namespace" => "namespace-f",
361
+ "podname" => "podname-b",
362
+ }
363
+ assert_equal(400, instance.find_group(metadata).limit)
364
+
365
+ metadata = {
366
+ "podname" => "podname-c",
367
+ }
368
+ assert_equal(400, instance.find_group(metadata).limit)
369
+
370
+ assert_equal(-1, instance.find_group({}).limit)
371
+ end
372
+ end
373
+ end
374
+
375
+ sub_test_case "files should be placed in groups" do
376
+ test "invalid regex pattern places files in default group" do
377
+ rule1 = create_rule_directive({}, 100) ## limits default groups
378
+ conf = ROOT_CONFIG + DEBUG_LOG_LEVEL + create_group_directive(tailing_group_pattern, '1m', rule1) + create_path_element("test*.txt") + SINGLE_LINE_CONFIG
379
+
380
+ d = create_driver(conf, false)
381
+ File.open("#{@tmp_dir}/test1.txt", 'w')
382
+ File.open("#{@tmp_dir}/test2.txt", 'w')
383
+ File.open("#{@tmp_dir}/test3.txt", 'w')
384
+
385
+ d.run do
386
+ ## checking default group_watcher's paths
387
+ instance = d.instance
388
+ key = instance.default_group_key
389
+
390
+ assert_equal(3, instance.log.logs.count{|a| a.match?("Cannot find group from metadata, Adding file in the default group\n")})
391
+ assert_equal(3, instance.group_watchers[key].size)
392
+ assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test1.txt'))
393
+ assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test2.txt'))
394
+ assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test3.txt'))
395
+ end
396
+ end
397
+
398
+ test "valid regex pattern places file in their respective groups" do
399
+ rule1 = create_rule_directive({
400
+ "namespace"=> "/test-namespace1/",
401
+ "podname"=> "/test-podname1/",
402
+ }, 100)
403
+ rule2 = create_rule_directive({
404
+ "namespace"=> "/test-namespace1/",
405
+ }, 200)
406
+ rule3 = create_rule_directive({
407
+ "podname"=> "/test-podname2/",
408
+ }, 300)
409
+ rule4 = create_rule_directive({}, 400)
410
+
411
+ path_element = create_path_element("test-podname*.log")
412
+
413
+ conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, '1m', rule4, rule3, rule2, rule1) + path_element + SINGLE_LINE_CONFIG
414
+ d = create_driver(conf, false)
415
+
416
+ file1 = File.join(@tmp_dir, "test-podname1_test-namespace1_test-container-15fabq.log")
417
+ file2 = File.join(@tmp_dir, "test-podname3_test-namespace1_test-container-15fabq.log")
418
+ file3 = File.join(@tmp_dir, "test-podname2_test-namespace2_test-container-15fabq.log")
419
+ file4 = File.join(@tmp_dir, "test-podname4_test-namespace3_test-container-15fabq.log")
420
+
421
+ d.run do
422
+ File.open(file1, 'w')
423
+ File.open(file2, 'w')
424
+ File.open(file3, 'w')
425
+ File.open(file4, 'w')
426
+
427
+ instance = d.instance
428
+ assert_equal(100, instance.find_group_from_metadata(file1).limit)
429
+ assert_equal(200, instance.find_group_from_metadata(file2).limit)
430
+ assert_equal(300, instance.find_group_from_metadata(file3).limit)
431
+ assert_equal(400, instance.find_group_from_metadata(file4).limit)
432
+ end
433
+ end
434
+ end
435
+
266
436
  sub_test_case "singleline" do
267
437
  data(flat: SINGLE_LINE_CONFIG,
268
438
  parse: PARSE_SINGLE_LINE_CONFIG)
269
439
  def test_emit(data)
270
440
  config = data
271
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
441
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
272
442
  f.puts "test1"
273
443
  f.puts "test2"
274
444
  }
@@ -276,7 +446,7 @@ class TailInputTest < Test::Unit::TestCase
276
446
  d = create_driver(config)
277
447
 
278
448
  d.run(expect_emits: 1) do
279
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
449
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
280
450
  f.puts "test3\ntest4"
281
451
  }
282
452
  end
@@ -292,11 +462,11 @@ class TailInputTest < Test::Unit::TestCase
292
462
 
293
463
  def test_emit_with_emit_unmatched_lines_true
294
464
  config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
295
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
465
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
296
466
 
297
467
  d = create_driver(config)
298
468
  d.run(expect_emits: 1) do
299
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
469
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
300
470
  f.puts "test line 1"
301
471
  f.puts "test line 2"
302
472
  f.puts "bad line 1"
@@ -328,7 +498,7 @@ class TailInputTest < Test::Unit::TestCase
328
498
  msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
329
499
 
330
500
  d.run(expect_emits: num_events, timeout: 2) do
331
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
501
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
332
502
  f.puts msg
333
503
  f.puts msg
334
504
  }
@@ -343,7 +513,7 @@ class TailInputTest < Test::Unit::TestCase
343
513
 
344
514
  sub_test_case "log throttling per file" do
345
515
  teardown do
346
- cleanup_file("#{TMP_DIR}/tail.txt")
516
+ cleanup_file("#{@tmp_dir}/tail.txt")
347
517
  end
348
518
 
349
519
  sub_test_case "reads_bytes_per_second w/o throttled" do
@@ -374,7 +544,7 @@ class TailInputTest < Test::Unit::TestCase
374
544
 
375
545
  d = create_driver(config)
376
546
  d.run(expect_emits: 2) do
377
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
547
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
378
548
  100.times do
379
549
  f.puts msg
380
550
  end
@@ -397,7 +567,7 @@ class TailInputTest < Test::Unit::TestCase
397
567
  start_time = Fluent::Clock.now
398
568
  d = create_driver(config)
399
569
  d.run(expect_emits: 2) do
400
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
570
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
401
571
  8000.times do
402
572
  f.puts msg
403
573
  end
@@ -436,7 +606,7 @@ class TailInputTest < Test::Unit::TestCase
436
606
  io_handler
437
607
  end
438
608
 
439
- File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
609
+ File.open("#{@tmp_dir}/tail.txt", "ab") do |f|
440
610
  100.times do
441
611
  f.puts msg
442
612
  end
@@ -446,7 +616,7 @@ class TailInputTest < Test::Unit::TestCase
446
616
  d.run do
447
617
  start_time = Fluent::Clock.now
448
618
  while Fluent::Clock.now - start_time < 0.8 do
449
- File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
619
+ File.open("#{@tmp_dir}/tail.txt", "ab") do |f|
450
620
  f.puts msg
451
621
  f.flush
452
622
  end
@@ -464,7 +634,7 @@ class TailInputTest < Test::Unit::TestCase
464
634
  num_lines = 1024 * 3
465
635
  msg = "08bytes"
466
636
 
467
- File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
637
+ File.open("#{@tmp_dir}/tail.txt", "wb") do |f|
468
638
  f.write("#{msg}\n" * num_lines)
469
639
  end
470
640
 
@@ -481,8 +651,8 @@ class TailInputTest < Test::Unit::TestCase
481
651
  d.run(timeout: 10) do
482
652
  while d.events.size < num_lines do
483
653
  if d.events.size > 0 && !rotated
484
- cleanup_file("#{TMP_DIR}/tail.txt")
485
- FileUtils.touch("#{TMP_DIR}/tail.txt")
654
+ cleanup_file("#{@tmp_dir}/tail.txt")
655
+ FileUtils.touch("#{@tmp_dir}/tail.txt")
486
656
  rotated = true
487
657
  end
488
658
  sleep 0.3
@@ -500,7 +670,7 @@ class TailInputTest < Test::Unit::TestCase
500
670
  num_lines = 1024 * 2
501
671
  msg = "08bytes"
502
672
 
503
- File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
673
+ File.open("#{@tmp_dir}/tail.txt", "wb") do |f|
504
674
  f.write("#{msg}\n" * num_lines)
505
675
  end
506
676
 
@@ -527,8 +697,8 @@ class TailInputTest < Test::Unit::TestCase
527
697
  d.run(timeout: 10) do
528
698
  until detached do
529
699
  if d.events.size > 0 && !rotated
530
- cleanup_file("#{TMP_DIR}/tail.txt")
531
- FileUtils.touch("#{TMP_DIR}/tail.txt")
700
+ cleanup_file("#{@tmp_dir}/tail.txt")
701
+ FileUtils.touch("#{@tmp_dir}/tail.txt")
532
702
  rotated = true
533
703
  end
534
704
  sleep 0.3
@@ -548,7 +718,7 @@ class TailInputTest < Test::Unit::TestCase
548
718
  parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
549
719
  def test_emit_with_read_from_head(data)
550
720
  config = data
551
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
721
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
552
722
  f.puts "test1"
553
723
  f.puts "test2"
554
724
  }
@@ -556,7 +726,7 @@ class TailInputTest < Test::Unit::TestCase
556
726
  d = create_driver(config)
557
727
 
558
728
  d.run(expect_emits: 2) do
559
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
729
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
560
730
  f.puts "test3"
561
731
  f.puts "test4"
562
732
  }
@@ -574,7 +744,7 @@ class TailInputTest < Test::Unit::TestCase
574
744
  parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
575
745
  def test_emit_without_watch_timer(data)
576
746
  config = data
577
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
747
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
578
748
  f.puts "test1"
579
749
  f.puts "test2"
580
750
  }
@@ -582,7 +752,7 @@ class TailInputTest < Test::Unit::TestCase
582
752
  d = create_driver(config)
583
753
 
584
754
  d.run(expect_emits: 1) do
585
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
755
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
586
756
  f.puts "test3"
587
757
  f.puts "test4"
588
758
  }
@@ -601,12 +771,12 @@ class TailInputTest < Test::Unit::TestCase
601
771
  omit "need inotify" unless Fluent.linux?
602
772
 
603
773
  config = config_element("ROOT", "", {
604
- "path" => "#{TMP_DIR}/tail*.txt",
774
+ "path" => "#{@tmp_dir}/tail*.txt",
605
775
  "tag" => "t1",
606
776
  })
607
777
  config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
608
778
 
609
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
779
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
610
780
  f.puts "test1"
611
781
  f.puts "test2"
612
782
  }
@@ -614,7 +784,7 @@ class TailInputTest < Test::Unit::TestCase
614
784
  d = create_driver(config, false)
615
785
 
616
786
  d.run(expect_emits: 1, timeout: 1) do
617
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
787
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
618
788
  f.puts "test3"
619
789
  f.puts "test4"
620
790
  }
@@ -632,7 +802,7 @@ class TailInputTest < Test::Unit::TestCase
632
802
  parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
633
803
  def test_emit_with_disable_stat_watcher(data)
634
804
  config = data
635
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
805
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
636
806
  f.puts "test1"
637
807
  f.puts "test2"
638
808
  }
@@ -640,7 +810,7 @@ class TailInputTest < Test::Unit::TestCase
640
810
  d = create_driver(config)
641
811
 
642
812
  d.run(expect_emits: 1) do
643
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
813
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
644
814
  f.puts "test3"
645
815
  f.puts "test4"
646
816
  }
@@ -656,7 +826,7 @@ class TailInputTest < Test::Unit::TestCase
656
826
  d = create_driver(SINGLE_LINE_CONFIG)
657
827
 
658
828
  d.run(expect_emits: 1, timeout: 3) do
659
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
829
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
660
830
  f.puts "test1\ntest2\n"
661
831
  }
662
832
  end
@@ -682,11 +852,14 @@ class TailInputTest < Test::Unit::TestCase
682
852
 
683
853
  def setup
684
854
  omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
855
+ super
685
856
  # Store default permission
686
857
  @default_permission = system_config.instance_variable_get(:@file_permission)
687
858
  end
688
859
 
689
860
  def teardown
861
+ return if Fluent.windows?
862
+ super
690
863
  # Restore default permission
691
864
  system_config.instance_variable_set(:@file_permission, @default_permission)
692
865
  end
@@ -700,7 +873,7 @@ class TailInputTest < Test::Unit::TestCase
700
873
  system_conf = parse_system(CONFIG_SYSTEM)
701
874
  sc = Fluent::SystemConfig.new(system_conf)
702
875
  Fluent::Engine.init(sc)
703
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
876
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
704
877
  f.puts "test1"
705
878
  f.puts "test2"
706
879
  }
@@ -708,7 +881,7 @@ class TailInputTest < Test::Unit::TestCase
708
881
  d = create_driver
709
882
 
710
883
  d.run(expect_emits: 1) do
711
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
884
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
712
885
  f.puts "test3"
713
886
  f.puts "test4"
714
887
  }
@@ -728,16 +901,18 @@ class TailInputTest < Test::Unit::TestCase
728
901
  end
729
902
 
730
903
  sub_test_case "rotate file" do
904
+ def create_driver(conf = SINGLE_LINE_CONFIG)
905
+ config = common_config + conf
906
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
907
+ end
908
+
731
909
  data(flat: SINGLE_LINE_CONFIG,
732
910
  parse: PARSE_SINGLE_LINE_CONFIG)
733
911
  def test_rotate_file(data)
734
912
  config = data
735
913
  events = sub_test_rotate_file(config, expect_emits: 2)
736
- assert_equal(4, events.length)
737
- assert_equal({"message" => "test3"}, events[0][2])
738
- assert_equal({"message" => "test4"}, events[1][2])
739
- assert_equal({"message" => "test5"}, events[2][2])
740
- assert_equal({"message" => "test6"}, events[3][2])
914
+ assert_equal(3.upto(6).collect { |i| {"message" => "test#{i}"} },
915
+ events.collect { |event| event[2] })
741
916
  end
742
917
 
743
918
  data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -745,13 +920,8 @@ class TailInputTest < Test::Unit::TestCase
745
920
  def test_rotate_file_with_read_from_head(data)
746
921
  config = data
747
922
  events = sub_test_rotate_file(config, expect_records: 6)
748
- assert_equal(6, events.length)
749
- assert_equal({"message" => "test1"}, events[0][2])
750
- assert_equal({"message" => "test2"}, events[1][2])
751
- assert_equal({"message" => "test3"}, events[2][2])
752
- assert_equal({"message" => "test4"}, events[3][2])
753
- assert_equal({"message" => "test5"}, events[4][2])
754
- assert_equal({"message" => "test6"}, events[5][2])
923
+ assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
924
+ events.collect { |event| event[2] })
755
925
  end
756
926
 
757
927
  data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -759,13 +929,8 @@ class TailInputTest < Test::Unit::TestCase
759
929
  def test_rotate_file_with_open_on_every_update(data)
760
930
  config = data
761
931
  events = sub_test_rotate_file(config, expect_records: 6)
762
- assert_equal(6, events.length)
763
- assert_equal({"message" => "test1"}, events[0][2])
764
- assert_equal({"message" => "test2"}, events[1][2])
765
- assert_equal({"message" => "test3"}, events[2][2])
766
- assert_equal({"message" => "test4"}, events[3][2])
767
- assert_equal({"message" => "test5"}, events[4][2])
768
- assert_equal({"message" => "test6"}, events[5][2])
932
+ assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
933
+ events.collect { |event| event[2] })
769
934
  end
770
935
 
771
936
  data(flat: SINGLE_LINE_CONFIG,
@@ -773,26 +938,21 @@ class TailInputTest < Test::Unit::TestCase
773
938
  def test_rotate_file_with_write_old(data)
774
939
  config = data
775
940
  events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
776
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
941
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
777
942
  rotated_file.puts "test7"
778
943
  rotated_file.puts "test8"
779
944
  rotated_file.flush
780
945
 
781
946
  sleep 1
782
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
947
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
783
948
  f.puts "test5"
784
949
  f.puts "test6"
785
950
  }
786
951
  }
787
952
  # This test sometimes fails and it shows a potential bug of in_tail
788
953
  # https://github.com/fluent/fluentd/issues/1434
789
- assert_equal(6, events.length)
790
- assert_equal({"message" => "test3"}, events[0][2])
791
- assert_equal({"message" => "test4"}, events[1][2])
792
- assert_equal({"message" => "test7"}, events[2][2])
793
- assert_equal({"message" => "test8"}, events[3][2])
794
- assert_equal({"message" => "test5"}, events[4][2])
795
- assert_equal({"message" => "test6"}, events[5][2])
954
+ assert_equal([3, 4, 7, 8, 5, 6].collect { |i| {"message" => "test#{i}"} },
955
+ events.collect { |event| event[2] })
796
956
  end
797
957
 
798
958
  data(flat: SINGLE_LINE_CONFIG,
@@ -804,21 +964,19 @@ class TailInputTest < Test::Unit::TestCase
804
964
  rotated_file.puts "test8"
805
965
  rotated_file.flush
806
966
  }
807
- assert_equal(4, events.length)
808
- assert_equal({"message" => "test3"}, events[0][2])
809
- assert_equal({"message" => "test4"}, events[1][2])
810
- assert_equal({"message" => "test7"}, events[2][2])
811
- assert_equal({"message" => "test8"}, events[3][2])
967
+ assert_equal([3, 4, 7, 8].collect { |i| {"message" => "test#{i}"} },
968
+ events.collect { |event| event[2] })
812
969
  end
813
970
 
814
- def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: nil)
815
- file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
971
+ def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: 5)
972
+ file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb")
816
973
  file.puts "test1"
817
974
  file.puts "test2"
818
975
  file.flush
819
976
 
820
977
  d = create_driver(config)
821
978
  d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
979
+ sleep(0.1) while d.instance.instance_variable_get(:@startup)
822
980
  size = d.emit_count
823
981
  file.puts "test3"
824
982
  file.puts "test4"
@@ -828,18 +986,18 @@ class TailInputTest < Test::Unit::TestCase
828
986
 
829
987
  if Fluent.windows?
830
988
  file.close
831
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
832
- file = File.open("#{TMP_DIR}/tail.txt", "ab")
989
+ FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
990
+ file = Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "ab")
833
991
  else
834
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
992
+ FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
835
993
  end
836
994
  if block_given?
837
995
  yield file
838
996
  else
839
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
997
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
840
998
  sleep 1
841
999
 
842
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1000
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
843
1001
  f.puts "test5"
844
1002
  f.puts "test6"
845
1003
  }
@@ -853,10 +1011,8 @@ class TailInputTest < Test::Unit::TestCase
853
1011
  end
854
1012
 
855
1013
  def test_truncate_file
856
- omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
857
-
858
1014
  config = SINGLE_LINE_CONFIG
859
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1015
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
860
1016
  f.puts "test1"
861
1017
  f.puts "test2"
862
1018
  f.flush
@@ -865,12 +1021,18 @@ class TailInputTest < Test::Unit::TestCase
865
1021
  d = create_driver(config)
866
1022
 
867
1023
  d.run(expect_emits: 2) do
868
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1024
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
869
1025
  f.puts "test3\ntest4"
870
1026
  f.flush
871
1027
  }
872
1028
  waiting(2) { sleep 0.1 until d.events.length == 2 }
873
- File.truncate("#{TMP_DIR}/tail.txt", 6)
1029
+ if Fluent.windows?
1030
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1031
+ f.puts("test1");
1032
+ }
1033
+ else
1034
+ File.truncate("#{@tmp_dir}/tail.txt", 6)
1035
+ end
874
1036
  end
875
1037
 
876
1038
  expected = {
@@ -889,10 +1051,8 @@ class TailInputTest < Test::Unit::TestCase
889
1051
  end
890
1052
 
891
1053
  def test_move_truncate_move_back
892
- omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
893
-
894
1054
  config = SINGLE_LINE_CONFIG
895
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1055
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
896
1056
  f.puts "test1"
897
1057
  f.puts "test2"
898
1058
  }
@@ -901,17 +1061,23 @@ class TailInputTest < Test::Unit::TestCase
901
1061
 
902
1062
  d.run(expect_emits: 1) do
903
1063
  if Fluent.windows?
904
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
1064
+ FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
905
1065
  else
906
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
1066
+ FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
907
1067
  end
908
1068
  sleep(1)
909
- File.truncate("#{TMP_DIR}/tail2.txt", 6)
1069
+ if Fluent.windows?
1070
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "wb") { |f|
1071
+ f.puts("test1");
1072
+ }
1073
+ else
1074
+ File.truncate("#{@tmp_dir}/tail2.txt", 6)
1075
+ end
910
1076
  sleep(1)
911
1077
  if Fluent.windows?
912
- FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt", force: true)
1078
+ FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt", force: true)
913
1079
  else
914
- FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
1080
+ FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt")
915
1081
  end
916
1082
  end
917
1083
 
@@ -923,17 +1089,17 @@ class TailInputTest < Test::Unit::TestCase
923
1089
  end
924
1090
 
925
1091
  def test_lf
926
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
1092
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
927
1093
 
928
1094
  d = create_driver
929
1095
 
930
1096
  d.run(expect_emits: 1) do
931
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1097
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
932
1098
  f.print "test3"
933
1099
  }
934
1100
  sleep 1
935
1101
 
936
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1102
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
937
1103
  f.puts "test4"
938
1104
  }
939
1105
  end
@@ -944,12 +1110,12 @@ class TailInputTest < Test::Unit::TestCase
944
1110
  end
945
1111
 
946
1112
  def test_whitespace
947
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
1113
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
948
1114
 
949
1115
  d = create_driver
950
1116
 
951
1117
  d.run(expect_emits: 1) do
952
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1118
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
953
1119
  f.puts " " # 4 spaces
954
1120
  f.puts " 4 spaces"
955
1121
  f.puts "4 spaces "
@@ -980,7 +1146,7 @@ class TailInputTest < Test::Unit::TestCase
980
1146
  d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
981
1147
 
982
1148
  d.run(expect_emits: 1) do
983
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1149
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
984
1150
  f.puts "test"
985
1151
  }
986
1152
  end
@@ -1002,7 +1168,7 @@ class TailInputTest < Test::Unit::TestCase
1002
1168
  utf8_message = cp932_message.encode(Encoding::UTF_8)
1003
1169
 
1004
1170
  d.run(expect_emits: 1) do
1005
- File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
1171
+ File.open("#{@tmp_dir}/tail.txt", "w:cp932") {|f|
1006
1172
  f.puts cp932_message
1007
1173
  }
1008
1174
  end
@@ -1025,7 +1191,7 @@ class TailInputTest < Test::Unit::TestCase
1025
1191
  utf8_message = utf16_message.encode(Encoding::UTF_8).strip
1026
1192
 
1027
1193
  d.run(expect_emits: 1) do
1028
- File.open("#{TMP_DIR}/tail.txt", "w:utf-16le") { |f|
1194
+ File.open("#{@tmp_dir}/tail.txt", "w:utf-16le") { |f|
1029
1195
  f.write utf16_message
1030
1196
  }
1031
1197
  end
@@ -1046,7 +1212,7 @@ class TailInputTest < Test::Unit::TestCase
1046
1212
  d = create_driver(conf)
1047
1213
 
1048
1214
  d.run(expect_emits: 1) do
1049
- File.open("#{TMP_DIR}/tail.txt", "w") { |f|
1215
+ File.open("#{@tmp_dir}/tail.txt", "w") { |f|
1050
1216
  f.write "te\x86st\n"
1051
1217
  }
1052
1218
  end
@@ -1061,11 +1227,11 @@ class TailInputTest < Test::Unit::TestCase
1061
1227
  parse: PARSE_MULTILINE_CONFIG)
1062
1228
  def test_multiline(data)
1063
1229
  config = data
1064
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1230
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1065
1231
 
1066
1232
  d = create_driver(config)
1067
1233
  d.run(expect_emits: 1) do
1068
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1234
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1069
1235
  f.puts "f test1"
1070
1236
  f.puts "s test2"
1071
1237
  f.puts "f test3"
@@ -1089,11 +1255,11 @@ class TailInputTest < Test::Unit::TestCase
1089
1255
  parse: PARSE_MULTILINE_CONFIG)
1090
1256
  def test_multiline_with_emit_unmatched_lines_true(data)
1091
1257
  config = data + config_element("", "", { "emit_unmatched_lines" => true })
1092
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1258
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1093
1259
 
1094
1260
  d = create_driver(config)
1095
1261
  d.run(expect_emits: 1) do
1096
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1262
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1097
1263
  f.puts "f test1"
1098
1264
  f.puts "s test2"
1099
1265
  f.puts "f test3"
@@ -1119,11 +1285,11 @@ class TailInputTest < Test::Unit::TestCase
1119
1285
  parse: PARSE_MULTILINE_CONFIG_WITH_NEWLINE)
1120
1286
  def test_multiline_with_emit_unmatched_lines2(data)
1121
1287
  config = data + config_element("", "", { "emit_unmatched_lines" => true })
1122
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1288
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1123
1289
 
1124
1290
  d = create_driver(config)
1125
1291
  d.run(expect_emits: 0, timeout: 1) do
1126
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1292
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1127
1293
  f.puts "s test0"
1128
1294
  f.puts "f test1"
1129
1295
  f.puts "f test2"
@@ -1145,15 +1311,15 @@ class TailInputTest < Test::Unit::TestCase
1145
1311
  data(flat: MULTILINE_CONFIG,
1146
1312
  parse: PARSE_MULTILINE_CONFIG)
1147
1313
  def test_multiline_with_flush_interval(data)
1148
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1314
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1149
1315
 
1150
1316
  config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
1151
1317
  d = create_driver(config)
1152
1318
 
1153
- assert_equal 2, d.instance.multiline_flush_interval
1319
+ assert_equal(2, d.instance.multiline_flush_interval)
1154
1320
 
1155
1321
  d.run(expect_emits: 1) do
1156
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1322
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1157
1323
  f.puts "f test1"
1158
1324
  f.puts "s test2"
1159
1325
  f.puts "f test3"
@@ -1188,7 +1354,7 @@ class TailInputTest < Test::Unit::TestCase
1188
1354
  d = create_driver(config + encoding_config)
1189
1355
 
1190
1356
  d.run(expect_emits: 1) do
1191
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
1357
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1192
1358
  f.puts "s test"
1193
1359
  }
1194
1360
  end
@@ -1211,7 +1377,7 @@ class TailInputTest < Test::Unit::TestCase
1211
1377
  cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1212
1378
  utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
1213
1379
  d.run(expect_emits: 1) do
1214
- File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
1380
+ File.open("#{@tmp_dir}/tail.txt", "w:cp932") { |f|
1215
1381
  f.puts cp932_message
1216
1382
  }
1217
1383
  end
@@ -1243,11 +1409,11 @@ class TailInputTest < Test::Unit::TestCase
1243
1409
  )
1244
1410
  def test_multiline_with_multiple_formats(data)
1245
1411
  config = data
1246
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1412
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1247
1413
 
1248
1414
  d = create_driver(config)
1249
1415
  d.run(expect_emits: 1) do
1250
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1416
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1251
1417
  f.puts "f test1"
1252
1418
  f.puts "s test2"
1253
1419
  f.puts "f test3"
@@ -1283,7 +1449,7 @@ class TailInputTest < Test::Unit::TestCase
1283
1449
  ])
1284
1450
  )
1285
1451
  def test_multilinelog_with_multiple_paths(data)
1286
- files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
1452
+ files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
1287
1453
  files.each { |file| File.open(file, "wb") { |f| } }
1288
1454
 
1289
1455
  config = data + config_element("", "", {
@@ -1328,12 +1494,12 @@ class TailInputTest < Test::Unit::TestCase
1328
1494
  ])
1329
1495
  )
1330
1496
  def test_multiline_without_firstline(data)
1331
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1497
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1332
1498
 
1333
1499
  config = data
1334
1500
  d = create_driver(config)
1335
1501
  d.run(expect_emits: 1) do
1336
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1502
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1337
1503
  f.puts "foo 1"
1338
1504
  f.puts "bar 1"
1339
1505
  f.puts "baz 1"
@@ -1353,35 +1519,22 @@ class TailInputTest < Test::Unit::TestCase
1353
1519
  sub_test_case "path" do
1354
1520
  # * path test
1355
1521
  # TODO: Clean up tests
1356
- EX_ROTATE_WAIT = 0
1357
- EX_FOLLOW_INODES = false
1358
-
1359
- EX_CONFIG = config_element("", "", {
1360
- "tag" => "tail",
1361
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1362
- "format" => "none",
1363
- "pos_file" => "#{TMP_DIR}/tail.pos",
1364
- "read_from_head" => true,
1365
- "refresh_interval" => 30,
1366
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1367
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1368
- })
1369
1522
  def test_expand_paths
1370
1523
  ex_paths = [
1371
1524
  create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1372
1525
  create_target_info('test/plugin/data/log/foo/bar.log'),
1373
1526
  create_target_info('test/plugin/data/log/test.log')
1374
1527
  ]
1375
- plugin = create_driver(EX_CONFIG, false).instance
1528
+ plugin = create_driver(ex_config, false).instance
1376
1529
  flexstub(Time) do |timeclass|
1377
1530
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1378
- assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1531
+ assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1379
1532
  end
1380
1533
 
1381
1534
  # Test exclusion
1382
- exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1535
+ exclude_config = ex_config + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1383
1536
  plugin = create_driver(exclude_config, false).instance
1384
- assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1537
+ assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1385
1538
  end
1386
1539
 
1387
1540
  def test_expand_paths_with_duplicate_configuration
@@ -1389,10 +1542,10 @@ class TailInputTest < Test::Unit::TestCase
1389
1542
  create_target_info('test/plugin/data/log/foo/bar.log'),
1390
1543
  create_target_info('test/plugin/data/log/test.log')
1391
1544
  ]
1392
- duplicate_config = EX_CONFIG.dup
1545
+ duplicate_config = ex_config.dup
1393
1546
  duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1394
- plugin = create_driver(EX_CONFIG, false).instance
1395
- assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1547
+ plugin = create_driver(ex_config, false).instance
1548
+ assert_equal(expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1396
1549
  end
1397
1550
 
1398
1551
  def test_expand_paths_with_timezone
@@ -1402,7 +1555,7 @@ class TailInputTest < Test::Unit::TestCase
1402
1555
  create_target_info('test/plugin/data/log/test.log')
1403
1556
  ]
1404
1557
  ['Asia/Taipei', '+08'].each do |tz_type|
1405
- taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
1558
+ taipei_config = ex_config + config_element("", "", {"path_timezone" => tz_type})
1406
1559
  plugin = create_driver(taipei_config, false).instance
1407
1560
 
1408
1561
  # Test exclude
@@ -1414,8 +1567,8 @@ class TailInputTest < Test::Unit::TestCase
1414
1567
  # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1415
1568
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1416
1569
 
1417
- assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1418
- assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1570
+ assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1571
+ assert_equal(ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1419
1572
  end
1420
1573
  end
1421
1574
  end
@@ -1433,35 +1586,44 @@ class TailInputTest < Test::Unit::TestCase
1433
1586
  "tag" => "tail",
1434
1587
  "path" => "test/plugin/data/log/**/*",
1435
1588
  "format" => "none",
1436
- "pos_file" => "#{TMP_DIR}/tail.pos"
1589
+ "pos_file" => "#{@tmp_dir}/tail.pos"
1437
1590
  })
1438
1591
 
1439
1592
  plugin = create_driver(config, false).instance
1440
- assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
1593
+ assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1441
1594
  end
1442
1595
 
1443
1596
  def test_unwatched_files_should_be_removed
1444
1597
  config = config_element("", "", {
1445
1598
  "tag" => "tail",
1446
- "path" => "#{TMP_DIR}/*.txt",
1599
+ "path" => "#{@tmp_dir}/*.txt",
1447
1600
  "format" => "none",
1448
- "pos_file" => "#{TMP_DIR}/tail.pos",
1601
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1449
1602
  "read_from_head" => true,
1450
1603
  "refresh_interval" => 1,
1451
1604
  })
1452
1605
  d = create_driver(config, false)
1453
1606
  d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1454
1607
  d.run(expect_emits: 1, shutdown: false) do
1455
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
1608
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f| f.puts "test3\n" }
1456
1609
  end
1457
1610
 
1458
- cleanup_directory(TMP_DIR)
1459
- waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
1611
+ cleanup_directory(@tmp_dir)
1612
+ waiting(20) { sleep 0.1 until Dir.glob("#{@tmp_dir}/*.txt").size == 0 } # Ensure file is deleted on Windows
1460
1613
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1461
1614
 
1462
- assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
1463
-
1464
- d.instance_shutdown
1615
+ assert_equal(
1616
+ {
1617
+ files: [],
1618
+ tails: []
1619
+ },
1620
+ {
1621
+ files: Dir.glob("#{@tmp_dir}/*.txt"),
1622
+ tails: d.instance.instance_variable_get(:@tails).keys
1623
+ }
1624
+ )
1625
+ ensure
1626
+ d.instance_shutdown if d && d.instance
1465
1627
  end
1466
1628
 
1467
1629
  def count_timer_object
@@ -1511,55 +1673,55 @@ class TailInputTest < Test::Unit::TestCase
1511
1673
  "rotate_wait" => "2s"
1512
1674
  }) + PARSE_SINGLE_LINE_CONFIG, false)
1513
1675
 
1514
- assert_equal readable_paths, d.instance.expand_paths.length
1515
- assert_equal result, d.instance.have_read_capability?
1676
+ assert_equal(readable_paths, d.instance.expand_paths.length)
1677
+ assert_equal(result, d.instance.have_read_capability?)
1516
1678
  end
1517
1679
  end
1518
1680
 
1519
1681
  def test_pos_file_dir_creation
1520
1682
  config = config_element("", "", {
1521
1683
  "tag" => "tail",
1522
- "path" => "#{TMP_DIR}/*.txt",
1684
+ "path" => "#{@tmp_dir}/*.txt",
1523
1685
  "format" => "none",
1524
- "pos_file" => "#{TMP_DIR}/pos/tail.pos",
1686
+ "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1525
1687
  "read_from_head" => true,
1526
1688
  "refresh_interval" => 1
1527
1689
  })
1528
1690
 
1529
- assert_path_not_exist("#{TMP_DIR}/pos")
1691
+ assert_path_not_exist("#{@tmp_dir}/pos")
1530
1692
  d = create_driver(config, false)
1531
1693
  d.run
1532
- assert_path_exist("#{TMP_DIR}/pos")
1533
- assert_equal '755', File.stat("#{TMP_DIR}/pos").mode.to_s(8)[-3, 3]
1694
+ assert_path_exist("#{@tmp_dir}/pos")
1695
+ assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1534
1696
  ensure
1535
- cleanup_directory(TMP_DIR)
1697
+ cleanup_directory(@tmp_dir)
1536
1698
  end
1537
1699
 
1538
1700
  def test_pos_file_dir_creation_with_system_dir_permission
1539
1701
  config = config_element("", "", {
1540
1702
  "tag" => "tail",
1541
- "path" => "#{TMP_DIR}/*.txt",
1703
+ "path" => "#{@tmp_dir}/*.txt",
1542
1704
  "format" => "none",
1543
- "pos_file" => "#{TMP_DIR}/pos/tail.pos",
1705
+ "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1544
1706
  "read_from_head" => true,
1545
1707
  "refresh_interval" => 1
1546
1708
  })
1547
1709
 
1548
- assert_path_not_exist("#{TMP_DIR}/pos")
1710
+ assert_path_not_exist("#{@tmp_dir}/pos")
1549
1711
 
1550
1712
  Fluent::SystemConfig.overwrite_system_config({ "dir_permission" => "744" }) do
1551
1713
  d = create_driver(config, false)
1552
1714
  d.run
1553
1715
  end
1554
1716
 
1555
- assert_path_exist("#{TMP_DIR}/pos")
1717
+ assert_path_exist("#{@tmp_dir}/pos")
1556
1718
  if Fluent.windows?
1557
- assert_equal '755', File.stat("#{TMP_DIR}/pos").mode.to_s(8)[-3, 3]
1719
+ assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1558
1720
  else
1559
- assert_equal '744', File.stat("#{TMP_DIR}/pos").mode.to_s(8)[-3, 3]
1721
+ assert_equal('744', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1560
1722
  end
1561
1723
  ensure
1562
- cleanup_directory(TMP_DIR)
1724
+ cleanup_directory(@tmp_dir)
1563
1725
  end
1564
1726
 
1565
1727
  def test_z_refresh_watchers
@@ -1568,7 +1730,7 @@ class TailInputTest < Test::Unit::TestCase
1568
1730
  create_target_info('test/plugin/data/log/foo/bar.log'),
1569
1731
  create_target_info('test/plugin/data/log/test.log'),
1570
1732
  ]
1571
- plugin = create_driver(EX_CONFIG, false).instance
1733
+ plugin = create_driver(ex_config, false).instance
1572
1734
  sio = StringIO.new
1573
1735
  plugin.instance_eval do
1574
1736
  @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
@@ -1591,8 +1753,7 @@ class TailInputTest < Test::Unit::TestCase
1591
1753
  end
1592
1754
 
1593
1755
  path = 'test/plugin/data/2010/01/20100102-030405.log'
1594
- target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
1595
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
1756
+ mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[path], Fluent::FileWrapper.stat(path).ino)
1596
1757
 
1597
1758
  Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1598
1759
  path = "test/plugin/data/2010/01/20100102-030406.log"
@@ -1612,9 +1773,9 @@ class TailInputTest < Test::Unit::TestCase
1612
1773
  test 'type of pos_file_compaction_interval is time' do
1613
1774
  tail = {
1614
1775
  "tag" => "tail",
1615
- "path" => "#{TMP_DIR}/*.txt",
1776
+ "path" => "#{@tmp_dir}/*.txt",
1616
1777
  "format" => "none",
1617
- "pos_file" => "#{TMP_DIR}/pos/tail.pos",
1778
+ "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1618
1779
  "refresh_interval" => 1,
1619
1780
  "read_from_head" => true,
1620
1781
  'pos_file_compaction_interval' => '24h',
@@ -1631,7 +1792,7 @@ class TailInputTest < Test::Unit::TestCase
1631
1792
  DummyWatcher = Struct.new("DummyWatcher", :tag)
1632
1793
 
1633
1794
  def test_tag
1634
- d = create_driver(EX_CONFIG, false)
1795
+ d = create_driver(ex_config, false)
1635
1796
  d.run {}
1636
1797
  plugin = d.instance
1637
1798
  mock(plugin.router).emit_stream('tail', anything).once
@@ -1715,13 +1876,13 @@ class TailInputTest < Test::Unit::TestCase
1715
1876
  test 'max_line_size' do |(label, size)|
1716
1877
  config = config_element("", "", {
1717
1878
  "tag" => "max_line_size",
1718
- "path" => "#{TMP_DIR}/with_long_lines.txt",
1879
+ "path" => "#{@tmp_dir}/with_long_lines.txt",
1719
1880
  "format" => "none",
1720
1881
  "read_from_head" => true,
1721
1882
  "max_line_size" => label,
1722
1883
  "log_level" => "debug"
1723
1884
  })
1724
- File.open("#{TMP_DIR}/with_long_lines.txt", "w+") do |f|
1885
+ File.open("#{@tmp_dir}/with_long_lines.txt", "w+") do |f|
1725
1886
  f.puts "foo"
1726
1887
  f.puts "x" * size # 'x' * size + \n > @max_line_size
1727
1888
  f.puts "bar"
@@ -1747,7 +1908,7 @@ class TailInputTest < Test::Unit::TestCase
1747
1908
  # Ensure that no fatal exception is raised when a file is missing and that
1748
1909
  # files that do exist are still tailed as expected.
1749
1910
  def test_missing_file
1750
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1911
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1751
1912
  f.puts "test1"
1752
1913
  f.puts "test2"
1753
1914
  }
@@ -1756,16 +1917,16 @@ class TailInputTest < Test::Unit::TestCase
1756
1917
  # since their interactions with the filesystem differ.
1757
1918
  config1 = config_element("", "", {
1758
1919
  "tag" => "t1",
1759
- "path" => "#{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt",
1920
+ "path" => "#{@tmp_dir}/non_existent_file.txt,#{@tmp_dir}/tail.txt",
1760
1921
  "format" => "none",
1761
1922
  "rotate_wait" => "2s",
1762
- "pos_file" => "#{TMP_DIR}/tail.pos"
1923
+ "pos_file" => "#{@tmp_dir}/tail.pos"
1763
1924
  })
1764
1925
  config2 = config1 + config_element("", "", { "read_from_head" => true })
1765
1926
  [config1, config2].each do |config|
1766
1927
  d = create_driver(config, false)
1767
1928
  d.run(expect_emits: 1) do
1768
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1929
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1769
1930
  f.puts "test3"
1770
1931
  f.puts "test4"
1771
1932
  }
@@ -1781,19 +1942,19 @@ class TailInputTest < Test::Unit::TestCase
1781
1942
 
1782
1943
  sub_test_case 'inode_processing' do
1783
1944
  def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
1784
- config = COMMON_FOLLOW_INODE_CONFIG
1945
+ config = common_follow_inode_config
1785
1946
 
1786
- path = "#{TMP_DIR}/tail.txt"
1947
+ path = "#{@tmp_dir}/tail.txt"
1787
1948
  ino = 1
1788
1949
  pos = 1234
1789
- File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
1950
+ File.open("#{@tmp_dir}/tail.pos", "wb") {|f|
1790
1951
  f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1791
1952
  }
1792
1953
 
1793
1954
  d = create_driver(config, false)
1794
1955
  d.run
1795
1956
 
1796
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1957
+ pos_file = File.open("#{@tmp_dir}/tail.pos", "r")
1797
1958
  pos_file.pos = 0
1798
1959
 
1799
1960
  assert_raise(EOFError) do
@@ -1802,21 +1963,21 @@ class TailInputTest < Test::Unit::TestCase
1802
1963
  end
1803
1964
 
1804
1965
  def test_should_write_latest_offset_after_rotate_wait
1805
- config = COMMON_FOLLOW_INODE_CONFIG
1806
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1966
+ config = common_follow_inode_config
1967
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1807
1968
  f.puts "test1"
1808
1969
  f.puts "test2"
1809
1970
  }
1810
1971
 
1811
1972
  d = create_driver(config, false)
1812
1973
  d.run(expect_emits: 2, shutdown: false) do
1813
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1814
- FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
1974
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
1975
+ FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
1815
1976
  sleep 1
1816
- File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
1977
+ File.open("#{@tmp_dir}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
1817
1978
  end
1818
1979
 
1819
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1980
+ pos_file = File.open("#{@tmp_dir}/tail.pos", "r")
1820
1981
  pos_file.pos = 0
1821
1982
  line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1822
1983
  waiting(5) {
@@ -1833,16 +1994,16 @@ class TailInputTest < Test::Unit::TestCase
1833
1994
  def test_should_remove_deleted_file
1834
1995
  config = config_element("", "", {"format" => "none"})
1835
1996
 
1836
- path = "#{TMP_DIR}/tail.txt"
1997
+ path = "#{@tmp_dir}/tail.txt"
1837
1998
  ino = 1
1838
1999
  pos = 1234
1839
- File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
2000
+ File.open("#{@tmp_dir}/tail.pos", "wb") {|f|
1840
2001
  f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1841
2002
  }
1842
2003
 
1843
2004
  d = create_driver(config)
1844
2005
  d.run do
1845
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
2006
+ pos_file = File.open("#{@tmp_dir}/tail.pos", "r")
1846
2007
  pos_file.pos = 0
1847
2008
  assert_equal([], pos_file.readlines)
1848
2009
  end
@@ -1850,8 +2011,8 @@ class TailInputTest < Test::Unit::TestCase
1850
2011
 
1851
2012
  def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
1852
2013
  config = config_element("ROOT", "", {
1853
- "path" => "#{TMP_DIR}/tail.txt*",
1854
- "pos_file" => "#{TMP_DIR}/tail.pos",
2014
+ "path" => "#{@tmp_dir}/tail.txt*",
2015
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1855
2016
  "tag" => "t1",
1856
2017
  "rotate_wait" => "1s",
1857
2018
  "refresh_interval" => "1s",
@@ -1863,14 +2024,14 @@ class TailInputTest < Test::Unit::TestCase
1863
2024
 
1864
2025
  d = create_driver(config, false)
1865
2026
 
1866
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2027
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1867
2028
  f.puts "test1"
1868
2029
  f.puts "test2"
1869
2030
  }
1870
- target_info = create_target_info("#{TMP_DIR}/tail.txt")
2031
+ target_info = create_target_info("#{@tmp_dir}/tail.txt")
1871
2032
 
1872
2033
  d.run(expect_emits: 1, shutdown: false) do
1873
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
2034
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
1874
2035
  end
1875
2036
 
1876
2037
 
@@ -1888,8 +2049,8 @@ class TailInputTest < Test::Unit::TestCase
1888
2049
 
1889
2050
  def test_should_read_from_head_on_file_renaming_with_star_in_pattern
1890
2051
  config = config_element("ROOT", "", {
1891
- "path" => "#{TMP_DIR}/tail.txt*",
1892
- "pos_file" => "#{TMP_DIR}/tail.pos",
2052
+ "path" => "#{@tmp_dir}/tail.txt*",
2053
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1893
2054
  "tag" => "t1",
1894
2055
  "rotate_wait" => "10s",
1895
2056
  "refresh_interval" => "1s",
@@ -1901,14 +2062,14 @@ class TailInputTest < Test::Unit::TestCase
1901
2062
 
1902
2063
  d = create_driver(config, false)
1903
2064
 
1904
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2065
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1905
2066
  f.puts "test1"
1906
2067
  f.puts "test2"
1907
2068
  }
1908
2069
 
1909
2070
  d.run(expect_emits: 2, shutdown: false) do
1910
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1911
- FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
2071
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2072
+ FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
1912
2073
  end
1913
2074
 
1914
2075
  events = d.events
@@ -1917,20 +2078,20 @@ class TailInputTest < Test::Unit::TestCase
1917
2078
  end
1918
2079
 
1919
2080
  def test_should_not_read_from_head_on_rotation_when_watching_inodes
1920
- config = COMMON_FOLLOW_INODE_CONFIG
2081
+ config = common_follow_inode_config
1921
2082
 
1922
2083
  d = create_driver(config, false)
1923
2084
 
1924
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2085
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1925
2086
  f.puts "test1"
1926
2087
  f.puts "test2"
1927
2088
  }
1928
2089
 
1929
2090
  d.run(expect_emits: 1, shutdown: false) do
1930
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
2091
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
1931
2092
  end
1932
2093
 
1933
- FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
2094
+ FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
1934
2095
  Timecop.travel(Time.now + 10) do
1935
2096
  sleep 2
1936
2097
  events = d.events
@@ -1941,23 +2102,23 @@ class TailInputTest < Test::Unit::TestCase
1941
2102
  end
1942
2103
 
1943
2104
  def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
1944
- config = COMMON_FOLLOW_INODE_CONFIG
2105
+ config = common_follow_inode_config
1945
2106
 
1946
2107
  d = create_driver(config, false)
1947
2108
 
1948
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2109
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1949
2110
  f.puts "test1"
1950
2111
  f.puts "test2"
1951
2112
  }
1952
- target_info = create_target_info("#{TMP_DIR}/tail.txt")
2113
+ target_info = create_target_info("#{@tmp_dir}/tail.txt")
1953
2114
 
1954
2115
  d.run(expect_emits: 2, shutdown: false) do
1955
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
1956
- cleanup_file("#{TMP_DIR}/tail.txt")
1957
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
2116
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2117
+ cleanup_file("#{@tmp_dir}/tail.txt")
2118
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test4\n"}
1958
2119
  end
1959
2120
 
1960
- new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
2121
+ new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
1961
2122
 
1962
2123
  pos_file = d.instance.instance_variable_get(:@pf)
1963
2124
 
@@ -1973,7 +2134,7 @@ class TailInputTest < Test::Unit::TestCase
1973
2134
 
1974
2135
  def test_should_close_watcher_after_rotate_wait
1975
2136
  now = Time.now
1976
- config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
2137
+ config = common_follow_inode_config + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1977
2138
 
1978
2139
  d = create_driver(config, false)
1979
2140
  d.instance.instance_eval do
@@ -1986,52 +2147,52 @@ class TailInputTest < Test::Unit::TestCase
1986
2147
  @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1987
2148
  end
1988
2149
 
1989
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2150
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1990
2151
  f.puts "test1"
1991
2152
  f.puts "test2"
1992
2153
  }
1993
- target_info = create_target_info("#{TMP_DIR}/tail.txt")
2154
+ target_info = create_target_info("#{@tmp_dir}/tail.txt")
1994
2155
  mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
1995
2156
  d.run(shutdown: false)
1996
- assert d.instance.instance_variable_get(:@tails)[target_info]
2157
+ assert d.instance.instance_variable_get(:@tails)[target_info.path]
1997
2158
 
1998
2159
  Timecop.travel(now + 10) do
1999
2160
  d.instance.instance_eval do
2000
- sleep 0.1 until @tails[target_info] == nil
2161
+ sleep 0.1 until @tails[target_info.path] == nil
2001
2162
  end
2002
- assert_nil d.instance.instance_variable_get(:@tails)[target_info]
2163
+ assert_nil d.instance.instance_variable_get(:@tails)[target_info.path]
2003
2164
  end
2004
2165
  d.instance_shutdown
2005
2166
  end
2006
2167
 
2007
2168
  def test_should_create_new_watcher_for_new_file_with_same_name
2008
2169
  now = Time.now
2009
- config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
2170
+ config = common_follow_inode_config + config_element('', '', {"limit_recently_modified" => "2s"})
2010
2171
 
2011
2172
  d = create_driver(config, false)
2012
2173
 
2013
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2174
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2014
2175
  f.puts "test1"
2015
2176
  f.puts "test2"
2016
2177
  }
2017
- path_ino = create_target_info("#{TMP_DIR}/tail.txt")
2178
+ path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2018
2179
 
2019
2180
  d.run(expect_emits: 1, shutdown: false) do
2020
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
2181
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2021
2182
  end
2022
2183
 
2023
- cleanup_file("#{TMP_DIR}/tail.txt")
2024
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2184
+ cleanup_file("#{@tmp_dir}/tail.txt")
2185
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2025
2186
  f.puts "test3"
2026
2187
  f.puts "test4"
2027
2188
  }
2028
- new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
2189
+ new_path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2029
2190
 
2030
2191
  Timecop.travel(now + 10) do
2031
2192
  sleep 3
2032
2193
  d.instance.instance_eval do
2033
- @tails[path_ino] == nil
2034
- @tails[new_path_ino] != nil
2194
+ @tails[path_ino.path] == nil
2195
+ @tails[new_path_ino.path] != nil
2035
2196
  end
2036
2197
  end
2037
2198
 
@@ -2043,19 +2204,19 @@ class TailInputTest < Test::Unit::TestCase
2043
2204
  end
2044
2205
 
2045
2206
  def test_truncate_file_with_follow_inodes
2046
- config = COMMON_FOLLOW_INODE_CONFIG
2207
+ config = common_follow_inode_config
2047
2208
 
2048
2209
  d = create_driver(config, false)
2049
2210
 
2050
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2211
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2051
2212
  f.puts "test1"
2052
2213
  f.puts "test2"
2053
2214
  }
2054
2215
 
2055
2216
  d.run(expect_emits: 3, shutdown: false) do
2056
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
2217
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2057
2218
  sleep 2
2058
- File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
2219
+ File.open("#{@tmp_dir}/tail.txt", "w+b") {|f| f.puts "test4\n"}
2059
2220
  end
2060
2221
 
2061
2222
  events = d.events
@@ -2069,15 +2230,15 @@ class TailInputTest < Test::Unit::TestCase
2069
2230
 
2070
2231
  # issue #3464
2071
2232
  def test_should_replace_target_info
2072
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2233
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2073
2234
  f.puts "test1\n"
2074
2235
  }
2075
- target_info = create_target_info("#{TMP_DIR}/tail.txt")
2236
+ target_info = create_target_info("#{@tmp_dir}/tail.txt")
2076
2237
  inodes = []
2077
2238
 
2078
2239
  config = config_element("ROOT", "", {
2079
- "path" => "#{TMP_DIR}/tail.txt*",
2080
- "pos_file" => "#{TMP_DIR}/tail.pos",
2240
+ "path" => "#{@tmp_dir}/tail.txt*",
2241
+ "pos_file" => "#{@tmp_dir}/tail.pos",
2081
2242
  "tag" => "t1",
2082
2243
  "refresh_interval" => "60s",
2083
2244
  "read_from_head" => "true",
@@ -2090,21 +2251,21 @@ class TailInputTest < Test::Unit::TestCase
2090
2251
  while d.events.size < 1 do
2091
2252
  sleep 0.1
2092
2253
  end
2093
- inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2094
- key.ino
2254
+ inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2255
+ tw.ino
2095
2256
  end
2096
2257
  assert_equal([target_info.ino], inodes)
2097
2258
 
2098
- cleanup_file("#{TMP_DIR}/tail.txt")
2099
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
2259
+ cleanup_file("#{@tmp_dir}/tail.txt")
2260
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test2\n"}
2100
2261
 
2101
2262
  while d.events.size < 2 do
2102
2263
  sleep 0.1
2103
2264
  end
2104
- inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2105
- key.ino
2265
+ inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2266
+ tw.ino
2106
2267
  end
2107
- new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
2268
+ new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2108
2269
  assert_not_equal(target_info.ino, new_target_info.ino)
2109
2270
  assert_equal([new_target_info.ino], inodes)
2110
2271
  end
@@ -2113,7 +2274,7 @@ class TailInputTest < Test::Unit::TestCase
2113
2274
 
2114
2275
  sub_test_case "tail_path" do
2115
2276
  def test_tail_path_with_singleline
2116
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
2277
+ File.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2117
2278
  f.puts "test1"
2118
2279
  f.puts "test2"
2119
2280
  }
@@ -2121,7 +2282,7 @@ class TailInputTest < Test::Unit::TestCase
2121
2282
  d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
2122
2283
 
2123
2284
  d.run(expect_emits: 1) do
2124
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
2285
+ File.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2125
2286
  f.puts "test3"
2126
2287
  f.puts "test4"
2127
2288
  }
@@ -2130,12 +2291,12 @@ class TailInputTest < Test::Unit::TestCase
2130
2291
  events = d.events
2131
2292
  assert_equal(true, events.length > 0)
2132
2293
  events.each do |emit|
2133
- assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
2294
+ assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2134
2295
  end
2135
2296
  end
2136
2297
 
2137
2298
  def test_tail_path_with_multiline_with_firstline
2138
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
2299
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2139
2300
 
2140
2301
  config = config_element("", "", {
2141
2302
  "path_key" => "path",
@@ -2145,7 +2306,7 @@ class TailInputTest < Test::Unit::TestCase
2145
2306
  })
2146
2307
  d = create_driver(config)
2147
2308
  d.run(expect_emits: 1) do
2148
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
2309
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2149
2310
  f.puts "f test1"
2150
2311
  f.puts "s test2"
2151
2312
  f.puts "f test3"
@@ -2160,12 +2321,12 @@ class TailInputTest < Test::Unit::TestCase
2160
2321
  events = d.events
2161
2322
  assert_equal(4, events.length)
2162
2323
  events.each do |emit|
2163
- assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
2324
+ assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2164
2325
  end
2165
2326
  end
2166
2327
 
2167
2328
  def test_tail_path_with_multiline_without_firstline
2168
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
2329
+ File.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2169
2330
 
2170
2331
  config = config_element("", "", {
2171
2332
  "path_key" => "path",
@@ -2176,7 +2337,7 @@ class TailInputTest < Test::Unit::TestCase
2176
2337
  })
2177
2338
  d = create_driver(config)
2178
2339
  d.run(expect_emits: 1) do
2179
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
2340
+ File.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2180
2341
  f.puts "foo 1"
2181
2342
  f.puts "bar 1"
2182
2343
  f.puts "baz 1"
@@ -2186,7 +2347,7 @@ class TailInputTest < Test::Unit::TestCase
2186
2347
  events = d.events
2187
2348
  assert(events.length > 0)
2188
2349
  events.each do |emit|
2189
- assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
2350
+ assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2190
2351
  end
2191
2352
  end
2192
2353
 
@@ -2194,7 +2355,7 @@ class TailInputTest < Test::Unit::TestCase
2194
2355
  if ENV["APPVEYOR"] && Fluent.windows?
2195
2356
  omit "This testcase is unstable on AppVeyor."
2196
2357
  end
2197
- files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
2358
+ files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
2198
2359
  files.each { |file| File.open(file, "wb") { |f| } }
2199
2360
 
2200
2361
  config = config_element("", "", {
@@ -2228,30 +2389,30 @@ class TailInputTest < Test::Unit::TestCase
2228
2389
 
2229
2390
  def test_limit_recently_modified
2230
2391
  now = Time.new(2010, 1, 2, 3, 4, 5)
2231
- FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
2232
- FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
2233
- FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
2392
+ FileUtils.touch("#{@tmp_dir}/tail_unwatch.txt", mtime: (now - 3601))
2393
+ FileUtils.touch("#{@tmp_dir}/tail_watch1.txt", mtime: (now - 3600))
2394
+ FileUtils.touch("#{@tmp_dir}/tail_watch2.txt", mtime: now)
2234
2395
 
2235
2396
  config = config_element('', '', {
2236
2397
  'tag' => 'tail',
2237
- 'path' => "#{TMP_DIR}/*.txt",
2398
+ 'path' => "#{@tmp_dir}/*.txt",
2238
2399
  'format' => 'none',
2239
2400
  'limit_recently_modified' => '3600s'
2240
2401
  })
2241
2402
 
2242
2403
  expected_files = [
2243
- create_target_info("#{TMP_DIR}/tail_watch1.txt"),
2244
- create_target_info("#{TMP_DIR}/tail_watch2.txt")
2404
+ create_target_info("#{@tmp_dir}/tail_watch1.txt"),
2405
+ create_target_info("#{@tmp_dir}/tail_watch2.txt")
2245
2406
  ]
2246
2407
 
2247
2408
  Timecop.freeze(now) do
2248
2409
  plugin = create_driver(config, false).instance
2249
- assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
2410
+ assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
2250
2411
  end
2251
2412
  end
2252
2413
 
2253
2414
  def test_skip_refresh_on_startup
2254
- FileUtils.touch("#{TMP_DIR}/tail.txt")
2415
+ FileUtils.touch("#{@tmp_dir}/tail.txt")
2255
2416
  config = config_element('', '', {
2256
2417
  'format' => 'none',
2257
2418
  'refresh_interval' => 1,
@@ -2259,37 +2420,42 @@ class TailInputTest < Test::Unit::TestCase
2259
2420
  })
2260
2421
  d = create_driver(config)
2261
2422
  d.run(shutdown: false) {}
2262
- assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
2423
+ assert_equal(0, d.instance.instance_variable_get(:@tails).keys.size)
2263
2424
  # detect a file at first execution of in_tail_refresh_watchers timer
2264
2425
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
2265
2426
  d.instance_shutdown
2266
2427
  end
2267
2428
 
2268
2429
  def test_ENOENT_error_after_setup_watcher
2269
- path = "#{TMP_DIR}/tail.txt"
2430
+ path = "#{@tmp_dir}/tail.txt"
2270
2431
  FileUtils.touch(path)
2271
2432
  config = config_element('', '', {
2272
2433
  'format' => 'none',
2273
2434
  })
2274
2435
  d = create_driver(config)
2275
- mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
2276
- cleanup_file(path)
2277
- tw
2278
- end
2436
+ file_deleted = false
2437
+ mock.proxy(d.instance).existence_path do |hash|
2438
+ unless file_deleted
2439
+ cleanup_file(path)
2440
+ file_deleted = true
2441
+ end
2442
+ hash
2443
+ end.twice
2279
2444
  assert_nothing_raised do
2280
2445
  d.run(shutdown: false) {}
2281
2446
  end
2282
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2447
+ assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2448
+ $log.out.logs.join("\n"))
2283
2449
  ensure
2284
2450
  d.instance_shutdown if d && d.instance
2285
2451
  end
2286
2452
 
2287
2453
  def test_EACCES_error_after_setup_watcher
2288
2454
  omit "Cannot test with root user" if Process::UID.eid == 0
2289
- path = "#{TMP_DIR}/noaccess/tail.txt"
2455
+ path = "#{@tmp_dir}/noaccess/tail.txt"
2290
2456
  begin
2291
- FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
2292
- FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2457
+ FileUtils.mkdir_p("#{@tmp_dir}/noaccess")
2458
+ FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2293
2459
  FileUtils.touch(path)
2294
2460
  config = config_element('', '', {
2295
2461
  'tag' => "tail",
@@ -2297,25 +2463,26 @@ class TailInputTest < Test::Unit::TestCase
2297
2463
  'format' => 'none',
2298
2464
  })
2299
2465
  d = create_driver(config, false)
2300
- mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
2301
- FileUtils.chmod(0000, "#{TMP_DIR}/noaccess")
2302
- tw
2303
- end
2466
+ mock.proxy(d.instance).existence_path do |hash|
2467
+ FileUtils.chmod(0000, "#{@tmp_dir}/noaccess")
2468
+ hash
2469
+ end.twice
2304
2470
  assert_nothing_raised do
2305
2471
  d.run(shutdown: false) {}
2306
2472
  end
2307
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2473
+ assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2474
+ $log.out.logs.join("\n"))
2308
2475
  end
2309
2476
  ensure
2310
2477
  d.instance_shutdown if d && d.instance
2311
- if File.exist?("#{TMP_DIR}/noaccess")
2312
- FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2313
- FileUtils.rm_rf("#{TMP_DIR}/noaccess")
2478
+ if File.exist?("#{@tmp_dir}/noaccess")
2479
+ FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2480
+ FileUtils.rm_rf("#{@tmp_dir}/noaccess")
2314
2481
  end
2315
2482
  end unless Fluent.windows?
2316
2483
 
2317
2484
  def test_EACCES
2318
- path = "#{TMP_DIR}/tail.txt"
2485
+ path = "#{@tmp_dir}/tail.txt"
2319
2486
  FileUtils.touch(path)
2320
2487
  config = config_element('', '', {
2321
2488
  'format' => 'none',
@@ -2333,8 +2500,8 @@ class TailInputTest < Test::Unit::TestCase
2333
2500
  end
2334
2501
 
2335
2502
  def test_shutdown_timeout
2336
- path = "#{TMP_DIR}/tail.txt"
2337
- File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
2503
+ File.open("#{@tmp_dir}/tail.txt", "wb") do |f|
2504
+ # Should be large enough to take too long time to consume
2338
2505
  (1024 * 1024 * 5).times do
2339
2506
  f.puts "{\"test\":\"fizzbuzz\"}"
2340
2507
  end
@@ -2346,18 +2513,127 @@ class TailInputTest < Test::Unit::TestCase
2346
2513
  'format' => 'json',
2347
2514
  'skip_refresh_on_startup' => true,
2348
2515
  })
2516
+ shutdown_start_time = 0
2517
+
2349
2518
  d = create_driver(config)
2350
2519
  mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
2520
+ mock.proxy(io_handler).ready_to_shutdown(anything) do
2521
+ shutdown_start_time = Fluent::Clock.now
2522
+ end
2351
2523
  io_handler.shutdown_timeout = 0.5
2352
2524
  io_handler
2353
2525
  end
2354
2526
 
2355
- start_time = Fluent::Clock.now
2356
2527
  assert_nothing_raised do
2357
2528
  d.run(expect_emits: 1)
2358
2529
  end
2359
2530
 
2360
- elapsed = Fluent::Clock.now - start_time
2361
- assert_true(elapsed > 0.5 && elapsed < 2.5)
2531
+ elapsed = Fluent::Clock.now - shutdown_start_time
2532
+ assert_true(elapsed > 0.5 && elapsed < 2.0,
2533
+ "elapsed time: #{elapsed}")
2534
+ end
2535
+
2536
+ sub_test_case "throttling logs at in_tail level" do
2537
+ data("file test1.log no_limit 5120 text: msg" => ["test1.log", 5120, "msg"],
2538
+ "file test2.log no_limit 1024 text: test" => ["test2.log", 1024, "test"])
2539
+ def test_lines_collected_with_no_throttling(data)
2540
+ file, num_lines, msg = data
2541
+
2542
+ pattern = "/^#{@tmp_dir}\/(?<file>.+)\.log$/"
2543
+ rule = create_rule_directive({
2544
+ "file" => "/test.*/",
2545
+ }, -1)
2546
+ group = create_group_directive(pattern, "1s", rule)
2547
+ path_element = create_path_element(file)
2548
+
2549
+ conf = ROOT_CONFIG + group + path_element + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG
2550
+
2551
+ File.open("#{@tmp_dir}/#{file}", 'wb') do |f|
2552
+ num_lines.times do
2553
+ f.puts "#{msg}\n"
2554
+ end
2555
+ end
2556
+
2557
+
2558
+ d = create_driver(conf, false)
2559
+ d.run(timeout: 3) do
2560
+ start_time = Fluent::Clock.now
2561
+
2562
+ assert_equal(num_lines, d.record_count)
2563
+ assert_equal({ "message" => msg }, d.events[0][2])
2564
+
2565
+ prev_count = d.record_count
2566
+ sleep(0.1) while d.emit_count < 1
2567
+ assert_true(Fluent::Clock.now - start_time < 2)
2568
+ ## after waiting for 1 (+ jitter) secs, limit will reset
2569
+ ## Plugin will start reading but it will encounter EOF Error
2570
+ ## since no logs are left to be read
2571
+ ## Hence, d.record_count = prev_count
2572
+ tail_watcher_interval = 1.0 # hard coded value in in_tail
2573
+ safety_ratio = 1.2
2574
+ jitter = tail_watcher_interval * safety_ratio
2575
+ sleep(1.0 + jitter)
2576
+ assert_equal(0, d.record_count - prev_count)
2577
+ end
2578
+ end
2579
+
2580
+ test "lines collected with throttling" do
2581
+ file = "podname1_namespace12_container-123456.log"
2582
+ limit = 1000
2583
+ rate_period = 2
2584
+ num_lines = 3000
2585
+ msg = "a" * 8190 # Total size = 8190 bytes + 2 (\n) bytes
2586
+
2587
+ rule = create_rule_directive({
2588
+ "namespace"=> "/namespace.+/",
2589
+ "podname"=> "/podname.+/",
2590
+ }, limit)
2591
+ path_element = create_path_element(file)
2592
+ conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, "#{rate_period}s", rule) + path_element + SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD
2593
+
2594
+ d = create_driver(conf, false)
2595
+ file_path = "#{@tmp_dir}/#{file}"
2596
+
2597
+ File.open(file_path, 'wb') do |f|
2598
+ num_lines.times do
2599
+ f.puts msg
2600
+ end
2601
+ end
2602
+
2603
+ d.run(timeout: 15) do
2604
+ sleep_interval = 0.1
2605
+ tail_watcher_interval = 1.0 # hard coded value in in_tail
2606
+ safety_ratio = 1.2
2607
+ lower_jitter = sleep_interval * safety_ratio
2608
+ upper_jitter = (tail_watcher_interval + sleep_interval) * safety_ratio
2609
+ lower_interval = rate_period - lower_jitter
2610
+ upper_interval = rate_period + upper_jitter
2611
+
2612
+ emit_count = 0
2613
+ prev_count = 0
2614
+
2615
+ while emit_count < 3 do
2616
+ start_time = Fluent::Clock.now
2617
+ sleep(sleep_interval) while d.emit_count <= emit_count
2618
+ elapsed_seconds = Fluent::Clock.now - start_time
2619
+ if emit_count > 0
2620
+ assert_true(elapsed_seconds > lower_interval && elapsed_seconds < upper_interval,
2621
+ "elapsed_seconds #{elapsed_seconds} is out of allowed range:\n" +
2622
+ " lower: #{lower_interval} [sec]\n" +
2623
+ " upper: #{upper_interval} [sec]")
2624
+ end
2625
+ assert_equal(limit, d.record_count - prev_count)
2626
+ emit_count = d.emit_count
2627
+ prev_count = d.record_count
2628
+ end
2629
+
2630
+ ## When all the lines are read and rate_period seconds are over
2631
+ ## limit will reset and since there are no more logs to be read,
2632
+ ## number_lines_read will be 0
2633
+ sleep upper_interval
2634
+ gw = d.instance.find_group_from_metadata(file_path)
2635
+ assert_equal(0, gw.current_paths[file_path].number_lines_read)
2636
+ end
2637
+ end
2362
2638
  end
2363
2639
  end