fluentd 1.12.0-x64-mingw32 → 1.13.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/workflows/linux-test.yaml +36 -0
  5. data/.github/workflows/macos-test.yaml +30 -0
  6. data/.github/workflows/windows-test.yaml +46 -0
  7. data/.gitlab-ci.yml +41 -19
  8. data/CHANGELOG.md +164 -2
  9. data/CONTRIBUTING.md +2 -2
  10. data/MAINTAINERS.md +5 -2
  11. data/README.md +7 -4
  12. data/example/counter.conf +1 -1
  13. data/fluentd.gemspec +5 -4
  14. data/lib/fluent/command/bundler_injection.rb +1 -1
  15. data/lib/fluent/command/ca_generate.rb +6 -3
  16. data/lib/fluent/command/cat.rb +19 -4
  17. data/lib/fluent/command/fluentd.rb +5 -2
  18. data/lib/fluent/command/plugin_config_formatter.rb +16 -1
  19. data/lib/fluent/command/plugin_generator.rb +31 -1
  20. data/lib/fluent/compat/parser.rb +2 -2
  21. data/lib/fluent/config/section.rb +2 -2
  22. data/lib/fluent/config/types.rb +2 -2
  23. data/lib/fluent/event.rb +3 -13
  24. data/lib/fluent/load.rb +0 -1
  25. data/lib/fluent/log.rb +1 -0
  26. data/lib/fluent/plugin/file_wrapper.rb +49 -4
  27. data/lib/fluent/plugin/formatter_ltsv.rb +2 -2
  28. data/lib/fluent/plugin/in_http.rb +11 -1
  29. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  30. data/lib/fluent/plugin/in_tail.rb +141 -41
  31. data/lib/fluent/plugin/in_tail/position_file.rb +15 -1
  32. data/lib/fluent/plugin/out_copy.rb +18 -5
  33. data/lib/fluent/plugin/out_exec_filter.rb +3 -3
  34. data/lib/fluent/plugin/out_forward.rb +74 -58
  35. data/lib/fluent/plugin/out_http.rb +9 -2
  36. data/lib/fluent/plugin/output.rb +11 -9
  37. data/lib/fluent/plugin/parser_csv.rb +2 -2
  38. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  39. data/lib/fluent/plugin/storage_local.rb +4 -4
  40. data/lib/fluent/plugin_helper/server.rb +4 -2
  41. data/lib/fluent/plugin_helper/service_discovery.rb +39 -1
  42. data/lib/fluent/plugin_helper/service_discovery/manager.rb +11 -5
  43. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  44. data/lib/fluent/supervisor.rb +28 -5
  45. data/lib/fluent/system_config.rb +16 -1
  46. data/lib/fluent/time.rb +57 -1
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
  49. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  50. data/test/command/test_cat.rb +96 -0
  51. data/test/command/test_fluentd.rb +38 -0
  52. data/test/command/test_plugin_config_formatter.rb +67 -0
  53. data/test/config/test_configurable.rb +1 -1
  54. data/test/config/test_system_config.rb +46 -0
  55. data/test/plugin/in_tail/test_io_handler.rb +4 -4
  56. data/test/plugin/in_tail/test_position_file.rb +59 -5
  57. data/test/plugin/test_file_wrapper.rb +115 -0
  58. data/test/plugin/test_in_exec.rb +1 -1
  59. data/test/plugin/test_in_http.rb +15 -0
  60. data/test/plugin/test_in_tail.rb +309 -26
  61. data/test/plugin/test_out_copy.rb +87 -0
  62. data/test/plugin/test_out_forward.rb +104 -11
  63. data/test/plugin/test_out_http.rb +20 -1
  64. data/test/plugin/test_output.rb +15 -3
  65. data/test/plugin/test_output_as_buffered_backup.rb +2 -0
  66. data/test/plugin/test_parser_csv.rb +14 -0
  67. data/test/plugin/test_parser_syslog.rb +14 -0
  68. data/test/plugin/test_sd_file.rb +1 -1
  69. data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
  70. data/test/plugin_helper/test_child_process.rb +5 -2
  71. data/test/plugin_helper/test_http_server_helper.rb +4 -2
  72. data/test/plugin_helper/test_server.rb +26 -7
  73. data/test/plugin_helper/test_service_discovery.rb +74 -14
  74. data/test/test_config.rb +2 -1
  75. data/test/test_event.rb +16 -0
  76. data/test/test_formatter.rb +30 -0
  77. data/test/test_output.rb +2 -2
  78. data/test/test_supervisor.rb +66 -0
  79. data/test/test_time_parser.rb +109 -0
  80. metadata +58 -31
  81. data/.travis.yml +0 -77
  82. data/appveyor.yml +0 -31
@@ -225,6 +225,30 @@ TEXT
225
225
  path to something
226
226
 
227
227
 
228
+ TEXT
229
+ assert_equal(expected, dumped_config)
230
+ end
231
+
232
+ test "input simple (table)" do
233
+ dumped_config = capture_stdout do
234
+ FluentPluginConfigFormatter.new(["--format=markdown", "--table", "input", "simple"]).call
235
+ end
236
+ expected = <<TEXT
237
+ ## Plugin helpers
238
+
239
+ * [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
240
+ * [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
241
+
242
+ * See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
243
+
244
+ ## TestFluentPluginConfigFormatter::SimpleInput
245
+
246
+ ### Configuration
247
+
248
+ |parameter|type|description|default|
249
+ |---|---|---|---|
250
+ |path|string (required)|path to something||
251
+
228
252
  TEXT
229
253
  assert_equal(expected, dumped_config)
230
254
  end
@@ -298,6 +322,49 @@ Default value: `normal`.
298
322
 
299
323
 
300
324
 
325
+ TEXT
326
+ assert_equal(expected, dumped_config)
327
+ end
328
+
329
+ test "output complex (table)" do
330
+ dumped_config = capture_stdout do
331
+ FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call
332
+ end
333
+ expected = <<TEXT
334
+ ## Plugin helpers
335
+
336
+ * [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
337
+ * [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
338
+
339
+ * See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
340
+
341
+ ## TestFluentPluginConfigFormatter::ComplexOutput
342
+
343
+
344
+ ### \\<authentication\\> section (required) (single)
345
+
346
+ ### Configuration
347
+
348
+ |parameter|type|description|default|
349
+ |---|---|---|---|
350
+ |username|string (required)|username||
351
+ |password|string (required)|password||
352
+
353
+
354
+ ### \\<parent\\> section (optional) (multiple)
355
+
356
+
357
+ #### \\<child\\> section (optional) (multiple)
358
+
359
+ ### Configuration
360
+
361
+ |parameter|type|description|default|
362
+ |---|---|---|---|
363
+ |names|array (required)|names||
364
+ |difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|
365
+
366
+
367
+
301
368
  TEXT
302
369
  assert_equal(expected, dumped_config)
303
370
  end
@@ -1453,7 +1453,7 @@ module Fluent::Config
1453
1453
  @example = ConfigurableSpec::ExampleWithSkipAccessor.new
1454
1454
  @example.configure(config_element('ROOT'))
1455
1455
  assert_equal 'example7', @example.instance_variable_get(:@name)
1456
- assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
1456
+ assert_raise NoMethodError do
1457
1457
  @example.name
1458
1458
  end
1459
1459
  end
@@ -143,5 +143,51 @@ module Fluent::Config
143
143
  sc.overwrite_variables(**s.for_system_config)
144
144
  assert_equal(level, sc.log_level)
145
145
  end
146
+
147
+ sub_test_case "log rotation" do
148
+ data('daily' => "daily",
149
+ 'weekly' => 'weekly',
150
+ 'monthly' => 'monthly')
151
+ test "symbols for rotate_age" do |age|
152
+ conf = parse_text(<<-EOS)
153
+ <system>
154
+ <log>
155
+ rotate_age #{age}
156
+ </log>
157
+ </system>
158
+ EOS
159
+ sc = Fluent::SystemConfig.new(conf)
160
+ assert_equal(age.to_sym, sc.log.rotate_age)
161
+ end
162
+
163
+ test "numeric number for rotate age" do
164
+ conf = parse_text(<<-EOS)
165
+ <system>
166
+ <log>
167
+ rotate_age 3
168
+ </log>
169
+ </system>
170
+ EOS
171
+ s = FakeSupervisor.new
172
+ sc = Fluent::SystemConfig.new(conf)
173
+ assert_equal(3, sc.log.rotate_age)
174
+ end
175
+
176
+ data(h: ['100', 100],
177
+ k: ['1k', 1024],
178
+ m: ['1m', 1024 * 1024],
179
+ g: ['1g', 1024 * 1024 * 1024])
180
+ test "numeric and SI prefix for rotate_size" do |(label, size)|
181
+ conf = parse_text(<<-EOS)
182
+ <system>
183
+ <log>
184
+ rotate_size #{label}
185
+ </log>
186
+ </system>
187
+ EOS
188
+ sc = Fluent::SystemConfig.new(conf)
189
+ assert_equal(size, sc.log.rotate_size)
190
+ end
191
+ end
146
192
  end
147
193
  end
@@ -30,7 +30,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
30
30
  end
31
31
 
32
32
  returned_lines = ''
33
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
33
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
34
34
  returned_lines << lines.join
35
35
  true
36
36
  end
@@ -62,7 +62,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
62
62
  end
63
63
 
64
64
  returned_lines = ''
65
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
65
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
66
66
  returned_lines << lines.join
67
67
  true
68
68
  end
@@ -93,7 +93,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
93
93
  end
94
94
 
95
95
  returned_lines = []
96
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
96
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
97
97
  returned_lines << lines.dup
98
98
  true
99
99
  end
@@ -119,7 +119,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
119
119
  end
120
120
 
121
121
  returned_lines = []
122
- r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
122
+ r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
123
123
  returned_lines << lines.dup
124
124
  true
125
125
  end
@@ -85,7 +85,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
85
85
  assert_equal 5, lines.size
86
86
  end
87
87
 
88
- test 'update seek postion of remained position entry' do
88
+ test 'update seek position of remained position entry' do
89
89
  pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
90
90
  target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
91
91
  target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
@@ -147,7 +147,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
147
147
  write_data(@file, TEST_CONTENT)
148
148
  pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
149
149
 
150
- valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
150
+ valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
151
151
  f = pf[valid_target_info]
152
152
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
153
153
  assert_equal 2, f.read_pos
@@ -177,7 +177,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
177
177
  assert_equal 0, f.read_inode
178
178
  assert_equal 0, f.read_pos
179
179
 
180
- pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
180
+ pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
181
181
 
182
182
  f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
183
183
  assert_equal 0, f.read_inode
@@ -193,7 +193,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
193
193
  test 'deletes entry by path' do
194
194
  write_data(@file, TEST_CONTENT)
195
195
  pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
196
- inode1 = Fluent::FileWrapper.stat(@file).ino
196
+ inode1 = File.stat(@file).ino
197
197
  target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
198
198
  p1 = pf[target_info1]
199
199
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
@@ -201,7 +201,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
201
201
  pf.unwatch(target_info1)
202
202
  assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
203
203
 
204
- inode2 = Fluent::FileWrapper.stat(@file).ino
204
+ inode2 = File.stat(@file).ino
205
205
  target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
206
206
  p2 = pf[target_info2]
207
207
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
@@ -282,4 +282,58 @@ class IntailPositionFileTest < Test::Unit::TestCase
282
282
  assert_equal 2, f.read_inode
283
283
  end
284
284
  end
285
+
286
+ sub_test_case "TargetInfo equality rules" do
287
+ sub_test_case "== operator" do
288
+ def test_equal
289
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
290
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
291
+
292
+ assert_equal t1, t2
293
+ end
294
+
295
+ def test_not_equal
296
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
297
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
298
+
299
+ assert_not_equal t1, t2
300
+ end
301
+ end
302
+
303
+ sub_test_case "eql? method" do
304
+ def test_eql?
305
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
306
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
307
+
308
+ assert do
309
+ t1.eql? t2
310
+ end
311
+ end
312
+
313
+ def test_not_eql?
314
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
315
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
316
+
317
+ assert do
318
+ !t1.eql? t2
319
+ end
320
+ end
321
+ end
322
+
323
+ sub_test_case "hash" do
324
+ def test_equal
325
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
326
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
327
+
328
+ assert_equal t1.hash, t2.hash
329
+ end
330
+
331
+ def test_not_equal
332
+ t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
333
+ t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
334
+
335
+ assert_not_equal t1.hash, t2.hash
336
+ end
337
+ end
338
+ end
285
339
  end
@@ -0,0 +1,115 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/file_wrapper'
3
+
4
+ class FileWrapperTest < Test::Unit::TestCase
5
+ require 'windows/file'
6
+ require 'windows/error'
7
+ include Windows::File
8
+ include Windows::Error
9
+
10
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
11
+
12
+ def setup
13
+ FileUtils.mkdir_p(TMP_DIR)
14
+ end
15
+
16
+ def teardown
17
+ FileUtils.rm_rf(TMP_DIR)
18
+ end
19
+
20
+ sub_test_case 'Win32Error' do
21
+ test 'equal' do
22
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
23
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
24
+ end
25
+
26
+ test 'different error code' do
27
+ assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
28
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
29
+ end
30
+
31
+ test 'different error message' do
32
+ assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
33
+ Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
34
+ end
35
+
36
+ test 'different class' do
37
+ assert_not_equal(Errno::EPIPE,
38
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
39
+ end
40
+
41
+ test 'ERROR_SHARING_VIOLATION message' do
42
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
43
+ "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
44
+ end
45
+
46
+ test 'ERROR_SHARING_VIOLATION with a message' do
47
+ assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
48
+ "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
49
+ " - cannot open the file")
50
+ end
51
+
52
+ test 'to_s' do
53
+ assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
54
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
55
+ end
56
+
57
+ test 'inspect' do
58
+ assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
59
+ Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
60
+ end
61
+
62
+ data('0' => [false, 0],
63
+ '9999' => [false, 9999],
64
+ '10000' => [true, 10000],
65
+ '10001' => [true, 10001])
66
+ test 'wsaerr?' do |data|
67
+ expected, code = data
68
+ assert_equal(expected, Fluent::Win32Error.new(code).wsaerr?)
69
+ end
70
+ end
71
+
72
+ sub_test_case 'WindowsFile exceptions' do
73
+ test 'nothing raised' do
74
+ begin
75
+ path = "#{TMP_DIR}/test_windows_file.txt"
76
+ file1 = file2 = nil
77
+ file1 = File.open(path, "wb") do |f|
78
+ end
79
+ assert_nothing_raised do
80
+ file2 = Fluent::WindowsFile.new(path)
81
+ ensure
82
+ file2.close
83
+ end
84
+ ensure
85
+ file1.close if file1
86
+ end
87
+ end
88
+
89
+ test 'Errno::ENOENT raised' do
90
+ path = "#{TMP_DIR}/nofile.txt"
91
+ file = nil
92
+ assert_raise(Errno::ENOENT) do
93
+ file = Fluent::WindowsFile.new(path)
94
+ ensure
95
+ file.close if file
96
+ end
97
+ end
98
+
99
+ test 'ERROR_SHARING_VIOLATION raised' do
100
+ begin
101
+ path = "#{TMP_DIR}/test_windows_file.txt"
102
+ file1 = file2 = nil
103
+ file1 = File.open(path, "wb")
104
+ win32err = Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)
105
+ assert_raise(Errno::EACCES.new(win32err.message)) do
106
+ file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
107
+ ensure
108
+ file2.close if file2
109
+ end
110
+ ensure
111
+ file1.close if file1
112
+ end
113
+ end
114
+ end
115
+ end if Fluent.windows?
@@ -255,7 +255,7 @@ EOC
255
255
  assert{ d.events.length > 0 }
256
256
  d.events.each do |event|
257
257
  assert_equal 'test', event[0]
258
- assert_match /LoadError/, event[2]['message']
258
+ assert_match(/LoadError/, event[2]['message'])
259
259
  end
260
260
  end
261
261
  end
@@ -770,6 +770,15 @@ class HttpInputTest < Test::Unit::TestCase
770
770
  end
771
771
  end
772
772
 
773
+ def test_get_request
774
+ d = create_driver(CONFIG)
775
+
776
+ d.run do
777
+ res = get("/cors.test", {}, {})
778
+ assert_equal "200", res.code
779
+ end
780
+ end
781
+
773
782
  def test_cors_preflight
774
783
  d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
775
784
 
@@ -985,6 +994,12 @@ class HttpInputTest < Test::Unit::TestCase
985
994
  assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
986
995
  end
987
996
 
997
+ def get(path, params, header = {})
998
+ http = Net::HTTP.new("127.0.0.1", PORT)
999
+ req = Net::HTTP::Get.new(path, header)
1000
+ http.request(req)
1001
+ end
1002
+
988
1003
  def options(path, params, header = {})
989
1004
  http = Net::HTTP.new("127.0.0.1", PORT)
990
1005
  req = Net::HTTP::Options.new(path, header)
@@ -6,6 +6,8 @@ require 'fluent/system_config'
6
6
  require 'net/http'
7
7
  require 'flexmock/test_unit'
8
8
  require 'timecop'
9
+ require 'tmpdir'
10
+ require 'securerandom'
9
11
 
10
12
  class TailInputTest < Test::Unit::TestCase
11
13
  include FlexMock::TestCase
@@ -22,28 +24,64 @@ class TailInputTest < Test::Unit::TestCase
22
24
  end
23
25
 
24
26
  def cleanup_directory(path)
25
- begin
26
- FileUtils.rm_f(path, secure: true)
27
- rescue ArgumentError
28
- FileUtils.rm_f(path) # For Ruby 2.6 or before.
27
+ unless Dir.exist?(path)
28
+ FileUtils.mkdir_p(path)
29
+ return
29
30
  end
30
- if File.exist?(path)
31
- FileUtils.remove_entry_secure(path, true)
31
+
32
+ if Fluent.windows?
33
+ Dir.glob("*", base: path).each do |name|
34
+ begin
35
+ cleanup_file(File.join(path, name))
36
+ rescue
37
+ # expect test driver block release already owned file handle.
38
+ end
39
+ end
40
+ else
41
+ begin
42
+ FileUtils.rm_f(path, secure:true)
43
+ rescue ArgumentError
44
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
45
+ end
46
+ if File.exist?(path)
47
+ FileUtils.remove_entry_secure(path, true)
48
+ end
32
49
  end
33
50
  FileUtils.mkdir_p(path)
34
51
  end
35
52
 
36
53
  def cleanup_file(path)
37
- begin
38
- FileUtils.rm_f(path, secure: true)
39
- rescue ArgumentError
40
- FileUtils.rm_f(path) # For Ruby 2.6 or before.
41
- end
42
- if File.exist?(path)
43
- # ensure files are closed for Windows, on which deleted files
44
- # are still visible from filesystem
45
- GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
46
- FileUtils.remove_entry_secure(path, true)
54
+ if Fluent.windows?
55
+ # On Windows, when the file or directory is removed and created
56
+ # frequently, there is a case that creating file or directory will
57
+ # fail. This situation is caused by pending file or directory
58
+ # deletion which is mentioned on win32 API document [1]
59
+ # As a workaround, execute rename and remove method.
60
+ #
61
+ # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
62
+ #
63
+ file = File.join(Dir.tmpdir, SecureRandom.hex(10))
64
+ begin
65
+ FileUtils.mv(path, file)
66
+ FileUtils.rm_rf(file, secure: true)
67
+ rescue ArgumentError
68
+ FileUtils.rm_rf(file) # For Ruby 2.6 or before.
69
+ end
70
+ if File.exist?(file)
71
+ # ensure files are closed for Windows, on which deleted files
72
+ # are still visible from filesystem
73
+ GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
74
+ FileUtils.remove_entry_secure(file, true)
75
+ end
76
+ else
77
+ begin
78
+ FileUtils.rm_f(path, secure: true)
79
+ rescue ArgumentError
80
+ FileUtils.rm_f(path) # For Ruby 2.6 or before.
81
+ end
82
+ if File.exist?(path)
83
+ FileUtils.remove_entry_secure(path, true)
84
+ end
47
85
  end
48
86
  end
49
87
 
@@ -118,6 +156,7 @@ class TailInputTest < Test::Unit::TestCase
118
156
  assert_equal 2, d.instance.rotate_wait
119
157
  assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
120
158
  assert_equal 1000, d.instance.read_lines_limit
159
+ assert_equal -1, d.instance.read_bytes_limit_per_second
121
160
  assert_equal false, d.instance.ignore_repeated_permission_error
122
161
  assert_nothing_raised do
123
162
  d.instance.have_read_capability?
@@ -157,6 +196,22 @@ class TailInputTest < Test::Unit::TestCase
157
196
  end
158
197
  end
159
198
 
199
+ sub_test_case "log throttling per file" do
200
+ test "w/o watcher timer is invalid" do
201
+ conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
202
+ assert_raise(Fluent::ConfigError) do
203
+ create_driver(conf)
204
+ end
205
+ end
206
+
207
+ test "valid" do
208
+ conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
209
+ assert_raise(Fluent::ConfigError) do
210
+ create_driver(conf)
211
+ end
212
+ end
213
+ end
214
+
160
215
  test "both enable_watch_timer and enable_stat_watcher are false" do
161
216
  assert_raise(Fluent::ConfigError) do
162
217
  create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -285,6 +340,138 @@ class TailInputTest < Test::Unit::TestCase
285
340
  assert num_events <= d.emit_count
286
341
  end
287
342
 
343
+ sub_test_case "log throttling per file" do
344
+ teardown do
345
+ cleanup_file("#{TMP_DIR}/tail.txt")
346
+ end
347
+
348
+ sub_test_case "reads_bytes_per_second w/o throttled" do
349
+ data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2, false],
350
+ "flat 8192 bytes, 2 events already read limit reached" => [:flat, 100, 8192, 2, true],
351
+ "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2, false],
352
+ "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
353
+ "flat #{8192*10} bytes, 20 events w/o stat watcher" => [:flat_without_stat, 100, (8192 * 10), 20],
354
+ "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
355
+ "parse #{8192*4} bytes, 8 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 4), 8],
356
+ "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
357
+ "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
358
+ "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2],
359
+ "flat 8k bytes with unit, 2 events w/o stat watcher" => [:flat_without_stat, 100, "8k", 2],
360
+ "flat #{8*10}k bytes with unit, 20 events" => [:flat, 100, "#{8*10}k", 20],
361
+ "flat #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:flat_without_stat, 100, "#{8*10}k", 20],
362
+ "parse #{8*4}k bytes with unit, 8 events" => [:parse, 100, "#{8*4}k", 8],
363
+ "parse #{8*4}k bytes with unit, 8 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*4}k", 8],
364
+ "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20],
365
+ "parse #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*10}k", 20])
366
+ def test_emit_with_read_bytes_limit_per_second(data)
367
+ config_style, limit, limit_bytes, num_events = data
368
+ case config_style
369
+ when :flat
370
+ config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
371
+ when :parse
372
+ config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
373
+ when :flat_without_stat
374
+ config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
375
+ when :parse_without_stat
376
+ config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
377
+ end
378
+
379
+ msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
380
+ start_time = Fluent::Clock.now
381
+
382
+ d = create_driver(config)
383
+ d.run(expect_emits: 2) do
384
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
385
+ 100.times do
386
+ f.puts msg
387
+ end
388
+ }
389
+ end
390
+
391
+ assert_true(Fluent::Clock.now - start_time > 1)
392
+ assert_equal(num_events.times.map { {"message" => msg} },
393
+ d.events.collect { |event| event[2] })
394
+ end
395
+
396
+ def test_read_bytes_limit_precede_read_lines_limit
397
+ config = CONFIG_READ_FROM_HEAD +
398
+ SINGLE_LINE_CONFIG +
399
+ config_element("", "", {
400
+ "read_lines_limit" => 1000,
401
+ "read_bytes_limit_per_second" => 8192
402
+ })
403
+ msg = 'abc'
404
+ start_time = Fluent::Clock.now
405
+ d = create_driver(config)
406
+ d.run(expect_emits: 2) do
407
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
408
+ 8000.times do
409
+ f.puts msg
410
+ end
411
+ }
412
+ end
413
+
414
+ assert_true(Fluent::Clock.now - start_time > 1)
415
+ assert_equal(4096.times.map { {"message" => msg} },
416
+ d.events.collect { |event| event[2] })
417
+ end
418
+ end
419
+
420
+ sub_test_case "reads_bytes_per_second w/ throttled already" do
421
+ data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
422
+ "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
423
+ "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
424
+ "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
425
+ "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2],
426
+ "flat #{8*10}k bytes with unit, 20 events" => [:flat, 100, "#{8*10}k", 20],
427
+ "parse #{8*4}k bytes with unit, 8 events" => [:parse, 100, "#{8*4}k", 8],
428
+ "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20])
429
+ def test_emit_with_read_bytes_limit_per_second(data)
430
+ config_style, limit, limit_bytes, num_events = data
431
+ case config_style
432
+ when :flat
433
+ config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
434
+ when :parse
435
+ config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
436
+ end
437
+ d = create_driver(config)
438
+ msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
439
+
440
+ mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
441
+ require 'fluent/config/types'
442
+ limit_bytes_value = Fluent::Config.size_value(limit_bytes)
443
+ io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
444
+ if Fluent.linux?
445
+ mock.proxy(io_handler).handle_notify.at_least(5)
446
+ else
447
+ mock.proxy(io_handler).handle_notify.twice
448
+ end
449
+ io_handler
450
+ end
451
+
452
+ File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
453
+ 100.times do
454
+ f.puts msg
455
+ end
456
+ end
457
+
458
+ # We should not do shutdown here due to hard timeout.
459
+ d.run do
460
+ start_time = Fluent::Clock.now
461
+ while Fluent::Clock.now - start_time < 0.8 do
462
+ File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
463
+ f.puts msg
464
+ f.flush
465
+ end
466
+ sleep 0.05
467
+ end
468
+ end
469
+
470
+ assert_equal([], d.events)
471
+ end
472
+ end
473
+ end
474
+
288
475
  data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
289
476
  parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
290
477
  def test_emit_with_read_from_head(data)
@@ -551,6 +738,7 @@ class TailInputTest < Test::Unit::TestCase
551
738
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
552
739
  f.puts "test1"
553
740
  f.puts "test2"
741
+ f.flush
554
742
  }
555
743
 
556
744
  d = create_driver(config)
@@ -560,19 +748,23 @@ class TailInputTest < Test::Unit::TestCase
560
748
  f.puts "test3\ntest4"
561
749
  f.flush
562
750
  }
563
- sleep 1
751
+ waiting(2) { sleep 0.1 until d.events.length == 2 }
564
752
  File.truncate("#{TMP_DIR}/tail.txt", 6)
565
753
  end
566
754
 
567
- events = d.events
568
- assert_equal(3, events.length)
569
- assert_equal({"message" => "test3"}, events[0][2])
570
- assert_equal({"message" => "test4"}, events[1][2])
571
- assert_equal({"message" => "test1"}, events[2][2])
572
- assert(events[0][1].is_a?(Fluent::EventTime))
573
- assert(events[1][1].is_a?(Fluent::EventTime))
574
- assert(events[2][1].is_a?(Fluent::EventTime))
575
- assert_equal(2, d.emit_count)
755
+ expected = {
756
+ emit_count: 2,
757
+ events: [
758
+ [Fluent::EventTime, {"message" => "test3"}],
759
+ [Fluent::EventTime, {"message" => "test4"}],
760
+ [Fluent::EventTime, {"message" => "test1"}],
761
+ ]
762
+ }
763
+ actual = {
764
+ emit_count: d.emit_count,
765
+ events: d.events.collect{|event| [event[1].class, event[2]]}
766
+ }
767
+ assert_equal(expected, actual)
576
768
  end
577
769
 
578
770
  def test_move_truncate_move_back
@@ -1876,4 +2068,95 @@ class TailInputTest < Test::Unit::TestCase
1876
2068
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
1877
2069
  d.instance_shutdown
1878
2070
  end
2071
+
2072
+ def test_ENOENT_error_after_setup_watcher
2073
+ path = "#{TMP_DIR}/tail.txt"
2074
+ FileUtils.touch(path)
2075
+ config = config_element('', '', {
2076
+ 'format' => 'none',
2077
+ })
2078
+ d = create_driver(config)
2079
+ mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
2080
+ cleanup_file(path)
2081
+ tw
2082
+ end
2083
+ assert_nothing_raised do
2084
+ d.run(shutdown: false) {}
2085
+ end
2086
+ d.instance_shutdown
2087
+ assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2088
+ end
2089
+
2090
+ def test_EACCES_error_after_setup_watcher
2091
+ path = "#{TMP_DIR}/noaccess/tail.txt"
2092
+ begin
2093
+ FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
2094
+ FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2095
+ FileUtils.touch(path)
2096
+ config = config_element('', '', {
2097
+ 'tag' => "tail",
2098
+ 'path' => path,
2099
+ 'format' => 'none',
2100
+ })
2101
+ d = create_driver(config, false)
2102
+ mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
2103
+ FileUtils.chmod(0000, "#{TMP_DIR}/noaccess")
2104
+ tw
2105
+ end
2106
+ assert_nothing_raised do
2107
+ d.run(shutdown: false) {}
2108
+ end
2109
+ d.instance_shutdown
2110
+ assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2111
+ end
2112
+ ensure
2113
+ FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2114
+ FileUtils.rm_rf("#{TMP_DIR}/noaccess")
2115
+ end unless Fluent.windows?
2116
+
2117
+ def test_EACCES
2118
+ path = "#{TMP_DIR}/tail.txt"
2119
+ FileUtils.touch(path)
2120
+ config = config_element('', '', {
2121
+ 'format' => 'none',
2122
+ })
2123
+ d = create_driver(config)
2124
+ mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
2125
+ raise Errno::EACCES
2126
+ end.at_least(1)
2127
+ assert_nothing_raised do
2128
+ d.run(shutdown: false) {}
2129
+ end
2130
+ d.instance_shutdown
2131
+ assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2132
+ end
2133
+
2134
+ def test_shutdown_timeout
2135
+ path = "#{TMP_DIR}/tail.txt"
2136
+ File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
2137
+ (1024 * 1024 * 5).times do
2138
+ f.puts "{\"test\":\"fizzbuzz\"}"
2139
+ end
2140
+ end
2141
+
2142
+ config =
2143
+ CONFIG_READ_FROM_HEAD +
2144
+ config_element('', '', {
2145
+ 'format' => 'json',
2146
+ 'skip_refresh_on_startup' => true,
2147
+ })
2148
+ d = create_driver(config)
2149
+ mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
2150
+ io_handler.shutdown_timeout = 0.5
2151
+ io_handler
2152
+ end
2153
+
2154
+ start_time = Fluent::Clock.now
2155
+ assert_nothing_raised do
2156
+ d.run(expect_emits: 1)
2157
+ end
2158
+
2159
+ elapsed = Fluent::Clock.now - start_time
2160
+ assert_true(elapsed > 0.5 && elapsed < 2.5)
2161
+ end
1879
2162
  end