fluentd 1.12.1-x64-mingw32 → 1.13.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/workflows/linux-test.yaml +36 -0
  5. data/.github/workflows/{build.yaml → macos-test.yaml} +8 -7
  6. data/.github/workflows/windows-test.yaml +46 -0
  7. data/.gitlab-ci.yml +19 -19
  8. data/CHANGELOG.md +163 -0
  9. data/CONTRIBUTING.md +2 -2
  10. data/MAINTAINERS.md +5 -2
  11. data/README.md +6 -3
  12. data/example/counter.conf +1 -1
  13. data/fluentd.gemspec +4 -2
  14. data/lib/fluent/command/bundler_injection.rb +1 -1
  15. data/lib/fluent/command/cat.rb +19 -4
  16. data/lib/fluent/command/fluentd.rb +1 -2
  17. data/lib/fluent/command/plugin_config_formatter.rb +2 -1
  18. data/lib/fluent/command/plugin_generator.rb +31 -1
  19. data/lib/fluent/compat/parser.rb +2 -2
  20. data/lib/fluent/config/section.rb +6 -1
  21. data/lib/fluent/config/types.rb +2 -2
  22. data/lib/fluent/event.rb +3 -13
  23. data/lib/fluent/load.rb +0 -1
  24. data/lib/fluent/log.rb +1 -0
  25. data/lib/fluent/plugin/file_wrapper.rb +49 -4
  26. data/lib/fluent/plugin/formatter_ltsv.rb +2 -2
  27. data/lib/fluent/plugin/in_http.rb +11 -1
  28. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  29. data/lib/fluent/plugin/in_tail.rb +140 -41
  30. data/lib/fluent/plugin/in_tail/position_file.rb +15 -1
  31. data/lib/fluent/plugin/out_copy.rb +18 -5
  32. data/lib/fluent/plugin/out_exec_filter.rb +3 -3
  33. data/lib/fluent/plugin/out_forward.rb +75 -61
  34. data/lib/fluent/plugin/output.rb +11 -9
  35. data/lib/fluent/plugin/parser_csv.rb +2 -2
  36. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  37. data/lib/fluent/plugin/service_discovery.rb +0 -15
  38. data/lib/fluent/plugin/storage_local.rb +4 -4
  39. data/lib/fluent/plugin_helper/http_server/router.rb +1 -1
  40. data/lib/fluent/plugin_helper/server.rb +4 -2
  41. data/lib/fluent/plugin_helper/service_discovery.rb +39 -1
  42. data/lib/fluent/plugin_helper/service_discovery/manager.rb +11 -5
  43. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  44. data/lib/fluent/supervisor.rb +16 -1
  45. data/lib/fluent/system_config.rb +14 -0
  46. data/lib/fluent/time.rb +57 -1
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
  49. data/test/command/test_cat.rb +99 -0
  50. data/test/command/test_fluentd.rb +8 -0
  51. data/test/config/test_configurable.rb +1 -1
  52. data/test/config/test_section.rb +9 -0
  53. data/test/config/test_system_config.rb +46 -0
  54. data/test/plugin/in_tail/test_io_handler.rb +4 -4
  55. data/test/plugin/in_tail/test_position_file.rb +58 -4
  56. data/test/plugin/test_file_wrapper.rb +115 -0
  57. data/test/plugin/test_in_exec.rb +1 -1
  58. data/test/plugin/test_in_forward.rb +59 -83
  59. data/test/plugin/test_in_http.rb +58 -40
  60. data/test/plugin/test_in_syslog.rb +66 -56
  61. data/test/plugin/test_in_tail.rb +298 -26
  62. data/test/plugin/test_in_tcp.rb +45 -32
  63. data/test/plugin/test_in_udp.rb +47 -33
  64. data/test/plugin/test_out_copy.rb +87 -0
  65. data/test/plugin/test_out_forward.rb +198 -91
  66. data/test/plugin/test_out_http.rb +1 -1
  67. data/test/plugin/test_out_stream.rb +18 -8
  68. data/test/plugin/test_output.rb +15 -3
  69. data/test/plugin/test_output_as_buffered_backup.rb +2 -0
  70. data/test/plugin/test_parser_csv.rb +14 -0
  71. data/test/plugin/test_parser_syslog.rb +14 -0
  72. data/test/plugin_helper/http_server/test_route.rb +1 -1
  73. data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
  74. data/test/plugin_helper/test_child_process.rb +6 -3
  75. data/test/plugin_helper/test_http_server_helper.rb +34 -27
  76. data/test/plugin_helper/test_server.rb +144 -139
  77. data/test/plugin_helper/test_service_discovery.rb +74 -14
  78. data/test/plugin_helper/test_socket.rb +16 -9
  79. data/test/test_config.rb +2 -1
  80. data/test/test_event.rb +16 -0
  81. data/test/test_formatter.rb +30 -0
  82. data/test/test_output.rb +2 -2
  83. data/test/test_supervisor.rb +35 -0
  84. data/test/test_time_parser.rb +109 -0
  85. metadata +55 -11
  86. data/.travis.yml +0 -77
  87. data/appveyor.yml +0 -31
data/README.md CHANGED
@@ -1,8 +1,11 @@
1
1
  Fluentd: Open-Source Log Collector
2
2
  ===================================
3
- Travis CI:
3
+ GitHub Actions:
4
4
 
5
- [<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
5
+ [![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
6
+ [![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
7
+ [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
8
+ [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
6
9
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
7
10
  [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
8
11
 
@@ -74,7 +77,7 @@ You can run specified test via `TEST` environment variable:
74
77
  - Website: https://www.fluentd.org/
75
78
  - Documentation: https://docs.fluentd.org/
76
79
  - Project repository: https://github.com/fluent
77
- - Discussion: https://groups.google.com/group/fluentd
80
+ - Discussion: https://discuss.fluentd.org/
78
81
  - Slack / Community: https://slack.fluentd.org
79
82
  - Newsletters: https://www.fluentd.org/newsletter
80
83
  - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
data/example/counter.conf CHANGED
@@ -3,7 +3,7 @@
3
3
  scope server1
4
4
  bind 127.0.0.1
5
5
  port 24321
6
- path tmp/back
6
+ backup_path tmp/back
7
7
  </counter_server>
8
8
  </system>
9
9
 
data/fluentd.gemspec CHANGED
@@ -28,12 +28,14 @@ Gem::Specification.new do |gem|
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
30
  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
31
+ gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
31
32
 
32
33
  # build gem for a certain platform. see also Rakefile
33
34
  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
34
35
  gem.platform = fake_platform unless fake_platform.empty?
35
36
  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
36
- gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
37
+ gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
38
+ gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
37
39
  gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
38
40
  gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
39
41
  gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -44,7 +46,7 @@ Gem::Specification.new do |gem|
44
46
  gem.add_development_dependency("flexmock", ["~> 2.0"])
45
47
  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
46
48
  gem.add_development_dependency("simplecov", ["~> 0.7"])
47
- gem.add_development_dependency("rr", ["~> 1.0"])
49
+ gem.add_development_dependency("rr", ["~> 3.0"])
48
50
  gem.add_development_dependency("timecop", ["~> 0.9"])
49
51
  gem.add_development_dependency("test-unit", ["~> 3.3"])
50
52
  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
@@ -40,6 +40,6 @@ else
40
40
  File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
41
41
  ] + ARGV
42
42
 
43
- exec *cmdline
43
+ exec(*cmdline)
44
44
  exit! 127
45
45
  end
@@ -101,7 +101,6 @@ rescue
101
101
  usage $!.to_s
102
102
  end
103
103
 
104
- require 'thread'
105
104
  require 'socket'
106
105
  require 'yajl'
107
106
  require 'msgpack'
@@ -153,14 +152,30 @@ class Writer
153
152
  super()
154
153
  end
155
154
 
155
+ def secondary_record?(record)
156
+ record.class != Hash &&
157
+ record.size == 2 &&
158
+ record.first.class == Fluent::EventTime &&
159
+ record.last.class == Hash
160
+ end
161
+
156
162
  def write(record)
157
- if record.class != Hash
158
- raise ArgumentError, "Input must be a map (got #{record.class})"
163
+ unless secondary_record?(record)
164
+ if record.class != Hash
165
+ raise ArgumentError, "Input must be a map (got #{record.class})"
166
+ end
159
167
  end
160
168
 
161
169
  time = Fluent::EventTime.now
162
170
  time = time.to_i if @time_as_integer
163
- entry = [time, record]
171
+ entry = if secondary_record?(record)
172
+ # Even though secondary contains Fluent::EventTime in record,
173
+ # fluent-cat just ignore it and set Fluent::EventTime.now instead.
174
+ # This specification is adopted to keep consistency.
175
+ [time, record.last]
176
+ else
177
+ [time, record]
178
+ end
164
179
  synchronize {
165
180
  unless write_impl([entry])
166
181
  # write failed
@@ -85,9 +85,8 @@ op.on('-o', '--log PATH', "log file path") {|s|
85
85
  opts[:log_path] = s
86
86
  }
87
87
 
88
- ROTATE_AGE = %w(daily weekly monthly)
89
88
  op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
90
- if ROTATE_AGE.include?(age)
89
+ if Fluent::Log::LOG_ROTATE_AGE.include?(age)
91
90
  opts[:log_rotate_age] = age
92
91
  else
93
92
  begin
@@ -44,6 +44,7 @@ class FluentPluginConfigFormatter
44
44
  @verbose = false
45
45
  @libs = []
46
46
  @plugin_dirs = []
47
+ @table = false
47
48
  @options = {}
48
49
 
49
50
  prepare_option_parser
@@ -162,7 +163,7 @@ class FluentPluginConfigFormatter
162
163
  else
163
164
  sections, params = base_section.partition {|_name, value| value[:section] }
164
165
  end
165
- if @table and not params.empty?
166
+ if @table && (not params.empty?)
166
167
  dumped << "### Configuration\n\n"
167
168
  dumped << "|parameter|type|description|default|\n"
168
169
  dumped << "|---|---|---|---|\n"
@@ -105,7 +105,7 @@ Generate a project skeleton for creating a Fluentd plugin
105
105
 
106
106
  Arguments:
107
107
  \ttype: #{SUPPORTED_TYPES.join(",")}
108
- \tname: Your plugin name
108
+ \tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
109
109
 
110
110
  Options:
111
111
  BANNER
@@ -151,6 +151,36 @@ BANNER
151
151
  underscore_name
152
152
  end
153
153
 
154
+ def gem_file_path
155
+ File.expand_path(File.join(File.dirname(__FILE__),
156
+ "../../../",
157
+ "Gemfile"))
158
+ end
159
+
160
+ def lock_file_path
161
+ File.expand_path(File.join(File.dirname(__FILE__),
162
+ "../../../",
163
+ "Gemfile.lock"))
164
+ end
165
+
166
+ def locked_gem_version(gem_name)
167
+ d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
168
+ d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
169
+ end
170
+
171
+ def rake_version
172
+ locked_gem_version("rake")
173
+ end
174
+
175
+ def test_unit_version
176
+ locked_gem_version("test-unit")
177
+ end
178
+
179
+ def bundler_version
180
+ d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
181
+ d.locked_gems.bundler_version.version
182
+ end
183
+
154
184
  def class_name
155
185
  "#{capitalized_name}#{type.capitalize}"
156
186
  end
@@ -244,10 +244,10 @@ module Fluent
244
244
  end
245
245
 
246
246
  def convert_value_to_nil(value)
247
- if value and @null_empty_string
247
+ if value && @null_empty_string
248
248
  value = (value == '') ? nil : value
249
249
  end
250
- if value and @null_value_pattern
250
+ if value && @null_value_pattern
251
251
  value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
252
252
  end
253
253
  value
@@ -50,6 +50,11 @@ module Fluent
50
50
  "<Fluent::Config::Section #{@params.to_json}>"
51
51
  end
52
52
 
53
+ # Used by PP and Pry
54
+ def pretty_print(q)
55
+ q.text(inspect)
56
+ end
57
+
53
58
  def nil?
54
59
  false
55
60
  end
@@ -179,7 +184,7 @@ module Fluent
179
184
  end
180
185
 
181
186
  if section_params[varname].nil?
182
- unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
187
+ unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
183
188
  logger.error "config error in:\n#{conf}" if logger
184
189
  raise ConfigError, "'#{name}' parameter is required but nil is specified"
185
190
  end
@@ -186,7 +186,7 @@ module Fluent
186
186
  return nil if val.nil?
187
187
 
188
188
  param = if val.is_a?(String)
189
- val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
189
+ val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
190
190
  else
191
191
  val
192
192
  end
@@ -213,7 +213,7 @@ module Fluent
213
213
  return nil if val.nil?
214
214
 
215
215
  param = if val.is_a?(String)
216
- val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
216
+ val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
217
217
  else
218
218
  val
219
219
  end
data/lib/fluent/event.rb CHANGED
@@ -254,19 +254,9 @@ module Fluent
254
254
  end
255
255
 
256
256
  def each(unpacker: nil, &block)
257
- if @unpacked_times
258
- @unpacked_times.each_with_index do |time, i|
259
- block.call(time, @unpacked_records[i])
260
- end
261
- else
262
- @unpacked_times = []
263
- @unpacked_records = []
264
- (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
265
- @unpacked_times << time
266
- @unpacked_records << record
267
- block.call(time, record)
268
- end
269
- @size = @unpacked_times.size
257
+ ensure_unpacked!(unpacker: unpacker)
258
+ @unpacked_times.each_with_index do |time, i|
259
+ block.call(time, @unpacked_records[i])
270
260
  end
271
261
  nil
272
262
  end
data/lib/fluent/load.rb CHANGED
@@ -1,4 +1,3 @@
1
- require 'thread'
2
1
  require 'socket'
3
2
  require 'fcntl'
4
3
  require 'time'
data/lib/fluent/log.rb CHANGED
@@ -49,6 +49,7 @@ module Fluent
49
49
  LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
50
50
 
51
51
  LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
52
+ LOG_ROTATE_AGE = %w(daily weekly monthly)
52
53
 
53
54
  def self.str_to_level(log_level_str)
54
55
  case log_level_str.downcase
@@ -46,6 +46,48 @@ module Fluent
46
46
  end
47
47
  end
48
48
 
49
+ class Win32Error < StandardError
50
+ require 'windows/error'
51
+ include Windows::Error
52
+
53
+ attr_reader :errcode, :msg
54
+
55
+ WSABASEERR = 10000
56
+
57
+ def initialize(errcode, msg = nil)
58
+ @errcode = errcode
59
+ @msg = msg
60
+ end
61
+
62
+ def format_english_message(errcode)
63
+ buf = 0.chr * 260
64
+ flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
65
+ english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
66
+ FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
67
+ buf.force_encoding(Encoding.default_external).strip
68
+ end
69
+
70
+ def to_s
71
+ msg = super
72
+ msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
73
+ msg << " - #{@msg}" if @msg
74
+ msg
75
+ end
76
+
77
+ def inspect
78
+ "#<#{to_s}>"
79
+ end
80
+
81
+ def ==(other)
82
+ return false if other.class != Win32Error
83
+ @errcode == other.errcode && @msg == other.msg
84
+ end
85
+
86
+ def wsaerr?
87
+ @errcode >= WSABASEERR
88
+ end
89
+ end
90
+
49
91
  # To open and get stat with setting FILE_SHARE_DELETE
50
92
  class WindowsFile
51
93
  require 'windows/file'
@@ -77,11 +119,14 @@ module Fluent
77
119
  @file_handle = CreateFile.call(@path, access, sharemode,
78
120
  0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
79
121
  if @file_handle == INVALID_HANDLE_VALUE
80
- err = GetLastError.call
81
- if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
82
- raise SystemCallError.new(2)
122
+ win32err = Win32Error.new(Win32::API.last_error, path)
123
+ errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
124
+ if errno == Errno::EINVAL::Errno || win32err.wsaerr?
125
+ # maybe failed to map
126
+ raise win32err
127
+ else
128
+ raise SystemCallError.new(win32err.message, errno)
83
129
  end
84
- raise SystemCallError.new(err)
85
130
  end
86
131
  end
87
132
 
@@ -27,14 +27,14 @@ module Fluent
27
27
 
28
28
  config_param :delimiter, :string, default: "\t".freeze
29
29
  config_param :label_delimiter, :string, default: ":".freeze
30
+ config_param :replacement, :string, default: " ".freeze
30
31
  config_param :add_newline, :bool, default: true
31
32
 
32
- # TODO: escaping for \t in values
33
33
  def format(tag, time, record)
34
34
  formatted = ""
35
35
  record.each do |label, value|
36
36
  formatted << @delimiter if formatted.length.nonzero?
37
- formatted << "#{label}#{@label_delimiter}#{value}"
37
+ formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
38
38
  end
39
39
  formatted << @newline if @add_newline
40
40
  formatted
@@ -461,6 +461,12 @@ module Fluent::Plugin
461
461
  RES_200_STATUS = "200 OK".freeze
462
462
  RES_403_STATUS = "403 Forbidden".freeze
463
463
 
464
+ # Azure App Service sends GET requests for health checking purpose.
465
+ # Respond with `200 OK` to accommodate it.
466
+ def handle_get_request
467
+ return send_response_and_close(RES_200_STATUS, {}, "")
468
+ end
469
+
464
470
  # Web browsers can send an OPTIONS request before performing POST
465
471
  # to check if cross-origin requests are supported.
466
472
  def handle_options_request
@@ -494,6 +500,10 @@ module Fluent::Plugin
494
500
  def on_message_complete
495
501
  return if closing?
496
502
 
503
+ if @parser.http_method == 'GET'.freeze
504
+ return handle_get_request()
505
+ end
506
+
497
507
  if @parser.http_method == 'OPTIONS'.freeze
498
508
  return handle_options_request()
499
509
  end
@@ -503,7 +513,7 @@ module Fluent::Plugin
503
513
  # For every incoming request, we check if we have some CORS
504
514
  # restrictions and allow listed origins through @cors_allow_origins.
505
515
  unless @cors_allow_origins.nil?
506
- unless @cors_allow_origins.include?('*') or include_cors_allow_origin
516
+ unless @cors_allow_origins.include?('*') || include_cors_allow_origin
507
517
  send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
508
518
  return
509
519
  end
@@ -338,7 +338,7 @@ module Fluent::Plugin
338
338
  obj.merge!(pe.statistics['output'] || {})
339
339
  end
340
340
 
341
- obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
341
+ obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
342
342
 
343
343
  # include all instance variables if :with_debug_info is set
344
344
  if opts[:with_debug_info]
@@ -56,6 +56,7 @@ module Fluent::Plugin
56
56
  @pf_file = nil
57
57
  @pf = nil
58
58
  @ignore_list = []
59
+ @shutdown_start_time = nil
59
60
  end
60
61
 
61
62
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -81,6 +82,8 @@ module Fluent::Plugin
81
82
  config_param :refresh_interval, :time, default: 60
82
83
  desc 'The number of reading lines at each IO.'
83
84
  config_param :read_lines_limit, :integer, default: 1000
85
+ desc 'The number of reading bytes per second'
86
+ config_param :read_bytes_limit_per_second, :size, default: -1
84
87
  desc 'The interval of flushing the buffer for multiline format'
85
88
  config_param :multiline_flush_interval, :time, default: nil
86
89
  desc 'Enable the option to emit unmatched lines.'
@@ -178,6 +181,16 @@ module Fluent::Plugin
178
181
  # parser is already created by parser helper
179
182
  @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
180
183
  @capability = Fluent::Capability.new(:current_process)
184
+ if @read_bytes_limit_per_second > 0
185
+ if !@enable_watch_timer
186
+ raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
187
+ end
188
+ min_bytes = TailWatcher::IOHandler::BYTES_TO_READ
189
+ if @read_bytes_limit_per_second < min_bytes
190
+ log.warn "Should specify greater equal than #{min_bytes}. Use #{min_bytes} for read_bytes_limit_per_second"
191
+ @read_bytes_limit_per_second = min_bytes
192
+ end
193
+ end
181
194
  end
182
195
 
183
196
  def configure_tag
@@ -244,6 +257,7 @@ module Fluent::Plugin
244
257
  end
245
258
 
246
259
  def shutdown
260
+ @shutdown_start_time = Fluent::Clock.now
247
261
  # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
248
262
  stop_watchers(existence_path, immediate: true, remove_watcher: false)
249
263
  @pf_file.close if @pf_file
@@ -290,7 +304,7 @@ module Fluent::Plugin
290
304
  end
291
305
  false
292
306
  end
293
- rescue Errno::ENOENT
307
+ rescue Errno::ENOENT, Errno::EACCES
294
308
  log.debug("#{p} is missing after refresh file list")
295
309
  false
296
310
  end
@@ -313,11 +327,17 @@ module Fluent::Plugin
313
327
  (paths - excluded).select { |path|
314
328
  FileTest.exist?(path)
315
329
  }.each { |path|
316
- target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
317
- if @follow_inodes
318
- hash[target_info.ino] = target_info
319
- else
320
- hash[target_info.path] = target_info
330
+ # Even we just checked for existence, there is a race condition here as
331
+ # of which stat() might fail with ENOENT. See #3224.
332
+ begin
333
+ target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
334
+ if @follow_inodes
335
+ hash[target_info.ino] = target_info
336
+ else
337
+ hash[target_info.path] = target_info
338
+ end
339
+ rescue Errno::ENOENT, Errno::EACCES => e
340
+ $log.warn "expand_paths: stat() for #{path} failed with #{e.class.name}. Skip file."
321
341
  end
322
342
  }
323
343
  hash
@@ -367,8 +387,6 @@ module Fluent::Plugin
367
387
  tw.register_watcher(tt)
368
388
  end
369
389
 
370
- tw.on_notify
371
-
372
390
  tw.watchers.each do |watcher|
373
391
  event_loop_attach(watcher)
374
392
  end
@@ -380,34 +398,48 @@ module Fluent::Plugin
380
398
  event_loop_detach(watcher)
381
399
  end
382
400
 
383
- tw.detach
401
+ tw.detach(@shutdown_start_time)
384
402
  tw.close
385
403
  end
386
404
  raise e
387
405
  end
388
406
 
389
- def start_watchers(targets_info)
390
- targets_info.each_value { |target_info|
391
- pe = nil
392
- if @pf
393
- pe = @pf[target_info]
394
- if @read_from_head && pe.read_inode.zero?
395
- begin
396
- pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
397
- rescue Errno::ENOENT
398
- $log.warn "#{target_info.path} not found. Continuing without tailing it."
399
- end
407
+ def construct_watcher(target_info)
408
+ pe = nil
409
+ if @pf
410
+ pe = @pf[target_info]
411
+ if @read_from_head && pe.read_inode.zero?
412
+ begin
413
+ pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
414
+ rescue Errno::ENOENT, Errno::EACCES
415
+ $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
400
416
  end
401
417
  end
418
+ end
402
419
 
403
- begin
404
- tw = setup_watcher(target_info, pe)
405
- rescue WatcherSetupError => e
406
- log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
407
- next
408
- end
420
+ begin
421
+ tw = setup_watcher(target_info, pe)
422
+ rescue WatcherSetupError => e
423
+ log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
424
+ return
425
+ end
426
+
427
+ begin
409
428
  target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
410
429
  @tails[target_info] = tw
430
+ tw.on_notify
431
+ rescue Errno::ENOENT, Errno::EACCES => e
432
+ $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
433
+ # explicitly detach and unwatch watcher `tw`.
434
+ tw.unwatched = true
435
+ detach_watcher(tw, target_info.ino, false)
436
+ end
437
+ end
438
+
439
+ def start_watchers(targets_info)
440
+ targets_info.each_value {|target_info|
441
+ construct_watcher(target_info)
442
+ break if before_shutdown?
411
443
  }
412
444
  end
413
445
 
@@ -460,9 +492,11 @@ module Fluent::Plugin
460
492
 
461
493
  if new_position_entry.read_inode == 0
462
494
  @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
495
+ @tails[new_target_info].on_notify
463
496
  end
464
497
  else
465
498
  @tails[new_target_info] = setup_watcher(new_target_info, pe)
499
+ @tails[new_target_info].on_notify
466
500
  end
467
501
  detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
468
502
  end
@@ -475,7 +509,7 @@ module Fluent::Plugin
475
509
  tw.watchers.each do |watcher|
476
510
  event_loop_detach(watcher)
477
511
  end
478
- tw.detach
512
+ tw.detach(@shutdown_start_time)
479
513
 
480
514
  tw.close if close_io
481
515
 
@@ -619,6 +653,7 @@ module Fluent::Plugin
619
653
  path: path,
620
654
  log: log,
621
655
  read_lines_limit: @read_lines_limit,
656
+ read_bytes_limit_per_second: @read_bytes_limit_per_second,
622
657
  open_on_every_update: @open_on_every_update,
623
658
  from_encoding: @from_encoding,
624
659
  encoding: @encoding,
@@ -686,8 +721,11 @@ module Fluent::Plugin
686
721
  @watchers << watcher
687
722
  end
688
723
 
689
- def detach
690
- @io_handler.on_notify if @io_handler
724
+ def detach(shutdown_start_time = nil)
725
+ if @io_handler
726
+ @io_handler.ready_to_shutdown(shutdown_start_time)
727
+ @io_handler.on_notify
728
+ end
691
729
  @line_buffer_timer_flusher&.close(self)
692
730
  end
693
731
 
@@ -701,7 +739,7 @@ module Fluent::Plugin
701
739
  def on_notify
702
740
  begin
703
741
  stat = Fluent::FileWrapper.stat(@path)
704
- rescue Errno::ENOENT
742
+ rescue Errno::ENOENT, Errno::EACCES
705
743
  # moved or deleted
706
744
  stat = nil
707
745
  end
@@ -767,16 +805,22 @@ module Fluent::Plugin
767
805
  end
768
806
 
769
807
  if watcher_needs_update
770
- # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
771
- # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
772
- if stat
773
- target_info = TargetInfo.new(@path, stat.ino)
774
- if @follow_inodes
775
- # don't want to swap state because we need latest read offset in pos file even after rotate_wait
808
+ if @follow_inodes
809
+ # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
810
+ # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
811
+ # don't want to swap state because we need latest read offset in pos file even after rotate_wait
812
+ if stat
813
+ target_info = TargetInfo.new(@path, stat)
776
814
  @update_watcher.call(target_info, @pe)
777
- else
778
- @update_watcher.call(target_info, swap_state(@pe))
779
815
  end
816
+ else
817
+ # Permit to handle if stat is nil (file not present).
818
+ # If a file is mv-ed and a new file is created during
819
+ # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
820
+ # and `#stop_watchers()` for the path because `target_paths_hash`
821
+ # always contains the path.
822
+ target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
823
+ @update_watcher.call(target_info, swap_state(@pe))
780
824
  end
781
825
  else
782
826
  @log.info "detected rotation of #{@path}"
@@ -856,10 +900,16 @@ module Fluent::Plugin
856
900
  end
857
901
 
858
902
  class IOHandler
859
- def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
903
+ BYTES_TO_READ = 8192
904
+ SHUTDOWN_TIMEOUT = 5
905
+
906
+ attr_accessor :shutdown_timeout
907
+
908
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
860
909
  @watcher = watcher
861
910
  @path = path
862
911
  @read_lines_limit = read_lines_limit
912
+ @read_bytes_limit_per_second = read_bytes_limit_per_second
863
913
  @receive_lines = receive_lines
864
914
  @open_on_every_update = open_on_every_update
865
915
  @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
@@ -868,6 +918,11 @@ module Fluent::Plugin
868
918
  @io = nil
869
919
  @notify_mutex = Mutex.new
870
920
  @log = log
921
+ @start_reading_time = nil
922
+ @number_bytes_read = 0
923
+ @shutdown_start_time = nil
924
+ @shutdown_timeout = SHUTDOWN_TIMEOUT
925
+ @shutdown_mutex = Mutex.new
871
926
 
872
927
  @log.info "following tail of #{@path}"
873
928
  end
@@ -876,6 +931,13 @@ module Fluent::Plugin
876
931
  @notify_mutex.synchronize { handle_notify }
877
932
  end
878
933
 
934
+ def ready_to_shutdown(shutdown_start_time = nil)
935
+ @shutdown_mutex.synchronize {
936
+ @shutdown_start_time =
937
+ shutdown_start_time || Fluent::Clock.now
938
+ }
939
+ end
940
+
879
941
  def close
880
942
  if @io && !@io.closed?
881
943
  @io.close
@@ -889,7 +951,35 @@ module Fluent::Plugin
889
951
 
890
952
  private
891
953
 
954
+ def limit_bytes_per_second_reached?
955
+ return false if @read_bytes_limit_per_second < 0 # not enabled by conf
956
+ return false if @number_bytes_read < @read_bytes_limit_per_second
957
+
958
+ @start_reading_time ||= Fluent::Clock.now
959
+ time_spent_reading = Fluent::Clock.now - @start_reading_time
960
+ @log.debug("time_spent_reading: #{time_spent_reading} #{ @watcher.path}")
961
+
962
+ if time_spent_reading < 1
963
+ true
964
+ else
965
+ @start_reading_time = nil
966
+ @number_bytes_read = 0
967
+ false
968
+ end
969
+ end
970
+
971
+ def should_shutdown_now?
972
+ # Ensure to read all remaining lines, but abort immediately if it
973
+ # seems to take too long time.
974
+ @shutdown_mutex.synchronize {
975
+ return false if @shutdown_start_time.nil?
976
+ return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout
977
+ }
978
+ end
979
+
892
980
  def handle_notify
981
+ return if limit_bytes_per_second_reached?
982
+
893
983
  with_io do |io|
894
984
  begin
895
985
  read_more = false
@@ -897,8 +987,17 @@ module Fluent::Plugin
897
987
  if !io.nil? && @lines.empty?
898
988
  begin
899
989
  while true
900
- @fifo << io.readpartial(8192, @iobuf)
990
+ @start_reading_time ||= Fluent::Clock.now
991
+ data = io.readpartial(BYTES_TO_READ, @iobuf)
992
+ @number_bytes_read += data.bytesize
993
+ @fifo << data
901
994
  @fifo.read_lines(@lines)
995
+
996
+ if limit_bytes_per_second_reached? || should_shutdown_now?
997
+ # Just get out from tailing loop.
998
+ read_more = false
999
+ break
1000
+ end
902
1001
  if @lines.size >= @read_lines_limit
903
1002
  # not to use too much memory in case the file is very large
904
1003
  read_more = true
@@ -928,7 +1027,7 @@ module Fluent::Plugin
928
1027
  rescue RangeError
929
1028
  io.close if io
930
1029
  raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
931
- rescue Errno::ENOENT
1030
+ rescue Errno::ENOENT, Errno::EACCES
932
1031
  nil
933
1032
  end
934
1033