fluentd 1.12.1-x64-mingw32 → 1.13.1-x64-mingw32

data/README.md

@@ -1,8 +1,11 @@
 Fluentd: Open-Source Log Collector
 ===================================
-Travis CI:
+GitHub Actions:
 
-[<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
+[![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
+[![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
+[![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
+[![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
@@ -74,7 +77,7 @@ You can run specified test via `TEST` environment variable:
 - Website: https://www.fluentd.org/
 - Documentation: https://docs.fluentd.org/
 - Project repository: https://github.com/fluent
-- Discussion: https://groups.google.com/group/fluentd
+- Discussion: https://discuss.fluentd.org/
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter
 - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)

data/example/counter.conf

@@ -3,7 +3,7 @@
     scope server1
     bind 127.0.0.1
     port 24321
-    path tmp/back
+    backup_path tmp/back
   </counter_server>
 </system>
 

data/fluentd.gemspec

@@ -28,12 +28,14 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -44,7 +46,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("flexmock", ["~> 2.0"])
   gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.7"])
-  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("rr", ["~> 3.0"])
   gem.add_development_dependency("timecop", ["~> 0.9"])
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])

data/lib/fluent/command/bundler_injection.rb

@@ -40,6 +40,6 @@ else
     File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
   ] + ARGV
 
-  exec *cmdline
+  exec(*cmdline)
   exit! 127
 end

data/lib/fluent/command/cat.rb

@@ -101,7 +101,6 @@ rescue
   usage $!.to_s
 end
 
-require 'thread'
 require 'socket'
 require 'yajl'
 require 'msgpack'
@@ -153,14 +152,30 @@ class Writer
     super()
   end
 
+  def secondary_record?(record)
+    record.class != Hash &&
+      record.size == 2 &&
+      record.first.class == Fluent::EventTime &&
+      record.last.class == Hash
+  end
+
   def write(record)
-    if record.class != Hash
-      raise ArgumentError, "Input must be a map (got #{record.class})"
+    unless secondary_record?(record)
+      if record.class != Hash
+        raise ArgumentError, "Input must be a map (got #{record.class})"
+      end
     end
 
     time = Fluent::EventTime.now
     time = time.to_i if @time_as_integer
-    entry = [time, record]
+    entry = if secondary_record?(record)
+              # Even though secondary contains Fluent::EventTime in record,
+              # fluent-cat just ignore it and set Fluent::EventTime.now instead.
+              # This specification is adopted to keep consistency.
+              [time, record.last]
+            else
+              [time, record]
+            end
     synchronize {
       unless write_impl([entry])
         # write failed

data/lib/fluent/command/fluentd.rb

@@ -85,9 +85,8 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
-ROTATE_AGE = %w(daily weekly monthly)
 op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
-  if ROTATE_AGE.include?(age)
+  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
     opts[:log_rotate_age] = age
   else
     begin

data/lib/fluent/command/plugin_config_formatter.rb

@@ -44,6 +44,7 @@ class FluentPluginConfigFormatter
     @verbose = false
     @libs = []
     @plugin_dirs = []
+    @table = false
     @options = {}
 
     prepare_option_parser
@@ -162,7 +163,7 @@ class FluentPluginConfigFormatter
     else
       sections, params = base_section.partition {|_name, value| value[:section] }
     end
-    if @table and not params.empty?
+    if @table && (not params.empty?)
       dumped << "### Configuration\n\n"
       dumped << "|parameter|type|description|default|\n"
       dumped << "|---|---|---|---|\n"

data/lib/fluent/command/plugin_generator.rb

@@ -105,7 +105,7 @@ Generate a project skeleton for creating a Fluentd plugin
 
 Arguments:
 \ttype: #{SUPPORTED_TYPES.join(",")}
-\tname: Your plugin name
+\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
 
 Options:
 BANNER
@@ -151,6 +151,36 @@ BANNER
     underscore_name
   end
 
+  def gem_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile"))
+  end
+
+  def lock_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile.lock"))
+  end
+
+  def locked_gem_version(gem_name)
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+  end
+
+  def rake_version
+    locked_gem_version("rake")
+  end
+
+  def test_unit_version
+    locked_gem_version("test-unit")
+  end
+
+  def bundler_version
+    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+    d.locked_gems.bundler_version.version
+  end
+
   def class_name
     "#{capitalized_name}#{type.capitalize}"
   end

data/lib/fluent/compat/parser.rb

@@ -244,10 +244,10 @@ module Fluent
         end
 
         def convert_value_to_nil(value)
-          if value and @null_empty_string
+          if value && @null_empty_string
             value = (value == '') ? nil : value
           end
-          if value and @null_value_pattern
+          if value && @null_value_pattern
             value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
           end
           value

data/lib/fluent/config/section.rb

@@ -50,6 +50,11 @@ module Fluent
         "<Fluent::Config::Section #{@params.to_json}>"
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       def nil?
         false
       end
@@ -179,7 +184,7 @@ module Fluent
             end
 
             if section_params[varname].nil?
-              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
                 logger.error "config error in:\n#{conf}" if logger
                 raise ConfigError, "'#{name}' parameter is required but nil is specified"
               end

data/lib/fluent/config/types.rb

@@ -186,7 +186,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -213,7 +213,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
               else
                 val
               end

data/lib/fluent/event.rb

@@ -254,19 +254,9 @@ module Fluent
     end
 
     def each(unpacker: nil, &block)
-      if @unpacked_times
-        @unpacked_times.each_with_index do |time, i|
-          block.call(time, @unpacked_records[i])
-        end
-      else
-        @unpacked_times = []
-        @unpacked_records = []
-        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
-          @unpacked_times << time
-          @unpacked_records << record
-          block.call(time, record)
-        end
-        @size = @unpacked_times.size
+      ensure_unpacked!(unpacker: unpacker)
+      @unpacked_times.each_with_index do |time, i|
+        block.call(time, @unpacked_records[i])
       end
       nil
     end

data/lib/fluent/load.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'

data/lib/fluent/log.rb

@@ -49,6 +49,7 @@ module Fluent
     LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
 
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
+    LOG_ROTATE_AGE = %w(daily weekly monthly)
 
     def self.str_to_level(log_level_str)
       case log_level_str.downcase

data/lib/fluent/plugin/file_wrapper.rb

@@ -46,6 +46,48 @@ module Fluent
     end
   end
 
+  class Win32Error < StandardError
+    require 'windows/error'
+    include Windows::Error
+
+    attr_reader :errcode, :msg
+
+    WSABASEERR = 10000
+
+    def initialize(errcode, msg = nil)
+      @errcode = errcode
+      @msg = msg
+    end
+
+    def format_english_message(errcode)
+      buf = 0.chr * 260
+      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
+      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
+      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
+      buf.force_encoding(Encoding.default_external).strip
+    end
+
+    def to_s
+      msg = super
+      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
+      msg << " - #{@msg}" if @msg
+      msg
+    end
+
+    def inspect
+      "#<#{to_s}>"
+    end
+
+    def ==(other)
+      return false if other.class != Win32Error
+      @errcode == other.errcode && @msg == other.msg
+    end
+
+    def wsaerr?
+      @errcode >= WSABASEERR
+    end
+  end
+
   # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
     require 'windows/file'
@@ -77,11 +119,14 @@ module Fluent
       @file_handle = CreateFile.call(@path, access, sharemode,
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
-        err = GetLastError.call
-        if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
-          raise SystemCallError.new(2)
+        win32err = Win32Error.new(Win32::API.last_error, path)
+        errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
+        if errno == Errno::EINVAL::Errno || win32err.wsaerr?
+          # maybe failed to map
+          raise win32err
+        else
+          raise SystemCallError.new(win32err.message, errno)
         end
-        raise SystemCallError.new(err)
       end
     end
 

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -27,14 +27,14 @@ module Fluent
 
       config_param :delimiter, :string, default: "\t".freeze
       config_param :label_delimiter, :string, default: ":".freeze
+      config_param :replacement, :string, default: " ".freeze
       config_param :add_newline, :bool, default: true
 
-      # TODO: escaping for \t in values
       def format(tag, time, record)
         formatted = ""
         record.each do |label, value|
           formatted << @delimiter if formatted.length.nonzero?
-          formatted << "#{label}#{@label_delimiter}#{value}"
+          formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
         end
         formatted << @newline if @add_newline
         formatted

data/lib/fluent/plugin/in_http.rb

@@ -461,6 +461,12 @@ module Fluent::Plugin
       RES_200_STATUS = "200 OK".freeze
       RES_403_STATUS = "403 Forbidden".freeze
 
+      # Azure App Service sends GET requests for health checking purpose.
+      # Respond with `200 OK` to accommodate it.
+      def handle_get_request
+          return send_response_and_close(RES_200_STATUS, {}, "")
+      end
+
       # Web browsers can send an OPTIONS request before performing POST
       # to check if cross-origin requests are supported.
       def handle_options_request
@@ -494,6 +500,10 @@ module Fluent::Plugin
       def on_message_complete
         return if closing?
 
+        if @parser.http_method == 'GET'.freeze
+          return handle_get_request()
+        end
+
         if @parser.http_method == 'OPTIONS'.freeze
           return handle_options_request()
         end
@@ -503,7 +513,7 @@ module Fluent::Plugin
         # For every incoming request, we check if we have some CORS
         # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?('*') or include_cors_allow_origin
+          unless @cors_allow_origins.include?('*') || include_cors_allow_origin
             send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -338,7 +338,7 @@ module Fluent::Plugin
         obj.merge!(pe.statistics['output'] || {})
       end
 
-      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
+      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
 
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]

data/lib/fluent/plugin/in_tail.rb

@@ -56,6 +56,7 @@ module Fluent::Plugin
       @pf_file = nil
       @pf = nil
       @ignore_list = []
+      @shutdown_start_time = nil
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -81,6 +82,8 @@ module Fluent::Plugin
     config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, default: 1000
+    desc 'The number of reading bytes per second'
+    config_param :read_bytes_limit_per_second, :size, default: -1
     desc 'The interval of flushing the buffer for multiline format'
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the option to emit unmatched lines.'
@@ -178,6 +181,16 @@ module Fluent::Plugin
       # parser is already created by parser helper
       @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
       @capability = Fluent::Capability.new(:current_process)
+      if @read_bytes_limit_per_second > 0
+        if !@enable_watch_timer
+          raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
+        end
+        min_bytes = TailWatcher::IOHandler::BYTES_TO_READ
+        if @read_bytes_limit_per_second < min_bytes
+          log.warn "Should specify greater equal than #{min_bytes}. Use #{min_bytes} for read_bytes_limit_per_second"
+          @read_bytes_limit_per_second = min_bytes
+        end
+      end
     end
 
     def configure_tag
@@ -244,6 +257,7 @@ module Fluent::Plugin
     end
 
     def shutdown
+      @shutdown_start_time = Fluent::Clock.now
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(existence_path, immediate: true, remove_watcher: false)
       @pf_file.close if @pf_file
@@ -290,7 +304,7 @@ module Fluent::Plugin
                 end
                 false
               end
-            rescue Errno::ENOENT
+            rescue Errno::ENOENT, Errno::EACCES
               log.debug("#{p} is missing after refresh file list")
               false
             end
@@ -313,11 +327,17 @@ module Fluent::Plugin
       (paths - excluded).select { |path|
         FileTest.exist?(path)
       }.each { |path|
-        target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
-        if @follow_inodes
-          hash[target_info.ino] = target_info
-        else
-          hash[target_info.path] = target_info
+        # Even we just checked for existence, there is a race condition here as
+        # of which stat() might fail with ENOENT. See #3224.
+        begin
+          target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+          if @follow_inodes
+            hash[target_info.ino] = target_info
+          else
+            hash[target_info.path] = target_info
+          end
+        rescue Errno::ENOENT, Errno::EACCES  => e
+          $log.warn "expand_paths: stat() for #{path} failed with #{e.class.name}. Skip file."
         end
       }
       hash
@@ -367,8 +387,6 @@ module Fluent::Plugin
         tw.register_watcher(tt)
       end
 
-      tw.on_notify
-
       tw.watchers.each do |watcher|
         event_loop_attach(watcher)
       end
@@ -380,34 +398,48 @@ module Fluent::Plugin
           event_loop_detach(watcher)
         end
 
-        tw.detach
+        tw.detach(@shutdown_start_time)
         tw.close
       end
       raise e
     end
 
-    def start_watchers(targets_info)
-      targets_info.each_value { |target_info|
-        pe = nil
-        if @pf
-          pe = @pf[target_info]
-          if @read_from_head && pe.read_inode.zero?
-            begin
-              pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
-            rescue Errno::ENOENT
-              $log.warn "#{target_info.path} not found. Continuing without tailing it."
-            end
+    def construct_watcher(target_info)
+      pe = nil
+      if @pf
+        pe = @pf[target_info]
+        if @read_from_head && pe.read_inode.zero?
+          begin
+            pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
+          rescue Errno::ENOENT, Errno::EACCES
+            $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
           end
         end
+      end
 
-        begin
-          tw = setup_watcher(target_info, pe)
-        rescue WatcherSetupError => e
-          log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
-          next
-        end
+      begin
+        tw = setup_watcher(target_info, pe)
+      rescue WatcherSetupError => e
+        log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
+        return
+      end
+
+      begin
         target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
         @tails[target_info] = tw
+        tw.on_notify
+      rescue Errno::ENOENT, Errno::EACCES => e
+        $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
+        # explicitly detach and unwatch watcher `tw`.
+        tw.unwatched = true
+        detach_watcher(tw, target_info.ino, false)
+      end
+    end
+
+    def start_watchers(targets_info)
+      targets_info.each_value {|target_info|
+        construct_watcher(target_info)
+        break if before_shutdown?
       }
     end
 
@@ -460,9 +492,11 @@ module Fluent::Plugin
 
         if new_position_entry.read_inode == 0
           @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
+          @tails[new_target_info].on_notify
         end
       else
         @tails[new_target_info] = setup_watcher(new_target_info, pe)
+        @tails[new_target_info].on_notify
       end
       detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
     end
@@ -475,7 +509,7 @@ module Fluent::Plugin
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
-      tw.detach
+      tw.detach(@shutdown_start_time)
 
       tw.close if close_io
 
@@ -619,6 +653,7 @@ module Fluent::Plugin
         path: path,
         log: log,
         read_lines_limit: @read_lines_limit,
+        read_bytes_limit_per_second: @read_bytes_limit_per_second,
         open_on_every_update: @open_on_every_update,
         from_encoding: @from_encoding,
         encoding: @encoding,
@@ -686,8 +721,11 @@ module Fluent::Plugin
         @watchers << watcher
       end
 
-      def detach
-        @io_handler.on_notify if @io_handler
+      def detach(shutdown_start_time = nil)
+        if @io_handler
+          @io_handler.ready_to_shutdown(shutdown_start_time)
+          @io_handler.on_notify
+        end
         @line_buffer_timer_flusher&.close(self)
       end
 
@@ -701,7 +739,7 @@ module Fluent::Plugin
       def on_notify
         begin
           stat = Fluent::FileWrapper.stat(@path)
-        rescue Errno::ENOENT
+        rescue Errno::ENOENT, Errno::EACCES
           # moved or deleted
           stat = nil
         end
@@ -767,16 +805,22 @@ module Fluent::Plugin
           end
 
           if watcher_needs_update
-            # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-            # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-            if stat
-              target_info = TargetInfo.new(@path, stat.ino)
-              if @follow_inodes
-                # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+            if @follow_inodes
+              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
+              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
+              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              if stat
+                target_info = TargetInfo.new(@path, stat)
                 @update_watcher.call(target_info, @pe)
-              else
-                @update_watcher.call(target_info, swap_state(@pe))
               end
+            else
+              # Permit to handle if stat is nil (file not present).
+              # If a file is mv-ed and a new file is created during
+              # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
+              # and `#stop_watchers()` for the path because `target_paths_hash`
+              # always contains the path.
+              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
+              @update_watcher.call(target_info, swap_state(@pe))
             end
           else
             @log.info "detected rotation of #{@path}"
@@ -856,10 +900,16 @@ module Fluent::Plugin
       end
 
       class IOHandler
-        def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
+        BYTES_TO_READ = 8192
+        SHUTDOWN_TIMEOUT = 5
+
+        attr_accessor :shutdown_timeout
+
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
+          @read_bytes_limit_per_second = read_bytes_limit_per_second
           @receive_lines = receive_lines
           @open_on_every_update = open_on_every_update
           @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
@@ -868,6 +918,11 @@ module Fluent::Plugin
           @io = nil
           @notify_mutex = Mutex.new
           @log = log
+          @start_reading_time = nil
+          @number_bytes_read = 0
+          @shutdown_start_time = nil
+          @shutdown_timeout = SHUTDOWN_TIMEOUT
+          @shutdown_mutex = Mutex.new
 
           @log.info "following tail of #{@path}"
         end
@@ -876,6 +931,13 @@ module Fluent::Plugin
           @notify_mutex.synchronize { handle_notify }
         end
 
+        def ready_to_shutdown(shutdown_start_time = nil)
+          @shutdown_mutex.synchronize {
+            @shutdown_start_time =
+              shutdown_start_time || Fluent::Clock.now
+          }
+        end
+
         def close
           if @io && !@io.closed?
             @io.close
@@ -889,7 +951,35 @@ module Fluent::Plugin
 
         private
 
+        def limit_bytes_per_second_reached?
+          return false if @read_bytes_limit_per_second < 0 # not enabled by conf
+          return false if @number_bytes_read < @read_bytes_limit_per_second
+
+          @start_reading_time ||= Fluent::Clock.now
+          time_spent_reading = Fluent::Clock.now - @start_reading_time
+          @log.debug("time_spent_reading: #{time_spent_reading} #{ @watcher.path}")
+
+          if time_spent_reading < 1
+            true
+          else
+            @start_reading_time = nil
+            @number_bytes_read = 0
+            false
+          end
+        end
+
+        def should_shutdown_now?
+          # Ensure to read all remaining lines, but abort immediately if it
+          # seems to take too long time.
+          @shutdown_mutex.synchronize {
+            return false if @shutdown_start_time.nil?
+            return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout
+          }
+        end
+
         def handle_notify
+          return if limit_bytes_per_second_reached?
+
           with_io do |io|
             begin
               read_more = false
@@ -897,8 +987,17 @@ module Fluent::Plugin
               if !io.nil? && @lines.empty?
                 begin
                   while true
-                    @fifo << io.readpartial(8192, @iobuf)
+                    @start_reading_time ||= Fluent::Clock.now
+                    data = io.readpartial(BYTES_TO_READ, @iobuf)
+                    @number_bytes_read += data.bytesize
+                    @fifo << data
                     @fifo.read_lines(@lines)
+
+                    if limit_bytes_per_second_reached? || should_shutdown_now?
+                      # Just get out from tailing loop.
+                      read_more = false
+                      break
+                    end
                     if @lines.size >= @read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true
@@ -928,7 +1027,7 @@ module Fluent::Plugin
         rescue RangeError
           io.close if io
           raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
-        rescue Errno::ENOENT
+        rescue Errno::ENOENT, Errno::EACCES
           nil
         end