fluentd 1.12.4-x86-mingw32 → 1.13.3-x86-mingw32

data/lib/fluent/command/fluentd.rb

@@ -85,9 +85,8 @@ op.on('-o', '--log PATH', "log file path") {|s|
   opts[:log_path] = s
 }
 
-ROTATE_AGE = %w(daily weekly monthly)
 op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
-  if ROTATE_AGE.include?(age)
+  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
     opts[:log_rotate_age] = age
   else
     begin

data/lib/fluent/command/plugin_generator.rb

@@ -27,7 +27,7 @@ class FluentPluginGenerator
   attr_reader :type, :name
   attr_reader :license_name
 
-  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
 
   def initialize(argv = ARGV)
     @argv = argv
@@ -164,8 +164,13 @@ BANNER
   end
 
   def locked_gem_version(gem_name)
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name(gem_name).version.version
+    end
   end
 
   def rake_version
@@ -177,8 +182,13 @@ BANNER
   end
 
   def bundler_version
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.bundler_version.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.bundler_version.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name("bundler").version.version
+    end
   end
 
   def class_name

data/lib/fluent/config.rb

@@ -62,7 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
-        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file") if $log
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/config/section.rb

@@ -50,6 +50,11 @@ module Fluent
         "<Fluent::Config::Section #{@params.to_json}>"
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       def nil?
         false
       end

data/lib/fluent/config/types.rb

@@ -20,6 +20,10 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
     def self.size_value(str, opts = {}, name = nil)
       return nil if str.nil?
 
@@ -104,6 +108,16 @@ module Fluent
       Config.string_value(val, opts, name)
     }
 
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
     def self.enum_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
@@ -176,6 +190,7 @@ module Fluent
         when :bool then Config.bool_value(value, opts, name)
         when :time then Config.time_value(value, opts, name)
         when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end

data/lib/fluent/config/v1_parser.rb

@@ -37,6 +37,7 @@ module Fluent
         super(strscan, eval_context)
         @include_basepath = include_basepath
         @fname = fname
+        @logger = defined?($log) ? $log : nil
       end
 
       def parse!
@@ -99,7 +100,7 @@ module Fluent
 
           elsif root_element && skip(/(\@include|include)#{SPACING}/)
             if !prev_match.start_with?('@')
-              $log.warn "'include' is deprecated. Use '@include' instead"
+              @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
             end
             parse_include(attrs, elems)
 
@@ -123,7 +124,7 @@ module Fluent
                     parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
                   else
                     # TODO: This is for backward compatibility. It will throw an error in the future.
-                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
+                    @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
                   end
                 end
 

data/lib/fluent/env.rb

@@ -15,13 +15,14 @@
 #
 
 require 'serverengine/utils'
+require 'fluent/oj_options'
 
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
-  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
+  DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
 

data/lib/fluent/log.rb

@@ -49,6 +49,7 @@ module Fluent
     LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
 
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
+    LOG_ROTATE_AGE = %w(daily weekly monthly)
 
     def self.str_to_level(log_level_str)
       case log_level_str.downcase

data/lib/fluent/oj_options.rb

@@ -0,0 +1,62 @@
+require 'fluent/config/types'
+
+module Fluent
+  class OjOptions
+    OPTIONS = {
+      'bigdecimal_load': :symbol,
+      'max_nesting': :integer,
+      'mode': :symbol,
+      'use_to_json': :bool
+    }
+
+    ALLOWED_VALUES = {
+      'bigdecimal_load': %i[bigdecimal float auto],
+      'mode': %i[strict null compat json rails object custom]
+    }
+
+    DEFAULTS = {
+      'bigdecimal_load': :float,
+      'mode': :compat,
+      'use_to_json': true
+    }
+
+    @@available = false
+
+    def self.available?
+      @@available
+    end
+
+    def self.load_env
+      options = self.get_options
+      begin
+        require 'oj'
+        Oj.default_options = options
+        @@available = true
+      rescue LoadError
+        @@available = false
+      end
+      options
+    end
+
+    private
+
+    def self.get_options
+      options = {}
+      DEFAULTS.each { |key, value| options[key] = value }
+
+      OPTIONS.each do |key, type|
+        env_value = ENV["FLUENT_OJ_OPTION_#{key.upcase}"]
+        next if env_value.nil?
+
+        cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true })
+        next if cast_value.nil?
+
+        next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value)
+
+        options[key.to_sym] = cast_value
+      end
+
+      options
+    end
+  end
+end

data/lib/fluent/plugin/file_wrapper.rb

@@ -52,6 +52,8 @@ module Fluent
 
     attr_reader :errcode, :msg
 
+    WSABASEERR = 10000
+
     def initialize(errcode, msg = nil)
       @errcode = errcode
       @msg = msg
@@ -80,6 +82,10 @@ module Fluent
       return false if other.class != Win32Error
       @errcode == other.errcode && @msg == other.msg
     end
+
+    def wsaerr?
+      @errcode >= WSABASEERR
+    end
   end
 
   # To open and get stat with setting FILE_SHARE_DELETE
@@ -113,11 +119,14 @@ module Fluent
       @file_handle = CreateFile.call(@path, access, sharemode,
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
-        err = Win32::API.last_error
-        if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
-          raise Errno::ENOENT
+        win32err = Win32Error.new(Win32::API.last_error, path)
+        errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
+        if errno == Errno::EINVAL::Errno || win32err.wsaerr?
+          # maybe failed to map
+          raise win32err
+        else
+          raise SystemCallError.new(win32err.message, errno)
         end
-        raise Win32Error.new(err, path)
       end
     end
 
@@ -146,7 +155,29 @@ module Fluent
       by_handle_file_information.unpack("I11Q1")[11] # fileindex
     end
 
+    # DeletePending is a Windows-specific file state that roughly means
+    # "this file is queued for deletion, so close any open handlers"
+    #
+    # This flag can be retrieved via GetFileInformationByHandleEx().
+    #
+    # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+    #
+    def delete_pending
+      file_standard_info = 0x01
+      bufsize = 1024
+      buf = '\0' * bufsize
+
+      unless GetFileInformationByHandleEx.call(@file_handle, file_standard_info, buf, bufsize)
+        return false
+      end
+
+      return buf.unpack("QQICC")[3] != 0
+    end
+
+    private :delete_pending
+
     def stat
+      raise Errno::ENOENT if delete_pending
       s = File.stat(@path)
       s.instance_variable_set :@ino, self.ino
       def s.ino; @ino; end

data/lib/fluent/plugin/formatter.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
 require 'fluent/time'

data/lib/fluent/plugin/formatter_json.rb

@@ -15,7 +15,7 @@
 #
 
 require 'fluent/plugin/formatter'
-require 'fluent/env'
+require 'fluent/oj_options'
 
 module Fluent
   module Plugin
@@ -30,12 +30,14 @@ module Fluent
       def configure(conf)
         super
 
-        begin
-          raise LoadError unless @json_parser == 'oj'
-          require 'oj'
-          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @dump_proc = Oj.method(:dump)
-        rescue LoadError
+        if @json_parser == 'oj'
+          if Fluent::OjOptions.available?
+            @dump_proc = Oj.method(:dump)
+          else
+            log.info "Oj isn't installed, fallback to Yajl as json parser"
+            @dump_proc = Yajl.method(:dump)
+          end
+        else
           @dump_proc = Yajl.method(:dump)
         end
 

data/lib/fluent/plugin/in_http.rb

@@ -461,6 +461,12 @@ module Fluent::Plugin
       RES_200_STATUS = "200 OK".freeze
       RES_403_STATUS = "403 Forbidden".freeze
 
+      # Azure App Service sends GET requests for health checking purpose.
+      # Respond with `200 OK` to accommodate it.
+      def handle_get_request
+          return send_response_and_close(RES_200_STATUS, {}, "")
+      end
+
       # Web browsers can send an OPTIONS request before performing POST
       # to check if cross-origin requests are supported.
       def handle_options_request
@@ -494,6 +500,10 @@ module Fluent::Plugin
       def on_message_complete
         return if closing?
 
+        if @parser.http_method == 'GET'.freeze
+          return handle_get_request()
+        end
+
         if @parser.http_method == 'OPTIONS'.freeze
           return handle_options_request()
         end

data/lib/fluent/plugin/in_tail.rb

@@ -56,6 +56,7 @@ module Fluent::Plugin
       @pf_file = nil
       @pf = nil
       @ignore_list = []
+      @shutdown_start_time = nil
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -81,6 +82,8 @@ module Fluent::Plugin
     config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, default: 1000
+    desc 'The number of reading bytes per second'
+    config_param :read_bytes_limit_per_second, :size, default: -1
     desc 'The interval of flushing the buffer for multiline format'
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the option to emit unmatched lines.'
@@ -178,6 +181,16 @@ module Fluent::Plugin
       # parser is already created by parser helper
       @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
       @capability = Fluent::Capability.new(:current_process)
+      if @read_bytes_limit_per_second > 0
+        if !@enable_watch_timer
+          raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
+        end
+        min_bytes = TailWatcher::IOHandler::BYTES_TO_READ
+        if @read_bytes_limit_per_second < min_bytes
+          log.warn "Should specify greater equal than #{min_bytes}. Use #{min_bytes} for read_bytes_limit_per_second"
+          @read_bytes_limit_per_second = min_bytes
+        end
+      end
     end
 
     def configure_tag
@@ -244,6 +257,7 @@ module Fluent::Plugin
     end
 
     def shutdown
+      @shutdown_start_time = Fluent::Clock.now
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(existence_path, immediate: true, remove_watcher: false)
       @pf_file.close if @pf_file
@@ -290,7 +304,7 @@ module Fluent::Plugin
                 end
                 false
               end
-            rescue Errno::ENOENT
+            rescue Errno::ENOENT, Errno::EACCES
               log.debug("#{p} is missing after refresh file list")
               false
             end
@@ -322,8 +336,8 @@ module Fluent::Plugin
           else
             hash[target_info.path] = target_info
           end
-        rescue Errno::ENOENT
-          $log.warn "expand_paths: stat() for #{path} failed with ENOENT. Skip file."
+        rescue Errno::ENOENT, Errno::EACCES  => e
+          $log.warn "expand_paths: stat() for #{path} failed with #{e.class.name}. Skip file."
         end
       }
       hash
@@ -373,8 +387,6 @@ module Fluent::Plugin
         tw.register_watcher(tt)
       end
 
-      tw.on_notify
-
       tw.watchers.each do |watcher|
         event_loop_attach(watcher)
       end
@@ -386,42 +398,49 @@ module Fluent::Plugin
           event_loop_detach(watcher)
         end
 
-        tw.detach
+        tw.detach(@shutdown_start_time)
         tw.close
       end
       raise e
     end
 
-    def start_watchers(targets_info)
-      targets_info.each_value { |target_info|
-        pe = nil
-        if @pf
-          pe = @pf[target_info]
-          if @read_from_head && pe.read_inode.zero?
-            begin
-              pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
-            rescue Errno::ENOENT
-              $log.warn "#{target_info.path} not found. Continuing without tailing it."
-            end
+    def construct_watcher(target_info)
+      pe = nil
+      if @pf
+        pe = @pf[target_info]
+        if @read_from_head && pe.read_inode.zero?
+          begin
+            pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
+          rescue Errno::ENOENT, Errno::EACCES
+            $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
           end
         end
+      end
 
-        begin
-          tw = setup_watcher(target_info, pe)
-        rescue WatcherSetupError => e
-          log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
-          next
-        end
+      begin
+        tw = setup_watcher(target_info, pe)
+      rescue WatcherSetupError => e
+        log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
+        return
+      end
 
-        begin
-          target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
-          @tails[target_info] = tw
-        rescue Errno::ENOENT
-          $log.warn "stat() for #{target_info.path} failed with ENOENT. Drop tail watcher for now."
-          # explicitly detach and unwatch watcher `tw`.
-          tw.unwatched = true
-          detach_watcher(tw, target_info.ino, false)
-        end
+      begin
+        target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
+        @tails.delete(target_info)
+        @tails[target_info] = tw
+        tw.on_notify
+      rescue Errno::ENOENT, Errno::EACCES => e
+        $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
+        # explicitly detach and unwatch watcher `tw`.
+        tw.unwatched = true
+        detach_watcher(tw, target_info.ino, false)
+      end
+    end
+
+    def start_watchers(targets_info)
+      targets_info.each_value {|target_info|
+        construct_watcher(target_info)
+        break if before_shutdown?
       }
     end
 
@@ -473,10 +492,19 @@ module Fluent::Plugin
         new_position_entry = @pf[target_info]
 
         if new_position_entry.read_inode == 0
+          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
+          # So it should be unwatched here explicitly.
+          rotated_tw.unwatched = true
+          # Make sure to delete old key, it has a different ino while the hash key is same.
+          @tails.delete(rotated_target_info)
           @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
+          @tails[new_target_info].on_notify
         end
       else
+        # Make sure to delete old key, it has a different ino while the hash key is same.
+        @tails.delete(rotated_target_info)
         @tails[new_target_info] = setup_watcher(new_target_info, pe)
+        @tails[new_target_info].on_notify
       end
       detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
     end
@@ -489,7 +517,7 @@ module Fluent::Plugin
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
-      tw.detach
+      tw.detach(@shutdown_start_time)
 
       tw.close if close_io
 
@@ -502,8 +530,25 @@ module Fluent::Plugin
     def detach_watcher_after_rotate_wait(tw, ino)
       # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
       # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
-      timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+      if @open_on_every_update
+        # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
+      elsif @read_bytes_limit_per_second < 0
+        # throttling isn't enabled, just wait @rotate_wait
+        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          detach_watcher(tw, ino)
+        end
+      else
+        # When the throttling feature is enabled, it might not reach EOF yet.
+        # Should ensure to read all contents before closing it, with keeping throttling.
+        start_time_to_wait = Fluent::Clock.now
+        timer = timer_execute(:in_tail_close_watcher, 1, repeat: true) do
+          elapsed = Fluent::Clock.now - start_time_to_wait
+          if tw.eof? && elapsed >= @rotate_wait
+            timer.detach
+            detach_watcher(tw, ino)
+          end
+        end
       end
     end
 
@@ -633,6 +678,7 @@ module Fluent::Plugin
         path: path,
         log: log,
         read_lines_limit: @read_lines_limit,
+        read_bytes_limit_per_second: @read_bytes_limit_per_second,
         open_on_every_update: @open_on_every_update,
         from_encoding: @from_encoding,
         encoding: @encoding,
@@ -700,8 +746,11 @@ module Fluent::Plugin
         @watchers << watcher
       end
 
-      def detach
-        @io_handler.on_notify if @io_handler
+      def detach(shutdown_start_time = nil)
+        if @io_handler
+          @io_handler.ready_to_shutdown(shutdown_start_time)
+          @io_handler.on_notify
+        end
         @line_buffer_timer_flusher&.close(self)
       end
 
@@ -712,10 +761,14 @@ module Fluent::Plugin
         end
       end
 
+      def eof?
+        @io_handler.eof?
+      end
+
       def on_notify
         begin
           stat = Fluent::FileWrapper.stat(@path)
-        rescue Errno::ENOENT
+        rescue Errno::ENOENT, Errno::EACCES
           # moved or deleted
           stat = nil
         end
@@ -786,7 +839,7 @@ module Fluent::Plugin
               # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
               # don't want to swap state because we need latest read offset in pos file even after rotate_wait
               if stat
-                target_info = TargetInfo.new(@path, stat)
+                target_info = TargetInfo.new(@path, stat.ino)
                 @update_watcher.call(target_info, @pe)
               end
             else
@@ -876,10 +929,16 @@ module Fluent::Plugin
       end
 
       class IOHandler
-        def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
+        BYTES_TO_READ = 8192
+        SHUTDOWN_TIMEOUT = 5
+
+        attr_accessor :shutdown_timeout
+
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
+          @read_bytes_limit_per_second = read_bytes_limit_per_second
           @receive_lines = receive_lines
           @open_on_every_update = open_on_every_update
           @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
@@ -888,6 +947,12 @@ module Fluent::Plugin
           @io = nil
           @notify_mutex = Mutex.new
           @log = log
+          @start_reading_time = nil
+          @number_bytes_read = 0
+          @shutdown_start_time = nil
+          @shutdown_timeout = SHUTDOWN_TIMEOUT
+          @shutdown_mutex = Mutex.new
+          @eof = false
 
           @log.info "following tail of #{@path}"
         end
@@ -896,6 +961,13 @@ module Fluent::Plugin
           @notify_mutex.synchronize { handle_notify }
         end
 
+        def ready_to_shutdown(shutdown_start_time = nil)
+          @shutdown_mutex.synchronize {
+            @shutdown_start_time =
+              shutdown_start_time || Fluent::Clock.now
+          }
+        end
+
         def close
           if @io && !@io.closed?
             @io.close
@@ -907,9 +979,41 @@ module Fluent::Plugin
           !!@io
         end
 
+        def eof?
+          @eof
+        end
+
         private
 
+        def limit_bytes_per_second_reached?
+          return false if @read_bytes_limit_per_second < 0 # not enabled by conf
+          return false if @number_bytes_read < @read_bytes_limit_per_second
+
+          @start_reading_time ||= Fluent::Clock.now
+          time_spent_reading = Fluent::Clock.now - @start_reading_time
+          @log.debug("time_spent_reading: #{time_spent_reading} #{ @watcher.path}")
+
+          if time_spent_reading < 1
+            true
+          else
+            @start_reading_time = nil
+            @number_bytes_read = 0
+            false
+          end
+        end
+
+        def should_shutdown_now?
+          # Ensure to read all remaining lines, but abort immediately if it
+          # seems to take too long time.
+          @shutdown_mutex.synchronize {
+            return false if @shutdown_start_time.nil?
+            return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout
+          }
+        end
+
         def handle_notify
+          return if limit_bytes_per_second_reached?
+
           with_io do |io|
             begin
               read_more = false
@@ -917,8 +1021,18 @@ module Fluent::Plugin
               if !io.nil? && @lines.empty?
                 begin
                   while true
-                    @fifo << io.readpartial(8192, @iobuf)
+                    @start_reading_time ||= Fluent::Clock.now
+                    data = io.readpartial(BYTES_TO_READ, @iobuf)
+                    @eof = false
+                    @number_bytes_read += data.bytesize
+                    @fifo << data
                     @fifo.read_lines(@lines)
+
+                    if limit_bytes_per_second_reached? || should_shutdown_now?
+                      # Just get out from tailing loop.
+                      read_more = false
+                      break
+                    end
                     if @lines.size >= @read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true
@@ -926,6 +1040,7 @@ module Fluent::Plugin
                     end
                   end
                 rescue EOFError
+                  @eof = true
                 end
               end
 
@@ -948,7 +1063,7 @@ module Fluent::Plugin
         rescue RangeError
           io.close if io
           raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
-        rescue Errno::ENOENT
+        rescue Errno::ENOENT, Errno::EACCES
           nil
         end
 
@@ -963,14 +1078,17 @@ module Fluent::Plugin
           else
             @io ||= open
             yield @io
+            @eof = true if @io.nil?
           end
         rescue WatcherSetupError => e
           close
+          @eof = true
           raise e
         rescue
           @log.error $!.to_s
           @log.error_backtrace
           close
+          @eof = true
         end
       end