fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/in_tail.rb

@@ -16,18 +16,42 @@
 
 require 'cool.io'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
 require 'fluent/event'
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/parser_multiline'
 
-module Fluent
-  class NewTailInput < Input
-    Plugin.register_input('tail', self)
+if Fluent.windows?
+  require_relative 'file_wrapper'
+else
+  Fluent::FileWrapper = File
+end
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('tail', self)
+
+    helpers :timer, :event_loop, :parser, :compat_parameters
+
+    class WatcherSetupError < StandardError
+      def initialize(msg)
+        @message = msg
+      end
+
+      def to_s
+        @message
+      end
+    end
+
+    FILE_PERMISSION = 0644
 
     def initialize
       super
       @paths = []
       @tails = {}
+      @pf_file = nil
+      @pf = nil
       @ignore_list = []
     end
 
@@ -43,6 +67,9 @@ module Fluent
     config_param :pos_file, :string, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, default: false
+    # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
+    # in_tail may raise a pos_file related error. This is a known issue but there is no such program on production.
+    # If we find such program / application, we will fix the problem.
     desc 'The interval of refreshing the list of watch file.'
     config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'
@@ -53,12 +80,16 @@ module Fluent
     config_param :emit_unmatched_lines, :bool, default: false
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'Enable the stat watcher based on inotify.'
+    config_param :enable_stat_watcher, :bool, default: true
     desc 'The encoding after conversion of the input.'
     config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
     config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
+    desc 'Open and close the file on every update instead of leaving it open until it gets rotated.'
+    config_param :open_on_every_update, :bool, default: false
     desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
     config_param :limit_recently_modified, :time, default: nil
     desc 'Enable the option to skip the refresh of watching list on startup.'
@@ -68,34 +99,56 @@ module Fluent
 
     attr_reader :paths
 
+    @@pos_file_paths = {}
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
+      unless parser_config["@type"]
+        raise Fluent::ConfigError, "parse/@type is required."
+      end
+
+      (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
+        parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
+      end
+
       super
 
+      if !@enable_watch_timer && !@enable_stat_watcher
+        raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
+      end
+
       @paths = @path.split(',').map {|path| path.strip }
       if @paths.empty?
-        raise ConfigError, "tail: 'path' parameter is required on tail input"
+        raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
 
-      unless @pos_file
+      # TODO: Use plugin_root_dir and storage plugin to store positions if available
+      if @pos_file
+        if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
+          plugin_id_using_this_path = @@pos_file_paths[@pos_file]
+          raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
+        end
+        @@pos_file_paths[@pos_file] = self.plugin_id
+      else
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
       end
 
-      configure_parser(conf)
       configure_tag
       configure_encoding
 
-      @multiline_mode = conf['format'] =~ /multiline/
+      @multiline_mode = parser_config["@type"] =~ /multiline/
       @receive_handler = if @multiline_mode
                            method(:parse_multilines)
                          else
                            method(:parse_singleline)
                          end
-    end
-
-    def configure_parser(conf)
-      @parser = Plugin.new_parser(conf['format'])
-      @parser.configure(conf)
+      @file_perm = system_config.file_permission || FILE_PERMISSION
+      @parser = parser_create(conf: parser_config)
     end
 
     def configure_tag
@@ -112,44 +165,52 @@ module Fluent
     def configure_encoding
       unless @encoding
         if @from_encoding
-          raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
+          raise Fluent::ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
         end
       end
 
       @encoding = parse_encoding_param(@encoding) if @encoding
       @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+      if @encoding && (@encoding == @from_encoding)
+        log.warn "'encoding' and 'from_encoding' are same encoding. No effect"
+      end
     end
 
     def parse_encoding_param(encoding_name)
       begin
         Encoding.find(encoding_name) if encoding_name
       rescue ArgumentError => e
-        raise ConfigError, e.message
+        raise Fluent::ConfigError, e.message
       end
     end
 
     def start
+      super
+
       if @pos_file
-        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
+        pos_file_dir = File.dirname(@pos_file)
+        FileUtils.mkdir_p(pos_file_dir) unless Dir.exist?(pos_file_dir)
+        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
         @pf_file.sync = true
         @pf = PositionFile.parse(@pf_file)
       end
 
-      @loop = Coolio::Loop.new
       refresh_watchers unless @skip_refresh_on_startup
-
-      @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
-      @refresh_trigger.attach(@loop)
-      @thread = Thread.new(&method(:run))
+      timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
     end
 
     def shutdown
-      @refresh_trigger.detach if @refresh_trigger && @refresh_trigger.attached?
-
-      stop_watchers(@tails.keys, true)
-      @loop.stop rescue nil # when all watchers are detached, `stop` raises RuntimeError. We can ignore this exception.
-      @thread.join
+      # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
+      stop_watchers(@tails.keys, immediate: true, remove_watcher: false)
       @pf_file.close if @pf_file
+
+      super
+    end
+
+    def close
+      super
+      # close file handles after all threads stopped (in #close of thread plugin helper)
+      close_watcher_handles
     end
 
     def expand_paths
@@ -160,20 +221,25 @@ module Fluent
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            is_file = !File.directory?(p)
-            if File.readable?(p) && is_file
-              if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
-                false
+            begin
+              is_file = !File.directory?(p)
+              if File.readable?(p) && is_file
+                if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                  false
+                else
+                  true
+                end
               else
-                true
-              end
-            else
-              if is_file
-                unless @ignore_list.include?(path)
-                  log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                  @ignore_list << path if @ignore_repeated_permission_error
+                if is_file
+                  unless @ignore_list.include?(path)
+                    log.warn "#{p} unreadable. It is excluded and would be examined next time."
+                    @ignore_list << path if @ignore_repeated_permission_error
+                  end
                 end
+                false
               end
+            rescue Errno::ENOENT
+              log.debug("#{p} is missing after refresh file list")
               false
             end
           }
@@ -195,18 +261,32 @@ module Fluent
       target_paths = expand_paths
       existence_paths = @tails.keys
 
+      log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" }
+
       unwatched = existence_paths - target_paths
       added = target_paths - existence_paths
 
-      stop_watchers(unwatched, false, true) unless unwatched.empty?
+      stop_watchers(unwatched, immediate: false, unwatched: true) unless unwatched.empty?
       start_watchers(added) unless added.empty?
     end
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
-      tw.attach(@loop)
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+      tw.attach do |watcher|
+        event_loop_attach(watcher.timer_trigger) if watcher.timer_trigger
+        event_loop_attach(watcher.stat_trigger) if watcher.stat_trigger
+      end
       tw
+    rescue => e
+      if tw
+        tw.detach { |watcher|
+          event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
+          event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
+        }
+        tw.close
+      end
+      raise e
     end
 
     def start_watchers(paths)
@@ -216,65 +296,86 @@ module Fluent
           pe = @pf[path]
           if @read_from_head && pe.read_inode.zero?
             begin
-              pe.update(File::Stat.new(path).ino, 0)
+              pe.update(Fluent::FileWrapper.stat(path).ino, 0)
             rescue Errno::ENOENT
               $log.warn "#{path} not found. Continuing without tailing it."
             end
           end
         end
 
-        @tails[path] = setup_watcher(path, pe)
+        begin
+          tw = setup_watcher(path, pe)
+        rescue WatcherSetupError => e
+          log.warn "Skip #{path} because unexpected setup error happens: #{e}"
+          next
+        end
+        @tails[path] = tw
       }
     end
 
-    def stop_watchers(paths, immediate = false, unwatched = false)
+    def stop_watchers(paths, immediate: false, unwatched: false, remove_watcher: true)
       paths.each { |path|
-        tw = @tails.delete(path)
+        tw = remove_watcher ? @tails.delete(path) : @tails[path]
         if tw
           tw.unwatched = unwatched
           if immediate
-            close_watcher(tw, false)
+            detach_watcher(tw, false)
           else
-            close_watcher_after_rotate_wait(tw)
+            detach_watcher_after_rotate_wait(tw)
           end
         end
       }
     end
 
+    def close_watcher_handles
+      @tails.keys.each do |path|
+        tw = @tails.delete(path)
+        if tw
+          tw.close
+        end
+      end
+    end
+
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(path, pe)
+      if @pf
+        unless pe.read_inode == @pf[path].read_inode
+          log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
+          return
+        end
+      end
       rotated_tw = @tails[path]
       @tails[path] = setup_watcher(path, pe)
-      close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
+      detach_watcher_after_rotate_wait(rotated_tw) if rotated_tw
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
     # It causes 'can't modify string; temporarily locked' error in IOHandler
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
-    def close_watcher(tw, close_io = true)
-      tw.close(close_io)
+    def detach_watcher(tw, close_io = true)
+      tw.detach { |watcher|
+        event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
+        event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
+      }
+      tw.close if close_io
       flush_buffer(tw)
       if tw.unwatched && @pf
         @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
       end
     end
 
-    def close_watcher_after_rotate_wait(tw)
-      closer = TailWatcher::Closer.new(@rotate_wait, tw, log, &method(:close_watcher))
-      closer.attach(@loop)
+    def detach_watcher_after_rotate_wait(tw)
+      # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
+      # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
+      timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+        detach_watcher(tw)
+      end
     end
 
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        if @encoding
-          if @from_encoding
-            lb.encode!(@encoding, @from_encoding)
-          else
-            lb.force_encoding(@encoding)
-          end
-        end
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -291,14 +392,7 @@ module Fluent
       end
     end
 
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
-    # @return true if no error or unrecoverable error happens in emit action. false if got BufferQueueLimitError
+    # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
@@ -309,7 +403,7 @@ module Fluent
               end
         begin
           router.emit_stream(tag, es)
-        rescue BufferQueueLimitError
+        rescue Fluent::Plugin::Buffer::BufferOverflowError
           return false
         rescue
           # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
@@ -323,13 +417,6 @@ module Fluent
     def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
-        if @encoding
-          if @from_encoding
-            line.encode!(@encoding, @from_encoding)
-          else
-            line.force_encoding(@encoding)
-          end
-        end
         @parser.parse(line) { |time, record|
           if time && record
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?
@@ -338,9 +425,9 @@ module Fluent
             if @emit_unmatched_lines
               record = {'unmatched_line' => line}
               record[@path_key] ||= tail_watcher.path unless @path_key.nil?
-              es.add(::Fluent::Engine.now, record)
+              es.add(Fluent::EventTime.now, record)
             end
-            log.warn "pattern not match: #{line.inspect}"
+            log.warn "pattern not matched: #{line.inspect}"
           end
         }
       rescue => e
@@ -350,7 +437,7 @@ module Fluent
     end
 
     def parse_singleline(lines, tail_watcher)
-      es = MultiEventStream.new
+      es = Fluent::MultiEventStream.new
       lines.each { |line|
         convert_line_to_event(line, es, tail_watcher)
       }
@@ -359,7 +446,7 @@ module Fluent
 
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
-      es = MultiEventStream.new
+      es = Fluent::MultiEventStream.new
       if @parser.has_firstline?
         tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
         lines.each { |line|
@@ -396,28 +483,35 @@ module Fluent
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
         @enable_watch_timer = enable_watch_timer
+        @enable_stat_watcher = enable_stat_watcher
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
+        @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
+        @timer_trigger = @enable_watch_timer ? TimerTrigger.new(1, log, &method(:on_notify)) : nil
 
-        @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
-
-        @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
+        @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
         @io_handler = nil
         @log = log
 
         @line_buffer_timer_flusher = line_buffer_timer_flusher
+        @from_encoding = from_encoding
+        @encoding = encoding
+        @open_on_every_update = open_on_every_update
       end
 
       attr_reader :path
+      attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
+      attr_reader :from_encoding, :encoding
+      attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
+      attr_accessor :timer_trigger
       attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
 
@@ -429,37 +523,40 @@ module Fluent
         @receive_lines.call(lines, self)
       end
 
-      def attach(loop)
-        @timer_trigger.attach(loop) if @enable_watch_timer
-        @stat_trigger.attach(loop)
+      def attach
         on_notify
+        yield self
       end
 
       def detach
-        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger.attached?
+        yield self
+        @io_handler.on_notify if @io_handler
       end
 
-      def close(close_io = true)
-        if close_io && @io_handler
-          @io_handler.on_notify
+      def close
+        if @io_handler
           @io_handler.close
+          @io_handler = nil
         end
-        detach
       end
 
       def on_notify
-        @rotate_handler.on_notify if @rotate_handler
+        begin
+          stat = Fluent::FileWrapper.stat(@path)
+        rescue Errno::ENOENT
+          # moved or deleted
+          stat = nil
+        end
+
+        @rotate_handler.on_notify(stat) if @rotate_handler
         @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher
-        return unless @io_handler
-        @io_handler.on_notify
+        @io_handler.on_notify if @io_handler
       end
 
-      def on_rotate(io)
-        if @io_handler == nil
-          if io
+      def on_rotate(stat)
+        if @io_handler.nil?
+          if stat
             # first time
-            stat = io.stat
             fsize = stat.size
             inode = stat.ino
 
@@ -469,14 +566,15 @@ module Fluent
               # assuming following situation:
               #   a) file was once renamed and backed, or
               #   b) symlink or hardlink to the same file is recreated
-              # in either case, seek to the saved position
-              pos = @pe.read_pos
+              # in either case of a and b, seek to the saved position
+              #   c) file was once renamed, truncated and then backed
+              # in this case, consider it truncated
+              @pe.update(inode, 0) if fsize < @pe.read_pos
             elsif last_inode != 0
               # this is FilePositionEntry and fluentd once started.
               # read data from the head of the rotated file.
               # logs never duplicate because this file is a rotated new file.
-              pos = 0
-              @pe.update(inode, pos)
+              @pe.update(inode, 0)
             else
               # this is MemoryPositionEntry or this is the first time fluentd started.
               # seek to the end of the any files.
@@ -485,161 +583,224 @@ module Fluent
               pos = @read_from_head ? 0 : fsize
               @pe.update(inode, pos)
             end
-            io.seek(pos)
-
-            @io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
+            @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
           else
             @io_handler = NullIOHandler.new
           end
         else
-          log_msg = "detected rotation of #{@path}"
-          log_msg << "; waiting #{@rotate_wait} seconds" if @io_handler.io  # wait rotate_time if previous file is exist
-          @log.info log_msg
+          watcher_needs_update = false
 
-          if io
-            stat = io.stat
+          if stat
             inode = stat.ino
             if inode == @pe.read_inode # truncated
-              @pe.update_pos(stat.size)
-              io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
+              @pe.update_pos(0)
               @io_handler.close
-              @io_handler = io_handler
-            elsif @io_handler.io.nil? # There is no previous file. Reuse TailWatcher
-              @pe.update(inode, io.pos)
-              io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
-              @io_handler = io_handler
+            elsif !@io_handler.opened? # There is no previous file. Reuse TailWatcher
+              @pe.update(inode, 0)
             else # file is rotated and new file found
-              detach
-              @update_watcher.call(@path, swap_state(@pe))
+              watcher_needs_update = true
+              # Handle the old log file before renewing TailWatcher [fluentd#1055]
+              @io_handler.on_notify
             end
           else # file is rotated and new file not found
             # Clear RotateHandler to avoid duplicated file watch in same path.
             @rotate_handler = nil
-            detach
-            @update_watcher.call(@path, swap_state(@pe))
+            watcher_needs_update = true
           end
-        end
 
-        def swap_state(pe)
-          # Use MemoryPositionEntry for rotated file temporary
-          mpe = MemoryPositionEntry.new
-          mpe.update(pe.read_inode, pe.read_pos)
-          @pe = mpe
-          @io_handler.pe = mpe # Don't re-create IOHandler because IOHandler has an internal buffer.
+          log_msg = "detected rotation of #{@path}"
+          log_msg << "; waiting #{@rotate_wait} seconds" if watcher_needs_update # wait rotate_time if previous file exists
+          @log.info log_msg
 
-          pe # This pe will be updated in on_rotate after TailWatcher is initialized
+          if watcher_needs_update
+            @update_watcher.call(@path, swap_state(@pe))
+          else
+            @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
+          end
         end
       end
 
-      class TimerWatcher < Coolio::TimerWatcher
-        def initialize(interval, repeat, log, &callback)
+      def swap_state(pe)
+        # Use MemoryPositionEntry for rotated file temporary
+        mpe = MemoryPositionEntry.new
+        mpe.update(pe.read_inode, pe.read_pos)
+        @pe = mpe
+        pe # This pe will be updated in on_rotate after TailWatcher is initialized
+      end
+
+      class TimerTrigger < Coolio::TimerWatcher
+        def initialize(interval, log, &callback)
           @callback = callback
           @log = log
-          super(interval, repeat)
+          super(interval, true)
         end
 
         def on_timer
           @callback.call
-        rescue
-          # TODO log?
-          @log.error $!.to_s
+        rescue => e
+          @log.error e.to_s
           @log.error_backtrace
         end
       end
 
       class StatWatcher < Coolio::StatWatcher
-        def initialize(path, log, &callback)
+        def initialize(watcher, &callback)
+          @watcher = watcher
           @callback = callback
-          @log = log
-          super(path)
+          super(watcher.path)
         end
 
         def on_change(prev, cur)
           @callback.call
         rescue
           # TODO log?
-          @log.error $!.to_s
-          @log.error_backtrace
+          @watcher.log.error $!.to_s
+          @watcher.log.error_backtrace
+        end
+      end
+
+      class FIFO
+        def initialize(from_encoding, encoding)
+          @from_encoding = from_encoding
+          @encoding = encoding
+          @need_enc = from_encoding != encoding
+          @buffer = ''.force_encoding(from_encoding)
+          @eol = "\n".encode(from_encoding).freeze
+        end
+
+        attr_reader :from_encoding, :encoding, :buffer
+
+        def <<(chunk)
+          # Although "chunk" is most likely transient besides String#force_encoding itself
+          # won't affect the actual content of it, it is also probable that "chunk" is
+          # a reused buffer and changing its encoding causes some problems on the caller side.
+          #
+          # Actually, the caller here is specific and "chunk" comes from IO#partial with
+          # the second argument, which the function always returns as a return value.
+          #
+          # Feeding a string that has its encoding attribute set to any double-byte or
+          # quad-byte encoding to IO#readpartial as the second arguments results in an
+          # assertion failure on Ruby < 2.4.0 for unknown reasons.
+          orig_encoding = chunk.encoding
+          chunk.force_encoding(from_encoding)
+          @buffer << chunk
+          # Thus the encoding needs to be reverted back here
+          chunk.force_encoding(orig_encoding)
+        end
+
+        def convert(s)
+          if @need_enc
+            s.encode!(@encoding, @from_encoding)
+          else
+            s
+          end
+        rescue
+          s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
         end
-      end
 
-      class Closer < Coolio::TimerWatcher
-        def initialize(interval, tw, log, &callback)
-          @callback = callback
-          @tw = tw
-          @log = log
-          super(interval, false)
+        def next_line
+          idx = @buffer.index(@eol)
+          convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
         end
 
-        def on_timer
-          @callback.call(@tw)
-        rescue => e
-          @log.error e.to_s
-          @log.error_backtrace(e.backtrace)
-        ensure
-          detach
+        def bytesize
+          @buffer.bytesize
         end
       end
 
       class IOHandler
-        def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
-          @log = log
-          @log.info "following tail of #{io.path}" if first
-          @io = io
-          @pe = pe
-          @read_lines_limit = read_lines_limit
+        def initialize(watcher, &receive_lines)
+          @watcher = watcher
           @receive_lines = receive_lines
-          @buffer = ''.force_encoding('ASCII-8BIT')
+          @fifo = FIFO.new(@watcher.from_encoding || Encoding::ASCII_8BIT, @watcher.encoding || Encoding::ASCII_8BIT)
           @iobuf = ''.force_encoding('ASCII-8BIT')
           @lines = []
+          @io = nil
+          @notify_mutex = Mutex.new
+          @watcher.log.info "following tail of #{@watcher.path}"
         end
 
-        attr_reader :io
-        attr_accessor :pe
-
         def on_notify
-          begin
-            read_more = false
+          @notify_mutex.synchronize { handle_notify }
+        end
 
-            if @lines.empty?
-              begin
-                while true
-                  if @buffer.empty?
-                    @io.readpartial(2048, @buffer)
-                  else
-                    @buffer << @io.readpartial(2048, @iobuf)
-                  end
-                  while idx = @buffer.index("\n".freeze)
-                    @lines << @buffer.slice!(0, idx + 1)
-                  end
-                  if @lines.size >= @read_lines_limit
-                    # not to use too much memory in case the file is very large
-                    read_more = true
-                    break
+        def handle_notify
+          with_io do |io|
+            begin
+              read_more = false
+
+              if !io.nil? && @lines.empty?
+                begin
+                  while true
+                    @fifo << io.readpartial(8192, @iobuf)
+                    while (line = @fifo.next_line)
+                      @lines << line
+                    end
+                    if @lines.size >= @watcher.read_lines_limit
+                      # not to use too much memory in case the file is very large
+                      read_more = true
+                      break
+                    end
                   end
+                rescue EOFError
                 end
-              rescue EOFError
               end
-            end
 
-            unless @lines.empty?
-              if @receive_lines.call(@lines)
-                @pe.update_pos(@io.pos - @buffer.bytesize)
-                @lines.clear
-              else
-                read_more = false
+              unless @lines.empty?
+                if @receive_lines.call(@lines)
+                  @watcher.pe.update_pos(io.pos - @fifo.bytesize)
+                  @lines.clear
+                else
+                  read_more = false
+                end
               end
-            end
-          end while read_more
-
-        rescue
-          @log.error $!.to_s
-          @log.error_backtrace
-          close
+            end while read_more
+          end
         end
 
         def close
-          @io.close unless @io.closed?
+          if @io && !@io.closed?
+            @io.close
+            @io = nil
+          end
+        end
+
+        def opened?
+          !!@io
+        end
+
+        def open
+          io = Fluent::FileWrapper.open(@watcher.path)
+          io.seek(@watcher.pe.read_pos + @fifo.bytesize)
+          io
+        rescue RangeError
+          io.close if io
+          raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def with_io
+          begin
+            if @watcher.open_on_every_update
+              io = open
+              begin
+                yield io
+              ensure
+                io.close unless io.nil?
+              end
+            else
+              @io ||= open
+              yield @io
+            end
+          rescue WatcherSetupError => e
+            close
+            raise e
+          rescue
+            @watcher.log.error $!.to_s
+            @watcher.log.error_backtrace
+            close
+          end
         end
       end
 
@@ -655,48 +816,43 @@ module Fluent
 
         def close
         end
+
+        def opened?
+          false
+        end
       end
 
       class RotateHandler
-        def initialize(path, log, &on_rotate)
-          @path = path
+        def initialize(watcher, &on_rotate)
+          @watcher = watcher
           @inode = nil
           @fsize = -1  # first
           @on_rotate = on_rotate
-          @log = log
         end
 
-        def on_notify
-          begin
-            stat = File.stat(@path)
-            inode = stat.ino
-            fsize = stat.size
-          rescue Errno::ENOENT
-            # moved or deleted
+        def on_notify(stat)
+          if stat.nil?
             inode = nil
             fsize = 0
+          else
+            inode = stat.ino
+            fsize = stat.size
           end
 
           begin
             if @inode != inode || fsize < @fsize
-              # rotated or truncated
-              begin
-                io = File.open(@path)
-              rescue Errno::ENOENT
-              end
-              @on_rotate.call(io)
+              @on_rotate.call(stat)
             end
             @inode = inode
             @fsize = fsize
           end
 
         rescue
-          @log.error $!.to_s
-          @log.error_backtrace
+          @watcher.log.error $!.to_s
+          @watcher.log.error_backtrace
         end
       end
 
-
       class LineBufferTimerFlusher
         def initialize(log, flush_interval, &flush_method)
           @log = log
@@ -721,12 +877,12 @@ module Fluent
       end
     end
 
-
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
 
-      def initialize(file, map, last_pos)
+      def initialize(file, file_mutex, map, last_pos)
         @file = file
+        @file_mutex = file_mutex
         @map = map
         @last_pos = last_pos
       end
@@ -736,31 +892,34 @@ module Fluent
           return m
         end
 
-        @file.pos = @last_pos
-        @file.write path
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "0000000000000000\t0000000000000000\n"
-        @last_pos = @file.pos
-
-        @map[path] = FilePositionEntry.new(@file, seek)
+        @file_mutex.synchronize {
+          @file.pos = @last_pos
+          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
+          seek = @last_pos + path.bytesize + 1
+          @last_pos = @file.pos
+          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+        }
       end
 
       def self.parse(file)
         compact(file)
 
+        file_mutex = Mutex.new
         map = {}
         file.pos = 0
         file.each_line {|line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
           pos = m[2].to_i(16)
           ino = m[3].to_i(16)
           seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, seek)
+          map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
         }
-        new(file, map, file.pos)
+        new(file, file_mutex, map, file.pos)
       end
 
       # Clean up unwatched file entries
@@ -768,7 +927,10 @@ module Fluent
         file.pos = 0
         existent_entries = file.each_line.map { |line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
           pos = m[2].to_i(16)
           ino = m[3].to_i(16)
@@ -791,536 +953,37 @@ module Fluent
       LN_OFFSET = 33
       SIZE = 34
 
-      def initialize(file, seek)
+      def initialize(file, file_mutex, seek, pos, inode)
         @file = file
+        @file_mutex = file_mutex
         @seek = seek
+        @pos = pos
+        @inode = inode
       end
 
       def update(ino, pos)
-        @file.pos = @seek
-        @file.write "%016x\t%016x" % [pos, ino]
-      end
-
-      def update_pos(pos)
-        @file.pos = @seek
-        @file.write "%016x" % pos
-      end
-
-      def read_inode
-        @file.pos = @seek + INO_OFFSET
-        raw = @file.read(16)
-        raw ? raw.to_i(16) : 0
-      end
-
-      def read_pos
-        @file.pos = @seek
-        raw = @file.read(16)
-        raw ? raw.to_i(16) : 0
-      end
-    end
-
-    class MemoryPositionEntry
-      def initialize
-        @pos = 0
-        @inode = 0
-      end
-
-      def update(ino, pos)
-        @inode = ino
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x\t%016x" % [pos, ino]
+        }
         @pos = pos
+        @inode = ino
       end
 
       def update_pos(pos)
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x" % pos
+        }
         @pos = pos
       end
 
-      def read_pos
-        @pos
-      end
-
       def read_inode
         @inode
       end
-    end
-  end
-
-  # This TailInput is for existence plugins which extends old in_tail
-  # This class will be removed after release v1.
-  class TailInput < Input
-    def initialize
-      super
-      @paths = []
-    end
-
-    config_param :path, :string
-    config_param :tag, :string
-    config_param :rotate_wait, :time, :default => 5
-    config_param :pos_file, :string, :default => nil
-
-    attr_reader :paths
-
-    def configure(conf)
-      super
-
-      @paths = @path.split(',').map {|path| path.strip }
-      if @paths.empty?
-        raise ConfigError, "tail: 'path' parameter is required on tail input"
-      end
-
-      unless @pos_file
-        $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
-        $log.warn "this parameter is highly recommended to save the position to resume tailing."
-      end
-
-      configure_parser(conf)
-    end
-
-    def configure_parser(conf)
-      @parser = TextParser.new
-      @parser.configure(conf)
-    end
-
-    def start
-      if @pos_file
-        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
-        @pf_file.sync = true
-        @pf = PositionFile.parse(@pf_file)
-      end
-
-      @loop = Coolio::Loop.new
-      @tails = @paths.map {|path|
-        pe = @pf ? @pf[path] : MemoryPositionEntry.new
-        tw = TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
-        tw.log = log
-        tw
-      }
-      @tails.each {|tail|
-        tail.attach(@loop)
-      }
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      @tails.each {|tail|
-        tail.close
-      }
-      @loop.stop
-      @thread.join
-      @pf_file.close if @pf_file
-    end
-
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
-      log.error_backtrace
-    end
-
-    def receive_lines(lines)
-      es = MultiEventStream.new
-      lines.each {|line|
-        begin
-          line.chomp!  # remove \n
-          time, record = parse_line(line)
-          if time && record
-            es.add(time, record)
-          else
-            log.warn "pattern not match: #{line.inspect}"
-          end
-        rescue
-          log.warn line.dump, :error=>$!.to_s
-          log.debug_backtrace
-        end
-      }
-
-      unless es.empty?
-        begin
-          router.emit_stream(@tag, es)
-        rescue
-          # ignore errors. Engine shows logs and backtraces.
-        end
-      end
-    end
-
-    def parse_line(line)
-      return @parser.parse(line)
-    end
-
-    class TailWatcher
-      def initialize(path, rotate_wait, pe, &receive_lines)
-        @path = path
-        @rotate_wait = rotate_wait
-        @pe = pe || MemoryPositionEntry.new
-        @receive_lines = receive_lines
-
-        @rotate_queue = []
-
-        @timer_trigger = TimerWatcher.new(1, true, &method(:on_notify))
-        @stat_trigger = StatWatcher.new(path, &method(:on_notify))
-
-        @rotate_handler = RotateHandler.new(path, &method(:on_rotate))
-        @io_handler = nil
-        @log = $log
-      end
-
-      # We use accessor approach to assign each logger, not passing log object at initialization,
-      # because several plugins depend on these internal classes.
-      # This approach avoids breaking plugins with new log_level option.
-      attr_accessor :log
-
-      def log=(logger)
-        @log = logger
-        @timer_trigger.log = logger
-        @stat_trigger.log = logger
-        @rotate_handler.log = logger
-      end
-
-      def attach(loop)
-        @timer_trigger.attach(loop)
-        @stat_trigger.attach(loop)
-        on_notify
-      end
-
-      def detach
-        @timer_trigger.detach if @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger.attached?
-      end
-
-      def close
-        @rotate_queue.reject! {|req|
-          req.io.close
-          true
-        }
-        detach
-      end
-
-      def on_notify
-        @rotate_handler.on_notify
-        return unless @io_handler
-        @io_handler.on_notify
-
-        # proceeds rotate queue
-        return if @rotate_queue.empty?
-        @rotate_queue.first.tick
-
-        while @rotate_queue.first.ready?
-          if io = @rotate_queue.first.io
-            stat = io.stat
-            inode = stat.ino
-            if inode == @pe.read_inode
-              # rotated file has the same inode number with the last file.
-              # assuming following situation:
-              #   a) file was once renamed and backed, or
-              #   b) symlink or hardlink to the same file is recreated
-              # in either case, seek to the saved position
-              pos = @pe.read_pos
-            else
-              pos = io.pos
-            end
-            @pe.update(inode, pos)
-            io_handler = IOHandler.new(io, @pe, log, &@receive_lines)
-          else
-            io_handler = NullIOHandler.new
-          end
-          @io_handler.close
-          @io_handler = io_handler
-          @rotate_queue.shift
-          break if @rotate_queue.empty?
-        end
-      end
-
-      def on_rotate(io)
-        if @io_handler == nil
-          if io
-            # first time
-            stat = io.stat
-            fsize = stat.size
-            inode = stat.ino
-
-            last_inode = @pe.read_inode
-            if inode == last_inode
-              # seek to the saved position
-              pos = @pe.read_pos
-            elsif last_inode != 0
-              # this is FilePositionEntry and fluentd once started.
-              # read data from the head of the rotated file.
-              # logs never duplicate because this file is a rotated new file.
-              pos = 0
-              @pe.update(inode, pos)
-            else
-              # this is MemoryPositionEntry or this is the first time fluentd started.
-              # seek to the end of the any files.
-              # logs may duplicate without this seek because it's not sure the file is
-              # existent file or rotated new file.
-              pos = fsize
-              @pe.update(inode, pos)
-            end
-            io.seek(pos)
-
-            @io_handler = IOHandler.new(io, @pe, log, &@receive_lines)
-          else
-            @io_handler = NullIOHandler.new
-          end
-
-        else
-          if io && @rotate_queue.find {|req| req.io == io }
-            return
-          end
-          last_io = @rotate_queue.empty? ? @io_handler.io : @rotate_queue.last.io
-          if last_io == nil
-            log.info "detected rotation of #{@path}"
-            # rotate imeediately if previous file is nil
-            wait = 0
-          else
-            log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
-            wait = @rotate_wait
-            wait -= @rotate_queue.first.wait unless @rotate_queue.empty?
-          end
-          @rotate_queue << RotationRequest.new(io, wait)
-        end
-      end
-
-      class TimerWatcher < Coolio::TimerWatcher
-        def initialize(interval, repeat, &callback)
-          @callback = callback
-          @log = $log
-          super(interval, repeat)
-        end
-
-        attr_accessor :log
-
-        def on_timer
-          @callback.call
-        rescue
-          # TODO log?
-          @log.error $!.to_s
-          @log.error_backtrace
-        end
-      end
-
-      class StatWatcher < Coolio::StatWatcher
-        def initialize(path, &callback)
-          @callback = callback
-          @log = $log
-          super(path)
-        end
-
-        attr_accessor :log
-
-        def on_change(prev, cur)
-          @callback.call
-        rescue
-          # TODO log?
-          @log.error $!.to_s
-          @log.error_backtrace
-        end
-      end
-
-      class RotationRequest
-        def initialize(io, wait)
-          @io = io
-          @wait = wait
-        end
-
-        attr_reader :io, :wait
-
-        def tick
-          @wait -= 1
-        end
-
-        def ready?
-          @wait <= 0
-        end
-      end
-
-      MAX_LINES_AT_ONCE = 1000
-
-      class IOHandler
-        def initialize(io, pe, log, &receive_lines)
-          @log = log
-          @log.info "following tail of #{io.path}"
-          @io = io
-          @pe = pe
-          @receive_lines = receive_lines
-          @buffer = ''.force_encoding('ASCII-8BIT')
-          @iobuf = ''.force_encoding('ASCII-8BIT')
-        end
-
-        attr_reader :io
-
-        def on_notify
-          begin
-            lines = []
-            read_more = false
-
-            begin
-              while true
-                if @buffer.empty?
-                  @io.read_nonblock(2048, @buffer)
-                else
-                  @buffer << @io.read_nonblock(2048, @iobuf)
-                end
-                while line = @buffer.slice!(/.*?\n/m)
-                  lines << line
-                end
-                if lines.size >= MAX_LINES_AT_ONCE
-                  # not to use too much memory in case the file is very large
-                  read_more = true
-                  break
-                end
-              end
-            rescue EOFError
-            end
-
-            unless lines.empty?
-              @receive_lines.call(lines)
-              @pe.update_pos(@io.pos - @buffer.bytesize)
-            end
-
-          end while read_more
-
-        rescue
-          @log.error $!.to_s
-          @log.error_backtrace
-          close
-        end
-
-        def close
-          @io.close unless @io.closed?
-        end
-      end
-
-      class NullIOHandler
-        def initialize
-        end
-
-        def io
-        end
-
-        def on_notify
-        end
-
-        def close
-        end
-      end
-
-      class RotateHandler
-        def initialize(path, &on_rotate)
-          @path = path
-          @inode = nil
-          @fsize = -1  # first
-          @on_rotate = on_rotate
-          @log = $log
-        end
-
-        attr_accessor :log
-
-        def on_notify
-          begin
-            io = File.open(@path)
-            stat = io.stat
-            inode = stat.ino
-            fsize = stat.size
-          rescue Errno::ENOENT
-            # moved or deleted
-            inode = nil
-            fsize = 0
-          end
-
-          begin
-            if @inode != inode || fsize < @fsize
-              # rotated or truncated
-              @on_rotate.call(io)
-              io = nil
-            end
-
-            @inode = inode
-            @fsize = fsize
-          ensure
-            io.close if io
-          end
-
-        rescue
-          @log.error $!.to_s
-          @log.error_backtrace
-        end
-      end
-    end
-
-
-    class PositionFile
-      def initialize(file, map, last_pos)
-        @file = file
-        @map = map
-        @last_pos = last_pos
-      end
-
-      def [](path)
-        if m = @map[path]
-          return m
-        end
-
-        @file.pos = @last_pos
-        @file.write path
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "0000000000000000\t00000000\n"
-        @last_pos = @file.pos
-
-        @map[path] = FilePositionEntry.new(@file, seek)
-      end
-
-      def self.parse(file)
-        map = {}
-        file.pos = 0
-        file.each_line {|line|
-          m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
-          path = m[1]
-          pos = m[2].to_i(16)
-          ino = m[3].to_i(16)
-          seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, seek)
-        }
-        new(file, map, file.pos)
-      end
-    end
-
-    # pos               inode
-    # ffffffffffffffff\tffffffff\n
-    class FilePositionEntry
-      POS_SIZE = 16
-      INO_OFFSET = 17
-      INO_SIZE = 8
-      LN_OFFSET = 25
-      SIZE = 26
-
-      def initialize(file, seek)
-        @file = file
-        @seek = seek
-      end
-
-      def update(ino, pos)
-        @file.pos = @seek
-        @file.write "%016x\t%08x" % [pos, ino]
-        @inode = ino
-      end
-
-      def update_pos(pos)
-        @file.pos = @seek
-        @file.write "%016x" % pos
-      end
-
-      def read_inode
-        @file.pos = @seek + INO_OFFSET
-        raw = @file.read(8)
-        raw ? raw.to_i(16) : 0
-      end
 
       def read_pos
-        @file.pos = @seek
-        raw = @file.read(16)
-        raw ? raw.to_i(16) : 0
+        @pos
       end
     end