fluentd 1.9.2 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05431f1c8da9d7be65503df9430efe567e0a6417d4d9bd9c32376f392528ccac
-  data.tar.gz: 0d309344866152cf644315ab84a43cd6586593b182bcb1e1548653dbb98ad360
+  metadata.gz: d67b7f6379705086047bb2d3adb34fc3c1dbb3c29b1dcd901b9cac62da28978f
+  data.tar.gz: ca8ee48ac0a2d435a87f189812e48c09d9d2947c06b6c2d3ce0e35a7e4c126e1
 SHA512:
-  metadata.gz: ee1f28cc7c10f374885b356d1273b7eec1af653314c06c4760f0ab0be2c33b39098e066d774261beda53af4022d6cae37dbdc87ce2896df64c03962f0b989eff
-  data.tar.gz: 61f1ae9453ab2dc31620cc264d1d6f68c2fb1b7bb694db8e9e4d535a6fc9a291a9ca93952f627961461e21f95a31a6a89491213c999742d28263a27021a8894d
+  metadata.gz: ab9d1deb64f93efa57575b28d2f72eb8ad4ef1874a4d86d6cdf38110968d3b7ea18773e658589151d338d6cc6b24187468c2864fa65616f6dbeb483fbe7bf600
+  data.tar.gz: 147c88ed0e30a82f2b8e72cb6600737c9c44a83eecb1d974d5dc26634918bfb725e813ca9f65d56689067c979e1f488f29a2c899dafaa41ea9128ad552f285d6

data/.gitignore

@@ -26,4 +26,5 @@ coverage/*
 .vagrant/
 cov-int/
 cov-fluentd.tar.gz
-.vscode
+.vscode
+.idea/

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # v1.9
 
+## Release v1.9.3 - 2020/03/05
+
+### Enhancement
+
+* in_tail: Emit buffered lines as `unmatched_line` at shutdown phase when `emit_unmatched_lines true`
+  https://github.com/fluent/fluentd/pull/2837
+* Specify directory mode explicitly
+  https://github.com/fluent/fluentd/pull/2827
+* server helper: Change SSLError log level to warn in accept
+  https://github.com/fluent/fluentd/pull/2861
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2829
+  https://github.com/fluent/fluentd/pull/2830
+  https://github.com/fluent/fluentd/pull/2832
+  https://github.com/fluent/fluentd/pull/2836
+  https://github.com/fluent/fluentd/pull/2838
+  https://github.com/fluent/fluentd/pull/2842
+  https://github.com/fluent/fluentd/pull/2843
+
+### Bug fix
+
+* buffer: Add seq to metadata that it can be unique
+  https://github.com/fluent/fluentd/pull/2824
+  https://github.com/fluent/fluentd/pull/2853
+* buffer: Use `Tempfile` as binmode for decompression
+  https://github.com/fluent/fluentd/pull/2847
+
+### Misc
+
+* Add `.idea` to git ignore file
+  https://github.com/fluent/fluentd/pull/2834
+* appveyor: Fix tests
+  https://github.com/fluent/fluentd/pull/2853
+  https://github.com/fluent/fluentd/pull/2855
+* Update pem for test
+  https://github.com/fluent/fluentd/pull/2839
+
 ## Release v1.9.2 - 2020/02/13
 
 ### Enhancement

data/lib/fluent/env.rb

@@ -22,6 +22,8 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
+  DEFAULT_DIR_PERMISSION = 0755
+  DEFAULT_FILE_PERMISSION = 0644
 
   def self.windows?
     ServerEngine.windows?

data/lib/fluent/plugin/buf_file.rb

@@ -31,8 +31,6 @@ module Fluent
       DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB
       DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB, same with v0.12 (TimeSlicedOutput + buf_file)
 
-      DIR_PERMISSION = 0755
-
       desc 'The path where buffer chunks are stored.'
       config_param :path, :string, default: nil
       desc 'The suffix of buffer chunks'
@@ -108,7 +106,7 @@ module Fluent
         if @dir_permission
           @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)
         else
-          @dir_permission = system_config.dir_permission || DIR_PERMISSION
+          @dir_permission = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
         end
       end
 
@@ -166,7 +164,17 @@ module Fluent
 
           case chunk.state
           when :staged
-            stage[chunk.metadata] = chunk
+            # unstaged chunk created at Buffer#write_step_by_step is identified as the staged chunk here because FileChunk#assume_chunk_state checks only the file name.
+            # https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L663
+            # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and changing them to staged mode in Buffer#write
+            # these chunks(unstaged chunks) has shared the same metadata
+            # So perform enqueue step again https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L364
+            if chunk_size_full?(chunk) || stage.key?(chunk.metadata)
+              chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
+              queue << chunk.enqueued!
+            else
+              stage[chunk.metadata] = chunk
+            end
           when :queued
             queue << chunk
           end

data/lib/fluent/plugin/buf_file_single.rb

@@ -32,7 +32,6 @@ module Fluent
       DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB
 
       PATH_SUFFIX = ".#{Fluent::Plugin::Buffer::FileSingleChunk::PATH_EXT}"
-      DIR_PERMISSION = 0755
 
       desc 'The path where buffer chunks are stored.'
       config_param :path, :string, default: nil
@@ -128,7 +127,7 @@ module Fluent
         @dir_permission = if @dir_permission
                             @dir_permission.to_i(8)
                           else
-                            system_config.dir_permission || DIR_PERMISSION
+                            system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
                           end
       end
 

data/lib/fluent/plugin/buffer.rb

@@ -60,7 +60,17 @@ module Fluent
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip], default: :text
 
-      Metadata = Struct.new(:timekey, :tag, :variables) do
+      Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
+        def initialize(timekey, tag, variables)
+          super(timekey, tag, variables, 0)
+        end
+
+        def dup_next
+          m = dup
+          m.seq = seq + 1
+          m
+        end
+
         def empty?
           timekey.nil? && tag.nil? && variables.nil?
         end
@@ -353,6 +363,8 @@ module Fluent
                 u = unstaged_chunks[m].pop
                 u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)
+                    # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
+                    u.metadata.seq = 0
                     synchronize {
                       @stage[m] = u.staged!
                       @stage_size += u.bytesize
@@ -416,6 +428,7 @@ module Fluent
             if chunk.empty?
               chunk.close
             else
+              chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
               @queue << chunk
               @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
               chunk.enqueued!
@@ -434,6 +447,7 @@ module Fluent
         synchronize do
           chunk.synchronize do
             metadata = chunk.metadata
+            metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
             @queue << chunk
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
             chunk.enqueued!
@@ -656,13 +670,15 @@ module Fluent
         # Then, will generate chunks not staged (not queued) to append rest data.
         staged_chunk_used = false
         modified_chunks = []
+        modified_metadata = metadata
         get_next_chunk = ->(){
           c = if staged_chunk_used
                 # Staging new chunk here is bad idea:
                 # Recovering whole state including newly staged chunks is much harder than current implementation.
-                generate_chunk(metadata)
+                modified_metadata = modified_metadata.dup_next
+                generate_chunk(modified_metadata)
               else
-                synchronize{ @stage[metadata] ||= generate_chunk(metadata).staged! }
+                synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
               end
           modified_chunks << c
           c

data/lib/fluent/plugin/buffer/chunk.rb

@@ -206,7 +206,7 @@ module Fluent
                 output_io = if chunk_io.is_a?(StringIO)
                               StringIO.new
                             else
-                              Tempfile.new('decompressed-data')
+                              Tempfile.new('decompressed-data').binmode
                             end
                 decompress(input_io: chunk_io, output_io: output_io)
                 output_io.seek(0, IO::SEEK_SET)

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -37,13 +37,11 @@ module Fluent
         # path_prefix: path prefix string, ended with '.'
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
-        FILE_PERMISSION = 0644
-
         attr_reader :path, :permission
 
         def initialize(metadata, path, mode, perm: nil, compress: :text)
           super(metadata, compress: compress)
-          perm ||= FILE_PERMISSION
+          perm ||= Fluent::DEFAULT_FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
           @meta = nil
@@ -232,6 +230,7 @@ module Fluent
           @metadata.timekey = data[:timekey]
           @metadata.tag = data[:tag]
           @metadata.variables = data[:variables]
+          @metadata.seq = data[:seq] || 0
         end
 
         def restore_metadata_partially(chunk)
@@ -243,6 +242,7 @@ module Fluent
           @metadata.timekey = nil
           @metadata.tag = nil
           @metadata.variables = nil
+          @metadata.seq = 0
         end
 
         def write_metadata(update: true)

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -32,14 +32,13 @@ module Fluent
         PATH_EXT = 'buf'
         PATH_SUFFIX = ".#{PATH_EXT}"
         PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n  # //n switch means explicit 'ASCII-8BIT' pattern
-        FILE_PERMISSION = 0644
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, key, perm: FILE_PERMISSION, compress: :text)
+        def initialize(metadata, path, mode, key, perm: Fluent::DEFAULT_FILE_PERMISSION, compress: :text)
           super(metadata, compress: compress)
           @key = key
-          perm ||= FILE_PERMISSION
+          perm ||= Fluent::DEFAULT_FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
 

data/lib/fluent/plugin/in_tail.rb

@@ -48,8 +48,6 @@ module Fluent::Plugin
       end
     end
 
-    FILE_PERMISSION = 0644
-
     def initialize
       super
       @paths = []
@@ -125,6 +123,8 @@ module Fluent::Plugin
         parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
       end
 
+      parser_config['unmatched_lines'] = conf['emit_unmatched_lines']
+
       super
 
       if !@enable_watch_timer && !@enable_stat_watcher
@@ -167,7 +167,8 @@ module Fluent::Plugin
                          else
                            method(:parse_singleline)
                          end
-      @file_perm = system_config.file_permission || FILE_PERMISSION
+      @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
+      @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       # parser is already created by parser helper
       @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
     end
@@ -210,7 +211,7 @@ module Fluent::Plugin
 
       if @pos_file
         pos_file_dir = File.dirname(@pos_file)
-        FileUtils.mkdir_p(pos_file_dir) unless Dir.exist?(pos_file_dir)
+        FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)
         @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
         @pf_file.sync = true
         @pf = PositionFile.load(@pf_file, logger: log)
@@ -317,19 +318,33 @@ module Fluent::Plugin
     end
 
     def setup_watcher(path, pe)
-      line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
-      tw.attach do |watcher|
-        event_loop_attach(watcher.timer_trigger) if watcher.timer_trigger
-        event_loop_attach(watcher.stat_trigger) if watcher.stat_trigger
+      line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
+      tw = TailWatcher.new(path, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+
+      if @enable_watch_timer
+        tt = TimerTrigger.new(1, log) { tw.on_notify }
+        tw.register_watcher(tt)
       end
+
+      if @enable_stat_watcher
+        tt = StatWatcher.new(path, log) { tw.on_notify }
+        tw.register_watcher(tt)
+      end
+
+      tw.on_notify
+
+      tw.watchers.each do |watcher|
+        event_loop_attach(watcher)
+      end
+
       tw
     rescue => e
       if tw
-        tw.detach { |watcher|
-          event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
-          event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
-        }
+        tw.watchers.each do |watcher|
+          event_loop_detach(watcher)
+        end
+
+        tw.detach
         tw.close
       end
       raise e
@@ -384,6 +399,8 @@ module Fluent::Plugin
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(path, pe)
+      log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
+
       if @pf
         unless pe.read_inode == @pf[path].read_inode
           log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
@@ -400,12 +417,13 @@ module Fluent::Plugin
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, close_io = true)
-      tw.detach { |watcher|
-        event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
-        event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
-      }
+      tw.watchers.each do |watcher|
+        event_loop_detach(watcher)
+      end
+      tw.detach
+
       tw.close if close_io
-      flush_buffer(tw)
+
       if tw.unwatched && @pf
         @pf.unwatch(tw.path)
       end
@@ -419,23 +437,31 @@ module Fluent::Plugin
       end
     end
 
-    def flush_buffer(tw)
-      if lb = tw.line_buffer
-        lb.chomp!
-        @parser.parse(lb) { |time, record|
-          if time && record
+    def flush_buffer(tw, buf)
+      buf.chomp!
+      @parser.parse(buf) { |time, record|
+        if time && record
+          tag = if @tag_prefix || @tag_suffix
+                  @tag_prefix + tw.tag + @tag_suffix
+                else
+                  @tag
+                end
+          record[@path_key] ||= tw.path unless @path_key.nil?
+          router.emit(tag, time, record)
+        else
+          if @emit_unmatched_lines
+            record = { 'unmatched_line' => buf }
+            record[@path_key] ||= tail_watcher.path unless @path_key.nil?
             tag = if @tag_prefix || @tag_suffix
                     @tag_prefix + tw.tag + @tag_suffix
                   else
                     @tag
                   end
-            record[@path_key] ||= tw.path unless @path_key.nil?
-            router.emit(tag, time, record)
-          else
-            log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
+            router.emit(tag, Fluent::EventTime.now, record)
           end
-        }
-      end
+          log.warn "got incomplete line at shutdown from #{tw.path}: #{buf.inspect}"
+        end
+      }
     end
 
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
@@ -490,11 +516,12 @@ module Fluent::Plugin
       es
     end
 
+    # No need to check if line_buffer_timer_flusher is nil, since line_buffer_timer_flusher should exist
     def parse_multilines(lines, tail_watcher)
-      lb = tail_watcher.line_buffer
+      lb = tail_watcher.line_buffer_timer_flusher.line_buffer
       es = Fluent::MultiEventStream.new
       if @parser.has_firstline?
-        tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
+        tail_watcher.line_buffer_timer_flusher.reset_timer
         lines.each { |line|
           if @parser.firstline?(line)
             if lb
@@ -524,26 +551,50 @@ module Fluent::Plugin
           }
         end
       end
-      tail_watcher.line_buffer = lb
+      tail_watcher.line_buffer_timer_flusher.line_buffer = lb
       es
     end
 
+    class StatWatcher < Coolio::StatWatcher
+      def initialize(path, log, &callback)
+        @callback = callback
+        @log = log
+        super(path)
+      end
+
+      def on_change(prev, cur)
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    class TimerTrigger < Coolio::TimerWatcher
+      def initialize(interval, log, &callback)
+        @log = log
+        @callback = callback
+        super(interval, true)
+      end
+
+      def on_timer
+        @callback.call
+      rescue => e
+        @log.error e.to_s
+        @log.error_backtrace
+      end
+    end
+
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
+      def initialize(path, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
         @path = path
-        @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
-        @enable_watch_timer = enable_watch_timer
-        @enable_stat_watcher = enable_stat_watcher
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
-        @timer_trigger = @enable_watch_timer ? TimerTrigger.new(1, log, &method(:on_notify)) : nil
-
-        @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
+        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
         @io_handler = nil
         @log = log
 
@@ -551,32 +602,26 @@ module Fluent::Plugin
         @from_encoding = from_encoding
         @encoding = encoding
         @open_on_every_update = open_on_every_update
+        @watchers = []
       end
 
       attr_reader :path
-      attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
-      attr_reader :from_encoding, :encoding
-      attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
-      attr_accessor :timer_trigger
-      attr_accessor :line_buffer, :line_buffer_timer_flusher
+      attr_reader :pe
+      attr_reader :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
+      attr_reader :watchers
 
       def tag
         @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
       end
 
-      def wrap_receive_lines(lines)
-        @receive_lines.call(lines, self)
-      end
-
-      def attach
-        on_notify
-        yield self
+      def register_watcher(watcher)
+        @watchers << watcher
       end
 
       def detach
-        yield self
         @io_handler.on_notify if @io_handler
+        @line_buffer_timer_flusher&.close(self)
       end
 
       def close
@@ -629,7 +674,7 @@ module Fluent::Plugin
               pos = @read_from_head ? 0 : fsize
               @pe.update(inode, pos)
             end
-            @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
+            @io_handler = io_handler
           else
             @io_handler = NullIOHandler.new
           end
@@ -654,18 +699,21 @@ module Fluent::Plugin
             watcher_needs_update = true
           end
 
-          log_msg = "detected rotation of #{@path}"
-          log_msg << "; waiting #{@rotate_wait} seconds" if watcher_needs_update # wait rotate_time if previous file exists
-          @log.info log_msg
-
           if watcher_needs_update
             @update_watcher.call(@path, swap_state(@pe))
           else
-            @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
+            @log.info "detected rotation of #{@path}"
+            @io_handler = io_handler
           end
         end
       end
 
+      def io_handler
+        IOHandler.new(self, path: @path, log: @log, read_lines_limit: @read_lines_limit, open_on_every_update: @open_on_every_update, from_encoding: @from_encoding, encoding: @encoding) do |lines|
+          @receive_lines.call(lines, self)
+        end
+      end
+
       def swap_state(pe)
         # Use MemoryPositionEntry for rotated file temporary
         mpe = MemoryPositionEntry.new
@@ -674,37 +722,6 @@ module Fluent::Plugin
         pe # This pe will be updated in on_rotate after TailWatcher is initialized
       end
 
-      class TimerTrigger < Coolio::TimerWatcher
-        def initialize(interval, log, &callback)
-          @callback = callback
-          @log = log
-          super(interval, true)
-        end
-
-        def on_timer
-          @callback.call
-        rescue => e
-          @log.error e.to_s
-          @log.error_backtrace
-        end
-      end
-
-      class StatWatcher < Coolio::StatWatcher
-        def initialize(watcher, &callback)
-          @watcher = watcher
-          @callback = callback
-          super(watcher.path)
-        end
-
-        def on_change(prev, cur)
-          @callback.call
-        rescue
-          # TODO log?
-          @watcher.log.error $!.to_s
-          @watcher.log.error_backtrace
-        end
-      end
-
       class FIFO
         def initialize(from_encoding, encoding)
           @from_encoding = from_encoding
@@ -764,15 +781,20 @@ module Fluent::Plugin
       end
 
       class IOHandler
-        def initialize(watcher, &receive_lines)
+        def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
           @watcher = watcher
+          @path = path
+          @read_lines_limit = read_lines_limit
           @receive_lines = receive_lines
-          @fifo = FIFO.new(@watcher.from_encoding || Encoding::ASCII_8BIT, @watcher.encoding || Encoding::ASCII_8BIT)
+          @open_on_every_update = open_on_every_update
+          @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
           @iobuf = ''.force_encoding('ASCII-8BIT')
           @lines = []
           @io = nil
           @notify_mutex = Mutex.new
-          @watcher.log.info "following tail of #{@watcher.path}"
+          @log = log
+
+          @log.info "following tail of #{@path}"
         end
 
         def on_notify
@@ -789,7 +811,7 @@ module Fluent::Plugin
                   while true
                     @fifo << io.readpartial(8192, @iobuf)
                     @fifo.read_lines(@lines)
-                    if @lines.size >= @watcher.read_lines_limit
+                    if @lines.size >= @read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true
                       break
@@ -823,37 +845,35 @@ module Fluent::Plugin
         end
 
         def open
-          io = Fluent::FileWrapper.open(@watcher.path)
+          io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
           io
         rescue RangeError
           io.close if io
-          raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
+          raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
         rescue Errno::ENOENT
           nil
         end
 
         def with_io
-          begin
-            if @watcher.open_on_every_update
-              io = open
-              begin
-                yield io
-              ensure
-                io.close unless io.nil?
-              end
-            else
-              @io ||= open
-              yield @io
+          if @open_on_every_update
+            io = open
+            begin
+              yield io
+            ensure
+              io.close unless io.nil?
             end
-          rescue WatcherSetupError => e
-            close
-            raise e
-          rescue
-            @watcher.log.error $!.to_s
-            @watcher.log.error_backtrace
-            close
+          else
+            @io ||= open
+            yield @io
           end
+        rescue WatcherSetupError => e
+          close
+          raise e
+        rescue
+          @log.error $!.to_s
+          @log.error_backtrace
+          close
         end
       end
 
@@ -876,8 +896,8 @@ module Fluent::Plugin
       end
 
       class RotateHandler
-        def initialize(watcher, &on_rotate)
-          @watcher = watcher
+        def initialize(log, &on_rotate)
+          @log = log
           @inode = nil
           @fsize = -1  # first
           @on_rotate = on_rotate
@@ -892,39 +912,50 @@ module Fluent::Plugin
             fsize = stat.size
           end
 
-          begin
-            if @inode != inode || fsize < @fsize
-              @on_rotate.call(stat)
-            end
-            @inode = inode
-            @fsize = fsize
+          if @inode != inode || fsize < @fsize
+            @on_rotate.call(stat)
           end
-
+          @inode = inode
+          @fsize = fsize
         rescue
-          @watcher.log.error $!.to_s
-          @watcher.log.error_backtrace
+          @log.error $!.to_s
+          @log.error_backtrace
         end
       end
 
       class LineBufferTimerFlusher
+        attr_accessor :line_buffer
+
         def initialize(log, flush_interval, &flush_method)
           @log = log
           @flush_interval = flush_interval
           @flush_method = flush_method
           @start = nil
+          @line_buffer = nil
         end
 
         def on_notify(tw)
-          if @start && @flush_interval
-            if Time.now - @start >= @flush_interval
-              @flush_method.call(tw)
-              tw.line_buffer = nil
-              @start = nil
-            end
+          unless @start && @flush_method
+            return
+          end
+
+          if Time.now - @start >= @flush_interval
+            @flush_method.call(tw, @line_buffer) if @line_buffer
+            @line_buffer = nil
+            @start = nil
           end
         end
 
+        def close(tw)
+          return unless @line_buffer
+
+          @flush_method.call(tw, @line_buffer)
+          @line_buffer = nil
+        end
+
         def reset_timer
+          return unless @flush_interval
+
           @start = Time.now
         end
       end