fluentd 1.16.0 → 1.16.7

data/lib/fluent/plugin/in_tail.rb

@@ -52,6 +52,7 @@ module Fluent::Plugin
       super
       @paths = []
       @tails = {}
+      @tails_rotate_wait = {}
       @pf_file = nil
       @pf = nil
       @ignore_list = []
@@ -267,6 +268,9 @@ module Fluent::Plugin
       @shutdown_start_time = Fluent::Clock.now
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(existence_path, immediate: true, remove_watcher: false)
+      @tails_rotate_wait.keys.each do |tw|
+        detach_watcher(tw, @tails_rotate_wait[tw][:ino], false)
+      end
       @pf_file.close if @pf_file
 
       super
@@ -275,6 +279,7 @@ module Fluent::Plugin
     def close
       super
       # close file handles after all threads stopped (in #close of thread plugin helper)
+      # It may be because we need to wait IOHanlder.ready_to_shutdown()
       close_watcher_handles
     end
 
@@ -370,17 +375,52 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-      
+
       log.debug {
         target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
       }
 
-      unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
+      if !@follow_inodes
+        need_unwatch_in_stop_watchers = true
+      else
+        # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
+        # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
+        # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
+        #
+        # This logic may work for `@follow_inodes false` too.
+        # Just limiting the case to suppress the impact to existing logics.
+        @pf&.unwatch_removed_targets(target_paths_hash)
+        need_unwatch_in_stop_watchers = false
+      end
+
+      removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
-      stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
+      # If an exisiting TailWatcher already follows a target path with the different inode,
+      # it means that the TailWatcher following the rotated file still exists. In this case,
+      # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we
+      # should output a warning log in order to prevent silent collection stops.
+      # (Such as https://github.com/fluent/fluentd/pull/4327)
+      # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)
+      # (The similar warning may work for `@follow_inodes true` too. Just limiting the case
+      # to suppress the impact to existing logics.)
+      unless @follow_inodes
+        target_paths_hash.each do |path, target|
+          next unless @tails.key?(path)
+          # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,
+          # which is very unstable parameter. (It can be `nil` or old).
+          # So, we need to use `TailWatcher.pe.read_inode`.
+          existing_watcher_inode = @tails[path].pe.read_inode
+          if existing_watcher_inode != target.ino
+            log.warn "Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.",
+              filepath: target.path
+          end
+        end
+      end
+
+      stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
     end
@@ -481,11 +521,32 @@ module Fluent::Plugin
           tw.close
         end
       end
+      @tails_rotate_wait.keys.each do |tw|
+        tw.close
+      end
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
-    def update_watcher(target_info, pe)
-      path = target_info.path
+    def update_watcher(tail_watcher, pe, new_inode)
+      # TODO we should use another callback for this.
+      # To supress impact to existing logics, limit the case to `@follow_inodes`.
+      # We may not need `@follow_inodes` condition.
+      if @follow_inodes && new_inode.nil?
+        # nil inode means the file disappeared, so we only need to stop it.
+        @tails.delete(tail_watcher.path)
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # Because of this problem, log duplication can occur during `rotate_wait`.
+        # Need to set `rotate_wait 0` for a workaround.
+        # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
+        # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
+        # and it causes the log duplication.
+        # (Other `detach_watcher_after_rotate_wait` may have the same problem.
+        #  We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
+        detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
+        return
+      end
+
+      path = tail_watcher.path
 
       log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
 
@@ -499,30 +560,29 @@ module Fluent::Plugin
         end
       end
 
-      rotated_tw = @tails[path]
+      new_target_info = TargetInfo.new(path, new_inode)
 
       if @follow_inodes
-        new_position_entry = @pf[target_info]
-
+        new_position_entry = @pf[new_target_info]
+        # If `refresh_watcher` find the new file before, this will not be zero.
+        # In this case, only we have to do is detaching the current tail_watcher.
         if new_position_entry.read_inode == 0
-          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
-          # So it should be unwatched here explicitly.
-          rotated_tw.unwatched = true if rotated_tw
-          @tails[path] = setup_watcher(target_info, new_position_entry)
+          @tails[path] = setup_watcher(new_target_info, new_position_entry)
           @tails[path].on_notify
         end
       else
-        @tails[path] = setup_watcher(target_info, pe)
+        @tails[path] = setup_watcher(new_target_info, pe)
         @tails[path].on_notify
       end
-      detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
+
+      detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
-    # TailWatcher#close is called by another thread at shutdown phase.
-    # It causes 'can't modify string; temporarily locked' error in IOHandler
-    # so adding close_io argument to avoid this problem.
-    # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
+      if @follow_inodes && tw.ino != ino
+        log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
+                  path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
+      end
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
@@ -530,7 +590,7 @@ module Fluent::Plugin
 
       tw.close if close_io
 
-      if tw.unwatched && @pf
+      if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])
         target_info = TargetInfo.new(tw.path, ino)
         @pf.unwatch(target_info)
       end
@@ -548,7 +608,11 @@ module Fluent::Plugin
       if @open_on_every_update
         # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
-      elsif throttling_is_enabled?(tw)
+      end
+
+      return if @tails_rotate_wait[tw]
+
+      if throttling_is_enabled?(tw)
         # When the throttling feature is enabled, it might not reach EOF yet.
         # Should ensure to read all contents before closing it, with keeping throttling.
         start_time_to_wait = Fluent::Clock.now
@@ -556,14 +620,18 @@ module Fluent::Plugin
           elapsed = Fluent::Clock.now - start_time_to_wait
           if tw.eof? && elapsed >= @rotate_wait
             timer.detach
+            @tails_rotate_wait.delete(tw)
             detach_watcher(tw, ino)
           end
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       else
         # when the throttling feature isn't enabled, just wait @rotate_wait
-        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+        timer = timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          @tails_rotate_wait.delete(tw)
           detach_watcher(tw, ino)
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       end
     end
 
@@ -778,7 +846,7 @@ module Fluent::Plugin
       attr_accessor :group_watcher
 
       def tag
-        @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
+        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
       end
 
       def register_watcher(watcher)
@@ -874,21 +942,21 @@ module Fluent::Plugin
 
           if watcher_needs_update
             if @follow_inodes
-              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
-              if stat
-                target_info = TargetInfo.new(@path, stat.ino)
-                @update_watcher.call(target_info, @pe)
-              end
+              # If stat is nil (file not present), NEED to stop and discard this watcher.
+              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`
+              #   can't recognize this TailWatcher needs to be stopped.
+              #   This can happens when the file is rotated.
+              #   If a notify comes before the new file for the path is created during rotation,
+              #   then it appears as if the file was resurrected once it disappeared.
+              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              @update_watcher.call(self, @pe, stat&.ino)
             else
               # Permit to handle if stat is nil (file not present).
               # If a file is mv-ed and a new file is created during
               # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
               # and `#stop_watchers()` for the path because `target_paths_hash`
               # always contains the path.
-              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
-              @update_watcher.call(target_info, swap_state(@pe))
+              @update_watcher.call(self, swap_state(@pe), stat&.ino)
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/in_tcp.rb

@@ -36,6 +36,10 @@ module Fluent::Plugin
     desc "The field name of the client's address."
     config_param :source_address_key, :string, default: nil
 
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
 
     desc 'The payload is read up to this character.'
@@ -102,6 +106,7 @@ module Fluent::Plugin
 
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
+      discard_till_next_delimiter = false
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
@@ -116,6 +121,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -131,6 +146,15 @@ module Fluent::Plugin
             end
           end
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       else
         server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
@@ -147,6 +171,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -161,6 +195,15 @@ module Fluent::Plugin
           end
           router.emit_stream(@tag, es)
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       end
     end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -163,7 +163,7 @@ module Fluent::Plugin
                     0
                   elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
-                  elsif @child_respawn =~ /^\d+$/
+                  elsif /^\d+$/.match?(@child_respawn)
                     @child_respawn.to_i
                   else
                     raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
       @rr = 0
 
       exit_callback = ->(status){
-        c = @children.select{|child| child.pid == status.pid }.first
+        c = @children.find{|child| child.pid == status.pid }
         if c
           unless self.stopped?
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig

data/lib/fluent/plugin/out_file.rb

@@ -172,6 +172,14 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
+          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
+            begin
+              v.validate!
+            rescue Fluent::ConfigError => e
+              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
+            end
+          end
+
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/output.rb

@@ -426,7 +426,7 @@ module Fluent
           end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          if (@secondary.class != SecondaryFileOutput) &&
+          if (@secondary.class.to_s != "Fluent::Plugin::SecondaryFileOutput") &&
              (self.class != @secondary.class) &&
              (@custom_format || @secondary.implement?(:custom_format))
             log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
@@ -824,7 +824,7 @@ module Fluent
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
-            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+            if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
               hash = {}
               tag_parts = metadata.tag.split('.')
               tag_parts.each_with_index do |part, i|

data/lib/fluent/plugin/parser_json.rb

@@ -50,23 +50,15 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          raise LoadError unless Fluent::OjOptions.available?
-          [Oj.method(:load), Oj::ParseError]
+          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
+
+          log&.info "Oj is not installed, and failing back to Yajl for json parser"
+          configure_json_parser(:yajl)
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError => ex
-        name = :yajl
-        if log
-          if /\boj\z/ =~ ex.message
-            log.info "Oj is not installed, and failing back to Yajl for json parser"
-          else
-            log.warn ex.message
-          end
-        end
-        retry
       end
 
       def parse(text)

data/lib/fluent/plugin_helper/event_loop.rb

@@ -99,7 +99,7 @@ module Fluent
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop_attached_watchers.reverse.each do |w|
+          @_event_loop_attached_watchers.reverse_each do |w|
             if w.attached?
               begin
                 w.detach
@@ -116,7 +116,7 @@ module Fluent
       def after_shutdown
         timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.reverse.each do |w|
+          @_event_loop.watchers.reverse_each do |w|
             begin
               w.detach
             rescue => e

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -119,7 +119,7 @@ module Fluent
         def self.validate_dot_keys(keys)
           keys.each { |key|
             next unless key.is_a?(String)
-            if /\s+/.match(key)
+            if /\s+/.match?(key)
               raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
             end
           }

data/lib/fluent/plugin_helper/server.rb

@@ -545,6 +545,10 @@ module Fluent
               data = @sock.recv(@max_bytes, @flags)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data)
           rescue => e
@@ -558,6 +562,10 @@ module Fluent
               data, addr = @sock.recvfrom(@max_bytes)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
           rescue => e

data/lib/fluent/plugin_helper/thread.rb

@@ -101,16 +101,16 @@ module Fluent
       end
 
       def thread_exist?(title)
-        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
       end
 
       def thread_started?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_started]
       end
 
       def thread_running?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_running]
       end
 

data/lib/fluent/plugin_id.rb

@@ -49,7 +49,7 @@ module Fluent
         # Thread::Backtrace::Location#path returns base filename or absolute path.
         # #absolute_path returns absolute_path always.
         # https://bugs.ruby-lang.org/issues/12159
-        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+        if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
           return true
         end
       end

data/lib/fluent/supervisor.rb

@@ -440,7 +440,7 @@ module Fluent
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
         logger: $log,
-        log: $log.out,
+        log: $log&.out,
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],
@@ -697,7 +697,7 @@ module Fluent
       actual_log_path = @log_path
 
       # We need to prepare a unique path for each worker since Windows locks files.
-      if Fluent.windows? && rotate
+      if Fluent.windows? && rotate && @log_path && @log_path != "-"
         actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
       end
 

data/lib/fluent/system_config.rb

@@ -62,7 +62,7 @@ module Fluent
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
         if Fluent::Log::LOG_ROTATE_AGE.include?(v)
-          v.to_sym
+          v
         else
           begin
             Integer(v)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.0'
+  VERSION = '1.16.7'
 
 end

data/lib/fluent/winsvc.rb

@@ -63,10 +63,12 @@ begin
     end
 
     def service_stop
-      set_event(@service_name)
-      if @pid > 0
-        Process.waitpid(@pid)
+      if @pid <= 0
+        set_event(@service_name)
+        return
       end
+
+      wait_supervisor_finished
     end
 
     def service_paramchange
@@ -91,6 +93,29 @@ begin
       ev.set
       ev.close
     end
+
+    def repeat_set_event_several_times_until_success(event_name)
+      retries = 0
+      max_retries = 10
+      delay_sec = 3
+
+      begin
+        set_event(event_name)
+      rescue Errno::ENOENT
+        # This error occurs when the supervisor process has not yet created the event.
+        # If STOP is immediately executed, this state will occur.
+        # Retry `set_event' to wait for the initialization of the supervisor.
+        retries += 1
+        raise if max_retries < retries
+        sleep(delay_sec)
+        retry
+      end
+    end
+
+    def wait_supervisor_finished
+      repeat_set_event_several_times_until_success(@service_name)
+      Process.waitpid(@pid)
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/templates/new_gem/test/helper.rb.erb

@@ -1,4 +1,3 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/<%= type %>"

data/test/command/test_cat.rb

@@ -18,7 +18,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port
+    @port = unused_port(protocol: :all)
   end
 
   def teardown
@@ -87,7 +87,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       d = create_driver
       d.run(expect_records: 1) do
         Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path))
+          stdin.write(File.read(path, File.size(path)))
           stdin.close
         end
       end