fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -53,10 +53,16 @@ module Fluent::Plugin
         }
       end
 
+      def unwatch_removed_targets(existing_targets)
+        @map.reject { |key, entry|
+          existing_targets.key?(key)
+        }.each_key { |key|
+          unwatch_key(key)
+        }
+      end
+
       def unwatch(target_info)
-        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
-          entry.update_pos(UNWATCHED_POSITION)
-        end
+        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
       end
 
       def load(existing_targets = nil)
@@ -96,6 +102,7 @@ module Fluent::Plugin
         end
 
         entries = fetch_compacted_entries
+        @logger&.debug "Compacted entries: ", entries.keys
 
         @file_mutex.synchronize do
           if last_modified == @file.mtime && size == @file.size
@@ -117,17 +124,31 @@ module Fluent::Plugin
 
       private
 
+      def unwatch_key(key)
+        if (entry = @map.delete(key))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
       def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries
+          @logger&.debug "Compacted entries: ", entries.keys
+
+          if existing_targets
+            entries = remove_deleted_files_entries(entries, existing_targets)
+            @logger&.debug "Remove missing entries.",
+                           existing_targets: existing_targets.keys,
+                           entries_after_removing: entries.keys
+          end
 
           @file.pos = 0
           @file.truncate(0)
-          @file.write(entries.join)
+          @file.write(entries.values.map(&:to_entry_fmt).join)
         end
       end
 
-      def fetch_compacted_entries(existing_targets = nil)
+      def fetch_compacted_entries
         entries = {}
 
         @file.pos = 0
@@ -145,31 +166,24 @@ module Fluent::Plugin
           if pos == UNWATCHED_POSITION
             @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
           else
-            if entries.include?(path)
-              @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
-            end
-
             if @follow_inodes
+              @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
               entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
+              @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
               entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, existing_targets)
         entries
       end
 
       def remove_deleted_files_entries(existent_entries, existing_targets)
-        if existing_targets
-          existent_entries.select { |path_or_ino|
-            existing_targets.key?(path_or_ino)
-          }
-        else
-          existent_entries
-        end
+        existent_entries.select { |path_or_ino|
+          existing_targets.key?(path_or_ino)
+        }
       end
     end
 

data/lib/fluent/plugin/in_tail.rb

@@ -370,17 +370,30 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-      
+
       log.debug {
         target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
       }
 
-      unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
+      if !@follow_inodes
+        need_unwatch_in_stop_watchers = true
+      else
+        # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
+        # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
+        # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
+        #
+        # This logic may work for `@follow_inodes false` too.
+        # Just limiting the case to supress the impact to existing logics.
+        @pf&.unwatch_removed_targets(target_paths_hash)
+        need_unwatch_in_stop_watchers = false
+      end
+
+      removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
-      stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
+      stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
     end
@@ -484,8 +497,26 @@ module Fluent::Plugin
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
-    def update_watcher(target_info, pe)
-      path = target_info.path
+    def update_watcher(tail_watcher, pe, new_inode)
+      # TODO we should use another callback for this.
+      # To supress impact to existing logics, limit the case to `@follow_inodes`.
+      # We may not need `@follow_inodes` condition.
+      if @follow_inodes && new_inode.nil?
+        # nil inode means the file disappeared, so we only need to stop it.
+        @tails.delete(tail_watcher.path)
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # Because of this problem, log duplication can occur during `rotate_wait`.
+        # Need to set `rotate_wait 0` for a workaround.
+        # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
+        # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
+        # and it causes the log duplication.
+        # (Other `detach_watcher_after_rotate_wait` may have the same problem.
+        #  We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
+        detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
+        return
+      end
+
+      path = tail_watcher.path
 
       log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
 
@@ -499,23 +530,22 @@ module Fluent::Plugin
         end
       end
 
-      rotated_tw = @tails[path]
+      new_target_info = TargetInfo.new(path, new_inode)
 
       if @follow_inodes
-        new_position_entry = @pf[target_info]
-
+        new_position_entry = @pf[new_target_info]
+        # If `refresh_watcher` find the new file before, this will not be zero.
+        # In this case, only we have to do is detaching the current tail_watcher.
         if new_position_entry.read_inode == 0
-          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
-          # So it should be unwatched here explicitly.
-          rotated_tw.unwatched = true if rotated_tw
-          @tails[path] = setup_watcher(target_info, new_position_entry)
+          @tails[path] = setup_watcher(new_target_info, new_position_entry)
           @tails[path].on_notify
         end
       else
-        @tails[path] = setup_watcher(target_info, pe)
+        @tails[path] = setup_watcher(new_target_info, pe)
         @tails[path].on_notify
       end
-      detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
+
+      detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
@@ -523,6 +553,10 @@ module Fluent::Plugin
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
+      if @follow_inodes && tw.ino != ino
+        log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
+                  path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
+      end
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
@@ -778,7 +812,7 @@ module Fluent::Plugin
       attr_accessor :group_watcher
 
       def tag
-        @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
+        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
       end
 
       def register_watcher(watcher)
@@ -874,21 +908,21 @@ module Fluent::Plugin
 
           if watcher_needs_update
             if @follow_inodes
-              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
-              if stat
-                target_info = TargetInfo.new(@path, stat.ino)
-                @update_watcher.call(target_info, @pe)
-              end
+              # If stat is nil (file not present), NEED to stop and discard this watcher.
+              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`
+              #   can't recognize this TailWatcher needs to be stopped.
+              #   This can happens when the file is rotated.
+              #   If a notify comes before the new file for the path is created during rotation,
+              #   then it appears as if the file was resurrected once it disappeared.
+              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              @update_watcher.call(self, @pe, stat&.ino)
             else
               # Permit to handle if stat is nil (file not present).
               # If a file is mv-ed and a new file is created during
               # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
               # and `#stop_watchers()` for the path because `target_paths_hash`
               # always contains the path.
-              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
-              @update_watcher.call(target_info, swap_state(@pe))
+              @update_watcher.call(self, swap_state(@pe), stat&.ino)
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/in_tcp.rb

@@ -36,10 +36,16 @@ module Fluent::Plugin
     desc "The field name of the client's address."
     config_param :source_address_key, :string, default: nil
 
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
 
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
 
     # in_forward like host/network restriction
     config_section :security, required: false, multi: false do
@@ -100,8 +106,9 @@ module Fluent::Plugin
 
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
+      discard_till_next_delimiter = false
       if @_extract_enabled && @_extract_tag_key
-        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -114,6 +121,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -129,9 +146,18 @@ module Fluent::Plugin
             end
           end
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       else
-        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -145,6 +171,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -159,6 +195,15 @@ module Fluent::Plugin
           end
           router.emit_stream(@tag, es)
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       end
     end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -163,7 +163,7 @@ module Fluent::Plugin
                     0
                   elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
-                  elsif @child_respawn =~ /^\d+$/
+                  elsif /^\d+$/.match?(@child_respawn)
                     @child_respawn.to_i
                   else
                     raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
       @rr = 0
 
       exit_callback = ->(status){
-        c = @children.select{|child| child.pid == status.pid }.first
+        c = @children.find{|child| child.pid == status.pid }
         if c
           unless self.stopped?
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -59,7 +59,13 @@ module Fluent::Plugin
             @ack_waitings = new_list
           end
 
-          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          begin
+            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          rescue IOError
+            @log.info "connection closed while waiting for readable sockets"
+            readable_sockets = nil
+          end
+
           if readable_sockets
             readable_sockets.each do |sock|
               results << read_ack_from_sock(sock)
@@ -109,13 +115,22 @@ module Fluent::Plugin
           raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
         rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
           raw_data = ''
+        rescue IOError
+          @log.info "socket closed while receiving ack response"
+          return nil, Result::FAILED
         end
 
         info = find(sock)
 
-        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-        # If this happens we assume the data wasn't delivered and retry it.
-        if raw_data.empty?
+        if info.nil?
+          # The info can be deleted by another thread during `sock.recv()` and `find()`.
+          # This is OK since another thread has completed to process the ack, so we can skip this.
+          # Note: exclusion mechanism about `collect_response()` may need to be considered.
+          @log.debug "could not find the ack info. this ack may be processed by another thread."
+          return nil, Result::FAILED
+        elsif raw_data.empty?
+          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+          # If this happens we assume the data wasn't delivered and retry it.
           @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
           # info.node.disable!
           return info, Result::FAILED

data/lib/fluent/plugin/out_forward.rb

@@ -521,8 +521,8 @@ module Fluent::Plugin
         when AckHandler::Result::SUCCESS
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
-          node.disable!
-          rollback_write(chunk_id, update_retry: false)
+          node&.disable!
+          rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
           rollback_write(chunk_id, update_retry: false)
         else

data/lib/fluent/plugin/out_secondary_file.rb

@@ -64,31 +64,29 @@ module Fluent::Plugin
     end
 
     def multi_workers_ready?
-      ### TODO: add hack to synchronize for multi workers
       true
     end
 
     def write(chunk)
       path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
-      path = generate_path(path_without_suffix)
-      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
-
-      case @compress
-      when :text
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          chunk.write_to(f)
-        }
-      when :gzip
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          gz = Zlib::GzipWriter.new(f)
-          chunk.write_to(gz)
-          gz.close
-        }
+      generate_path(path_without_suffix) do |path|
+        FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+        case @compress
+        when :text
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            chunk.write_to(f)
+          }
+        when :gzip
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            gz = Zlib::GzipWriter.new(f)
+            chunk.write_to(gz)
+            gz.close
+          }
+        end
       end
-
-      path
     end
 
     private
@@ -117,15 +115,34 @@ module Fluent::Plugin
 
     def generate_path(path_without_suffix)
       if @append
-        "#{path_without_suffix}#{@suffix}"
-      else
+        path = "#{path_without_suffix}#{@suffix}"
+        synchronize_path(path) do
+          yield path
+        end
+        return path
+      end
+
+      begin
         i = 0
         loop do
           path = "#{path_without_suffix}.#{i}#{@suffix}"
-          return path unless File.exist?(path)
+          break unless File.exist?(path)
           i += 1
         end
+        synchronize_path(path) do
+          # If multiple processes or threads select the same path and another
+          # one entered this locking block first, the file should already
+          # exist and this one should retry to find new path.
+          raise FileAlreadyExist if File.exist?(path)
+          yield path
+        end
+      rescue FileAlreadyExist
+        retry
       end
+      path
+    end
+
+    class FileAlreadyExist < StandardError
     end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -99,7 +99,6 @@ module Fluent
         config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
 
         config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
-        config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
       end
 
       config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
@@ -199,6 +198,7 @@ module Fluent
       def initialize
         super
         @counter_mutex = Mutex.new
+        @flush_thread_mutex = Mutex.new
         @buffering = false
         @delayed_commit = false
         @as_secondary = false
@@ -378,6 +378,7 @@ module Fluent
           buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
           @buffer = Plugin.new_buffer(buffer_type, parent: self)
           @buffer.configure(buffer_conf)
+          keep_buffer_config_compat
           @buffer.enable_update_timekeys if @chunk_key_time
 
           @flush_at_shutdown = @buffer_config.flush_at_shutdown
@@ -425,7 +426,9 @@ module Fluent
           end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
+          if (@secondary.class.to_s != "Fluent::Plugin::SecondaryFileOutput") &&
+             (self.class != @secondary.class) &&
+             (@custom_format || @secondary.implement?(:custom_format))
             log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
@@ -435,6 +438,12 @@ module Fluent
         self
       end
 
+      def keep_buffer_config_compat
+        # Need this to call `@buffer_config.disable_chunk_backup` just as before,
+        # since some plugins may use this option in this way.
+        @buffer_config[:disable_chunk_backup] = @buffer.disable_chunk_backup
+      end
+
       def start
         super
 
@@ -591,6 +600,42 @@ module Fluent
         super
       end
 
+      def actual_flush_thread_count
+        return 0 unless @buffering
+        return @buffer_config.flush_thread_count unless @as_secondary
+        @primary_instance.buffer_config.flush_thread_count
+      end
+
+      # Ensures `path` (filename or filepath) processable
+      # only by the current thread in the current process.
+      # For multiple workers, the lock is shared if `path` is the same value.
+      # For multiple threads, the lock is shared by all threads in the same process.
+      def synchronize_path(path)
+        synchronize_path_in_workers(path) do
+          synchronize_in_threads do
+            yield
+          end
+        end
+      end
+
+      def synchronize_path_in_workers(path)
+        need_worker_lock = system_config.workers > 1
+        if need_worker_lock
+          acquire_worker_lock(path) { yield }
+        else
+          yield
+        end
+      end
+
+      def synchronize_in_threads
+        need_thread_lock = actual_flush_thread_count > 1
+        if need_thread_lock
+          @flush_thread_mutex.synchronize { yield }
+        else
+          yield
+        end
+      end
+
       def support_in_v12_style?(feature)
         # for plugins written in v0.12 styles
         case feature
@@ -779,7 +824,7 @@ module Fluent
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
-            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+            if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
               hash = {}
               tag_parts = metadata.tag.split('.')
               tag_parts.each_with_index do |part, i|
@@ -1240,18 +1285,10 @@ module Fluent
       end
 
       def backup_chunk(chunk, using_secondary, delayed_commit)
-        if @buffer_config.disable_chunk_backup
+        if @buffer.disable_chunk_backup
           log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
         else
-          unique_id = dump_unique_id_hex(chunk.unique_id)
-          safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
-          backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
-          backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
-          backup_dir = File.dirname(backup_file)
-
-          log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
-          File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
+          @buffer.backup(chunk.unique_id) { |f|
             chunk.write_to(f)
           }
         end

data/lib/fluent/plugin/parser_json.rb

@@ -60,7 +60,7 @@ module Fluent
       rescue LoadError => ex
         name = :yajl
         if log
-          if /\boj\z/ =~ ex.message
+          if /\boj\z/.match?(ex.message)
             log.info "Oj is not installed, and failing back to Yajl for json parser"
           else
             log.warn ex.message

data/lib/fluent/plugin_helper/event_loop.rb

@@ -99,7 +99,7 @@ module Fluent
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop_attached_watchers.reverse.each do |w|
+          @_event_loop_attached_watchers.reverse_each do |w|
             if w.attached?
               begin
                 w.detach
@@ -116,7 +116,7 @@ module Fluent
       def after_shutdown
         timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.reverse.each do |w|
+          @_event_loop.watchers.reverse_each do |w|
             begin
               w.detach
             rescue => e

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -21,6 +21,7 @@ require 'async/http/endpoint'
 require 'fluent/plugin_helper/http_server/app'
 require 'fluent/plugin_helper/http_server/router'
 require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/log/console_adapter'
 
 module Fluent
   module PluginHelper
@@ -38,7 +39,7 @@ module Fluent
           scheme = tls_context ? 'https' : 'http'
           @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
           @router = Router.new(default_app)
-          @reactor = Async::Reactor.new(nil, logger: @logger)
+          @reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(@logger))
 
           opts = if tls_context
                    { ssl_context: tls_context }