fluentd 1.18.0 → 1.19.0

data/lib/fluent/config/parser.rb

@@ -92,10 +92,23 @@ module Fluent
         else
           basepath = '/'
           fname = path
-          require 'open-uri'
-          URI.open(uri) {|f|
+          parser_proc = ->(f) {
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
+
+          case u.scheme
+          when 'http', 'https', 'ftp'
+            # URI#open can be able to handle URIs for http, https and ftp.
+            require 'open-uri'
+            u.open(&parser_proc)
+          else
+            # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
+            # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
+            #   - uri is 'c:/path/to/file'
+            #   - u.path is '/path/to/file' and u.scheme is 'c'
+            # Therefore, the condition of the if statement above is not met and it is handled here.
+            File.open(uri, &parser_proc)
+          end
         end
 
       rescue SystemCallError => e
@@ -104,4 +117,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/config/section.rb

@@ -150,7 +150,7 @@ module Fluent
             end
           end
           unless section_params.has_key?(proxy.argument.first)
-            logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but sometimes it's nil (e.g, in tests)
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
           # argument should NOT be deprecated... (argument always has a value: '')
@@ -253,7 +253,7 @@ module Fluent
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
-          next if e.unused_in && e.unused_in.empty? # the section is used at least once
+          next if e.unused_in&.empty? # the section is used at least once
 
           if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
             e.unused_in = []

data/lib/fluent/config/types.rb

@@ -71,7 +71,7 @@ module Fluent
       else
         # Current parser passes comment without actual values, e.g. "param #foo".
         # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
-        if str.respond_to?('start_with?') && str.start_with?('#')
+        if str.respond_to?(:start_with?) && str.start_with?('#')
           true
         elsif opts[:strict]
           raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"

data/lib/fluent/config/v1_parser.rb

@@ -83,7 +83,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/o)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"
@@ -98,7 +98,7 @@ module Fluent
             new_e.v1_config = true
             elems << new_e
 
-          elsif root_element && skip(/(\@include|include)#{SPACING}/)
+          elsif root_element && skip(/(\@include|include)#{SPACING}/o)
             if !prev_match.start_with?('@')
               @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
             end
@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = URI.open(uri) { |f| f.read }
+          data = u.open { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/counter/store.rb

@@ -156,7 +156,7 @@ module Fluent
         }
       end
 
-      # value is Hash. value requires these fileds.
+      # value is Hash. value requires these fields.
       # :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
       def build_value(data)
         type = data['type'] || 'numeric'

data/lib/fluent/engine.rb

@@ -177,7 +177,7 @@ module Fluent
 
     # @param conf [Fluent::Config]
     # @param supervisor [Bool]
-    # @reutrn nil
+    # @return nil
     def reload_config(conf, supervisor: false)
       @root_agent_mutex.synchronize do
         # configure first to reduce down time while restarting

data/lib/fluent/env.rb

@@ -21,6 +21,7 @@ require 'fluent/oj_options'
 
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
+  DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
@@ -34,10 +35,10 @@ module Fluent
   end
 
   def self.linux?
-    /linux/ === RUBY_PLATFORM
+    RUBY_PLATFORM.include?("linux")
   end
 
   def self.macos?
-    /darwin/ =~ RUBY_PLATFORM
+    RUBY_PLATFORM.include?("darwin")
   end
 end

data/lib/fluent/event.rb

@@ -62,9 +62,9 @@ module Fluent
       out.full_pack
     end
 
-    def to_compressed_msgpack_stream(time_int: false, packer: nil)
+    def to_compressed_msgpack_stream(time_int: false, packer: nil, type: :gzip)
       packed = to_msgpack_stream(time_int: time_int, packer: packer)
-      compress(packed)
+      compress(packed, type: type)
     end
 
     def to_msgpack_stream_forced_integer(packer: nil)
@@ -247,7 +247,7 @@ module Fluent
     end
 
     # This method returns MultiEventStream, because there are no reason
-    # to surve binary serialized by msgpack.
+    # to survey binary serialized by msgpack.
     def slice(index, num)
       ensure_unpacked!
       MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
@@ -268,10 +268,11 @@ module Fluent
   end
 
   class CompressedMessagePackEventStream < MessagePackEventStream
-    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
-      super
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip)
+      super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)
       @decompressed_data = nil
       @compressed_data = data
+      @type = compress
     end
 
     def empty?
@@ -303,7 +304,7 @@ module Fluent
 
     def ensure_decompressed!
       return if @decompressed_data
-      @data = @decompressed_data = decompress(@data)
+      @data = @decompressed_data = decompress(@data, type: @type)
     end
   end
 

data/lib/fluent/log/console_adapter.rb

@@ -19,11 +19,9 @@ require 'console'
 module Fluent
   class Log
     # Async gem which is used by http_server helper switched logger mechanism to
-    # Console gem which isn't complatible with Ruby's standard Logger (since
+    # Console gem which isn't compatible with Ruby's standard Logger (since
     # v1.17). This class adapts it to Fluentd's logger mechanism.
-    class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
-      Console::Output::Terminal : Console::Terminal::Logger
-
+    class ConsoleAdapter < Console::Output::Terminal
       def self.wrap(logger)
         _, level = Console::Logger::LEVELS.find { |key, value|
           if logger.level <= 0
@@ -58,10 +56,10 @@ module Fluent
           level = 'warn'
         end
 
-        @io.seek(0)
-        @io.truncate(0)
+        @stream.seek(0)
+        @stream.truncate(0)
         super
-        @logger.send(level, @io.string.chomp)
+        @logger.send(level, @stream.string.chomp)
       end
     end
   end

data/lib/fluent/log.rb

@@ -138,6 +138,7 @@ module Fluent
       @optional_attrs = nil
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
+      @forced_stacktrace_level = nil
       @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
       @ignore_same_log_interval = opts[:ignore_same_log_interval]
 
@@ -173,6 +174,7 @@ module Fluent
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
+      clone.force_stacktrace_level(@forced_stacktrace_level)
       # optional headers/attrs are not copied, because new PluginLogger should have another one of it
       clone
     end
@@ -240,6 +242,14 @@ module Fluent
       nil
     end
 
+    def force_stacktrace_level?
+      not @forced_stacktrace_level.nil?
+    end
+
+    def force_stacktrace_level(level)
+      @forced_stacktrace_level = level
+    end
+
     def enable_debug(b=true)
       @debug_mode = b
       self
@@ -500,6 +510,16 @@ module Fluent
     def dump_stacktrace(type, backtrace, level)
       return if @level > level
 
+      dump_stacktrace_internal(
+        type,
+        backtrace,
+        force_stacktrace_level? ? @forced_stacktrace_level : level,
+      )
+    end
+
+    def dump_stacktrace_internal(type, backtrace, level)
+      return if @level > level
+
       time = Time.now
 
       if @format == :text
@@ -633,6 +653,9 @@ module Fluent
       if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
       end
+      if logger.instance_variable_defined?(:@forced_stacktrace_level)
+        @forced_stacktrace_level = logger.instance_variable_get(:@forced_stacktrace_level)
+      end
       if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
         @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
       end

data/lib/fluent/plugin/bare_output.rb

@@ -40,22 +40,6 @@ module Fluent
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
-      def num_errors
-        @num_errors_metrics.get
-      end
-
-      def emit_count
-        @emit_count_metrics.get
-      end
-
-      def emit_size
-        @emit_size_metrics.get
-      end
-
-      def emit_records
-        @emit_records_metrics.get
-      end
-
       def initialize
         super
         @counter_mutex = Mutex.new

data/lib/fluent/plugin/base.rb

@@ -84,7 +84,7 @@ module Fluent
           yield
         end
         # Update access time to prevent tmpwatch from deleting a lock file.
-        FileUtils.touch(lock_path);
+        FileUtils.touch(lock_path)
       end
 
       def string_safe_encoding(str)
@@ -206,7 +206,7 @@ module Fluent
       end
 
       def reloadable_plugin?
-        # Engine can't capture all class variables. so it's forbbiden to use class variables in each plugins if enabling reload.
+        # Engine can't capture all class variables. so it's forbidden to use class variables in each plugins if enabling reload.
         self.class.class_variables.empty?
       end
     end

data/lib/fluent/plugin/buf_file.rb

@@ -191,7 +191,7 @@ module Fluent
         queue.sort_by!{ |chunk| chunk.modified_at }
 
         # If one of the files is corrupted, other files may also be corrupted and be undetected.
-        # The time priods of each chunk are helpful to check the data.
+        # The time periods of each chunk are helpful to check the data.
         if exist_broken_file
           log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
           (stage.values + queue).each { |chunk|
@@ -229,6 +229,20 @@ module Fluent
         File.unlink(path, path + '.meta') rescue nil
       end
 
+      def evacuate_chunk(chunk)
+        unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk)
+          raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}."
+        end
+
+        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
+        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+
+        FileUtils.copy([chunk.path, chunk.meta_path], backup_dir)
+        log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
+      rescue => e
+        log.error "unexpected error while evacuating chunk files.", error: e
+      end
+
       private
 
       def escaped_patterns(patterns)

data/lib/fluent/plugin/buf_file_single.rb

@@ -202,7 +202,7 @@ module Fluent
         queue.sort_by!(&:modified_at)
 
         # If one of the files is corrupted, other files may also be corrupted and be undetected.
-        # The time priods of each chunk are helpful to check the data.
+        # The time periods of each chunk are helpful to check the data.
         if exist_broken_file
           log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
           (stage.values + queue).each { |chunk|
@@ -241,6 +241,20 @@ module Fluent
         File.unlink(path) rescue nil
       end
 
+      def evacuate_chunk(chunk)
+        unless chunk.is_a?(Fluent::Plugin::Buffer::FileSingleChunk)
+          raise ArgumentError, "The chunk must be FileSingleChunk, but it was #{chunk.class}."
+        end
+
+        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
+        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+
+        FileUtils.copy(chunk.path, backup_dir)
+        log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
+      rescue => e
+        log.error "unexpected error while evacuating chunk files.", error: e
+      end
+
       private
 
       def escaped_patterns(patterns)

data/lib/fluent/plugin/buffer/chunk.rb

@@ -59,8 +59,11 @@ module Fluent
           @size = 0
           @created_at = Fluent::Clock.real_now
           @modified_at = Fluent::Clock.real_now
-
-          extend Decompressable if compress == :gzip
+          if compress == :gzip
+            extend GzipDecompressable
+          elsif compress == :zstd
+            extend ZstdDecompressable
+          end
         end
 
         attr_reader :unique_id, :metadata, :state
@@ -85,10 +88,17 @@ module Fluent
 
         # data is array of formatted record string
         def append(data, **kwargs)
-          raise ArgumentError, '`compress: gzip` can be used for Compressable module' if kwargs[:compress] == :gzip
-          adding = ''.b
-          data.each do |d|
-            adding << d.b
+          raise ArgumentError, "`compress: #{kwargs[:compress]}` can be used for Compressable module" if kwargs[:compress] == :gzip || kwargs[:compress] == :zstd
+          begin
+            adding = data.join.force_encoding(Encoding::ASCII_8BIT)
+          rescue
+            # Fallback
+            # Array#join throws an exception if data contains strings with a different encoding.
+            # Although such cases may be rare, it should be considered as a safety precaution.
+            adding = ''.force_encoding(Encoding::ASCII_8BIT)
+            data.each do |d|
+              adding << d.b
+            end
           end
           concat(adding, data.size)
         end
@@ -165,23 +175,23 @@ module Fluent
         end
 
         def read(**kwargs)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           raise NotImplementedError, "Implement this method in child class"
         end
 
         def open(**kwargs, &block)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           raise NotImplementedError, "Implement this method in child class"
         end
 
         def write_to(io, **kwargs)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           open do |i|
             IO.copy_stream(i, io)
           end
         end
 
-        module Decompressable
+        module GzipDecompressable
           include Fluent::Plugin::Compressable
 
           def append(data, **kwargs)
@@ -234,6 +244,60 @@ module Fluent
             end
           end
         end
+
+        module ZstdDecompressable
+          include Fluent::Plugin::Compressable
+
+          def append(data, **kwargs)
+            if kwargs[:compress] == :zstd
+              io = StringIO.new
+              stream = Zstd::StreamWriter.new(io)
+              data.each do |d|
+                stream.write(d)
+              end
+              stream.finish
+              concat(io.string, data.size)
+            else
+              super
+            end
+          end
+
+          def open(**kwargs, &block)
+            if kwargs[:compressed] == :zstd
+              super
+            else
+              super(**kwargs) do |chunk_io|
+                output_io = if chunk_io.is_a?(StringIO)
+                              StringIO.new
+                            else
+                              Tempfile.new('decompressed-data')
+                            end
+                output_io.binmode if output_io.is_a?(Tempfile)
+                decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
+                output_io.seek(0, IO::SEEK_SET)
+                yield output_io
+              end
+            end
+          end
+
+          def read(**kwargs)
+            if kwargs[:compressed] == :zstd
+              super
+            else
+              decompress(super,type: :zstd)
+            end
+          end
+
+          def write_to(io, **kwargs)
+            open(compressed: :zstd) do |chunk_io|
+              if kwargs[:compressed] == :zstd
+                IO.copy_stream(chunk_io, io)
+              else
+                decompress(input_io: chunk_io, output_io: io, type: :zstd)
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -37,7 +37,7 @@ module Fluent
         # path_prefix: path prefix string, ended with '.'
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
-        attr_reader :path, :permission
+        attr_reader :path, :meta_path, :permission
 
         def initialize(metadata, path, mode, perm: nil, compress: :text)
           super(metadata, compress: compress)
@@ -219,13 +219,17 @@ module Fluent
             # old type of restore
             data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
           end
+          raise FileChunkError, "invalid meta data" if data.nil? || !data.is_a?(Hash)
+          raise FileChunkError, "invalid unique_id" unless data[:id]
+          raise FileChunkError, "invalid created_at" unless data[:c].to_i > 0
+          raise FileChunkError, "invalid modified_at" unless data[:m].to_i > 0
 
           now = Fluent::Clock.real_now
 
-          @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
+          @unique_id = data[:id]
           @size = data[:s] || 0
-          @created_at = data.fetch(:c, now.to_i)
-          @modified_at = data.fetch(:m, now.to_i)
+          @created_at = data[:c]
+          @modified_at = data[:m]
 
           @metadata.timekey = data[:timekey]
           @metadata.tag = data[:tag]
@@ -285,7 +289,7 @@ module Fluent
             @chunk.binmode
           rescue => e
             # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
-            # If other cases are possible, we will change erorr handling with proper classes.
+            # If other cases are possible, we will change error handling with proper classes.
             raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
           end
           begin

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -243,11 +243,11 @@ module Fluent
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::RFC2396_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI::DEFAULT_PARSER.unescape(key)
+          URI::RFC2396_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)
@@ -259,7 +259,7 @@ module Fluent
             @chunk.binmode
           rescue => e
             # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
-            # If other cases are possible, we will change erorr handling with proper classes.
+            # If other cases are possible, we will change error handling with proper classes.
             raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
           end
 

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -68,13 +68,13 @@ module Fluent
 
         def purge
           super
-          @chunk = ''.force_encoding("ASCII-8BIT")
+          @chunk.clear
           @chunk_bytes = @size = @adding_bytes = @adding_size = 0
           true
         end
 
         def read(**kwargs)
-          @chunk
+          @chunk.dup
         end
 
         def open(**kwargs, &block)

data/lib/fluent/plugin/buffer.rb

@@ -64,7 +64,7 @@ module Fluent
       config_param :queued_chunks_limit_size, :integer, default: nil
 
       desc 'Compress buffered data.'
-      config_param :compress, :enum, list: [:text, :gzip], default: :text
+      config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
 
       desc 'If true, chunks are thrown away when unrecoverable error happens'
       config_param :disable_chunk_backup, :bool, default: false
@@ -196,6 +196,8 @@ module Fluent
         @mutex = Mutex.new
       end
 
+      # The metrics_create method defines getter methods named stage_byte_size and queue_byte_size.
+      # For compatibility, stage_size, stage_size=, queue_size, and queue_size= are still available.
       def stage_size
         @stage_size_metrics.get
       end
@@ -385,7 +387,7 @@ module Fluent
           end
 
           errors = []
-          # Buffer plugin estimates there's no serious error cause: will commit for all chunks eigher way
+          # Buffer plugin estimates there's no serious error cause: will commit for all chunks either way
           operated_chunks.each do |chunk|
             begin
               chunk.commit
@@ -523,7 +525,7 @@ module Fluent
           chunks = @stage.values
           chunks.concat(@queue)
           @timekeys = chunks.each_with_object({}) do |chunk, keys|
-            if chunk.metadata && chunk.metadata.timekey
+            if chunk.metadata&.timekey
               t = chunk.metadata.timekey
               keys[t] = keys.fetch(t, 0) + 1
             end
@@ -623,6 +625,7 @@ module Fluent
           until @queue.empty?
             begin
               q = @queue.shift
+              evacuate_chunk(q)
               log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
               q.purge
             rescue => e
@@ -634,6 +637,25 @@ module Fluent
         end
       end
 
+      def evacuate_chunk(chunk)
+        # Overwrite this on demand.
+        #
+        # Note: Difference from the `backup` feature.
+        #       The `backup` feature is for unrecoverable errors, mainly for bad chunks.
+        #       On the other hand, this feature is for normal chunks.
+        #       The main motivation for this feature is to enable recovery by evacuating buffer files
+        #       when the retry limit is reached due to external factors such as network issues.
+        #
+        # Note: Difference from the `secondary` feature.
+        #       The `secondary` feature is not suitable for recovery.
+        #       It can be difficult to recover files made by `out_secondary_file` because the metadata
+        #       is lost.
+        #       For file buffers, the easiest way for recovery is to evacuate the chunk files as is.
+        #       Once the issue is recovered, we can put back the chunk files, and restart Fluentd to
+        #       load them.
+        #       This feature enables it.
+      end
+
       def chunk_size_over?(chunk)
         chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)
       end
@@ -923,8 +945,6 @@ module Fluent
           return
         end
 
-        safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
-        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
         backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
         backup_dir = File.dirname(backup_file)
 
@@ -938,11 +958,19 @@ module Fluent
       def optimistic_queued?(metadata = nil)
         if metadata
           n = @queued_num[metadata]
-          n && n.nonzero?
+          n&.nonzero?
         else
           !@queue.empty?
         end
       end
+
+      def safe_owner_id
+        owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+      end
+
+      def backup_base_dir
+        system_config.root_dir || DEFAULT_BACKUP_DIR
+      end
     end
   end
 end