fluentd 1.15.2-x64-mingw32 → 1.16.0-x64-mingw32

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/log/console_adapter.rb

@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'console/terminal/logger'
+
+module Fluent
+  class Log
+    # Async gem which is used by http_server helper switched logger mechanism to
+    # Console gem which isn't complatible with Ruby's standard Logger (since
+    # v1.17). This class adapts it to Fluentd's logger mechanism.
+    class ConsoleAdapter < Console::Terminal::Logger
+      def self.wrap(logger)
+        _, level = Console::Logger::LEVELS.find { |key, value|
+          if logger.level <= 0
+            key == :debug
+          else
+            value == logger.level - 1
+          end
+        }
+        Console::Logger.new(ConsoleAdapter.new(logger), level: level)
+      end
+
+      def initialize(logger)
+        @logger = logger
+        # When `verbose` is `true`, following items will be added as a prefix or
+        # suffix of the subject:
+        #   * Severity
+        #   * Object ID
+        #   * PID
+        #   * Time
+        # Severity and Time are added by Fluentd::Log too so they are redundant.
+        # PID is the worker's PID so it's also redundant.
+        # Object ID will be too verbose in usual cases.
+        # So set it as `false` here to suppress redundant items.
+        super(StringIO.new, verbose: false)
+      end
+
+      def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
+        if LEVEL_TEXT.include?(severity.to_s)
+          level = severity
+        else
+          @logger.warn("Unknown severity: #{severity}")
+          level = 'warn'
+        end
+
+        @io.seek(0)
+        @io.truncate(0)
+        super
+        @logger.send(level, @io.string.chomp)
+      end
+    end
+  end
+end

data/lib/fluent/log.rb

@@ -67,8 +67,29 @@ module Fluent
       LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
     end
 
+    # Create a unique path for each process.
+    #
+    # >>> per_process_path("C:/tmp/test.log", :worker, 1)
+    # C:/tmp/test-1.log
+    # >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
+    # C:/tmp/test-supervisor-0.log
+    def self.per_process_path(path, process_type, worker_id)
+      path = Pathname(path)
+      ext = path.extname
+
+      if process_type == :supervisor
+        suffix = "-#{process_type}-0#{ext}"  # "-0" for backword compatibility.
+      else
+        suffix = "-#{worker_id}#{ext}"
+      end
+      return path.sub_ext(suffix).to_s
+    end
+
     def initialize(logger, opts={})
-      # overwrites logger.level= so that config reloading resets level of Fluentd::Log
+      # When ServerEngine changes the logger.level, the Fluentd logger level should also change.
+      # So overwrites logger.level= below.
+      # However, currently Fluentd doesn't use the ServerEngine's reloading feature,
+      # so maybe we don't need this overwriting anymore.
       orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
       me = self
       # The original ruby logger sets the number as each log level like below.
@@ -92,6 +113,7 @@ module Fluent
       # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
       logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
 
+      @path = opts[:path]
       @logger = logger
       @out = logger.instance_variable_get(:@logdev)
       @level = logger.level + 1
@@ -102,7 +124,8 @@ module Fluent
       @time_format = nil
       @formatter = nil
 
-      self.format = :text
+      self.format = opts.fetch(:format, :text)
+      self.time_format = opts[:time_format] if opts.key?(:time_format)
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -154,8 +177,12 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
     attr_accessor :out
+    # Strictly speaking, we should also change @logger.level when the setter of @level is called.
+    # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
+    # Since We overwrites logger.level= so that @logger.level is applied to @level,
+    # we need to find a good way to do this, otherwise we will end up in an endless loop.
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
 
@@ -202,9 +229,12 @@ module Fluent
       @time_formatter = Strftime.new(@time_format) rescue nil
     end
 
+    def stdout?
+      @out == $stdout
+    end
+
     def reopen!
-      # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
-      @logger.reopen! if @logger
+      @out.reopen(@path, "a") if @path && @path != "-"
       nil
     end
 

data/lib/fluent/oj_options.rb

@@ -4,14 +4,13 @@ module Fluent
   class OjOptions
     OPTIONS = {
       'bigdecimal_load': :symbol,
-      'max_nesting': :integer,
       'mode': :symbol,
       'use_to_json': :bool
     }
 
     ALLOWED_VALUES = {
       'bigdecimal_load': %i[bigdecimal float auto],
-      'mode': %i[strict null compat json rails object custom]
+      'mode': %i[strict null compat json rails custom]
     }
 
     DEFAULTS = {

data/lib/fluent/plugin/base.rb

@@ -53,14 +53,12 @@ module Fluent
       end
 
       def configure(conf)
-        if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
-          workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
-                      conf.target_worker_ids.size
-                    else
-                      1
-                    end
-          system_config_override(workers: workers)
+        raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
+
+        if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
+          system_config_override(workers: conf.target_worker_ids.size)
         end
+
         super(conf, system_config.strict_config_value)
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true

data/lib/fluent/plugin/buf_file.rb

@@ -139,13 +139,20 @@ module Fluent
       def resume
         stage = {}
         queue = []
+        exist_broken_file = false
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
         Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
-          log.debug { "restoring buffer file: path = #{path}" }
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
 
           m = new_metadata() # this metadata will be overwritten by resuming .meta file content
                              # so it should not added into @metadata_list for now
@@ -158,6 +165,7 @@ module Fluent
           begin
             chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            exist_broken_file = true
             handle_broken_files(path, mode, e)
             next
           end
@@ -182,6 +190,15 @@ module Fluent
 
         queue.sort_by!{ |chunk| chunk.modified_at }
 
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
         return stage, queue
       end
 
@@ -195,8 +212,20 @@ module Fluent
       end
 
       def handle_broken_files(path, mode, e)
-        log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
-        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
         File.unlink(path, path + '.meta') rescue nil
       end
 

data/lib/fluent/plugin/buf_file_single.rb

@@ -160,13 +160,20 @@ module Fluent
       def resume
         stage = {}
         queue = []
+        exist_broken_file = false
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
         Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
-          log.debug { "restoring buffer file: path = #{path}" }
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
 
           m = new_metadata() # this metadata will be updated in FileSingleChunk.new
           mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
@@ -179,6 +186,7 @@ module Fluent
             chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
             chunk.restore_size(@chunk_format) if @calc_num_records
           rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
+            exist_broken_file = true
             handle_broken_files(path, mode, e)
             next
           end
@@ -193,6 +201,15 @@ module Fluent
 
         queue.sort_by!(&:modified_at)
 
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
         return stage, queue
       end
 
@@ -207,8 +224,20 @@ module Fluent
       end
 
       def handle_broken_files(path, mode, e)
-        log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
-        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
         File.unlink(path) rescue nil
       end
 

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -204,7 +204,7 @@ module Fluent
           end
         end
 
-        # used only for queued v0.12 buffer path
+        # used only for queued v0.12 buffer path or broken files
         def self.unique_id_from_path(path)
           if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
             return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')

data/lib/fluent/plugin/buffer.rb

@@ -66,6 +66,9 @@ module Fluent
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip], default: :text
 
+      desc 'If true, chunks are thrown away when unrecoverable error happens'
+      config_param :disable_chunk_backup, :bool, default: false
+
       Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
         def initialize(timekey, tag, variables)
           super(timekey, tag, variables, 0)
@@ -903,6 +906,24 @@ module Fluent
         { 'buffer' => stats }
       end
 
+      def backup(chunk_unique_id)
+        unique_id = dump_unique_id_hex(chunk_unique_id)
+
+        if @disable_chunk_backup
+          log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
+          return
+        end
+
+        safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
+        backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
+        backup_dir = File.dirname(backup_file)
+
+        log.warn "bad chunk is moved to #{backup_file}"
+        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+        File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
+      end
+
       private
 
       def optimistic_queued?(metadata = nil)

data/lib/fluent/plugin/in_tail.rb

@@ -25,12 +25,7 @@ require 'fluent/variable_store'
 require 'fluent/capability'
 require 'fluent/plugin/in_tail/position_file'
 require 'fluent/plugin/in_tail/group_watch'
-
-if Fluent.windows?
-  require_relative 'file_wrapper'
-else
-  Fluent::FileWrapper = File
-end
+require 'fluent/file_wrapper'
 
 module Fluent::Plugin
   class TailInput < Fluent::Plugin::Input

data/lib/fluent/plugin/in_tcp.rb

@@ -40,6 +40,8 @@ module Fluent::Plugin
 
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
 
     # in_forward like host/network restriction
     config_section :security, required: false, multi: false do
@@ -101,7 +103,7 @@ module Fluent::Plugin
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
       if @_extract_enabled && @_extract_tag_key
-        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -131,7 +133,7 @@ module Fluent::Plugin
           buf.slice!(0, pos) if pos > 0
         end
       else
-        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next

data/lib/fluent/plugin/out_file.rb

@@ -188,10 +188,6 @@ module Fluent::Plugin
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
-
-      if @need_lock && @append && @fluentd_lock_dir.nil?
-        raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
-      end
     end
 
     def multi_workers_ready?

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -59,7 +59,13 @@ module Fluent::Plugin
             @ack_waitings = new_list
           end
 
-          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          begin
+            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          rescue IOError
+            @log.info "connection closed while waiting for readable sockets"
+            readable_sockets = nil
+          end
+
           if readable_sockets
             readable_sockets.each do |sock|
               results << read_ack_from_sock(sock)
@@ -109,13 +115,22 @@ module Fluent::Plugin
           raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
         rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
           raw_data = ''
+        rescue IOError
+          @log.info "socket closed while receiving ack response"
+          return nil, Result::FAILED
         end
 
         info = find(sock)
 
-        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-        # If this happens we assume the data wasn't delivered and retry it.
-        if raw_data.empty?
+        if info.nil?
+          # The info can be deleted by another thread during `sock.recv()` and `find()`.
+          # This is OK since another thread has completed to process the ack, so we can skip this.
+          # Note: exclusion mechanism about `collect_response()` may need to be considered.
+          @log.debug "could not find the ack info. this ack may be processed by another thread."
+          return nil, Result::FAILED
+        elsif raw_data.empty?
+          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+          # If this happens we assume the data wasn't delivered and retry it.
           @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
           # info.node.disable!
           return info, Result::FAILED

data/lib/fluent/plugin/out_forward.rb

@@ -521,8 +521,8 @@ module Fluent::Plugin
         when AckHandler::Result::SUCCESS
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
-          node.disable!
-          rollback_write(chunk_id, update_retry: false)
+          node&.disable!
+          rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
           rollback_write(chunk_id, update_retry: false)
         else