fluentd 1.15.3-x64-mingw32 → 1.16.1-x64-mingw32

data/lib/fluent/log/console_adapter.rb

@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'console/terminal/logger'
+
+module Fluent
+  class Log
+    # Async gem which is used by http_server helper switched logger mechanism to
+    # Console gem which isn't complatible with Ruby's standard Logger (since
+    # v1.17). This class adapts it to Fluentd's logger mechanism.
+    class ConsoleAdapter < Console::Terminal::Logger
+      def self.wrap(logger)
+        _, level = Console::Logger::LEVELS.find { |key, value|
+          if logger.level <= 0
+            key == :debug
+          else
+            value == logger.level - 1
+          end
+        }
+        Console::Logger.new(ConsoleAdapter.new(logger), level: level)
+      end
+
+      def initialize(logger)
+        @logger = logger
+        # When `verbose` is `true`, following items will be added as a prefix or
+        # suffix of the subject:
+        #   * Severity
+        #   * Object ID
+        #   * PID
+        #   * Time
+        # Severity and Time are added by Fluentd::Log too so they are redundant.
+        # PID is the worker's PID so it's also redundant.
+        # Object ID will be too verbose in usual cases.
+        # So set it as `false` here to suppress redundant items.
+        super(StringIO.new, verbose: false)
+      end
+
+      def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
+        if LEVEL_TEXT.include?(severity.to_s)
+          level = severity
+        else
+          @logger.warn("Unknown severity: #{severity}")
+          level = 'warn'
+        end
+
+        @io.seek(0)
+        @io.truncate(0)
+        super
+        @logger.send(level, @io.string.chomp)
+      end
+    end
+  end
+end

data/lib/fluent/log.rb

@@ -67,8 +67,29 @@ module Fluent
       LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
     end
 
+    # Create a unique path for each process.
+    #
+    # >>> per_process_path("C:/tmp/test.log", :worker, 1)
+    # C:/tmp/test-1.log
+    # >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
+    # C:/tmp/test-supervisor-0.log
+    def self.per_process_path(path, process_type, worker_id)
+      path = Pathname(path)
+      ext = path.extname
+
+      if process_type == :supervisor
+        suffix = "-#{process_type}-0#{ext}"  # "-0" for backword compatibility.
+      else
+        suffix = "-#{worker_id}#{ext}"
+      end
+      return path.sub_ext(suffix).to_s
+    end
+
     def initialize(logger, opts={})
-      # overwrites logger.level= so that config reloading resets level of Fluentd::Log
+      # When ServerEngine changes the logger.level, the Fluentd logger level should also change.
+      # So overwrites logger.level= below.
+      # However, currently Fluentd doesn't use the ServerEngine's reloading feature,
+      # so maybe we don't need this overwriting anymore.
       orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
       me = self
       # The original ruby logger sets the number as each log level like below.
@@ -92,6 +113,7 @@ module Fluent
       # So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
       logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
 
+      @path = opts[:path]
       @logger = logger
       @out = logger.instance_variable_get(:@logdev)
       @level = logger.level + 1
@@ -102,7 +124,8 @@ module Fluent
       @time_format = nil
       @formatter = nil
 
-      self.format = :text
+      self.format = opts.fetch(:format, :text)
+      self.time_format = opts[:time_format] if opts.key?(:time_format)
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -154,8 +177,12 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
     attr_accessor :out
+    # Strictly speaking, we should also change @logger.level when the setter of @level is called.
+    # Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
+    # Since We overwrites logger.level= so that @logger.level is applied to @level,
+    # we need to find a good way to do this, otherwise we will end up in an endless loop.
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
 
@@ -202,9 +229,12 @@ module Fluent
       @time_formatter = Strftime.new(@time_format) rescue nil
     end
 
+    def stdout?
+      @out == $stdout
+    end
+
     def reopen!
-      # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
-      @logger.reopen! if @logger
+      @out.reopen(@path, "a") if @path && @path != "-"
       nil
     end
 

data/lib/fluent/plugin/base.rb

@@ -53,14 +53,12 @@ module Fluent
       end
 
       def configure(conf)
-        if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
-          workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
-                      conf.target_worker_ids.size
-                    else
-                      1
-                    end
-          system_config_override(workers: workers)
+        raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
+
+        if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
+          system_config_override(workers: conf.target_worker_ids.size)
         end
+
         super(conf, system_config.strict_config_value)
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true

data/lib/fluent/plugin/buf_file.rb

@@ -139,13 +139,20 @@ module Fluent
       def resume
         stage = {}
         queue = []
+        exist_broken_file = false
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
         Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
-          log.debug { "restoring buffer file: path = #{path}" }
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
 
           m = new_metadata() # this metadata will be overwritten by resuming .meta file content
                              # so it should not added into @metadata_list for now
@@ -158,6 +165,7 @@ module Fluent
           begin
             chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            exist_broken_file = true
             handle_broken_files(path, mode, e)
             next
           end
@@ -182,6 +190,15 @@ module Fluent
 
         queue.sort_by!{ |chunk| chunk.modified_at }
 
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
         return stage, queue
       end
 
@@ -195,8 +212,20 @@ module Fluent
       end
 
       def handle_broken_files(path, mode, e)
-        log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
-        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
         File.unlink(path, path + '.meta') rescue nil
       end
 

data/lib/fluent/plugin/buf_file_single.rb

@@ -160,13 +160,20 @@ module Fluent
       def resume
         stage = {}
         queue = []
+        exist_broken_file = false
 
         patterns = [@path]
         patterns.unshift @additional_resume_path if @additional_resume_path
         Dir.glob(escaped_patterns(patterns)) do |path|
           next unless File.file?(path)
 
-          log.debug { "restoring buffer file: path = #{path}" }
+          if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
+            # When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
+            # since there may be a power failure or similar failure.
+            log.warn { "restoring buffer file: path = #{path}" }
+          else
+            log.debug { "restoring buffer file: path = #{path}" }
+          end
 
           m = new_metadata() # this metadata will be updated in FileSingleChunk.new
           mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
@@ -179,6 +186,7 @@ module Fluent
             chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
             chunk.restore_size(@chunk_format) if @calc_num_records
           rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
+            exist_broken_file = true
             handle_broken_files(path, mode, e)
             next
           end
@@ -193,6 +201,15 @@ module Fluent
 
         queue.sort_by!(&:modified_at)
 
+        # If one of the files is corrupted, other files may also be corrupted and be undetected.
+        # The time priods of each chunk are helpful to check the data.
+        if exist_broken_file
+          log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
+          (stage.values + queue).each { |chunk|
+            log.info "  #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
+          }
+        end
+
         return stage, queue
       end
 
@@ -207,8 +224,20 @@ module Fluent
       end
 
       def handle_broken_files(path, mode, e)
-        log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
-        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
+        unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
+        backup(unique_id) { |f|
+          File.open(path, 'rb') { |chunk|
+            chunk.set_encoding(Encoding::ASCII_8BIT)
+            chunk.sync = true
+            chunk.binmode
+            IO.copy_stream(chunk, f)
+          }
+        }
+      rescue => error
+        log.error "backup failed. Delete corresponding files.", :err_msg => error.message
+      ensure
+        log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
         File.unlink(path) rescue nil
       end
 

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -204,7 +204,7 @@ module Fluent
           end
         end
 
-        # used only for queued v0.12 buffer path
+        # used only for queued v0.12 buffer path or broken files
         def self.unique_id_from_path(path)
           if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
             return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')

data/lib/fluent/plugin/buffer.rb

@@ -66,6 +66,9 @@ module Fluent
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip], default: :text
 
+      desc 'If true, chunks are thrown away when unrecoverable error happens'
+      config_param :disable_chunk_backup, :bool, default: false
+
       Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
         def initialize(timekey, tag, variables)
           super(timekey, tag, variables, 0)
@@ -903,6 +906,24 @@ module Fluent
         { 'buffer' => stats }
       end
 
+      def backup(chunk_unique_id)
+        unique_id = dump_unique_id_hex(chunk_unique_id)
+
+        if @disable_chunk_backup
+          log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
+          return
+        end
+
+        safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
+        backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
+        backup_dir = File.dirname(backup_file)
+
+        log.warn "bad chunk is moved to #{backup_file}"
+        FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
+        File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
+      end
+
       private
 
       def optimistic_queued?(metadata = nil)

data/lib/fluent/plugin/in_tcp.rb

@@ -36,10 +36,16 @@ module Fluent::Plugin
     desc "The field name of the client's address."
     config_param :source_address_key, :string, default: nil
 
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
 
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
 
     # in_forward like host/network restriction
     config_section :security, required: false, multi: false do
@@ -100,8 +106,9 @@ module Fluent::Plugin
 
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
+      discard_till_next_delimiter = false
       if @_extract_enabled && @_extract_tag_key
-        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -114,6 +121,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -129,9 +146,18 @@ module Fluent::Plugin
             end
           end
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       else
-        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
             conn.close
             next
@@ -145,6 +171,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -159,6 +195,15 @@ module Fluent::Plugin
           end
           router.emit_stream(@tag, es)
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       end
     end

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -59,7 +59,13 @@ module Fluent::Plugin
             @ack_waitings = new_list
           end
 
-          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          begin
+            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          rescue IOError
+            @log.info "connection closed while waiting for readable sockets"
+            readable_sockets = nil
+          end
+
           if readable_sockets
             readable_sockets.each do |sock|
               results << read_ack_from_sock(sock)
@@ -109,13 +115,22 @@ module Fluent::Plugin
           raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
         rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
           raw_data = ''
+        rescue IOError
+          @log.info "socket closed while receiving ack response"
+          return nil, Result::FAILED
         end
 
         info = find(sock)
 
-        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-        # If this happens we assume the data wasn't delivered and retry it.
-        if raw_data.empty?
+        if info.nil?
+          # The info can be deleted by another thread during `sock.recv()` and `find()`.
+          # This is OK since another thread has completed to process the ack, so we can skip this.
+          # Note: exclusion mechanism about `collect_response()` may need to be considered.
+          @log.debug "could not find the ack info. this ack may be processed by another thread."
+          return nil, Result::FAILED
+        elsif raw_data.empty?
+          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+          # If this happens we assume the data wasn't delivered and retry it.
           @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
           # info.node.disable!
           return info, Result::FAILED

data/lib/fluent/plugin/out_forward.rb

@@ -521,8 +521,8 @@ module Fluent::Plugin
         when AckHandler::Result::SUCCESS
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
-          node.disable!
-          rollback_write(chunk_id, update_retry: false)
+          node&.disable!
+          rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
           rollback_write(chunk_id, update_retry: false)
         else

data/lib/fluent/plugin/out_secondary_file.rb

@@ -64,31 +64,29 @@ module Fluent::Plugin
     end
 
     def multi_workers_ready?
-      ### TODO: add hack to synchronize for multi workers
       true
     end
 
     def write(chunk)
       path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
-      path = generate_path(path_without_suffix)
-      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
-
-      case @compress
-      when :text
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          chunk.write_to(f)
-        }
-      when :gzip
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          gz = Zlib::GzipWriter.new(f)
-          chunk.write_to(gz)
-          gz.close
-        }
+      generate_path(path_without_suffix) do |path|
+        FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+        case @compress
+        when :text
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            chunk.write_to(f)
+          }
+        when :gzip
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            gz = Zlib::GzipWriter.new(f)
+            chunk.write_to(gz)
+            gz.close
+          }
+        end
       end
-
-      path
     end
 
     private
@@ -117,15 +115,34 @@ module Fluent::Plugin
 
     def generate_path(path_without_suffix)
       if @append
-        "#{path_without_suffix}#{@suffix}"
-      else
+        path = "#{path_without_suffix}#{@suffix}"
+        synchronize_path(path) do
+          yield path
+        end
+        return path
+      end
+
+      begin
         i = 0
         loop do
           path = "#{path_without_suffix}.#{i}#{@suffix}"
-          return path unless File.exist?(path)
+          break unless File.exist?(path)
           i += 1
         end
+        synchronize_path(path) do
+          # If multiple processes or threads select the same path and another
+          # one entered this locking block first, the file should already
+          # exist and this one should retry to find new path.
+          raise FileAlreadyExist if File.exist?(path)
+          yield path
+        end
+      rescue FileAlreadyExist
+        retry
       end
+      path
+    end
+
+    class FileAlreadyExist < StandardError
     end
   end
 end