fluentd 1.15.2-x86-mingw32 → 1.16.1-x86-mingw32

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -59,7 +59,13 @@ module Fluent::Plugin
             @ack_waitings = new_list
           end
 
-          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          begin
+            readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          rescue IOError
+            @log.info "connection closed while waiting for readable sockets"
+            readable_sockets = nil
+          end
+
           if readable_sockets
             readable_sockets.each do |sock|
               results << read_ack_from_sock(sock)
@@ -109,13 +115,22 @@ module Fluent::Plugin
           raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
         rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
           raw_data = ''
+        rescue IOError
+          @log.info "socket closed while receiving ack response"
+          return nil, Result::FAILED
         end
 
         info = find(sock)
 
-        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-        # If this happens we assume the data wasn't delivered and retry it.
-        if raw_data.empty?
+        if info.nil?
+          # The info can be deleted by another thread during `sock.recv()` and `find()`.
+          # This is OK since another thread has completed to process the ack, so we can skip this.
+          # Note: exclusion mechanism about `collect_response()` may need to be considered.
+          @log.debug "could not find the ack info. this ack may be processed by another thread."
+          return nil, Result::FAILED
+        elsif raw_data.empty?
+          # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+          # If this happens we assume the data wasn't delivered and retry it.
           @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
           # info.node.disable!
           return info, Result::FAILED

data/lib/fluent/plugin/out_forward.rb

@@ -521,8 +521,8 @@ module Fluent::Plugin
         when AckHandler::Result::SUCCESS
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
-          node.disable!
-          rollback_write(chunk_id, update_retry: false)
+          node&.disable!
+          rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
           rollback_write(chunk_id, update_retry: false)
         else

data/lib/fluent/plugin/out_secondary_file.rb

@@ -64,31 +64,29 @@ module Fluent::Plugin
     end
 
     def multi_workers_ready?
-      ### TODO: add hack to synchronize for multi workers
       true
     end
 
     def write(chunk)
       path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
-      path = generate_path(path_without_suffix)
-      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
-
-      case @compress
-      when :text
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          chunk.write_to(f)
-        }
-      when :gzip
-        File.open(path, "ab", @file_perm) {|f|
-          f.flock(File::LOCK_EX)
-          gz = Zlib::GzipWriter.new(f)
-          chunk.write_to(gz)
-          gz.close
-        }
+      generate_path(path_without_suffix) do |path|
+        FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+        case @compress
+        when :text
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            chunk.write_to(f)
+          }
+        when :gzip
+          File.open(path, "ab", @file_perm) {|f|
+            f.flock(File::LOCK_EX)
+            gz = Zlib::GzipWriter.new(f)
+            chunk.write_to(gz)
+            gz.close
+          }
+        end
       end
-
-      path
     end
 
     private
@@ -117,15 +115,34 @@ module Fluent::Plugin
 
     def generate_path(path_without_suffix)
       if @append
-        "#{path_without_suffix}#{@suffix}"
-      else
+        path = "#{path_without_suffix}#{@suffix}"
+        synchronize_path(path) do
+          yield path
+        end
+        return path
+      end
+
+      begin
         i = 0
         loop do
           path = "#{path_without_suffix}.#{i}#{@suffix}"
-          return path unless File.exist?(path)
+          break unless File.exist?(path)
           i += 1
         end
+        synchronize_path(path) do
+          # If multiple processes or threads select the same path and another
+          # one entered this locking block first, the file should already
+          # exist and this one should retry to find new path.
+          raise FileAlreadyExist if File.exist?(path)
+          yield path
+        end
+      rescue FileAlreadyExist
+        retry
       end
+      path
+    end
+
+    class FileAlreadyExist < StandardError
     end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -99,7 +99,6 @@ module Fluent
         config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
 
         config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
-        config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
       end
 
       config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
@@ -199,6 +198,7 @@ module Fluent
       def initialize
         super
         @counter_mutex = Mutex.new
+        @flush_thread_mutex = Mutex.new
         @buffering = false
         @delayed_commit = false
         @as_secondary = false
@@ -378,6 +378,7 @@ module Fluent
           buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
           @buffer = Plugin.new_buffer(buffer_type, parent: self)
           @buffer.configure(buffer_conf)
+          keep_buffer_config_compat
           @buffer.enable_update_timekeys if @chunk_key_time
 
           @flush_at_shutdown = @buffer_config.flush_at_shutdown
@@ -425,7 +426,9 @@ module Fluent
           end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
+          if (@secondary.class.to_s != "Fluent::Plugin::SecondaryFileOutput") &&
+             (self.class != @secondary.class) &&
+             (@custom_format || @secondary.implement?(:custom_format))
             log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
@@ -435,6 +438,12 @@ module Fluent
         self
       end
 
+      def keep_buffer_config_compat
+        # Need this to call `@buffer_config.disable_chunk_backup` just as before,
+        # since some plugins may use this option in this way.
+        @buffer_config[:disable_chunk_backup] = @buffer.disable_chunk_backup
+      end
+
       def start
         super
 
@@ -591,6 +600,42 @@ module Fluent
         super
       end
 
+      def actual_flush_thread_count
+        return 0 unless @buffering
+        return @buffer_config.flush_thread_count unless @as_secondary
+        @primary_instance.buffer_config.flush_thread_count
+      end
+
+      # Ensures `path` (filename or filepath) processable
+      # only by the current thread in the current process.
+      # For multiple workers, the lock is shared if `path` is the same value.
+      # For multiple threads, the lock is shared by all threads in the same process.
+      def synchronize_path(path)
+        synchronize_path_in_workers(path) do
+          synchronize_in_threads do
+            yield
+          end
+        end
+      end
+
+      def synchronize_path_in_workers(path)
+        need_worker_lock = system_config.workers > 1
+        if need_worker_lock
+          acquire_worker_lock(path) { yield }
+        else
+          yield
+        end
+      end
+
+      def synchronize_in_threads
+        need_thread_lock = actual_flush_thread_count > 1
+        if need_thread_lock
+          @flush_thread_mutex.synchronize { yield }
+        else
+          yield
+        end
+      end
+
       def support_in_v12_style?(feature)
         # for plugins written in v0.12 styles
         case feature
@@ -1240,18 +1285,10 @@ module Fluent
       end
 
       def backup_chunk(chunk, using_secondary, delayed_commit)
-        if @buffer_config.disable_chunk_backup
+        if @buffer.disable_chunk_backup
           log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
         else
-          unique_id = dump_unique_id_hex(chunk.unique_id)
-          safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
-          backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
-          backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
-          backup_dir = File.dirname(backup_file)
-
-          log.warn "bad chunk is moved to #{backup_file}"
-          FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
-          File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
+          @buffer.backup(chunk.unique_id) { |f|
             chunk.write_to(f)
           }
         end

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -21,6 +21,7 @@ require 'async/http/endpoint'
 require 'fluent/plugin_helper/http_server/app'
 require 'fluent/plugin_helper/http_server/router'
 require 'fluent/plugin_helper/http_server/methods'
+require 'fluent/log/console_adapter'
 
 module Fluent
   module PluginHelper
@@ -38,7 +39,7 @@ module Fluent
           scheme = tls_context ? 'https' : 'http'
           @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
           @router = Router.new(default_app)
-          @reactor = Async::Reactor.new(nil, logger: @logger)
+          @reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(@logger))
 
           opts = if tls_context
                    { ssl_context: tls_context }

data/lib/fluent/plugin_helper/server.rb

@@ -545,6 +545,10 @@ module Fluent
               data = @sock.recv(@max_bytes, @flags)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data)
           rescue => e
@@ -558,6 +562,10 @@ module Fluent
               data, addr = @sock.recvfrom(@max_bytes)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
           rescue => e