fluentd 1.17.1 → 1.19.0

data/lib/fluent/plugin/multi_output.rb

@@ -56,22 +56,6 @@ module Fluent
         @enable_size_metrics = false
       end
 
-      def num_errors
-        @num_errors_metrics.get
-      end
-
-      def emit_count
-        @emit_count_metrics.get
-      end
-
-      def emit_size
-        @emit_size_metrics.get
-      end
-
-      def emit_records
-        @emit_records_metrics.get
-      end
-
       def statistics
         stats = {
           'num_errors' => @num_errors_metrics.get,
@@ -91,7 +75,7 @@ module Fluent
         super
 
         @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
-        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_count", help_text: "Number of count emits")
         @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
         @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
         @enable_size_metrics = !!system_config.enable_size_metrics

data/lib/fluent/plugin/out_buffer.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class BufferOutput < Output
+    Fluent::Plugin.register_output("buffer", self)
+    helpers :event_emitter
+
+    config_section :buffer do
+      config_set_default :@type, "file"
+      config_set_default :chunk_keys, ["tag"]
+      config_set_default :flush_mode, :interval
+      config_set_default :flush_interval, 10
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      return if chunk.empty?
+      router.emit_stream(chunk.metadata.tag, Fluent::MessagePackEventStream.new(chunk.read))
+    end
+  end
+end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -193,8 +193,8 @@ module Fluent::Plugin
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
           end
           c.mutex.synchronize do
-            (c.writeio && c.writeio.close) rescue nil
-            (c.readio && c.readio.close) rescue nil
+            c.writeio&.close rescue nil
+            c.readio&.close rescue nil
             c.pid = c.readio = c.writeio = nil
           end
         end

data/lib/fluent/plugin/out_file.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'zlib'
 require 'time'
+require 'pathname'
 
 require 'fluent/plugin/output'
 require 'fluent/config/error'
@@ -29,11 +30,12 @@ module Fluent::Plugin
 
     helpers :formatter, :inject, :compat_parameters
 
-    SUPPORTED_COMPRESS = [:text, :gz, :gzip]
+    SUPPORTED_COMPRESS = [:text, :gz, :gzip, :zstd]
     SUPPORTED_COMPRESS_MAP = {
       text: nil,
       gz: :gzip,
       gzip: :gzip,
+      zstd: :zstd,
     }
 
     DEFAULT_TIMEKEY = 60 * 60 * 24
@@ -53,6 +55,8 @@ module Fluent::Plugin
     config_param :recompress, :bool, default: false
     desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
     config_param :symlink_path, :string, default: nil
+    desc "Use relative path for symlink target (default: false)"
+    config_param :symlink_path_use_relative, :bool, default: false
 
     config_section :format do
       config_set_default :@type, 'out_file'
@@ -96,7 +100,12 @@ module Fluent::Plugin
         if chunk.metadata == @latest_metadata
           sym_path = @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk)
           FileUtils.mkdir_p(File.dirname(sym_path), mode: @_output_plugin_for_symlink.dir_perm)
-          FileUtils.ln_sf(chunk.path, sym_path)
+          if @_output_plugin_for_symlink.symlink_path_use_relative
+            relative_path = Pathname.new(chunk.path).relative_path_from(Pathname.new(File.dirname(sym_path)))
+            FileUtils.ln_sf(relative_path, sym_path)
+          else
+            FileUtils.ln_sf(chunk.path, sym_path)
+          end
         end
         chunk
       end
@@ -184,18 +193,15 @@ module Fluent::Plugin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self
         end
+
+        if @compress != :text && @buffer.compress != :text && @buffer.compress != @compress_method
+          raise Fluent::ConfigError, "You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})"
+        end
       end
 
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
-
-      # https://github.com/fluent/fluentd/issues/3569
-      @need_ruby_on_macos_workaround = false
-      if @append && Fluent.macos?
-        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
-        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
-      end
     end
 
     def multi_workers_ready?
@@ -212,17 +218,17 @@ module Fluent::Plugin
       FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
       writer = case
-               when @compress_method.nil?
-                 method(:write_without_compression)
-               when @compress_method == :gzip
-                 if @buffer.compress != :gzip || @recompress
-                   method(:write_gzip_with_compression)
-                 else
-                   method(:write_gzip_from_gzipped_chunk)
-                 end
-               else
-                 raise "BUG: unknown compression method #{@compress_method}"
-               end
+                when @compress_method.nil?
+                  method(:write_without_compression)
+                when @compress_method != :text
+                  if @buffer.compress == :text || @recompress
+                    method(:write_with_compression).curry.call(@compress_method)
+                  else
+                    method(:write_from_compressed_chunk).curry.call(@compress_method)
+                  end
+                else
+                  raise "BUG: unknown compression method #{@compress_method}"
+                end
 
       if @append
         if @need_lock
@@ -244,26 +250,26 @@ module Fluent::Plugin
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        if @need_ruby_on_macos_workaround
-          content = chunk.read()
-          f.puts content
-        else
-          chunk.write_to(f)
-        end
+        chunk.write_to(f)
       end
     end
 
-    def write_gzip_with_compression(path, chunk)
+    def write_with_compression(type, path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        gz = Zlib::GzipWriter.new(f)
+        gz = nil
+        if type == :gzip
+          gz = Zlib::GzipWriter.new(f)
+        elsif type == :zstd
+          gz = Zstd::StreamWriter.new(f)
+        end
         chunk.write_to(gz, compressed: :text)
         gz.close
       end
     end
 
-    def write_gzip_from_gzipped_chunk(path, chunk)
+    def write_from_compressed_chunk(type, path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f, compressed: :gzip)
+        chunk.write_to(f, compressed: type)
       end
     end
 
@@ -280,6 +286,7 @@ module Fluent::Plugin
     def compression_suffix(compress)
       case compress
       when :gzip then '.gz'
+      when :zstd then '.zstd'
       when nil then ''
       else
         raise ArgumentError, "unknown compression type #{compress}"

data/lib/fluent/plugin/out_forward/connection_manager.rb

@@ -24,7 +24,7 @@ module Fluent::Plugin
       # @param log [Logger]
       # @param secure [Boolean]
       # @param connection_factory [Proc]
-      # @param SocketCache [Fluent::ForwardOutput::SocketCache]
+      # @param socket_cache [Fluent::ForwardOutput::SocketCache]
       def initialize(log:, secure:, connection_factory:, socket_cache:)
         @log = log
         @secure = secure
@@ -36,7 +36,7 @@ module Fluent::Plugin
         @socket_cache && @socket_cache.clear
       end
 
-      # @param ack [Fluent::Plugin::ForwardOutput::AckHander::Ack|nil]
+      # @param ack [Fluent::Plugin::ForwardOutput::AckHandler::Ack|nil]
       def connect(host:, port:, hostname:, ack: nil, &block)
         if @socket_cache
           return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)

data/lib/fluent/plugin/out_forward.rb

@@ -87,7 +87,7 @@ module Fluent::Plugin
     config_param :verify_connection_at_startup, :bool, default: false
 
     desc 'Compress buffered data.'
-    config_param :compress, :enum, list: [:text, :gzip], default: :text
+    config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
 
     desc 'The default version of TLS transport.'
     config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
@@ -251,10 +251,14 @@ module Fluent::Plugin
       end
 
       unless @as_secondary
-        if @compress == :gzip && @buffer.compress == :text
-          @buffer.compress = :gzip
-        elsif @compress == :text && @buffer.compress == :gzip
-          log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+        if @buffer.compress == :text
+          @buffer.compress = @compress
+        else
+          if @compress == :text
+            log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+          elsif @compress != @buffer.compress
+            raise Fluent::ConfigError, "You cannot specify different compression formats for Buffer (Buffer: #{@buffer.compress}, Self: #{@compress})"
+          end
         end
       end
 
@@ -267,9 +271,15 @@ module Fluent::Plugin
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
+
+      if @compress == :zstd
+        log.warn "zstd compression feature is an experimental new feature supported since v1.19.0." +
+                 " Please make sure that the destination server also supports this feature before using it." +
+                 " in_forward plugin for Fluentd supports it since v1.19.0."
+      end
+
       @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
       @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
-
     end
 
     def multi_workers_ready?
@@ -295,7 +305,7 @@ module Fluent::Plugin
       unless @heartbeat_type == :none
         if @heartbeat_type == :udp
           @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)
-          server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
+          server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heartbeat_response_recv))
         end
         timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
       end
@@ -481,7 +491,7 @@ module Fluent::Plugin
       end
     end
 
-    def on_udp_heatbeat_response_recv(data, sock)
+    def on_udp_heartbeat_response_recv(data, sock)
       sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
       if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
         # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
@@ -567,8 +577,8 @@ module Fluent::Plugin
 
         @handshake = HandshakeProtocol.new(
           log: @log,
-          hostname: sender.security && sender.security.self_hostname,
-          shared_key: server.shared_key || (sender.security && sender.security.shared_key) || '',
+          hostname: sender.security&.self_hostname,
+          shared_key: server.shared_key || sender.security&.shared_key || '',
           password: server.password || '',
           username: server.username || '',
         )
@@ -584,7 +594,7 @@ module Fluent::Plugin
       attr_accessor :usock
 
       attr_reader :state
-      attr_reader :sockaddr  # used by on_udp_heatbeat_response_recv
+      attr_reader :sockaddr  # used by on_udp_heartbeat_response_recv
       attr_reader :failure # for test
 
       def validate_host_resolution!
@@ -711,7 +721,7 @@ module Fluent::Plugin
           end
         when :udp
           @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, dest_addr)
-          # response is going to receive at on_udp_heatbeat_response_recv
+          # response is going to receive at on_udp_heartbeat_response_recv
           false
         when :none # :none doesn't use this class
           raise "BUG: heartbeat_type none must not use Node"
@@ -744,7 +754,7 @@ module Fluent::Plugin
       def resolve_dns!
         addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)
         addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
-        @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heatbeat_response_recv
+        @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_udp_heartbeat_response_recv
         addrinfo[3]
       end
       private :resolve_dns!

data/lib/fluent/plugin/out_http.rb

@@ -270,7 +270,7 @@ module Fluent::Plugin
                               OpenSSL::SSL::VERIFY_PEER
                             end
         opt[:ciphers] = @tls_ciphers
-        opt[:ssl_version] = @tls_version
+        opt = Fluent::TLS.set_version_to_options(opt, @tls_version, nil, nil)
       end
 
       opt

data/lib/fluent/plugin/out_secondary_file.rb

@@ -98,11 +98,11 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
       end
 
-      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
+      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match?(/tag(\[\d+\])?/) })
         raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
       end
 
-      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
+      vars = placeholders.reject { |placeholder| placeholder.match?(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
 
       if ph = vars.find { |v| !@chunk_keys.include?(v) }
         raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"

data/lib/fluent/plugin/out_stdout.rb

@@ -25,6 +25,9 @@ module Fluent::Plugin
     DEFAULT_LINE_FORMAT_TYPE = 'stdout'
     DEFAULT_FORMAT_TYPE = 'json'
 
+    desc "If Fluentd logger outputs logs to a file (with -o option), this plugin outputs events to the file as well."
+    config_param :use_logger, :bool, default: true
+
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']
       config_set_default :flush_at_shutdown, true
@@ -44,6 +47,10 @@ module Fluent::Plugin
       true
     end
 
+    def dest_io
+      @use_logger ? $log : $stdout
+    end
+
     attr_accessor :formatter
 
     def configure(conf)
@@ -57,9 +64,9 @@ module Fluent::Plugin
     def process(tag, es)
       es = inject_values_to_event_stream(tag, es)
       es.each {|time,record|
-        $log.write(format(tag, time, record))
+        dest_io.write(format(tag, time, record))
       }
-      $log.flush
+      dest_io.flush
     end
 
     def format(tag, time, record)
@@ -68,7 +75,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      chunk.write_to($log)
+      chunk.write_to(dest_io)
     end
   end
 end

data/lib/fluent/plugin/out_stream.rb

@@ -92,8 +92,8 @@ module Fluent
 
     def initialize
       super
-      $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
-      $log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
+      log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
+      log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
     end
 
     config_param :port, :integer, default: LISTEN_PORT
@@ -114,7 +114,7 @@ module Fluent
 
     def initialize
       super
-      $log.warn "'unix' output is obsoleted and will be removed."
+      log.warn "'unix' output is obsoleted and will be removed."
     end
 
     config_param :path, :string

data/lib/fluent/plugin/output.rb

@@ -171,30 +171,6 @@ module Fluent
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
-      def num_errors
-        @num_errors_metrics.get
-      end
-
-      def emit_count
-        @emit_count_metrics.get
-      end
-
-      def emit_size
-        @emit_size_metrics.get
-      end
-
-      def emit_records
-        @emit_records_metrics.get
-      end
-
-      def write_count
-        @write_count_metrics.get
-      end
-
-      def rollback_count
-        @rollback_count_metrics.get
-      end
-
       def initialize
         super
         @counter_mutex = Mutex.new
@@ -210,9 +186,11 @@ module Fluent
         @emit_records_metrics = nil
         @emit_size_metrics = nil
         @write_count_metrics = nil
+        @write_secondary_count_metrics = nil
         @rollback_count_metrics = nil
         @flush_time_count_metrics = nil
         @slow_flush_count_metrics = nil
+        @drop_oldest_chunk_count_metrics = nil
         @enable_size_metrics = false
 
         # How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
@@ -278,9 +256,11 @@ module Fluent
         @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
         @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
         @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
+        @write_secondary_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_secondary_count", help_text: "Number of writing events in secondary")
         @rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
         @flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
         @slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
+        @drop_oldest_chunk_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "drop_oldest_chunk_count", help_text: "Number of count that old chunk were discarded with drop_oldest_chunk")
 
         if has_buffer_section
           unless implement?(:buffered) || implement?(:delayed_commit)
@@ -572,7 +552,7 @@ module Fluent
             @output_flush_threads.each do |state|
               # to wakeup thread and make it to stop by itself
               state.mutex.synchronize {
-                if state.thread && state.thread.status
+                if state.thread&.status
                   state.next_clock = 0
                   state.cond_var.signal
                 end
@@ -999,6 +979,7 @@ module Fluent
               if oldest
                 log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
                 @buffer.purge_chunk(oldest.unique_id)
+                @drop_oldest_chunk_count_metrics.inc
               else
                 log.error "no queued chunks to be dropped for drop_oldest_chunk"
               end
@@ -1014,13 +995,17 @@ module Fluent
       end
 
       FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
-      FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }
       FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
-      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer, type: :zstd) }
 
       def generate_format_proc
         if @buffer && @buffer.compress == :gzip
-          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
+          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_GZIP : FORMAT_COMPRESSED_MSGPACK_STREAM_GZIP
+        elsif @buffer && @buffer.compress == :zstd
+          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT_ZSTD : FORMAT_COMPRESSED_MSGPACK_STREAM_ZSTD
         else
           @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
         end
@@ -1036,17 +1021,17 @@ module Fluent
       #   iteration of event stream, and it should be done just once even if total event stream size
       #   is bigger than chunk_limit_size because of performance.
       def handle_stream_with_custom_format(tag, es, enqueue: false)
-        meta_and_data = {}
+        meta_and_data = Hash.new { |h, k| h[k] = [] }
         records = 0
         es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
           meta = metadata(tag, time, record)
-          meta_and_data[meta] ||= []
           res = format(tag, time, record)
           if res
             meta_and_data[meta] << res
             records += 1
           end
         end
+        meta_and_data.default_proc = nil
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
@@ -1057,14 +1042,14 @@ module Fluent
 
       def handle_stream_with_standard_format(tag, es, enqueue: false)
         format_proc = generate_format_proc
-        meta_and_data = {}
+        meta_and_data = Hash.new { |h, k| h[k] = MultiEventStream.new }
         records = 0
         es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
           meta = metadata(tag, time, record)
-          meta_and_data[meta] ||= MultiEventStream.new
           meta_and_data[meta].add(time, record)
           records += 1
         end
+        meta_and_data.default_proc = nil
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
@@ -1146,7 +1131,7 @@ module Fluent
 
       def try_rollback_write
         @dequeued_chunks_mutex.synchronize do
-          while @dequeued_chunks.first && @dequeued_chunks.first.expired?
+          while @dequeued_chunks.first&.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
               @rollback_count_metrics.inc
@@ -1208,6 +1193,7 @@ module Fluent
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
             @write_count_metrics.inc
+            @write_secondary_count_metrics.inc if using_secondary
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1220,6 +1206,7 @@ module Fluent
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
             @write_count_metrics.inc
+            @write_secondary_count_metrics.inc if using_secondary
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1384,11 +1371,12 @@ module Fluent
       end
 
       def submit_flush_once
+        return unless @buffer_config.flush_thread_count > 0
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
         state.mutex.synchronize {
-          if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
+          if state.thread&.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
             state.next_clock = 0
             state.cond_var.signal
           else
@@ -1406,6 +1394,7 @@ module Fluent
       end
 
       def submit_flush_all
+        return unless @buffer_config.flush_thread_count > 0
         while !@retry && @buffer.queued?
           submit_flush_once
           sleep @buffer_config.flush_thread_burst_interval
@@ -1433,7 +1422,7 @@ module Fluent
       def flush_thread_wakeup
         @output_flush_threads.each do |state|
           state.mutex.synchronize {
-            if state.thread && state.thread.status
+            if state.thread&.status
               state.next_clock = 0
               state.cond_var.signal
             end
@@ -1585,9 +1574,11 @@ module Fluent
           'retry_count' => @num_errors_metrics.get,
           'emit_count' => @emit_count_metrics.get,
           'write_count' => @write_count_metrics.get,
+          'write_secondary_count' => @write_secondary_count_metrics.get,
           'rollback_count' => @rollback_count_metrics.get,
           'slow_flush_count' => @slow_flush_count_metrics.get,
           'flush_time_count' => @flush_time_count_metrics.get,
+          'drop_oldest_chunk_count' => @drop_oldest_chunk_count_metrics.get,
         }
 
         if @buffer && @buffer.respond_to?(:statistics)

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -26,13 +26,13 @@ module Fluent
       end
 
       def owner
-        if instance_variable_defined?("@_owner")
+        if instance_variable_defined?(:@_owner)
           @_owner
         end
       end
 
       def log
-        if instance_variable_defined?("@log")
+        if instance_variable_defined?(:@log)
           @log
         end
       end