fluentd 1.14.0-x86-mingw32 → 1.14.4-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 585487ebe4fdf640b517d8673d446dcc683acae55d6a8b58a3255e3e523e6824
-  data.tar.gz: e4ce59e08623ad835cf979e870d46f98fcc89a43eac7d180ace8e88e4bda7e64
+  metadata.gz: '0735519c797e326cb6fcd64a4e27a0e0872af520ae52d24802460c51869417dc'
+  data.tar.gz: a6524d76380af765663e85378c6e1bda25a83f01bfd5ee034152f0f672d44071
 SHA512:
-  metadata.gz: 14a8b359210ce14d816009b9afecbd7f749b68c96655fad2d6ea7700991bba771ca3834ecc4916c8edb26d69ad5f01006b4674a1b21fb4640c390457bd9835dc
-  data.tar.gz: a7dd58d022d2388fd4c2de96ce454405eb7bdba9cb2aba87379849567958e2da47591def7d5baac18f09977a9fde4ec356aea9ca409f7c204e73fc0965f772d6
+  metadata.gz: af913a0847c7bd11dc534cf935ae1ae73f7b447836a8bca8abd8cd4a151dfc7cdf4e72fefe577c265dbccc1829f2c0dc6fa31d23a046b869b1f69ac972570435
+  data.tar.gz: b0360be02ae35ba1b1a5212fbb5815a7b2fb1bf620b6cf5459f5acf4768dd19c1f7825b17ae9bc00983eaea13bcd7c3036ae7a6f802e4684dc7963f7387d9ffe

data/.drone.yml

@@ -1,13 +1,13 @@
 kind: pipeline
-name: fluentd-test-arm64-2-6-3
+name: fluentd-test-arm64-3-0
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-2-6-3
-  image: arm64v8/ruby:2.6.3
+- name: fluentd-test-arm64-3-0
+  image: arm64v8/ruby:3.0
   commands:
   - apt update
   - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
   - bundle exec rake test
 ---
 kind: pipeline
-name: fluentd-test-arm64-latest
+name: fluentd-test-arm64-2-7
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-latest
-  image: arm64v8/ruby:latest
+- name: fluentd-test-arm64-2-7
+  image: arm64v8/ruby:2.7
   commands:
   - apt update
   - apt -y install libgmp3-dev

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -50,6 +50,7 @@ body:
       label: Your Configuration
       description: |
         Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
     validations:
       required: true
   - type: textarea

data/CHANGELOG.md

@@ -1,3 +1,104 @@
+# v1.14.4
+
+## Release v1.14.4 - 2022/01/06
+
+### Enhancement
+
+* `in_tail`: Add option to skip long lines (`max_line_size`)
+  https://github.com/fluent/fluentd/pull/3565
+
+### Bug fix
+
+* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
+  https://github.com/fluent/fluentd/pull/3560
+* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
+  https://github.com/fluent/fluentd/pull/3579
+* test: Fix unstable test cases
+  https://github.com/fluent/fluentd/pull/3574
+  https://github.com/fluent/fluentd/pull/3577
+
+# v1.14.3
+
+## Release v1.14.3 - 2021/11/26
+
+### Enhancement
+
+* Changed to accept `http_parser.rb` 0.8.0.
+  `http_parser.rb` 0.8.0 is ready for Ractor.
+  https://github.com/fluent/fluentd/pull/3544
+
+### Bug fix
+
+* in_tail: Fixed a bug that no new logs are read when
+  `enable_stat_watcher true` and `enable_watch_timer false` is set.
+  https://github.com/fluent/fluentd/pull/3541
+* in_tail: Fixed a bug that the beginning and initial lines are lost
+  after startup when `read_from_head false` and path includes wildcard '*'.
+  https://github.com/fluent/fluentd/pull/3542
+* Fixed a bug that processing messages were lost when
+  BufferChunkOverflowError was thrown even though only a specific
+  message size exceeds chunk_limit_size.
+  https://github.com/fluent/fluentd/pull/3553
+  https://github.com/fluent/fluentd/pull/3562
+
+### Misc
+
+* Bump up required version of `win32-service` gem.
+  newer version is required to implement additional `fluent-ctl` commands.
+  https://github.com/fluent/fluentd/pull/3556
+
+# v1.14.2
+
+## Release v1.14.2 - 2021/10/29
+
+IMPORTANT: This release contain the fix for CVE-2021-41186 -
+ReDoS vulnerability in `parser_apache2`.
+This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
+We recommend to upgrade Fluentd to v1.14.2 or use patched version of
+`parser_apache2` plugin.
+
+### Enhancement
+
+* fluent-cat: Add `--event-time` option to send specified event time for testing.
+
+### Bug fix
+
+* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
+  https://github.com/fluent/fluentd/pull/3524
+* Fixed ReDoS vulnerability in parser_apache2.
+  This vulnerability is caused by a certain pattern of a broken apache log.
+
+# v1.14.1
+
+## Release v1.14.1 - 2021/09/29
+
+### Enhancement
+
+* in_tail: Added file related metrics.
+  These metrics should be collected same as fluent-bit's in_tail.
+  https://github.com/fluent/fluentd/pull/3504
+* out_forward: Changed to use metrics mechanism for node statistics
+  https://github.com/fluent/fluentd/pull/3506
+
+### Bug fix
+
+* in_tail: Fixed a crash bug that it raise undefined method of eof? error.
+  This error may happen only when `read_bytes_limit_per_second` was specified.
+  https://github.com/fluent/fluentd/pull/3500
+* out_forward: Fixed a bug that node statistics information is not included correctly.
+  https://github.com/fluent/fluentd/pull/3503
+  https://github.com/fluent/fluentd/pull/3507
+* Fixed a error when using `@include` directive
+  It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
+  https://github.com/fluent/fluentd/pull/3517
+* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
+  It didn't work even if a user set it.
+  https://github.com/fluent/fluentd/pull/3515
+* Fixed not to output nanoseconds field of next retry time in warning log
+  Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
+  next_retry_seconds -> next_retry_time)
+  https://github.com/fluent/fluentd/pull/3518
+
 # v1.14.0
 
 ## Release v1.14.0 - 2021/08/30

data/README.md

@@ -88,6 +88,8 @@ You can run specified test via `TEST` environment variable:
 
 A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
 
+See [SECURITY](SECURITY.md) to contact us about vulnerability.
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+## Supported Versions
+
+| Version   | Supported          |
+| -------   | ------------------ |
+| 1.14.x    | :white_check_mark: |
+| <= 1.13.x | :x:                |
+
+## Reporting a Vulnerability
+
+Please contact to current active maintainers. (in alphabetical order)
+
+* ashie@clear-code.com
+* fujimoto@clear-code.com
+* hatake@calyptia.com
+* hayashi@clear-code.com
+

data/fluentd.gemspec

@@ -23,11 +23,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
     gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
-    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])

data/lib/fluent/command/cat.rb

@@ -35,6 +35,7 @@ format = 'json'
 message_key = 'message'
 time_as_integer = false
 retry_limit = 5
+event_time = nil
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
   retry_limit = n
 }
 
+op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
+  event_time = v
+}
+
 singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -134,7 +139,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
     @tag = tag
     @connector = connector
     @socket = false
@@ -148,6 +153,7 @@ class Writer
     @retry_wait = 1
     @retry_limit = retry_limit
     @time_as_integer = time_as_integer
+    @event_time = event_time
 
     super()
   end
@@ -166,7 +172,11 @@ class Writer
       end
     end
 
-    time = Fluent::EventTime.now
+    time = if @event_time
+             Fluent::EventTime.parse(@event_time)
+           else
+             Fluent::EventTime.now
+           end
     time = time.to_i if @time_as_integer
     entry = if secondary_record?(record)
               # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
 w.start
 
 case format

data/lib/fluent/config/parser.rb

@@ -93,7 +93,7 @@ module Fluent
           basepath = '/'
           fname = path
           require 'open-uri'
-          open(uri) {|f|
+          URI.open(uri) {|f|
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
         end

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = open(uri) { |f| f.read }
+          data = URI.open(uri) { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/env.rb

@@ -33,4 +33,8 @@ module Fluent
   def self.linux?
     /linux/ === RUBY_PLATFORM
   end
+
+  def self.macos?
+    /darwin/ =~ RUBY_PLATFORM
+  end
 end

data/lib/fluent/plugin/buf_file.rb

@@ -39,8 +39,8 @@ module Fluent
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-      config_param :file_permission, :string, default: nil # '0644'
-      config_param :dir_permission,  :string, default: nil # '0755'
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
 
       def initialize
         super

data/lib/fluent/plugin/buffer.rb

@@ -332,12 +332,14 @@ module Fluent
         unstaged_chunks = {} # metadata => [chunk, chunk, ...]
         chunks_to_enqueue = []
         staged_bytesizes_by_chunk = {}
+        # track internal BufferChunkOverflowError in write_step_by_step
+        buffer_chunk_overflow_errors = []
 
         begin
           # sort metadata to get lock of chunks in same order with other threads
           metadata_and_data.keys.sort.each do |metadata|
             data = metadata_and_data[metadata]
-            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
+            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize, error|
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
               if chunk.staged?
@@ -352,6 +354,9 @@ module Fluent
                 unstaged_chunks[metadata] ||= []
                 unstaged_chunks[metadata] << chunk
               end
+              if error && !error.empty?
+                buffer_chunk_overflow_errors << error
+              end
             end
           end
 
@@ -444,6 +449,10 @@ module Fluent
             end
             chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
           end
+          unless buffer_chunk_overflow_errors.empty?
+            # Notify delayed BufferChunkOverflowError here
+            raise BufferChunkOverflowError, buffer_chunk_overflow_errors.join(", ")
+          end
         end
       end
 
@@ -716,6 +725,7 @@ module Fluent
 
       def write_step_by_step(metadata, data, format, splits_count, &block)
         splits = []
+        errors = []
         if splits_count > data.size
           splits_count = data.size
         end
@@ -757,22 +767,65 @@ module Fluent
             raise ShouldRetry unless chunk.writable?
             staged_chunk_used = true if chunk.staged?
 
-            original_bytesize = chunk.bytesize
+            original_bytesize = committed_bytesize = chunk.bytesize
             begin
               while writing_splits_index < splits.size
                 split = splits[writing_splits_index]
+                formatted_split = format ? format.call(split) : nil
+
+                if split.size == 1 # Check BufferChunkOverflowError
+                  determined_bytesize = nil
+                  if @compress != :text
+                    determined_bytesize = nil
+                  elsif formatted_split
+                    determined_bytesize = formatted_split.bytesize
+                  elsif split.first.respond_to?(:bytesize)
+                    determined_bytesize = split.first.bytesize
+                  end
+
+                  if determined_bytesize && determined_bytesize > @chunk_limit_size
+                    # It is a obvious case that BufferChunkOverflowError should be raised here.
+                    # But if it raises here, already processed 'split' or
+                    # the proceeding 'split' will be lost completely.
+                    # So it is a last resort to delay raising such a exception
+                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
+                  end
+
+                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                    # The split will (might) cause size over so keep already processed
+                    # 'split' content here (allow performance regression a bit).
+                    chunk.commit
+                    committed_bytesize = chunk.bytesize
+                  end
+                end
+
                 if format
-                  chunk.concat(format.call(split), split.size)
+                  chunk.concat(formatted_split, split.size)
                 else
                   chunk.append(split, compress: @compress)
                 end
+                adding_bytes = chunk.bytesize - committed_bytesize
 
                 if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
                   chunk.rollback
-
-                  if split.size == 1 && original_bytesize == 0
-                    big_record_size = format ? format.call(split).bytesize : split.first.bytesize
-                    raise BufferChunkOverflowError, "a #{big_record_size}bytes record is larger than buffer chunk limit size"
+                  committed_bytesize = chunk.bytesize
+
+                  if split.size == 1 # Check BufferChunkOverflowError again
+                    if adding_bytes > @chunk_limit_size
+                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                      writing_splits_index += 1
+                      next
+                    else
+                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                      # After that, re-process current split again.
+                      # New chunk should be allocated, to do it, modify @stage and so on.
+                      synchronize { @stage.delete(modified_metadata) }
+                      staged_chunk_used = false
+                      chunk.unstaged!
+                      break
+                    end
                   end
 
                   if chunk_size_full?(chunk) || split.size == 1
@@ -795,7 +848,8 @@ module Fluent
               raise
             end
 
-            block.call(chunk, chunk.bytesize - original_bytesize)
+            block.call(chunk, chunk.bytesize - original_bytesize, errors)
+            errors = []
           end
         end
       rescue ShouldRetry

data/lib/fluent/plugin/in_tail.rb

@@ -38,6 +38,7 @@ module Fluent::Plugin
     helpers :timer, :event_loop, :parser, :compat_parameters
 
     RESERVED_CHARS = ['/', '*', '%'].freeze
+    MetricsInfo = Struct.new(:opened, :closed, :rotated)
 
     class WatcherSetupError < StandardError
       def initialize(msg)
@@ -57,6 +58,8 @@ module Fluent::Plugin
       @pf = nil
       @ignore_list = []
       @shutdown_start_time = nil
+      @metrics = nil
+      @startup = true
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -110,6 +113,8 @@ module Fluent::Plugin
     config_param :path_timezone, :string, default: nil
     desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
     config_param :follow_inodes, :bool, default: false
+    desc 'Maximum length of line. The longer line is just skipped.'
+    config_param :max_line_size, :size, default: nil
 
     config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
       config_argument :usage, :string, default: 'in_tail_parser'
@@ -191,6 +196,10 @@ module Fluent::Plugin
           @read_bytes_limit_per_second = min_bytes
         end
       end
+      opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
+      closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
+      rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
+      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     def configure_tag
@@ -363,19 +372,25 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-
-      log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" }
+      
+      log.debug {
+        target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
+        existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
+        "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
+      }
 
       unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
       stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
+      @startup = false if @startup
     end
 
     def setup_watcher(target_info, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
+      read_from_head = !@startup || @read_from_head
+      tw = TailWatcher.new(target_info, pe, log, read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -383,7 +398,7 @@ module Fluent::Plugin
       end
 
       if @enable_stat_watcher
-        tt = StatWatcher.new(path, log) { tw.on_notify }
+        tt = StatWatcher.new(target_info.path, log) { tw.on_notify }
         tw.register_watcher(tt)
       end
 
@@ -581,6 +596,14 @@ module Fluent::Plugin
 
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
+      lines = lines.reject do |line|
+        skip_line = @max_line_size ? line.bytesize > @max_line_size : false
+        if skip_line
+          log.warn "received line length is longer than #{@max_line_size}"
+          log.debug "skipped line: #{line.chomp}"
+        end
+        skip_line
+      end
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
         tag = if @tag_prefix || @tag_suffix
@@ -670,6 +693,19 @@ module Fluent::Plugin
       es
     end
 
+    def statistics
+      stats = super
+
+      stats = {
+        'input' => stats["input"].merge({
+          'opened_file_count' => @metrics.opened.get,
+          'closed_file_count' => @metrics.closed.get,
+          'rotated_file_count' => @metrics.rotated.get,
+        })
+      }
+      stats
+    end
+
     private
 
     def io_handler(watcher, path)
@@ -682,6 +718,7 @@ module Fluent::Plugin
         open_on_every_update: @open_on_every_update,
         from_encoding: @from_encoding,
         encoding: @encoding,
+        metrics: @metrics,
         &method(:receive_lines)
       )
     end
@@ -717,7 +754,7 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
+      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
         @path = target_info.path
         @ino = target_info.ino
         @pe = pe || MemoryPositionEntry.new
@@ -729,6 +766,7 @@ module Fluent::Plugin
         @line_buffer_timer_flusher = line_buffer_timer_flusher
         @io_handler = nil
         @io_handler_build = io_handler_build
+        @metrics = metrics
         @watchers = []
       end
 
@@ -762,7 +800,7 @@ module Fluent::Plugin
       end
 
       def eof?
-        @io_handler.eof?
+        @io_handler.nil? || @io_handler.eof?
       end
 
       def on_notify
@@ -855,6 +893,7 @@ module Fluent::Plugin
             @log.info "detected rotation of #{@path}"
             @io_handler = io_handler
           end
+          @metrics.rotated.inc
         end
       end
 
@@ -934,7 +973,7 @@ module Fluent::Plugin
 
         attr_accessor :shutdown_timeout
 
-        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
@@ -953,6 +992,7 @@ module Fluent::Plugin
           @shutdown_timeout = SHUTDOWN_TIMEOUT
           @shutdown_mutex = Mutex.new
           @eof = false
+          @metrics = metrics
 
           @log.info "following tail of #{@path}"
         end
@@ -972,6 +1012,7 @@ module Fluent::Plugin
           if @io && !@io.closed?
             @io.close
             @io = nil
+            @metrics.closed.inc
           end
         end
 
@@ -1059,6 +1100,7 @@ module Fluent::Plugin
         def open
           io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
+          @metrics.opened.inc
           io
         rescue RangeError
           io.close if io
@@ -1111,6 +1153,10 @@ module Fluent::Plugin
         def opened?
           false
         end
+
+        def eof?
+          true
+        end
       end
 
       class RotateHandler

data/lib/fluent/plugin/out_copy.rb

@@ -46,7 +46,7 @@ module Fluent::Plugin
         @ignore_errors << (store.arg.include?('ignore_error'))
         @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
-      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
         log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
       end
     end

data/lib/fluent/plugin/out_file.rb

@@ -181,6 +181,13 @@ module Fluent::Plugin
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
+
+      # https://github.com/fluent/fluentd/issues/3569
+      @need_ruby_on_macos_workaround = false
+      if @append && Fluent.macos?
+        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
+        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
+      end
     end
 
     def multi_workers_ready?
@@ -223,7 +230,12 @@ module Fluent::Plugin
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f)
+        if @need_ruby_on_macos_workaround
+          content = chunk.read()
+          f.puts content
+        else
+          chunk.write_to(f)
+        end
       end
     end