fluentd 1.14.1-x64-mingw32 → 1.14.5-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b95a24ab25f46ab88d9e5bfc09a0f4374dbbb606523fade2d3dd0fb175227e2
-  data.tar.gz: 6569c24837680722e497009210e7bf64b1b3a50c9fe4b4ef4d6a89db8cbb0faf
+  metadata.gz: 9cbb8685c529600692b7829b8d06421da681b10408350bc94c11dc17d8392f60
+  data.tar.gz: d3c4c191b81c61c4506dc72d689d0a990bd3f885f37265dc27bd91db52ff8e68
 SHA512:
-  metadata.gz: b06c10a7461f156e7da183c18e2ebe47d8e35d2dda2fd61ca607ddac6daa084c9b3e367952927211598a856b08e12c6aaccfe0f7be34c7a23acaac87bec0b858
-  data.tar.gz: 31dac402718a9e78416f2481704eec0ac35c154006c14f73c60c62149dc83bf53d55d3aad1b94a57f9dd45ae158b4eac979542de4e2d90ff41339881c16175e0
+  metadata.gz: 1afc007c616fbf9a2c11e2995300af667be5a143722e5575a00348d7e7a499442d86cd1f93997d5d63c206283a70a48e14f7c60f9b52de9d2df8d1def5a40f91
+  data.tar.gz: 80b0fbe5ffa8ddb7ace7808318d273a549e3ac56d864f2e650cf8c424cef082c7672d15fbdd7d5cdcd77360d94c92a8d24658ede33e55a150bdb9bf6c432f47d

data/.drone.yml

@@ -1,13 +1,13 @@
 kind: pipeline
-name: fluentd-test-arm64-2-6-3
+name: fluentd-test-arm64-3-0
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-2-6-3
-  image: arm64v8/ruby:2.6.3
+- name: fluentd-test-arm64-3-0
+  image: arm64v8/ruby:3.0
   commands:
   - apt update
   - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
   - bundle exec rake test
 ---
 kind: pipeline
-name: fluentd-test-arm64-latest
+name: fluentd-test-arm64-2-7
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-latest
-  image: arm64v8/ruby:latest
+- name: fluentd-test-arm64-2-7
+  image: arm64v8/ruby:2.7
   commands:
   - apt update
   - apt -y install libgmp3-dev

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -50,6 +50,7 @@ body:
       label: Your Configuration
       description: |
         Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
     validations:
       required: true
   - type: textarea

data/CHANGELOG.md

@@ -1,3 +1,97 @@
+# v1.14.5
+
+## Release v1.14.5 - 2022/02/09
+
+### Enhancement
+
+* Add support for "application/x-ndjson" to `in_http`
+  https://github.com/fluent/fluentd/pull/3616
+* Add support for ucrt binary for Windows
+  https://github.com/fluent/fluentd/pull/3613
+
+### Bug fixes
+
+* Don't retry when `retry_max_times == 0`
+  https://github.com/fluent/fluentd/pull/3608
+* Fix hang-up issue during TLS handshake in `out_forward`
+  https://github.com/fluent/fluentd/pull/3601
+* Bump up required ServerEngine to v2.2.5
+  https://github.com/fluent/fluentd/pull/3599
+* Fix "invalid byte sequence is replaced" warning on Kubernetes
+  https://github.com/fluent/fluentd/pull/3596
+* Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
+  https://github.com/fluent/fluentd/pull/3592
+
+# v1.14.4
+
+## Release v1.14.4 - 2022/01/06
+
+### Enhancement
+
+* `in_tail`: Add option to skip long lines (`max_line_size`)
+  https://github.com/fluent/fluentd/pull/3565
+
+### Bug fix
+
+* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
+  https://github.com/fluent/fluentd/pull/3560
+* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
+  https://github.com/fluent/fluentd/pull/3579
+* test: Fix unstable test cases
+  https://github.com/fluent/fluentd/pull/3574
+  https://github.com/fluent/fluentd/pull/3577
+
+# v1.14.3
+
+## Release v1.14.3 - 2021/11/26
+
+### Enhancement
+
+* Changed to accept `http_parser.rb` 0.8.0.
+  `http_parser.rb` 0.8.0 is ready for Ractor.
+  https://github.com/fluent/fluentd/pull/3544
+
+### Bug fix
+
+* in_tail: Fixed a bug that no new logs are read when
+  `enable_stat_watcher true` and `enable_watch_timer false` is set.
+  https://github.com/fluent/fluentd/pull/3541
+* in_tail: Fixed a bug that the beginning and initial lines are lost
+  after startup when `read_from_head false` and path includes wildcard '*'.
+  https://github.com/fluent/fluentd/pull/3542
+* Fixed a bug that processing messages were lost when
+  BufferChunkOverflowError was thrown even though only a specific
+  message size exceeds chunk_limit_size.
+  https://github.com/fluent/fluentd/pull/3553
+  https://github.com/fluent/fluentd/pull/3562
+
+### Misc
+
+* Bump up required version of `win32-service` gem.
+  newer version is required to implement additional `fluent-ctl` commands.
+  https://github.com/fluent/fluentd/pull/3556
+
+# v1.14.2
+
+## Release v1.14.2 - 2021/10/29
+
+IMPORTANT: This release contain the fix for CVE-2021-41186 -
+ReDoS vulnerability in `parser_apache2`.
+This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
+We recommend to upgrade Fluentd to v1.14.2 or use patched version of
+`parser_apache2` plugin.
+
+### Enhancement
+
+* fluent-cat: Add `--event-time` option to send specified event time for testing.
+
+### Bug fix
+
+* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
+  https://github.com/fluent/fluentd/pull/3524
+* Fixed ReDoS vulnerability in parser_apache2.
+  This vulnerability is caused by a certain pattern of a broken apache log.
+
 # v1.14.1
 
 ## Release v1.14.1 - 2021/09/29

data/README.md

@@ -1,17 +1,12 @@
 Fluentd: Open-Source Log Collector
 ===================================
-GitHub Actions:
 
 [![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
 [![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
 [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
+[![Testing on Arm64](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
 [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
-
-Drone CI for Arm64:
-
-[![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
 
 [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
 
@@ -19,16 +14,6 @@ Drone CI for Arm64:
 <img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
 </p>
 
-An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
-
-## Example Use Cases
-
-Use Case | Description | Diagram
--------- | ------------|:---------:
-Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
-Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
-Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
-
 ## Quick Start
 
     $ gem install fluentd
@@ -66,12 +51,6 @@ You can run specified test via `TEST` environment variable:
     $ bundle exec rake test TEST=test/test_specified_path.rb
     $ bundle exec rake test TEST=test/test_*.rb
 
-## Fluentd UI: Admin GUI
-
-[Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
-
-<p align="center"><img width="500" src="https://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
-
 ## More Information
 
 - Website: https://www.fluentd.org/
@@ -88,6 +67,8 @@ You can run specified test via `TEST` environment variable:
 
 A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
 
+See [SECURITY](SECURITY.md) to contact us about vulnerability.
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/Rakefile

@@ -14,7 +14,7 @@ namespace :build do
   desc 'Build gems for all platforms'
   task :all do
     Bundler.with_clean_env do
-      %w[ruby x86-mingw32 x64-mingw32].each do |name|
+      %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute
       end

data/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+## Supported Versions
+
+| Version   | Supported          |
+| -------   | ------------------ |
+| 1.14.x    | :white_check_mark: |
+| <= 1.13.x | :x:                |
+
+## Reporting a Vulnerability
+
+Please contact to current active maintainers. (in alphabetical order)
+
+* ashie@clear-code.com
+* fujimoto@clear-code.com
+* hatake@calyptia.com
+* hayashi@clear-code.com
+

data/fluentd.gemspec

@@ -22,12 +22,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
     gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
-    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -51,5 +51,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
+  gem.add_development_dependency("async", "~> 1.23")
   gem.add_development_dependency("async-http", ">= 0.50.0")
 end

data/lib/fluent/command/cat.rb

@@ -35,6 +35,7 @@ format = 'json'
 message_key = 'message'
 time_as_integer = false
 retry_limit = 5
+event_time = nil
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
   retry_limit = n
 }
 
+op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
+  event_time = v
+}
+
 singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -134,7 +139,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
     @tag = tag
     @connector = connector
     @socket = false
@@ -148,6 +153,7 @@ class Writer
     @retry_wait = 1
     @retry_limit = retry_limit
     @time_as_integer = time_as_integer
+    @event_time = event_time
 
     super()
   end
@@ -166,7 +172,11 @@ class Writer
       end
     end
 
-    time = Fluent::EventTime.now
+    time = if @event_time
+             Fluent::EventTime.parse(@event_time)
+           else
+             Fluent::EventTime.now
+           end
     time = time.to_i if @time_as_integer
     entry = if secondary_record?(record)
               # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
 w.start
 
 case format

data/lib/fluent/config/error.rb

@@ -29,4 +29,16 @@ module Fluent
 
   class SetDefault < Exception
   end
+
+  class NotFoundPluginError < ConfigError
+    attr_reader :type, :kind
+
+    def initialize(msg, type: nil, kind: nil)
+      @msg = msg
+      @type = type
+      @kind = kind
+
+      super(msg)
+    end
+  end
 end

data/lib/fluent/env.rb

@@ -33,4 +33,8 @@ module Fluent
   def self.linux?
     /linux/ === RUBY_PLATFORM
   end
+
+  def self.macos?
+    /darwin/ =~ RUBY_PLATFORM
+  end
 end

data/lib/fluent/plugin/base.rb

@@ -72,8 +72,8 @@ module Fluent
 
       def string_safe_encoding(str)
         unless str.valid_encoding?
-          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
           str = str.scrub('?')
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
         end
         yield str
       end

data/lib/fluent/plugin/buf_file.rb

@@ -39,8 +39,8 @@ module Fluent
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-      config_param :file_permission, :string, default: nil # '0644'
-      config_param :dir_permission,  :string, default: nil # '0755'
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
 
       def initialize
         super

data/lib/fluent/plugin/buffer.rb

@@ -332,12 +332,14 @@ module Fluent
         unstaged_chunks = {} # metadata => [chunk, chunk, ...]
         chunks_to_enqueue = []
         staged_bytesizes_by_chunk = {}
+        # track internal BufferChunkOverflowError in write_step_by_step
+        buffer_chunk_overflow_errors = []
 
         begin
           # sort metadata to get lock of chunks in same order with other threads
           metadata_and_data.keys.sort.each do |metadata|
             data = metadata_and_data[metadata]
-            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
+            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize, error|
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
               if chunk.staged?
@@ -352,6 +354,9 @@ module Fluent
                 unstaged_chunks[metadata] ||= []
                 unstaged_chunks[metadata] << chunk
               end
+              if error && !error.empty?
+                buffer_chunk_overflow_errors << error
+              end
             end
           end
 
@@ -444,6 +449,10 @@ module Fluent
             end
             chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
           end
+          unless buffer_chunk_overflow_errors.empty?
+            # Notify delayed BufferChunkOverflowError here
+            raise BufferChunkOverflowError, buffer_chunk_overflow_errors.join(", ")
+          end
         end
       end
 
@@ -716,6 +725,7 @@ module Fluent
 
       def write_step_by_step(metadata, data, format, splits_count, &block)
         splits = []
+        errors = []
         if splits_count > data.size
           splits_count = data.size
         end
@@ -757,22 +767,65 @@ module Fluent
             raise ShouldRetry unless chunk.writable?
             staged_chunk_used = true if chunk.staged?
 
-            original_bytesize = chunk.bytesize
+            original_bytesize = committed_bytesize = chunk.bytesize
             begin
               while writing_splits_index < splits.size
                 split = splits[writing_splits_index]
+                formatted_split = format ? format.call(split) : nil
+
+                if split.size == 1 # Check BufferChunkOverflowError
+                  determined_bytesize = nil
+                  if @compress != :text
+                    determined_bytesize = nil
+                  elsif formatted_split
+                    determined_bytesize = formatted_split.bytesize
+                  elsif split.first.respond_to?(:bytesize)
+                    determined_bytesize = split.first.bytesize
+                  end
+
+                  if determined_bytesize && determined_bytesize > @chunk_limit_size
+                    # It is a obvious case that BufferChunkOverflowError should be raised here.
+                    # But if it raises here, already processed 'split' or
+                    # the proceeding 'split' will be lost completely.
+                    # So it is a last resort to delay raising such a exception
+                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
+                  end
+
+                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                    # The split will (might) cause size over so keep already processed
+                    # 'split' content here (allow performance regression a bit).
+                    chunk.commit
+                    committed_bytesize = chunk.bytesize
+                  end
+                end
+
                 if format
-                  chunk.concat(format.call(split), split.size)
+                  chunk.concat(formatted_split, split.size)
                 else
                   chunk.append(split, compress: @compress)
                 end
+                adding_bytes = chunk.bytesize - committed_bytesize
 
                 if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
                   chunk.rollback
-
-                  if split.size == 1 && original_bytesize == 0
-                    big_record_size = format ? format.call(split).bytesize : split.first.bytesize
-                    raise BufferChunkOverflowError, "a #{big_record_size}bytes record is larger than buffer chunk limit size"
+                  committed_bytesize = chunk.bytesize
+
+                  if split.size == 1 # Check BufferChunkOverflowError again
+                    if adding_bytes > @chunk_limit_size
+                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                      writing_splits_index += 1
+                      next
+                    else
+                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                      # After that, re-process current split again.
+                      # New chunk should be allocated, to do it, modify @stage and so on.
+                      synchronize { @stage.delete(modified_metadata) }
+                      staged_chunk_used = false
+                      chunk.unstaged!
+                      break
+                    end
                   end
 
                   if chunk_size_full?(chunk) || split.size == 1
@@ -795,7 +848,8 @@ module Fluent
               raise
             end
 
-            block.call(chunk, chunk.bytesize - original_bytesize)
+            block.call(chunk, chunk.bytesize - original_bytesize, errors)
+            errors = []
           end
         end
       rescue ShouldRetry

data/lib/fluent/plugin/in_http.rb

@@ -314,8 +314,16 @@ module Fluent::Plugin
         @parser_json.parse(js) do |_time, record|
           return nil, record
         end
+      elsif ndjson = params['ndjson']
+        events = []
+        ndjson.split(/\r?\n/).each do |js|
+          @parser_json.parse(js) do |_time, record|
+            events.push(record)
+          end
+        end
+        return nil, events
       else
-        raise "'json' or 'msgpack' parameter is required"
+        raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
     end
 
@@ -567,6 +575,8 @@ module Fluent::Plugin
           params['json'] = @body
         elsif @content_type =~ /^application\/msgpack/
           params['msgpack'] = @body
+        elsif @content_type =~ /^application\/x-ndjson/
+          params['ndjson'] = @body
         end
         path_info = uri.path
 

data/lib/fluent/plugin/in_tail.rb

@@ -59,6 +59,7 @@ module Fluent::Plugin
       @ignore_list = []
       @shutdown_start_time = nil
       @metrics = nil
+      @startup = true
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -112,6 +113,8 @@ module Fluent::Plugin
     config_param :path_timezone, :string, default: nil
     desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
     config_param :follow_inodes, :bool, default: false
+    desc 'Maximum length of line. The longer line is just skipped.'
+    config_param :max_line_size, :size, default: nil
 
     config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
       config_argument :usage, :string, default: 'in_tail_parser'
@@ -369,19 +372,25 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-
-      log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" }
+      
+      log.debug {
+        target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
+        existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
+        "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
+      }
 
       unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
       stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
+      @startup = false if @startup
     end
 
     def setup_watcher(target_info, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
+      read_from_head = !@startup || @read_from_head
+      tw = TailWatcher.new(target_info, pe, log, read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -389,7 +398,7 @@ module Fluent::Plugin
       end
 
       if @enable_stat_watcher
-        tt = StatWatcher.new(path, log) { tw.on_notify }
+        tt = StatWatcher.new(target_info.path, log) { tw.on_notify }
         tw.register_watcher(tt)
       end
 
@@ -587,6 +596,14 @@ module Fluent::Plugin
 
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
+      lines = lines.reject do |line|
+        skip_line = @max_line_size ? line.bytesize > @max_line_size : false
+        if skip_line
+          log.warn "received line length is longer than #{@max_line_size}"
+          log.debug "skipped line: #{line.chomp}"
+        end
+        skip_line
+      end
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
         tag = if @tag_prefix || @tag_suffix

data/lib/fluent/plugin/out_file.rb

@@ -181,6 +181,13 @@ module Fluent::Plugin
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
+
+      # https://github.com/fluent/fluentd/issues/3569
+      @need_ruby_on_macos_workaround = false
+      if @append && Fluent.macos?
+        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
+        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
+      end
     end
 
     def multi_workers_ready?
@@ -223,7 +230,12 @@ module Fluent::Plugin
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f)
+        if @need_ruby_on_macos_workaround
+          content = chunk.read()
+          f.puts content
+        else
+          chunk.write_to(f)
+        end
       end
     end