fluentd 1.17.1 → 1.19.0

data/lib/fluent/source_only_buffer_agent.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/agent'
+require 'fluent/system_config'
+
+module Fluent
+  class SourceOnlyBufferAgent < Agent
+    # Use INSTANCE_ID to use the same base dir as the other workers.
+    # This will make recovery easier.
+    BUFFER_DIR_NAME = Fluent::INSTANCE_ID
+
+    def initialize(log:, system_config:)
+      super(log: log)
+
+      @default_buffer_path = File.join(system_config.root_dir || DEFAULT_BACKUP_DIR, 'source-only-buffer', BUFFER_DIR_NAME)
+      @optional_buffer_config = system_config.source_only_buffer.to_h.transform_keys(&:to_s)
+      @base_buffer_dir = nil
+      @actual_buffer_dir = nil
+    end
+
+    def configure(flush: false)
+      buffer_config = @optional_buffer_config.compact
+      buffer_config['flush_at_shutdown'] = flush ? 'true' : 'false'
+      buffer_config['flush_thread_count'] = 0 unless flush
+      buffer_config['path'] ||= @default_buffer_path
+
+      super(
+        Config::Element.new('SOURCE_ONLY_BUFFER', '', {}, [
+          Config::Element.new('match', '**', {'@type' => 'buffer', '@label' => '@ROOT'}, [
+            Config::Element.new('buffer', '', buffer_config, [])
+          ])
+        ])
+      )
+
+      @base_buffer_dir = buffer_config['path']
+      # It can be "#{@base_buffer_dir}/worker#{fluentd_worker_id}/" when using multiple workers
+      @actual_buffer_dir = File.dirname(outputs[0].buffer.path)
+
+      unless flush
+        log.info "with-source-only: the emitted data will be stored in the buffer files under" +
+                 " #{@base_buffer_dir}. You can send SIGWINCH to the supervisor process to cancel" +
+                 " with-source-only mode and process data."
+      end
+    end
+
+    def cleanup
+      unless (Dir.empty?(@actual_buffer_dir) rescue true)
+        log.warn "some buffer files remain in #{@base_buffer_dir}." +
+                 " Please consider recovering or saving the buffer files in the directory." +
+                 " To recover them, you can set the buffer path manually to system config and" +
+                 " retry, i.e., restart Fluentd with with-source-only mode and send SIGWINCH again." +
+                 " Config Example:\n#{config_example_to_recover(@base_buffer_dir)}"
+        return
+      end
+
+      begin
+        FileUtils.remove_dir(@base_buffer_dir)
+      rescue Errno::ENOENT
+        # This worker doesn't need to do anything. Another worker may remove the dir first.
+      rescue => e
+        log.warn "failed to remove the buffer directory: #{@base_buffer_dir}", error: e
+      end
+    end
+
+    def emit_error_event(tag, time, record, error)
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
+      log.warn "SourceOnlyBufferAgent: dump an error event:", error_info
+    end
+
+    def handle_emits_error(tag, es, error)
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
+      log.warn "SourceOnlyBufferAgent: emit transaction failed:", error_info
+      log.warn_backtrace
+    end
+
+    private
+
+    def config_example_to_recover(path)
+      <<~EOC
+        <system>
+          <source_only_buffer>
+            path #{path}
+          </source_only_buffer>
+        </system>
+      EOC
+    end
+  end
+end

data/lib/fluent/static_config_analysis.rb

@@ -18,7 +18,7 @@ require 'fluent/config'
 require 'fluent/plugin'
 
 module Fluent
-  # Static Analysis means analysing all plugins and Fluent::Element without invokeing Plugin#configure
+  # Static Analysis means analysing all plugins and Fluent::Element without invoking Plugin#configure
   class StaticConfigAnalysis
     module Elem
       Input = Struct.new(:plugin, :config)
@@ -72,6 +72,7 @@ module Fluent
       available_worker_ids = [*0...@workers]
 
       ret = []
+      supported_directives = %w[source match filter label]
       conf.elements(name: 'worker').each do |config|
         ids = parse_worker_id(config)
         ids.each do |id|
@@ -83,7 +84,7 @@ module Fluent
         end
 
         config.elements.each do |elem|
-          unless %w[source match filter label].include?(elem.name)
+          unless supported_directives.include?(elem.name)
             raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
           end
         end

data/lib/fluent/supervisor.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'open3'
 require 'pathname'
+require 'find'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -43,11 +44,16 @@ module Fluent
       @rpc_endpoint = nil
       @rpc_server = nil
       @counter = nil
+      @socket_manager_server = nil
+      @starting_new_supervisor_with_zero_downtime = false
+      @new_supervisor_pid = nil
+      start_in_parallel = ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
+      @zero_downtime_restart_mutex = Mutex.new
 
       @fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
       ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
 
-      if config[:rpc_endpoint]
+      if config[:rpc_endpoint] and not start_in_parallel
         @rpc_endpoint = config[:rpc_endpoint]
         @enable_get_dump = config[:enable_get_dump]
         run_rpc_server
@@ -59,16 +65,27 @@ module Fluent
         install_supervisor_signal_handlers
       end
 
-      if counter = config[:counter_server]
+      if counter = config[:counter_server] and not start_in_parallel
         run_counter_server(counter)
       end
 
       if config[:disable_shared_socket]
         $log.info "shared socket for multiple workers is disabled"
+      elsif start_in_parallel
+        begin
+          raise "[BUG] SERVERENGINE_SOCKETMANAGER_PATH env var must exist when starting in parallel" unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')
+          @socket_manager_server = ServerEngine::SocketManager::Server.share_sockets_with_another_server(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+          $log.info "zero-downtime-restart: took over the shared sockets", path: ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+        rescue => e
+          $log.error "zero-downtime-restart: cancel sequence because failed to take over the shared sockets", error: e
+          raise
+        end
       else
-        server = ServerEngine::SocketManager::Server.open
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
+        @socket_manager_server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_server.path.to_s
       end
+
+      stop_parallel_old_supervisor_after_delay if start_in_parallel
     end
 
     def after_run
@@ -76,7 +93,9 @@ module Fluent
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
       cleanup_lock_dir
-      Fluent::Supervisor.cleanup_resources
+      Fluent::Supervisor.cleanup_socketmanager_path unless @starting_new_supervisor_with_zero_downtime
+    ensure
+      notify_new_supervisor_that_old_one_has_stopped if @starting_new_supervisor_with_zero_downtime
     end
 
     def cleanup_lock_dir
@@ -109,6 +128,13 @@ module Fluent
         end
         nil
       }
+      unless Fluent.windows?
+        @rpc_server.mount_proc('/api/processes.zeroDowntimeRestart') { |req, res|
+          $log.debug "fluentd RPC got /api/processes.zeroDowntimeRestart request"
+          Process.kill :USR2, Process.pid
+          nil
+        }
+      end
       @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
         $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
         if Fluent.windows?
@@ -137,27 +163,24 @@ module Fluent
 
       @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
         $log.debug "fluentd RPC got /api/config.gracefulReload request"
-        if Fluent.windows?
-          supervisor_sigusr2_handler
-        else
-          Process.kill :USR2, Process.pid
-        end
-
+        graceful_reload
         nil
       }
 
-      @rpc_server.mount_proc('/api/config.getDump') { |req, res|
-        $log.debug "fluentd RPC got /api/config.getDump request"
-        $log.info "get dump in-memory config via HTTP"
-        res.body = supervisor_get_dump_config_handler
-        [nil, nil, res]
-      } if @enable_get_dump
+      if @enable_get_dump
+        @rpc_server.mount_proc('/api/config.getDump') { |req, res|
+          $log.debug "fluentd RPC got /api/config.getDump request"
+          $log.info "get dump in-memory config via HTTP"
+          res.body = supervisor_get_dump_config_handler
+          [nil, nil, res]
+        }
+      end
 
       @rpc_server.start
     end
 
     def stop_rpc_server
-      @rpc_server.shutdown
+      @rpc_server&.shutdown
     end
 
     def run_counter_server(counter_conf)
@@ -172,6 +195,44 @@ module Fluent
       @counter.stop
     end
 
+    def stop_parallel_old_supervisor_after_delay
+      Thread.new do
+        # Delay to wait the new workers to start up.
+        # Even if it takes a long time to start the new workers and stop the old Fluentd first,
+        # it is no problem because the socket buffer works, as long as the capacity is not exceeded.
+        sleep 10
+        old_pid = ENV["FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"]&.to_i
+        if old_pid
+          $log.info "zero-downtime-restart: stop the old supervisor"
+          Process.kill :TERM, old_pid
+        end
+      rescue => e
+        $log.warn "zero-downtime-restart: failed to stop the old supervisor." +
+                  " If the old one does not exist, please send SIGWINCH to this new process to start to work fully." +
+                  " If it exists, something went wrong. Please kill the old one manually.",
+                  error: e
+      end
+    end
+
+    def notify_new_supervisor_that_old_one_has_stopped
+      if config[:pid_path]
+        new_pid = File.read(config[:pid_path]).to_i
+      else
+        raise "[BUG] new_supervisor_pid is not saved" unless @new_supervisor_pid
+        new_pid = @new_supervisor_pid
+      end
+
+      $log.info "zero-downtime-restart: notify the new supervisor (pid: #{new_pid}) that old one has stopped"
+      Process.kill :WINCH, new_pid
+    rescue => e
+      $log.error(
+        "zero-downtime-restart: failed to notify the new supervisor." +
+        " Please send SIGWINCH to the new supervisor process manually" +
+        " if it does not start to work fully.",
+        error: e
+      )
+    end
+
     def install_supervisor_signal_handlers
       return if Fluent.windows?
 
@@ -187,7 +248,16 @@ module Fluent
 
       trap :USR2 do
         $log.debug 'fluentd supervisor process got SIGUSR2'
-        supervisor_sigusr2_handler
+        if Fluent.windows?
+          graceful_reload
+        else
+          zero_downtime_restart
+        end
+      end
+
+      trap :WINCH do
+        $log.debug 'fluentd supervisor process got SIGWINCH'
+        cancel_source_only
       end
     end
 
@@ -254,7 +324,7 @@ module Fluent
             when :usr1
               supervisor_sigusr1_handler
             when :usr2
-              supervisor_sigusr2_handler
+              graceful_reload
             when :cont
               supervisor_dump_handler_for_windows
             when :stop_event_thread
@@ -284,7 +354,7 @@ module Fluent
       send_signal_to_workers(:USR1)
     end
 
-    def supervisor_sigusr2_handler
+    def graceful_reload
       conf = nil
       t = Thread.new do
         $log.info 'Reloading new config'
@@ -312,6 +382,79 @@ module Fluent
       $log.error "Failed to reload config file: #{e}"
     end
 
+    def zero_downtime_restart
+      Thread.new do
+        @zero_downtime_restart_mutex.synchronize do
+          $log.info "start zero-downtime-restart sequence"
+
+          if @starting_new_supervisor_with_zero_downtime
+            $log.warn "zero-downtime-restart: canceled because it is already starting"
+            Thread.exit
+          end
+          if ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
+            $log.warn "zero-downtime-restart: canceled because the previous sequence is still running"
+            Thread.exit
+          end
+
+          @starting_new_supervisor_with_zero_downtime = true
+          commands = [ServerEngine.ruby_bin_path, $0] + ARGV
+          env_to_add = {
+            "SERVERENGINE_SOCKETMANAGER_INTERNAL_TOKEN" => ServerEngine::SocketManager::INTERNAL_TOKEN,
+            "FLUENT_RUNNING_IN_PARALLEL_WITH_OLD" => "#{Process.pid}",
+          }
+          pid = Process.spawn(env_to_add, commands.join(" "))
+          @new_supervisor_pid = pid unless config[:daemonize]
+
+          if config[:daemonize]
+            Thread.new(pid) do |pid|
+              _, status = Process.wait2(pid)
+              # check if `ServerEngine::Daemon#daemonize_with_double_fork` succeeded or not
+              unless status.success?
+                @starting_new_supervisor_with_zero_downtime = false
+                $log.error "zero-downtime-restart: failed because new supervisor exits unexpectedly"
+              end
+            end
+          else
+            Thread.new(pid) do |pid|
+              _, status = Process.wait2(pid)
+              @starting_new_supervisor_with_zero_downtime = false
+              $log.error "zero-downtime-restart: failed because new supervisor exits unexpectedly", status: status
+            end
+          end
+        end
+      rescue => e
+        $log.error "zero-downtime-restart: failed", error: e
+        @starting_new_supervisor_with_zero_downtime = false
+      end
+    end
+
+    def cancel_source_only
+      if ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
+        if config[:rpc_endpoint]
+          begin
+            @rpc_endpoint = config[:rpc_endpoint]
+            @enable_get_dump = config[:enable_get_dump]
+            run_rpc_server
+          rescue => e
+            $log.error "failed to start RPC server", error: e
+          end
+        end
+
+        if counter = config[:counter_server]
+          begin
+            run_counter_server(counter)
+          rescue => e
+            $log.error "failed to start counter server", error: e
+          end
+        end
+
+        $log.info "zero-downtime-restart: done all sequences, now new processes start to work fully"
+        ENV.delete("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
+      end
+
+      send_signal_to_workers(:WINCH)
+    end
+
     def supervisor_dump_handler_for_windows
       # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
       # and it is implemented before the implementation of the function for Windows.
@@ -409,6 +552,7 @@ module Fluent
       main_cmd = config[:main_cmd]
       env = {
         'SERVERENGINE_WORKER_ID' => @worker_id.to_i.to_s,
+        'FLUENT_INSTANCE_ID' => Fluent::INSTANCE_ID,
       }
       @pm = process_manager.spawn(env, *main_cmd)
     end
@@ -440,11 +584,11 @@ module Fluent
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
         logger: $log,
-        log: $log.out,
+        log: $log&.out,
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],
-        chumask: params['chumask'],
+        chumask: params['chumask'].is_a?(Integer) ? params['chumask'] : params['chumask']&.to_i(8),
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -486,7 +630,8 @@ module Fluent
         suppress_repeated_stacktrace: true,
         ignore_repeated_log_interval: nil,
         without_source: nil,
-        enable_input_metrics: nil,
+        with_source_only: nil,
+        enable_input_metrics: true,
         enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
@@ -499,12 +644,11 @@ module Fluent
       }
     end
 
-    def self.cleanup_resources
-      unless Fluent.windows?
-        if ENV.has_key?('SERVERENGINE_SOCKETMANAGER_PATH')
-          FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
-        end
-      end
+    def self.cleanup_socketmanager_path
+      return if Fluent.windows?
+      return unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')
+
+      FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
     end
 
     def initialize(cl_opt)
@@ -518,7 +662,6 @@ module Fluent
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
       @conf_encoding = opt[:conf_encoding]
-      @log_path = opt[:log_path]
       @show_plugin_config = opt[:show_plugin_config]
       @libs = opt[:libs]
       @plugin_dirs = opt[:plugin_dirs]
@@ -527,13 +670,15 @@ module Fluent
       @chumask = opt[:chumask]
       @signame = opt[:signame]
 
-      # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
+      # TODO: `@log_path`, `@log_rotate_age` and `@log_rotate_size` should be removed
       # since it should be merged with SystemConfig in `build_system_config()`.
-      # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
+      # We should always use `system_config.log.path`, `system_config.log.rotate_age`
+      # and `system_config.log.rotate_size`.
       # However, currently, there is a bug that `system_config.log` parameters
       # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
       # parameters are not merged in `build_system_config()`.
       # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
+      @log_path = opt[:log_path]
       @log_rotate_age = opt[:log_rotate_age]
       @log_rotate_size = opt[:log_rotate_size]
 
@@ -549,6 +694,10 @@ module Fluent
         raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@system_config.workers}"
       end
 
+      if Fluent.windows? && @system_config.with_source_only
+        raise Fluent::ConfigError, "with-source-only is not supported on Windows"
+      end
+
       root_dir = @system_config.root_dir
       if root_dir
         if File.exist?(root_dir)
@@ -567,12 +716,24 @@ module Fluent
       begin
         ServerEngine::Privilege.change(@chuser, @chgroup)
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
-        Fluent::Engine.init(@system_config, supervisor_mode: true)
+        Fluent::Engine.init(@system_config, supervisor_mode: true, start_in_parallel: ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"))
         Fluent::Engine.run_configure(@conf, dry_run: dry_run)
       rescue Fluent::ConfigError => e
         $log.error 'config error', file: @config_path, error: e
         $log.debug_backtrace
         exit!(1)
+      rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError
+        if e.respond_to?(:path)
+          $log.error e.message, path: e.path, error: e
+        else
+          $log.error e.message, error: e
+        end
+        $log.debug_backtrace
+        exit!(1)
+      rescue => e
+        $log.error "unexpected error", error: e
+        $log.debug_backtrace
+        exit!(1)
       end
 
       if dry_run
@@ -600,9 +761,13 @@ module Fluent
         raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}"
       end
 
+      if Fluent.windows? && @system_config.with_source_only
+        raise Fluent::ConfigError, "with-source-only is not supported on Windows"
+      end
+
       install_main_process_signal_handlers
 
-      # This is the only log messsage for @standalone_worker
+      # This is the only log message for @standalone_worker
       $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker
 
       main_process do
@@ -612,10 +777,10 @@ module Fluent
           File.umask(@chumask.to_i(8))
         end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
-        Fluent::Engine.init(@system_config)
+        Fluent::Engine.init(@system_config, start_in_parallel: ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"))
         Fluent::Engine.run_configure(@conf)
         Fluent::Engine.run
-        self.class.cleanup_resources if @standalone_worker
+        self.class.cleanup_socketmanager_path if @standalone_worker
         exit 0
       end
     end
@@ -642,6 +807,10 @@ module Fluent
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 
+      build_additional_configurations do |additional_conf|
+        @conf += additional_conf
+      end
+
       @libs.each do |lib|
         require lib
       end
@@ -690,6 +859,7 @@ module Fluent
 
       # TODO: we should remove this logic. This merging process should be done
       # in `build_system_config()`.
+      @log_path ||= system_config.log.path
       @log_rotate_age ||= system_config.log.rotate_age
       @log_rotate_size ||= system_config.log.rotate_size
 
@@ -738,11 +908,12 @@ module Fluent
         ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
         ignore_same_log_interval: system_config.ignore_same_log_interval,
       )
+      $log.force_stacktrace_level(system_config.log.forced_stacktrace_level) if system_config.force_stacktrace_level?
       $log.enable_color(false) if actual_log_path
       $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
 
       $log.info "init #{process_type} logger",
-                path: actual_log_path, 
+                path: actual_log_path,
                 rotate_age: @log_rotate_age,
                 rotate_size: @log_rotate_size
     end
@@ -771,6 +942,7 @@ module Fluent
         'inline_config' => @inline_config,
         'chuser' => @chuser,
         'chgroup' => @chgroup,
+        'chumask' => @chumask,
         'fluentd_conf_path' => @config_path,
         'fluentd_conf' => @conf.to_s,
         'use_v1_config' => @use_v1_config,
@@ -832,12 +1004,20 @@ module Fluent
         end
 
         trap :USR2 do
+          # Leave the old GracefulReload feature, just in case.
+          # We can send SIGUSR2 to the worker process to use this old GracefulReload feature.
+          # (Note: Normally, we can send SIGUSR2 to the supervisor process to use
+          #  zero-downtime-restart feature as GracefulReload on non-Windows.)
           reload_config
         end
 
         trap :CONT do
           dump_non_windows
         end
+
+        trap :WINCH do
+          cancel_source_only
+        end
       end
     end
 
@@ -891,6 +1071,18 @@ module Fluent
       end
     end
 
+    def cancel_source_only
+      Thread.new do
+        begin
+          $log.debug "fluentd main process get SIGWINCH"
+          $log.info "try to cancel with-source-only mode"
+          Fluent::Engine.cancel_source_only!
+        rescue Exception => e
+          $log.warn "failed to cancel source only", error: e
+        end
+      end
+    end
+
     def reload_config
       Thread.new do
         $log.debug('worker got SIGUSR2')
@@ -904,6 +1096,10 @@ module Fluent
             type: @config_file_type,
           )
 
+          build_additional_configurations do |additional_conf|
+            conf += additional_conf
+          end
+
           Fluent::VariableStore.try_to_reset do
             Fluent::Engine.reload_config(conf)
           end
@@ -995,7 +1191,7 @@ module Fluent
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       # Prefer the options explicitly specified in the command line
-      # 
+      #
       # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
       # not merged with the command line options since they are not in
       # `SYSTEM_CONFIG_PARAMETERS`.
@@ -1010,6 +1206,29 @@ module Fluent
       system_config
     end
 
+    def build_additional_configurations
+      if @system_config.config_include_dir&.empty?
+        $log.info :supervisor, 'configuration include directory is disabled'
+        return
+      end
+      begin
+        supported_suffixes = [".conf", ".yaml", ".yml"]
+        Find.find(@system_config.config_include_dir) do |path|
+          next if File.directory?(path)
+          next unless supported_suffixes.include?(File.extname(path))
+          # NOTE: both types of normal config (.conf) and YAML will be loaded.
+          # Thus, it does not care whether @config_path is .conf or .yml.
+          $log.info :supervisor, 'loading additional configuration file', path: path
+          yield Fluent::Config.build(config_path: path,
+                                     encoding: @conf_encoding,
+                                     use_v1_config: @use_v1_config,
+                                     type: :guess)
+        end
+      rescue Errno::ENOENT
+        $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir
+      end
+    end
+
     RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
 
     def build_spawn_command