fluentd 1.13.3 → 1.16.5

data/lib/fluent/supervisor.rb

@@ -16,6 +16,7 @@
 
 require 'fileutils'
 require 'open3'
+require 'pathname'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -31,12 +32,6 @@ require 'fluent/variable_store'
 require 'serverengine'
 
 if Fluent.windows?
-  require 'windows/library'
-  require 'windows/synchronize'
-  require 'windows/system_info'
-  include Windows::Library
-  include Windows::Synchronize
-  include Windows::SystemInfo
   require 'win32/ipc'
   require 'win32/event'
 end
@@ -49,6 +44,9 @@ module Fluent
       @rpc_server = nil
       @counter = nil
 
+      @fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
+      ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
+
       if config[:rpc_endpoint]
         @rpc_endpoint = config[:rpc_endpoint]
         @enable_get_dump = config[:enable_get_dump]
@@ -68,9 +66,8 @@ module Fluent
       if config[:disable_shared_socket]
         $log.info "shared socket for multiple workers is disabled"
       else
-        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-        ServerEngine::SocketManager::Server.open(socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+        server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
       end
     end
 
@@ -78,21 +75,27 @@ module Fluent
       stop_windows_event_thread if Fluent.windows?
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
+      cleanup_lock_dir
       Fluent::Supervisor.cleanup_resources
     end
 
+    def cleanup_lock_dir
+      FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, "fluentd-*.lock")))
+      FileUtils.rmdir(@fluentd_lock_dir)
+    end
+
     def run_rpc_server
       @rpc_server = RPC::Server.new(@rpc_endpoint, $log)
 
       # built-in RPC for signals
       @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
         $log.debug "fluentd RPC got /api/processes.interruptWorkers request"
-        Process.kill :INT, $$
+        Process.kill :INT, Process.pid
         nil
       }
       @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
         $log.debug "fluentd RPC got /api/processes.killWorkers request"
-        Process.kill :TERM, $$
+        Process.kill :TERM, Process.pid
         nil
       }
       @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
@@ -101,8 +104,8 @@ module Fluent
           supervisor_sigusr1_handler
           stop(true)
         else
-          Process.kill :USR1, $$
-          Process.kill :TERM, $$
+          Process.kill :USR1, Process.pid
+          Process.kill :TERM, Process.pid
         end
         nil
       }
@@ -111,7 +114,7 @@ module Fluent
         if Fluent.windows?
           supervisor_sigusr1_handler
         else
-          Process.kill :USR1, $$
+          Process.kill :USR1, Process.pid
         end
         nil
       }
@@ -121,7 +124,7 @@ module Fluent
           # restart worker with auto restarting by killing
           kill_worker
         else
-          Process.kill :HUP, $$
+          Process.kill :HUP, Process.pid
         end
         nil
       }
@@ -137,7 +140,7 @@ module Fluent
         if Fluent.windows?
           supervisor_sigusr2_handler
         else
-          Process.kill :USR2, $$
+          Process.kill :USR2, Process.pid
         end
 
         nil
@@ -209,50 +212,57 @@ module Fluent
     def install_windows_event_handler
       return unless Fluent.windows?
 
-      @pid_signame = "fluentd_#{$$}"
+      @pid_signame = "fluentd_#{Process.pid}"
       @signame = config[:signame]
 
       Thread.new do
         ipc = Win32::Ipc.new(nil)
         events = [
-          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
-          Win32::Event.new("#{@pid_signame}"),
-          Win32::Event.new("#{@pid_signame}_HUP"),
-          Win32::Event.new("#{@pid_signame}_USR1"),
-          Win32::Event.new("#{@pid_signame}_USR2"),
+          {win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
+          {win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
+          {win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
+          {win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
         ]
         if @signame
           signame_events = [
-            Win32::Event.new("#{@signame}"),
-            Win32::Event.new("#{@signame}_HUP"),
-            Win32::Event.new("#{@signame}_USR1"),
-            Win32::Event.new("#{@signame}_USR2"),
+            {win32_event: Win32::Event.new("#{@signame}"), action: :stop},
+            {win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
+            {win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
+            {win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
+            {win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
           ]
           events.concat(signame_events)
         end
         begin
           loop do
-            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
-            if idx > 0 && idx <= events.length
-              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            infinite = 0xFFFFFFFF
+            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, infinite)
+            event_idx = ipc_idx - 1
+
+            if event_idx >= 0 && event_idx < events.length
+              $log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
             else
-              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+              $log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
             end
-            case idx
-            when 2, 6
+            case events[event_idx][:action]
+            when :stop
               stop(true)
-            when 3, 7
+            when :hup
               supervisor_sighup_handler
-            when 4, 8
+            when :usr1
               supervisor_sigusr1_handler
-            when 5, 9
+            when :usr2
               supervisor_sigusr2_handler
-            when 1
+            when :cont
+              supervisor_dump_handler_for_windows
+            when :stop_event_thread
               break
             end
           end
         ensure
-          events.each { |event| event.close }
+          events.each { |event| event[:win32_event].close }
         end
       end
     end
@@ -302,6 +312,26 @@ module Fluent
       $log.error "Failed to reload config file: #{e}"
     end
 
+    def supervisor_dump_handler_for_windows
+      # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
+      # and it is implemented before the implementation of the function for Windows.
+      # It is possible to trap SIGCONT and handle it here also on UNIX-like,
+      # but for backward compatibility, this handler is currently for a Windows-only.
+      raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
+
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error "failed to dump: #{e}"
+        end
+      end
+
+      send_signal_to_workers(:CONT)
+    rescue => e
+      $log.error "failed to dump: #{e}"
+    end
+
     def kill_worker
       if config[:worker_pid]
         pids = config[:worker_pid].clone
@@ -324,14 +354,18 @@ module Fluent
       { conf: @fluentd_conf }
     end
 
+    def dump
+      super unless @stop
+    end
+
     private
 
     def reopen_log
-      if (log = config[:logger_initializer])
+      if $log
         # Creating new thread due to mutex can't lock
         # in main thread during trap context
         Thread.new do
-          log.reopen!
+          $log.reopen!
         end
       end
     end
@@ -358,6 +392,14 @@ module Fluent
         restart(true)
       when :USR2
         reload
+      when :CONT
+        dump_all_windows_workers
+      end
+    end
+
+    def dump_all_windows_workers
+      @monitors.each do |m|
+        m.send_command("DUMP\n")
       end
     end
   end
@@ -374,46 +416,14 @@ module Fluent
     def after_start
       (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
     end
+
+    def dump
+      super unless @stop
+    end
   end
 
   class Supervisor
-    def self.load_config(path, params = {})
-      pre_loadtime = 0
-      pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
-      pre_config_mtime = nil
-      pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
-      config_mtime = File.mtime(path)
-
-      # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
-      if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
-        return params['pre_conf']
-      end
-
-      log_level = params['log_level']
-      suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
-      ignore_repeated_log_interval = params['ignore_repeated_log_interval']
-      ignore_same_log_interval = params['ignore_same_log_interval']
-
-      log_path = params['log_path']
-      chuser = params['chuser']
-      chgroup = params['chgroup']
-      log_rotate_age = params['log_rotate_age']
-      log_rotate_size = params['log_rotate_size']
-
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
-                  ignore_same_log_interval: ignore_same_log_interval}
-      logger_initializer = Supervisor::LoggerInitializer.new(
-        log_path, log_level, chuser, chgroup, log_opts,
-        log_rotate_age: log_rotate_age,
-        log_rotate_size: log_rotate_size
-      )
-      # this #init sets initialized logger to $log
-      logger_initializer.init(:supervisor, 0)
-      logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
-      logger = $log
-
-      command_sender = Fluent.windows? ? "pipe" : "signal"
-
+    def self.serverengine_config(params = {})
       # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
       pid_path = params['daemonize']
       daemonize = !!params['daemonize']
@@ -429,17 +439,12 @@ module Fluent
         unrecoverable_exit_codes: [2],
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
-        logger: logger,
-        log: logger.out,
-        log_path: log_path,
-        log_level: log_level,
-        logger_initializer: logger_initializer,
-        chuser: chuser,
-        chgroup: chgroup,
-        chumask: 0,
-        suppress_repeated_stacktrace: suppress_repeated_stacktrace,
-        ignore_repeated_log_interval: ignore_repeated_log_interval,
-        ignore_same_log_interval: ignore_same_log_interval,
+        logger: $log,
+        log: $log.out,
+        log_level: params['log_level'],
+        chuser: params['chuser'],
+        chgroup: params['chgroup'],
+        chumask: params['chumask'],
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -448,112 +453,22 @@ module Fluent
                                  File.join(File.dirname(__FILE__), 'daemon.rb'),
                                  ServerModule.name,
                                  WorkerModule.name,
-                                 path,
                                  JSON.dump(params)],
-        command_sender: command_sender,
+        command_sender: Fluent.windows? ? "pipe" : "signal",
+        config_path: params['fluentd_conf_path'],
         fluentd_conf: params['fluentd_conf'],
         conf_encoding: params['conf_encoding'],
         inline_config: params['inline_config'],
-        config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
-        disable_shared_socket: params['disable_shared_socket']
+        disable_shared_socket: params['disable_shared_socket'],
+        restart_worker_interval: params['restart_worker_interval'],
       }
-      if daemonize
-        se_config[:pid_path] = pid_path
-      end
-      pre_params = params.dup
-      params['pre_loadtime'] = Time.now.to_i
-      params['pre_config_mtime'] = config_mtime
-      params['pre_conf'] = se_config
-      # prevent pre_conf from being too big by reloading many times.
-      pre_params['pre_conf'] = nil
-      params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
+      se_config[:pid_path] = pid_path if daemonize
 
       se_config
     end
 
-    class LoggerInitializer
-      def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
-        @path = path
-        @level = level
-        @chuser = chuser
-        @chgroup = chgroup
-        @opts = opts
-        @log_rotate_age = log_rotate_age
-        @log_rotate_size = log_rotate_size
-      end
-
-      def worker_id_suffixed_path(worker_id, path)
-        require 'pathname'
-
-        Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
-      end
-
-      def init(process_type, worker_id)
-        @opts[:process_type] = process_type
-        @opts[:worker_id] = worker_id
-
-        if @path && @path != "-"
-          unless File.exist?(@path)
-            FileUtils.mkdir_p(File.dirname(@path))
-          end
-
-          @logdev = if @log_rotate_age || @log_rotate_size
-                     Fluent::LogDeviceIO.new(Fluent.windows? ?
-                                               worker_id_suffixed_path(worker_id, @path) : @path,
-                                             shift_age: @log_rotate_age, shift_size: @log_rotate_size)
-                   else
-                     File.open(@path, "a")
-                   end
-          if @chuser || @chgroup
-            chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
-            chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
-            File.chown(chuid, chgid, @path)
-          end
-        else
-          @logdev = STDOUT
-        end
-
-        dl_opts = {}
-        # subtract 1 to match serverengine daemon logger side logging severity.
-        dl_opts[:log_level] = @level - 1
-        dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
-        dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
-        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
-        $log = Fluent::Log.new(logger, @opts)
-        $log.enable_color(false) if @path
-        $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
-      end
-
-      def stdout?
-        @logdev == STDOUT
-      end
-
-      def reopen!
-        if @path && @path != "-"
-          @logdev.reopen(@path, "a")
-        end
-        self
-      end
-
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
-        $log.format = format if format
-        $log.time_format = time_format if time_format
-        $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
-        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
-
-        if @path && log_dir_perm
-          File.chmod(log_dir_perm || 0755, File.dirname(@path))
-        end
-      end
-
-      def level=(level)
-        @level = level
-        $log.level = level
-      end
-    end
-
     def self.default_options
       {
         config_path: Fluent::DEFAULT_CONFIG_PATH,
@@ -565,18 +480,22 @@ module Fluent
         setup_path: nil,
         chuser: nil,
         chgroup: nil,
+        chumask: "0",
         root_dir: nil,
         suppress_interval: 0,
         suppress_repeated_stacktrace: true,
         ignore_repeated_log_interval: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
         supervise: true,
         standalone_worker: false,
         signame: nil,
         conf_encoding: 'utf-8',
-        disable_shared_socket: nil
+        disable_shared_socket: nil,
+        config_file_type: :guess,
       }
     end
 
@@ -588,7 +507,11 @@ module Fluent
       end
     end
 
-    def initialize(opt)
+    def initialize(cl_opt)
+      @cl_opt = cl_opt
+      opt = self.class.default_options.merge(cl_opt)
+
+      @config_file_type = opt[:config_file_type]
       @daemonize = opt[:daemonize]
       @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
@@ -601,34 +524,19 @@ module Fluent
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
+      @chumask = opt[:chumask]
+      @signame = opt[:signame]
 
+      # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
+      # since it should be merged with SystemConfig in `build_system_config()`.
+      # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
+      # However, currently, there is a bug that `system_config.log` parameters
+      # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
+      # parameters are not merged in `build_system_config()`.
+      # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
       @log_rotate_age = opt[:log_rotate_age]
       @log_rotate_size = opt[:log_rotate_size]
-      @signame = opt[:signame]
 
-      @cl_opt = opt
-      @conf = nil
-      # parse configuration immediately to initialize logger in early stage
-      if @config_path and File.exist?(@config_path)
-        @conf = Fluent::Config.build(config_path: @config_path,
-                                     encoding: @conf_encoding ? @conf_encoding : 'utf-8',
-                                     additional_config: @inline_config ? @inline_config : nil,
-                                     use_v1_config: !!@use_v1_config)
-        @system_config = build_system_config(@conf)
-        if @system_config.log
-          @log_rotate_age ||= @system_config.log.rotate_age
-          @log_rotate_size ||= @system_config.log.rotate_size
-        end
-        @conf = nil
-      end
-
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
-                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
-      @log = LoggerInitializer.new(
-        @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
-        log_rotate_age: @log_rotate_age,
-        log_rotate_size: @log_rotate_size
-      )
       @finished = false
     end
 
@@ -649,7 +557,7 @@ module Fluent
           end
         else
           begin
-            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
+            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
           rescue => e
             raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end
@@ -686,12 +594,6 @@ module Fluent
     end
 
     def run_worker
-      begin
-        require 'sigdump/setup'
-      rescue Exception
-        # ignore LoadError and others (related with signals): it may raise these errors in Windows
-      end
-
       Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
 
       if @standalone_worker && @system_config.workers != 1
@@ -707,7 +609,7 @@ module Fluent
         create_socket_manager if @standalone_worker
         if @standalone_worker
           ServerEngine::Privilege.change(@chuser, @chgroup)
-          File.umask(0)
+          File.umask(@chumask.to_i(8))
         end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
@@ -719,17 +621,7 @@ module Fluent
     end
 
     def configure(supervisor: false)
-      if supervisor
-        @log.init(:supervisor, 0)
-      else
-        worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
-        process_type = case
-                       when @standalone_worker then :standalone
-                       when worker_id == 0 then :worker0
-                       else :workers
-                       end
-        @log.init(process_type, worker_id)
-      end
+      setup_global_logger(supervisor: supervisor)
 
       if @show_plugin_config
         show_plugin_config
@@ -739,17 +631,14 @@ module Fluent
         $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
         @inline_config = STDIN.read
       end
-      @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
-      @system_config = build_system_config(@conf)
-
-      @log.level = @system_config.log_level
-      @log.apply_options(
-        format: @system_config.log.format,
-        time_format: @system_config.log.time_format,
-        log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
-        ignore_same_log_interval: @system_config.ignore_same_log_interval
+      @conf = Fluent::Config.build(
+        config_path: @config_path,
+        encoding: @conf_encoding,
+        additional_config: @inline_config,
+        use_v1_config: @use_v1_config,
+        type: @config_file_type,
       )
+      @system_config = build_system_config(@conf)
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 
@@ -774,10 +663,93 @@ module Fluent
 
     private
 
+    def setup_global_logger(supervisor: false)
+      if supervisor
+        worker_id = 0
+        process_type = :supervisor
+      else
+        worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
+        process_type = case
+                       when @standalone_worker then :standalone
+                       when worker_id == 0 then :worker0
+                       else :workers
+                       end
+      end
+
+      # Parse configuration immediately to initialize logger in early stage.
+      # Since we can't confirm the log messages in this parsing process,
+      # we must parse the config again after initializing logger.
+      conf = Fluent::Config.build(
+        config_path: @config_path,
+        encoding: @conf_encoding,
+        additional_config: @inline_config,
+        use_v1_config: @use_v1_config,
+        type: @config_file_type,
+      )
+      system_config = build_system_config(conf)
+
+      # TODO: we should remove this logic. This merging process should be done
+      # in `build_system_config()`.
+      @log_rotate_age ||= system_config.log.rotate_age
+      @log_rotate_size ||= system_config.log.rotate_size
+
+      rotate = @log_rotate_age || @log_rotate_size
+      actual_log_path = @log_path
+
+      # We need to prepare a unique path for each worker since Windows locks files.
+      if Fluent.windows? && rotate && @log_path && @log_path != "-"
+        actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
+      end
+
+      if actual_log_path && actual_log_path != "-"
+        FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
+        if rotate
+          logdev = Fluent::LogDeviceIO.new(
+            actual_log_path,
+            shift_age: @log_rotate_age,
+            shift_size: @log_rotate_size,
+          )
+        else
+          logdev = File.open(actual_log_path, "a")
+        end
+
+        if @chuser || @chgroup
+          chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
+          chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
+          File.chown(chuid, chgid, actual_log_path)
+        end
+
+        if system_config.dir_permission
+          File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
+        end
+      else
+        logdev = STDOUT
+      end
+
+      $log = Fluent::Log.new(
+        # log_level: subtract 1 to match serverengine daemon logger side logging severity.
+        ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
+        path: actual_log_path,
+        process_type: process_type,
+        worker_id: worker_id,
+        format: system_config.log.format,
+        time_format: system_config.log.time_format,
+        suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
+        ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: system_config.ignore_same_log_interval,
+      )
+      $log.enable_color(false) if actual_log_path
+      $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
+
+      $log.info "init #{process_type} logger",
+                path: actual_log_path, 
+                rotate_age: @log_rotate_age,
+                rotate_size: @log_rotate_size
+    end
+
     def create_socket_manager
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      server = ServerEngine::SocketManager::Server.open
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
     end
 
     def show_plugin_config
@@ -797,32 +769,30 @@ module Fluent
         'main_cmd' => fluentd_spawn_cmd,
         'daemonize' => @daemonize,
         'inline_config' => @inline_config,
-        'log_path' => @log_path,
-        'log_rotate_age' => @log_rotate_age,
-        'log_rotate_size' => @log_rotate_size,
         'chuser' => @chuser,
         'chgroup' => @chgroup,
+        'fluentd_conf_path' => @config_path,
+        'fluentd_conf' => @conf.to_s,
         'use_v1_config' => @use_v1_config,
         'conf_encoding' => @conf_encoding,
         'signame' => @signame,
-        'fluentd_conf' => @conf.to_s,
 
         'workers' => @system_config.workers,
         'root_dir' => @system_config.root_dir,
         'log_level' => @system_config.log_level,
-        'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
-        'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
         'rpc_endpoint' => @system_config.rpc_endpoint,
         'enable_get_dump' => @system_config.enable_get_dump,
         'counter_server' => @system_config.counter_server,
-        'log_format' => @system_config.log.format,
-        'log_time_format' => @system_config.log.time_format,
-        'disable_shared_socket' => @system_config.disable_shared_socket
+        'disable_shared_socket' => @system_config.disable_shared_socket,
+        'restart_worker_interval' => @system_config.restart_worker_interval,
       }
 
-      se = ServerEngine.create(ServerModule, WorkerModule){
-        Fluent::Supervisor.load_config(@config_path, params)
+      se = ServerEngine.create(ServerModule, WorkerModule) {
+        # Note: This is called only at the initialization of ServerEngine, since
+        # Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
+        Fluent::Supervisor.serverengine_config(params)
       }
+
       se.run
     end
 
@@ -864,6 +834,10 @@ module Fluent
         trap :USR2 do
           reload_config
         end
+
+        trap :CONT do
+          dump_non_windows
+        end
       end
     end
 
@@ -891,6 +865,9 @@ module Fluent
           when "RELOAD"
             $log.debug "fluentd main process get #{cmd} command"
             reload_config
+          when "DUMP"
+            $log.debug "fluentd main process get #{cmd} command"
+            dump_windows
           else
             $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
@@ -905,7 +882,7 @@ module Fluent
         begin
           $log.debug "fluentd main process get SIGUSR1"
           $log.info "force flushing buffered events"
-          @log.reopen!
+          $log.reopen!
           Fluent::Engine.flush!
           $log.debug "flushing thread: flushed"
         rescue Exception => e
@@ -924,6 +901,7 @@ module Fluent
             encoding: @conf_encoding,
             additional_config: @inline_config,
             use_v1_config: @use_v1_config,
+            type: @config_file_type,
           )
 
           Fluent::VariableStore.try_to_reset do
@@ -940,9 +918,27 @@ module Fluent
       end
     end
 
+    def dump_non_windows
+      begin
+        Sigdump.dump unless @finished
+      rescue => e
+        $log.error("failed to dump: #{e}")
+      end
+    end
+
+    def dump_windows
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error("failed to dump: #{e}")
+        end
+      end
+    end
+
     def logging_with_console_output
       yield $log
-      unless @log.stdout?
+      unless $log.stdout?
         logger = ServerEngine::DaemonLogger.new(STDOUT)
         log = Fluent::Log.new(logger)
         log.level = @system_config.log_level
@@ -998,15 +994,15 @@ module Fluent
 
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
+      # Prefer the options explicitly specified in the command line
+      # 
+      # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
+      # not merged with the command line options since they are not in
+      # `SYSTEM_CONFIG_PARAMETERS`.
+      # We have to fix this bug.
       opt = {}
       Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
         if @cl_opt.key?(param) && !@cl_opt[param].nil?
-          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
-            # info level can't be specified via command line option.
-            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
-            next
-          end
-
           opt[param] = @cl_opt[param]
         end
       end
@@ -1034,6 +1030,11 @@ module Fluent
         fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
       end
 
+      if @system_config.enable_jit
+        $log.info "enable Ruby JIT for workers (--jit)"
+        fluentd_spawn_cmd << '--jit'
+      end
+
       # Adding `-h` so that it can avoid ruby's command blocking
       # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
       _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
@@ -1049,4 +1050,27 @@ module Fluent
       fluentd_spawn_cmd
     end
   end
+
+  module FluentSigdump
+    def self.dump_windows
+      raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
+
+      # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
+      # but `/tmp` dir may not exist on Windows by default.
+      # So use the systemroot-temp-dir instead.
+      dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
+        ? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
+        : get_path_with_pid(ENV['SIGDUMP_PATH'])
+
+      require 'sigdump'
+      Sigdump.dump(dump_filepath)
+
+      $log.info "dump to #{dump_filepath}."
+    end
+
+    def self.get_path_with_pid(raw_path)
+      path = Pathname.new(raw_path)
+      path.sub_ext("-#{Process.pid}#{path.extname}").to_s
+    end
+  end
 end