fluentd 1.8.0.rc2 → 1.8.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c70380cd4a9f0f78150f7332a5e484a9f9523786fabae5c1bda5761cf21617e
-  data.tar.gz: 65e167aa06b31cc5aebce1e8d1bbb61d80e8e1ae9e1690dd58dc799f51683503
+  metadata.gz: 2f75be7f2480c2f0fbdddf1999dc3163b91f6bd75fb53b71ff3d7a0ee4f670af
+  data.tar.gz: 5089bdce54388f855748befd4234ceb945f4ef7c6aa0f2a4f1aeea21adb516da
 SHA512:
-  metadata.gz: f19fccb4e6d84dec936215ef8388d650a5063f30e0456c9aae5f47d2d0f1b73b86f01c95fc7e3d30e939b3d72e4224fcc5c8c1d69f59a0db7fc64b239cd9f8a2
-  data.tar.gz: dd4d743c7d1d73199c35458ba3cb09e1dd6dda5a49b2f30575d6696f573909b642b1e860687a2e6bec8575db14869cfa29039cb472eb75b00abfb944bf368e0a
+  metadata.gz: 6dfd02456cd7a47aaf483759b57075cc0b8c0bbe322a41c20cd6df7f9a163150af07a21201e8514b0ba097f50dc56ae7004c4116f2f4c2b5347eebde303a19a2
+  data.tar.gz: 1ec01c129c043f8fafee99360a3a625907cff7666f9a605093110b4b1cc57691e7fd1eba543e2003568a9aa48d714a853b2644e83d2823f5773f49615bc0d265

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,7 +1,6 @@
 ---
 name:  Bug Report
 about: Create a report to help us improve. If you have questions about Fluentd and plugins, please direct these to https://groups.google.com/forum/#!forum/fluentd
-labels: 'bug'
 
 ---
 

data/CHANGELOG.md

@@ -13,16 +13,23 @@
   https://github.com/fluent/fluentd/pull/2660
 * Relax tzinfo dependency to accept v1
   https://github.com/fluent/fluentd/pull/2673
-* clean up internal classes / code
+* log: Deprecate top-level match for capturing fluentd logs
+  https://github.com/fluent/fluentd/pull/2689
+* clean up internal classes / methods / code
   https://github.com/fluent/fluentd/pull/2647
   https://github.com/fluent/fluentd/pull/2648
   https://github.com/fluent/fluentd/pull/2653
   https://github.com/fluent/fluentd/pull/2654
   https://github.com/fluent/fluentd/pull/2657
   https://github.com/fluent/fluentd/pull/2667
+  https://github.com/fluent/fluentd/pull/2674
+  https://github.com/fluent/fluentd/pull/2677
+  https://github.com/fluent/fluentd/pull/2680
 
 ### Bug fixes
 
+* Fix dry-run mode
+  https://github.com/fluent/fluentd/pull/2651
 * suppress warning
   https://github.com/fluent/fluentd/pull/2652
 * suppress keyword argument warning for ruby2.7
@@ -31,6 +38,12 @@
   https://github.com/fluent/fluentd/pull/2666
 * time: Properly show class names in error message
   https://github.com/fluent/fluentd/pull/2671
+* Fix a potential bug that ThreadError may occur on SIGUSR1
+  https://github.com/fluent/fluentd/pull/2678
+* server helper: Ignore ECONNREFUSED in TLS accept to avoid fluentd restart
+  https://github.com/fluent/fluentd/pull/2695
+* server helper: Fix IPv6 dual stack mode issue for tcp socket.
+  https://github.com/fluent/fluentd/pull/2697
 
 
 # v1.7

data/lib/fluent/agent.rb

@@ -62,7 +62,9 @@ module Fluent
 
       # initialize <match> and <filter> elements
       conf.elements('filter', 'match').each { |e|
-        next if e.for_another_worker?
+        if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+          next
+        end
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type']
         raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type

data/lib/fluent/command/fluentd.rb

@@ -309,7 +309,6 @@ end
 
 exit 0 if early_exit
 
-require 'fluent/supervisor'
 if opts[:supervise]
   if Fluent.windows?
     if opts[:log_path] && opts[:log_path] != "-"
@@ -321,11 +320,16 @@ if opts[:supervise]
       end
     end
   end
-  Fluent::Supervisor.new(opts).run_supervisor
+
+  supervisor = Fluent::Supervisor.new(opts)
+  supervisor.configure(supervisor: true)
+  supervisor.run_supervisor(dry_run: opts[:dry_run])
 else
   if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
     puts "Error: multi workers is not supported with --no-supervisor"
     exit 2
   end
-  Fluent::Supervisor.new(opts).run_worker
+  worker = Fluent::Supervisor.new(opts)
+  worker.configure
+  worker.run_worker
 end

data/lib/fluent/engine.rb

@@ -41,16 +41,16 @@ module Fluent
       @fluent_log_event_router = nil
       @system_config = SystemConfig.new
 
-      @dry_run_mode = false
+      @supervisor_mode = false
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
 
-    attr_reader :root_agent, :system_config
-    attr_accessor :dry_run_mode
+    attr_reader :root_agent, :system_config, :supervisor_mode
 
-    def init(system_config)
+    def init(system_config, supervisor_mode: false)
       @system_config = system_config
+      @supervisor_mode = supervisor_mode
 
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
@@ -75,34 +75,29 @@ module Fluent
       end
     end
 
-    def run_configure(conf)
+    def run_configure(conf, dry_run: false)
       configure(conf)
-      conf.check_not_fetched { |key, e|
+      conf.check_not_fetched do |key, e|
         parent_name, plugin_name = e.unused_in
-        if parent_name
-          message = if plugin_name
-                      "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
-                    else
-                      "section <#{e.name}> is not used in <#{parent_name}>"
-                    end
-          if e.for_every_workers?
-            $log.warn :worker0, message
-          elsif e.for_this_worker?
-            $log.warn message
-          end
-          next
+        message = if parent_name && plugin_name
+                    "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
+                  elsif parent_name
+                    "section <#{e.name}> is not used in <#{parent_name}>"
+                  elsif e.name != 'system' && !(@without_source && e.name == 'source')
+                    "parameter '#{key}' in #{e.to_s.strip} is not used."
+                  else
+                    nil
+                  end
+        next if message.nil?
+
+        if dry_run && @supervisor_mode
+          $log.warn :supervisor, message
+        elsif e.for_every_workers?
+          $log.warn :worker0, message
+        elsif e.for_this_worker?
+          $log.warn message
         end
-        unless e.name == 'system'
-          unless @without_source && e.name == 'source'
-            message = "parameter '#{key}' in #{e.to_s.strip} is not used."
-            if e.for_every_workers?
-              $log.warn :worker0, message
-            elsif e.for_this_worker?
-              $log.warn message
-            end
-          end
-        end
-      }
+      end
     end
 
     def configure(conf)
@@ -120,6 +115,7 @@ module Fluent
     end
 
     def add_plugin_dir(dir)
+      $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::Plugin.add_plugin_dir')
       Plugin.add_plugin_dir(dir)
     end
 
@@ -181,6 +177,10 @@ module Fluent
     end
 
     def worker_id
+      if @supervisor_mode
+        return -1
+      end
+
       return @_worker_id if @_worker_id
       # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
       # so it's (almost) a single worker, worker_id=0

data/lib/fluent/fluent_log_event_router.rb

@@ -51,19 +51,23 @@ module Fluent
 
         unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
         unless unmatched_tags.empty?
-          $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
         end
 
       rescue ArgumentError # ArgumentError "#{label_name} label not found"
         # use default event router if <label @FLUENT_LOG> is missing in configuration
         root_log_event_router = root_agent.event_router
-
-        if Fluent::Log.event_tags.any? { |t| root_log_event_router.match?(t) }
+        event_tags = Fluent::Log.event_tags
+        if event_tags.any? { |t| root_log_event_router.match?(t) }
           log_event_router = root_log_event_router
 
-          unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
-          unless unmatched_tags.empty?
-            $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          unmatched_tags = event_tags.select { |t| !log_event_router.match?(t) }
+          if unmatched_tags.empty?
+            $log.warn "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+          else
+            matched_sections = (event_tags - unmatched_tags).map { |tag| "<match #{tag}>" }.join(', ')
+            $log.warn "define #{matched_sections} to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+            $log.warn "match for some tags of log events are not defined in top level (to be ignored)", tags: unmatched_tags
           end
         end
       end

data/lib/fluent/plugin/base.rb

@@ -52,7 +52,7 @@ module Fluent
       end
 
       def configure(conf)
-        if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
+        if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
           workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
                       conf.target_worker_ids.size
                     else

data/lib/fluent/plugin/buf_file.rb

@@ -175,12 +175,8 @@ module Fluent
 
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
-        if @file_permission
-          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
-        else
-          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
-        end
-
+        perm = @file_permission || system_config.file_permission
+        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress)
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 
         return chunk

data/lib/fluent/plugin/buf_file_single.rb

@@ -195,11 +195,8 @@ module Fluent
 
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
-        if @file_permission
-          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: @file_permission, compress: @compress)
-        else
-          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, compress: @compress)
-        end
+        perm = @file_permission || system_config.file_permission
+        chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: perm, compress: @compress)
 
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -37,14 +37,13 @@ module Fluent
         # path_prefix: path prefix string, ended with '.'
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
-        include SystemConfig::Mixin
-
         FILE_PERMISSION = 0644
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+        def initialize(metadata, path, mode, perm: nil, compress: :text)
           super(metadata, compress: compress)
+          perm ||= FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
           @meta = nil

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -29,8 +29,6 @@ module Fluent
         ## buffer chunk path          : /path/to/directory/fsb.key.b513b61c9791029c2513b61c9791029c2.buf
         ## state: b/q - 'b'(on stage), 'q'(enqueued)
 
-        include SystemConfig::Mixin
-
         PATH_EXT = 'buf'
         PATH_SUFFIX = ".#{PATH_EXT}"
         PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n  # //n switch means explicit 'ASCII-8BIT' pattern
@@ -38,9 +36,10 @@ module Fluent
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, key, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+        def initialize(metadata, path, mode, key, perm: FILE_PERMISSION, compress: :text)
           super(metadata, compress: compress)
           @key = key
+          perm ||= FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
 

data/lib/fluent/plugin_helper/server.rb

@@ -355,7 +355,11 @@ module Fluent
         sock = if shared
                  server_socket_manager_client.listen_tcp(bind, port)
                else
-                 TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
+                 # TCPServer.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.
+                 # backlog will be set by the caller, we don't need to set backlog here
+                 tsock = Addrinfo.tcp(bind, port).listen
+                 tsock.autoclose = false
+                 TCPServer.for_fd(tsock.fileno)
                end
         # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
         sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
@@ -722,7 +726,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/root_agent.rb

@@ -89,7 +89,7 @@ module Fluent
               raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
             end
             available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
-            if used_worker_ids.include?(target_worker_id) && !Fluent::Engine.dry_run_mode
+            if used_worker_ids.include?(target_worker_id)
               raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
             end
             used_worker_ids << target_worker_id
@@ -100,9 +100,6 @@ module Fluent
               end
             end
 
-            # On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
-            target_worker_ids = [0] if Fluent::Engine.dry_run_mode
-
             unless target_worker_ids.empty?
               e.set_target_worker_ids(target_worker_ids.uniq)
             end
@@ -113,9 +110,6 @@ module Fluent
             raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
           end
 
-          ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
-          target_worker_id = 0 if Fluent::Engine.dry_run_mode
-
           e.elements.each do |elem|
             unless ['source', 'match', 'filter', 'label'].include?(elem.name)
               raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
@@ -132,7 +126,9 @@ module Fluent
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
       conf.elements(name: 'label').each { |e|
-        next if e.for_another_worker?
+        if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+          next
+        end
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -154,7 +150,9 @@ module Fluent
         log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
       else
         conf.elements(name: 'source').each { |e|
-          next if e.for_another_worker?
+          if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+            next
+          end
           type = e['@type']
           raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)

data/lib/fluent/supervisor.rb

@@ -182,9 +182,9 @@ module Fluent
       if log = config[:logger_initializer]
         # Creating new thread due to mutex can't lock
         # in main thread during trap context
-        Thread.new {
+        Thread.new do
           log.reopen!
-        }.run
+        end
       end
 
       if config[:worker_pid]
@@ -400,9 +400,9 @@ module Fluent
         self
       end
 
-      def apply_options(opts)
-        $log.format = opts[:format] if opts[:format]
-        $log.time_format = opts[:time_format] if opts[:time_format]
+      def apply_options(format: nil, time_format: nil)
+        $log.format = format if format
+        $log.time_format = time_format if time_format
       end
 
       def level=(level)
@@ -444,71 +444,75 @@ module Fluent
 
     def initialize(opt)
       @daemonize = opt[:daemonize]
-      @supervise = opt[:supervise]
       @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
       @conf_encoding = opt[:conf_encoding]
       @log_path = opt[:log_path]
-      @dry_run = opt[:dry_run]
       @show_plugin_config = opt[:show_plugin_config]
       @libs = opt[:libs]
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
-      @process_name = nil
 
-      @workers = opt[:workers]
-      @root_dir = opt[:root_dir]
-      @log_level = opt[:log_level]
       @log_rotate_age = opt[:log_rotate_age]
       @log_rotate_size = opt[:log_rotate_size]
-      @suppress_interval = opt[:suppress_interval]
-      @suppress_config_dump = opt[:suppress_config_dump]
-      @log_event_verbose = opt[:log_event_verbose]
-      @without_source = opt[:without_source]
       @signame = opt[:signame]
 
-      @suppress_repeated_stacktrace = opt[:suppress_repeated_stacktrace]
-      log_opts = {suppress_repeated_stacktrace: @suppress_repeated_stacktrace}
+      @cl_opt = opt
+      @conf = nil
+
+      log_opts = { suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace] }
       @log = LoggerInitializer.new(
-        @log_path, @log_level, @chuser, @chgroup, log_opts,
+        @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
         log_rotate_size: @log_rotate_size
       )
       @finished = false
     end
 
-    def run_supervisor
-      @log.init(:supervisor, 0)
-      show_plugin_config if @show_plugin_config
-      read_config
-      set_system_config
-      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
-
-      $log.info :supervisor, "parsing config file is succeeded", path: @config_path
+    def run_supervisor(dry_run: false)
+      if dry_run
+        $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
+      end
 
-      if @workers < 1
-        raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@workers}"
+      if @system_config.workers < 1
+        raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@system_config.workers}"
       end
 
-      if @root_dir
-        if File.exist?(@root_dir)
-          unless Dir.exist?(@root_dir)
-            raise Fluent::InvalidRootDirectory, "non directory entry exists:#{@root_dir}"
+      root_dir = @system_config.root_dir
+      if root_dir
+        if File.exist?(root_dir)
+          unless Dir.exist?(root_dir)
+            raise Fluent::InvalidRootDirectory, "non directory entry exists:#{root_dir}"
           end
         else
           begin
-            FileUtils.mkdir_p(@root_dir)
+            FileUtils.mkdir_p(root_dir)
           rescue => e
-            raise Fluent::InvalidRootDirectory, "failed to create root directory:#{@root_dir}, #{e.inspect}"
+            raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end
         end
       end
 
-      dry_run_cmd if @dry_run
-      supervise
+      begin
+        ServerEngine::Privilege.change(@chuser, @chgroup)
+        MessagePackFactory.init
+        Fluent::Engine.init(@system_config, supervisor_mode: true)
+        Fluent::Engine.run_configure(@conf, dry_run: dry_run)
+      rescue Fluent::ConfigError => e
+        $log.error 'config error', file: @config_path, error: e
+        $log.debug_backtrace
+        exit!(1)
+      end
+
+      if dry_run
+        $log.info 'finsihed dry run mode'
+        exit 0
+      else
+        supervise
+      end
     end
 
     def options
@@ -517,7 +521,7 @@ module Fluent
         'pid_file' => @daemonize,
         'plugin_dirs' => @plugin_dirs,
         'log_path' => @log_path,
-        'root_dir' => @root_dir,
+        'root_dir' => @system_config.root_dir,
       }
     end
 
@@ -527,22 +531,11 @@ module Fluent
       rescue Exception
         # ignore LoadError and others (related with signals): it may raise these errors in Windows
       end
-      worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
-      process_type = case
-                     when @standalone_worker then :standalone
-                     when worker_id == 0 then :worker0
-                     else :workers
-                     end
-      @log.init(process_type, worker_id)
-      show_plugin_config if @show_plugin_config
-      read_config
-      set_system_config
-      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
 
-      Process.setproctitle("worker:#{@process_name}") if @process_name
+      Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
 
-      if @standalone_worker && @workers != 1
-        raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@workers}"
+      if @standalone_worker && @system_config.workers != 1
+        raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}"
       end
 
       install_main_process_signal_handlers
@@ -552,16 +545,60 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        change_privilege if @standalone_worker
+        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
         MessagePackFactory.init
-        init_engine
-        run_configure
-        run_engine
+        Fluent::Engine.init(@system_config)
+        Fluent::Engine.run_configure(@conf)
+        Fluent::Engine.run
         self.class.cleanup_resources if @standalone_worker
         exit 0
       end
     end
 
+    def configure(supervisor: false)
+      if supervisor
+        @log.init(:supervisor, 0)
+      else
+        worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
+        process_type = case
+                       when @standalone_worker then :standalone
+                       when worker_id == 0 then :worker0
+                       else :workers
+                       end
+        @log.init(process_type, worker_id)
+      end
+
+      if @show_plugin_config
+        show_plugin_config
+      end
+
+      @conf = read_config
+      @system_config = build_system_config(@conf)
+
+      @log.level = @system_config.log_level
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
+
+      @libs.each do |lib|
+        require lib
+      end
+
+      @plugin_dirs.each do |dir|
+        if Dir.exist?(dir)
+          dir = File.expand_path(dir)
+          Fluent::Plugin.add_plugin_dir(dir)
+        end
+      end
+
+      if supervisor
+        # plugins / configuration dumps
+        Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|
+          $log.info("gem '#{spec.name}' version '#{spec.version}'")
+        end
+      end
+    end
+
     private
 
     def create_socket_manager
@@ -570,33 +607,6 @@ module Fluent
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
-    def dry_run_cmd
-      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
-      @system_config.suppress_config_dump = true
-      dry_run
-      exit 0
-    rescue => e
-      $log.error "dry run failed: #{e}"
-      exit 1
-    end
-
-    ## Set Engine's dry_run_mode true to override all target_id of worker sections
-    def dry_run
-      begin
-        Fluent::Engine.dry_run_mode = true
-        change_privilege
-        MessagePackFactory.init
-        init_engine(supervisor: true)
-        run_configure
-      rescue Fluent::ConfigError => e
-        $log.error "config error", file: @config_path, error: e
-        $log.debug_backtrace
-        exit!(1)
-      ensure
-        Fluent::Engine.dry_run_mode = false
-      end
-    end
-
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
       $log.info "show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}"
@@ -604,10 +614,7 @@ module Fluent
     end
 
     def supervise
-      # Make dumpable conf, which is set corresponding_proxies for all elements in all worker sections
-      dry_run
-
-      Process.setproctitle("supervisor:#{@process_name}") if @process_name
+      Process.setproctitle("supervisor:#{@system_config.process_name}") if @system_config.process_name
       $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
 
       rubyopt = ENV["RUBYOPT"]
@@ -619,24 +626,24 @@ module Fluent
 
       $log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
 
-      params = {}
-      params['main_cmd'] = fluentd_spawn_cmd
-      params['daemonize'] = @daemonize
-      params['inline_config'] = @inline_config
-      params['log_path'] = @log_path
-      params['log_rotate_age'] = @log_rotate_age
-      params['log_rotate_size'] = @log_rotate_size
-      params['chuser'] = @chuser
-      params['chgroup'] = @chgroup
-      params['use_v1_config'] = @use_v1_config
-      params['conf_encoding'] = @conf_encoding
-
-      # system config parameters
-      params['workers'] = @workers
-      params['root_dir'] = @root_dir
-      params['log_level'] = @log_level
-      params['suppress_repeated_stacktrace'] = @suppress_repeated_stacktrace
-      params['signame'] = @signame
+      params = {
+        'main_cmd' => fluentd_spawn_cmd,
+        'daemonize' => @daemonize,
+        'inline_config' => @inline_config,
+        'log_path' => @log_path,
+        'log_rotate_age' => @log_rotate_age,
+        'log_rotate_size' => @log_rotate_size,
+        'chuser' => @chuser,
+        'chgroup' => @chgroup,
+        'use_v1_config' => @use_v1_config,
+        'conf_encoding' => @conf_encoding,
+        'signame' => @signame,
+
+        'workers' => @system_config.workers,
+        'root_dir' => @system_config.root_dir,
+        'log_level' => @system_config.log_level,
+        'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
+      }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
@@ -703,7 +710,7 @@ module Fluent
     def flush_buffer
       # Creating new thread due to mutex can't lock
       # in main thread during trap context
-      Thread.new {
+      Thread.new do
         begin
           $log.debug "fluentd main process get SIGUSR1"
           $log.info "force flushing buffered events"
@@ -713,7 +720,7 @@ module Fluent
         rescue Exception => e
           $log.warn "flushing thread error: #{e}"
         end
-      }.run
+      end
     end
 
     def logging_with_console_output
@@ -721,18 +728,18 @@ module Fluent
       unless @log.stdout?
         logger = ServerEngine::DaemonLogger.new(STDOUT)
         log = Fluent::Log.new(logger)
-        log.level = @log_level
+        log.level = @system_config.log_level
         console = log.enable_debug
         yield console
       end
     end
 
     def main_process(&block)
-      if @process_name
-        if @workers > 1
-          Process.setproctitle("worker:#{@process_name}#{ENV['SERVERENGINE_WORKER_ID']}")
+      if @system_config.process_name
+        if @system_config.workers > 1
+          Process.setproctitle("worker:#{@system_config.process_name}#{ENV['SERVERENGINE_WORKER_ID']}")
         else
-          Process.setproctitle("worker:#{@process_name}")
+          Process.setproctitle("worker:#{@system_config.process_name}")
         end
       end
 
@@ -781,47 +788,25 @@ module Fluent
       elsif @inline_config
         config_data << "\n" << @inline_config.gsub("\\n","\n")
       end
-      @conf = Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
-    end
-
-    def set_system_config
-      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
-      @system_config.attach(self)
-      @system_config.apply(self)
+      Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
     end
 
-    def change_privilege
-      ServerEngine::Privilege.change(@chuser, @chgroup)
-    end
-
-    def init_engine(supervisor: false)
-      Fluent::Engine.init(@system_config)
-
-      @libs.each {|lib|
-        require lib
-      }
-
-      @plugin_dirs.each {|dir|
-        if Dir.exist?(dir)
-          dir = File.expand_path(dir)
-          Fluent::Engine.add_plugin_dir(dir)
-        end
-      }
+    def build_system_config(conf)
+      system_config = SystemConfig.create(conf)
+      opt = {}
+      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if @cl_opt.key?(param) && !@cl_opt[param].nil?
+          if param == :log_level && opt[:log_level] == Fluent::Log::LEVEL_INFO
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            next
+          end
 
-      if supervisor
-        # plugins / configuration dumps
-        Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|
-          $log.info("gem '#{spec.name}' version '#{spec.version}'")
+          opt[param] = @cl_opt[param]
         end
       end
-    end
-
-    def run_configure
-      Fluent::Engine.run_configure(@conf)
-    end
-
-    def run_engine
-      Fluent::Engine.run
+      system_config.overwrite_variables(opt)
+      system_config
     end
   end
 end