fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb

@@ -0,0 +1,43 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      class RoundRobinBalancer
+        def initialize
+          @services = []
+          @mutex = Mutex.new
+        end
+
+        def rebalance(services)
+          @mutex.synchronize do
+            @services = services
+          end
+        end
+
+        def select_service
+          s = @mutex.synchronize do
+            s = @services.shift
+            @services.push(s)
+            s
+          end
+          yield(s)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_id.rb

@@ -23,6 +23,7 @@ module Fluent
     def initialize
       super
       @_plugin_root_dir = nil
+      @id = nil
     end
 
     def configure(conf)
@@ -76,5 +77,11 @@ module Fluent
       @_plugin_root_dir = dir.freeze
       dir
     end
+
+    def stop
+      @@configured_ids.delete(@id)
+
+      super
+    end
   end
 end

data/lib/fluent/root_agent.rb

@@ -89,7 +89,7 @@ module Fluent
               raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
             end
             available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
-            if used_worker_ids.include?(target_worker_id) && !Fluent::Engine.dry_run_mode
+            if used_worker_ids.include?(target_worker_id)
               raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
             end
             used_worker_ids << target_worker_id
@@ -100,9 +100,6 @@ module Fluent
               end
             end
 
-            # On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
-            target_worker_ids = [0] if Fluent::Engine.dry_run_mode
-
             unless target_worker_ids.empty?
               e.set_target_worker_ids(target_worker_ids.uniq)
             end
@@ -113,9 +110,6 @@ module Fluent
             raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
           end
 
-          ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
-          target_worker_id = 0 if Fluent::Engine.dry_run_mode
-
           e.elements.each do |elem|
             unless ['source', 'match', 'filter', 'label'].include?(elem.name)
               raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
@@ -132,7 +126,9 @@ module Fluent
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
       conf.elements(name: 'label').each { |e|
-        next if e.for_another_worker?
+        if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+          next
+        end
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -154,7 +150,9 @@ module Fluent
         log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
       else
         conf.elements(name: 'source').each { |e|
-          next if e.for_another_worker?
+          if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+            next
+          end
           type = e['@type']
           raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)

data/lib/fluent/supervisor.rb

@@ -25,6 +25,7 @@ require 'fluent/log'
 require 'fluent/plugin'
 require 'fluent/rpc'
 require 'fluent/system_config'
+require 'fluent/msgpack_factory'
 require 'serverengine'
 
 if Fluent.windows?
@@ -41,7 +42,6 @@ end
 module Fluent
   module ServerModule
     def before_run
-      @start_time = Time.now
       @rpc_server = nil
       @counter = nil
 
@@ -121,7 +121,7 @@ module Fluent
       }
 
       @rpc_server.mount_proc('/api/config.getDump') { |req, res|
-        $log.debug "fluentd RPC got /api/config.dump request"
+        $log.debug "fluentd RPC got /api/config.getDump request"
         $log.info "get dump in-memory config via HTTP"
         res.body = supervisor_get_dump_config_handler
         [nil, nil, res]
@@ -182,9 +182,9 @@ module Fluent
       if log = config[:logger_initializer]
         # Creating new thread due to mutex can't lock
         # in main thread during trap context
-        Thread.new {
+        Thread.new do
           log.reopen!
-        }.run
+        end
       end
 
       if config[:worker_pid]
@@ -210,11 +210,11 @@ module Fluent
     end
 
     def supervisor_dump_config_handler
-      $log.info config[:fluentd_conf].to_s
+      $log.info config[:fluentd_conf]
     end
 
     def supervisor_get_dump_config_handler
-      {conf: config[:fluentd_conf].to_s}
+      {conf: config[:fluentd_conf]}
     end
   end
 
@@ -234,7 +234,6 @@ module Fluent
 
   class Supervisor
     def self.load_config(path, params = {})
-
       pre_loadtime = 0
       pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
       pre_config_mtime = nil
@@ -246,23 +245,6 @@ module Fluent
         return params['pre_conf']
       end
 
-      config_fname = File.basename(path)
-      config_basedir = File.dirname(path)
-      # Assume fluent.conf encoding is UTF-8
-      config_data = File.open(path, "r:#{params['conf_encoding']}:utf-8") {|f| f.read }
-      inline_config = params['inline_config']
-      if inline_config == '-'
-        config_data << "\n" << STDIN.read
-      elsif inline_config
-        config_data << "\n" << inline_config.gsub("\\n","\n")
-      end
-      fluentd_conf = Fluent::Config.parse(config_data, config_fname, config_basedir, params['use_v1_config'])
-      system_config = SystemConfig.create(fluentd_conf)
-
-      # these params must NOT be configured via system config here.
-      # these may be overridden by command line params.
-      workers = params['workers']
-      root_dir = params['root_dir']
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
 
@@ -271,9 +253,6 @@ module Fluent
       chgroup = params['chgroup']
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
-      rpc_endpoint = system_config.rpc_endpoint
-      enable_get_dump = system_config.enable_get_dump
-      counter_server = system_config.counter_server
 
       log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
       logger_initializer = Supervisor::LoggerInitializer.new(
@@ -290,43 +269,41 @@ module Fluent
       # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
       pid_path = params['daemonize']
       daemonize = !!params['daemonize']
-      main_cmd = params['main_cmd']
-      signame = params['signame']
 
       se_config = {
-          worker_type: 'spawn',
-          workers: workers,
-          log_stdin: false,
-          log_stdout: false,
-          log_stderr: false,
-          enable_heartbeat: true,
-          auto_heartbeat: false,
-          unrecoverable_exit_codes: [2],
-          stop_immediately_at_unrecoverable_exit: true,
-          root_dir: root_dir,
-          logger: logger,
-          log: logger.out,
-          log_path: log_path,
-          log_level: log_level,
-          logger_initializer: logger_initializer,
-          chuser: chuser,
-          chgroup: chgroup,
-          chumask: 0,
-          suppress_repeated_stacktrace: suppress_repeated_stacktrace,
-          daemonize: daemonize,
-          rpc_endpoint: rpc_endpoint,
-          counter_server: counter_server,
-          enable_get_dump: enable_get_dump,
-          windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
-                                   File.join(File.dirname(__FILE__), 'daemon.rb'),
-                                   ServerModule.name,
-                                   WorkerModule.name,
-                                   path,
-                                   JSON.dump(params)],
-          command_sender: command_sender,
-          fluentd_conf: fluentd_conf,
-          main_cmd: main_cmd,
-          signame: signame,
+        worker_type: 'spawn',
+        workers: params['workers'],
+        log_stdin: false,
+        log_stdout: false,
+        log_stderr: false,
+        enable_heartbeat: true,
+        auto_heartbeat: false,
+        unrecoverable_exit_codes: [2],
+        stop_immediately_at_unrecoverable_exit: true,
+        root_dir: params['root_dir'],
+        logger: logger,
+        log: logger.out,
+        log_path: log_path,
+        log_level: log_level,
+        logger_initializer: logger_initializer,
+        chuser: chuser,
+        chgroup: chgroup,
+        chumask: 0,
+        suppress_repeated_stacktrace: suppress_repeated_stacktrace,
+        daemonize: daemonize,
+        rpc_endpoint: params['rpc_endpoint'],
+        counter_server: params['counter_server'],
+        enable_get_dump: params['enable_get_dump'],
+        windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
+                                 File.join(File.dirname(__FILE__), 'daemon.rb'),
+                                 ServerModule.name,
+                                 WorkerModule.name,
+                                 path,
+                                 JSON.dump(params)],
+        command_sender: command_sender,
+        fluentd_conf: params['fluentd_conf'],
+        main_cmd: params['main_cmd'],
+        signame: params['signame'],
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -339,7 +316,7 @@ module Fluent
       pre_params['pre_conf'] = nil
       params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
 
-      return se_config
+      se_config
     end
 
     class LoggerInitializer
@@ -364,6 +341,10 @@ module Fluent
         @opts[:worker_id] = worker_id
 
         if @path && @path != "-"
+          unless File.exist?(@path)
+            FileUtils.mkdir_p(File.dirname(@path))
+          end
+
           @logdev = if @log_rotate_age || @log_rotate_size
                      Fluent::LogDeviceIO.new(Fluent.windows? ?
                                                worker_id_suffixed_path(worker_id, @path) : @path,
@@ -400,9 +381,9 @@ module Fluent
         self
       end
 
-      def apply_options(opts)
-        $log.format = opts[:format] if opts[:format]
-        $log.time_format = opts[:time_format] if opts[:time_format]
+      def apply_options(format: nil, time_format: nil)
+        $log.format = format if format
+        $log.time_format = time_format if time_format
       end
 
       def level=(level)
@@ -427,6 +408,7 @@ module Fluent
         suppress_repeated_stacktrace: true,
         without_source: nil,
         use_v1_config: true,
+        strict_config_value: nil,
         supervise: true,
         standalone_worker: false,
         signame: nil,
@@ -444,72 +426,75 @@ module Fluent
 
     def initialize(opt)
       @daemonize = opt[:daemonize]
-      @supervise = opt[:supervise]
       @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
       @conf_encoding = opt[:conf_encoding]
       @log_path = opt[:log_path]
-      @dry_run = opt[:dry_run]
       @show_plugin_config = opt[:show_plugin_config]
       @libs = opt[:libs]
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
-      @rpc_server = nil
-      @process_name = nil
 
-      @workers = opt[:workers]
-      @root_dir = opt[:root_dir]
-      @log_level = opt[:log_level]
       @log_rotate_age = opt[:log_rotate_age]
       @log_rotate_size = opt[:log_rotate_size]
-      @suppress_interval = opt[:suppress_interval]
-      @suppress_config_dump = opt[:suppress_config_dump]
-      @log_event_verbose = opt[:log_event_verbose]
-      @without_source = opt[:without_source]
       @signame = opt[:signame]
 
-      @suppress_repeated_stacktrace = opt[:suppress_repeated_stacktrace]
-      log_opts = {suppress_repeated_stacktrace: @suppress_repeated_stacktrace}
+      @cl_opt = opt
+      @conf = nil
+
+      log_opts = { suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace] }
       @log = LoggerInitializer.new(
-        @log_path, @log_level, @chuser, @chgroup, log_opts,
+        @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
         log_rotate_size: @log_rotate_size
       )
       @finished = false
     end
 
-    def run_supervisor
-      @log.init(:supervisor, 0)
-      show_plugin_config if @show_plugin_config
-      read_config
-      set_system_config
-      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
-
-      $log.info :supervisor, "parsing config file is succeeded", path: @config_path
+    def run_supervisor(dry_run: false)
+      if dry_run
+        $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
+      end
 
-      if @workers < 1
-        raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@workers}"
+      if @system_config.workers < 1
+        raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@system_config.workers}"
       end
 
-      if @root_dir
-        if File.exist?(@root_dir)
-          unless Dir.exist?(@root_dir)
-            raise Fluent::InvalidRootDirectory, "non directory entry exists:#{@root_dir}"
+      root_dir = @system_config.root_dir
+      if root_dir
+        if File.exist?(root_dir)
+          unless Dir.exist?(root_dir)
+            raise Fluent::InvalidRootDirectory, "non directory entry exists:#{root_dir}"
           end
         else
           begin
-            FileUtils.mkdir_p(@root_dir)
+            FileUtils.mkdir_p(root_dir)
           rescue => e
-            raise Fluent::InvalidRootDirectory, "failed to create root directory:#{@root_dir}, #{e.inspect}"
+            raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end
         end
       end
 
-      dry_run_cmd if @dry_run
-      supervise
+      begin
+        ServerEngine::Privilege.change(@chuser, @chgroup)
+        MessagePackFactory.init
+        Fluent::Engine.init(@system_config, supervisor_mode: true)
+        Fluent::Engine.run_configure(@conf, dry_run: dry_run)
+      rescue Fluent::ConfigError => e
+        $log.error 'config error', file: @config_path, error: e
+        $log.debug_backtrace
+        exit!(1)
+      end
+
+      if dry_run
+        $log.info 'finsihed dry run mode'
+        exit 0
+      else
+        supervise
+      end
     end
 
     def options
@@ -518,7 +503,7 @@ module Fluent
         'pid_file' => @daemonize,
         'plugin_dirs' => @plugin_dirs,
         'log_path' => @log_path,
-        'root_dir' => @root_dir,
+        'root_dir' => @system_config.root_dir,
       }
     end
 
@@ -528,22 +513,11 @@ module Fluent
       rescue Exception
         # ignore LoadError and others (related with signals): it may raise these errors in Windows
       end
-      worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
-      process_type = case
-                     when @standalone_worker then :standalone
-                     when worker_id == 0 then :worker0
-                     else :workers
-                     end
-      @log.init(process_type, worker_id)
-      show_plugin_config if @show_plugin_config
-      read_config
-      set_system_config
-      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
 
-      Process.setproctitle("worker:#{@process_name}") if @process_name
+      Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
 
-      if @standalone_worker && @workers != 1
-        raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@workers}"
+      if @standalone_worker && @system_config.workers != 1
+        raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}"
       end
 
       install_main_process_signal_handlers
@@ -553,15 +527,64 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        change_privilege if @standalone_worker
-        init_engine
-        run_configure
-        run_engine
+        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        MessagePackFactory.init
+        Fluent::Engine.init(@system_config)
+        Fluent::Engine.run_configure(@conf)
+        Fluent::Engine.run
         self.class.cleanup_resources if @standalone_worker
         exit 0
       end
     end
 
+    def configure(supervisor: false)
+      if supervisor
+        @log.init(:supervisor, 0)
+      else
+        worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
+        process_type = case
+                       when @standalone_worker then :standalone
+                       when worker_id == 0 then :worker0
+                       else :workers
+                       end
+        @log.init(process_type, worker_id)
+      end
+
+      if @show_plugin_config
+        show_plugin_config
+      end
+
+      if @inline_config == '-'
+        @inline_config = STDIN.read
+      end
+
+      @conf = read_config
+      @system_config = build_system_config(@conf)
+
+      @log.level = @system_config.log_level
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
+
+      @libs.each do |lib|
+        require lib
+      end
+
+      @plugin_dirs.each do |dir|
+        if Dir.exist?(dir)
+          dir = File.expand_path(dir)
+          Fluent::Plugin.add_plugin_dir(dir)
+        end
+      end
+
+      if supervisor
+        # plugins / configuration dumps
+        Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|
+          $log.info("gem '#{spec.name}' version '#{spec.version}'")
+        end
+      end
+    end
+
     private
 
     def create_socket_manager
@@ -570,32 +593,6 @@ module Fluent
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
-    def dry_run_cmd
-      $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode", ruby: RUBY_VERSION
-      @system_config.suppress_config_dump = true
-      dry_run
-      exit 0
-    rescue => e
-      $log.error "dry run failed: #{e}"
-      exit 1
-    end
-
-    ## Set Engine's dry_run_mode true to override all target_id of worker sections
-    def dry_run
-      begin
-        Fluent::Engine.dry_run_mode = true
-        change_privilege
-        init_engine
-        run_configure
-      rescue Fluent::ConfigError => e
-        $log.error "config error", file: @config_path, error: e
-        $log.debug_backtrace
-        exit!(1)
-      ensure
-        Fluent::Engine.dry_run_mode = false
-      end
-    end
-
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
       $log.info "show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}"
@@ -603,10 +600,7 @@ module Fluent
     end
 
     def supervise
-      # Make dumpable conf, which is set corresponding_proxies for all elements in all worker sections
-      dry_run
-
-      Process.setproctitle("supervisor:#{@process_name}") if @process_name
+      Process.setproctitle("supervisor:#{@system_config.process_name}") if @system_config.process_name
       $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
 
       rubyopt = ENV["RUBYOPT"]
@@ -618,24 +612,28 @@ module Fluent
 
       $log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
 
-      params = {}
-      params['main_cmd'] = fluentd_spawn_cmd
-      params['daemonize'] = @daemonize
-      params['inline_config'] = @inline_config
-      params['log_path'] = @log_path
-      params['log_rotate_age'] = @log_rotate_age
-      params['log_rotate_size'] = @log_rotate_size
-      params['chuser'] = @chuser
-      params['chgroup'] = @chgroup
-      params['use_v1_config'] = @use_v1_config
-      params['conf_encoding'] = @conf_encoding
-
-      # system config parameters
-      params['workers'] = @workers
-      params['root_dir'] = @root_dir
-      params['log_level'] = @log_level
-      params['suppress_repeated_stacktrace'] = @suppress_repeated_stacktrace
-      params['signame'] = @signame
+      params = {
+        'main_cmd' => fluentd_spawn_cmd,
+        'daemonize' => @daemonize,
+        'inline_config' => @inline_config,
+        'log_path' => @log_path,
+        'log_rotate_age' => @log_rotate_age,
+        'log_rotate_size' => @log_rotate_size,
+        'chuser' => @chuser,
+        'chgroup' => @chgroup,
+        'use_v1_config' => @use_v1_config,
+        'conf_encoding' => @conf_encoding,
+        'signame' => @signame,
+        'fluentd_conf' => @conf.to_s,
+
+        'workers' => @system_config.workers,
+        'root_dir' => @system_config.root_dir,
+        'log_level' => @system_config.log_level,
+        'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
+        'rpc_endpoint' => @system_config.rpc_endpoint,
+        'enable_get_dump' => @system_config.enable_get_dump,
+        'counter_server' => @system_config.counter_server,
+      }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
@@ -702,7 +700,7 @@ module Fluent
     def flush_buffer
       # Creating new thread due to mutex can't lock
       # in main thread during trap context
-      Thread.new {
+      Thread.new do
         begin
           $log.debug "fluentd main process get SIGUSR1"
           $log.info "force flushing buffered events"
@@ -712,7 +710,7 @@ module Fluent
         rescue Exception => e
           $log.warn "flushing thread error: #{e}"
         end
-      }.run
+      end
     end
 
     def logging_with_console_output
@@ -720,18 +718,18 @@ module Fluent
       unless @log.stdout?
         logger = ServerEngine::DaemonLogger.new(STDOUT)
         log = Fluent::Log.new(logger)
-        log.level = @log_level
+        log.level = @system_config.log_level
         console = log.enable_debug
         yield console
       end
     end
 
     def main_process(&block)
-      if @process_name
-        if @workers > 1
-          Process.setproctitle("worker:#{@process_name}#{ENV['SERVERENGINE_WORKER_ID']}")
+      if @system_config.process_name
+        if @system_config.workers > 1
+          Process.setproctitle("worker:#{@system_config.process_name}#{ENV['SERVERENGINE_WORKER_ID']}")
         else
-          Process.setproctitle("worker:#{@process_name}")
+          Process.setproctitle("worker:#{@system_config.process_name}")
         end
       end
 
@@ -772,48 +770,31 @@ module Fluent
     end
 
     def read_config
-      @config_fname = File.basename(@config_path)
-      @config_basedir = File.dirname(@config_path)
-      @config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
-      if @inline_config == '-'
-        @config_data << "\n" << STDIN.read
-      elsif @inline_config
-        @config_data << "\n" << @inline_config.gsub("\\n","\n")
+      config_fname = File.basename(@config_path)
+      config_basedir = File.dirname(@config_path)
+      config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
+      if @inline_config
+        config_data << "\n" << @inline_config.gsub("\\n", "\n")
       end
-      @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
-    end
-
-    def set_system_config
-      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
-      @system_config.attach(self)
-      @system_config.apply(self)
-    end
-
-    def change_privilege
-      ServerEngine::Privilege.change(@chuser, @chgroup)
-    end
-
-    def init_engine
-      Fluent::Engine.init(@system_config)
-
-      @libs.each {|lib|
-        require lib
-      }
+      Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
+    end
+
+    def build_system_config(conf)
+      system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
+      opt = {}
+      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if @cl_opt.key?(param) && !@cl_opt[param].nil?
+          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            next
+          end
 
-      @plugin_dirs.each {|dir|
-        if Dir.exist?(dir)
-          dir = File.expand_path(dir)
-          Fluent::Engine.add_plugin_dir(dir)
+          opt[param] = @cl_opt[param]
         end
-      }
-    end
-
-    def run_configure
-      Fluent::Engine.run_configure(@conf)
-    end
-
-    def run_engine
-      Fluent::Engine.run
+      end
+      system_config.overwrite_variables(opt)
+      system_config
     end
   end
 end