fluentd 1.8.0.rc3 → 1.8.0

data/lib/fluent/configurable.rb

@@ -74,7 +74,7 @@ module Fluent
       Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
     end
 
-    def configure(conf)
+    def configure(conf, strict_config_value=false)
       @config = conf
 
       logger = if self.respond_to?(:log)
@@ -91,7 +91,7 @@ module Fluent
 
       # In the nested section, can't get plugin class through proxies so get plugin class here
       plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
-      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class, [], strict_config_value)
       @config_root_section = root
 
       root.instance_eval{ @params.keys }.each do |param_name|

data/lib/fluent/engine.rb

@@ -103,9 +103,9 @@ module Fluent
     def configure(conf)
       @root_agent.configure(conf)
 
-      @fleunt_log_event_router = FluentLogEventRouter.build(@root_agent)
+      @fluent_log_event_router = FluentLogEventRouter.build(@root_agent)
 
-      if @fleunt_log_event_router.emittable?
+      if @fluent_log_event_router.emittable?
         $log.enable_event(true)
       end
 
@@ -145,7 +145,7 @@ module Fluent
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
 
-        @fleunt_log_event_router.start
+        @fluent_log_event_router.start
 
         $log.info "fluentd worker is now running", worker: worker_id
         sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
@@ -159,12 +159,12 @@ module Fluent
 
       unless @log_event_verbose
         $log.enable_event(false)
-        @fleunt_log_event_router.graceful_stop
+        @fluent_log_event_router.graceful_stop
       end
       $log.info "shutting down fluentd worker", worker: worker_id
       shutdown
 
-      @fleunt_log_event_router.stop
+      @fluent_log_event_router.stop
     end
 
     def stop
@@ -173,7 +173,7 @@ module Fluent
     end
 
     def push_log_event(tag, time, record)
-      @fleunt_log_event_router.emit_event([tag, time, record])
+      @fluent_log_event_router.emit_event([tag, time, record])
     end
 
     def worker_id

data/lib/fluent/plugin/base.rb

@@ -60,7 +60,7 @@ module Fluent
                     end
           system_config_override(workers: workers)
         end
-        super
+        super(conf, system_config.strict_config_value)
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true
         self

data/lib/fluent/plugin/formatter_stdout.rb

@@ -28,7 +28,7 @@ module Fluent
       def configure(conf)
         super
 
-        @time_formatter = Strftime.new(TIME_FORMAT)
+        @time_formatter = Strftime.new(@time_format || TIME_FORMAT)
         @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
         @sub_formatter.configure(conf)
       end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -64,7 +64,8 @@ module Fluent::Plugin
       def config_ltsv(_req)
         obj = {
           'pid' => Process.pid,
-          'ppid' => Process.ppid
+          'ppid' => Process.ppid,
+          'version' => Fluent::VERSION,
         }.merge(@agent.fluentd_opts)
 
         render_ltsv([obj])
@@ -73,7 +74,8 @@ module Fluent::Plugin
       def config_json(req)
         obj = {
           'pid' => Process.pid,
-          'ppid' => Process.ppid
+          'ppid' => Process.ppid,
+          'version' => Fluent::VERSION,
         }.merge(@agent.fluentd_opts)
         opts = build_option(req)
 

data/lib/fluent/plugin/out_forward.rb

@@ -406,16 +406,16 @@ module Fluent::Plugin
     def statistics
       stats = super
       services = discovery_manager.services
-      healty_nodes_count = 0
+      healthy_nodes_count = 0
       registed_nodes_count = services.size
       services.each do |s|
         if s.available?
-          healty_nodes_count += 1
+          healthy_nodes_count += 1
         end
       end
 
       stats.merge(
-        'healty_nodes_count' => healty_nodes_count,
+        'healthy_nodes_count' => healthy_nodes_count,
         'registered_nodes_count' => registed_nodes_count,
       )
     end

data/lib/fluent/plugin/output.rb

@@ -40,7 +40,7 @@ module Fluent
       helpers_internal :thread, :retry_state
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
-      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
+      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
       CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
       CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
 
@@ -707,7 +707,7 @@ module Fluent
       end
 
       def get_placeholders_keys(str)
-        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
+        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map(&:first).reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
       end
 
       # TODO: optimize this code
@@ -759,11 +759,19 @@ module Fluent
             @chunk_keys.each do |key|
               hash["${#{key}}"] = metadata.variables[key.to_sym]
             end
-            rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
+
+            rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
+              hash.fetch(matched) do
+                log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
+                ''
+              end
+            end
           end
+
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
             log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end
+
           rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -194,7 +194,9 @@ module Fluent
           hash['time_type'] = 'unixtime'
         end
         if conf.has_key?('localtime') || conf.has_key?('utc')
-          if conf.has_key?('localtime') && conf.has_key?('utc')
+          utc = to_bool(conf['utc'])
+          localtime = to_bool(conf['localtime'])
+          if conf.has_key?('localtime') && conf.has_key?('utc') && !(localtime ^ utc)
             raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
           elsif conf.has_key?('localtime')
             hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
@@ -217,6 +219,14 @@ module Fluent
         conf
       end
 
+      def to_bool(v)
+        if  v.is_a?(FalseClass) || v == 'false' || v.nil?
+          false
+        else
+          true
+        end
+      end
+
       def compat_parameters_extract(conf)
         return unless conf.elements('extract').empty?
         return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')

data/lib/fluent/supervisor.rb

@@ -210,11 +210,11 @@ module Fluent
     end
 
     def supervisor_dump_config_handler
-      $log.info config[:fluentd_conf].to_s
+      $log.info config[:fluentd_conf]
     end
 
     def supervisor_get_dump_config_handler
-      {conf: config[:fluentd_conf].to_s}
+      {conf: config[:fluentd_conf]}
     end
   end
 
@@ -234,7 +234,6 @@ module Fluent
 
   class Supervisor
     def self.load_config(path, params = {})
-
       pre_loadtime = 0
       pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
       pre_config_mtime = nil
@@ -246,23 +245,6 @@ module Fluent
         return params['pre_conf']
       end
 
-      config_fname = File.basename(path)
-      config_basedir = File.dirname(path)
-      # Assume fluent.conf encoding is UTF-8
-      config_data = File.open(path, "r:#{params['conf_encoding']}:utf-8") {|f| f.read }
-      inline_config = params['inline_config']
-      if inline_config == '-'
-        config_data << "\n" << STDIN.read
-      elsif inline_config
-        config_data << "\n" << inline_config.gsub("\\n","\n")
-      end
-      fluentd_conf = Fluent::Config.parse(config_data, config_fname, config_basedir, params['use_v1_config'])
-      system_config = SystemConfig.create(fluentd_conf)
-
-      # these params must NOT be configured via system config here.
-      # these may be overridden by command line params.
-      workers = params['workers']
-      root_dir = params['root_dir']
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
 
@@ -271,9 +253,6 @@ module Fluent
       chgroup = params['chgroup']
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
-      rpc_endpoint = system_config.rpc_endpoint
-      enable_get_dump = system_config.enable_get_dump
-      counter_server = system_config.counter_server
 
       log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
       logger_initializer = Supervisor::LoggerInitializer.new(
@@ -290,43 +269,41 @@ module Fluent
       # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
       pid_path = params['daemonize']
       daemonize = !!params['daemonize']
-      main_cmd = params['main_cmd']
-      signame = params['signame']
 
       se_config = {
-          worker_type: 'spawn',
-          workers: workers,
-          log_stdin: false,
-          log_stdout: false,
-          log_stderr: false,
-          enable_heartbeat: true,
-          auto_heartbeat: false,
-          unrecoverable_exit_codes: [2],
-          stop_immediately_at_unrecoverable_exit: true,
-          root_dir: root_dir,
-          logger: logger,
-          log: logger.out,
-          log_path: log_path,
-          log_level: log_level,
-          logger_initializer: logger_initializer,
-          chuser: chuser,
-          chgroup: chgroup,
-          chumask: 0,
-          suppress_repeated_stacktrace: suppress_repeated_stacktrace,
-          daemonize: daemonize,
-          rpc_endpoint: rpc_endpoint,
-          counter_server: counter_server,
-          enable_get_dump: enable_get_dump,
-          windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
-                                   File.join(File.dirname(__FILE__), 'daemon.rb'),
-                                   ServerModule.name,
-                                   WorkerModule.name,
-                                   path,
-                                   JSON.dump(params)],
-          command_sender: command_sender,
-          fluentd_conf: fluentd_conf,
-          main_cmd: main_cmd,
-          signame: signame,
+        worker_type: 'spawn',
+        workers: params['workers'],
+        log_stdin: false,
+        log_stdout: false,
+        log_stderr: false,
+        enable_heartbeat: true,
+        auto_heartbeat: false,
+        unrecoverable_exit_codes: [2],
+        stop_immediately_at_unrecoverable_exit: true,
+        root_dir: params['root_dir'],
+        logger: logger,
+        log: logger.out,
+        log_path: log_path,
+        log_level: log_level,
+        logger_initializer: logger_initializer,
+        chuser: chuser,
+        chgroup: chgroup,
+        chumask: 0,
+        suppress_repeated_stacktrace: suppress_repeated_stacktrace,
+        daemonize: daemonize,
+        rpc_endpoint: params['rpc_endpoint'],
+        counter_server: params['counter_server'],
+        enable_get_dump: params['enable_get_dump'],
+        windows_daemon_cmdline: [ServerEngine.ruby_bin_path,
+                                 File.join(File.dirname(__FILE__), 'daemon.rb'),
+                                 ServerModule.name,
+                                 WorkerModule.name,
+                                 path,
+                                 JSON.dump(params)],
+        command_sender: command_sender,
+        fluentd_conf: params['fluentd_conf'],
+        main_cmd: params['main_cmd'],
+        signame: params['signame'],
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -339,7 +316,7 @@ module Fluent
       pre_params['pre_conf'] = nil
       params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
 
-      return se_config
+      se_config
     end
 
     class LoggerInitializer
@@ -364,6 +341,10 @@ module Fluent
         @opts[:worker_id] = worker_id
 
         if @path && @path != "-"
+          unless File.exist?(@path)
+            FileUtils.mkdir_p(File.dirname(@path))
+          end
+
           @logdev = if @log_rotate_age || @log_rotate_size
                      Fluent::LogDeviceIO.new(Fluent.windows? ?
                                                worker_id_suffixed_path(worker_id, @path) : @path,
@@ -427,6 +408,7 @@ module Fluent
         suppress_repeated_stacktrace: true,
         without_source: nil,
         use_v1_config: true,
+        strict_config_value: nil,
         supervise: true,
         standalone_worker: false,
         signame: nil,
@@ -572,6 +554,10 @@ module Fluent
         show_plugin_config
       end
 
+      if @inline_config == '-'
+        @inline_config = STDIN.read
+      end
+
       @conf = read_config
       @system_config = build_system_config(@conf)
 
@@ -638,11 +624,15 @@ module Fluent
         'use_v1_config' => @use_v1_config,
         'conf_encoding' => @conf_encoding,
         'signame' => @signame,
+        'fluentd_conf' => @conf.to_s,
 
         'workers' => @system_config.workers,
         'root_dir' => @system_config.root_dir,
         'log_level' => @system_config.log_level,
         'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
+        'rpc_endpoint' => @system_config.rpc_endpoint,
+        'enable_get_dump' => @system_config.enable_get_dump,
+        'counter_server' => @system_config.counter_server,
       }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
@@ -783,20 +773,18 @@ module Fluent
       config_fname = File.basename(@config_path)
       config_basedir = File.dirname(@config_path)
       config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
-      if @inline_config == '-'
-        config_data << "\n" << STDIN.read
-      elsif @inline_config
-        config_data << "\n" << @inline_config.gsub("\\n","\n")
+      if @inline_config
+        config_data << "\n" << @inline_config.gsub("\\n", "\n")
       end
       Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
     end
 
     def build_system_config(conf)
-      system_config = SystemConfig.create(conf)
+      system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       opt = {}
       Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
         if @cl_opt.key?(param) && !@cl_opt[param].nil?
-          if param == :log_level && opt[:log_level] == Fluent::Log::LEVEL_INFO
+          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
             # info level can't be specified via command line option.
             # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
             next

data/lib/fluent/system_config.rb

@@ -27,11 +27,12 @@ module Fluent
       :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
+      :strict_config_value
     ]
 
     config_param :workers,   :integer, default: 1
     config_param :root_dir,  :string, default: nil
-    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
@@ -40,6 +41,7 @@ module Fluent
     config_param :rpc_endpoint,    :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name,    :string, default: nil
+    config_param :strict_config_value, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -73,12 +75,12 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
-    def self.create(conf)
+    def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
       raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
-      SystemConfig.new(systems.first)
+      SystemConfig.new(systems.first, strict_config_value)
     end
 
     def self.blank_system_config
@@ -95,14 +97,26 @@ module Fluent
       end
     end
 
-    def initialize(conf=nil)
+    def initialize(conf=nil, strict_config_value=false)
       super()
       conf ||= SystemConfig.blank_system_config
-      configure(conf)
+      configure(conf, strict_config_value)
     end
 
-    def configure(conf)
-      super
+    def configure(conf, strict_config_value=false)
+      strict = strict_config_value
+      if !strict && conf && conf.has_key?("strict_config_value")
+        strict = Fluent::Config.bool_value(conf["strict_config_value"])
+      end
+
+      begin
+        super(conf, strict)
+      rescue ConfigError => e
+        $log.error "config error in:\n#{conf}"
+        $log.error 'config error', error: e
+        $log.debug_backtrace
+        exit!(1)
+      end
 
       @log_level = Log.str_to_level(@log_level.to_s) if @log_level
     end